diff --git a/.golangci.yml b/.golangci.yml index a9a96b2c1..898f0fbc5 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -81,6 +81,7 @@ linters: - ireturn # this one may be interesting to control allocations - gosmopolitan - maligned + - musttag - nilnil - nlreturn - nonamedreturns @@ -123,9 +124,11 @@ issues: - gosec - musttag - nestif + - perfsprint - revive - nolintlint - wrapcheck + - wsl - text: "log is a global variable" linters: - gochecknoglobals diff --git a/cli/app.go b/cli/app.go index 8ca4a1b8d..9cb6f802d 100644 --- a/cli/app.go +++ b/cli/app.go @@ -365,7 +365,7 @@ func safetyFlagVar(cmd *kingpin.CmdClause, result *maintenance.SafetyParameters) "full": maintenance.SafetyFull, } - cmd.Flag("safety", "Safety level").Default("full").PreAction(func(pc *kingpin.ParseContext) error { + cmd.Flag("safety", "Safety level").Default("full").PreAction(func(_ *kingpin.ParseContext) error { r, ok := safetyByName[str] if !ok { return errors.Errorf("unhandled safety level") diff --git a/cli/command_content_stats.go b/cli/command_content_stats.go index af399f5ec..b280688cd 100644 --- a/cli/command_content_stats.go +++ b/cli/command_content_stats.go @@ -150,6 +150,7 @@ func(b content.Info) error { totalSizeOfContentsUnder[s] += int64(b.GetPackedLength()) } } + return nil }) diff --git a/cli/command_content_verify.go b/cli/command_content_verify.go index 694863ba5..26a688a5f 100644 --- a/cli/command_content_verify.go +++ b/cli/command_content_verify.go @@ -133,7 +133,7 @@ func (c *commandContentVerify) getTotalContentCount(ctx context.Context, rep rep if err := rep.ContentReader().IterateContents(ctx, content.IterateOptions{ Range: c.contentRange.contentIDRange(), IncludeDeleted: c.contentVerifyIncludeDeleted, - }, func(ci content.Info) error { + }, func(_ content.Info) error { if err := ctx.Err(); err != nil { return errors.Wrap(err, "context error") } diff --git a/cli/command_index_recover.go b/cli/command_index_recover.go index a145f92fc..6eba78a2a 100644 --- a/cli/command_index_recover.go +++ b/cli/command_index_recover.go @@ -111,7 +111,7 @@ func (c *commandIndexRecover) recoverIndexesFromAllPacks(ctx context.Context, re go func() { for _, prefix := range prefixes { //nolint:errcheck - rep.BlobStorage().ListBlobs(ctx, prefix, func(bm blob.Metadata) error { + rep.BlobStorage().ListBlobs(ctx, prefix, func(_ blob.Metadata) error { discoveringBlobCount.Add(1) return nil }) @@ -151,6 +151,7 @@ func (c *commandIndexRecover) recoverIndexesFromAllPacks(ctx context.Context, re finishedBlobs := processedBlobCount.Load() log(ctx).Debugf("worker %v got %v", worker, cnt) + cnt++ if tt.ShouldOutput(time.Second) { diff --git a/cli/command_mount.go b/cli/command_mount.go index bd1a82dc3..cd407a535 100644 --- a/cli/command_mount.go +++ b/cli/command_mount.go @@ -61,6 +61,7 @@ func (c *commandMount) run(ctx context.Context, rep repo.Repository) error { entry = snapshotfs.AllSourcesEntry(rep) } else { var err error + entry, err = snapshotfs.FilesystemDirectoryFromIDWithPath(ctx, rep, c.mountObjectID, false) if err != nil { return errors.Wrapf(err, "unable to get directory entry for %v", c.mountObjectID) diff --git a/cli/command_policy_set_scheduling.go b/cli/command_policy_set_scheduling.go index 0f7233ad5..98468ffc8 100644 --- a/cli/command_policy_set_scheduling.go +++ b/cli/command_policy_set_scheduling.go @@ -64,8 +64,8 @@ func (c *policySchedulingFlags) setScheduleFromFlags(ctx context.Context, sp *po timesOfDay = append(timesOfDay, timeOfDay) } } - *changeCount++ + *changeCount++ sp.TimesOfDay = policy.SortAndDedupeTimesOfDay(timesOfDay) if timesOfDay == nil { @@ -142,7 +142,7 @@ func splitCronExpressions(expr string) []string { func (c *policySchedulingFlags) setManualFromFlags(ctx context.Context, sp *policy.SchedulingPolicy, changeCount *int) error { // Cannot set both schedule and manual setting - if len(c.policySetInterval) > 0 || len(c.policySetTimesOfDay) > 0 || len(c.policySetCron) > 0 { + if len(c.policySetInterval) > 0 || len(c.policySetTimesOfDay) > 0 || c.policySetCron != "" { return errors.New("cannot set manual field when scheduling snapshots") } diff --git a/cli/command_repository_create.go b/cli/command_repository_create.go index 776116905..4fdc8cae4 100644 --- a/cli/command_repository_create.go +++ b/cli/command_repository_create.go @@ -99,7 +99,7 @@ func (c *commandRepositoryCreate) newRepositoryOptionsFromFlags() *repo.NewRepos func (c *commandRepositoryCreate) ensureEmpty(ctx context.Context, s blob.Storage) error { hasDataError := errors.Errorf("has data") - err := s.ListBlobs(ctx, "", func(cb blob.Metadata) error { + err := s.ListBlobs(ctx, "", func(_ blob.Metadata) error { return hasDataError }) diff --git a/cli/command_repository_repair.go b/cli/command_repository_repair.go index e2564f73d..d6abeb2e7 100644 --- a/cli/command_repository_repair.go +++ b/cli/command_repository_repair.go @@ -84,6 +84,7 @@ func (c *commandRepositoryRepair) recoverFormatBlob(ctx context.Context, st blob for _, prefix := range prefixes { err := st.ListBlobs(ctx, blob.ID(prefix), func(bi blob.Metadata) error { log(ctx).Infof("looking for replica of format blob in %v...", bi.BlobID) + if b, err := format.RecoverFormatBlob(ctx, st, bi.BlobID, bi.Length); err == nil { if !c.repairDryRun { if puterr := st.PutBlob(ctx, format.KopiaRepositoryBlobID, gather.FromSlice(b), blob.PutOptions{}); puterr != nil { diff --git a/cli/command_repository_sync.go b/cli/command_repository_sync.go index 2a36f9b2d..a8db85a7a 100644 --- a/cli/command_repository_sync.go +++ b/cli/command_repository_sync.go @@ -236,15 +236,17 @@ func (c *commandRepositorySyncTo) runSyncBlobs(ctx context.Context, src blob.Rea eg.Go(func() error { for m := range copyCh { log(ctx).Debugf("[%v] Copying %v (%v bytes)...\n", workerID, m.BlobID, m.Length) + if err := c.syncCopyBlob(ctx, m, src, dst); err != nil { return errors.Wrapf(err, "error copying %v", m.BlobID) } numBlobs, bytesCopied := totalCopied.Add(m.Length) - progressMutex.Lock() eta := "unknown" speed := "-" + progressMutex.Lock() + if est, ok := tt.Estimate(float64(bytesCopied), float64(totalBytes)); ok { eta = fmt.Sprintf("%v (%v)", est.Remaining, formatTimestamp(est.EstimatedEndTime)) speed = units.BytesPerSecondsString(est.SpeedPerSecond) @@ -253,15 +255,18 @@ func (c *commandRepositorySyncTo) runSyncBlobs(ctx context.Context, src blob.Rea c.outputSyncProgress( fmt.Sprintf(" Copied %v blobs (%v), Speed: %v, ETA: %v", numBlobs, units.BytesString(bytesCopied), speed, eta)) + progressMutex.Unlock() } for m := range deleteCh { log(ctx).Debugf("[%v] Deleting %v (%v bytes)...\n", workerID, m.BlobID, m.Length) + if err := syncDeleteBlob(ctx, m, dst); err != nil { return errors.Wrapf(err, "error deleting %v", m.BlobID) } } + return nil }) } diff --git a/cli/command_repository_upgrade.go b/cli/command_repository_upgrade.go index f8a737e84..fc68404b9 100644 --- a/cli/command_repository_upgrade.go +++ b/cli/command_repository_upgrade.go @@ -48,11 +48,12 @@ type commandRepositoryUpgrade struct { func (c *commandRepositoryUpgrade) setup(svc advancedAppServices, parent commandParent) { // override the parent, the upgrade sub-command becomes the new parent here-onwards - parent = parent.Command("upgrade", fmt.Sprintf("Upgrade repository format.\n\n%s", warningColor.Sprint(experimentalWarning))).Hidden(). - Validate(func(tmpCmd *kingpin.CmdClause) error { + parent = parent.Command("upgrade", "Upgrade repository format.\n\n"+warningColor.Sprint(experimentalWarning)).Hidden(). + Validate(func(_ *kingpin.CmdClause) error { if v := os.Getenv(c.svc.EnvName(upgradeLockFeatureEnv)); v == "" { return errors.Errorf("please set %q env variable to use this feature", upgradeLockFeatureEnv) } + return nil }) diff --git a/cli/command_restore.go b/cli/command_restore.go index 7ed854c6c..2d1909290 100644 --- a/cli/command_restore.go +++ b/cli/command_restore.go @@ -509,13 +509,13 @@ func (c *commandRestore) tryToConvertPathToID(ctx context.Context, rep repo.Repo func createSnapshotTimeFilter(timespec string) (func(*snapshot.Manifest, int, int) bool, error) { if timespec == "" || timespec == "latest" { - return func(m *snapshot.Manifest, i, total int) bool { + return func(_ *snapshot.Manifest, i, _ int) bool { return i == 0 }, nil } if timespec == "oldest" { - return func(m *snapshot.Manifest, i, total int) bool { + return func(_ *snapshot.Manifest, i, total int) bool { return i == total-1 }, nil } @@ -525,7 +525,7 @@ func createSnapshotTimeFilter(timespec string) (func(*snapshot.Manifest, int, in return nil, err } - return func(m *snapshot.Manifest, i, total int) bool { + return func(m *snapshot.Manifest, _, _ int) bool { return m.StartTime.ToTime().Before(t) }, nil } diff --git a/cli/command_server_start.go b/cli/command_server_start.go index c744aa201..ac6964f2c 100644 --- a/cli/command_server_start.go +++ b/cli/command_server_start.go @@ -199,7 +199,7 @@ func (c *commandServerStart) run(ctx context.Context) error { httpServer := &http.Server{ ReadHeaderTimeout: 15 * time.Second, //nolint:gomnd Addr: stripProtocol(c.sf.serverAddress), - BaseContext: func(l net.Listener) context.Context { + BaseContext: func(_ net.Listener) context.Context { return ctx }, } diff --git a/cli/command_snapshot_create.go b/cli/command_snapshot_create.go index c998ed87c..613490597 100644 --- a/cli/command_snapshot_create.go +++ b/cli/command_snapshot_create.go @@ -463,7 +463,6 @@ func (c *commandSnapshotCreate) getContentToSnapshot(ctx context.Context, dir st if c.sourceOverride != "" { info, err = parseFullSource(c.sourceOverride, rep.ClientOptions().Hostname, rep.ClientOptions().Username) - if err != nil { return nil, info, false, errors.Wrapf(err, "invalid source override %v", c.sourceOverride) } diff --git a/cli/command_snapshot_list.go b/cli/command_snapshot_list.go index 2e133765c..7f1007805 100644 --- a/cli/command_snapshot_list.go +++ b/cli/command_snapshot_list.go @@ -65,7 +65,7 @@ func (c *commandSnapshotList) setup(svc appServices, parent commandParent) { func findSnapshotsForSource(ctx context.Context, rep repo.Repository, sourceInfo snapshot.SourceInfo, tags map[string]string) (manifestIDs []manifest.ID, err error) { var result []manifest.ID - for len(sourceInfo.Path) > 0 { + for sourceInfo.Path != "" { list, err := snapshot.ListSnapshotManifests(ctx, rep, &sourceInfo, tags) if err != nil { return nil, errors.Wrapf(err, "error listing manifests for %v", sourceInfo) diff --git a/cli/command_snapshot_verify.go b/cli/command_snapshot_verify.go index 5c45f2cf3..b34659453 100644 --- a/cli/command_snapshot_verify.go +++ b/cli/command_snapshot_verify.go @@ -137,10 +137,12 @@ func (c *commandSnapshotVerify) loadSourceManifests(ctx context.Context, rep rep if err != nil { return nil, errors.Wrapf(err, "error parsing %q", srcStr) } + man, err := snapshot.ListSnapshotManifests(ctx, rep, &src, nil) if err != nil { return nil, errors.Wrapf(err, "unable to list snapshot manifests for %v", src) } + manifestIDs = append(manifestIDs, man...) } } diff --git a/cli/storage_azure.go b/cli/storage_azure.go index 1c9ec017f..b5f0803b8 100644 --- a/cli/storage_azure.go +++ b/cli/storage_azure.go @@ -30,7 +30,7 @@ func (c *storageAzureFlags) Setup(svc StorageProviderServices, cmd *kingpin.CmdC var pointInTimeStr string - pitPreAction := func(pc *kingpin.ParseContext) error { + pitPreAction := func(_ *kingpin.ParseContext) error { if pointInTimeStr != "" { t, err := time.Parse(time.RFC3339, pointInTimeStr) if err != nil { diff --git a/cli/storage_s3.go b/cli/storage_s3.go index d3a74f786..77f1b9ce4 100644 --- a/cli/storage_s3.go +++ b/cli/storage_s3.go @@ -34,7 +34,7 @@ func (c *storageS3Flags) Setup(svc StorageProviderServices, cmd *kingpin.CmdClau var pointInTimeStr string - pitPreAction := func(pc *kingpin.ParseContext) error { + pitPreAction := func(_ *kingpin.ParseContext) error { if pointInTimeStr != "" { t, err := time.Parse(time.RFC3339, pointInTimeStr) if err != nil { diff --git a/internal/epoch/epoch_manager.go b/internal/epoch/epoch_manager.go index 1546a71f1..6f8c085d4 100644 --- a/internal/epoch/epoch_manager.go +++ b/internal/epoch/epoch_manager.go @@ -650,6 +650,7 @@ func (e *Manager) loadUncompactedEpochs(ctx context.Context, min, max int) (map[ defer mu.Unlock() result[n] = bm + return nil }) } diff --git a/internal/epoch/epoch_utils.go b/internal/epoch/epoch_utils.go index b980da69c..dbc6a4442 100644 --- a/internal/epoch/epoch_utils.go +++ b/internal/epoch/epoch_utils.go @@ -18,7 +18,7 @@ func epochNumberFromBlobID(blobID blob.ID) (int, bool) { s = s[0:p] } - for len(s) > 0 && !unicode.IsDigit(rune(s[0])) { + for s != "" && !unicode.IsDigit(rune(s[0])) { s = s[1:] } @@ -43,7 +43,7 @@ func epochRangeFromBlobID(blobID blob.ID) (min, max int, ok bool) { first := parts[0] second := parts[1] - for len(first) > 0 && !unicode.IsDigit(rune(first[0])) { + for first != "" && !unicode.IsDigit(rune(first[0])) { first = first[1:] } diff --git a/internal/ospath/ospath.go b/internal/ospath/ospath.go index e9dca65b5..92b94b719 100644 --- a/internal/ospath/ospath.go +++ b/internal/ospath/ospath.go @@ -31,7 +31,7 @@ func IsAbs(s string) bool { if strings.HasPrefix(s, "\\\\") { parts := strings.Split(s[2:], "\\") - return len(parts) > 1 && len(parts[1]) > 0 + return len(parts) > 1 && parts[1] != "" } } diff --git a/internal/parallelwork/parallel_work_queue.go b/internal/parallelwork/parallel_work_queue.go index 144c385bf..303fa99bc 100644 --- a/internal/parallelwork/parallel_work_queue.go +++ b/internal/parallelwork/parallel_work_queue.go @@ -80,7 +80,9 @@ func (v *Queue) Process(ctx context.Context, workers int) error { } err := callback() + v.completed(ctx) + if err != nil { return err } diff --git a/internal/server/server.go b/internal/server/server.go index e964487d6..486a56fca 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -241,7 +241,7 @@ func isAuthenticated(rc requestContext) bool { } func (s *Server) isAuthCookieValid(username, cookieValue string) bool { - tok, err := jwt.ParseWithClaims(cookieValue, &jwt.RegisteredClaims{}, func(t *jwt.Token) (interface{}, error) { + tok, err := jwt.ParseWithClaims(cookieValue, &jwt.RegisteredClaims{}, func(_ *jwt.Token) (interface{}, error) { return s.authCookieSigningKey, nil }) if err != nil { @@ -381,6 +381,7 @@ func (s *Server) handleRequestPossiblyNotConnected(isAuthorized isAuthorizedFunc http.Error(rc.w, "error reading request body", http.StatusInternalServerError) return } + rc.body = body if s.options.LogRequests { @@ -392,8 +393,10 @@ func (s *Server) handleRequestPossiblyNotConnected(isAuthorized isAuthorizedFunc e := json.NewEncoder(rc.w) e.SetIndent("", " ") - var v interface{} - var err *apiError + var ( + v any + err *apiError + ) // process the request while ignoring the cancellation signal // to ensure all goroutines started by it won't be canceled @@ -804,6 +807,7 @@ func (s *Server) ServeStaticFiles(m *mux.Router, fs http.FileSystem) { } http.ServeContent(w, r, "/", clock.Now(), bytes.NewReader(s.patchIndexBytes(sessionID, indexBytes))) + return } diff --git a/internal/server/source_manager.go b/internal/server/source_manager.go index bd036f6b2..3a4be3c05 100644 --- a/internal/server/source_manager.go +++ b/internal/server/source_manager.go @@ -336,6 +336,7 @@ func (s *sourceManager) snapshotInternal(ctx context.Context, ctrl uitask.Contro }, }, func(ctx context.Context, w repo.RepositoryWriter) error { log(ctx).Debugf("uploading %v", s.src) + u := snapshotfs.NewUploader(w) ctrl.OnCancel(u.Cancel) @@ -359,8 +360,8 @@ func (s *sourceManager) snapshotInternal(ctx context.Context, ctrl uitask.Contro s.setUploader(u) manifest, err := u.Upload(ctx, localEntry, policyTree, s.src, manifestsSinceLastCompleteSnapshot...) - prog.report(true) + prog.report(true) s.setUploader(nil) if err != nil { @@ -385,6 +386,7 @@ func (s *sourceManager) snapshotInternal(ctx context.Context, ctrl uitask.Contro } log(ctx).Debugf("created snapshot %v", snapshotID) + return nil }) } diff --git a/internal/tlsutil/tlsutil.go b/internal/tlsutil/tlsutil.go index 56c075f95..dba9c83d3 100644 --- a/internal/tlsutil/tlsutil.go +++ b/internal/tlsutil/tlsutil.go @@ -142,6 +142,8 @@ func verifyPeerCertificate(sha256Fingerprint string) func(rawCerts [][]byte, ver sha256Fingerprint = strings.ToLower(sha256Fingerprint) return func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { + _ = verifiedChains + var serverCerts []string for _, c := range rawCerts { diff --git a/repo/api_server_repository.go b/repo/api_server_repository.go index 46cd379f7..a8ebd8fea 100644 --- a/repo/api_server_repository.go +++ b/repo/api_server_repository.go @@ -177,7 +177,7 @@ func (r *apiServerRepository) NewWriter(ctx context.Context, opt WriteSessionOpt w.afterFlush = nil if w.wso.OnUpload == nil { - w.wso.OnUpload = func(i int64) {} + w.wso.OnUpload = func(_ int64) {} } r.addRef() @@ -200,7 +200,7 @@ func (r *apiServerRepository) GetContent(ctx context.Context, contentID content. var tmp gather.WriteBuffer defer tmp.Close() - err := r.contentCache.GetOrLoad(ctx, contentID.String(), func(output *gather.WriteBuffer) error { + err := r.contentCache.GetOrLoad(ctx, contentID.String(), func(_ *gather.WriteBuffer) error { var result []byte if err := r.cli.Get(ctx, "contents/"+contentID.String(), content.ErrContentNotFound, &result); err != nil { @@ -320,7 +320,7 @@ func openRestAPIRepository(ctx context.Context, si *APIServerInfo, password stri immutableServerRepositoryParameters: par, cli: cli, wso: WriteSessionOptions{ - OnUpload: func(i int64) {}, + OnUpload: func(_ int64) {}, }, } diff --git a/repo/blob/azure/azure_pit.go b/repo/blob/azure/azure_pit.go index dc186ee0f..a5f15c86e 100644 --- a/repo/blob/azure/azure_pit.go +++ b/repo/blob/azure/azure_pit.go @@ -189,7 +189,7 @@ func maybePointInTimeStore(ctx context.Context, s *azStorage, pointInTime *time. pointInTime: *pointInTime, // not used for the check } - err := pit.getBlobVersions(ctx, format.KopiaRepositoryBlobID, func(vm versionMetadata) error { + err := pit.getBlobVersions(ctx, format.KopiaRepositoryBlobID, func(_ versionMetadata) error { return nil }) if err != nil { diff --git a/repo/blob/azure/azure_storage.go b/repo/blob/azure/azure_storage.go index 55464bd91..808c7b2f2 100644 --- a/repo/blob/azure/azure_storage.go +++ b/repo/blob/azure/azure_storage.go @@ -427,7 +427,8 @@ func New(ctx context.Context, opt *Options, isCreate bool) (blob.Storage, error) // verify Azure connection is functional by listing blobs in a bucket, which will fail if the container // does not exist. We list with a prefix that will not exist, to avoid iterating through any objects. nonExistentPrefix := fmt.Sprintf("kopia-azure-storage-initializing-%v", clock.Now().UnixNano()) - if err := raw.ListBlobs(ctx, blob.ID(nonExistentPrefix), func(md blob.Metadata) error { + + if err := raw.ListBlobs(ctx, blob.ID(nonExistentPrefix), func(_ blob.Metadata) error { return nil }); err != nil { return nil, errors.Wrap(err, "unable to list from the bucket") diff --git a/repo/blob/config.go b/repo/blob/config.go index ac6cb5ed3..215fe1282 100644 --- a/repo/blob/config.go +++ b/repo/blob/config.go @@ -7,8 +7,6 @@ ) // ConnectionInfo represents JSON-serializable configuration of a blob storage. -// -//nolint:musttag // we use custom JSON marshaling. type ConnectionInfo struct { Type string Config interface{} diff --git a/repo/blob/gcs/gcs_storage.go b/repo/blob/gcs/gcs_storage.go index c28ac90cd..b5b58ee28 100644 --- a/repo/blob/gcs/gcs_storage.go +++ b/repo/blob/gcs/gcs_storage.go @@ -273,10 +273,10 @@ func New(ctx context.Context, opt *Options, isCreate bool) (blob.Storage, error) // verify GCS connection is functional by listing blobs in a bucket, which will fail if the bucket // does not exist. We list with a prefix that will not exist, to avoid iterating through any objects. nonExistentPrefix := fmt.Sprintf("kopia-gcs-storage-initializing-%v", clock.Now().UnixNano()) - err = gcs.ListBlobs(ctx, blob.ID(nonExistentPrefix), func(md blob.Metadata) error { + + err = gcs.ListBlobs(ctx, blob.ID(nonExistentPrefix), func(_ blob.Metadata) error { return nil }) - if err != nil { return nil, errors.Wrap(err, "unable to list from the bucket") } diff --git a/repo/blob/gdrive/gdrive_storage.go b/repo/blob/gdrive/gdrive_storage.go index cb1daf8ac..ac7e9b9ab 100644 --- a/repo/blob/gdrive/gdrive_storage.go +++ b/repo/blob/gdrive/gdrive_storage.go @@ -115,6 +115,7 @@ func (gdrive *gdriveStorage) GetMetadata(ctx context.Context, blobID blob.ID) (b } entry.FileID = file.Id + return file, err }) if err != nil { @@ -158,6 +159,7 @@ func (gdrive *gdriveStorage) PutBlob(ctx context.Context, blobID blob.ID, data b } var file *drive.File + mtime := "" if !opts.SetModTime.IsZero() { @@ -198,7 +200,6 @@ func (gdrive *gdriveStorage) PutBlob(ctx context.Context, blobID blob.ID, data b ). Context(ctx). Do() - if err != nil { return nil, errors.Wrapf(translateError(err), "Update in PutBlob(%s)", blobID) } @@ -221,11 +222,16 @@ func (gdrive *gdriveStorage) DeleteBlob(ctx context.Context, blobID blob.ID) err handleError := func(err error) error { if errors.Is(err, blob.ErrBlobNotFound) { log(ctx).Warnf("Trying to non-existent DeleteBlob(%s)", blobID) + entry.FileID = "" + return nil - } else if err != nil { + } + + if err != nil { return errors.Wrapf(err, "DeleteBlob(%s)", blobID) } + return nil } @@ -240,7 +246,9 @@ func (gdrive *gdriveStorage) DeleteBlob(ctx context.Context, blobID blob.ID) err } entry.FileID = "" + gdrive.fileIDCache.RecordBlobChange(blobID, "") + return nil, nil }) @@ -249,7 +257,7 @@ func (gdrive *gdriveStorage) DeleteBlob(ctx context.Context, blobID blob.ID) err func (gdrive *gdriveStorage) ListBlobs(ctx context.Context, prefix blob.ID, callback func(blob.Metadata) error) error { // Tracks blob matches in cache but not returned by API. - unvisitedIds := make(map[blob.ID]bool) + unvisitedIDs := make(map[blob.ID]bool) consumer := func(files *drive.FileList) error { for _, file := range files.Files { @@ -261,7 +269,7 @@ func (gdrive *gdriveStorage) ListBlobs(ctx context.Context, prefix blob.ID, call } // Mark blob as visited. - delete(unvisitedIds, blobID) + delete(unvisitedIDs, blobID) bm, err := parseBlobMetadata(file, blobID) if err != nil { @@ -285,9 +293,9 @@ func (gdrive *gdriveStorage) ListBlobs(ctx context.Context, prefix blob.ID, call gdrive.fileIDCache.VisitBlobChanges(func(blobID blob.ID, fileID string) { if matchesPrefix(blobID, prefix) { if fileID != "" { - unvisitedIds[blobID] = true + unvisitedIDs[blobID] = true } else { - delete(unvisitedIds, blobID) + delete(unvisitedIDs, blobID) } } }) @@ -299,8 +307,8 @@ func (gdrive *gdriveStorage) ListBlobs(ctx context.Context, prefix blob.ID, call } // Catch any blobs that the API didn't return. - if len(unvisitedIds) != 0 { - for blobID := range unvisitedIds { + if len(unvisitedIDs) != 0 { + for blobID := range unvisitedIDs { bm, err := gdrive.GetMetadata(ctx, blobID) if err != nil { return errors.Wrapf(translateError(err), "GetMetadata in ListBlobs(%s)", prefix) @@ -560,10 +568,10 @@ func New(ctx context.Context, opt *Options, isCreate bool) (blob.Storage, error) // verify Drive connection is functional by listing blobs in a bucket, which will fail if the bucket // does not exist. We list with a prefix that will not exist, to avoid iterating through any objects. nonExistentPrefix := fmt.Sprintf("kopia-gdrive-storage-initializing-%v", clock.Now().UnixNano()) - err = gdrive.ListBlobs(ctx, blob.ID(nonExistentPrefix), func(md blob.Metadata) error { + + err = gdrive.ListBlobs(ctx, blob.ID(nonExistentPrefix), func(_ blob.Metadata) error { return nil }) - if err != nil { return nil, errors.Wrap(err, "unable to list from the folder") } diff --git a/repo/blob/rclone/rclone_storage_test.go b/repo/blob/rclone/rclone_storage_test.go index e976008c6..6bcbaf3da 100644 --- a/repo/blob/rclone/rclone_storage_test.go +++ b/repo/blob/rclone/rclone_storage_test.go @@ -15,6 +15,7 @@ "github.com/google/uuid" "github.com/pkg/errors" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" @@ -271,7 +272,7 @@ func TestRCloneProviders(t *testing.T) { defer wg.Done() for j := 0; j < 3; j++ { - require.NoError(t, st.PutBlob(ctx, blob.ID(fmt.Sprintf("%v-%v-%v", prefix, i, j)), gather.FromSlice([]byte{1, 2, 3}), blob.PutOptions{})) + assert.NoError(t, st.PutBlob(ctx, blob.ID(fmt.Sprintf("%v-%v-%v", prefix, i, j)), gather.FromSlice([]byte{1, 2, 3}), blob.PutOptions{})) } }() } diff --git a/repo/blob/s3/s3_storage.go b/repo/blob/s3/s3_storage.go index d267a8a3a..3fb05117a 100644 --- a/repo/blob/s3/s3_storage.go +++ b/repo/blob/s3/s3_storage.go @@ -378,7 +378,6 @@ func newStorageWithCredentials(ctx context.Context, creds *credentials.Credentia var err error minioOpts.Transport, err = getCustomTransport(opt) - if err != nil { return nil, err } diff --git a/repo/blob/s3/s3_storage_test.go b/repo/blob/s3/s3_storage_test.go index f540e6287..d26bc3bae 100644 --- a/repo/blob/s3/s3_storage_test.go +++ b/repo/blob/s3/s3_storage_test.go @@ -648,7 +648,6 @@ func createClient(tb testing.TB, opt *Options) *minio.Client { var err error transport, err = getCustomTransport(opt) - if err != nil { tb.Fatalf("unable to get proper transport: %v", err) } diff --git a/repo/blob/sftp/sftp_storage.go b/repo/blob/sftp/sftp_storage.go index efbeb57e3..7e8c76480 100644 --- a/repo/blob/sftp/sftp_storage.go +++ b/repo/blob/sftp/sftp_storage.go @@ -131,6 +131,7 @@ func (s *sftpImpl) GetBlobFromPath(ctx context.Context, dirPath, fullPath string if err != nil { return errors.Wrapf(err, "unrecognized error when opening SFTP file %v", fullPath) } + defer r.Close() //nolint:errcheck if length < 0 { diff --git a/repo/content/committed_content_index.go b/repo/content/committed_content_index.go index fc222b5a4..1769c7420 100644 --- a/repo/content/committed_content_index.go +++ b/repo/content/committed_content_index.go @@ -324,6 +324,7 @@ func (c *committedContentIndex) fetchIndexBlobs(ctx context.Context, isPermissiv c.log.Errorf("skipping bad read of index blob %v", indexBlobID) continue } + return errors.Wrapf(err, "error loading index blob %v", indexBlobID) } @@ -331,6 +332,7 @@ func (c *committedContentIndex) fetchIndexBlobs(ctx context.Context, isPermissiv return errors.Wrap(err, "unable to add to committed content cache") } } + return nil }) } diff --git a/repo/content/committed_read_manager.go b/repo/content/committed_read_manager.go index 4b4174c96..c45ffc9d5 100644 --- a/repo/content/committed_read_manager.go +++ b/repo/content/committed_read_manager.go @@ -587,7 +587,7 @@ func (sm *SharedManager) CloseShared(ctx context.Context) error { func (sm *SharedManager) AlsoLogToContentLog(ctx context.Context) context.Context { sm.repoLogManager.Enable() - return logging.WithAdditionalLogger(ctx, func(module string) logging.Logger { + return logging.WithAdditionalLogger(ctx, func(_ string) logging.Logger { return sm.log }) } diff --git a/repo/content/content_formatter_test.go b/repo/content/content_formatter_test.go index bc79b68be..fe01e8f9f 100644 --- a/repo/content/content_formatter_test.go +++ b/repo/content/content_formatter_test.go @@ -8,6 +8,7 @@ "testing" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/kopia/kopia/internal/blobtesting" @@ -142,7 +143,7 @@ func mustCreateFormatProvider(t *testing.T, f *format.ContentFormat) format.Prov t.Helper() fop, err := format.NewFormattingOptionsProvider(f, nil) - require.NoError(t, err) + assert.NoError(t, err) return fop } diff --git a/repo/content/content_manager.go b/repo/content/content_manager.go index 0222a7fe6..ae8097b17 100644 --- a/repo/content/content_manager.go +++ b/repo/content/content_manager.go @@ -350,7 +350,9 @@ func (bm *WriteManager) addToPackUnlocked(ctx context.Context, contentID ID, dat func (bm *WriteManager) DisableIndexFlush(ctx context.Context) { bm.lock() defer bm.unlock(ctx) + bm.log.Debugf("DisableIndexFlush()") + bm.disableIndexFlushCount++ } @@ -359,7 +361,9 @@ func (bm *WriteManager) DisableIndexFlush(ctx context.Context) { func (bm *WriteManager) EnableIndexFlush(ctx context.Context) { bm.lock() defer bm.unlock(ctx) + bm.log.Debugf("EnableIndexFlush()") + bm.disableIndexFlushCount-- } diff --git a/repo/content/content_manager_test.go b/repo/content/content_manager_test.go index a68e1a0b9..338b80ac0 100644 --- a/repo/content/content_manager_test.go +++ b/repo/content/content_manager_test.go @@ -17,6 +17,7 @@ "github.com/google/go-cmp/cmp" "github.com/pkg/errors" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/kopia/kopia/internal/blobtesting" @@ -1227,6 +1228,7 @@ func (s *contentManagerSuite) verifyAllDataPresent(ctx context.Context, t *testi bm := s.newTestContentManagerWithCustomTime(t, st, nil) defer bm.CloseShared(ctx) + _ = bm.IterateContents(ctx, IterateOptions{}, func(ci Info) error { delete(contentIDs, ci.GetContentID()) return nil @@ -2514,7 +2516,8 @@ func verifyContent(ctx context.Context, t *testing.T, bm *WriteManager, contentI b2, err := bm.GetContent(ctx, contentID) if err != nil { - t.Fatalf("unable to read content %q: %v", contentID, err) + t.Errorf("unable to read content %q: %v", contentID, err) + return } @@ -2533,6 +2536,8 @@ func writeContentAndVerify(ctx context.Context, t *testing.T, bm *WriteManager, contentID, err := bm.WriteContent(ctx, gather.FromSlice(b), "", NoCompression) if err != nil { t.Errorf("err: %v", err) + + return contentID } if got, want := contentID, hashValue(t, b); got != want { @@ -2606,7 +2611,7 @@ func hashValue(t *testing.T, b []byte) ID { h.Write(b) id, err := IDFromHash("", h.Sum(nil)) - require.NoError(t, err) + assert.NoError(t, err) return id } diff --git a/repo/grpc_repository_client.go b/repo/grpc_repository_client.go index 5fee712d9..c6fe31beb 100644 --- a/repo/grpc_repository_client.go +++ b/repo/grpc_repository_client.go @@ -939,7 +939,7 @@ func newGRPCAPIRepositoryForConnection( par *immutableServerRepositoryParameters, ) (*grpcRepositoryClient, error) { if opt.OnUpload == nil { - opt.OnUpload = func(i int64) {} + opt.OnUpload = func(_ int64) {} } rr := &grpcRepositoryClient{ @@ -954,6 +954,7 @@ func newGRPCAPIRepositoryForConnection( return inSessionWithoutRetry(ctx, rr, func(ctx context.Context, sess *grpcInnerSession) (*grpcRepositoryClient, error) { p := sess.repoParams + hf, err := hashing.CreateHashFunc(p) if err != nil { return nil, errors.Wrap(err, "unable to create hash function") diff --git a/repo/maintenance/blob_gc.go b/repo/maintenance/blob_gc.go index df3e8f9a8..f55c9fb9b 100644 --- a/repo/maintenance/blob_gc.go +++ b/repo/maintenance/blob_gc.go @@ -46,6 +46,7 @@ func DeleteUnreferencedBlobs(ctx context.Context, rep repo.DirectRepositoryWrite if err := rep.BlobStorage().DeleteBlob(ctx, bm.BlobID); err != nil { return errors.Wrapf(err, "unable to delete blob %q", bm.BlobID) } + cnt, del := deleted.Add(bm.Length) if cnt%100 == 0 { log(ctx).Infof(" deleted %v unreferenced blobs (%v)", cnt, units.BytesString(del)) diff --git a/repo/maintenance/blob_retain.go b/repo/maintenance/blob_retain.go index 041ecee10..3a8f16510 100644 --- a/repo/maintenance/blob_retain.go +++ b/repo/maintenance/blob_retain.go @@ -96,6 +96,7 @@ func ExtendBlobRetentionTime(ctx context.Context, rep repo.DirectRepositoryWrite } atomic.AddUint32(toExtend, 1) + return nil }) diff --git a/repo/maintenance/content_rewrite.go b/repo/maintenance/content_rewrite.go index 221c3a764..1931de254 100644 --- a/repo/maintenance/content_rewrite.go +++ b/repo/maintenance/content_rewrite.go @@ -174,6 +174,7 @@ func(b content.Info) error { if int(b.GetFormatVersion()) == opt.FormatVersion && strings.HasPrefix(string(b.GetPackBlobID()), string(opt.PackPrefix)) { ch <- contentInfoOrError{Info: b} } + return nil }) } diff --git a/repo/maintenance/maintenance_run.go b/repo/maintenance/maintenance_run.go index e18fc22c3..694fad11c 100644 --- a/repo/maintenance/maintenance_run.go +++ b/repo/maintenance/maintenance_run.go @@ -388,6 +388,7 @@ func runTaskDeleteOrphanedBlobsFull(ctx context.Context, runParams RunParameters _, err := DeleteUnreferencedBlobs(ctx, runParams.rep, DeleteUnreferencedBlobsOptions{ NotAfterTime: runParams.MaintenanceStartTime, }, safety) + return err }) } @@ -398,6 +399,7 @@ func runTaskDeleteOrphanedBlobsQuick(ctx context.Context, runParams RunParameter NotAfterTime: runParams.MaintenanceStartTime, Prefix: content.PackBlobIDPrefixSpecial, }, safety) + return err }) } diff --git a/repo/manifest/committed_manifest_manager.go b/repo/manifest/committed_manifest_manager.go index 418a13ccf..c43d058ce 100644 --- a/repo/manifest/committed_manifest_manager.go +++ b/repo/manifest/committed_manifest_manager.go @@ -160,9 +160,11 @@ func (m *committedManifestManager) loadCommittedContentsLocked(ctx context.Conte return err } + mu.Lock() manifests[ci.GetContentID()] = man mu.Unlock() + return nil }) if err == nil { diff --git a/repo/object/object_reader.go b/repo/object/object_reader.go index 53080c48d..5e4da900b 100644 --- a/repo/object/object_reader.go +++ b/repo/object/object_reader.go @@ -65,6 +65,7 @@ func (r *objectReader) Read(buffer []byte) (int, error) { if toCopy == 0 { // EOF on current chunk r.closeCurrentChunk() + r.currentChunkIndex++ continue diff --git a/repo/object/objectid.go b/repo/object/objectid.go index d078cfd93..d6c16e90f 100644 --- a/repo/object/objectid.go +++ b/repo/object/objectid.go @@ -162,19 +162,19 @@ func IndirectObjectID(indexObjectID ID) ID { func ParseID(s string) (ID, error) { var id ID - for len(s) > 0 && s[0] == 'I' { + for s != "" && s[0] == 'I' { id.indirection++ s = s[1:] } - if len(s) > 0 && s[0] == 'Z' { + if s != "" && s[0] == 'Z' { id.compression = true s = s[1:] } - if len(s) > 0 && s[0] == 'D' { + if s != "" && s[0] == 'D' { // no-op, legacy case s = s[1:] } diff --git a/repo/open.go b/repo/open.go index ca2d72fea..2186e2d81 100644 --- a/repo/open.go +++ b/repo/open.go @@ -407,9 +407,11 @@ func wrapLockingStorage(st blob.Storage, r format.BlobStorageConfiguration) blob if strings.HasPrefix(string(id), prefix) { opts.RetentionMode = r.RetentionMode opts.RetentionPeriod = r.RetentionPeriod + break } } + return nil }) } diff --git a/repo/splitter/splitter_buzhash32.go b/repo/splitter/splitter_buzhash32.go index 32b901ed6..09984c543 100644 --- a/repo/splitter/splitter_buzhash32.go +++ b/repo/splitter/splitter_buzhash32.go @@ -57,6 +57,7 @@ func (rs *buzhash32Splitter) NextSplitPoint(b []byte) int { for i, b := range b[0:fp] { rs.rh.Roll(b) + rs.count++ if rs.rh.Sum32()&rs.mask == 0 { diff --git a/repo/splitter/splitter_rabinkarp64.go b/repo/splitter/splitter_rabinkarp64.go index 3618544d3..9dcd9d8a5 100644 --- a/repo/splitter/splitter_rabinkarp64.go +++ b/repo/splitter/splitter_rabinkarp64.go @@ -57,6 +57,7 @@ func (rs *rabinKarp64Splitter) NextSplitPoint(b []byte) int { for i, b := range b[0:fp] { rs.rh.Roll(b) + rs.count++ if rs.rh.Sum64()&rs.mask == 0 { diff --git a/snapshot/policy/policy_manager.go b/snapshot/policy/policy_manager.go index 0100ebe66..68ba71d8b 100644 --- a/snapshot/policy/policy_manager.go +++ b/snapshot/policy/policy_manager.go @@ -65,7 +65,7 @@ func GetPolicyHierarchy(ctx context.Context, rep repo.Repository, si snapshot.So var md []*manifest.EntryMetadata // Find policies applying to paths all the way up to the root. - for tmp := si; len(si.Path) > 0; { + for tmp := si; si.Path != ""; { manifests, err := rep.FindManifests(ctx, LabelsForSource(tmp)) if err != nil { return nil, errors.Wrapf(err, "unable to find manifest for source %v", tmp) diff --git a/snapshot/snapshotfs/dir_rewriter.go b/snapshot/snapshotfs/dir_rewriter.go index e309a76ab..23d8c5b12 100644 --- a/snapshot/snapshotfs/dir_rewriter.go +++ b/snapshot/snapshotfs/dir_rewriter.go @@ -258,6 +258,8 @@ func RewriteKeep(ctx context.Context, parentPath string, input *snapshot.DirEntr // the error. func RewriteAsStub(rep repo.RepositoryWriter) RewriteFailedEntryCallback { return func(ctx context.Context, parentPath string, input *snapshot.DirEntry, originalErr error) (*snapshot.DirEntry, error) { + _ = parentPath + var buf bytes.Buffer e := json.NewEncoder(&buf) diff --git a/snapshot/snapshotfs/estimate.go b/snapshot/snapshotfs/estimate.go index 931e0afad..1f79c7934 100644 --- a/snapshot/snapshotfs/estimate.go +++ b/snapshot/snapshotfs/estimate.go @@ -83,6 +83,8 @@ func Estimate(ctx context.Context, entry fs.Directory, policyTree *policy.Tree, }() onIgnoredFile := func(ctx context.Context, relativePath string, e fs.Entry, pol *policy.Tree) { + _ = pol + if e.IsDir() { if len(ed) < maxExamplesPerBucket { ed = append(ed, relativePath) diff --git a/snapshot/snapshotfs/snapshot_storage_stats.go b/snapshot/snapshotfs/snapshot_storage_stats.go index d9ac7304e..d923416bf 100644 --- a/snapshot/snapshotfs/snapshot_storage_stats.go +++ b/snapshot/snapshotfs/snapshot_storage_stats.go @@ -33,6 +33,8 @@ func CalculateStorageStats(ctx context.Context, rep repo.Repository, manifests [ tw, twerr := NewTreeWalker(ctx, TreeWalkerOptions{ EntryCallback: func(ctx context.Context, entry fs.Entry, oid object.ID, entryPath string) error { + _ = entryPath + if !entry.IsDir() { atomic.AddInt32(&unique.FileObjectCount, 1) atomic.AddInt32(&runningTotal.FileObjectCount, 1) diff --git a/snapshot/snapshotfs/snapshot_tree_walker.go b/snapshot/snapshotfs/snapshot_tree_walker.go index fd945a7f6..b54ad9e1c 100644 --- a/snapshot/snapshotfs/snapshot_tree_walker.go +++ b/snapshot/snapshotfs/snapshot_tree_walker.go @@ -131,7 +131,7 @@ func (w *TreeWalker) processDirEntry(ctx context.Context, dir fs.Directory, entr childPath := path.Join(entryPath, ent2.Name()) if ag.CanShareWork(w.wp) { - ag.RunAsync(w.wp, func(c *workshare.Pool[any], request any) { + ag.RunAsync(w.wp, func(_ *workshare.Pool[any], _ any) { w.processEntry(ctx, ent2, childPath) }, nil) } else { diff --git a/snapshot/snapshotfs/upload.go b/snapshot/snapshotfs/upload.go index cc6366779..a928c6ff5 100644 --- a/snapshot/snapshotfs/upload.go +++ b/snapshot/snapshotfs/upload.go @@ -190,7 +190,7 @@ func (u *Uploader) uploadFileInternal(ctx context.Context, parentCheckpointRegis if wg.CanShareWork(u.workerPool) { // another goroutine is available, delegate to them - wg.RunAsync(u.workerPool, func(c *workshare.Pool[*uploadWorkItem], request *uploadWorkItem) { + wg.RunAsync(u.workerPool, func(_ *workshare.Pool[*uploadWorkItem], _ *uploadWorkItem) { parts[i], partErrors[i] = u.uploadFileData(ctx, parentCheckpointRegistry, f, uuid.NewString(), offset, length, comp) }, nil) } else { @@ -809,7 +809,7 @@ func (u *Uploader) processDirectoryEntries( entryRelativePath := path.Join(dirRelativePath, entry2.Name()) if wg.CanShareWork(u.workerPool) { - wg.RunAsync(u.workerPool, func(c *workshare.Pool[*uploadWorkItem], wi *uploadWorkItem) { + wg.RunAsync(u.workerPool, func(_ *workshare.Pool[*uploadWorkItem], wi *uploadWorkItem) { wi.err = u.processSingle(ctx, entry2, entryRelativePath, parentDirBuilder, policyTree, prevDirs, localDirPathOrEmpty, parentCheckpointRegistry) }, &uploadWorkItem{}) } else { @@ -1154,6 +1154,7 @@ func uploadDirInternal( } checkpointManifest := thisCheckpointBuilder.Build(fs.UTCTimestampFromTime(directory.ModTime()), IncompleteReasonCheckpoint) + oid, err := writeDirManifest(ctx, u.repo, dirRelativePath, checkpointManifest) if err != nil { return nil, errors.Wrap(err, "error writing dir manifest") diff --git a/snapshot/snapshotfs/upload_os_snapshot_windows.go b/snapshot/snapshotfs/upload_os_snapshot_windows.go index eebe39af2..fe219dd32 100644 --- a/snapshot/snapshotfs/upload_os_snapshot_windows.go +++ b/snapshot/snapshotfs/upload_os_snapshot_windows.go @@ -71,7 +71,6 @@ func createOSSnapshot(ctx context.Context, root fs.Directory, _ *policy.OSSnapsh } newRoot, err = localfs.Directory(filepath.Join(sc.DeviceObject, rel)) - if err != nil { return nil, nil, err } diff --git a/snapshot/snapshotgc/gc.go b/snapshot/snapshotgc/gc.go index f8bf468c5..1f4d64e5b 100644 --- a/snapshot/snapshotgc/gc.go +++ b/snapshot/snapshotgc/gc.go @@ -35,7 +35,7 @@ func findInUseContentIDs(ctx context.Context, rep repo.Repository, used *bigmap. } w, twerr := snapshotfs.NewTreeWalker(ctx, snapshotfs.TreeWalkerOptions{ - EntryCallback: func(ctx context.Context, entry fs.Entry, oid object.ID, entryPath string) error { + EntryCallback: func(ctx context.Context, _ fs.Entry, oid object.ID, _ string) error { contentIDs, verr := rep.VerifyObject(ctx, oid) if verr != nil { return errors.Wrapf(verr, "error verifying %v", oid) @@ -128,6 +128,7 @@ func runInternal(ctx context.Context, rep repo.DirectRepositoryWriter, gcDelete if err := rep.ContentManager().UndeleteContent(ctx, ci.GetContentID()); err != nil { return errors.Wrapf(err, "Could not undelete referenced content: %v", ci) } + undeleted.Add(int64(ci.GetPackedLength())) } @@ -154,6 +155,7 @@ func runInternal(ctx context.Context, rep repo.DirectRepositoryWriter, gcDelete if cnt%100000 == 0 { log(ctx).Infof("... found %v unused contents so far (%v bytes)", cnt, units.BytesString(totalSize)) + if gcDelete { if err := rep.Flush(ctx); err != nil { return errors.Wrap(err, "flush error") diff --git a/tests/htmlui_e2e_test/htmlui_e2e_test.go b/tests/htmlui_e2e_test/htmlui_e2e_test.go index eb0ab8698..3c72e75e7 100644 --- a/tests/htmlui_e2e_test/htmlui_e2e_test.go +++ b/tests/htmlui_e2e_test/htmlui_e2e_test.go @@ -12,6 +12,7 @@ "github.com/chromedp/cdproto/page" "github.com/chromedp/chromedp" "github.com/chromedp/chromedp/kb" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/kopia/kopia/internal/testutil" @@ -90,8 +91,8 @@ func runInBrowser(t *testing.T, run func(ctx context.Context, sp *testutil.Serve t.Logf("dialog opening: %v", do.Message) go func() { - require.Equal(t, tc.expectedDialogText, do.Message) - require.NoError(t, chromedp.Run(ctx, page.HandleJavaScriptDialog(tc.dialogResponse))) + assert.Equal(t, tc.expectedDialogText, do.Message) + assert.NoError(t, chromedp.Run(ctx, page.HandleJavaScriptDialog(tc.dialogResponse))) tc.expectedDialogText = "" }() } diff --git a/tools/gettool/autodownload/autodownload.go b/tools/gettool/autodownload/autodownload.go index 583fa1910..3caffb16f 100644 --- a/tools/gettool/autodownload/autodownload.go +++ b/tools/gettool/autodownload/autodownload.go @@ -213,6 +213,10 @@ type InvalidChecksumError struct { } func (e InvalidChecksumError) Error() string { + if e.expected == "" { + return fmt.Sprintf("missing checksum: %v", e.actual) + } + return fmt.Sprintf("invalid checksum: %v, wanted %v", e.actual, e.expected) } @@ -247,7 +251,7 @@ func downloadInternal(url, dir string, checksum map[string]string, stripPathComp switch { case checksum[url] == "": checksum[url] = actualChecksum - return errors.Errorf("missing checksum - calculated as %v", actualChecksum) + return InvalidChecksumError{actualChecksum, ""} case checksum[url] != actualChecksum: return InvalidChecksumError{actualChecksum, checksum[url]} diff --git a/tools/gettool/checksums.txt b/tools/gettool/checksums.txt index cdf63cd1a..30ab61283 100644 --- a/tools/gettool/checksums.txt +++ b/tools/gettool/checksums.txt @@ -7,12 +7,12 @@ https://github.com/git-chglog/git-chglog/releases/download/v0.15.1/git-chglog_0. https://github.com/gohugoio/hugo/releases/download/v0.113.0/hugo_extended_0.113.0_darwin-universal.tar.gz: 1557f896f34743d241e1aecab588be273dde59692b362a9f4488231a2595b2ae https://github.com/gohugoio/hugo/releases/download/v0.113.0/hugo_extended_0.113.0_linux-amd64.tar.gz: e04bccfa81df6c727f1c03bc858eb21d6f95123d311cafe245f4485d289123f3 https://github.com/gohugoio/hugo/releases/download/v0.113.0/hugo_extended_0.113.0_windows-amd64.zip: 3eabfbfad1431939058e6f7e76573c6bac1fee92f3a7b1ac5739c555940f0e0e -https://github.com/golangci/golangci-lint/releases/download/v1.55.2/golangci-lint-1.55.2-darwin-amd64.tar.gz: 632e96e6d5294fbbe7b2c410a49c8fa01c60712a0af85a567de85bcc1623ea21 -https://github.com/golangci/golangci-lint/releases/download/v1.55.2/golangci-lint-1.55.2-darwin-arm64.tar.gz: 234463f059249f82045824afdcdd5db5682d0593052f58f6a3039a0a1c3899f6 -https://github.com/golangci/golangci-lint/releases/download/v1.55.2/golangci-lint-1.55.2-linux-amd64.tar.gz: ca21c961a33be3bc15e4292dc40c98c8dcc5463a7b6768a3afc123761630c09c -https://github.com/golangci/golangci-lint/releases/download/v1.55.2/golangci-lint-1.55.2-linux-arm64.tar.gz: 8eb0cee9b1dbf0eaa49871798c7f8a5b35f2960c52d776a5f31eb7d886b92746 -https://github.com/golangci/golangci-lint/releases/download/v1.55.2/golangci-lint-1.55.2-linux-armv6.tar.gz: 3195f3e0f37d353fd5bd415cabcd4e263f5c29d3d0ffb176c26ff3d2c75eb3bb -https://github.com/golangci/golangci-lint/releases/download/v1.55.2/golangci-lint-1.55.2-windows-amd64.zip: f57d434d231d43417dfa631587522f8c1991220b43c8ffadb9c7bd279508bf81 +https://github.com/golangci/golangci-lint/releases/download/v1.56.2/golangci-lint-1.56.2-darwin-amd64.tar.gz: 15c4d19a2c85a04f67779047dbb9467ba176c71fff762a0d514a21bb75e4b42c +https://github.com/golangci/golangci-lint/releases/download/v1.56.2/golangci-lint-1.56.2-darwin-arm64.tar.gz: 5f9ecda712c7ae08fbf872336fae3db866720e5865903d4c53903184b2a2c2dc +https://github.com/golangci/golangci-lint/releases/download/v1.56.2/golangci-lint-1.56.2-linux-amd64.tar.gz: e1c313fb5fc85a33890fdee5dbb1777d1f5829c84d655a47a55688f3aad5e501 +https://github.com/golangci/golangci-lint/releases/download/v1.56.2/golangci-lint-1.56.2-linux-arm64.tar.gz: 0041594fde41ce43b75e65476a050fe9057881d8b5bccd472f18357e2ead3e04 +https://github.com/golangci/golangci-lint/releases/download/v1.56.2/golangci-lint-1.56.2-linux-armv6.tar.gz: 4820c7b0a2832812bc243328c5046bc06cca71874ca31f0001e2f8f5effaa0d7 +https://github.com/golangci/golangci-lint/releases/download/v1.56.2/golangci-lint-1.56.2-windows-amd64.zip: d02df32c581281ef46af62d4ab71da6e24b47ef3e93e7f05d719fada74440185 https://github.com/goreleaser/goreleaser/releases/download/v0.176.0/goreleaser_Darwin_arm64.tar.gz: 1f95e6561974f4766d8833438b646b06930563ca9867447ea03edb623d876c75 https://github.com/goreleaser/goreleaser/releases/download/v0.176.0/goreleaser_Darwin_x86_64.tar.gz: 17ecad881a50e32f033da5a200c8417d37cae70f09e925645452937998aca506 https://github.com/goreleaser/goreleaser/releases/download/v0.176.0/goreleaser_Linux_arm64.tar.gz: 8bf2a9b9e84498bfa239f2fe91b2d555642c87ab9d3f5d37f29e6e97116910a3 diff --git a/tools/gettool/gettool.go b/tools/gettool/gettool.go index 94dfdc601..1fc1f997a 100644 --- a/tools/gettool/gettool.go +++ b/tools/gettool/gettool.go @@ -9,6 +9,7 @@ "flag" "fmt" "log" + "os" "path/filepath" "runtime" "sort" @@ -131,7 +132,7 @@ func (ti ToolInfo) actualURL(version, goos, goarch string) string { goarch = flag.String("goarch", runtime.GOARCH, "Override GOARCH") testAll = flag.Bool("test-all", false, "Unpacks the package for all GOOS/ARCH combinations") - regenerateChecksums = flag.Bool("regenerate-checksums", false, "Regenerate checksums") + regenerateChecksums = flag.String("regenerate-checksums", "", "Regenerate checksums") ) //nolint:gochecknoglobals @@ -179,6 +180,7 @@ func main() { } checksums := parseEmbeddedChecksums() + downloadedChecksums := map[string]string{} var errorCount int @@ -193,20 +195,20 @@ func main() { toolName := parts[0] toolVersion := parts[1] - if err := downloadTool(toolName, toolVersion, checksums, &errorCount); err != nil { + if err := downloadTool(toolName, toolVersion, checksums, downloadedChecksums, &errorCount); err != nil { log.Fatalf("unable to download %v version %v: %v", toolName, toolVersion, err) } } // all good - if errorCount == 0 && !*regenerateChecksums { + if errorCount == 0 && *regenerateChecksums == "" { return } // on failure print current checksums, so they can be copy/pasted as the new baseline var lines []string - for k, v := range checksums { + for k, v := range downloadedChecksums { lines = append(lines, fmt.Sprintf("%v: %v", k, v)) } @@ -216,14 +218,33 @@ func main() { fmt.Println(l) } - if *regenerateChecksums { + if *regenerateChecksums != "" { + if err := writeLinesToFile(lines); err != nil { + log.Fatal(err) + } + return } log.Fatalf("Error(s) encountered, see log messages above.") } -func downloadTool(toolName, toolVersion string, checksums map[string]string, errorCount *int) error { +func writeLinesToFile(lines []string) error { + f, err := os.Create(*regenerateChecksums) + if err != nil { + return errors.Wrap(err, "writeLinesToFile") + } + + defer f.Close() //nolint:errcheck + + for _, l := range lines { + fmt.Fprintln(f, l) + } + + return nil +} + +func downloadTool(toolName, toolVersion string, oldChecksums, downloadedChecksums map[string]string, errorCount *int) error { t, ok := tools[toolName] if !ok { return errors.Errorf("unsupported tool: %q", toolName) @@ -236,7 +257,7 @@ func downloadTool(toolName, toolVersion string, checksums map[string]string, err continue } - if err := autodownload.Download(u, filepath.Join(*outputDir, ba.goos, ba.goarch), checksums, t.stripPathComponents); err != nil { + if err := autodownload.Download(u, filepath.Join(*outputDir, ba.goos, ba.goarch), oldChecksums, t.stripPathComponents); err != nil { log.Printf("ERROR %v: %v", u, err) *errorCount++ @@ -246,20 +267,21 @@ func downloadTool(toolName, toolVersion string, checksums map[string]string, err return nil } - if *regenerateChecksums { + if *regenerateChecksums != "" { for _, ba := range buildArchitectures { u := t.actualURL(toolVersion, ba.goos, ba.goarch) if u == "" { continue } - if checksums[u] != "" { + if oldChecksums[u] != "" { + downloadedChecksums[u] = oldChecksums[u] continue } log.Printf("downloading %v...", u) - if err := autodownload.Download(u, filepath.Join(*outputDir, ba.goos, ba.goarch), checksums, t.stripPathComponents); err != nil { + if err := autodownload.Download(u, filepath.Join(*outputDir, ba.goos, ba.goarch), downloadedChecksums, t.stripPathComponents); err != nil { log.Printf("ERROR %v: %v", u, err) *errorCount++ @@ -276,7 +298,7 @@ func downloadTool(toolName, toolVersion string, checksums map[string]string, err fmt.Printf("Downloading %v version %v from %v...\n", toolName, toolVersion, u) - if err := autodownload.Download(u, *outputDir, checksums, t.stripPathComponents); err != nil { + if err := autodownload.Download(u, *outputDir, oldChecksums, t.stripPathComponents); err != nil { return errors.Wrap(err, "unable to download") } diff --git a/tools/tools.mk b/tools/tools.mk index 13f8e4a97..2437ebea3 100644 --- a/tools/tools.mk +++ b/tools/tools.mk @@ -102,7 +102,7 @@ retry:= endif # tool versions -GOLANGCI_LINT_VERSION=1.55.2 +GOLANGCI_LINT_VERSION=1.56.2 CHECKLOCKS_VERSION=e8c1fff214d0ecf02cfe5aa9c62d11174130c339 NODE_VERSION=18.16.0 HUGO_VERSION=0.113.0 @@ -303,9 +303,8 @@ verify-all-tool-checksums: --tool $(ALL_TOOL_VERSIONS) regenerate-checksums: - go run github.com/kopia/kopia/tools/gettool --regenerate-checksums \ + go run github.com/kopia/kopia/tools/gettool --regenerate-checksums $(CURDIR)/tools/gettool/checksums.txt \ --output-dir /tmp/all-tools \ --tool $(ALL_TOOL_VERSIONS) all-tools: $(gotestsum) $(npm) $(linter) $(maybehugo) -