diff --git a/block/block_manager.go b/block/block_manager.go index d537662f4..014dc13e6 100644 --- a/block/block_manager.go +++ b/block/block_manager.go @@ -750,7 +750,7 @@ func (bm *Manager) writePackDataNotLocked(ctx context.Context, data []byte) (str return "", fmt.Errorf("unable to read crypto bytes: %v", err) } - physicalBlockID := string(fmt.Sprintf("%v%x", PackBlockPrefix, blockID)) + physicalBlockID := fmt.Sprintf("%v%x", PackBlockPrefix, blockID) atomic.AddInt32(&bm.stats.WrittenBlocks, 1) atomic.AddInt64(&bm.stats.WrittenBytes, int64(len(data))) @@ -763,7 +763,7 @@ func (bm *Manager) writePackDataNotLocked(ctx context.Context, data []byte) (str func (bm *Manager) encryptAndWriteBlockNotLocked(ctx context.Context, data []byte, prefix string) (string, error) { hash := bm.hashData(data) - physicalBlockID := string(prefix + hex.EncodeToString(hash)) + physicalBlockID := prefix + hex.EncodeToString(hash) // Encrypt the block in-place. atomic.AddInt64(&bm.stats.EncryptedBytes, int64(len(data))) @@ -900,7 +900,7 @@ func (bm *Manager) decryptAndVerifyPayload(formatVersion byte, payload []byte, o } func (bm *Manager) getPhysicalBlockInternal(ctx context.Context, blockID string) ([]byte, error) { - payload, err := bm.cache.getBlock(ctx, string(blockID), blockID, 0, -1) + payload, err := bm.cache.getBlock(ctx, blockID, blockID, 0, -1) if err != nil { return nil, err } @@ -932,8 +932,7 @@ func getPackedBlockIV(blockID string) ([]byte, error) { return hex.DecodeString(blockID[len(blockID)-(aes.BlockSize*2):]) } -func getPhysicalBlockIV(b string) ([]byte, error) { - s := string(b) +func getPhysicalBlockIV(s string) ([]byte, error) { if p := strings.Index(s, "-"); p >= 0 { s = s[0:p] } @@ -993,7 +992,7 @@ func listIndexBlocksFromStorage(ctx context.Context, st storage.Storage) ([]Inde } ii := IndexInfo{ - FileName: string(it.BlockID), + FileName: it.BlockID, Timestamp: it.TimeStamp, Length: it.Length, } diff --git a/block/local_storage_cache.go b/block/local_storage_cache.go index b2dfdc4c5..f8bbefb90 100644 --- a/block/local_storage_cache.go +++ b/block/local_storage_cache.go @@ -51,7 +51,7 @@ func (c *localStorageCache) getBlock(ctx context.Context, cacheKey string, physi log.Warn().Msgf("unable to read cache %v: %v", cacheKey, err) } - b, err = c.st.GetBlock(ctx, string(physicalBlockID), offset, length) + b, err = c.st.GetBlock(ctx, physicalBlockID, offset, length) if err == storage.ErrBlockNotFound { // not found in underlying storage return nil, err @@ -76,12 +76,12 @@ func (c *localStorageCache) writeToCacheBestEffort(ctx context.Context, cacheKey func (c *localStorageCache) putBlock(ctx context.Context, blockID string, data []byte) error { c.deleteListCache(ctx) - return c.st.PutBlock(ctx, string(blockID), bytes.NewReader(data)) + return c.st.PutBlock(ctx, blockID, bytes.NewReader(data)) } func (c *localStorageCache) deleteBlock(ctx context.Context, blockID string) error { c.deleteListCache(ctx) - return c.st.DeleteBlock(ctx, string(blockID)) + return c.st.DeleteBlock(ctx, blockID) } func (c *localStorageCache) listIndexBlocks(ctx context.Context) ([]IndexInfo, error) { diff --git a/block/null_block_cache.go b/block/null_block_cache.go index 5fc7bfd67..5e9018ea1 100644 --- a/block/null_block_cache.go +++ b/block/null_block_cache.go @@ -12,11 +12,11 @@ type nullBlockCache struct { } func (c nullBlockCache) getBlock(ctx context.Context, cacheKey string, blockID string, offset, length int64) ([]byte, error) { - return c.st.GetBlock(ctx, string(blockID), offset, length) + return c.st.GetBlock(ctx, blockID, offset, length) } func (c nullBlockCache) putBlock(ctx context.Context, blockID string, data []byte) error { - return c.st.PutBlock(ctx, string(blockID), bytes.NewReader(data)) + return c.st.PutBlock(ctx, blockID, bytes.NewReader(data)) } func (c nullBlockCache) listIndexBlocks(ctx context.Context) ([]IndexInfo, error) { @@ -27,7 +27,7 @@ func (c nullBlockCache) deleteListCache(ctx context.Context) { } func (c nullBlockCache) deleteBlock(ctx context.Context, blockID string) error { - return c.st.DeleteBlock(ctx, string(blockID)) + return c.st.DeleteBlock(ctx, blockID) } func (c nullBlockCache) close() error { diff --git a/block/simple_committed_block_index.go b/block/simple_committed_block_index.go index 58c6ee09f..e1672f4c9 100644 --- a/block/simple_committed_block_index.go +++ b/block/simple_committed_block_index.go @@ -57,7 +57,7 @@ func (b *simpleCommittedBlockIndex) hasIndexBlockID(indexBlockID string) (bool, } func (b *simpleCommittedBlockIndex) indexBlockPath(indexBlockID string) string { - return filepath.Join(b.dirname, string(indexBlockID+simpleIndexSuffix)) + return filepath.Join(b.dirname, indexBlockID+simpleIndexSuffix) } func (b *simpleCommittedBlockIndex) addBlockToCache(indexBlockID string, data []byte) error { @@ -159,7 +159,7 @@ func (b *simpleCommittedBlockIndex) use(packFiles []string) error { } for _, e := range packFiles { - fname := string(e) + simpleIndexSuffix + fname := e + simpleIndexSuffix delete(remaining, fname) fullpath := filepath.Join(b.dirname, fname) ndx, err := b.openIndex(fullpath) diff --git a/cli/analytics.go b/cli/analytics.go index bc8a959ee..c21d36564 100644 --- a/cli/analytics.go +++ b/cli/analytics.go @@ -24,7 +24,7 @@ var ( clientIDFile = app.Flag("client-id-file", "Path to client ID file, which enables anonymous usage reporting if present").Default(filepath.Join(ospath.ConfigDir(), "client_id.txt")).String() - analyticsConsent = app.Flag("analytics-consent", "Consent to send analytics").Default("ask").Enum("yes", "no", "ask") + analyticsConsent = app.Flag("analytics-consent", "Consent to send analytics").Default("ask").Enum("agree", "disagree", "ask") globalGAClient *ga.Client gaClientOnce sync.Once @@ -79,10 +79,10 @@ func promptForAnalyticsConsent() { clientID := fmt.Sprintf("%x", x) switch *analyticsConsent { - case "no": + case "disagree": disableAnalytics() - case "yes": + case "agree": enableAnalytics(clientID) case "ask": diff --git a/cli/command_block_gc.go b/cli/command_block_gc.go index 3b52b91f8..27cc44117 100644 --- a/cli/command_block_gc.go +++ b/cli/command_block_gc.go @@ -4,7 +4,6 @@ "context" "fmt" "os" - "strings" "github.com/kopia/kopia/block" "github.com/kopia/kopia/repo" @@ -23,7 +22,7 @@ func runBlockGarbageCollectAction(ctx context.Context, rep *repo.Repository) err } usedPackBlocks := findPackBlocksInUse(infos) - ch := rep.Storage.ListBlocks(ctx, "") + ch := rep.Storage.ListBlocks(ctx, block.PackBlockPrefix) var unused []string var totalBytes int64 @@ -33,22 +32,15 @@ func runBlockGarbageCollectAction(ctx context.Context, rep *repo.Repository) err return fmt.Errorf("error listing storage blocks: %v", bi.Error) } - if strings.HasPrefix(bi.BlockID, "n") { - continue - } - if strings.HasPrefix(bi.BlockID, "kopia") { - continue - } - allPackBlocks++ - u := usedPackBlocks[string(bi.BlockID)] + u := usedPackBlocks[bi.BlockID] if u > 0 { log.Printf("pack %v, in use by %v blocks", bi.BlockID, u) continue } - totalBytes += int64(bi.Length) + totalBytes += bi.Length unused = append(unused, bi.BlockID) } fmt.Fprintf(os.Stderr, "Found %v/%v pack blocks in use.\n", len(usedPackBlocks), allPackBlocks) diff --git a/cli/command_block_index_show.go b/cli/command_block_index_show.go index e0dbd9a51..c5b6392a6 100644 --- a/cli/command_block_index_show.go +++ b/cli/command_block_index_show.go @@ -21,9 +21,8 @@ func getIndexBlocksToShow(ctx context.Context, rep *repo.Repository) ([]string, error) { var blockIDs []string - for _, id := range *blockIndexShowIDs { - blockIDs = append(blockIDs, string(id)) - } + + blockIDs = append(blockIDs, *blockIndexShowIDs...) if len(blockIDs) == 1 && blockIDs[0] == "active" { b, err := rep.Blocks.IndexBlocks(ctx) diff --git a/cli/command_block_rewrite.go b/cli/command_block_rewrite.go index ebc5379e4..29bdd17d0 100644 --- a/cli/command_block_rewrite.go +++ b/cli/command_block_rewrite.go @@ -55,52 +55,88 @@ func runRewriteBlocksAction(ctx context.Context, rep *repo.Repository) error { } func getBlocksToRewrite(ctx context.Context, rep *repo.Repository) ([]block.Info, error) { + // get blocks listed on command line + result, err := getBlockInfos(ctx, rep, *blockRewriteIDs) + if err != nil { + return nil, err + } + + // add all blocks from short packs + if *blockRewriteShortPacks { + threshold := uint32(rep.Blocks.Format.MaxPackSize * 6 / 10) + info, err := getBlocksInShortPacks(ctx, rep, threshold) + if err != nil { + return nil, err + } + + result = append(result, info...) + } + + // add all blocks with given format version + if *blockRewriteFormatVersion != -1 { + info, err := getBlocksWithFormatVersion(ctx, rep, *blockRewriteFormatVersion) + if err != nil { + return nil, err + } + + result = append(result, info...) + } + + return result, nil +} + +func getBlockInfos(ctx context.Context, rep *repo.Repository, blockIDs []string) ([]block.Info, error) { var result []block.Info - for _, blockID := range *blockRewriteIDs { + for _, blockID := range blockIDs { i, err := rep.Blocks.BlockInfo(ctx, blockID) if err != nil { return nil, fmt.Errorf("unable to get info for block %q: %v", blockID, err) } result = append(result, i) } + return result, nil +} - if *blockRewriteShortPacks { - infos, err := rep.Blocks.ListBlockInfos("", true) - if err != nil { - return nil, fmt.Errorf("unable to list index blocks: %v", err) - } +func getBlocksWithFormatVersion(ctx context.Context, rep *repo.Repository, version int) ([]block.Info, error) { + var result []block.Info - threshold := uint32(rep.Blocks.Format.MaxPackSize * 6 / 10) - shortPackBlocks, err := findShortPackBlocks(infos, threshold) - if err != nil { - return nil, fmt.Errorf("unable to find short pack blocks: %v", err) - } - log.Printf("found %v short pack blocks", len(shortPackBlocks)) + infos, err := rep.Blocks.ListBlockInfos("", true) + if err != nil { + return nil, fmt.Errorf("unable to list index blocks: %v", err) + } - if len(shortPackBlocks) <= 1 { - fmt.Printf("Nothing to do, found %v short pack blocks\n", len(shortPackBlocks)) - } else { - for _, b := range infos { - if shortPackBlocks[b.PackFile] && strings.HasPrefix(string(b.PackFile), *blockRewritePackPrefix) { - result = append(result, b) - } - } + for _, b := range infos { + if int(b.FormatVersion) == *blockRewriteFormatVersion && strings.HasPrefix(b.PackFile, *blockRewritePackPrefix) { + result = append(result, b) } } - if *blockRewriteFormatVersion != -1 { - infos, err := rep.Blocks.ListBlockInfos("", true) - if err != nil { - return nil, fmt.Errorf("unable to list index blocks: %v", err) - } + return result, nil +} +func getBlocksInShortPacks(ctx context.Context, rep *repo.Repository, threshold uint32) ([]block.Info, error) { + var result []block.Info + + infos, err := rep.Blocks.ListBlockInfos("", true) + if err != nil { + return nil, fmt.Errorf("unable to list index blocks: %v", err) + } + + shortPackBlocks, err := findShortPackBlocks(infos, threshold) + if err != nil { + return nil, fmt.Errorf("unable to find short pack blocks: %v", err) + } + log.Printf("found %v short pack blocks", len(shortPackBlocks)) + + if len(shortPackBlocks) <= 1 { + fmt.Printf("Nothing to do, found %v short pack blocks\n", len(shortPackBlocks)) + } else { for _, b := range infos { - if int(b.FormatVersion) == *blockRewriteFormatVersion && strings.HasPrefix(string(b.PackFile), *blockRewritePackPrefix) { + if shortPackBlocks[b.PackFile] && strings.HasPrefix(b.PackFile, *blockRewritePackPrefix) { result = append(result, b) } } } - return result, nil }