From 06b4bec2461fab7c771cb95b8cc4fdd844d71b2f Mon Sep 17 00:00:00 2001 From: Jarek Kowalski Date: Sat, 10 Feb 2018 12:50:21 -0800 Subject: [PATCH] added 'storage sweep' subcommand which removes unreferenced storage blocks --- block/block_manager.go | 9 +++++ cli/command_block_index_show.go | 21 ++++++++++-- cli/command_storage_sweep.go | 60 +++++++++++++++++++++++++++++++++ repo/connect.go | 2 +- repo/format_block.go | 7 ++-- repo/open.go | 2 +- 6 files changed, 93 insertions(+), 8 deletions(-) create mode 100644 cli/command_storage_sweep.go diff --git a/block/block_manager.go b/block/block_manager.go index 904a97949..6cefa5acf 100644 --- a/block/block_manager.go +++ b/block/block_manager.go @@ -560,6 +560,15 @@ func (bm *Manager) WriteBlock(data []byte, prefix string) (string, error) { return blockID, err } +// IsStorageBlockInUse determines whether given storage block is in use by currently loaded pack indexes. +func (bm *Manager) IsStorageBlockInUse(storageBlockID string) bool { + bm.lock() + defer bm.unlock() + + bm.ensurePackIndexesLoaded() + return bm.packBlockIDToIndex[storageBlockID] != nil +} + // Repackage reorganizes all pack blocks belonging to a given group that are not bigger than given size. func (bm *Manager) Repackage(maxLength uint64) error { bm.lock() diff --git a/cli/command_block_index_show.go b/cli/command_block_index_show.go index 636820355..acf4cdb16 100644 --- a/cli/command_block_index_show.go +++ b/cli/command_block_index_show.go @@ -21,7 +21,24 @@ func runShowBlockIndexesAction(context *kingpin.ParseContext) error { rep := mustOpenRepository(nil) defer rep.Close() - for _, blockID := range *blockIndexShowIDs { + blockIDs := *blockIndexShowIDs + if len(blockIDs) == 1 || blockIDs[0] == "active" { + b, err := rep.Blocks.ActiveIndexBlocks() + if err != nil { + return err + } + + sort.Slice(b, func(i, j int) bool { + return b[i].Timestamp.Before(b[j].Timestamp) + }) + + blockIDs = nil + for _, bi := range b { + blockIDs = append(blockIDs, bi.BlockID) + } + } + + for _, blockID := range blockIDs { data, err := rep.Blocks.GetBlock(blockID) if err != nil { return fmt.Errorf("can't read block %q: %v", blockID, err) @@ -33,7 +50,7 @@ func runShowBlockIndexesAction(context *kingpin.ParseContext) error { } for _, ndx := range d.Indexes { - fmt.Printf("pack %v len: %v created %v\n", ndx.PackBlockId, ndx.PackLength, time.Unix(0, int64(ndx.CreateTimeNanos)).Local()) + fmt.Printf("pack:%v len:%v created:%v\n", ndx.PackBlockId, ndx.PackLength, time.Unix(0, int64(ndx.CreateTimeNanos)).Local()) type unpacked struct { blockID string offset uint32 diff --git a/cli/command_storage_sweep.go b/cli/command_storage_sweep.go new file mode 100644 index 000000000..7b23469e1 --- /dev/null +++ b/cli/command_storage_sweep.go @@ -0,0 +1,60 @@ +package cli + +import ( + "fmt" + "time" + + "github.com/kopia/kopia/repo" + + kingpin "gopkg.in/alecthomas/kingpin.v2" +) + +var ( + storageSweepCommand = storageCommands.Command("sweep", "Remove unused storage blocks") + storageSweepAgeThreshold = storageSweepCommand.Flag("min-age", "Minimum age of storage block to delete").Default("24h").Duration() + storageSweepPrintUnused = storageSweepCommand.Flag("print", "Do not delete blocks, but print which ones would be deleted").Short('n').Bool() +) + +func runStorageSweepAction(context *kingpin.ParseContext) error { + rep := mustOpenRepository(nil) + defer rep.Close() + + indexBlocks, err := rep.Blocks.ActiveIndexBlocks() + if err != nil { + return err + } + + inUseIndexBlocks := map[string]bool{} + for _, ib := range indexBlocks { + inUseIndexBlocks[ib.BlockID] = true + } + + ch, cancel := rep.Storage.ListBlocks("") + defer cancel() + + for bm := range ch { + if bm.Error != nil { + return bm.Error + } + + age := time.Since(bm.TimeStamp) + inUse := rep.Blocks.IsStorageBlockInUse(bm.BlockID) || inUseIndexBlocks[bm.BlockID] || bm.BlockID == repo.FormatBlockID + keep := inUse || age < *storageSweepAgeThreshold + if !keep { + if *storageSweepPrintUnused { + fmt.Printf("unused block %v age %v\n", bm.BlockID, age) + } else { + fmt.Printf("deleting unused block %v age %v\n", bm.BlockID, age) + if err := rep.Storage.DeleteBlock(bm.BlockID); err != nil { + return err + } + } + } + } + + return nil +} + +func init() { + storageSweepCommand.Action(runStorageSweepAction) +} diff --git a/repo/connect.go b/repo/connect.go index 306ef6686..f5fe72972 100644 --- a/repo/connect.go +++ b/repo/connect.go @@ -32,7 +32,7 @@ type ConnectOptions struct { // Connect connects to the repository in the specified storage and persists the configuration and credentials in the file provided. func Connect(ctx context.Context, configFile string, st storage.Storage, creds auth.Credentials, opt ConnectOptions) error { - formatBytes, err := st.GetBlock(formatBlockID, 0, -1) + formatBytes, err := st.GetBlock(FormatBlockID, 0, -1) if err != nil { return fmt.Errorf("unable to read format block: %v", err) } diff --git a/repo/format_block.go b/repo/format_block.go index 54012a673..bd82230dd 100644 --- a/repo/format_block.go +++ b/repo/format_block.go @@ -14,9 +14,8 @@ import ( "github.com/kopia/kopia/storage" ) -const ( - formatBlockID = "kopia.repository" -) +// FormatBlockID is the identifier of a storage block that describes repository format. +const FormatBlockID = "kopia.repository" var ( purposeAESKey = []byte("AES") @@ -59,7 +58,7 @@ func writeFormatBlock(st storage.Storage, f *formatBlock) error { return fmt.Errorf("unable to marshal format block: %v", err) } - if err := st.PutBlock(formatBlockID, buf.Bytes()); err != nil { + if err := st.PutBlock(FormatBlockID, buf.Bytes()); err != nil { return fmt.Errorf("unable to write format block: %v", err) } diff --git a/repo/open.go b/repo/open.go index e3ff4778c..9ffe1af4b 100644 --- a/repo/open.go +++ b/repo/open.go @@ -172,7 +172,7 @@ func readAndCacheFormatBlock(st storage.Storage, cacheDirectory string) (*format } } - b, err := st.GetBlock(formatBlockID, 0, -1) + b, err := st.GetBlock(FormatBlockID, 0, -1) if err != nil { return nil, err }