mirror of
https://github.com/kopia/kopia.git
synced 2026-01-28 00:08:04 -05:00
d6477377f9
instead of having time-based naming, block manager will perform occasional compaction at startup time and delete unwanted blocks The protocol is safe from concurrency standpoint and provides eventual consistency. We only delete blocks after creating compacted blocks. We retry loading index until (any) consistent index is fetched (possibly from cache) and all underlying blocks are also available, not necessarily the latest ones. TODO - we need to periodically snapshot block index contents, so that if we have a bug somewhere in compaction code, we have a way of restoring working indexes.
70 lines
2.0 KiB
Go
70 lines
2.0 KiB
Go
package block
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"path/filepath"
|
|
"time"
|
|
|
|
"github.com/kopia/kopia/storage"
|
|
"github.com/kopia/kopia/storage/filesystem"
|
|
)
|
|
|
|
type blockCache interface {
|
|
getBlock(ctx context.Context, cacheKey string, physicalBlockID PhysicalBlockID, offset, length int64) ([]byte, error)
|
|
putBlock(ctx context.Context, blockID PhysicalBlockID, data []byte) error
|
|
listIndexBlocks(ctx context.Context) ([]IndexInfo, error)
|
|
deleteListCache(ctx context.Context)
|
|
deleteBlock(ctx context.Context, blockID PhysicalBlockID) error
|
|
close() error
|
|
}
|
|
|
|
// CachingOptions specifies configuration of local cache.
|
|
type CachingOptions struct {
|
|
CacheDirectory string `json:"cacheDirectory,omitempty"`
|
|
MaxCacheSizeBytes int64 `json:"maxCacheSize,omitempty"`
|
|
MaxListCacheDurationSec int `json:"maxListCacheDuration,omitempty"`
|
|
IgnoreListCache bool `json:"-"`
|
|
HMACSecret []byte `json:"-"`
|
|
}
|
|
|
|
func newBlockCache(ctx context.Context, st storage.Storage, caching CachingOptions) (blockCache, error) {
|
|
if caching.MaxCacheSizeBytes == 0 || caching.CacheDirectory == "" {
|
|
return nullBlockCache{st}, nil
|
|
}
|
|
|
|
blockCacheDir := filepath.Join(caching.CacheDirectory, "blocks")
|
|
|
|
if _, err := os.Stat(blockCacheDir); os.IsNotExist(err) {
|
|
if err := os.MkdirAll(blockCacheDir, 0700); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
cacheStorage, err := filesystem.New(context.Background(), &filesystem.Options{
|
|
Path: blockCacheDir,
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
c := &localStorageCache{
|
|
st: st,
|
|
cacheStorage: cacheStorage,
|
|
maxSizeBytes: caching.MaxCacheSizeBytes,
|
|
hmacSecret: append([]byte(nil), caching.HMACSecret...),
|
|
listCacheDuration: time.Duration(caching.MaxListCacheDurationSec) * time.Second,
|
|
closed: make(chan struct{}),
|
|
}
|
|
|
|
if caching.IgnoreListCache {
|
|
c.deleteListCache(ctx)
|
|
}
|
|
|
|
if err := c.sweepDirectory(ctx); err != nil {
|
|
return nil, err
|
|
}
|
|
go c.sweepDirectoryPeriodically(ctx)
|
|
|
|
return c, nil
|
|
}
|