mirror of
https://github.com/kopia/kopia.git
synced 2026-01-27 07:48:06 -05:00
cead806a3f
* blob: changed default shards from {3,3} to {1,3}
Turns out for very large repository around 100TB (5M blobs),
we end up creating max ~16M directories which is way too much
and slows down listing. Currently each leaf directory only has a handful
of files.
Simple sharding of {3} should work much better and will end up creating
directories with meaningful shard sizes - 12 K files per directory
should not be too slow and will reduce the overhead of listing by
4096 times.
The change is done in a backwards-compatible way and will respect
custom sharding (.shards) file written by previous 0.9 builds
as well as older repositories that don't have the .shards file (which
we assume to be {3,3}).
* fixed compat tests
57 lines
1.5 KiB
Go
57 lines
1.5 KiB
Go
package cache
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"path/filepath"
|
|
"time"
|
|
|
|
"github.com/pkg/errors"
|
|
|
|
"github.com/kopia/kopia/internal/ctxutil"
|
|
"github.com/kopia/kopia/internal/ospath"
|
|
"github.com/kopia/kopia/repo/blob"
|
|
"github.com/kopia/kopia/repo/blob/filesystem"
|
|
"github.com/kopia/kopia/repo/blob/sharded"
|
|
)
|
|
|
|
// DirMode is the directory mode for all caches.
|
|
const DirMode = 0o700
|
|
|
|
// Storage is the storage interface required by the cache and is implemented by the filesystem Storage.
|
|
type Storage interface {
|
|
blob.Storage
|
|
TouchBlob(ctx context.Context, contentID blob.ID, threshold time.Duration) error
|
|
}
|
|
|
|
// NewStorageOrNil returns cache.Storage backed by the provided directory.
|
|
func NewStorageOrNil(ctx context.Context, cacheDir string, maxBytes int64, subdir string) (Storage, error) {
|
|
if maxBytes <= 0 || cacheDir == "" {
|
|
return nil, nil
|
|
}
|
|
|
|
if !ospath.IsAbs(cacheDir) {
|
|
return nil, errors.Errorf("cache dir %q was not absolute", cacheDir)
|
|
}
|
|
|
|
contentCacheDir := filepath.Join(cacheDir, subdir)
|
|
|
|
if _, err := os.Stat(contentCacheDir); os.IsNotExist(err) {
|
|
if mkdirerr := os.MkdirAll(contentCacheDir, DirMode); mkdirerr != nil {
|
|
return nil, errors.Wrap(mkdirerr, "error creating cache directory")
|
|
}
|
|
}
|
|
|
|
fs, err := filesystem.New(ctxutil.Detach(ctx), &filesystem.Options{
|
|
Path: contentCacheDir,
|
|
Options: sharded.Options{
|
|
DirectoryShards: []int{2},
|
|
},
|
|
}, false)
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "error initializing filesystem cache")
|
|
}
|
|
|
|
return fs.(Storage), nil
|
|
}
|