mirror of
https://github.com/kopia/kopia.git
synced 2026-01-27 07:48:06 -05:00
cead806a3f
* blob: changed default shards from {3,3} to {1,3}
Turns out for very large repository around 100TB (5M blobs),
we end up creating max ~16M directories which is way too much
and slows down listing. Currently each leaf directory only has a handful
of files.
Simple sharding of {3} should work much better and will end up creating
directories with meaningful shard sizes - 12 K files per directory
should not be too slow and will reduce the overhead of listing by
4096 times.
The change is done in a backwards-compatible way and will respect
custom sharding (.shards) file written by previous 0.9 builds
as well as older repositories that don't have the .shards file (which
we assume to be {3,3}).
* fixed compat tests
78 lines
2.0 KiB
Go
78 lines
2.0 KiB
Go
package repotesting
|
|
|
|
import (
|
|
"context"
|
|
"sync"
|
|
"testing"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/pkg/errors"
|
|
|
|
"github.com/kopia/kopia/repo/blob"
|
|
)
|
|
|
|
// reconnectableStorage implements wraps a blob.Storage provider which allows it
|
|
// to be reconnected later even if the underlying provider does not have ConnectionInfo().
|
|
// the wrapper is cached in a global map and ConnectionInfo() exposes UUID which is the
|
|
// map key.
|
|
type reconnectableStorage struct {
|
|
blob.Storage
|
|
|
|
opt *reconnectableStorageOptions
|
|
}
|
|
|
|
const reconnectableStorageType = "reconnectable"
|
|
|
|
// reconnectableStorageOptions provides options to reconnectable storage.
|
|
type reconnectableStorageOptions struct {
|
|
UUID string
|
|
}
|
|
|
|
// newReconnectableStorage wraps the provided storage that may or may not be round-trippable
|
|
// in a wrapper that globally caches storage instance and ensures its connection info is
|
|
// round-trippable.
|
|
func newReconnectableStorage(tb testing.TB, st blob.Storage) blob.Storage {
|
|
tb.Helper()
|
|
|
|
st2 := reconnectableStorage{st, &reconnectableStorageOptions{UUID: uuid.NewString()}}
|
|
|
|
reconnectableStorageByUUID.Store(st2.opt.UUID, st2)
|
|
tb.Cleanup(func() {
|
|
reconnectableStorageByUUID.Delete(st2.opt.UUID)
|
|
})
|
|
|
|
return st2
|
|
}
|
|
|
|
var reconnectableStorageByUUID sync.Map
|
|
|
|
func (s reconnectableStorage) ConnectionInfo() blob.ConnectionInfo {
|
|
return blob.ConnectionInfo{
|
|
Type: reconnectableStorageType,
|
|
Config: s.opt,
|
|
}
|
|
}
|
|
|
|
func init() {
|
|
blob.AddSupportedStorage(
|
|
reconnectableStorageType,
|
|
func() interface{} { return &reconnectableStorageOptions{} },
|
|
func(ctx context.Context, o interface{}, isCreate bool) (blob.Storage, error) {
|
|
opt, ok := o.(*reconnectableStorageOptions)
|
|
if !ok {
|
|
return nil, errors.Errorf("invalid options %T", o)
|
|
}
|
|
|
|
if opt.UUID == "" {
|
|
return nil, errors.Errorf("missing UUID")
|
|
}
|
|
|
|
v, ok := reconnectableStorageByUUID.Load(opt.UUID)
|
|
if !ok {
|
|
return nil, errors.Errorf("reconnectable storage not found: %v", opt.UUID)
|
|
}
|
|
|
|
return v.(blob.Storage), nil
|
|
})
|
|
}
|