mirror of
https://github.com/kopia/kopia.git
synced 2026-01-27 15:58:03 -05:00
cead806a3f
* blob: changed default shards from {3,3} to {1,3}
Turns out for very large repository around 100TB (5M blobs),
we end up creating max ~16M directories which is way too much
and slows down listing. Currently each leaf directory only has a handful
of files.
Simple sharding of {3} should work much better and will end up creating
directories with meaningful shard sizes - 12 K files per directory
should not be too slow and will reduce the overhead of listing by
4096 times.
The change is done in a backwards-compatible way and will respect
custom sharding (.shards) file written by previous 0.9 builds
as well as older repositories that don't have the .shards file (which
we assume to be {3,3}).
* fixed compat tests
70 lines
1.8 KiB
Go
70 lines
1.8 KiB
Go
package cli
|
|
|
|
import (
|
|
"context"
|
|
|
|
"github.com/alecthomas/kingpin"
|
|
"github.com/pkg/errors"
|
|
|
|
"github.com/kopia/kopia/repo"
|
|
"github.com/kopia/kopia/repo/blob"
|
|
)
|
|
|
|
type storageFromConfigFlags struct {
|
|
connectFromConfigFile string
|
|
connectFromConfigToken string
|
|
|
|
sps storageProviderServices
|
|
}
|
|
|
|
func (c *storageFromConfigFlags) setup(sps storageProviderServices, cmd *kingpin.CmdClause) {
|
|
cmd.Flag("file", "Path to the configuration file").StringVar(&c.connectFromConfigFile)
|
|
cmd.Flag("token", "Configuration token").StringVar(&c.connectFromConfigToken)
|
|
|
|
c.sps = sps
|
|
}
|
|
|
|
func (c *storageFromConfigFlags) connect(ctx context.Context, isCreate bool, formatVersion int) (blob.Storage, error) {
|
|
if isCreate {
|
|
return nil, errors.New("not supported")
|
|
}
|
|
|
|
if c.connectFromConfigFile != "" {
|
|
return c.connectToStorageFromConfigFile(ctx)
|
|
}
|
|
|
|
if c.connectFromConfigToken != "" {
|
|
return c.connectToStorageFromConfigToken(ctx)
|
|
}
|
|
|
|
return nil, errors.New("either --file or --token must be provided")
|
|
}
|
|
|
|
func (c *storageFromConfigFlags) connectToStorageFromConfigFile(ctx context.Context) (blob.Storage, error) {
|
|
cfg, err := repo.LoadConfigFromFile(c.connectFromConfigFile)
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "unable to open config")
|
|
}
|
|
|
|
if cfg.Storage == nil {
|
|
return nil, errors.Errorf("connection file does not specify blob storage connection parameters, kopia server connections are not supported")
|
|
}
|
|
|
|
// nolint:wrapcheck
|
|
return blob.NewStorage(ctx, *cfg.Storage, false)
|
|
}
|
|
|
|
func (c *storageFromConfigFlags) connectToStorageFromConfigToken(ctx context.Context) (blob.Storage, error) {
|
|
ci, pass, err := repo.DecodeToken(c.connectFromConfigToken)
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "invalid token")
|
|
}
|
|
|
|
if pass != "" {
|
|
c.sps.setPasswordFromToken(pass)
|
|
}
|
|
|
|
// nolint:wrapcheck
|
|
return blob.NewStorage(ctx, ci, false)
|
|
}
|