mirror/kopia
1
0
Fork 0
mirror of https://github.com/kopia/kopia.git synced 2026-05-19 04:04:56 -04:00
Code Issues Packages Projects Releases Wiki Activity

feat(repository): added hard size limit to the on-disk cache (#3238)

Browse Source 
* test(providers): added capacity limits to blobtesting.mapStorage

* refactor(general): added mutex map which dynamically allocates and releases named mutexes

* refactor(repository): refactored cache cleanup and limit enforcement

* refactor(repository): plumb through cache size limits in the repository

* feat(cli): added CLI options to set cache size limits

* unified flag setting and field naming

* Update cli/command_cache_set.go

Co-authored-by: Shikhar Mall <mall.shikhar.in@gmail.com>

* pr feedback

---------

Co-authored-by: Shikhar Mall <mall.shikhar.in@gmail.com>
This commit is contained in:
Jarek Kowalski
2023-08-24 09:38:56 -07:00
committed by GitHub
parent 1fd1ff0151
commit fe55dcb6a2
20 changed files with 654 additions and 295 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
cli/command_cache_set.go
View File

@@ -4,6 +4,7 @@
"context"
"time"
"github.com/alecthomas/kingpin/v2"
"github.com/pkg/errors"
"github.com/kopia/kopia/internal/units"
@@ -11,14 +12,36 @@
"github.com/kopia/kopia/repo/content"
)
type cacheSizeFlags struct {
contentCacheSizeMB int64
contentCacheSizeLimitMB int64
contentMinSweepAge time.Duration
metadataCacheSizeMB int64
metadataCacheSizeLimitMB int64
metadataMinSweepAge time.Duration
maxListCacheDuration time.Duration
indexMinSweepAge time.Duration
}
func (c *cacheSizeFlags) setup(cmd *kingpin.CmdClause) {
// do not use Defaults here, since this structure is shared between connect/create/set commands
// each command will set their default values in code.
cmd.Flag("content-cache-size-mb", "Desired size of local content cache (soft limit)").PlaceHolder("MB").Int64Var(&c.contentCacheSizeMB)
cmd.Flag("content-cache-size-limit-mb", "Maximum size of local content cache (hard limit)").PlaceHolder("MB").Int64Var(&c.contentCacheSizeLimitMB)
cmd.Flag("content-min-sweep-age", "Minimal age of content cache item to be subject to sweeping").DurationVar(&c.contentMinSweepAge)
cmd.Flag("metadata-cache-size-mb", "Desired size of local metadata cache (soft limit)").PlaceHolder("MB").Int64Var(&c.metadataCacheSizeMB)
cmd.Flag("metadata-cache-size-limit-mb", "Maximum size of local metadata cache (hard limit)").PlaceHolder("MB").Int64Var(&c.metadataCacheSizeLimitMB)
cmd.Flag("metadata-min-sweep-age", "Minimal age of metadata cache item to be subject to sweeping").DurationVar(&c.metadataMinSweepAge)
cmd.Flag("index-min-sweep-age", "Minimal age of index cache item to be subject to sweeping").DurationVar(&c.indexMinSweepAge)
cmd.Flag("max-list-cache-duration", "Duration of index cache").DurationVar(&c.maxListCacheDuration)
}
type commandCacheSetParams struct {
directory string
contentCacheSizeMB int64
maxMetadataCacheSizeMB int64
maxListCacheDuration time.Duration
contentMinSweepAge time.Duration
metadataMinSweepAge time.Duration
indexMinSweepAge time.Duration
directory string
cacheSizeFlags
svc appServices
}
@@ -30,14 +53,14 @@ func (c *commandCacheSetParams) setup(svc appServices, parent commandParent) {
c.metadataMinSweepAge = -1
c.indexMinSweepAge = -1
c.maxListCacheDuration = -1
c.contentCacheSizeLimitMB = -1
c.contentCacheSizeMB = -1
c.metadataCacheSizeLimitMB = -1
c.metadataCacheSizeMB = -1
c.cacheSizeFlags.setup(cmd)
cmd.Flag("cache-directory", "Directory where to store cache files").StringVar(&c.directory)
cmd.Flag("content-cache-size-mb", "Size of local content cache").PlaceHolder("MB").Default("-1").Int64Var(&c.contentCacheSizeMB)
cmd.Flag("content-min-sweep-age", "Minimal age of content cache item to be subject to sweeping").DurationVar(&c.contentMinSweepAge)
cmd.Flag("metadata-cache-size-mb", "Size of local metadata cache").PlaceHolder("MB").Default("-1").Int64Var(&c.maxMetadataCacheSizeMB)
cmd.Flag("metadata-min-sweep-age", "Minimal age of metadata cache item to be subject to sweeping").DurationVar(&c.metadataMinSweepAge)
cmd.Flag("index-min-sweep-age", "Minimal age of index cache item to be subject to sweeping").DurationVar(&c.indexMinSweepAge)
cmd.Flag("max-list-cache-duration", "Duration of index cache").DurationVar(&c.maxListCacheDuration)
cmd.Action(svc.repositoryWriterAction(c.run))
c.svc = svc
}
@@ -59,14 +82,28 @@ func (c *commandCacheSetParams) run(ctx context.Context, _ repo.RepositoryWriter
if v := c.contentCacheSizeMB; v != -1 {
v *= 1e6 // convert MB to bytes
log(ctx).Infof("changing content cache size to %v", units.BytesString(v))
opts.MaxCacheSizeBytes = v
opts.ContentCacheSizeBytes = v
changed++
}
if v := c.maxMetadataCacheSizeMB; v != -1 {
if v := c.contentCacheSizeLimitMB; v != -1 {
v *= 1e6 // convert MB to bytes
log(ctx).Infof("changing content cache size limit to %v", units.BytesString(v))
opts.ContentCacheSizeLimitBytes = v
changed++
}
if v := c.metadataCacheSizeMB; v != -1 {
v *= 1e6 // convert MB to bytes
log(ctx).Infof("changing metadata cache size to %v", units.BytesString(v))
opts.MaxMetadataCacheSizeBytes = v
opts.MetadataCacheSizeBytes = v
changed++
}
if v := c.metadataCacheSizeLimitMB; v != -1 {
v *= 1e6 // convert MB to bytes
log(ctx).Infof("changing metadata cache size limit to %v", units.BytesString(v))
opts.MetadataCacheSizeLimitBytes = v
changed++
}