Remove content.CompactOptions.MinSmallBlobs

Use MaxSmallBlobs instead. MaxSmallBlobs was not being really used.
Replaced uses of MinSmallBlobs with MaxSmallBlobs and removed
MinSmallBlobs
This commit is contained in:
Julio Lopez
2020-02-06 11:25:05 -08:00
committed by Jarek Kowalski
parent f509c46a3d
commit 4625e5ba9e
5 changed files with 12 additions and 32 deletions

View File

@@ -9,7 +9,6 @@
var (
optimizeCommand = indexCommands.Command("optimize", "Optimize indexes blobs.")
optimizeMinSmallBlobs = optimizeCommand.Flag("min-small-blobs", "Minimum number of small index blobs that can be left after compaction.").Default("1").Int()
optimizeMaxSmallBlobs = optimizeCommand.Flag("max-small-blobs", "Maximum number of small index blobs that can be left after compaction.").Default("1").Int()
optimizeSkipDeletedOlderThan = optimizeCommand.Flag("skip-deleted-older-than", "Skip deleted blobs above given age").Duration()
optimizeAllIndexes = optimizeCommand.Flag("all", "Optimize all indexes, even those above maximum size.").Bool()
@@ -17,7 +16,6 @@
func runOptimizeCommand(ctx context.Context, rep *repo.Repository) error {
return rep.Content.CompactIndexes(ctx, content.CompactOptions{
MinSmallBlobs: *optimizeMinSmallBlobs,
MaxSmallBlobs: *optimizeMaxSmallBlobs,
AllIndexes: *optimizeAllIndexes,
SkipDeletedOlderThan: *optimizeSkipDeletedOlderThan,

View File

@@ -11,28 +11,22 @@
const verySmallContentFraction = 20 // blobs less than 1/verySmallContentFraction of maxPackSize are considered 'very small'
var autoCompactionOptions = CompactOptions{
MinSmallBlobs: 4 * parallelFetches, // nolint:gomnd
MaxSmallBlobs: 64, // nolint:gomnd
MaxSmallBlobs: 4 * parallelFetches, // nolint:gomnd
}
// CompactOptions provides options for compaction
type CompactOptions struct {
MinSmallBlobs int
MaxSmallBlobs int
AllIndexes bool
SkipDeletedOlderThan time.Duration
}
// CompactIndexes performs compaction of index blobs ensuring that # of small contents is between minSmallContentCount and maxSmallContentCount
// CompactIndexes performs compaction of index blobs ensuring that # of small index blobs is below opt.maxSmallBlobs
func (bm *Manager) CompactIndexes(ctx context.Context, opt CompactOptions) error {
bm.lock()
defer bm.unlock()
log.Debugf("CompactIndexes(%+v)", opt)
if opt.MaxSmallBlobs < opt.MinSmallBlobs {
return errors.Errorf("invalid content counts")
}
bm.lock()
defer bm.unlock()
indexBlobs, _, err := bm.loadPackIndexesUnlocked(ctx)
if err != nil {
@@ -70,13 +64,13 @@ func (bm *Manager) getContentsToCompact(indexBlobs []IndexBlobInfo, opt CompactO
}
}
if len(nonCompactedBlobs) < opt.MinSmallBlobs {
if len(nonCompactedBlobs) < opt.MaxSmallBlobs {
// current count is below min allowed - nothing to do
formatLog.Debugf("no small contents to compact")
return nil
}
if len(verySmallBlobs) > len(nonCompactedBlobs)/2 && len(mediumSizedBlobs)+1 < opt.MinSmallBlobs {
if len(verySmallBlobs) > len(nonCompactedBlobs)/2 && len(mediumSizedBlobs)+1 < opt.MaxSmallBlobs {
formatLog.Debugf("compacting %v very small contents", len(verySmallBlobs))
return verySmallBlobs
}

View File

@@ -374,10 +374,7 @@ func TestContentManagerConcurrency(t *testing.T) {
t.Errorf("unexpected index count before compaction: %v, wanted %v", got, want)
}
if err := bm4.CompactIndexes(ctx, CompactOptions{
MinSmallBlobs: 1,
MaxSmallBlobs: 1,
}); err != nil {
if err := bm4.CompactIndexes(ctx, CompactOptions{MaxSmallBlobs: 1}); err != nil {
t.Errorf("compaction error: %v", err)
}
@@ -393,10 +390,7 @@ func TestContentManagerConcurrency(t *testing.T) {
verifyContent(ctx, t, bm5, bm2content, seededRandomData(32, 100))
verifyContent(ctx, t, bm5, bm3content, seededRandomData(33, 100))
if err := bm5.CompactIndexes(ctx, CompactOptions{
MinSmallBlobs: 1,
MaxSmallBlobs: 1,
}); err != nil {
if err := bm5.CompactIndexes(ctx, CompactOptions{MaxSmallBlobs: 1}); err != nil {
t.Errorf("compaction error: %v", err)
}
}
@@ -1233,10 +1227,7 @@ func verifyVersionCompat(t *testing.T, writeVersion int) {
// make sure we can read everything
verifyContentManagerDataSet(ctx, t, mgr, dataSet)
if err := mgr.CompactIndexes(ctx, CompactOptions{
MinSmallBlobs: 1,
MaxSmallBlobs: 1,
}); err != nil {
if err := mgr.CompactIndexes(ctx, CompactOptions{MaxSmallBlobs: 1}); err != nil {
t.Fatalf("unable to compact indexes: %v", err)
}

View File

@@ -124,7 +124,7 @@ func TestPackingSimple(t *testing.T) {
verify(ctx, t, env.Repository, oid2a, []byte(content2), "packed-object-2")
verify(ctx, t, env.Repository, oid3a, []byte(content3), "packed-object-3")
if err := env.Repository.Content.CompactIndexes(ctx, content.CompactOptions{MinSmallBlobs: 1, MaxSmallBlobs: 1}); err != nil {
if err := env.Repository.Content.CompactIndexes(ctx, content.CompactOptions{MaxSmallBlobs: 1}); err != nil {
t.Errorf("optimize error: %v", err)
}
@@ -134,7 +134,7 @@ func TestPackingSimple(t *testing.T) {
verify(ctx, t, env.Repository, oid2a, []byte(content2), "packed-object-2")
verify(ctx, t, env.Repository, oid3a, []byte(content3), "packed-object-3")
if err := env.Repository.Content.CompactIndexes(ctx, content.CompactOptions{MinSmallBlobs: 1, MaxSmallBlobs: 1}); err != nil {
if err := env.Repository.Content.CompactIndexes(ctx, content.CompactOptions{MaxSmallBlobs: 1}); err != nil {
t.Errorf("optimize error: %v", err)
}

View File

@@ -280,10 +280,7 @@ func(ci content.Info) error {
}
func compact(ctx context.Context, t *testing.T, r *repo.Repository) error {
return r.Content.CompactIndexes(ctx, content.CompactOptions{
MinSmallBlobs: 1,
MaxSmallBlobs: 1,
})
return r.Content.CompactIndexes(ctx, content.CompactOptions{MaxSmallBlobs: 1})
}
func flush(ctx context.Context, t *testing.T, r *repo.Repository) error {