From 8ff6b6d06007fc800cc9d7d42efb1914bb795fdd Mon Sep 17 00:00:00 2001 From: Jarek Kowalski Date: Sat, 10 Sep 2022 20:11:07 -0700 Subject: [PATCH] fix(snapshots): fixed --parallel flag when snapshotting (#2392) Previously the value of --parallel flag was (unintentionally) capped at max number of CPUs. This PR fixes the logic. --- snapshot/snapshotfs/upload.go | 7 ++++++- snapshot/snapshotfs/upload_test.go | 4 +++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/snapshot/snapshotfs/upload.go b/snapshot/snapshotfs/upload.go index 70ee9151a..1ae6f50be 100644 --- a/snapshot/snapshotfs/upload.go +++ b/snapshot/snapshotfs/upload.go @@ -727,8 +727,13 @@ func (u *Uploader) maybeIgnoreCachedEntry(ctx context.Context, ent fs.Entry) fs. func (u *Uploader) effectiveParallelFileReads(pol *policy.Policy) int { p := u.ParallelUploads - max := pol.UploadPolicy.MaxParallelFileReads.OrDefault(runtime.NumCPU()) + if p > 0 { + // command-line override takes precedence. + return p + } + // use policy setting or number of CPUs. + max := pol.UploadPolicy.MaxParallelFileReads.OrDefault(runtime.NumCPU()) if p < 1 || p > max { return max } diff --git a/snapshot/snapshotfs/upload_test.go b/snapshot/snapshotfs/upload_test.go index 740835d91..49ffc11fa 100644 --- a/snapshot/snapshotfs/upload_test.go +++ b/snapshot/snapshotfs/upload_test.go @@ -679,7 +679,7 @@ func TestParallelUploadUploadsBlobsInParallel(t *testing.T) { th := newUploadTestHarness(ctx, t) u := NewUploader(th.repo) - u.ParallelUploads = 10 + u.ParallelUploads = 13 // no faults for first blob write - session marker. th.faulty.AddFault(blobtesting.MethodPutBlob) @@ -705,6 +705,8 @@ func TestParallelUploadUploadsBlobsInParallel(t *testing.T) { policyTree := policy.BuildTree(nil, policy.DefaultPolicy) + require.Equal(t, 13, u.effectiveParallelFileReads(policyTree.EffectivePolicy())) + si := snapshot.SourceInfo{ UserName: "user", Host: "host",