From 463f30ccfbaff07d642076fce688c86eaa3baec6 Mon Sep 17 00:00:00 2001 From: Zoltan Kochan Date: Fri, 21 Nov 2025 15:29:56 +0100 Subject: [PATCH] perf: increase the default network concurrency on machines with many CPU cores (#10215) close #10068 --- .changeset/small-results-hunt.md | 7 +++++++ pkg-manager/package-requester/src/packageRequester.ts | 8 ++------ worker/src/index.ts | 2 +- 3 files changed, 10 insertions(+), 7 deletions(-) create mode 100644 .changeset/small-results-hunt.md diff --git a/.changeset/small-results-hunt.md b/.changeset/small-results-hunt.md new file mode 100644 index 0000000000..0f75272f93 --- /dev/null +++ b/.changeset/small-results-hunt.md @@ -0,0 +1,7 @@ +--- +"@pnpm/package-requester": minor +"@pnpm/worker": minor +"pnpm": minor +--- + +Increase the network concurrency on machines with many CPU cores. We pick a network concurrency that is not less than 16 and not more than 64 and it is calculated by the number of pnpm workers multiplied by 3 [#10068](https://github.com/pnpm/pnpm/issues/10068). diff --git a/pkg-manager/package-requester/src/packageRequester.ts b/pkg-manager/package-requester/src/packageRequester.ts index 0e0c369935..bc0544300e 100644 --- a/pkg-manager/package-requester/src/packageRequester.ts +++ b/pkg-manager/package-requester/src/packageRequester.ts @@ -1,5 +1,4 @@ import { createReadStream, promises as fs } from 'fs' -import os from 'os' import path from 'path' import { getFilePathByModeInCafs as _getFilePathByModeInCafs, @@ -44,7 +43,7 @@ import { } from '@pnpm/store-controller-types' import { type DependencyManifest, type SupportedArchitectures } from '@pnpm/types' import { depPathToFilename } from '@pnpm/dependency-path' -import { readPkgFromCafs as _readPkgFromCafs } from '@pnpm/worker' +import { calcMaxWorkers, readPkgFromCafs as _readPkgFromCafs } from '@pnpm/worker' import { familySync } from 'detect-libc' import PQueue from 'p-queue' import pDefer from 'p-defer' @@ -111,10 +110,7 @@ export function createPackageRequester ( } { opts = opts || {} - // A lower bound of 16 is enforced to prevent performance degradation, - // especially in CI environments. Tests with a threshold lower than 16 - // have shown consistent underperformance. - const networkConcurrency = opts.networkConcurrency ?? Math.max(os.availableParallelism?.() ?? os.cpus().length, 16) + const networkConcurrency = opts.networkConcurrency ?? Math.min(64, Math.max(calcMaxWorkers() * 3, 16)) const requestsQueue = new PQueue({ concurrency: networkConcurrency, }) diff --git a/worker/src/index.ts b/worker/src/index.ts index 41e638df04..879e1a8a6c 100644 --- a/worker/src/index.ts +++ b/worker/src/index.ts @@ -52,7 +52,7 @@ function createTarballWorkerPool (): WorkerPool { return workerPool } -function calcMaxWorkers () { +export function calcMaxWorkers (): number { if (process.env.PNPM_MAX_WORKERS) { return parseInt(process.env.PNPM_MAX_WORKERS) }