feat: add workspace-concurrency cli option for pack and publish command (#9493)

* refactor: set the default `workspaceConcurrency` to `Math.min(os.availableParallelism(), 4)`

* feat(plugin-commands-publishing): add `workspace-concurrency` cli option for pack and publish

* feat(recursive): add support for `recursive pack`

* feat: get default workspaceConcurrency from config package

* test(config): mock cpus to support Node.js 18

.changeset/hot-tires-report.md

@@ -0,0 +1,10 @@
+---
+"@pnpm/plugin-commands-installation": patch
+"@pnpm/plugin-commands-script-runners": patch
+"@pnpm/plugin-commands-rebuild": patch
+"@pnpm/plugin-commands-releasing": patch
+"@pnpm/build-modules": patch
+"@pnpm/config": patch
+---
+
+Set the default `workspaceConcurrency` to `Math.min(os.availableParallelism(), 4)`

.changeset/new-waves-remain.md

@@ -0,0 +1,5 @@
+---
+"pnpm": patch
+---
+
+Add support for `recursive pack`

.changeset/tricky-wolves-trade.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/plugin-commands-publishing": patch
+---
+
+Add `workspace-concurrency` cli option for pack and publish

config/config/src/concurrency.ts

@@ -1,12 +1,32 @@
 import os from 'os'
 
+const MaxDefaultWorkspaceConcurrency: number = 4
+
+let cacheAvailableParallelism: number | undefined
+
+export function getAvailableParallelism (cache: boolean = true): number {
+  if (cache && Number(cacheAvailableParallelism) > 0) {
+    return cacheAvailableParallelism!
+  }
+  cacheAvailableParallelism = Math.max(1, os.availableParallelism?.() ?? os.cpus().length)
+  return cacheAvailableParallelism
+}
+
+export function resetAvailableParallelismCache (): void {
+  cacheAvailableParallelism = undefined
+}
+
+export function getDefaultWorkspaceConcurrency (cache?: boolean): number {
+  return Math.min(MaxDefaultWorkspaceConcurrency, getAvailableParallelism(cache))
+}
+
 export function getWorkspaceConcurrency (option: number | undefined): number {
-  if (typeof option !== 'number') return 4
+  if (typeof option !== 'number') return getDefaultWorkspaceConcurrency()
 
   if (option <= 0) {
     // If option is <= 0, it uses the amount of cores minus the absolute of the number given
     // but always returning at least 1
-    return Math.max(1, (os.availableParallelism?.() ?? os.cpus().length) - Math.abs(option))
+    return Math.max(1, getAvailableParallelism() - Math.abs(option))
   }
 
   return option

config/config/src/index.ts

@@ -29,7 +29,7 @@ import {
   type VerifyDepsBeforeRun,
   type WantedPackageManager,
 } from './Config'
-import { getWorkspaceConcurrency } from './concurrency'
+import { getDefaultWorkspaceConcurrency, getWorkspaceConcurrency } from './concurrency'
 import { readWorkspaceManifest } from '@pnpm/workspace.read-manifest'
 
 import { types } from './types'
@@ -38,6 +38,7 @@ export { types }
 
 export { getOptionsFromRootManifest, getOptionsFromPnpmSettings, type OptionsFromRootManifest } from './getOptionsFromRootManifest'
 export * from './readLocalConfig'
+export { getDefaultWorkspaceConcurrency, getWorkspaceConcurrency } from './concurrency'
 
 export type { Config, UniversalOptions, WantedPackageManager, VerifyDepsBeforeRun }
 
@@ -191,7 +192,7 @@ export async function getConfig (opts: {
     'verify-deps-before-run': false,
     'verify-store-integrity': true,
     'virtual-store-dir': 'node_modules/.pnpm',
-    'workspace-concurrency': 4,
+    'workspace-concurrency': getDefaultWorkspaceConcurrency(),
     'workspace-prefix': opts.workspaceDir,
     'embed-readme': false,
     'registry-supports-time-field': false,

config/config/test/concurrency.test.ts

@@ -1,8 +1,47 @@
-import { cpus } from 'os'
-import { getWorkspaceConcurrency } from '../lib/concurrency'
+import os, { cpus } from 'os'
+import { getDefaultWorkspaceConcurrency, resetAvailableParallelismCache, getWorkspaceConcurrency } from '../lib/concurrency'
 
 const hostCores = cpus().length
 
+beforeEach(() => {
+  resetAvailableParallelismCache()
+})
+
+afterEach(() => {
+  resetAvailableParallelismCache()
+  jest.restoreAllMocks()
+})
+
+function mockAvailableParallelism (value: number) {
+  if ('availableParallelism' in os) {
+    jest.spyOn(os, 'availableParallelism').mockReturnValue(value)
+  }
+  jest.spyOn(os, 'cpus').mockReturnValue(Array(value).fill(cpus()[0]))
+}
+
+test('getDefaultWorkspaceConcurrency: cpu num < 4', () => {
+  mockAvailableParallelism(1)
+  expect(getDefaultWorkspaceConcurrency(false)).toBe(1)
+})
+
+test('getDefaultWorkspaceConcurrency: cpu num > 4', () => {
+  mockAvailableParallelism(5)
+  expect(getDefaultWorkspaceConcurrency(false)).toBe(4)
+})
+
+test('getDefaultWorkspaceConcurrency: cpu num = 4', () => {
+  mockAvailableParallelism(4)
+  expect(getDefaultWorkspaceConcurrency(false)).toBe(4)
+})
+
+test('getDefaultWorkspaceConcurrency: using cache', () => {
+  mockAvailableParallelism(4)
+  expect(getDefaultWorkspaceConcurrency()).toBe(4)
+
+  mockAvailableParallelism(5)
+  expect(getDefaultWorkspaceConcurrency()).toBe(4)
+})
+
 test('default workspace concurrency', () => {
   const n = getWorkspaceConcurrency(undefined)
 

exec/build-modules/package.json

@@ -33,6 +33,7 @@
   },
   "dependencies": {
     "@pnpm/calc-dep-state": "workspace:*",
+    "@pnpm/config": "workspace:*",
     "@pnpm/core-loggers": "workspace:*",
     "@pnpm/deps.graph-sequencer": "workspace:*",
     "@pnpm/fs.hard-link-dir": "workspace:*",

exec/build-modules/src/index.ts

@@ -2,6 +2,7 @@ import assert from 'assert'
 import path from 'path'
 import util from 'util'
 import { calcDepState, type DepsStateCache } from '@pnpm/calc-dep-state'
+import { getWorkspaceConcurrency } from '@pnpm/config'
 import { skippedOptionalDependencyLogger, ignoredScriptsLogger } from '@pnpm/core-loggers'
 import { runPostinstallHooks } from '@pnpm/lifecycle'
 import { linkBins, linkBinsOfPackages } from '@pnpm/link-bins'
@@ -87,7 +88,7 @@ export async function buildModules<T extends string> (
       }
     )
   })
-  await runGroups(opts.childConcurrency ?? 4, groups)
+  await runGroups(getWorkspaceConcurrency(opts.childConcurrency), groups)
   if (opts.ignoredBuiltDependencies?.length) {
     for (const ignoredBuild of opts.ignoredBuiltDependencies) {
       // We already ignore the build of this dependency.

exec/build-modules/tsconfig.json

@@ -9,6 +9,9 @@
     "../../__typings__/**/*.d.ts"
   ],
   "references": [
+    {
+      "path": "../../config/config"
+    },
     {
       "path": "../../deps/graph-sequencer"
     },

exec/plugin-commands-rebuild/src/recursive.ts

@@ -7,6 +7,7 @@ import {
 import {
   type Config,
   readLocalConfig,
+  getWorkspaceConcurrency,
 } from '@pnpm/config'
 import { logger } from '@pnpm/logger'
 import { sortPackages } from '@pnpm/sort-packages'
@@ -110,7 +111,7 @@ export async function recursiveRebuild (
     )
     return
   }
-  const limitRebuild = pLimit(opts.workspaceConcurrency ?? 4)
+  const limitRebuild = pLimit(getWorkspaceConcurrency(opts.workspaceConcurrency))
   for (const chunk of chunks) {
     // eslint-disable-next-line no-await-in-loop
     await Promise.all(chunk.map(async (rootDir) =>

exec/plugin-commands-script-runners/src/exec.ts

@@ -2,7 +2,7 @@ import path from 'path'
 import { docsUrl, type RecursiveSummary, throwOnCommandFail, readProjectManifestOnly } from '@pnpm/cli-utils'
 import { type LifecycleMessage, lifecycleLogger } from '@pnpm/core-loggers'
 import { FILTERING, UNIVERSAL_OPTIONS } from '@pnpm/common-cli-options-help'
-import { type Config, types } from '@pnpm/config'
+import { type Config, types, getWorkspaceConcurrency } from '@pnpm/config'
 import { type CheckDepsStatusOptions } from '@pnpm/deps.status'
 import { makeNodeRequireOption } from '@pnpm/lifecycle'
 import { logger } from '@pnpm/logger'
@@ -174,7 +174,7 @@ export async function handler (
   if (!params[0]) {
     throw new PnpmError('EXEC_MISSING_COMMAND', '\'pnpm exec\' requires a command to run')
   }
-  const limitRun = pLimit(opts.workspaceConcurrency ?? 4)
+  const limitRun = pLimit(getWorkspaceConcurrency(opts.workspaceConcurrency))
 
   if (opts.verifyDepsBeforeRun) {
     await runDepsStatusCheck(opts)

exec/plugin-commands-script-runners/src/run.ts

@@ -8,7 +8,7 @@ import {
 import { type CompletionFunc } from '@pnpm/command'
 import { prepareExecutionEnv } from '@pnpm/plugin-commands-env'
 import { FILTERING, UNIVERSAL_OPTIONS } from '@pnpm/common-cli-options-help'
-import { type Config, types as allTypes } from '@pnpm/config'
+import { type Config, types as allTypes, getWorkspaceConcurrency } from '@pnpm/config'
 import { PnpmError } from '@pnpm/error'
 import { type CheckDepsStatusOptions } from '@pnpm/deps.status'
 import {
@@ -264,7 +264,7 @@ so you may run "pnpm -w run ${scriptName}"`,
       hint: buildCommandNotFoundHint(scriptName, manifest.scripts),
     })
   }
-  const concurrency = opts.workspaceConcurrency ?? 4
+  const concurrency = getWorkspaceConcurrency(opts.workspaceConcurrency)
 
   const lifecycleOpts: RunLifecycleHookOptions = {
     depPath: dir,

exec/plugin-commands-script-runners/src/runRecursive.ts

@@ -2,7 +2,7 @@ import assert from 'assert'
 import path from 'path'
 import util from 'util'
 import { throwOnCommandFail } from '@pnpm/cli-utils'
-import { type Config } from '@pnpm/config'
+import { type Config, getWorkspaceConcurrency } from '@pnpm/config'
 import { prepareExecutionEnv } from '@pnpm/plugin-commands-env'
 import { PnpmError } from '@pnpm/error'
 import {
@@ -64,7 +64,7 @@ export async function runRecursive (
     })
   }
 
-  const limitRun = pLimit(opts.workspaceConcurrency ?? 4)
+  const limitRun = pLimit(getWorkspaceConcurrency(opts.workspaceConcurrency))
   const stdio =
     !opts.stream &&
     (opts.workspaceConcurrency === 1 ||
@@ -145,7 +145,7 @@ export async function runRecursive (
             workspaceDir: opts.workspaceDir,
           }
           const _runScript = runScript.bind(null, { manifest: pkg.package.manifest, lifecycleOpts, runScriptOptions, passedThruArgs })
-          const groupEnd = (opts.workspaceConcurrency ?? 4) > 1
+          const groupEnd = getWorkspaceConcurrency(opts.workspaceConcurrency) > 1
             ? undefined
             : groupStart(formatSectionName({
               name: pkg.package.manifest.name,

pkg-manager/plugin-commands-installation/src/recursive.ts

@@ -8,6 +8,7 @@ import {
   type Config,
   type OptionsFromRootManifest,
   getOptionsFromRootManifest,
+  getWorkspaceConcurrency,
   readLocalConfig,
 } from '@pnpm/config'
 import { PnpmError } from '@pnpm/error'
@@ -294,7 +295,7 @@ export async function recursive (
 
   const pkgPaths = (Object.keys(opts.selectedProjectsGraph) as ProjectRootDir[]).sort()
 
-  const limitInstallation = pLimit(opts.workspaceConcurrency ?? 4)
+  const limitInstallation = pLimit(getWorkspaceConcurrency(opts.workspaceConcurrency))
   await Promise.all(pkgPaths.map(async (rootDir) =>
     limitInstallation(async () => {
       const hooks = opts.ignorePnpmfile

pnpm-lock.yaml

@@ -2312,6 +2312,9 @@ importers:
       '@pnpm/calc-dep-state':
         specifier: workspace:*
         version: link:../../packages/calc-dep-state
+      '@pnpm/config':
+        specifier: workspace:*
+        version: link:../../config/config
       '@pnpm/core-loggers':
         specifier: workspace:*
         version: link:../../packages/core-loggers

pnpm/src/cmd/recursive.ts

@@ -1,6 +1,7 @@
 import { docsUrl } from '@pnpm/cli-utils'
 import { FILTERING } from '@pnpm/common-cli-options-help'
 import { WANTED_LOCKFILE } from '@pnpm/constants'
+import { getDefaultWorkspaceConcurrency } from '@pnpm/config'
 import renderHelp from 'render-help'
 
 export const rcOptionsTypes = (): Record<string, unknown> => ({})
@@ -70,6 +71,10 @@ and must recompile all your C++ addons with the new binary.',
             description: 'Publishes packages to the npm registry. Only publishes a package if its version is not taken in the registry.',
             name: 'publish [--tag <tag>] [--access <public|restricted>]',
           },
+          {
+            description: 'Create tarballs for each package.',
+            name: 'pack [-- <args>...]',
+          },
         ],
       },
       {
@@ -81,7 +86,7 @@ and must recompile all your C++ addons with the new binary.',
             name: '--no-bail',
           },
           {
-            description: 'Set the maximum number of concurrency. Default is 4. For unlimited concurrency use Infinity.',
+            description: `Set the maximum number of concurrency. Default is ${getDefaultWorkspaceConcurrency()}. For unlimited concurrency use Infinity.`,
             name: '--workspace-concurrency <number>',
           },
           {

releasing/plugin-commands-publishing/src/pack.ts

@@ -3,7 +3,7 @@ import path from 'path'
 import { createGzip } from 'zlib'
 import { type Catalogs } from '@pnpm/catalogs.types'
 import { PnpmError } from '@pnpm/error'
-import { types as allTypes, type UniversalOptions, type Config } from '@pnpm/config'
+import { types as allTypes, type UniversalOptions, type Config, getWorkspaceConcurrency, getDefaultWorkspaceConcurrency } from '@pnpm/config'
 import { readProjectManifest } from '@pnpm/cli-utils'
 import { createExportableManifest } from '@pnpm/exportable-manifest'
 import { packlist } from '@pnpm/fs.packlist'
@@ -41,6 +41,7 @@ export function cliOptionsTypes (): Record<string, unknown> {
       'pack-destination',
       'pack-gzip-level',
       'json',
+      'workspace-concurrency',
     ], allTypes),
   }
 }
@@ -73,6 +74,10 @@ export function help (): string {
             name: '--recursive',
             shortAlias: '-r',
           },
+          {
+            description: `Set the maximum number of concurrency. Default is ${getDefaultWorkspaceConcurrency()}. For unlimited concurrency use Infinity.`,
+            name: '--workspace-concurrency <number>',
+          },
         ],
       },
       FILTERING,
@@ -132,7 +137,7 @@ export async function handler (opts: PackOptions): Promise<string> {
 
     const chunks = sortPackages(selectedProjectsGraph)
 
-    const limitPack = pLimit(opts.workspaceConcurrency ?? 4)
+    const limitPack = pLimit(getWorkspaceConcurrency(opts.workspaceConcurrency))
     const resolvedOpts = { ...opts }
     if (opts.out) {
       resolvedOpts.out = path.resolve(opts.dir, opts.out)

releasing/plugin-commands-publishing/src/publish.ts

@@ -2,7 +2,7 @@ import { promises as fs, existsSync } from 'fs'
 import path from 'path'
 import { docsUrl, readProjectManifest } from '@pnpm/cli-utils'
 import { FILTERING } from '@pnpm/common-cli-options-help'
-import { type Config, types as allTypes } from '@pnpm/config'
+import { type Config, types as allTypes, getDefaultWorkspaceConcurrency } from '@pnpm/config'
 import { PnpmError } from '@pnpm/error'
 import { runLifecycleHook, type RunLifecycleHookOptions } from '@pnpm/lifecycle'
 import { runNpm } from '@pnpm/run-npm'
@@ -32,6 +32,7 @@ export function rcOptionsTypes (): Record<string, unknown> {
     'tag',
     'unsafe-perm',
     'embed-readme',
+    'workspace-concurrency',
   ], allTypes)
 }
 
@@ -101,6 +102,10 @@ export function help (): string {
             name: '--recursive',
             shortAlias: '-r',
           },
+          {
+            description: `Set the maximum number of concurrency. Default is ${getDefaultWorkspaceConcurrency()}. For unlimited concurrency use Infinity.`,
+            name: '--workspace-concurrency <number>',
+          },
         ],
       },
       FILTERING,

releasing/plugin-commands-publishing/src/recursivePublish.ts

@@ -1,6 +1,6 @@
 import path from 'path'
 import { createResolver } from '@pnpm/client'
-import { type Config } from '@pnpm/config'
+import { type Config, getWorkspaceConcurrency } from '@pnpm/config'
 import { logger } from '@pnpm/logger'
 import { pickRegistryForPackage } from '@pnpm/pick-registry-for-package'
 import { type ResolveFunction } from '@pnpm/resolver-base'
@@ -105,7 +105,7 @@ export async function recursivePublish (
       appendedArgs.push(`--otp=${opts.cliOptions['otp'] as string}`)
     }
     const chunks = sortPackages(opts.selectedProjectsGraph)
-    const limitPublish = pLimit(opts.workspaceConcurrency ?? 4)
+    const limitPublish = pLimit(getWorkspaceConcurrency(opts.workspaceConcurrency))
     const tag = opts.tag ?? 'latest'
     for (const chunk of chunks) {
       // eslint-disable-next-line no-await-in-loop