feat(pack): add package filtering for pack command (#4351) (#9477)

* feat(pack): add package filtering for pack command (#4351)

* feat(plugin-commands-publishing): parallelly run recusive pack and publish

* refactor: pack.ts

* feat(pack): support absolute pack-destination path

* feat: get `pack-destination` configuration from npmrc

* refactor: pack.ts

* docs: update changeset

* refactor: pack.ts

close #4351

---------

Co-authored-by: Zoltan Kochan <z@kochan.io>

.changeset/cruel-onions-fix.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/plugin-commands-publishing": patch
+---
+
+Parallelly run recursive pack and publish

.changeset/eight-trams-vanish.md

@@ -0,0 +1,6 @@
+---
+"@pnpm/plugin-commands-publishing": patch
+"@pnpm/config": patch
+---
+
+Get `pack-destination` configuration from settings.

.changeset/thirty-islands-juggle.md

@@ -0,0 +1,8 @@
+---
+"@pnpm/plugin-commands-publishing": minor
+"pnpm": minor
+---
+
+Added support for recursively running pack in every project of a workspace [#4351](https://github.com/pnpm/pnpm/issues/4351).
+
+Now you can run `pnpm -r pack` to pack all packages in the workspace.

config/config/src/types.ts

@@ -67,6 +67,7 @@ export const types = Object.assign({
   'npm-path': String,
   offline: Boolean,
   'only-built-dependencies': [String],
+  'pack-destination': String,
   'pack-gzip-level': Number,
   'package-import-method': ['auto', 'hardlink', 'clone', 'copy'],
   'patches-dir': String,

pnpm-lock.yaml

@@ -6561,6 +6561,9 @@ importers:
       p-filter:
         specifier: 'catalog:'
         version: 2.1.0
+      p-limit:
+        specifier: 'catalog:'
+        version: 3.1.0
       ramda:
         specifier: 'catalog:'
         version: '@pnpm/ramda@0.28.1'
@@ -14820,6 +14823,7 @@ packages:
   verdaccio@5.20.1:
     resolution: {integrity: sha512-zKQXYubQOfl2w09gO9BR7U9ZZkFPPby8tvV+na86/2vGZnY79kNSVnSbK8CM1bpJHTCQ80AGsmIGovg2FgXhdQ==}
     engines: {node: '>=12.18'}
+    deprecated: this version is deprecated, please migrate to 6.x versions
     hasBin: true
 
   verror@1.10.0:

releasing/plugin-commands-publishing/package.json

@@ -56,6 +56,7 @@
     "enquirer": "catalog:",
     "execa": "catalog:",
     "p-filter": "catalog:",
+    "p-limit": "catalog:",
     "ramda": "catalog:",
     "realpath-missing": "catalog:",
     "render-help": "catalog:",

releasing/plugin-commands-publishing/src/pack.ts

@@ -8,7 +8,7 @@ import { readProjectManifest } from '@pnpm/cli-utils'
 import { createExportableManifest } from '@pnpm/exportable-manifest'
 import { packlist } from '@pnpm/fs.packlist'
 import { getBinsFromPackageManifest } from '@pnpm/package-bins'
-import { type ProjectManifest, type DependencyManifest } from '@pnpm/types'
+import { type ProjectManifest, type Project, type ProjectRootDir, type ProjectsGraph, type DependencyManifest } from '@pnpm/types'
 import { glob } from 'tinyglobby'
 import pick from 'ramda/src/pick'
 import realpathMissing from 'realpath-missing'
@@ -17,6 +17,10 @@ import tar from 'tar-stream'
 import { runScriptsIfPresent } from './publish'
 import chalk from 'chalk'
 import validateNpmPackageName from 'validate-npm-package-name'
+import pLimit from 'p-limit'
+import { FILTERING } from '@pnpm/common-cli-options-help'
+import { sortPackages } from '@pnpm/sort-packages'
+import { logger } from '@pnpm/logger'
 
 const LICENSE_GLOB = 'LICEN{S,C}E{,.*}' // cspell:disable-line
 
@@ -31,9 +35,10 @@ export function rcOptionsTypes (): Record<string, unknown> {
 
 export function cliOptionsTypes (): Record<string, unknown> {
   return {
-    'pack-destination': String,
     out: String,
+    recursive: Boolean,
     ...pick([
+      'pack-destination',
       'pack-gzip-level',
       'json',
     ], allTypes),
@@ -63,38 +68,109 @@ export function help (): string {
             description: 'Customizes the output path for the tarball. Use `%s` and `%v` to include the package name and version, e.g., `%s.tgz` or `some-dir/%s-%v.tgz`. By default, the tarball is saved in the current working directory with the name `<package-name>-<version>.tgz`.',
             name: '--out <path>',
           },
+          {
+            description: 'Pack all packages from the workspace',
+            name: '--recursive',
+            shortAlias: '-r',
+          },
         ],
       },
+      FILTERING,
     ],
   })
 }
 
-export type PackOptions = Pick<UniversalOptions, 'dir'> & Pick<Config, 'catalogs' | 'ignoreScripts' | 'rawConfig' | 'embedReadme' | 'packGzipLevel' | 'nodeLinker'> & Partial<Pick<Config, 'extraBinPaths' | 'extraEnv'>> & {
+export type PackOptions = Pick<UniversalOptions, 'dir'> & Pick<Config, 'catalogs'
+| 'ignoreScripts'
+| 'rawConfig'
+| 'embedReadme'
+| 'packGzipLevel'
+| 'nodeLinker'
+> & Partial<Pick<Config, 'extraBinPaths'
+| 'extraEnv'
+| 'recursive'
+| 'selectedProjectsGraph'
+| 'workspaceConcurrency'
+| 'workspaceDir'
+>> & {
   argv: {
     original: string[]
   }
   engineStrict?: boolean
   packDestination?: string
   out?: string
-  workspaceDir?: string
   json?: boolean
+  unicode?: boolean
+}
+
+export interface PackResultJson {
+  name: string
+  version: string
+  filename: string
+  files: Array<{ path: string }>
 }
 
 export async function handler (opts: PackOptions): Promise<string> {
-  const { publishedManifest, tarballPath, contents } = await api(opts)
-  if (opts.json) {
-    return JSON.stringify({
-      name: publishedManifest.name,
-      version: publishedManifest.version,
-      filename: tarballPath,
-      files: contents.map((path) => ({ path })),
-    }, null, 2)
-  }
-  return `${chalk.blueBright('Tarball Contents')}
-${contents.join('\n')}
+  const packedPackages: PackResultJson[] = []
 
+  if (opts.recursive) {
+    const selectedProjectsGraph = opts.selectedProjectsGraph as ProjectsGraph
+    const pkgsToPack: Project[] = []
+    for (const { package: pkg } of Object.values(selectedProjectsGraph)) {
+      if (pkg.manifest.name && pkg.manifest.version) {
+        pkgsToPack.push(pkg)
+      }
+    }
+    const packedPkgDirs = new Set<ProjectRootDir>(pkgsToPack.map(({ rootDir }) => rootDir))
+
+    if (packedPkgDirs.size === 0) {
+      logger.info({
+        message: 'There are no packages that should be packed',
+        prefix: opts.dir,
+      })
+    }
+
+    const chunks = sortPackages(selectedProjectsGraph)
+
+    const limitPack = pLimit(opts.workspaceConcurrency ?? 4)
+    const resolvedOpts = { ...opts }
+    if (opts.out) {
+      resolvedOpts.out = path.resolve(opts.dir, opts.out)
+    } else if (opts.packDestination) {
+      resolvedOpts.packDestination = path.resolve(opts.dir, opts.packDestination)
+    } else {
+      resolvedOpts.packDestination = path.resolve(opts.dir)
+    }
+    for (const chunk of chunks) {
+      // eslint-disable-next-line no-await-in-loop
+      await Promise.all(chunk.map(pkgDir =>
+        limitPack(async () => {
+          if (!packedPkgDirs.has(pkgDir)) return
+          const pkg = selectedProjectsGraph[pkgDir].package
+          const packResult = await api({
+            ...resolvedOpts,
+            dir: pkg.rootDir,
+          })
+          packedPackages.push(toPackResultJson(packResult))
+        })
+      ))
+    }
+  } else {
+    const packResult = await api(opts)
+    packedPackages.push(toPackResultJson(packResult))
+  }
+
+  if (opts.json) {
+    return JSON.stringify(packedPackages.length > 1 ? packedPackages : packedPackages[0], null, 2)
+  }
+
+  return packedPackages.map(
+    ({ name, version, filename, files }) => `${opts.unicode ? '📦 ' : 'package:'} ${name}@${version}
+${chalk.blueBright('Tarball Contents')}
+${files.map(({ path }) => path).join('\n')}
 ${chalk.blueBright('Tarball Details')}
-${tarballPath}`
+${filename}`
+  ).join('\n\n')
 }
 
 export async function api (opts: PackOptions): Promise<PackResult> {
@@ -275,3 +351,13 @@ async function createPublishManifest (opts: {
     modulesDir,
   })
 }
+
+function toPackResultJson (packResult: PackResult): PackResultJson {
+  const { publishedManifest, contents, tarballPath } = packResult
+  return {
+    name: publishedManifest.name as string,
+    version: publishedManifest.version as string,
+    filename: tarballPath,
+    files: contents.map((file) => ({ path: file })),
+  }
+}

releasing/plugin-commands-publishing/src/recursivePublish.ts

@@ -7,6 +7,7 @@ import { type ResolveFunction } from '@pnpm/resolver-base'
 import { sortPackages } from '@pnpm/sort-packages'
 import { type Registries, type ProjectRootDir } from '@pnpm/types'
 import pFilter from 'p-filter'
+import pLimit from 'p-limit'
 import pick from 'ramda/src/pick'
 import writeJsonFile from 'write-json-file'
 import { publish } from './publish'
@@ -20,6 +21,7 @@ export type PublishRecursiveOpts = Required<Pick<Config,
 | 'rawConfig'
 | 'registries'
 | 'workspaceDir'
+| 'workspaceConcurrency'
 >> &
 Partial<Pick<Config,
 | 'tag'
@@ -103,37 +105,41 @@ export async function recursivePublish (
       appendedArgs.push(`--otp=${opts.cliOptions['otp'] as string}`)
     }
     const chunks = sortPackages(opts.selectedProjectsGraph)
+    const limitPublish = pLimit(opts.workspaceConcurrency ?? 4)
     const tag = opts.tag ?? 'latest'
     for (const chunk of chunks) {
-      // NOTE: It should be possible to publish these packages concurrently.
-      // However, looks like that requires too much resources for some CI envs.
-      // See related issue: https://github.com/pnpm/pnpm/issues/6968
-      for (const pkgDir of chunk) {
-        if (!publishedPkgDirs.has(pkgDir)) continue
-        const pkg = opts.selectedProjectsGraph[pkgDir].package
-        const registry = pkg.manifest.publishConfig?.registry ?? pickRegistryForPackage(opts.registries, pkg.manifest.name!)
-        // eslint-disable-next-line no-await-in-loop
-        const publishResult = await publish({
-          ...opts,
-          dir: pkg.rootDir,
-          argv: {
-            original: [
-              'publish',
-              '--tag',
-              tag,
-              '--registry',
-              registry,
-              ...appendedArgs,
-            ],
-          },
-          gitChecks: false,
-          recursive: false,
-        }, [pkg.rootDir])
-        if (publishResult?.manifest != null) {
-          publishedPackages.push(pick(['name', 'version'], publishResult.manifest))
-        } else if (publishResult?.exitCode) {
-          return { exitCode: publishResult.exitCode }
-        }
+      // eslint-disable-next-line no-await-in-loop
+      const publishResults = await Promise.all(chunk.map(pkgDir =>
+        limitPublish(async () => {
+          if (!publishedPkgDirs.has(pkgDir)) return
+          const pkg = opts.selectedProjectsGraph[pkgDir].package
+          const registry = pkg.manifest.publishConfig?.registry ?? pickRegistryForPackage(opts.registries, pkg.manifest.name!)
+
+          const publishResult = await publish({
+            ...opts,
+            dir: pkg.rootDir,
+            argv: {
+              original: [
+                'publish',
+                '--tag',
+                tag,
+                '--registry',
+                registry,
+                ...appendedArgs,
+              ],
+            },
+            gitChecks: false,
+            recursive: false,
+          }, [pkg.rootDir])
+          if (publishResult?.manifest != null) {
+            publishedPackages.push(pick(['name', 'version'], publishResult.manifest))
+          }
+          return publishResult
+        })
+      ))
+      const failedPublish = publishResults.find((result) => result?.exitCode)
+      if (failedPublish) {
+        return { exitCode: failedPublish.exitCode! }
       }
     }
   }

releasing/plugin-commands-publishing/test/pack.ts

@@ -1,9 +1,12 @@
 import fs from 'fs'
 import path from 'path'
 import { pack } from '@pnpm/plugin-commands-publishing'
-import { prepare, tempDir } from '@pnpm/prepare'
+import { prepare, preparePackages, tempDir } from '@pnpm/prepare'
 import tar from 'tar'
 import chalk from 'chalk'
+import { sync as writeYamlFile } from 'write-yaml-file'
+import { filterPackagesFromDir } from '@pnpm/workspace.filter-packages-from-dir'
+import { type PackResultJson } from '../src/pack'
 import { DEFAULT_OPTS } from './utils'
 
 test('pack: package with package.json', async () => {
@@ -512,12 +515,12 @@ test('pack: should display packed contents order by name', async () => {
     extraBinPaths: [],
   })
 
-  expect(output).toBe(`${chalk.blueBright('Tarball Contents')}
+  expect(output).toBe(`package: test-publish-package.json@0.0.0
+${chalk.blueBright('Tarball Contents')}
 a.js
 b.js
 package.json
 src/index.ts
-
 ${chalk.blueBright('Tarball Details')}
 test-publish-package.json-0.0.0.tgz`)
 })
@@ -561,3 +564,164 @@ test('pack: display in json format', async () => {
     ],
   }, null, 2))
 })
+
+test('pack: recursive pack and display in json format', async () => {
+  const dir = tempDir()
+
+  const pkg1 = {
+    name: '@pnpmtest/test-recursive-pack-project-1',
+    version: '1.0.0',
+
+    dependencies: {
+      'is-positive': '1.0.0',
+    },
+  }
+  const pkg2 = {
+    name: '@pnpmtest/test-recursive-pack-project-2',
+    version: '1.0.0',
+
+    dependencies: {
+      'is-negative': '1.0.0',
+    },
+  }
+
+  prepare({
+    name: '@pnpmtest/test-recursive-pack-project',
+    version: '0.0.0',
+  }, {
+    tempDir: dir,
+  })
+
+  const pkgs = [
+    pkg1,
+    pkg2,
+    // This will be packed because the pack command does not check whether it is in the registry
+    {
+      name: 'is-positive',
+      version: '1.0.0',
+
+      scripts: {
+        prepublishOnly: 'exit 1',
+      },
+    },
+    // This will be packed because the pack command does not check whether it is a private package
+    {
+      name: 'i-am-private',
+      version: '1.0.0',
+
+      private: true,
+      scripts: {
+        prepublishOnly: 'exit 1',
+      },
+    },
+  ]
+
+  preparePackages(pkgs, {
+    tempDir: path.join(dir, 'project'),
+  })
+
+  writeYamlFile(path.join(dir, 'pnpm-workspace.yaml'), { packages: pkgs.filter(pkg => pkg).map(pkg => pkg.name) })
+
+  const { selectedProjectsGraph } = await filterPackagesFromDir(dir, [])
+
+  const output = await pack.handler({
+    ...DEFAULT_OPTS,
+    argv: { original: [] },
+    dir,
+    extraBinPaths: [],
+    json: true,
+    recursive: true,
+    selectedProjectsGraph,
+  })
+
+  const json: PackResultJson[] = JSON.parse(output)
+
+  expect(Array.isArray(json)).toBeTruthy()
+  expect(json).toHaveLength(5)
+
+  for (const pkg of json) {
+    expect(pkg).toHaveProperty('name')
+    expect(pkg).toHaveProperty('version')
+    expect(pkg).toHaveProperty('filename')
+    expect(pkg).toHaveProperty('files')
+    expect(Array.isArray(pkg.files)).toBeTruthy()
+    for (const file of pkg.files) {
+      expect(file).toHaveProperty('path')
+    }
+    expect(fs.existsSync(pkg.filename)).toBeTruthy()
+  }
+})
+
+test('pack: recursive pack with filter', async () => {
+  const dir = tempDir()
+
+  const pkg1 = {
+    name: '@pnpmtest/test-recursive-pack-project-1',
+    version: '1.0.0',
+
+    dependencies: {
+      'is-positive': '1.0.0',
+    },
+  }
+  const pkg2 = {
+    name: '@pnpmtest/test-recursive-pack-project-2',
+    version: '1.0.0',
+
+    dependencies: {
+      'is-negative': '1.0.0',
+    },
+  }
+
+  prepare({
+    name: '@pnpmtest/test-recursive-pack-project',
+    version: '0.0.0',
+  }, {
+    tempDir: dir,
+  })
+
+  const pkgs = [
+    pkg1,
+    pkg2,
+    {
+      name: 'is-positive',
+      version: '1.0.0',
+
+      scripts: {
+        prepublishOnly: 'exit 1',
+      },
+    },
+    {
+      name: 'i-am-private',
+      version: '1.0.0',
+
+      private: true,
+      scripts: {
+        prepublishOnly: 'exit 1',
+      },
+    },
+  ]
+
+  preparePackages(pkgs, {
+    tempDir: path.join(dir, 'project'),
+  })
+
+  writeYamlFile(path.join(dir, 'pnpm-workspace.yaml'), { packages: pkgs.filter(pkg => pkg).map(pkg => pkg.name) })
+
+  const { selectedProjectsGraph } = await filterPackagesFromDir(dir, [{ namePattern: '@pnpmtest/*' }])
+
+  const output = await pack.handler({
+    ...DEFAULT_OPTS,
+    argv: { original: [] },
+    dir,
+    extraBinPaths: [],
+    selectedProjectsGraph,
+    recursive: true,
+    unicode: false,
+  })
+
+  expect(output).toContain('package: @pnpmtest/test-recursive-pack-project@0.0.0')
+  expect(output).toContain('package: @pnpmtest/test-recursive-pack-project-1@1.0.0')
+  expect(output).toContain('package: @pnpmtest/test-recursive-pack-project-2@1.0.0')
+  expect(output).not.toContain('package: is-positive')
+  expect(output).not.toContain('package: i-am-private')
+})