fix: packing the same file twice during publish (#7250)

close #6997

.changeset/lazy-mirrors-mate.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/fs.packlist": major
+---
+
+Initial release.

.changeset/rich-coins-walk.md

@@ -0,0 +1,10 @@
+---
+"@pnpm/plugin-commands-publishing": patch
+"@pnpm/plugin-commands-patching": patch
+"@pnpm/directory-fetcher": patch
+"pnpm": patch
+---
+
+`pnpm publish` should not pack the same file twice sometimes [#6997](https://github.com/pnpm/pnpm/issues/6997).
+
+The fix was to update `npm-packlist` to the latest version.

cspell.json

@@ -129,6 +129,7 @@
     "noproxy",
     "nosystem",
     "nothrow",
+    "npmcli",
     "npmignore",
     "npmjs",
     "ofjergrg",

fetching/directory-fetcher/package.json

@@ -35,15 +35,14 @@
   },
   "dependencies": {
     "@pnpm/fetcher-base": "workspace:*",
+    "@pnpm/fs.packlist": "workspace:*",
     "@pnpm/read-project-manifest": "workspace:*",
     "@pnpm/resolver-base": "workspace:*",
-    "@pnpm/types": "workspace:*",
-    "npm-packlist": "^5.1.3"
+    "@pnpm/types": "workspace:*"
   },
   "devDependencies": {
     "@pnpm/directory-fetcher": "workspace:*",
     "@pnpm/test-fixtures": "workspace:*",
-    "@types/npm-packlist": "^3.0.0",
     "@zkochan/rimraf": "^2.1.3"
   },
   "exports": {

fetching/directory-fetcher/src/index.ts

@@ -2,9 +2,9 @@ import { promises as fs, type Stats } from 'fs'
 import path from 'path'
 import type { DirectoryFetcher, DirectoryFetcherOptions } from '@pnpm/fetcher-base'
 import { logger } from '@pnpm/logger'
+import { packlist } from '@pnpm/fs.packlist'
 import { safeReadProjectManifestOnly } from '@pnpm/read-project-manifest'
 import { type DependencyManifest } from '@pnpm/types'
-import packlist from 'npm-packlist'
 
 const directoryFetcherLogger = logger('directory-fetcher')
 
@@ -128,7 +128,7 @@ async function fetchPackageFilesFromDir (
   dir: string,
   opts: FetchFromDirOpts
 ) {
-  const files = await packlist({ path: dir })
+  const files = await packlist(dir)
   const filesIndex: Record<string, string> = Object.fromEntries(files.map((file) => [file, path.join(dir, file)]))
   let manifest: DependencyManifest | undefined
   if (opts.readManifest) {

fetching/directory-fetcher/tsconfig.json

@@ -12,6 +12,9 @@
     {
       "path": "../../__utils__/test-fixtures"
     },
+    {
+      "path": "../../fs/packlist"
+    },
     {
       "path": "../../packages/types"
     },

fs/packlist/README.md

@@ -0,0 +1,25 @@
+# @pnpm/fs.packlist
+
+> Get a list of the files to add from a directory into an npm package
+
+<!--@shields('npm')-->
+[![npm version](https://img.shields.io/npm/v/packlist.svg)](https://www.npmjs.com/package/@pnpm/fs.packlist)
+<!--/@-->
+
+## Installation
+
+```sh
+pnpm add @pnpm/fs.packlist
+```
+
+## Usage
+
+```js
+const { packlist } = require('path')
+
+const files = packlist('/package-dir')
+```
+
+## License
+
+MIT © [Zoltan Kochan](https://www.kochan.io)

fs/packlist/package.json

@@ -0,0 +1,42 @@
+{
+  "name": "@pnpm/fs.packlist",
+  "version": "0.0.0",
+  "description": "Get a list of the files to add from a directory into an npm package",
+  "main": "lib/index.js",
+  "files": [
+    "lib",
+    "!*.map"
+  ],
+  "types": "lib/index.d.ts",
+  "scripts": {
+    "lint": "eslint \"src/**/*.ts\"",
+    "prepublishOnly": "pnpm run compile",
+    "compile": "tsc --build && pnpm run lint --fix",
+    "test": "pnpm run compile"
+  },
+  "repository": "https://github.com/pnpm/pnpm/blob/main/fs/packlist",
+  "keywords": [
+    "pnpm8"
+  ],
+  "engines": {
+    "node": ">=16.14"
+  },
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/pnpm/pnpm/issues"
+  },
+  "homepage": "https://github.com/pnpm/pnpm/blob/main/fs/packlist#readme",
+  "dependencies": {
+    "@npmcli/arborist": "7.2.0",
+    "npm-packlist": "^8.0.0"
+  },
+  "funding": "https://opencollective.com/pnpm",
+  "devDependencies": {
+    "@pnpm/fs.packlist": "workspace:*",
+    "@types/npm-packlist": "^7.0.2",
+    "@types/npmcli__arborist": "5.6.4"
+  },
+  "exports": {
+    ".": "./lib/index.js"
+  }
+}

fs/packlist/src/index.ts

@@ -0,0 +1,8 @@
+import Arborist from '@npmcli/arborist'
+import npmPacklist from 'npm-packlist'
+
+export async function packlist (pkgDir: string) {
+  const arborist = new Arborist(({ path: pkgDir }))
+  const tree = await arborist.loadActual()
+  return npmPacklist(tree)
+}

fs/packlist/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "extends": "@pnpm/tsconfig",
+  "compilerOptions": {
+    "outDir": "lib",
+    "rootDir": "src"
+  },
+  "include": [
+    "src/**/*.ts",
+    "../../__typings__/**/*.d.ts"
+  ],
+  "references": [],
+  "composite": true
+}

fs/packlist/tsconfig.lint.json

@@ -0,0 +1,8 @@
+{
+  "extends": "./tsconfig.json",
+  "include": [
+    "src/**/*.ts",
+    "test/**/*.ts",
+    "../../__typings__/**/*.d.ts"
+  ]
+}

patching/plugin-commands-patching/package.json

@@ -37,7 +37,6 @@
     "@pnpm/registry-mock": "3.16.0",
     "@pnpm/test-fixtures": "workspace:*",
     "@types/normalize-path": "^3.0.1",
-    "@types/npm-packlist": "^3.0.0",
     "@types/ramda": "0.28.20",
     "@types/semver": "7.5.3",
     "write-yaml-file": "^5.0.0"
@@ -47,6 +46,7 @@
     "@pnpm/config": "workspace:*",
     "@pnpm/constants": "workspace:*",
     "@pnpm/error": "workspace:*",
+    "@pnpm/fs.packlist": "workspace:*",
     "@pnpm/lockfile-file": "workspace:*",
     "@pnpm/lockfile-utils": "workspace:*",
     "@pnpm/modules-yaml": "workspace:*",
@@ -62,7 +62,6 @@
     "escape-string-regexp": "^4.0.0",
     "fast-glob": "^3.3.1",
     "normalize-path": "^3.0.0",
-    "npm-packlist": "^5.1.3",
     "ramda": "npm:@pnpm/ramda@0.28.1",
     "realpath-missing": "^1.1.0",
     "render-help": "^1.0.3",

patching/plugin-commands-patching/src/patchCommit.ts

@@ -2,6 +2,7 @@ import fs from 'fs'
 import path from 'path'
 import { docsUrl } from '@pnpm/cli-utils'
 import { type Config, types as allTypes } from '@pnpm/config'
+import { packlist } from '@pnpm/fs.packlist'
 import { install } from '@pnpm/plugin-commands-installation'
 import { readPackageJsonFromDir } from '@pnpm/read-package-json'
 import { tryReadProjectManifest } from '@pnpm/read-project-manifest'
@@ -15,7 +16,6 @@ import renderHelp from 'render-help'
 import tempy from 'tempy'
 import { writePackage } from './writePackage'
 import { type ParseWantedDependencyResult, parseWantedDependency } from '@pnpm/parse-wanted-dependency'
-import packlist from 'npm-packlist'
 import { type GetPatchedDependencyOptions, getVersionsFromLockfile } from './getPatchedDependency'
 
 export const rcOptionsTypes = cliOptionsTypes
@@ -157,7 +157,7 @@ function removeTrailingAndLeadingSlash (p: string) {
  * This is required in order for the diff to not include files that are not part of the package.
  */
 async function preparePkgFilesForDiff (src: string): Promise<string> {
-  const files = Array.from(new Set((await packlist({ path: src })).map((f) => path.join(f))))
+  const files = Array.from(new Set((await packlist(src)).map((f) => path.join(f))))
   // If there are no extra files in the source directories, then there is no reason
   // to copy.
   if (await areAllFilesInPkg(files, src)) {

patching/plugin-commands-patching/tsconfig.json

@@ -27,6 +27,9 @@
     {
       "path": "../../fetching/pick-fetcher"
     },
+    {
+      "path": "../../fs/packlist"
+    },
     {
       "path": "../../lockfile/lockfile-file"
     },

pnpm/test/install/hooks.ts

@@ -255,7 +255,6 @@ test('readPackage hook from pnpmfile at root of workspace', async () => {
   expect(lockfile.packages!['/is-negative@1.0.0'].dependencies).toStrictEqual({
     '@pnpm.e2e/dep-of-pkg-with-1-dep': '100.1.0',
   })
-  /* eslint-enable @typescript-eslint/no-unnecessary-type-assertion */
 })
 
 test('readPackage hook during update', async () => {

pnpm/test/recursive/misc.ts

@@ -231,7 +231,6 @@ test('recursive installation of packages in workspace ignores hooks in packages'
   const depPaths = Object.keys(lockfile.packages ?? [])
   expect(depPaths).not.toContain('/@pnpm.e2e/dep-of-pkg-with-1-dep@100.1.0')
   expect(depPaths).toContain('/is-number@1.0.0')
-  /* eslint-enable @typescript-eslint/no-unnecessary-type-assertion */
 })
 
 test('ignores .pnpmfile.cjs during recursive installation when --ignore-pnpmfile is used', async () => {

releasing/plugin-commands-publishing/package.json

@@ -38,7 +38,6 @@
     "@pnpm/registry-mock": "3.16.0",
     "@types/cross-spawn": "^6.0.4",
     "@types/is-windows": "^1.0.1",
-    "@types/npm-packlist": "^3.0.0",
     "@types/proxyquire": "^1.3.30",
     "@types/ramda": "0.28.20",
     "@types/sinon": "^10.0.20",
@@ -59,6 +58,7 @@
     "@pnpm/config": "workspace:*",
     "@pnpm/error": "workspace:*",
     "@pnpm/exportable-manifest": "workspace:*",
+    "@pnpm/fs.packlist": "workspace:*",
     "@pnpm/git-utils": "workspace:*",
     "@pnpm/lifecycle": "workspace:*",
     "@pnpm/package-bins": "workspace:*",
@@ -71,7 +71,6 @@
     "enquirer": "^2.4.1",
     "execa": "npm:safe-execa@0.1.2",
     "fast-glob": "^3.3.1",
-    "npm-packlist": "^5.1.3",
     "p-filter": "^2.1.0",
     "ramda": "npm:@pnpm/ramda@0.28.1",
     "realpath-missing": "^1.1.0",

releasing/plugin-commands-publishing/src/pack.ts

@@ -5,6 +5,7 @@ import { PnpmError } from '@pnpm/error'
 import { types as allTypes, type UniversalOptions, type Config } from '@pnpm/config'
 import { readProjectManifest } from '@pnpm/cli-utils'
 import { createExportableManifest } from '@pnpm/exportable-manifest'
+import { packlist } from '@pnpm/fs.packlist'
 import { getBinsFromPackageManifest } from '@pnpm/package-bins'
 import { type DependencyManifest } from '@pnpm/types'
 import fg from 'fast-glob'
@@ -12,7 +13,6 @@ import pick from 'ramda/src/pick'
 import realpathMissing from 'realpath-missing'
 import renderHelp from 'render-help'
 import tar from 'tar-stream'
-import packlist from 'npm-packlist'
 import { runScriptsIfPresent } from './publish'
 
 const LICENSE_GLOB = 'LICEN{S,C}E{,.*}' // cspell:disable-line
@@ -95,7 +95,7 @@ export async function handler (
     throw new PnpmError('PACKAGE_VERSION_NOT_FOUND', `Package version is not defined in the ${manifestFileName}.`)
   }
   const tarballName = `${manifest.name.replace('@', '').replace('/', '-')}-${manifest.version}.tgz`
-  const files = await packlist({ path: dir })
+  const files = await packlist(dir)
   const filesMap: Record<string, string> = Object.fromEntries(files.map((file) => [`package/${file}`, path.join(dir, file)]))
   // cspell:disable-next-line
   if (opts.workspaceDir != null && dir !== opts.workspaceDir && !files.some((file) => /LICEN[CS]E(\..+)?/i.test(file))) {

releasing/plugin-commands-publishing/tsconfig.json

@@ -30,6 +30,9 @@
     {
       "path": "../../exec/run-npm"
     },
+    {
+      "path": "../../fs/packlist"
+    },
     {
       "path": "../../packages/error"
     },