fix: linking packages with invalid file names (#3232)

close #3229
close #2266

.changeset/pretty-knives-kick.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/package-store": patch
+---
+
+Rename files with invalid names if linking fails.

packages/package-store/package.json

@@ -32,13 +32,14 @@
     "path-temp": "^2.0.0",
     "ramda": "^0.27.1",
     "rename-overwrite": "^3.0.0",
+    "sanitize-filename": "^1.6.3",
     "ssri": "6.0.1",
     "write-json-file": "^4.3.0"
   },
   "devDependencies": {
     "@pnpm/client": "workspace:2.0.23",
     "@pnpm/logger": "^3.2.3",
-    "@types/mz": "^2.7.3",
+    "@pnpm/prepare": "workspace:0.0.18",
     "@types/ramda": "^0.27.35",
     "@types/ssri": "^7.1.0",
     "tempy": "^1.0.0"

packages/package-store/src/fs/importIndexedDir.ts

@@ -5,6 +5,7 @@ import makeEmptyDir = require('make-empty-dir')
 import fs = require('mz/fs')
 import pathTemp = require('path-temp')
 import renameOverwrite = require('rename-overwrite')
+import sanitizeFilename = require('sanitize-filename')
 
 const filenameConflictsLogger = pnpmLogger('_filename-conflicts')
 
@@ -23,24 +24,49 @@ export default async function importIndexedDir (
     try {
       await rimraf(stage)
     } catch (err) {} // eslint-disable-line:no-empty
-    if (err['code'] !== 'EEXIST') throw err
-
-    const { uniqueFileMap, conflictingFileNames } = getUniqueFileMap(filenames)
-    if (Object.keys(conflictingFileNames).length === 0) throw err
-    filenameConflictsLogger.debug({
-      conflicts: conflictingFileNames,
-      writingTo: newDir,
-    })
-    globalWarn(
-      `Not all files were linked to "${path.relative(process.cwd(), newDir)}". ` +
-      'Some of the files have equal names in different case, ' +
-      'which is an issue on case-insensitive filesystems. ' +
-      `The conflicting file names are: ${JSON.stringify(conflictingFileNames)}`
-    )
-    await importIndexedDir(importFile, newDir, uniqueFileMap)
+    if (err['code'] === 'EEXIST') {
+      const { uniqueFileMap, conflictingFileNames } = getUniqueFileMap(filenames)
+      if (Object.keys(conflictingFileNames).length === 0) throw err
+      filenameConflictsLogger.debug({
+        conflicts: conflictingFileNames,
+        writingTo: newDir,
+      })
+      globalWarn(
+        `Not all files were linked to "${path.relative(process.cwd(), newDir)}". ` +
+        'Some of the files have equal names in different case, ' +
+        'which is an issue on case-insensitive filesystems. ' +
+        `The conflicting file names are: ${JSON.stringify(conflictingFileNames)}`
+      )
+      await importIndexedDir(importFile, newDir, uniqueFileMap)
+      return
+    }
+    if (err['code'] === 'ENOENT') {
+      const { sanitizedFilenames, invalidFilenames } = sanitizeFilenames(filenames)
+      if (invalidFilenames.length === 0) throw err
+      globalWarn(`\
+The package linked to "${path.relative(process.cwd(), newDir)}" had \
+files with invalid names: ${invalidFilenames.join(', ')}. \
+They were renamed.`)
+      await importIndexedDir(importFile, newDir, sanitizedFilenames)
+      return
+    }
+    throw err
   }
 }
 
+function sanitizeFilenames (filenames: Record<string, string>) {
+  const sanitizedFilenames: Record<string, string> = {}
+  const invalidFilenames: string[] = []
+  for (const [filename, src] of Object.entries(filenames)) {
+    const sanitizedFilename = filename.split('/').map((f) => sanitizeFilename(f)).join('/')
+    if (sanitizedFilename !== filename) {
+      invalidFilenames.push(filename)
+    }
+    sanitizedFilenames[sanitizedFilename] = src
+  }
+  return { sanitizedFilenames, invalidFilenames }
+}
+
 async function tryImportIndexedDir (importFile: ImportFile, newDir: string, filenames: Record<string, string>) {
   await makeEmptyDir(newDir, { recursive: true })
   const alldirs = new Set<string>()

packages/package-store/test/importingPkgWithInvalidFiles.ts

@@ -0,0 +1,19 @@
+import { promises as fs } from 'fs'
+import path from 'path'
+import createImportPackage from '@pnpm/package-store/lib/storeController/createImportPackage'
+import { prepareEmpty } from '@pnpm/prepare'
+
+test('importing a package with invalid files', async () => {
+  prepareEmpty()
+  const importPackage = createImportPackage('copy')
+  const target = path.resolve('target')
+  await importPackage(target, {
+    filesMap: {
+      'foo?bar/qar>zoo.txt': __filename,
+      '1*2.txt': __filename,
+    },
+    force: false,
+    fromStore: false,
+  })
+  expect(await (await fs.readdir(target)).length).toBe(2)
+})

packages/package-store/tsconfig.json

@@ -9,6 +9,9 @@
     "../../typings/**/*.d.ts"
   ],
   "references": [
+    {
+      "path": "../../privatePackages/prepare"
+    },
     {
       "path": "../cafs"
     },

pnpm-lock.yaml

@@ -1572,13 +1572,14 @@ importers:
       path-temp: 2.0.0
       ramda: 0.27.1
       rename-overwrite: 3.1.0
+      sanitize-filename: 1.6.3
       ssri: 6.0.1
       write-json-file: 4.3.0
     devDependencies:
       '@pnpm/client': link:../client
       '@pnpm/logger': 3.2.3
       '@pnpm/package-store': 'link:'
-      '@types/mz': 2.7.3
+      '@pnpm/prepare': link:../../privatePackages/prepare
       '@types/ramda': 0.27.38
       '@types/ssri': 7.1.0
       tempy: 1.0.0
@@ -1590,10 +1591,10 @@ importers:
       '@pnpm/logger': ^3.2.3
       '@pnpm/package-requester': workspace:13.0.0
       '@pnpm/package-store': 'link:'
+      '@pnpm/prepare': workspace:0.0.18
       '@pnpm/resolver-base': workspace:7.1.1
       '@pnpm/store-controller-types': workspace:10.0.0
       '@pnpm/types': workspace:6.4.0
-      '@types/mz': ^2.7.3
       '@types/ramda': ^0.27.35
       '@types/ssri': ^7.1.0
       '@zkochan/rimraf': ^1.0.0
@@ -1605,6 +1606,7 @@ importers:
       path-temp: ^2.0.0
       ramda: ^0.27.1
       rename-overwrite: ^3.0.0
+      sanitize-filename: ^1.6.3
       ssri: 6.0.1
       tempy: ^1.0.0
       write-json-file: ^4.3.0
@@ -13252,6 +13254,12 @@ packages:
       node: 6.* || 8.* || >= 10.*
     hasBin: true
 
+  /sanitize-filename/1.6.3:
+    resolution: {integrity: sha512-y/52Mcy7aw3gRm7IrcGDFx/bCk4AhRh2eI9luHOQM86nZsqwiRkkq2GekHXBBD+SmPidc8i2PqtYZl+pWJ8Oeg==}
+    dependencies:
+      truncate-utf8-bytes: 1.0.2
+    dev: false
+
   /saxes/3.1.11:
     resolution: {integrity: sha512-Ydydq3zC+WYDJK1+gRxRapLIED9PWeSuuS41wqyoRmzvhhh9nc+QQrVMKJYzJFULazeGhzSV0QleN2wD3boh2g==}
     dependencies:
@@ -14336,6 +14344,12 @@ packages:
     engines:
       node: '>=0.10.0'
 
+  /truncate-utf8-bytes/1.0.2:
+    resolution: {integrity: sha1-QFkjkJWS1W94pYGENLC3hInKXys=}
+    dependencies:
+      utf8-byte-length: 1.0.4
+    dev: false
+
   /ts-jest/26.5.3_jest@26.6.3+typescript@4.2.3:
     resolution: {integrity: sha512-nBiiFGNvtujdLryU7MiMQh1iPmnZ/QvOskBbD2kURiI1MwqvxlxNnaAB/z9TbslMqCsSbu5BXvSSQPc5tvHGeA==}
     dependencies:
@@ -14692,6 +14706,10 @@ packages:
     engines:
       node: '>=0.10.0'
 
+  /utf8-byte-length/1.0.4:
+    resolution: {integrity: sha1-9F8VDExm7uloGGUFq5P8u4rWv2E=}
+    dev: false
+
   /util-deprecate/1.0.2:
     resolution: {integrity: sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=}