perf: use rclonefile to fix cloning on macOS and Windows Dev Drive (#7031)

close #5001

---------

Co-authored-by: Ignacio Aldama Vicente <sr.drabx@gmail.com>

.github/workflows/release.yml

@@ -34,7 +34,8 @@ jobs:
           version: latest-8
           standalone: true
       - name: pnpm install
-        run: pnpm install
+        # We use --force because we want all artifacts of @reflink/reflink to be installed.
+        run: pnpm install --force
       - name: Publish Packages
         env:
           # setting the "npm_config_//registry.npmjs.org/:_authToken" env variable directly doesn't work.

.pnpmfile.cjs

@@ -0,0 +1,15 @@
+module.exports = {
+  hooks: {
+    readPackage: (manifest) => {
+      if (manifest.name === '@reflink/reflink') {
+        for (const depName of Object.keys(manifest.optionalDependencies)) {
+          // We don't need refclone on Linux as Node.js supports reflinks out of the box on Linux
+          if (depName.includes('linux')) {
+            delete manifest.optionalDependencies[depName]
+          }
+        }
+      }
+      return manifest
+    }
+  }
+}

fs/indexed-pkg-importer/package.json

@@ -24,6 +24,7 @@
     "p-limit": "^3.1.0",
     "path-exists": "^4.0.0",
     "path-temp": "^2.1.0",
+    "@reflink/reflink": "0.1.6",
     "rename-overwrite": "^4.0.3",
     "sanitize-filename": "^1.6.3"
   },

fs/indexed-pkg-importer/src/index.ts

@@ -22,7 +22,7 @@ function createImportPackage (packageImportMethod?: 'auto' | 'hardlink' | 'copy'
   switch (packageImportMethod ?? 'auto') {
   case 'clone':
     packageImportMethodLogger.debug({ method: 'clone' })
-    return clonePkg
+    return clonePkg.bind(null, createCloneFunction())
   case 'hardlink':
     packageImportMethodLogger.debug({ method: 'hardlink' })
     return hardlinkPkg.bind(null, linkOrCopy)
@@ -49,9 +49,10 @@ function createAutoImporter (): ImportIndexedPackage {
     opts: ImportOptions
   ): string | undefined {
     try {
-      if (!clonePkg(to, opts)) return undefined
+      const _clonePkg = clonePkg.bind(null, createCloneFunction())
+      if (!_clonePkg(to, opts)) return undefined
       packageImportMethodLogger.debug({ method: 'clone' })
-      auto = clonePkg
+      auto = _clonePkg
       return 'clone'
     } catch (err: any) { // eslint-disable-line
       // ignore
@@ -87,9 +88,10 @@ function createCloneOrCopyImporter (): ImportIndexedPackage {
     opts: ImportOptions
   ): string | undefined {
     try {
-      if (!clonePkg(to, opts)) return undefined
+      const _clonePkg = clonePkg.bind(null, createCloneFunction())
+      if (!_clonePkg(to, opts)) return undefined
       packageImportMethodLogger.debug({ method: 'clone' })
-      auto = clonePkg
+      auto = _clonePkg
       return 'clone'
     } catch (err: any) { // eslint-disable-line
       // ignore
@@ -100,21 +102,33 @@ function createCloneOrCopyImporter (): ImportIndexedPackage {
   }
 }
 
+type CloneFunction = (src: string, dest: string) => void
+
 function clonePkg (
+  clone: CloneFunction,
   to: string,
   opts: ImportOptions
 ) {
   const pkgJsonPath = path.join(to, 'package.json')
 
   if (opts.resolvedFrom !== 'store' || opts.force || !existsSync(pkgJsonPath)) {
-    importIndexedDir(cloneFile, to, opts.filesMap, opts)
+    importIndexedDir(clone, to, opts.filesMap, opts)
     return 'clone'
   }
   return undefined
 }
 
-function cloneFile (from: string, to: string) {
-  fs.copyFileSync(from, to, constants.COPYFILE_FICLONE_FORCE)
+function createCloneFunction (): CloneFunction {
+  // Node.js currently does not natively support reflinks on Windows and macOS.
+  // Hence, we use a third party solution.
+  if (process.platform === 'win32' || process.platform === 'darwin') {
+    // eslint-disable-next-line
+    const { reflinkFileSync } = require('@reflink/reflink')
+    return reflinkFileSync
+  }
+  return (src: string, dest: string) => {
+    fs.copyFileSync(src, dest, constants.COPYFILE_FICLONE_FORCE)
+  }
 }
 
 function hardlinkPkg (

fs/indexed-pkg-importer/test/createImportPackage.test.ts

@@ -4,6 +4,8 @@ import { createIndexedPkgImporter } from '@pnpm/fs.indexed-pkg-importer'
 import gfs from '@pnpm/graceful-fs'
 import { globalInfo } from '@pnpm/logger'
 
+const testOnLinuxOnly = (process.platform === 'darwin' || process.platform === 'win32') ? test.skip : test
+
 jest.mock('@pnpm/graceful-fs', () => {
   const { access, promises } = jest.requireActual('fs')
   const fsMock = {
@@ -36,7 +38,7 @@ beforeEach(() => {
   ;(globalInfo as jest.Mock).mockReset()
 })
 
-test('packageImportMethod=auto: clone files by default', () => {
+testOnLinuxOnly('packageImportMethod=auto: clone files by default', () => {
   const importPackage = createIndexedPkgImporter('auto')
   expect(importPackage('project/package', {
     filesMap: {
@@ -58,7 +60,7 @@ test('packageImportMethod=auto: clone files by default', () => {
   )
 })
 
-test('packageImportMethod=auto: link files if cloning fails', () => {
+testOnLinuxOnly('packageImportMethod=auto: link files if cloning fails', () => {
   const importPackage = createIndexedPkgImporter('auto')
   ;(gfs.copyFileSync as jest.Mock).mockImplementation(() => {
     throw new Error('This file system does not support cloning')
@@ -90,7 +92,7 @@ test('packageImportMethod=auto: link files if cloning fails', () => {
   expect(gfs.linkSync).toBeCalledWith(path.join('hash2'), path.join('project2', 'package_tmp', 'index.js'))
 })
 
-test('packageImportMethod=auto: link files if cloning fails and even hard linking fails but not with EXDEV error', () => {
+testOnLinuxOnly('packageImportMethod=auto: link files if cloning fails and even hard linking fails but not with EXDEV error', () => {
   const importPackage = createIndexedPkgImporter('auto')
   ;(gfs.copyFileSync as jest.Mock).mockImplementation(() => {
     throw new Error('This file system does not support cloning')
@@ -114,7 +116,7 @@ test('packageImportMethod=auto: link files if cloning fails and even hard linkin
   expect(gfs.copyFileSync).toBeCalledTimes(1)
 })
 
-test('packageImportMethod=auto: chooses copying if cloning and hard linking is not possible', () => {
+testOnLinuxOnly('packageImportMethod=auto: chooses copying if cloning and hard linking is not possible', () => {
   const importPackage = createIndexedPkgImporter('auto')
   ;(gfs.copyFileSync as jest.Mock).mockImplementation((src: string, dest: string, flags?: number) => {
     if (flags === fs.constants.COPYFILE_FICLONE_FORCE) {
@@ -135,7 +137,7 @@ test('packageImportMethod=auto: chooses copying if cloning and hard linking is n
   expect(gfs.copyFileSync).toBeCalledTimes(2)
 })
 
-test('packageImportMethod=hardlink: fall back to copying if hardlinking fails', () => {
+testOnLinuxOnly('packageImportMethod=hardlink: fall back to copying if hardlinking fails', () => {
   const importPackage = createIndexedPkgImporter('hardlink')
   ;(gfs.linkSync as jest.Mock).mockImplementation((src: string, dest: string) => {
     if (dest.endsWith('license')) {

pnpm/bundle.ts

@@ -0,0 +1,42 @@
+import { build } from 'esbuild'
+
+/**
+ * We publish the "pnpm" package bundled with all refclone artifacts for macOS and Windows.
+ * Unfortunately, we need to do this because otherwise corepack wouldn't be able to install
+ * pnpm with reflink support. Reflink is only unpacking the pnpm tarball and does no additional actions.
+ */
+;(async () => {
+  try {
+    await build({
+      entryPoints: ['lib/pnpm.js'],
+      bundle: true,
+      platform: 'node',
+      outfile: 'dist/pnpm.cjs',
+      external: ['node-gyp'],
+      define: {
+        'process.env.npm_package_name': JSON.stringify(
+          process.env.npm_package_name
+        ),
+        'process.env.npm_package_version': JSON.stringify(
+          process.env.npm_package_version
+        ),
+      },
+      loader: {
+        '.node': 'copy',
+      },
+    })
+
+    await build({
+      entryPoints: ['../worker/lib/worker.js'],
+      bundle: true,
+      platform: 'node',
+      outfile: 'dist/worker.js',
+      loader: {
+        '.node': 'copy',
+      },
+    })
+  } catch (err) {
+    console.error(err)
+    process.exit(1)
+  }
+})()

pnpm/dev/pd.js

@@ -53,6 +53,9 @@ const pnpmPackageJson = JSON.parse(fs.readFileSync(pathLib.join(__dirname, 'pack
     sourcemap: true, // nice for local debugging
     logLevel: 'warning', // keeps esbuild quiet unless there's a problem
     plugins: [spnpmImportsPlugin],
+    loader: {
+      '.node': 'binary',
+    }
   })
 
   // Require the file just built by esbuild

pnpm/package-macos-arm64.json

@@ -1,6 +1,11 @@
 {
   "pkg": {
-    "assets": ["dist/worker.js", "dist/pnpmrc", "dist/scripts/*"],
+    "assets": [
+      "dist/worker.js",
+      "dist/pnpmrc",
+      "dist/scripts/*",
+      "dist/refclone.darwin-arm64-*.node"
+    ],
     "targets": ["node18-macos-arm64"],
     "outputPath": "../macos-arm64"
   }

pnpm/package-macos-x64.json

@@ -1,6 +1,11 @@
 {
   "pkg": {
-    "assets": ["dist/worker.js", "dist/pnpmrc", "dist/scripts/*"],
+    "assets": [
+      "dist/worker.js",
+      "dist/pnpmrc",
+      "dist/scripts/*",
+      "dist/refclone.darwin-x64-*.node"
+    ],
     "targets": ["node18-macos-x64"],
     "outputPath": "../macos-x64"
   }

pnpm/package-win-x64.json

@@ -1,6 +1,11 @@
 {
   "pkg": {
-    "assets": ["dist/worker.js", "dist/pnpmrc", "dist/scripts/*"],
+    "assets": [
+      "dist/worker.js",
+      "dist/pnpmrc",
+      "dist/scripts/*",
+      "dist/refclone.win32-x64-*.node"
+    ],
     "targets": ["node18-win-x64"],
     "outputPath": "../win-x64"
   }

pnpm/package.json

@@ -82,7 +82,6 @@
     "chalk": "^4.1.2",
     "ci-info": "^3.8.0",
     "cross-spawn": "^7.0.3",
-    "cross-var-no-babel": "^1.2.0",
     "deep-require-cwd": "1.0.0",
     "delay": "^5.0.0",
     "dir-is-case-sensitive": "^2.0.0",
@@ -153,7 +152,7 @@
     "url": "git+https://github.com/pnpm/pnpm.git"
   },
   "scripts": {
-    "bundle": "cross-var esbuild lib/pnpm.js --bundle --platform=node --outfile=dist/pnpm.cjs --external:node-gyp --define:process.env.npm_package_name=\\\"$npm_package_name\\\" --define:process.env.npm_package_version=\\\"$npm_package_version\\\" && esbuild ../worker/lib/worker.js --bundle --platform=node --outfile=dist/worker.js",
+    "bundle": "ts-node bundle.ts",
     "start": "tsc --watch",
     "lint": "eslint \"src/**/*.ts\" \"test/**/*.ts\"",
     "pretest:e2e": "rimraf node_modules/.bin/pnpm",