perf: use a worker for hard linking directories (#7154)

.changeset/cold-meals-push.md

@@ -0,0 +1,7 @@
+---
+"@pnpm/plugin-commands-rebuild": minor
+"@pnpm/build-modules": minor
+"@pnpm/worker": patch
+---
+
+Directory hard linking moved to the worker.

.changeset/fresh-camels-design.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/fs.hard-link-dir": major
+---
+
+Changed to be sync.

exec/build-modules/package.json

@@ -44,6 +44,7 @@
     "@pnpm/read-package-json": "workspace:*",
     "@pnpm/store-controller-types": "workspace:*",
     "@pnpm/types": "workspace:*",
+    "@pnpm/worker": "workspace:*",
     "p-defer": "^3.0.0",
     "ramda": "npm:@pnpm/ramda@0.28.1",
     "run-groups": "^3.0.1"

exec/build-modules/src/index.ts

@@ -4,7 +4,7 @@ import { skippedOptionalDependencyLogger } from '@pnpm/core-loggers'
 import { runPostinstallHooks } from '@pnpm/lifecycle'
 import { linkBins, linkBinsOfPackages } from '@pnpm/link-bins'
 import { logger } from '@pnpm/logger'
-import { hardLinkDir } from '@pnpm/fs.hard-link-dir'
+import { hardLinkDir } from '@pnpm/worker'
 import { readPackageJsonFromDir, safeReadPackageJsonFromDir } from '@pnpm/read-package-json'
 import { type StoreController } from '@pnpm/store-controller-types'
 import { applyPatchToDir } from '@pnpm/patching.apply-patch'

exec/build-modules/tsconfig.json

@@ -36,6 +36,9 @@
     {
       "path": "../../store/store-controller-types"
     },
+    {
+      "path": "../../worker"
+    },
     {
       "path": "../lifecycle"
     }

exec/plugin-commands-rebuild/package.json

@@ -54,7 +54,6 @@
     "@pnpm/core-loggers": "workspace:*",
     "@pnpm/dependency-path": "workspace:*",
     "@pnpm/error": "workspace:*",
-    "@pnpm/fs.hard-link-dir": "workspace:*",
     "@pnpm/get-context": "workspace:*",
     "@pnpm/deps.graph-sequencer": "workspace:*",
     "@pnpm/lifecycle": "workspace:*",
@@ -70,6 +69,7 @@
     "@pnpm/store-controller-types": "workspace:*",
     "@pnpm/store.cafs": "workspace:*",
     "@pnpm/types": "workspace:*",
+    "@pnpm/worker": "workspace:*",
     "@pnpm/workspace.find-packages": "workspace:*",
     "load-json-file": "^6.2.0",
     "mem": "^8.1.1",

exec/plugin-commands-rebuild/src/implementation/index.ts

@@ -27,7 +27,7 @@ import { createOrConnectStoreController } from '@pnpm/store-connection-manager'
 import { type ProjectManifest } from '@pnpm/types'
 import { createAllowBuildFunction } from '@pnpm/builder.policy'
 import * as dp from '@pnpm/dependency-path'
-import { hardLinkDir } from '@pnpm/fs.hard-link-dir'
+import { hardLinkDir } from '@pnpm/worker'
 import loadJsonFile from 'load-json-file'
 import runGroups from 'run-groups'
 import { graphSequencer } from '@pnpm/deps.graph-sequencer'

exec/plugin-commands-rebuild/tsconfig.json

@@ -33,9 +33,6 @@
     {
       "path": "../../deps/graph-sequencer"
     },
-    {
-      "path": "../../fs/hard-link-dir"
-    },
     {
       "path": "../../lockfile/lockfile-types"
     },
@@ -81,6 +78,9 @@
     {
       "path": "../../store/store-controller-types"
     },
+    {
+      "path": "../../worker"
+    },
     {
       "path": "../../workspace/filter-workspace-packages"
     },

fs/hard-link-dir/src/index.ts

@@ -1,70 +1,64 @@
 import path from 'path'
-import { promises as fs } from 'fs'
+import fs from 'fs'
 import { globalWarn } from '@pnpm/logger'
 
-export async function hardLinkDir (src: string, destDirs: string[]) {
+export function hardLinkDir (src: string, destDirs: string[]) {
   if (destDirs.length === 0) return
   // Don't try to hard link the source directory to itself
   destDirs = destDirs.filter((destDir) => path.relative(destDir, src) !== '')
-  await _hardLinkDir(src, destDirs, true)
+  _hardLinkDir(src, destDirs, true)
 }
 
-async function _hardLinkDir (src: string, destDirs: string[], isRoot?: boolean) {
+function _hardLinkDir (src: string, destDirs: string[], isRoot?: boolean) {
   let files: string[] = []
   try {
-    files = await fs.readdir(src)
+    files = fs.readdirSync(src)
   } catch (err: any) { // eslint-disable-line
     if (!isRoot || err.code !== 'ENOENT') throw err
     globalWarn(`Source directory not found when creating hardLinks for: ${src}. Creating destinations as empty: ${destDirs.join(', ')}`)
-    await Promise.all(
-      destDirs.map((dir) => fs.mkdir(dir, { recursive: true }))
-    )
+    for (const dir of destDirs) {
+      fs.mkdirSync(dir, { recursive: true })
+    }
     return
   }
-  await Promise.all(
-    files.map(async (file) => {
-      if (file === 'node_modules') return
-      const srcFile = path.join(src, file)
-      if ((await fs.lstat(srcFile)).isDirectory()) {
-        const destSubdirs = await Promise.all(
-          destDirs.map(async (destDir) => {
-            const destSubdir = path.join(destDir, file)
-            try {
-              await fs.mkdir(destSubdir, { recursive: true })
-            } catch (err: any) { // eslint-disable-line
-              if (err.code !== 'EEXIST') throw err
-            }
-            return destSubdir
-          })
-        )
-        await _hardLinkDir(srcFile, destSubdirs)
-        return
+  for (const file of files) {
+    if (file === 'node_modules') continue
+    const srcFile = path.join(src, file)
+    if (fs.lstatSync(srcFile).isDirectory()) {
+      const destSubdirs = destDirs.map((destDir) => {
+        const destSubdir = path.join(destDir, file)
+        try {
+          fs.mkdirSync(destSubdir, { recursive: true })
+        } catch (err: any) { // eslint-disable-line
+          if (err.code !== 'EEXIST') throw err
+        }
+        return destSubdir
+      })
+      _hardLinkDir(srcFile, destSubdirs)
+      continue
+    }
+    for (const destDir of destDirs) {
+      const destFile = path.join(destDir, file)
+      try {
+        linkOrCopyFile(srcFile, destFile)
+      } catch (err: any) { // eslint-disable-line
+        if (err.code === 'ENOENT') {
+          // Ignore broken symlinks
+          continue
+        }
+        throw err
       }
-      await Promise.all(
-        destDirs.map(async (destDir) => {
-          const destFile = path.join(destDir, file)
-          try {
-            await linkOrCopyFile(srcFile, destFile)
-          } catch (err: any) { // eslint-disable-line
-            if (err.code === 'ENOENT') {
-              // Ignore broken symlinks
-              return
-            }
-            throw err
-          }
-        })
-      )
-    })
-  )
+    }
+  }
 }
 
-async function linkOrCopyFile (srcFile: string, destFile: string) {
+function linkOrCopyFile (srcFile: string, destFile: string) {
   try {
-    await linkOrCopy(srcFile, destFile)
+    linkOrCopy(srcFile, destFile)
   } catch (err: any) { // eslint-disable-line
     if (err.code === 'ENOENT') {
-      await fs.mkdir(path.dirname(destFile), { recursive: true })
-      await linkOrCopy(srcFile, destFile)
+      fs.mkdirSync(path.dirname(destFile), { recursive: true })
+      linkOrCopy(srcFile, destFile)
       return
     }
     if (err.code !== 'EEXIST') {
@@ -77,11 +71,11 @@ async function linkOrCopyFile (srcFile: string, destFile: string) {
  * This function could be optimized because we don't really need to try linking again
  * if linking failed once.
  */
-async function linkOrCopy (srcFile: string, destFile: string) {
+function linkOrCopy (srcFile: string, destFile: string) {
   try {
-    await fs.link(srcFile, destFile)
+    fs.linkSync(srcFile, destFile)
   } catch (err: any) { // eslint-disable-line
     if (err.code !== 'EXDEV') throw err
-    await fs.copyFile(srcFile, destFile)
+    fs.copyFileSync(srcFile, destFile)
   }
 }

fs/hard-link-dir/test/index.ts

@@ -3,7 +3,7 @@ import path from 'path'
 import { tempDir as createTempDir } from '@pnpm/prepare'
 import { hardLinkDir } from '@pnpm/fs.hard-link-dir'
 
-test('hardLinkDirectory()', async () => {
+test('hardLinkDirectory()', () => {
   const tempDir = createTempDir()
   const srcDir = path.join(tempDir, 'source')
   const dest1Dir = path.join(tempDir, 'dest1')
@@ -18,7 +18,7 @@ test('hardLinkDirectory()', async () => {
   fs.writeFileSync(path.join(srcDir, 'subdir/file.txt'), 'Hello World')
   fs.writeFileSync(path.join(srcDir, 'node_modules/file.txt'), 'Hello World')
 
-  await hardLinkDir(srcDir, [dest1Dir, dest2Dir])
+  hardLinkDir(srcDir, [dest1Dir, dest2Dir])
 
   // It should link the files from the root
   expect(fs.readFileSync(path.join(dest1Dir, 'file.txt'), 'utf8')).toBe('Hello World')
@@ -33,12 +33,12 @@ test('hardLinkDirectory()', async () => {
   expect(fs.existsSync(path.join(dest2Dir, 'node_modules/file.txt'))).toBe(false)
 })
 
-test("don't fail on missing source and dest directories", async () => {
+test("don't fail on missing source and dest directories", () => {
   const tempDir = createTempDir()
   const missingDirSrc = path.join(tempDir, 'missing_source')
   const missingDirDest = path.join(tempDir, 'missing_dest')
 
-  await hardLinkDir(missingDirSrc, [missingDirDest])
+  hardLinkDir(missingDirSrc, [missingDirDest])
 
   // It should create an empty dest dir if src does not exist
   expect(fs.existsSync(missingDirSrc)).toBe(false)

pkg-manager/core/test/install/lifecycleScripts.ts

@@ -10,6 +10,7 @@ import {
   type MutatedProject,
   mutateModules,
 } from '@pnpm/core'
+import { restartWorkerPool } from '@pnpm/worker'
 import rimraf from '@zkochan/rimraf'
 import isWindows from 'is-windows'
 import loadJsonFile from 'load-json-file'
@@ -656,6 +657,7 @@ test('ignore-dep-scripts', async () => {
 })
 
 test('run pre/postinstall scripts in a workspace that uses node-linker=hoisted', async () => {
+  await restartWorkerPool()
   const projects = preparePackages([
     {
       location: 'project-1',

pnpm-lock.yaml

@@ -1090,6 +1090,9 @@ importers:
       '@pnpm/types':
         specifier: workspace:*
         version: link:../../packages/types
+      '@pnpm/worker':
+        specifier: workspace:*
+        version: link:../../worker
       p-defer:
         specifier: ^3.0.0
         version: 3.0.0
@@ -1194,9 +1197,6 @@ importers:
       '@pnpm/error':
         specifier: workspace:*
         version: link:../../packages/error
-      '@pnpm/fs.hard-link-dir':
-        specifier: workspace:*
-        version: link:../../fs/hard-link-dir
       '@pnpm/get-context':
         specifier: workspace:*
         version: link:../../pkg-manager/get-context
@@ -1242,6 +1242,9 @@ importers:
       '@pnpm/types':
         specifier: workspace:*
         version: link:../../packages/types
+      '@pnpm/worker':
+        specifier: workspace:*
+        version: link:../../worker
       '@pnpm/workspace.find-packages':
         specifier: workspace:*
         version: link:../../workspace/find-packages
@@ -6036,6 +6039,9 @@ importers:
       '@pnpm/error':
         specifier: workspace:*
         version: link:../packages/error
+      '@pnpm/fs.hard-link-dir':
+        specifier: workspace:*
+        version: link:../fs/hard-link-dir
       '@pnpm/graceful-fs':
         specifier: workspace:*
         version: link:../fs/graceful-fs
@@ -6050,7 +6056,7 @@ importers:
         version: link:../fs/symlink-dependency
       '@rushstack/worker-pool':
         specifier: 0.4.9
-        version: 0.4.9
+        version: 0.4.9(@types/node@16.18.58)
       load-json-file:
         specifier: ^6.2.0
         version: 6.2.0
@@ -8345,13 +8351,15 @@ packages:
       '@reflink/reflink-win32-x64-msvc': 0.1.12
     dev: false
 
-  /@rushstack/worker-pool@0.4.9:
+  /@rushstack/worker-pool@0.4.9(@types/node@16.18.58):
     resolution: {integrity: sha512-ibAOeQCuz3g0c88GGawAPO2LVOTZE3uPh4DCEJILZS9SEv9opEUObsovC18EHPgeIuFy4HkoJT+t7l8LURZjIw==}
     peerDependencies:
       '@types/node': '*'
     peerDependenciesMeta:
       '@types/node':
         optional: true
+    dependencies:
+      '@types/node': 16.18.58
     dev: false
 
   /@sinclair/typebox@0.27.8:
@@ -8635,7 +8643,6 @@ packages:
 
   /@types/node@16.18.58:
     resolution: {integrity: sha512-YGncyA25/MaVtQkjWW9r0EFBukZ+JulsLcVZBlGUfIb96OBMjkoRWwQo5IEWJ8Fj06Go3GHw+bjYDitv6BaGsA==}
-    dev: true
 
   /@types/node@18.18.5:
     resolution: {integrity: sha512-4slmbtwV59ZxitY4ixUZdy1uRLf9eSIvBWPQxNjhHYWEtn0FryfKpyS2cvADYXTayWdKEIsJengncrVvkI4I6A==}
@@ -9069,7 +9076,7 @@ packages:
     resolution: {integrity: sha512-YmG+oTBCyrAoMIx5g2I9CfyurSpHyoan+9SCj7laaFKseOe3lFEyIVKvwRBQMmSt8uzh+eY5RWeQnoyyOs6AbA==}
     engines: {node: '>=14.15.0'}
     peerDependencies:
-      '@yarnpkg/fslib': 3.0.0-rc.45
+      '@yarnpkg/fslib': 3.0.0-rc.25
     dependencies:
       '@types/emscripten': 1.39.8
       '@yarnpkg/fslib': 3.0.0-rc.25

worker/package.json

@@ -35,6 +35,7 @@
     "@pnpm/cafs-types": "workspace:*",
     "@pnpm/create-cafs-store": "workspace:*",
     "@pnpm/error": "workspace:*",
+    "@pnpm/fs.hard-link-dir": "workspace:*",
     "@pnpm/graceful-fs": "workspace:*",
     "@pnpm/store.cafs": "workspace:*",
     "@pnpm/symlink-dependency": "workspace:*",

worker/src/index.ts

@@ -9,6 +9,7 @@ import {
   type AddDirToStoreMessage,
   type LinkPkgMessage,
   type SymlinkAllModulesMessage,
+  type HardLinkDirMessage,
 } from './types'
 
 let workerPool: WorkerPool | undefined
@@ -220,3 +221,25 @@ export async function symlinkAllModules (
     } as SymlinkAllModulesMessage)
   })
 }
+
+export async function hardLinkDir (src: string, destDirs: string[]): Promise<void> {
+  if (!workerPool) {
+    workerPool = createTarballWorkerPool()
+  }
+  const localWorker = await workerPool.checkoutWorkerAsync(true)
+  await new Promise<void>((resolve, reject) => {
+    localWorker.once('message', ({ status, error }: any) => { // eslint-disable-line @typescript-eslint/no-explicit-any
+      workerPool!.checkinWorker(localWorker)
+      if (status === 'error') {
+        reject(new PnpmError('HARDLINK_FAILED', error as string))
+        return
+      }
+      resolve()
+    })
+    localWorker.postMessage({
+      type: 'hardLinkDir',
+      src,
+      destDirs,
+    } as HardLinkDirMessage)
+  })
+}

worker/src/types.ts

@@ -54,3 +54,9 @@ export interface ReadPkgFromCafsMessage {
   readManifest: boolean
   verifyStoreIntegrity: boolean
 }
+
+export interface HardLinkDirMessage {
+  type: 'hardLinkDir'
+  src: string
+  destDirs: string[]
+}

worker/src/worker.ts

@@ -3,6 +3,7 @@ import fs from 'fs'
 import gfs from '@pnpm/graceful-fs'
 import * as crypto from 'crypto'
 import { createCafsStore } from '@pnpm/create-cafs-store'
+import { hardLinkDir } from '@pnpm/fs.hard-link-dir'
 import {
   checkPkgFilesIntegrity,
   createCafs,
@@ -22,6 +23,7 @@ import {
   type LinkPkgMessage,
   type SymlinkAllModulesMessage,
   type TarballExtractMessage,
+  type HardLinkDirMessage,
 } from './types'
 
 const INTEGRITY_REGEX: RegExp = /^([^-]+)-([A-Za-z0-9+/=]+)$/
@@ -33,7 +35,7 @@ const cafsStoreCache = new Map<string, ReturnType<typeof createCafsStore>>()
 const cafsLocker = new Map<string, number>()
 
 async function handleMessage (
-  message: TarballExtractMessage | LinkPkgMessage | AddDirToStoreMessage | ReadPkgFromCafsMessage | SymlinkAllModulesMessage | false
+  message: TarballExtractMessage | LinkPkgMessage | AddDirToStoreMessage | ReadPkgFromCafsMessage | SymlinkAllModulesMessage | HardLinkDirMessage | false
 ): Promise<void> {
   if (message === false) {
     parentPort!.off('message', handleMessage)
@@ -94,6 +96,11 @@ async function handleMessage (
       parentPort!.postMessage(symlinkAllModules(message))
       break
     }
+    case 'hardLinkDir': {
+      hardLinkDir(message.src, message.destDirs)
+      parentPort!.postMessage({ status: 'success' })
+      break
+    }
     }
   } catch (e: any) { // eslint-disable-line
     parentPort!.postMessage({ status: 'error', error: e.toString() })

worker/tsconfig.json

@@ -12,6 +12,9 @@
     {
       "path": "../fs/graceful-fs"
     },
+    {
+      "path": "../fs/hard-link-dir"
+    },
     {
       "path": "../fs/symlink-dependency"
     },