fix: retry filesystem operations on EAGAIN (#9959)

* fix: retry filesystem operations on EAGAIN

filesystem operations can raise EAGAIN to tell the application to try
again later. This is especially often the case under ZFS.

fix: move wrapped functions to graceful-fs directly

* fix: retry filesystem operations on EAGAIN

* fix: retry filesystem operations on EAGAIN

* fix: indexed-pkg-importer

* test: fix

* docs: add changeset

---------

Co-authored-by: Zoltan Kochan <z@kochan.io>

.changeset/crazy-socks-nail.md

@@ -0,0 +1,9 @@
+---
+"@pnpm/fs.indexed-pkg-importer": patch
+"@pnpm/fs.hard-link-dir": patch
+"@pnpm/graceful-fs": patch
+"@pnpm/store.cafs": patch
+"pnpm": patch
+---
+
+Retry filesystem operations on EAGAIN errors [#9959](https://github.com/pnpm/pnpm/pull/9959).

cspell.json

@@ -52,6 +52,7 @@
     "dislink",
     "dpkg",
     "duplexify",
+    "eagain",
     "ebadplatform",
     "ebusy",
     "ehrkoext",

fs/graceful-fs/src/index.ts

@@ -1,12 +1,14 @@
-import { promisify } from 'util'
+import util, { promisify } from 'util'
 import gfs from 'graceful-fs'
 
 export default { // eslint-disable-line
   copyFile: promisify(gfs.copyFile),
-  copyFileSync: gfs.copyFileSync,
+  copyFileSync: withEagainRetry(gfs.copyFileSync),
   createReadStream: gfs.createReadStream,
   link: promisify(gfs.link),
-  linkSync: gfs.linkSync,
+  linkSync: withEagainRetry(gfs.linkSync),
+  mkdirSync: withEagainRetry(gfs.mkdirSync),
+  renameSync: withEagainRetry(gfs.renameSync),
   readFile: promisify(gfs.readFile),
   readFileSync: gfs.readFileSync,
   readdirSync: gfs.readdirSync,
@@ -14,5 +16,29 @@ export default { // eslint-disable-line
   statSync: gfs.statSync,
   unlinkSync: gfs.unlinkSync,
   writeFile: promisify(gfs.writeFile),
-  writeFileSync: gfs.writeFileSync,
+  writeFileSync: withEagainRetry(gfs.writeFileSync),
+}
+
+function withEagainRetry<T extends unknown[], R> (
+  fn: (...args: T) => R,
+  maxRetries: number = 15
+): (...args: T) => R {
+  return (...args: T): R => {
+    let attempts = 0
+    while (attempts <= maxRetries) {
+      try {
+        return fn(...args)
+      } catch (err: unknown) {
+        if (util.types.isNativeError(err) && 'code' in err && err.code === 'EAGAIN' && attempts < maxRetries) {
+          attempts++
+          // Exponential backoff: wait 2^attempts milliseconds, max 300ms
+          const delay = Math.min(Math.pow(2, attempts), 300)
+          Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, delay)
+          continue
+        }
+        throw err
+      }
+    }
+    throw new Error('Unreachable')
+  }
 }

fs/hard-link-dir/package.json

@@ -32,6 +32,9 @@
     "prepublishOnly": "pnpm run compile",
     "compile": "tsc --build && pnpm run lint --fix"
   },
+  "dependencies": {
+    "@pnpm/graceful-fs": "workspace:*"
+  },
   "peerDependencies": {
     "@pnpm/logger": "catalog:"
   },

fs/hard-link-dir/src/index.ts

@@ -3,6 +3,7 @@ import path from 'path'
 import util from 'util'
 import fs from 'fs'
 import { globalWarn } from '@pnpm/logger'
+import gfs from '@pnpm/graceful-fs'
 
 export function hardLinkDir (src: string, destDirs: string[]): void {
   if (destDirs.length === 0) return
@@ -19,7 +20,7 @@ function _hardLinkDir (src: string, destDirs: string[], isRoot?: boolean) {
     if (!isRoot || !((util.types.isNativeError(err) && 'code' in err && err.code === 'ENOENT'))) throw err
     globalWarn(`Source directory not found when creating hardLinks for: ${src}. Creating destinations as empty: ${destDirs.join(', ')}`)
     for (const dir of destDirs) {
-      fs.mkdirSync(dir, { recursive: true })
+      gfs.mkdirSync(dir, { recursive: true })
     }
     return
   }
@@ -30,7 +31,7 @@ function _hardLinkDir (src: string, destDirs: string[], isRoot?: boolean) {
       const destSubdirs = destDirs.map((destDir) => {
         const destSubdir = path.join(destDir, file)
         try {
-          fs.mkdirSync(destSubdir, { recursive: true })
+          gfs.mkdirSync(destSubdir, { recursive: true })
         } catch (err: unknown) {
           if (!(util.types.isNativeError(err) && 'code' in err && err.code === 'EEXIST')) throw err
         }
@@ -60,7 +61,7 @@ function linkOrCopyFile (srcFile: string, destFile: string): void {
   } catch (err: unknown) {
     assert(util.types.isNativeError(err))
     if ('code' in err && err.code === 'ENOENT') {
-      fs.mkdirSync(path.dirname(destFile), { recursive: true })
+      gfs.mkdirSync(path.dirname(destFile), { recursive: true })
       linkOrCopy(srcFile, destFile)
       return
     }
@@ -76,9 +77,9 @@ function linkOrCopyFile (srcFile: string, destFile: string): void {
  */
 function linkOrCopy (srcFile: string, destFile: string): void {
   try {
-    fs.linkSync(srcFile, destFile)
+    gfs.linkSync(srcFile, destFile)
   } catch (err: unknown) {
     if (!(util.types.isNativeError(err) && 'code' in err && err.code === 'EXDEV')) throw err
-    fs.copyFileSync(srcFile, destFile)
+    gfs.copyFileSync(srcFile, destFile)
   }
 }

fs/hard-link-dir/tsconfig.json

@@ -14,6 +14,9 @@
     },
     {
       "path": "../../packages/logger"
+    },
+    {
+      "path": "../graceful-fs"
     }
   ]
 }

fs/indexed-pkg-importer/src/importIndexedDir.ts

@@ -8,6 +8,7 @@ import { sync as makeEmptyDir } from 'make-empty-dir'
 import sanitizeFilename from 'sanitize-filename'
 import { fastPathTemp as pathTemp } from 'path-temp'
 import renameOverwrite from 'rename-overwrite'
+import gfs from '@pnpm/graceful-fs'
 
 const filenameConflictsLogger = logger('_filename-conflicts')
 
@@ -143,7 +144,7 @@ function moveOrMergeModulesDirs (src: string, dest: string): void {
 
 function renameEvenAcrossDevices (src: string, dest: string): void {
   try {
-    fs.renameSync(src, dest)
+    gfs.renameSync(src, dest)
   } catch (err: unknown) {
     if (!(util.types.isNativeError(err) && 'code' in err && err.code === 'EXDEV')) throw err
     copySync(src, dest)

fs/indexed-pkg-importer/test/createImportPackage.test.ts

@@ -8,18 +8,21 @@ import { jest } from '@jest/globals'
 const testOnLinuxOnly = (process.platform === 'darwin' || process.platform === 'win32') ? test.skip : test
 
 jest.mock('@pnpm/graceful-fs', () => {
-  const { access, promises } = jest.requireActual<typeof fs>('fs')
+  const { access } = jest.requireActual<typeof fs>('fs')
   const fsMock = {
-    mkdirSync: promises.mkdir,
-    readdirSync: promises.readdir,
     access,
     copyFileSync: jest.fn(),
+    readdirSync: jest.fn(),
     linkSync: jest.fn(),
+    mkdirSync: jest.fn(),
+    renameSync: jest.fn(),
+    writeFileSync: jest.fn(),
     statSync: jest.fn(),
   }
   return {
     __esModule: true,
     default: fsMock,
+    ...fsMock,
   }
 })
 jest.mock('path-temp', () => ({ fastPathTemp: (file: string) => `${file}_tmp` }))
@@ -36,6 +39,8 @@ jest.mock('@pnpm/logger', () => ({
 beforeEach(() => {
   jest.mocked(gfs.copyFileSync).mockClear()
   jest.mocked(gfs.linkSync).mockClear()
+  jest.mocked(gfs.mkdirSync).mockClear()
+  jest.mocked(gfs.renameSync).mockClear()
   jest.mocked(globalInfo).mockReset()
 })
 

pnpm-lock.yaml

@@ -3291,6 +3291,10 @@ importers:
         version: 4.1.9
 
   fs/hard-link-dir:
+    dependencies:
+      '@pnpm/graceful-fs':
+        specifier: workspace:*
+        version: link:../graceful-fs
     devDependencies:
       '@pnpm/fs.hard-link-dir':
         specifier: workspace:*

store/cafs/src/writeFile.ts

@@ -1,5 +1,5 @@
-import fs from 'fs'
 import path from 'path'
+import fs from '@pnpm/graceful-fs'
 
 const dirs = new Set()