fix: use graceful-fs for read/write files

.changeset/fuzzy-penguins-greet.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/graceful-fs": major
+---
+
+Project created.

packages/cafs/package.json

@@ -15,12 +15,12 @@
   "license": "MIT",
   "dependencies": {
     "@pnpm/fetcher-base": "workspace:11.0.0",
+    "@pnpm/graceful-fs": "workspace:0.0.0",
     "@pnpm/store-controller-types": "workspace:11.0.2",
     "@zkochan/rimraf": "^2.1.1",
     "concat-stream": "^2.0.0",
     "decompress-maybe": "^1.0.0",
     "get-stream": "^6.0.0",
-    "graceful-fs": "^4.2.6",
     "p-limit": "^3.1.0",
     "path-temp": "^2.0.0",
     "rename-overwrite": "^4.0.0",

packages/cafs/src/addFilesFromDir.ts

@@ -1,10 +1,11 @@
-import { createReadStream, promises as fs } from 'fs'
+import { promises as fs } from 'fs'
 import path from 'path'
 import {
   DeferredManifestPromise,
   FilesIndex,
   FileWriteResult,
 } from '@pnpm/fetcher-base'
+import gfs from '@pnpm/graceful-fs'
 import pLimit from 'p-limit'
 import { parseJsonBuffer } from './parseJson'
 
@@ -48,15 +49,15 @@ async function _retrieveFileIntegrities (
         const relativePath = path.relative(rootDir, fullPath)
         const writeResult = limit(async () => {
           if ((deferredManifest != null) && rootDir === currDir && file === 'package.json') {
-            const buffer = await fs.readFile(fullPath)
+            const buffer = await gfs.readFile(fullPath)
             parseJsonBuffer(buffer, deferredManifest)
             return cafs.addBuffer(buffer, stat.mode)
           }
           if (stat.size < MAX_BULK_SIZE) {
-            const buffer = await fs.readFile(fullPath)
+            const buffer = await gfs.readFile(fullPath)
             return cafs.addBuffer(buffer, stat.mode)
           }
-          return cafs.addStream(createReadStream(fullPath), stat.mode)
+          return cafs.addStream(gfs.createReadStream(fullPath), stat.mode)
         })
         index[relativePath] = {
           mode: stat.mode,

packages/cafs/src/checkFilesIntegrity.ts

@@ -1,16 +1,14 @@
 import { promises as fs } from 'fs'
-import gracefulFS from 'graceful-fs'
 import { DeferredManifestPromise, PackageFileInfo } from '@pnpm/fetcher-base'
+import gfs from '@pnpm/graceful-fs'
 import rimraf from '@zkochan/rimraf'
 import pLimit from 'p-limit'
 import ssri from 'ssri'
-import { promisify } from 'util'
 import { getFilePathByModeInCafs } from './getFilePathInCafs'
 import { parseJsonBuffer } from './parseJson'
 
 const limit = pLimit(20)
 const MAX_BULK_SIZE = 1 * 1024 * 1024 // 1MB
-const readFile = promisify(gracefulFS.readFile)
 
 export interface PackageFilesIndex {
   // name and version are nullable for backward compatibility
@@ -72,7 +70,7 @@ async function verifyFile (
     return verifyFileIntegrity(filename, fstat, deferredManifest)
   }
   if (deferredManifest != null) {
-    parseJsonBuffer(await readFile(filename), deferredManifest)
+    parseJsonBuffer(await gfs.readFile(filename), deferredManifest)
   }
   // If a file was not edited, we are skipping integrity check.
   // We assume that nobody will manually remove a file in the store and create a new one.
@@ -86,13 +84,13 @@ export async function verifyFileIntegrity (
 ) {
   try {
     if (expectedFile.size > MAX_BULK_SIZE && (deferredManifest == null)) {
-      const ok = Boolean(await ssri.checkStream(gracefulFS.createReadStream(filename), expectedFile.integrity))
+      const ok = Boolean(await ssri.checkStream(gfs.createReadStream(filename), expectedFile.integrity))
       if (!ok) {
         await rimraf(filename)
       }
       return ok
     }
-    const data = await readFile(filename)
+    const data = await gfs.readFile(filename)
     const ok = Boolean(ssri.checkData(data, expectedFile.integrity))
     if (!ok) {
       await rimraf(filename)

packages/cafs/src/writeFile.ts

@@ -1,5 +1,6 @@
 import { promises as fs } from 'fs'
 import path from 'path'
+import gfs from '@pnpm/graceful-fs'
 
 const dirs = new Set()
 
@@ -9,7 +10,7 @@ export default async function (
   mode?: number
 ) {
   await makeDirForFile(fileDest)
-  await fs.writeFile(fileDest, buffer, { mode })
+  await gfs.writeFile(fileDest, buffer, { mode })
 }
 
 async function makeDirForFile (fileDest: string) {

packages/cafs/tsconfig.json

@@ -12,6 +12,9 @@
     {
       "path": "../fetcher-base"
     },
+    {
+      "path": "../graceful-fs"
+    },
     {
       "path": "../store-controller-types"
     }

packages/graceful-fs/README.md

@@ -0,0 +1,15 @@
+# @pnpm/graceful-fs
+
+> Promisified graceful-fs
+
+[![npm version](https://img.shields.io/npm/v/@pnpm/graceful-fs.svg)](https://www.npmjs.com/package/@pnpm/graceful-fs)
+
+## Installation
+
+```sh
+<pnpm|npm|yarn> add @pnpm/graceful-fs
+```
+
+## License
+
+MIT

packages/graceful-fs/jest.config.js

@@ -0,0 +1,3 @@
+const config = require('../../jest.config.js');
+
+module.exports = Object.assign({}, config, {});

packages/graceful-fs/package.json

@@ -0,0 +1,33 @@
+{
+  "name": "@pnpm/graceful-fs",
+  "version": "0.0.0",
+  "description": "Promisified graceful-fs",
+  "main": "lib/index.js",
+  "typings": "lib/index.d.ts",
+  "files": [
+    "lib",
+    "!*.map"
+  ],
+  "engines": {
+    "node": ">=12.17"
+  },
+  "scripts": {
+    "lint": "eslint -c ../../eslint.json src/**/*.ts",
+    "test": "pnpm run compile",
+    "prepublishOnly": "pnpm run compile",
+    "compile": "rimraf lib tsconfig.tsbuildinfo && tsc --build && pnpm run lint -- --fix"
+  },
+  "repository": "https://github.com/pnpm/pnpm/blob/master/packages/graceful-fs",
+  "keywords": [
+    "pnpm"
+  ],
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/pnpm/pnpm/issues"
+  },
+  "homepage": "https://github.com/pnpm/pnpm/blob/master/packages/graceful-fs#readme",
+  "dependencies": {
+    "graceful-fs": "^4.2.6"
+  },
+  "funding": "https://opencollective.com/pnpm"
+}

packages/graceful-fs/src/index.ts

@@ -0,0 +1,8 @@
+import gfs from 'graceful-fs'
+import { promisify } from 'util'
+
+export default {
+  createReadStream: gfs.createReadStream,
+  readFile: promisify(gfs.readFile),
+  writeFile: promisify(gfs.writeFile),
+}

packages/graceful-fs/tsconfig.json

@@ -0,0 +1,12 @@
+{
+  "extends": "@pnpm/tsconfig",
+  "compilerOptions": {
+    "outDir": "lib",
+    "rootDir": "src"
+  },
+  "include": [
+    "src/**/*.ts",
+    "../../typings/**/*.d.ts"
+  ],
+  "references": []
+}

packages/graceful-fs/tsconfig.lint.json

@@ -0,0 +1,8 @@
+{
+  "extends": "./tsconfig.json",
+  "include": [
+    "src/**/*.ts",
+    "test/**/*.ts",
+    "../../typings/**/*.d.ts"
+  ]
+}

packages/local-resolver/package.json

@@ -32,6 +32,7 @@
   "homepage": "https://github.com/pnpm/pnpm/blob/master/packages/local-resolver#readme",
   "dependencies": {
     "@pnpm/error": "workspace:2.0.0",
+    "@pnpm/graceful-fs": "workspace:0.0.0",
     "@pnpm/read-project-manifest": "workspace:2.0.1",
     "@pnpm/resolver-base": "workspace:8.0.1",
     "@pnpm/types": "workspace:7.1.0",

packages/local-resolver/src/index.ts

@@ -1,5 +1,5 @@
-import fs from 'fs'
 import PnpmError from '@pnpm/error'
+import gfs from '@pnpm/graceful-fs'
 import { readProjectManifestOnly } from '@pnpm/read-project-manifest'
 import {
   DirectoryResolution,
@@ -77,5 +77,5 @@ export default async function resolveLocal (
 }
 
 async function getFileIntegrity (filename: string) {
-  return (await ssri.fromStream(fs.createReadStream(filename))).toString()
+  return (await ssri.fromStream(gfs.createReadStream(filename))).toString()
 }

packages/local-resolver/tsconfig.json

@@ -12,6 +12,9 @@
     {
       "path": "../error"
     },
+    {
+      "path": "../graceful-fs"
+    },
     {
       "path": "../read-project-manifest"
     },

packages/npm-resolver/package.json

@@ -36,6 +36,7 @@
     "@pnpm/core-loggers": "workspace:6.0.1",
     "@pnpm/error": "workspace:2.0.0",
     "@pnpm/fetching-types": "workspace:2.1.0",
+    "@pnpm/graceful-fs": "workspace:0.0.0",
     "@pnpm/resolve-workspace-range": "workspace:2.1.0",
     "@pnpm/resolver-base": "workspace:8.0.1",
     "@pnpm/types": "workspace:7.1.0",

packages/npm-resolver/src/pickPackage.ts

@@ -2,6 +2,7 @@ import { promises as fs } from 'fs'
 import path from 'path'
 import PnpmError from '@pnpm/error'
 import logger from '@pnpm/logger'
+import gfs from '@pnpm/graceful-fs'
 import { VersionSelectors } from '@pnpm/resolver-base'
 import { PackageManifest } from '@pnpm/types'
 import getRegistryName from 'encode-registry'
@@ -162,7 +163,7 @@ async function saveMeta (pkgMirror: string, meta: PackageMeta): Promise<void> {
     createdDirs.add(dir)
   }
   const temp = pathTemp(dir)
-  await fs.writeFile(temp, JSON.stringify(meta))
+  await gfs.writeFile(temp, JSON.stringify(meta))
   await renameOverwrite(temp, pkgMirror)
 }
 

packages/npm-resolver/tsconfig.json

@@ -21,6 +21,9 @@
     {
       "path": "../fetching-types"
     },
+    {
+      "path": "../graceful-fs"
+    },
     {
       "path": "../resolve-workspace-range"
     },

packages/package-requester/package.json

@@ -38,6 +38,7 @@
     "@pnpm/core-loggers": "workspace:6.0.1",
     "@pnpm/error": "workspace:2.0.0",
     "@pnpm/fetcher-base": "workspace:11.0.0",
+    "@pnpm/graceful-fs": "workspace:0.0.0",
     "@pnpm/read-package-json": "workspace:5.0.1",
     "@pnpm/resolver-base": "workspace:8.0.1",
     "@pnpm/store-controller-types": "workspace:11.0.2",

packages/package-requester/src/packageRequester.ts

@@ -18,6 +18,7 @@ import {
   FetchResult,
   PackageFilesResponse,
 } from '@pnpm/fetcher-base'
+import gfs from '@pnpm/graceful-fs'
 import logger from '@pnpm/logger'
 import readPackage from '@pnpm/read-package-json'
 import {
@@ -493,7 +494,7 @@ Actual package in the store by the given integrity: ${pkgFilesIndex.name}@${pkgF
 
       if (isLocalTarballDep && opts.pkg.resolution['integrity']) { // eslint-disable-line @typescript-eslint/dot-notation
         await fs.mkdir(target, { recursive: true })
-        await fs.writeFile(path.join(target, TARBALL_INTEGRITY_FILENAME), opts.pkg.resolution['integrity'], 'utf8') // eslint-disable-line @typescript-eslint/dot-notation
+        await gfs.writeFile(path.join(target, TARBALL_INTEGRITY_FILENAME), opts.pkg.resolution['integrity'], 'utf8') // eslint-disable-line @typescript-eslint/dot-notation
       }
 
       files.resolve({
@@ -517,7 +518,7 @@ async function writeJsonFile (filePath: string, data: Object) {
   // So by using cafs API, we'll improve performance.
   await fs.mkdir(targetDir, { recursive: true })
   const temp = pathTemp(targetDir)
-  await fs.writeFile(temp, JSON.stringify(data))
+  await gfs.writeFile(temp, JSON.stringify(data))
   await renameOverwrite(temp, filePath)
 }
 
@@ -536,7 +537,7 @@ async function tarballIsUpToDate (
 ) {
   let currentIntegrity!: string
   try {
-    currentIntegrity = (await fs.readFile(path.join(pkgInStoreLocation, TARBALL_INTEGRITY_FILENAME), 'utf8'))
+    currentIntegrity = (await gfs.readFile(path.join(pkgInStoreLocation, TARBALL_INTEGRITY_FILENAME), 'utf8'))
   } catch (err) {
     return false
   }

packages/package-requester/tsconfig.json

@@ -27,6 +27,9 @@
     {
       "path": "../fetcher-base"
     },
+    {
+      "path": "../graceful-fs"
+    },
     {
       "path": "../read-package-json"
     },

packages/read-project-manifest/package.json

@@ -29,11 +29,11 @@
   "homepage": "https://github.com/pnpm/pnpm/blob/master/packages/read-project-manifest#readme",
   "dependencies": {
     "@pnpm/error": "workspace:2.0.0",
+    "@pnpm/graceful-fs": "workspace:0.0.0",
     "@pnpm/types": "workspace:7.1.0",
     "@pnpm/write-project-manifest": "workspace:2.0.1",
     "detect-indent": "^6.0.0",
     "fast-deep-equal": "^3.1.3",
-    "graceful-fs": "^4.2.6",
     "is-windows": "^1.0.2",
     "json5": "^2.1.3",
     "parse-json": "^5.1.0",

packages/read-project-manifest/src/readFile.ts

@@ -1,13 +1,9 @@
-import { promisify } from 'util'
-import gracefulFs from 'graceful-fs'
+import gfs from '@pnpm/graceful-fs'
 import { ProjectManifest } from '@pnpm/types'
 import JSON5 from 'json5'
 import parseJson from 'parse-json'
 import stripBom from 'strip-bom'
 
-// We use graceful-fs to avoid EMFILE errors.
-const readFile = promisify(gracefulFs.readFile)
-
 export async function readJson5File (filePath: string) {
   const text = await readFileWithoutBom(filePath)
   try {
@@ -36,5 +32,5 @@ export async function readJsonFile (filePath: string) {
 }
 
 async function readFileWithoutBom (path: string) {
-  return stripBom(await readFile(path, 'utf8'))
+  return stripBom(await gfs.readFile(path, 'utf8'))
 }

packages/read-project-manifest/tsconfig.json

@@ -12,6 +12,9 @@
     {
       "path": "../error"
     },
+    {
+      "path": "../graceful-fs"
+    },
     {
       "path": "../types"
     },

packages/tarball-fetcher/package.json

@@ -37,6 +37,7 @@
     "@pnpm/error": "workspace:2.0.0",
     "@pnpm/fetcher-base": "workspace:11.0.0",
     "@pnpm/fetching-types": "workspace:2.1.0",
+    "@pnpm/graceful-fs": "workspace:0.0.0",
     "@pnpm/prepare-package": "workspace:1.0.0",
     "@zkochan/retry": "^0.2.0",
     "ramda": "^0.27.1",

packages/tarball-fetcher/src/index.ts

@@ -1,5 +1,4 @@
 import path from 'path'
-import fs from 'fs'
 import PnpmError from '@pnpm/error'
 import {
   Cafs,
@@ -13,6 +12,7 @@ import {
   GetCredentials,
   RetryTimeoutOptions,
 } from '@pnpm/fetching-types'
+import gfs from '@pnpm/graceful-fs'
 import ssri from 'ssri'
 import createDownloader, {
   DownloadFunction,
@@ -101,7 +101,7 @@ async function fetchFromLocalTarball (
   }
 ): Promise<FetchResult> {
   try {
-    const tarballStream = fs.createReadStream(tarball)
+    const tarballStream = gfs.createReadStream(tarball)
     const [fetchResult] = (
       await Promise.all([
         cafs.addFilesFromTarball(tarballStream, opts.manifest),

packages/tarball-fetcher/tsconfig.json

@@ -24,6 +24,9 @@
     {
       "path": "../fetching-types"
     },
+    {
+      "path": "../graceful-fs"
+    },
     {
       "path": "../prepare-package"
     }

pnpm-lock.yaml

@@ -131,6 +131,7 @@ importers:
     specifiers:
       '@pnpm/cafs': 'link:'
       '@pnpm/fetcher-base': workspace:11.0.0
+      '@pnpm/graceful-fs': workspace:0.0.0
       '@pnpm/store-controller-types': workspace:11.0.2
       '@types/concat-stream': ^1.6.0
       '@types/node': ^14.14.33
@@ -140,7 +141,6 @@ importers:
       concat-stream: ^2.0.0
       decompress-maybe: ^1.0.0
       get-stream: ^6.0.0
-      graceful-fs: ^4.2.6
       p-limit: ^3.1.0
       path-temp: ^2.0.0
       rename-overwrite: ^4.0.0
@@ -150,12 +150,12 @@ importers:
       tempy: ^1.0.0
     dependencies:
       '@pnpm/fetcher-base': link:../fetcher-base
+      '@pnpm/graceful-fs': link:../graceful-fs
       '@pnpm/store-controller-types': link:../store-controller-types
       '@zkochan/rimraf': 2.1.1
       concat-stream: 2.0.0
       decompress-maybe: 1.0.0
       get-stream: 6.0.1
-      graceful-fs: 4.2.6
       p-limit: 3.1.0
       path-temp: 2.0.0
       rename-overwrite: 4.0.0
@@ -730,6 +730,15 @@ importers:
       '@pnpm/global-bin-dir': 'link:'
       is-windows: 1.0.2
 
+  packages/graceful-fs:
+    specifiers:
+      '@pnpm/graceful-fs': 'link:'
+      graceful-fs: ^4.2.6
+    dependencies:
+      graceful-fs: 4.2.6
+    devDependencies:
+      '@pnpm/graceful-fs': 'link:'
+
   packages/headless:
     specifiers:
       '@pnpm/assert-project': workspace:*
@@ -976,6 +985,7 @@ importers:
   packages/local-resolver:
     specifiers:
       '@pnpm/error': workspace:2.0.0
+      '@pnpm/graceful-fs': workspace:0.0.0
       '@pnpm/local-resolver': 'link:'
       '@pnpm/read-project-manifest': workspace:2.0.1
       '@pnpm/resolver-base': workspace:8.0.1
@@ -986,6 +996,7 @@ importers:
       ssri: ^8.0.1
     dependencies:
       '@pnpm/error': link:../error
+      '@pnpm/graceful-fs': link:../graceful-fs
       '@pnpm/read-project-manifest': link:../read-project-manifest
       '@pnpm/resolver-base': link:../resolver-base
       '@pnpm/types': link:../types
@@ -1334,6 +1345,7 @@ importers:
       '@pnpm/error': workspace:2.0.0
       '@pnpm/fetch': workspace:3.1.0
       '@pnpm/fetching-types': workspace:2.1.0
+      '@pnpm/graceful-fs': workspace:0.0.0
       '@pnpm/logger': ^4.0.0
       '@pnpm/npm-resolver': 'link:'
       '@pnpm/resolve-workspace-range': workspace:2.1.0
@@ -1363,6 +1375,7 @@ importers:
       '@pnpm/core-loggers': link:../core-loggers
       '@pnpm/error': link:../error
       '@pnpm/fetching-types': link:../fetching-types
+      '@pnpm/graceful-fs': link:../graceful-fs
       '@pnpm/resolve-workspace-range': link:../resolve-workspace-range
       '@pnpm/resolver-base': link:../resolver-base
       '@pnpm/types': link:../types
@@ -1475,6 +1488,7 @@ importers:
       '@pnpm/core-loggers': workspace:6.0.1
       '@pnpm/error': workspace:2.0.0
       '@pnpm/fetcher-base': workspace:11.0.0
+      '@pnpm/graceful-fs': workspace:0.0.0
       '@pnpm/logger': ^4.0.0
       '@pnpm/package-requester': 'link:'
       '@pnpm/package-store': workspace:12.0.5
@@ -1506,6 +1520,7 @@ importers:
       '@pnpm/core-loggers': link:../core-loggers
       '@pnpm/error': link:../error
       '@pnpm/fetcher-base': link:../fetcher-base
+      '@pnpm/graceful-fs': link:../graceful-fs
       '@pnpm/read-package-json': link:../read-package-json
       '@pnpm/resolver-base': link:../resolver-base
       '@pnpm/store-controller-types': link:../store-controller-types
@@ -2546,6 +2561,7 @@ importers:
   packages/read-project-manifest:
     specifiers:
       '@pnpm/error': workspace:2.0.0
+      '@pnpm/graceful-fs': workspace:0.0.0
       '@pnpm/read-project-manifest': 'link:'
       '@pnpm/types': workspace:7.1.0
       '@pnpm/write-project-manifest': workspace:2.0.1
@@ -2554,7 +2570,6 @@ importers:
       '@types/parse-json': ^4.0.0
       detect-indent: ^6.0.0
       fast-deep-equal: ^3.1.3
-      graceful-fs: ^4.2.6
       is-windows: ^1.0.2
       json5: ^2.1.3
       parse-json: ^5.1.0
@@ -2564,11 +2579,11 @@ importers:
       tempy: ^1.0.0
     dependencies:
       '@pnpm/error': link:../error
+      '@pnpm/graceful-fs': link:../graceful-fs
       '@pnpm/types': link:../types
       '@pnpm/write-project-manifest': link:../write-project-manifest
       detect-indent: 6.0.0
       fast-deep-equal: 3.1.3
-      graceful-fs: 4.2.6
       is-windows: 1.0.2
       json5: 2.2.0
       parse-json: 5.2.0
@@ -3009,6 +3024,7 @@ importers:
       '@pnpm/fetch': workspace:3.1.0
       '@pnpm/fetcher-base': workspace:11.0.0
       '@pnpm/fetching-types': workspace:2.1.0
+      '@pnpm/graceful-fs': workspace:0.0.0
       '@pnpm/logger': ^4.0.0
       '@pnpm/package-store': workspace:12.0.5
       '@pnpm/prepare-package': workspace:1.0.0
@@ -3026,6 +3042,7 @@ importers:
       '@pnpm/error': link:../error
       '@pnpm/fetcher-base': link:../fetcher-base
       '@pnpm/fetching-types': link:../fetching-types
+      '@pnpm/graceful-fs': link:../graceful-fs
       '@pnpm/prepare-package': link:../prepare-package
       '@zkochan/retry': 0.2.0
       ramda: 0.27.1