From 9b344c89828d4147180a01c20dbca810f93832e9 Mon Sep 17 00:00:00 2001 From: Zoltan Kochan Date: Thu, 6 Nov 2025 01:01:06 +0100 Subject: [PATCH] perf: use v8 serialize/deserialize instead of JSON (#9971) close #9965 --- .changeset/ripe-friends-dream.md | 19 +++++ .changeset/short-cougars-arrive.md | 5 ++ .changeset/young-points-dress.md | 9 +++ cache/api/package.json | 1 + cache/api/src/cacheList.ts | 2 +- cache/api/src/cacheView.ts | 6 +- cache/api/tsconfig.json | 3 + cache/commands/test/cacheDelete.cmd.test.ts | 4 +- cache/commands/test/cacheList.cmd.test.ts | 12 +-- exec/plugin-commands-rebuild/package.json | 1 + exec/plugin-commands-rebuild/test/index.ts | 25 ++++-- exec/plugin-commands-rebuild/tsconfig.json | 3 + fs/v8-file/README.md | 17 ++++ fs/v8-file/package.json | 43 ++++++++++ fs/v8-file/src/index.ts | 31 +++++++ fs/v8-file/tsconfig.json | 12 +++ fs/v8-file/tsconfig.lint.json | 8 ++ modules-mounter/daemon/package.json | 2 +- .../daemon/src/createFuseHandlers.ts | 4 +- modules-mounter/daemon/tsconfig.json | 3 + pkg-manager/core/package.json | 1 + pkg-manager/core/test/install/lockfileOnly.ts | 8 +- pkg-manager/core/test/install/patch.ts | 18 ++--- pkg-manager/core/test/install/sideEffects.ts | 14 ++-- pkg-manager/core/tsconfig.json | 3 + pkg-manager/headless/package.json | 1 + pkg-manager/headless/test/index.ts | 11 +-- pkg-manager/headless/tsconfig.json | 3 + pkg-manager/package-requester/package.json | 2 +- pkg-manager/package-requester/test/index.ts | 4 +- pkg-manager/package-requester/tsconfig.json | 3 + pnpm-lock.yaml | 81 ++++++++++++------- pnpm/package.json | 1 + pnpm/test/install/misc.ts | 14 ++-- pnpm/tsconfig.json | 3 + resolving/npm-resolver/package.json | 3 +- resolving/npm-resolver/src/pickPackage.ts | 13 +-- resolving/npm-resolver/test/index.ts | 14 ++-- .../npm-resolver/test/resolveJsr.test.ts | 4 +- resolving/npm-resolver/test/utils/index.ts | 4 +- resolving/npm-resolver/tsconfig.json | 3 + reviewing/license-scanner/package.json | 2 +- reviewing/license-scanner/src/getPkgInfo.ts | 4 +- .../license-scanner/test/getPkgInfo.spec.ts | 2 +- reviewing/license-scanner/tsconfig.json | 3 + store/cafs/src/getFilePathInCafs.ts | 4 +- store/package-store/package.json | 2 +- .../src/storeController/prune.ts | 8 +- store/package-store/tsconfig.json | 3 + .../package.json | 2 +- .../src/catIndex.ts | 4 +- .../src/findHash.ts | 9 ++- .../test/findHash.ts | 8 +- .../tsconfig.json | 3 + store/plugin-commands-store/package.json | 2 +- .../src/storeStatus/index.ts | 4 +- store/plugin-commands-store/tsconfig.json | 3 + store/server/package.json | 3 +- store/server/test/index.ts | 12 +-- store/server/tsconfig.json | 6 ++ worker/package.json | 2 +- worker/src/start.ts | 19 ++--- worker/tsconfig.json | 3 + 63 files changed, 384 insertions(+), 142 deletions(-) create mode 100644 .changeset/ripe-friends-dream.md create mode 100644 .changeset/short-cougars-arrive.md create mode 100644 .changeset/young-points-dress.md create mode 100644 fs/v8-file/README.md create mode 100644 fs/v8-file/package.json create mode 100644 fs/v8-file/src/index.ts create mode 100644 fs/v8-file/tsconfig.json create mode 100644 fs/v8-file/tsconfig.lint.json diff --git a/.changeset/ripe-friends-dream.md b/.changeset/ripe-friends-dream.md new file mode 100644 index 0000000000..c7d8ae68a8 --- /dev/null +++ b/.changeset/ripe-friends-dream.md @@ -0,0 +1,19 @@ +--- +"@pnpm/plugin-commands-store-inspecting": major +"@pnpm/package-requester": major +"@pnpm/plugin-commands-rebuild": major +"@pnpm/plugin-commands-store": major +"@pnpm/license-scanner": major +"@pnpm/mount-modules": major +"@pnpm/npm-resolver": major +"@pnpm/headless": major +"@pnpm/package-store": major +"@pnpm/core": major +"@pnpm/cache.commands": major +"@pnpm/server": major +"@pnpm/store.cafs": major +"@pnpm/cache.api": major +"@pnpm/worker": major +--- + +Switched to from .json file to .v8 files in the global store and cache. diff --git a/.changeset/short-cougars-arrive.md b/.changeset/short-cougars-arrive.md new file mode 100644 index 0000000000..357a0da509 --- /dev/null +++ b/.changeset/short-cougars-arrive.md @@ -0,0 +1,5 @@ +--- +"@pnpm/fs.v8-file": major +--- + +Initial release. diff --git a/.changeset/young-points-dress.md b/.changeset/young-points-dress.md new file mode 100644 index 0000000000..b10fbefb8b --- /dev/null +++ b/.changeset/young-points-dress.md @@ -0,0 +1,9 @@ +--- +"pnpm": major +--- + +JSON files in the store and cache are replaced with binary files generated with v8.serialize. + +Reading and deserializing a binary file generated with v8.serialize appears to be about 23% faster than reading and parsing a JSON file. Other operations are similar in speed. Therefore, we have switched to V8 binary files for storing data in pnpm’s global store and cache. The disadvantage of this approach is that V8 files generated by newer versions of Node.js cannot be deserialized by older versions. In such cases, we ignore the cache. + +Related PR: [#9971](https://github.com/pnpm/pnpm/pull/9971). diff --git a/cache/api/package.json b/cache/api/package.json index bdbcc0b896..3da3fe4e8d 100644 --- a/cache/api/package.json +++ b/cache/api/package.json @@ -33,6 +33,7 @@ "dependencies": { "@pnpm/config": "workspace:*", "@pnpm/constants": "workspace:*", + "@pnpm/fs.v8-file": "workspace:*", "@pnpm/npm-resolver": "workspace:*", "@pnpm/store.cafs": "workspace:*", "encode-registry": "catalog:", diff --git a/cache/api/src/cacheList.ts b/cache/api/src/cacheList.ts index 00554b916d..b9c3804597 100644 --- a/cache/api/src/cacheList.ts +++ b/cache/api/src/cacheList.ts @@ -13,7 +13,7 @@ export async function cacheList (opts: { cacheDir: string, registry?: string, re export async function findMetadataFiles (opts: { cacheDir: string, registry?: string }, filter: string[]): Promise { const prefix = opts.registry ? `${getRegistryName(opts.registry)}` : '*' - const patterns = filter.length ? filter.map((filter) => `${prefix}/${filter}.json`) : [`${prefix}/**`] + const patterns = filter.length ? filter.map((filter) => `${prefix}/${filter}.v8`) : [`${prefix}/**`] const metaFiles = await glob(patterns, { cwd: opts.cacheDir, expandDirectories: false, diff --git a/cache/api/src/cacheView.ts b/cache/api/src/cacheView.ts index ce0ef3a16a..2171d97d0d 100644 --- a/cache/api/src/cacheView.ts +++ b/cache/api/src/cacheView.ts @@ -1,6 +1,7 @@ import fs from 'fs' import path from 'path' import { glob } from 'tinyglobby' +import { safeReadV8FileSync } from '@pnpm/fs.v8-file' import { getIndexFilePathInCafs } from '@pnpm/store.cafs' import { type PackageMeta } from '@pnpm/npm-resolver' import getRegistryName from 'encode-registry' @@ -14,13 +15,14 @@ interface CachedVersions { export async function cacheView (opts: { cacheDir: string, storeDir: string, registry?: string }, packageName: string): Promise { const prefix = opts.registry ? `${getRegistryName(opts.registry)}` : '*' - const metaFilePaths = (await glob(`${prefix}/${packageName}.json`, { + const metaFilePaths = (await glob(`${prefix}/${packageName}.v8`, { cwd: opts.cacheDir, expandDirectories: false, })).sort() const metaFilesByPath: Record = {} for (const filePath of metaFilePaths) { - const metaObject = JSON.parse(fs.readFileSync(path.join(opts.cacheDir, filePath), 'utf8')) as PackageMeta + const metaObject = safeReadV8FileSync(path.join(opts.cacheDir, filePath)) + if (!metaObject) continue const cachedVersions: string[] = [] const nonCachedVersions: string[] = [] for (const [version, manifest] of Object.entries(metaObject.versions)) { diff --git a/cache/api/tsconfig.json b/cache/api/tsconfig.json index a175a5b5a3..a975674207 100644 --- a/cache/api/tsconfig.json +++ b/cache/api/tsconfig.json @@ -12,6 +12,9 @@ { "path": "../../config/config" }, + { + "path": "../../fs/v8-file" + }, { "path": "../../packages/constants" }, diff --git a/cache/commands/test/cacheDelete.cmd.test.ts b/cache/commands/test/cacheDelete.cmd.test.ts index 7b5f4779f7..89565e9f10 100644 --- a/cache/commands/test/cacheDelete.cmd.test.ts +++ b/cache/commands/test/cacheDelete.cmd.test.ts @@ -49,7 +49,7 @@ describe('cache delete', () => { pnpmHomeDir: storeDir, }, ['list']) - expect(result).toBe(`localhost+${REGISTRY_MOCK_PORT}/is-negative.json -registry.npmjs.org/is-negative.json`) + expect(result).toBe(`localhost+${REGISTRY_MOCK_PORT}/is-negative.v8 +registry.npmjs.org/is-negative.v8`) }) }) diff --git a/cache/commands/test/cacheList.cmd.test.ts b/cache/commands/test/cacheList.cmd.test.ts index cec0f7ebda..7718422e20 100644 --- a/cache/commands/test/cacheList.cmd.test.ts +++ b/cache/commands/test/cacheList.cmd.test.ts @@ -44,9 +44,9 @@ describe('cache', () => { pnpmHomeDir: storeDir, }, ['list']) - expect(result).toBe(`localhost+${REGISTRY_MOCK_PORT}/is-negative.json -registry.npmjs.org/is-negative.json -registry.npmjs.org/is-positive.json`) + expect(result).toBe(`localhost+${REGISTRY_MOCK_PORT}/is-negative.v8 +registry.npmjs.org/is-negative.v8 +registry.npmjs.org/is-positive.v8`) }) test('list all metadata from the cache related to the specified registry', async () => { const result = await cache.handler({ @@ -57,8 +57,8 @@ registry.npmjs.org/is-positive.json`) pnpmHomeDir: storeDir, }, ['list']) - expect(result).toBe(`registry.npmjs.org/is-negative.json -registry.npmjs.org/is-positive.json`) + expect(result).toBe(`registry.npmjs.org/is-negative.v8 +registry.npmjs.org/is-positive.v8`) }) test('list all metadata from the cache that matches a pattern', async () => { const result = await cache.handler({ @@ -67,7 +67,7 @@ registry.npmjs.org/is-positive.json`) pnpmHomeDir: storeDir, }, ['list', '*-positive']) - expect(result).toBe('registry.npmjs.org/is-positive.json') + expect(result).toBe('registry.npmjs.org/is-positive.v8') }) test('list registries', async () => { const result = await cache.handler({ diff --git a/exec/plugin-commands-rebuild/package.json b/exec/plugin-commands-rebuild/package.json index 1ebb6224fa..20190521cd 100644 --- a/exec/plugin-commands-rebuild/package.json +++ b/exec/plugin-commands-rebuild/package.json @@ -74,6 +74,7 @@ "devDependencies": { "@pnpm/assert-project": "workspace:*", "@pnpm/crypto.object-hasher": "workspace:*", + "@pnpm/fs.v8-file": "workspace:*", "@pnpm/logger": "workspace:*", "@pnpm/plugin-commands-rebuild": "workspace:*", "@pnpm/prepare": "workspace:*", diff --git a/exec/plugin-commands-rebuild/test/index.ts b/exec/plugin-commands-rebuild/test/index.ts index 6b0c5a85f2..153040bc98 100644 --- a/exec/plugin-commands-rebuild/test/index.ts +++ b/exec/plugin-commands-rebuild/test/index.ts @@ -1,7 +1,9 @@ /// import fs from 'fs' import path from 'path' -import { getIndexFilePathInCafs } from '@pnpm/store.cafs' +import v8 from 'v8' +import { readV8FileSync } from '@pnpm/fs.v8-file' +import { getIndexFilePathInCafs, type PackageFilesIndex } from '@pnpm/store.cafs' import { ENGINE_NAME, STORE_VERSION, WANTED_LOCKFILE } from '@pnpm/constants' import { hashObject } from '@pnpm/crypto.object-hasher' import { rebuild } from '@pnpm/plugin-commands-rebuild' @@ -9,7 +11,6 @@ import { prepare } from '@pnpm/prepare' import { getIntegrity, REGISTRY_MOCK_PORT } from '@pnpm/registry-mock' import { fixtures } from '@pnpm/test-fixtures' import execa from 'execa' -import { loadJsonFileSync } from 'load-json-file' import sinon from 'sinon' import { DEFAULT_OPTS } from './utils/index.js' @@ -76,7 +77,7 @@ test('rebuilds dependencies', async () => { } const cacheIntegrityPath = getIndexFilePathInCafs(path.join(storeDir, STORE_VERSION), getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0') - const cacheIntegrity = loadJsonFileSync(cacheIntegrityPath) // eslint-disable-line @typescript-eslint/no-explicit-any + const cacheIntegrity = readV8FileSync(cacheIntegrityPath)! expect(cacheIntegrity!.sideEffects).toBeTruthy() const sideEffectsKey = `${ENGINE_NAME};deps=${hashObject({ id: `@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0:${getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0')}`, @@ -88,7 +89,7 @@ test('rebuilds dependencies', async () => { }, })}` expect(cacheIntegrity).toHaveProperty(['sideEffects', sideEffectsKey, 'added', 'generated-by-postinstall.js']) - delete cacheIntegrity!.sideEffects[sideEffectsKey].added['generated-by-postinstall.js'] + delete cacheIntegrity!.sideEffects![sideEffectsKey].added!['generated-by-postinstall.js'] }) test('skipIfHasSideEffectsCache', async () => { @@ -109,12 +110,20 @@ test('skipIfHasSideEffectsCache', async () => { ]) const cacheIntegrityPath = getIndexFilePathInCafs(path.join(storeDir, STORE_VERSION), getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0') - let cacheIntegrity = loadJsonFileSync(cacheIntegrityPath) // eslint-disable-line @typescript-eslint/no-explicit-any + let cacheIntegrity = readV8FileSync(cacheIntegrityPath)! const sideEffectsKey = `${ENGINE_NAME};deps=${hashObject({ '@pnpm.e2e/hello-world-js-bin@1.0.0': {} })}` cacheIntegrity.sideEffects = { - [sideEffectsKey]: { added: { foo: 'bar' } }, + [sideEffectsKey]: { + added: { + foo: { + integrity: 'bar', + mode: 1, + size: 1, + }, + }, + }, } - fs.writeFileSync(cacheIntegrityPath, JSON.stringify(cacheIntegrity, null, 2), 'utf8') + fs.writeFileSync(cacheIntegrityPath, v8.serialize(cacheIntegrity)) let modules = project.readModulesManifest() expect(modules!.pendingBuilds).toStrictEqual([ @@ -136,7 +145,7 @@ test('skipIfHasSideEffectsCache', async () => { expect(modules).toBeTruthy() expect(modules!.pendingBuilds).toHaveLength(0) - cacheIntegrity = loadJsonFileSync(cacheIntegrityPath) // eslint-disable-line @typescript-eslint/no-explicit-any + cacheIntegrity = readV8FileSync(cacheIntegrityPath)! expect(cacheIntegrity!.sideEffects).toBeTruthy() expect(cacheIntegrity).toHaveProperty(['sideEffects', sideEffectsKey, 'added', 'foo']) }) diff --git a/exec/plugin-commands-rebuild/tsconfig.json b/exec/plugin-commands-rebuild/tsconfig.json index 274fab41ef..772a5351e0 100644 --- a/exec/plugin-commands-rebuild/tsconfig.json +++ b/exec/plugin-commands-rebuild/tsconfig.json @@ -42,6 +42,9 @@ { "path": "../../deps/graph-sequencer" }, + { + "path": "../../fs/v8-file" + }, { "path": "../../lockfile/types" }, diff --git a/fs/v8-file/README.md b/fs/v8-file/README.md new file mode 100644 index 0000000000..2aba304b7a --- /dev/null +++ b/fs/v8-file/README.md @@ -0,0 +1,17 @@ +# @pnpm/fs.v8-file + +> Reading/writing binary files serialized with v8 + +[![npm version](https://img.shields.io/npm/v/@pnpm/fs.v8-file)](https://www.npmjs.com/package/@pnpm/fs.v8-file) + +Reading and deserializing a binary file generated with v8.serialize appears to be about 23% faster than reading and parsing a JSON file. Other operations are similar in speed. Therefore, we use V8 binary files instead of JSON files to store data in pnpm’s global store and cache. The disadvantage of this approach is that V8 files generated by newer versions of Node.js cannot be deserialized by older versions. + +## Installation + +```sh +pnpm add @pnpm/fs.v8-file +``` + +## License + +MIT diff --git a/fs/v8-file/package.json b/fs/v8-file/package.json new file mode 100644 index 0000000000..308386def0 --- /dev/null +++ b/fs/v8-file/package.json @@ -0,0 +1,43 @@ +{ + "name": "@pnpm/fs.v8-file", + "version": "1000.0.0-0", + "description": "Reading/writing binary files serialized with v8", + "keywords": [ + "pnpm", + "pnpm10" + ], + "license": "MIT", + "funding": "https://opencollective.com/pnpm", + "repository": "https://github.com/pnpm/pnpm/tree/main/fs/v8-file", + "homepage": "https://github.com/pnpm/pnpm/tree/main/fs/v8-file#readme", + "bugs": { + "url": "https://github.com/pnpm/pnpm/issues" + }, + "type": "module", + "main": "lib/index.js", + "types": "lib/index.d.ts", + "exports": { + ".": "./lib/index.js" + }, + "files": [ + "lib", + "!*.map" + ], + "scripts": { + "lint": "eslint \"src/**/*.ts\"", + "test": "pnpm run compile", + "prepublishOnly": "pnpm run compile", + "compile": "tsc --build && pnpm run lint --fix" + }, + "dependencies": { + }, + "devDependencies": { + "@pnpm/fs.v8-file": "workspace:*" + }, + "engines": { + "node": ">=20.19" + }, + "jest": { + "preset": "@pnpm/jest-config" + } +} diff --git a/fs/v8-file/src/index.ts b/fs/v8-file/src/index.ts new file mode 100644 index 0000000000..2c238b3d4f --- /dev/null +++ b/fs/v8-file/src/index.ts @@ -0,0 +1,31 @@ +import fs from 'fs' +import v8 from 'v8' +import util from 'util' + +export function safeReadV8FileSync (filePath: string): T | undefined { + try { + return readV8FileSync(filePath) + } catch (error) { + if (util.types.isNativeError(error) && 'code' in error && error.code === 'ENOENT') return undefined + throw error + } +} + +export function readV8FileSync (filePath: string): T | undefined { + const buffer: Buffer = fs.readFileSync(filePath) + try { + return v8.deserialize(buffer) + } catch { + return undefined + } +} + +export function readV8FileStrictSync (filePath: string): T { + const buffer: Buffer = fs.readFileSync(filePath) + return v8.deserialize(buffer) +} + +export async function readV8FileStrictAsync (filePath: string): Promise { + const buffer: Buffer = await fs.promises.readFile(filePath) + return v8.deserialize(buffer) +} diff --git a/fs/v8-file/tsconfig.json b/fs/v8-file/tsconfig.json new file mode 100644 index 0000000000..c6f0399f60 --- /dev/null +++ b/fs/v8-file/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "@pnpm/tsconfig", + "compilerOptions": { + "outDir": "lib", + "rootDir": "src" + }, + "include": [ + "src/**/*.ts", + "../../__typings__/**/*.d.ts" + ], + "references": [] +} diff --git a/fs/v8-file/tsconfig.lint.json b/fs/v8-file/tsconfig.lint.json new file mode 100644 index 0000000000..1bbe711971 --- /dev/null +++ b/fs/v8-file/tsconfig.lint.json @@ -0,0 +1,8 @@ +{ + "extends": "./tsconfig.json", + "include": [ + "src/**/*.ts", + "test/**/*.ts", + "../../__typings__/**/*.d.ts" + ] +} diff --git a/modules-mounter/daemon/package.json b/modules-mounter/daemon/package.json index ec508dfecb..813c60b130 100644 --- a/modules-mounter/daemon/package.json +++ b/modules-mounter/daemon/package.json @@ -38,6 +38,7 @@ "dependencies": { "@pnpm/config": "workspace:*", "@pnpm/dependency-path": "workspace:*", + "@pnpm/fs.v8-file": "workspace:*", "@pnpm/lockfile.fs": "workspace:*", "@pnpm/lockfile.utils": "workspace:*", "@pnpm/logger": "workspace:*", @@ -45,7 +46,6 @@ "@pnpm/store.cafs": "workspace:*", "@pnpm/types": "workspace:*", "hyperdrive-schemas": "catalog:", - "load-json-file": "catalog:", "normalize-path": "catalog:" }, "peerDependencies": { diff --git a/modules-mounter/daemon/src/createFuseHandlers.ts b/modules-mounter/daemon/src/createFuseHandlers.ts index 6a31717577..05b6eb088a 100644 --- a/modules-mounter/daemon/src/createFuseHandlers.ts +++ b/modules-mounter/daemon/src/createFuseHandlers.ts @@ -1,5 +1,6 @@ // cspell:ignore ents import fs from 'fs' +import { readV8FileStrictSync } from '@pnpm/fs.v8-file' import { getIndexFilePathInCafs, getFilePathByModeInCafs, type PackageFilesIndex } from '@pnpm/store.cafs' import { type LockfileObject, readWantedLockfile, type PackageSnapshot, type TarballResolution } from '@pnpm/lockfile.fs' import { @@ -7,7 +8,6 @@ import { } from '@pnpm/lockfile.utils' import { type DepPath } from '@pnpm/types' import schemas from 'hyperdrive-schemas' -import { loadJsonFileSync } from 'load-json-file' import Fuse from 'fuse-native' import * as cafsExplorer from './cafsExplorer.js' import { makeVirtualNodeModules } from './makeVirtualNodeModules.js' @@ -185,7 +185,7 @@ export function createFuseHandlersFromLockfile (lockfile: LockfileObject, storeD pkgSnapshotCache.set(depPath, { ...nameVer, pkgSnapshot, - index: loadJsonFileSync(indexPath), // TODO: maybe make it async? + index: readV8FileStrictSync(indexPath), // TODO: maybe make it async? }) } return pkgSnapshotCache.get(depPath) diff --git a/modules-mounter/daemon/tsconfig.json b/modules-mounter/daemon/tsconfig.json index 9585337e8f..d7ff9211ea 100644 --- a/modules-mounter/daemon/tsconfig.json +++ b/modules-mounter/daemon/tsconfig.json @@ -12,6 +12,9 @@ { "path": "../../config/config" }, + { + "path": "../../fs/v8-file" + }, { "path": "../../lockfile/fs" }, diff --git a/pkg-manager/core/package.json b/pkg-manager/core/package.json index 9f33a4437c..632fbeb15b 100644 --- a/pkg-manager/core/package.json +++ b/pkg-manager/core/package.json @@ -126,6 +126,7 @@ "@pnpm/assert-project": "workspace:*", "@pnpm/assert-store": "workspace:*", "@pnpm/core": "workspace:*", + "@pnpm/fs.v8-file": "workspace:*", "@pnpm/git-utils": "workspace:*", "@pnpm/lockfile.types": "workspace:*", "@pnpm/logger": "workspace:*", diff --git a/pkg-manager/core/test/install/lockfileOnly.ts b/pkg-manager/core/test/install/lockfileOnly.ts index 47cd8c70e6..3467a6a548 100644 --- a/pkg-manager/core/test/install/lockfileOnly.ts +++ b/pkg-manager/core/test/install/lockfileOnly.ts @@ -19,9 +19,9 @@ test('install with lockfileOnly = true', async () => { const { cafsHasNot } = assertStore(opts.storeDir) cafsHasNot('@pnpm.e2e/pkg-with-1-dep', '100.0.0') - expect(fs.existsSync(path.join(opts.cacheDir, `${ABBREVIATED_META_DIR}/localhost+${REGISTRY_MOCK_PORT}/@pnpm.e2e/pkg-with-1-dep.json`))).toBeTruthy() + expect(fs.existsSync(path.join(opts.cacheDir, `${ABBREVIATED_META_DIR}/localhost+${REGISTRY_MOCK_PORT}/@pnpm.e2e/pkg-with-1-dep.v8`))).toBeTruthy() cafsHasNot('@pnpm.e2e/dep-of-pkg-with-1-dep', '100.1.0') - expect(fs.existsSync(path.join(opts.cacheDir, `${ABBREVIATED_META_DIR}/localhost+${REGISTRY_MOCK_PORT}/@pnpm.e2e/dep-of-pkg-with-1-dep.json`))).toBeTruthy() + expect(fs.existsSync(path.join(opts.cacheDir, `${ABBREVIATED_META_DIR}/localhost+${REGISTRY_MOCK_PORT}/@pnpm.e2e/dep-of-pkg-with-1-dep.v8`))).toBeTruthy() project.hasNot('@pnpm.e2e/pkg-with-1-dep') expect(manifest.dependencies!['@pnpm.e2e/pkg-with-1-dep']).toBeTruthy() @@ -37,9 +37,9 @@ test('install with lockfileOnly = true', async () => { await install(manifest, opts) cafsHasNot('@pnpm.e2e/pkg-with-1-dep', '100.0.0') - expect(fs.existsSync(path.join(opts.cacheDir, `${ABBREVIATED_META_DIR}/localhost+${REGISTRY_MOCK_PORT}/@pnpm.e2e/pkg-with-1-dep.json`))).toBeTruthy() + expect(fs.existsSync(path.join(opts.cacheDir, `${ABBREVIATED_META_DIR}/localhost+${REGISTRY_MOCK_PORT}/@pnpm.e2e/pkg-with-1-dep.v8`))).toBeTruthy() cafsHasNot('@pnpm.e2e/dep-of-pkg-with-1-dep', '100.1.0') - expect(fs.existsSync(path.join(opts.cacheDir, `${ABBREVIATED_META_DIR}/localhost+${REGISTRY_MOCK_PORT}/@pnpm.e2e/dep-of-pkg-with-1-dep.json`))).toBeTruthy() + expect(fs.existsSync(path.join(opts.cacheDir, `${ABBREVIATED_META_DIR}/localhost+${REGISTRY_MOCK_PORT}/@pnpm.e2e/dep-of-pkg-with-1-dep.v8`))).toBeTruthy() project.hasNot('@pnpm.e2e/pkg-with-1-dep') expect(project.readCurrentLockfile()).toBeFalsy() diff --git a/pkg-manager/core/test/install/patch.ts b/pkg-manager/core/test/install/patch.ts index 946f7c3e4e..45940f691c 100644 --- a/pkg-manager/core/test/install/patch.ts +++ b/pkg-manager/core/test/install/patch.ts @@ -5,11 +5,11 @@ import { ENGINE_NAME } from '@pnpm/constants' import { install } from '@pnpm/core' import { type IgnoredScriptsLog } from '@pnpm/core-loggers' import { createHexHashFromFile } from '@pnpm/crypto.hash' +import { readV8FileStrictSync } from '@pnpm/fs.v8-file' import { prepareEmpty } from '@pnpm/prepare' import { fixtures } from '@pnpm/test-fixtures' import { jest } from '@jest/globals' import { sync as rimraf } from '@zkochan/rimraf' -import { loadJsonFileSync } from 'load-json-file' import sinon from 'sinon' import { testDefaults } from '../utils/index.js' @@ -56,8 +56,8 @@ test('patch package with exact version', async () => { }) expect(lockfile.snapshots[`is-positive@1.0.0(patch_hash=${patchFileHash})`]).toBeTruthy() - const filesIndexFile = path.join(opts.storeDir, 'index/c7/1ccf199e0fdae37aad13946b937d67bcd35fa111b84d21b3a19439cfdc2812-is-positive@1.0.0.json') - const filesIndex = loadJsonFileSync(filesIndexFile) + const filesIndexFile = path.join(opts.storeDir, 'index/c7/1ccf199e0fdae37aad13946b937d67bcd35fa111b84d21b3a19439cfdc2812-is-positive@1.0.0.v8') + const filesIndex = readV8FileStrictSync(filesIndexFile) const sideEffectsKey = `${ENGINE_NAME};patch=${patchFileHash}` const patchedFileIntegrity = filesIndex.sideEffects?.[sideEffectsKey].added?.['index.js']?.integrity expect(patchedFileIntegrity).toBeTruthy() @@ -151,8 +151,8 @@ test('patch package with version range', async () => { }) expect(lockfile.snapshots[`is-positive@1.0.0(patch_hash=${patchFileHash})`]).toBeTruthy() - const filesIndexFile = path.join(opts.storeDir, 'index/c7/1ccf199e0fdae37aad13946b937d67bcd35fa111b84d21b3a19439cfdc2812-is-positive@1.0.0.json') - const filesIndex = loadJsonFileSync(filesIndexFile) + const filesIndexFile = path.join(opts.storeDir, 'index/c7/1ccf199e0fdae37aad13946b937d67bcd35fa111b84d21b3a19439cfdc2812-is-positive@1.0.0.v8') + const filesIndex = readV8FileStrictSync(filesIndexFile) const sideEffectsKey = `${ENGINE_NAME};patch=${patchFileHash}` const patchedFileIntegrity = filesIndex.sideEffects?.[sideEffectsKey].added?.['index.js']?.integrity expect(patchedFileIntegrity).toBeTruthy() @@ -318,8 +318,8 @@ test('patch package when scripts are ignored', async () => { }) expect(lockfile.snapshots[`is-positive@1.0.0(patch_hash=${patchFileHash})`]).toBeTruthy() - const filesIndexFile = path.join(opts.storeDir, 'index/c7/1ccf199e0fdae37aad13946b937d67bcd35fa111b84d21b3a19439cfdc2812-is-positive@1.0.0.json') - const filesIndex = loadJsonFileSync(filesIndexFile) + const filesIndexFile = path.join(opts.storeDir, 'index/c7/1ccf199e0fdae37aad13946b937d67bcd35fa111b84d21b3a19439cfdc2812-is-positive@1.0.0.v8') + const filesIndex = readV8FileStrictSync(filesIndexFile) const sideEffectsKey = `${ENGINE_NAME};patch=${patchFileHash}` const patchedFileIntegrity = filesIndex.sideEffects?.[sideEffectsKey].added?.['index.js']?.integrity expect(patchedFileIntegrity).toBeTruthy() @@ -406,8 +406,8 @@ test('patch package when the package is not in onlyBuiltDependencies list', asyn }) expect(lockfile.snapshots[`is-positive@1.0.0(patch_hash=${patchFileHash})`]).toBeTruthy() - const filesIndexFile = path.join(opts.storeDir, 'index/c7/1ccf199e0fdae37aad13946b937d67bcd35fa111b84d21b3a19439cfdc2812-is-positive@1.0.0.json') - const filesIndex = loadJsonFileSync(filesIndexFile) + const filesIndexFile = path.join(opts.storeDir, 'index/c7/1ccf199e0fdae37aad13946b937d67bcd35fa111b84d21b3a19439cfdc2812-is-positive@1.0.0.v8') + const filesIndex = readV8FileStrictSync(filesIndexFile) const sideEffectsKey = `${ENGINE_NAME};patch=${patchFileHash}` const patchedFileIntegrity = filesIndex.sideEffects?.[sideEffectsKey].added?.['index.js']?.integrity expect(patchedFileIntegrity).toBeTruthy() diff --git a/pkg-manager/core/test/install/sideEffects.ts b/pkg-manager/core/test/install/sideEffects.ts index aa6e64c1f8..62678b7b48 100644 --- a/pkg-manager/core/test/install/sideEffects.ts +++ b/pkg-manager/core/test/install/sideEffects.ts @@ -1,14 +1,14 @@ import fs from 'fs' import path from 'path' +import v8 from 'v8' import { addDependenciesToPackage, install } from '@pnpm/core' import { hashObject } from '@pnpm/crypto.object-hasher' +import { readV8FileStrictSync } from '@pnpm/fs.v8-file' import { getIndexFilePathInCafs, getFilePathByModeInCafs, type PackageFilesIndex } from '@pnpm/store.cafs' import { getIntegrity, REGISTRY_MOCK_PORT } from '@pnpm/registry-mock' import { prepareEmpty } from '@pnpm/prepare' import { ENGINE_NAME } from '@pnpm/constants' import { sync as rimraf } from '@zkochan/rimraf' -import { loadJsonFileSync } from 'load-json-file' -import { writeJsonFileSync } from 'write-json-file' import { testDefaults } from '../utils/index.js' const ENGINE_DIR = `${process.platform}-${process.arch}-node-${process.version.split('.')[0]}` @@ -83,7 +83,7 @@ test('using side effects cache', async () => { const { updatedManifest: manifest } = await addDependenciesToPackage({}, ['@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0'], opts) const filesIndexFile = getIndexFilePathInCafs(opts.storeDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0') - const filesIndex = loadJsonFileSync(filesIndexFile) + const filesIndex = readV8FileStrictSync(filesIndexFile) expect(filesIndex.sideEffects).toBeTruthy() // files index has side effects const sideEffectsKey = `${ENGINE_NAME};deps=${hashObject({ id: `@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0:${getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0')}`, @@ -97,7 +97,7 @@ test('using side effects cache', async () => { expect(filesIndex.sideEffects).toHaveProperty([sideEffectsKey, 'added', 'generated-by-preinstall.js']) expect(filesIndex.sideEffects).toHaveProperty([sideEffectsKey, 'added', 'generated-by-postinstall.js']) delete filesIndex.sideEffects![sideEffectsKey].added?.['generated-by-postinstall.js'] - writeJsonFileSync(filesIndexFile, filesIndex) + fs.writeFileSync(filesIndexFile, v8.serialize(filesIndex)) rimraf('node_modules') rimraf('pnpm-lock.yaml') // to avoid headless install @@ -164,7 +164,7 @@ test('uploading errors do not interrupt installation', async () => { expect(fs.existsSync('node_modules/@pnpm.e2e/pre-and-postinstall-scripts-example/generated-by-postinstall.js')).toBeTruthy() const filesIndexFile = getIndexFilePathInCafs(opts.storeDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0') - const filesIndex = loadJsonFileSync(filesIndexFile) + const filesIndex = readV8FileStrictSync(filesIndexFile) expect(filesIndex.sideEffects).toBeFalsy() }) @@ -181,7 +181,7 @@ test('a postinstall script does not modify the original sources added to the sto expect(fs.readFileSync('node_modules/@pnpm/postinstall-modifies-source/empty-file.txt', 'utf8')).toContain('hello') const filesIndexFile = getIndexFilePathInCafs(opts.storeDir, getIntegrity('@pnpm/postinstall-modifies-source', '1.0.0'), '@pnpm/postinstall-modifies-source@1.0.0') - const filesIndex = loadJsonFileSync(filesIndexFile) + const filesIndex = readV8FileStrictSync(filesIndexFile) const patchedFileIntegrity = filesIndex.sideEffects?.[`${ENGINE_NAME};deps=${hashObject({ id: `@pnpm/postinstall-modifies-source@1.0.0:${getIntegrity('@pnpm/postinstall-modifies-source', '1.0.0')}`, deps: {}, @@ -206,7 +206,7 @@ test('a corrupted side-effects cache is ignored', async () => { const { updatedManifest: manifest } = await addDependenciesToPackage({}, ['@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0'], opts) const filesIndexFile = getIndexFilePathInCafs(opts.storeDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0') - const filesIndex = loadJsonFileSync(filesIndexFile) + const filesIndex = readV8FileStrictSync(filesIndexFile) expect(filesIndex.sideEffects).toBeTruthy() // files index has side effects const sideEffectsKey = `${ENGINE_NAME};deps=${hashObject({ id: `@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0:${getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0')}`, diff --git a/pkg-manager/core/tsconfig.json b/pkg-manager/core/tsconfig.json index f76c9b5210..f18ec3463a 100644 --- a/pkg-manager/core/tsconfig.json +++ b/pkg-manager/core/tsconfig.json @@ -66,6 +66,9 @@ { "path": "../../fs/symlink-dependency" }, + { + "path": "../../fs/v8-file" + }, { "path": "../../hooks/read-package-hook" }, diff --git a/pkg-manager/headless/package.json b/pkg-manager/headless/package.json index 828d4cee73..c54da0f2e3 100644 --- a/pkg-manager/headless/package.json +++ b/pkg-manager/headless/package.json @@ -81,6 +81,7 @@ "@jest/globals": "catalog:", "@pnpm/assert-project": "workspace:*", "@pnpm/crypto.object-hasher": "workspace:*", + "@pnpm/fs.v8-file": "workspace:*", "@pnpm/headless": "workspace:*", "@pnpm/logger": "workspace:*", "@pnpm/prepare": "workspace:*", diff --git a/pkg-manager/headless/test/index.ts b/pkg-manager/headless/test/index.ts index 25a0b00d29..536a9b5a3a 100644 --- a/pkg-manager/headless/test/index.ts +++ b/pkg-manager/headless/test/index.ts @@ -1,9 +1,11 @@ /// import fs from 'fs' +import v8 from 'v8' import path from 'path' import { assertProject } from '@pnpm/assert-project' import { hashObject } from '@pnpm/crypto.object-hasher' -import { getIndexFilePathInCafs } from '@pnpm/store.cafs' +import { readV8FileStrictSync } from '@pnpm/fs.v8-file' +import { getIndexFilePathInCafs, type PackageFilesIndex } from '@pnpm/store.cafs' import { ENGINE_NAME, WANTED_LOCKFILE } from '@pnpm/constants' import { type PackageManifestLog, @@ -23,7 +25,6 @@ import { jest } from '@jest/globals' import { sync as rimraf } from '@zkochan/rimraf' import { loadJsonFileSync } from 'load-json-file' import sinon from 'sinon' -import { writeJsonFileSync } from 'write-json-file' import { testDefaults } from './utils/testDefaults.js' const f = fixtures(import.meta.dirname) @@ -680,7 +681,7 @@ test.each([['isolated'], ['hoisted']])('using side effects cache with nodeLinker await headlessInstall(opts) const cacheIntegrityPath = getIndexFilePathInCafs(opts.storeDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0') - const cacheIntegrity = loadJsonFileSync(cacheIntegrityPath) // eslint-disable-line @typescript-eslint/no-explicit-any + const cacheIntegrity = readV8FileStrictSync(cacheIntegrityPath) expect(cacheIntegrity!.sideEffects).toBeTruthy() const sideEffectsKey = `${ENGINE_NAME};deps=${hashObject({ id: `@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0:${getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0')}`, @@ -692,10 +693,10 @@ test.each([['isolated'], ['hoisted']])('using side effects cache with nodeLinker }, })}` expect(cacheIntegrity).toHaveProperty(['sideEffects', sideEffectsKey, 'added', 'generated-by-postinstall.js']) - delete cacheIntegrity!.sideEffects[sideEffectsKey].added['generated-by-postinstall.js'] + delete cacheIntegrity!.sideEffects![sideEffectsKey].added!['generated-by-postinstall.js'] expect(cacheIntegrity).toHaveProperty(['sideEffects', sideEffectsKey, 'added', 'generated-by-preinstall.js']) - writeJsonFileSync(cacheIntegrityPath, cacheIntegrity) + fs.writeFileSync(cacheIntegrityPath, v8.serialize(cacheIntegrity)) prefix = f.prepare('side-effects') const opts2 = await testDefaults({ diff --git a/pkg-manager/headless/tsconfig.json b/pkg-manager/headless/tsconfig.json index 652b9a9a31..99f42d326e 100644 --- a/pkg-manager/headless/tsconfig.json +++ b/pkg-manager/headless/tsconfig.json @@ -42,6 +42,9 @@ { "path": "../../fs/symlink-dependency" }, + { + "path": "../../fs/v8-file" + }, { "path": "../../lockfile/filtering" }, diff --git a/pkg-manager/package-requester/package.json b/pkg-manager/package-requester/package.json index 73471e761a..0c44f2cf79 100644 --- a/pkg-manager/package-requester/package.json +++ b/pkg-manager/package-requester/package.json @@ -64,6 +64,7 @@ "@pnpm/cafs-types": "workspace:*", "@pnpm/client": "workspace:*", "@pnpm/create-cafs-store": "workspace:*", + "@pnpm/fs.v8-file": "workspace:*", "@pnpm/logger": "workspace:*", "@pnpm/package-requester": "workspace:*", "@pnpm/registry-mock": "catalog:", @@ -73,7 +74,6 @@ "@types/semver": "catalog:", "@types/ssri": "catalog:", "delay": "catalog:", - "load-json-file": "catalog:", "nock": "catalog:", "normalize-path": "catalog:", "tempy": "catalog:" diff --git a/pkg-manager/package-requester/test/index.ts b/pkg-manager/package-requester/test/index.ts index fa5b46ba13..1661232d7f 100644 --- a/pkg-manager/package-requester/test/index.ts +++ b/pkg-manager/package-requester/test/index.ts @@ -3,6 +3,7 @@ import fs from 'fs' import path from 'path' import { type PackageFilesIndex } from '@pnpm/store.cafs' import { createClient } from '@pnpm/client' +import { readV8FileStrictSync } from '@pnpm/fs.v8-file' import { streamParser } from '@pnpm/logger' import { createPackageRequester, type PackageResponse } from '@pnpm/package-requester' import { createCafsStore } from '@pnpm/create-cafs-store' @@ -12,7 +13,6 @@ import delay from 'delay' import { depPathToFilename } from '@pnpm/dependency-path' import { restartWorkerPool } from '@pnpm/worker' import { jest } from '@jest/globals' -import { loadJsonFileSync } from 'load-json-file' import nock from 'nock' import normalize from 'normalize-path' import { temporaryDirectory } from 'tempy' @@ -385,7 +385,7 @@ test('fetchPackageToStore()', async () => { expect(Object.keys(files.filesIndex).sort()).toStrictEqual(['package.json', 'index.js', 'license', 'readme.md'].sort()) expect(files.resolvedFrom).toBe('remote') - const indexFile = loadJsonFileSync(fetchResult.filesIndexFile) + const indexFile = readV8FileStrictSync(fetchResult.filesIndexFile) expect(indexFile).toBeTruthy() expect(typeof indexFile.files['package.json'].checkedAt).toBeTruthy() diff --git a/pkg-manager/package-requester/tsconfig.json b/pkg-manager/package-requester/tsconfig.json index 7826cb0ae8..e91eed619b 100644 --- a/pkg-manager/package-requester/tsconfig.json +++ b/pkg-manager/package-requester/tsconfig.json @@ -24,6 +24,9 @@ { "path": "../../fs/graceful-fs" }, + { + "path": "../../fs/v8-file" + }, { "path": "../../packages/core-loggers" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ac2e30ad9d..b56319863d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1254,6 +1254,9 @@ importers: '@pnpm/constants': specifier: workspace:* version: link:../../packages/constants + '@pnpm/fs.v8-file': + specifier: workspace:* + version: link:../../fs/v8-file '@pnpm/npm-resolver': specifier: workspace:* version: link:../../resolving/npm-resolver @@ -2800,6 +2803,9 @@ importers: '@pnpm/crypto.object-hasher': specifier: workspace:* version: link:../../crypto/object-hasher + '@pnpm/fs.v8-file': + specifier: workspace:* + version: link:../../fs/v8-file '@pnpm/logger': specifier: workspace:* version: link:../../packages/logger @@ -3462,6 +3468,12 @@ importers: specifier: workspace:* version: 'link:' + fs/v8-file: + devDependencies: + '@pnpm/fs.v8-file': + specifier: workspace:* + version: 'link:' + hooks/pnpmfile: dependencies: '@pnpm/core-loggers': @@ -4156,6 +4168,9 @@ importers: '@pnpm/dependency-path': specifier: workspace:* version: link:../../packages/dependency-path + '@pnpm/fs.v8-file': + specifier: workspace:* + version: link:../../fs/v8-file '@pnpm/lockfile.fs': specifier: workspace:* version: link:../../lockfile/fs @@ -4177,9 +4192,6 @@ importers: hyperdrive-schemas: specifier: 'catalog:' version: 2.0.0 - load-json-file: - specifier: 'catalog:' - version: 7.0.1 normalize-path: specifier: 'catalog:' version: 3.0.0 @@ -5041,6 +5053,9 @@ importers: '@pnpm/core': specifier: workspace:* version: 'link:' + '@pnpm/fs.v8-file': + specifier: workspace:* + version: link:../../fs/v8-file '@pnpm/git-utils': specifier: workspace:* version: link:../../packages/git-utils @@ -5320,6 +5335,9 @@ importers: '@pnpm/crypto.object-hasher': specifier: workspace:* version: link:../../crypto/object-hasher + '@pnpm/fs.v8-file': + specifier: workspace:* + version: link:../../fs/v8-file '@pnpm/headless': specifier: workspace:* version: 'link:' @@ -5686,6 +5704,9 @@ importers: '@pnpm/create-cafs-store': specifier: workspace:* version: link:../../store/create-cafs-store + '@pnpm/fs.v8-file': + specifier: workspace:* + version: link:../../fs/v8-file '@pnpm/logger': specifier: workspace:* version: link:../../packages/logger @@ -5713,9 +5734,6 @@ importers: delay: specifier: 'catalog:' version: 6.0.0 - load-json-file: - specifier: 'catalog:' - version: 7.0.1 nock: specifier: 'catalog:' version: 13.3.4 @@ -6465,6 +6483,9 @@ importers: '@pnpm/find-workspace-dir': specifier: workspace:* version: link:../workspace/find-workspace-dir + '@pnpm/fs.v8-file': + specifier: workspace:* + version: link:../fs/v8-file '@pnpm/lockfile.types': specifier: workspace:* version: link:../lockfile/types @@ -7321,6 +7342,9 @@ importers: '@pnpm/fetching-types': specifier: workspace:* version: link:../../network/fetching-types + '@pnpm/fs.v8-file': + specifier: workspace:* + version: link:../../fs/v8-file '@pnpm/graceful-fs': specifier: workspace:* version: link:../../fs/graceful-fs @@ -7354,9 +7378,6 @@ importers: encode-registry: specifier: 'catalog:' version: 3.0.1 - load-json-file: - specifier: 'catalog:' - version: 7.0.1 lru-cache: specifier: 'catalog:' version: 11.1.0 @@ -7418,6 +7439,9 @@ importers: '@types/ssri': specifier: 'catalog:' version: 7.1.5 + load-json-file: + specifier: 'catalog:' + version: 7.0.1 nock: specifier: 'catalog:' version: 13.3.4 @@ -7526,6 +7550,9 @@ importers: '@pnpm/error': specifier: workspace:* version: link:../../packages/error + '@pnpm/fs.v8-file': + specifier: workspace:* + version: link:../../fs/v8-file '@pnpm/lockfile.detect-dep-types': specifier: workspace:* version: link:../../lockfile/detect-dep-types @@ -7553,9 +7580,6 @@ importers: '@pnpm/types': specifier: workspace:* version: link:../../packages/types - load-json-file: - specifier: 'catalog:' - version: 7.0.1 p-limit: specifier: 'catalog:' version: 7.1.0 @@ -8066,6 +8090,9 @@ importers: '@pnpm/fetcher-base': specifier: workspace:* version: link:../../fetching/fetcher-base + '@pnpm/fs.v8-file': + specifier: workspace:* + version: link:../../fs/v8-file '@pnpm/package-requester': specifier: workspace:* version: link:../../pkg-manager/package-requester @@ -8087,9 +8114,6 @@ importers: '@zkochan/rimraf': specifier: 'catalog:' version: 3.0.2 - load-json-file: - specifier: 'catalog:' - version: 7.0.1 ramda: specifier: 'catalog:' version: '@pnpm/ramda@0.28.1' @@ -8203,6 +8227,9 @@ importers: '@pnpm/error': specifier: workspace:* version: link:../../packages/error + '@pnpm/fs.v8-file': + specifier: workspace:* + version: link:../../fs/v8-file '@pnpm/get-context': specifier: workspace:* version: link:../../pkg-manager/get-context @@ -8236,9 +8263,6 @@ importers: dint: specifier: 'catalog:' version: 5.1.0 - load-json-file: - specifier: 'catalog:' - version: 7.0.1 p-filter: specifier: 'catalog:' version: 4.1.0 @@ -8309,6 +8333,9 @@ importers: '@pnpm/error': specifier: workspace:* version: link:../../packages/error + '@pnpm/fs.v8-file': + specifier: workspace:* + version: link:../../fs/v8-file '@pnpm/graceful-fs': specifier: workspace:* version: link:../../fs/graceful-fs @@ -8333,9 +8360,6 @@ importers: chalk: specifier: 'catalog:' version: 5.6.0 - load-json-file: - specifier: 'catalog:' - version: 7.0.1 render-help: specifier: 'catalog:' version: 1.0.3 @@ -8377,6 +8401,9 @@ importers: '@pnpm/client': specifier: workspace:* version: link:../../pkg-manager/client + '@pnpm/fs.v8-file': + specifier: workspace:* + version: link:../../fs/v8-file '@pnpm/logger': specifier: workspace:* version: link:../../packages/logger @@ -8389,6 +8416,9 @@ importers: '@pnpm/server': specifier: workspace:* version: 'link:' + '@pnpm/store.cafs': + specifier: workspace:* + version: link:../cafs '@types/uuid': specifier: 'catalog:' version: 8.3.4 @@ -8401,9 +8431,6 @@ importers: is-port-reachable: specifier: 'catalog:' version: 4.0.0 - load-json-file: - specifier: 'catalog:' - version: 7.0.1 node-fetch: specifier: 'catalog:' version: 3.3.2 @@ -8647,6 +8674,9 @@ importers: '@pnpm/fs.hard-link-dir': specifier: workspace:* version: link:../fs/hard-link-dir + '@pnpm/fs.v8-file': + specifier: workspace:* + version: link:../fs/v8-file '@pnpm/graceful-fs': specifier: workspace:* version: link:../fs/graceful-fs @@ -8662,9 +8692,6 @@ importers: is-windows: specifier: 'catalog:' version: 1.0.2 - load-json-file: - specifier: 'catalog:' - version: 7.0.1 p-limit: specifier: 'catalog:' version: 7.1.0 diff --git a/pnpm/package.json b/pnpm/package.json index 026b0ba660..2d27fc1d10 100644 --- a/pnpm/package.json +++ b/pnpm/package.json @@ -78,6 +78,7 @@ "devDependencies": { "@jest/globals": "catalog:", "@pnpm/assert-project": "workspace:*", + "@pnpm/fs.v8-file": "workspace:*", "@pnpm/byline": "catalog:", "@pnpm/cache.commands": "workspace:*", "@pnpm/cli-meta": "workspace:*", diff --git a/pnpm/test/install/misc.ts b/pnpm/test/install/misc.ts index 0bdecfe2c3..742b715121 100644 --- a/pnpm/test/install/misc.ts +++ b/pnpm/test/install/misc.ts @@ -1,20 +1,20 @@ import fs from 'fs' import path from 'path' +import v8 from 'v8' import { STORE_VERSION, WANTED_LOCKFILE } from '@pnpm/constants' +import { readV8FileStrictSync } from '@pnpm/fs.v8-file' import { type LockfileObject } from '@pnpm/lockfile.types' import { prepare, prepareEmpty, preparePackages } from '@pnpm/prepare' import { readPackageJsonFromDir } from '@pnpm/read-package-json' import { readProjectManifest } from '@pnpm/read-project-manifest' import { getIntegrity } from '@pnpm/registry-mock' -import { getIndexFilePathInCafs } from '@pnpm/store.cafs' +import { getIndexFilePathInCafs, type PackageFilesIndex } from '@pnpm/store.cafs' import { writeProjectManifest } from '@pnpm/write-project-manifest' import { fixtures } from '@pnpm/test-fixtures' import dirIsCaseSensitive from 'dir-is-case-sensitive' import { sync as readYamlFile } from 'read-yaml-file' import { sync as rimraf } from '@zkochan/rimraf' import isWindows from 'is-windows' -import { loadJsonFileSync } from 'load-json-file' -import { writeJsonFileSync } from 'write-json-file' import { sync as writeYamlFile } from 'write-yaml-file' import crossSpawn from 'cross-spawn' import { @@ -159,7 +159,7 @@ test("don't fail on case insensitive filesystems when package has 2 files with s project.has('@pnpm.e2e/with-same-file-in-different-cases') - const { files: integrityFile } = loadJsonFileSync<{ files: object }>(project.getPkgIndexFilePath('@pnpm.e2e/with-same-file-in-different-cases', '1.0.0')) + const { files: integrityFile } = readV8FileStrictSync(project.getPkgIndexFilePath('@pnpm.e2e/with-same-file-in-different-cases', '1.0.0')) const packageFiles = Object.keys(integrityFile).sort() expect(packageFiles).toStrictEqual(['Foo.js', 'foo.js', 'package.json']) @@ -453,12 +453,12 @@ test('installation fails when the stored package name and version do not match t await execPnpm(['add', '@pnpm.e2e/dep-of-pkg-with-1-dep@100.1.0', ...settings]) const cacheIntegrityPath = getIndexFilePathInCafs(path.join(storeDir, STORE_VERSION), getIntegrity('@pnpm.e2e/dep-of-pkg-with-1-dep', '100.1.0'), '@pnpm.e2e/dep-of-pkg-with-1-dep@100.1.0') - const cacheIntegrity = loadJsonFileSync(cacheIntegrityPath) // eslint-disable-line @typescript-eslint/no-explicit-any + const cacheIntegrity = readV8FileStrictSync(cacheIntegrityPath) cacheIntegrity.name = 'foo' - writeJsonFileSync(cacheIntegrityPath, { + fs.writeFileSync(cacheIntegrityPath, v8.serialize({ ...cacheIntegrity, name: 'foo', - }) + })) rimraf('node_modules') await expect( diff --git a/pnpm/tsconfig.json b/pnpm/tsconfig.json index 3d9152c3b9..307d5c0163 100644 --- a/pnpm/tsconfig.json +++ b/pnpm/tsconfig.json @@ -74,6 +74,9 @@ { "path": "../exec/run-npm" }, + { + "path": "../fs/v8-file" + }, { "path": "../lockfile/plugin-commands-audit" }, diff --git a/resolving/npm-resolver/package.json b/resolving/npm-resolver/package.json index 171168b5ab..b73b086008 100644 --- a/resolving/npm-resolver/package.json +++ b/resolving/npm-resolver/package.json @@ -38,6 +38,7 @@ "@pnpm/crypto.hash": "workspace:*", "@pnpm/error": "workspace:*", "@pnpm/fetching-types": "workspace:*", + "@pnpm/fs.v8-file": "workspace:*", "@pnpm/graceful-fs": "workspace:*", "@pnpm/pick-registry-for-package": "workspace:*", "@pnpm/registry.pkg-metadata-filter": "workspace:*", @@ -49,7 +50,6 @@ "@pnpm/workspace.spec-parser": "workspace:*", "@zkochan/retry": "catalog:", "encode-registry": "catalog:", - "load-json-file": "catalog:", "lru-cache": "catalog:", "normalize-path": "catalog:", "p-limit": "catalog:", @@ -75,6 +75,7 @@ "@types/ramda": "catalog:", "@types/semver": "catalog:", "@types/ssri": "catalog:", + "load-json-file": "catalog:", "nock": "catalog:", "tempy": "catalog:" }, diff --git a/resolving/npm-resolver/src/pickPackage.ts b/resolving/npm-resolver/src/pickPackage.ts index eabf1efee2..9fc0cf8480 100644 --- a/resolving/npm-resolver/src/pickPackage.ts +++ b/resolving/npm-resolver/src/pickPackage.ts @@ -1,12 +1,13 @@ +import v8 from 'v8' import { promises as fs } from 'fs' import path from 'path' import { createHexHash } from '@pnpm/crypto.hash' import { PnpmError } from '@pnpm/error' import { logger } from '@pnpm/logger' +import { readV8FileStrictAsync } from '@pnpm/fs.v8-file' import gfs from '@pnpm/graceful-fs' import { type PackageMeta, type PackageInRegistry } from '@pnpm/registry.types' import getRegistryName from 'encode-registry' -import { loadJsonFile } from 'load-json-file' import pLimit, { type LimitFunction } from 'p-limit' import { fastPathTemp as pathTemp } from 'path-temp' import { pick } from 'ramda' @@ -134,7 +135,7 @@ export async function pickPackage ( } const registryName = getRegistryName(opts.registry) - const pkgMirror = path.join(ctx.cacheDir, ctx.metaDir, registryName, `${encodePkgName(spec.name)}.json`) + const pkgMirror = path.join(ctx.cacheDir, ctx.metaDir, registryName, `${encodePkgName(spec.name)}.v8`) return runLimited(pkgMirror, async (limit) => { let metaCachedInStore: PackageMeta | null | undefined @@ -210,7 +211,7 @@ export async function pickPackage ( ctx.metaCache.set(spec.name, meta) if (!opts.dryRun) { // We stringify this meta here to avoid saving any mutations that could happen to the meta object. - const stringifiedMeta = JSON.stringify(meta) + const stringifiedMeta = v8.serialize(meta) // eslint-disable-next-line @typescript-eslint/no-floating-promises runLimited(pkgMirror, (limit) => limit(async () => { try { @@ -283,15 +284,15 @@ function encodePkgName (pkgName: string): string { async function loadMeta (pkgMirror: string): Promise { try { - return await loadJsonFile(pkgMirror) - } catch (err: any) { // eslint-disable-line + return await readV8FileStrictAsync(pkgMirror) + } catch { return null } } const createdDirs = new Set() -async function saveMeta (pkgMirror: string, meta: string): Promise { +async function saveMeta (pkgMirror: string, meta: Buffer): Promise { const dir = path.dirname(pkgMirror) if (!createdDirs.has(dir)) { await fs.mkdir(dir, { recursive: true }) diff --git a/resolving/npm-resolver/test/index.ts b/resolving/npm-resolver/test/index.ts index 240ac22e7d..6415a71c9b 100644 --- a/resolving/npm-resolver/test/index.ts +++ b/resolving/npm-resolver/test/index.ts @@ -73,7 +73,7 @@ test('resolveFromNpm()', async () => { // The resolve function does not wait for the package meta cache file to be saved // so we must delay for a bit in order to read it - const meta = await retryLoadJsonFile(path.join(cacheDir, ABBREVIATED_META_DIR, 'registry.npmjs.org/is-positive.json')) // eslint-disable-line @typescript-eslint/no-explicit-any + const meta = await retryLoadJsonFile(path.join(cacheDir, ABBREVIATED_META_DIR, 'registry.npmjs.org/is-positive.v8')) // eslint-disable-line @typescript-eslint/no-explicit-any expect(meta.name).toBeTruthy() expect(meta.versions).toBeTruthy() expect(meta['dist-tags']).toBeTruthy() @@ -95,7 +95,7 @@ test('resolveFromNpm() does not save mutated meta to the cache', async () => { // The resolve function does not wait for the package meta cache file to be saved // so we must delay for a bit in order to read it - const meta = await retryLoadJsonFile(path.join(cacheDir, ABBREVIATED_META_DIR, 'registry.npmjs.org/is-positive.json')) // eslint-disable-line @typescript-eslint/no-explicit-any + const meta = await retryLoadJsonFile(path.join(cacheDir, ABBREVIATED_META_DIR, 'registry.npmjs.org/is-positive.v8')) // eslint-disable-line @typescript-eslint/no-explicit-any expect(meta.versions['1.0.0'].version).toBe('1.0.0') }) @@ -116,7 +116,7 @@ test('resolveFromNpm() should save metadata to a unique file when the package na // The resolve function does not wait for the package meta cache file to be saved // so we must delay for a bit in order to read it - const meta = await retryLoadJsonFile(path.join(cacheDir, ABBREVIATED_META_DIR, `registry.npmjs.org/JSON_${createHexHash('JSON')}.json`)) // eslint-disable-line @typescript-eslint/no-explicit-any + const meta = await retryLoadJsonFile(path.join(cacheDir, ABBREVIATED_META_DIR, `registry.npmjs.org/JSON_${createHexHash('JSON')}.v8`)) // eslint-disable-line @typescript-eslint/no-explicit-any expect(meta.name).toBeTruthy() expect(meta.versions).toBeTruthy() expect(meta['dist-tags']).toBeTruthy() @@ -642,7 +642,7 @@ test('offline resolution fails when package meta not found in the store', async await expect(resolveFromNpm({ alias: 'is-positive', bareSpecifier: '1.0.0' }, {})).rejects .toThrow( - new PnpmError('NO_OFFLINE_META', `Failed to resolve is-positive@1.0.0 in package mirror ${path.join(cacheDir, ABBREVIATED_META_DIR, 'registry.npmjs.org/is-positive.json')}`) + new PnpmError('NO_OFFLINE_META', `Failed to resolve is-positive@1.0.0 in package mirror ${path.join(cacheDir, ABBREVIATED_META_DIR, 'registry.npmjs.org/is-positive.v8')}`) ) }) @@ -947,7 +947,7 @@ test('resolve when tarball URL is requested from the registry', async () => { // The resolve function does not wait for the package meta cache file to be saved // so we must delay for a bit in order to read it - const meta = await retryLoadJsonFile(path.join(cacheDir, ABBREVIATED_META_DIR, 'registry.npmjs.org/is-positive.json')) // eslint-disable-line @typescript-eslint/no-explicit-any + const meta = await retryLoadJsonFile(path.join(cacheDir, ABBREVIATED_META_DIR, 'registry.npmjs.org/is-positive.v8')) // eslint-disable-line @typescript-eslint/no-explicit-any expect(meta.name).toBeTruthy() expect(meta.versions).toBeTruthy() expect(meta['dist-tags']).toBeTruthy() @@ -979,7 +979,7 @@ test('resolve when tarball URL is requested from the registry and alias is not s // The resolve function does not wait for the package meta cache file to be saved // so we must delay for a bit in order to read it - const meta = await retryLoadJsonFile(path.join(cacheDir, ABBREVIATED_META_DIR, 'registry.npmjs.org/is-positive.json')) // eslint-disable-line @typescript-eslint/no-explicit-any + const meta = await retryLoadJsonFile(path.join(cacheDir, ABBREVIATED_META_DIR, 'registry.npmjs.org/is-positive.v8')) // eslint-disable-line @typescript-eslint/no-explicit-any expect(meta.name).toBeTruthy() expect(meta.versions).toBeTruthy() expect(meta['dist-tags']).toBeTruthy() @@ -1732,7 +1732,7 @@ test('resolveFromNpm() should always return the name of the package that is spec // The resolve function does not wait for the package meta cache file to be saved // so we must delay for a bit in order to read it - const meta = await retryLoadJsonFile(path.join(cacheDir, ABBREVIATED_META_DIR, 'registry.npmjs.org/is-positive.json')) // eslint-disable-line @typescript-eslint/no-explicit-any + const meta = await retryLoadJsonFile(path.join(cacheDir, ABBREVIATED_META_DIR, 'registry.npmjs.org/is-positive.v8')) // eslint-disable-line @typescript-eslint/no-explicit-any expect(meta.name).toBeTruthy() expect(meta.versions).toBeTruthy() expect(meta['dist-tags']).toBeTruthy() diff --git a/resolving/npm-resolver/test/resolveJsr.test.ts b/resolving/npm-resolver/test/resolveJsr.test.ts index cf6bb78ef6..294dafae18 100644 --- a/resolving/npm-resolver/test/resolveJsr.test.ts +++ b/resolving/npm-resolver/test/resolveJsr.test.ts @@ -70,7 +70,7 @@ test('resolveFromJsr() on jsr', async () => { // The resolve function does not wait for the package meta cache file to be saved // so we must delay for a bit in order to read it - const meta = await retryLoadJsonFile(path.join(cacheDir, ABBREVIATED_META_DIR, 'npm.jsr.io/@jsr/rus__greet.json')) // eslint-disable-line @typescript-eslint/no-explicit-any + const meta = await retryLoadJsonFile(path.join(cacheDir, ABBREVIATED_META_DIR, 'npm.jsr.io/@jsr/rus__greet.v8')) // eslint-disable-line @typescript-eslint/no-explicit-any expect(meta).toMatchObject({ name: expect.any(String), versions: expect.any(Object), @@ -114,7 +114,7 @@ test('resolveFromJsr() on jsr with alias renaming', async () => { // The resolve function does not wait for the package meta cache file to be saved // so we must delay for a bit in order to read it - const meta = await retryLoadJsonFile(path.join(cacheDir, ABBREVIATED_META_DIR, 'npm.jsr.io/@jsr/rus__greet.json')) // eslint-disable-line @typescript-eslint/no-explicit-any + const meta = await retryLoadJsonFile(path.join(cacheDir, ABBREVIATED_META_DIR, 'npm.jsr.io/@jsr/rus__greet.v8')) // eslint-disable-line @typescript-eslint/no-explicit-any expect(meta).toMatchObject({ name: expect.any(String), versions: expect.any(Object), diff --git a/resolving/npm-resolver/test/utils/index.ts b/resolving/npm-resolver/test/utils/index.ts index c038aebcf4..7737782051 100644 --- a/resolving/npm-resolver/test/utils/index.ts +++ b/resolving/npm-resolver/test/utils/index.ts @@ -1,4 +1,4 @@ -import { loadJsonFile } from 'load-json-file' +import { readV8FileStrictAsync } from '@pnpm/fs.v8-file' export async function retryLoadJsonFile (filePath: string): Promise { let retry = 0 @@ -6,7 +6,7 @@ export async function retryLoadJsonFile (filePath: string): Promise { while (true) { await delay(500) try { - return await loadJsonFile(filePath) + return await readV8FileStrictAsync(filePath) } catch (err: any) { // eslint-disable-line if (retry > 2) throw err retry++ diff --git a/resolving/npm-resolver/tsconfig.json b/resolving/npm-resolver/tsconfig.json index e9d64d071d..0f7a43d6ef 100644 --- a/resolving/npm-resolver/tsconfig.json +++ b/resolving/npm-resolver/tsconfig.json @@ -21,6 +21,9 @@ { "path": "../../fs/graceful-fs" }, + { + "path": "../../fs/v8-file" + }, { "path": "../../network/fetch" }, diff --git a/reviewing/license-scanner/package.json b/reviewing/license-scanner/package.json index 69dcc8e34c..0e1b911b3e 100644 --- a/reviewing/license-scanner/package.json +++ b/reviewing/license-scanner/package.json @@ -35,6 +35,7 @@ "@pnpm/dependency-path": "workspace:*", "@pnpm/directory-fetcher": "workspace:*", "@pnpm/error": "workspace:*", + "@pnpm/fs.v8-file": "workspace:*", "@pnpm/lockfile.detect-dep-types": "workspace:*", "@pnpm/lockfile.fs": "workspace:*", "@pnpm/lockfile.types": "workspace:*", @@ -44,7 +45,6 @@ "@pnpm/read-package-json": "workspace:*", "@pnpm/store.cafs": "workspace:*", "@pnpm/types": "workspace:*", - "load-json-file": "catalog:", "p-limit": "catalog:", "path-absolute": "catalog:", "ramda": "catalog:", diff --git a/reviewing/license-scanner/src/getPkgInfo.ts b/reviewing/license-scanner/src/getPkgInfo.ts index 74bf4de53a..5ad84ddc1e 100644 --- a/reviewing/license-scanner/src/getPkgInfo.ts +++ b/reviewing/license-scanner/src/getPkgInfo.ts @@ -12,11 +12,11 @@ import { type PackageFileInfo, type PackageFilesIndex, } from '@pnpm/store.cafs' -import { loadJsonFile } from 'load-json-file' import { PnpmError } from '@pnpm/error' import { type LicensePackage } from './licenses.js' import { type DirectoryResolution, type PackageSnapshot, pkgSnapshotToResolution, type Resolution } from '@pnpm/lockfile.utils' import { fetchFromDir } from '@pnpm/directory-fetcher' +import { readV8FileStrictAsync } from '@pnpm/fs.v8-file' const limitPkgReads = pLimit(4) @@ -276,7 +276,7 @@ export async function readPackageIndexFile ( } try { - const { files } = await loadJsonFile(pkgIndexFilePath) + const { files } = await readV8FileStrictAsync(pkgIndexFilePath) return { local: false, files, diff --git a/reviewing/license-scanner/test/getPkgInfo.spec.ts b/reviewing/license-scanner/test/getPkgInfo.spec.ts index c8d9c870f0..8bebb3b34b 100644 --- a/reviewing/license-scanner/test/getPkgInfo.spec.ts +++ b/reviewing/license-scanner/test/getPkgInfo.spec.ts @@ -30,6 +30,6 @@ describe('licences', () => { virtualStoreDirMaxLength: 120, } ) - ).rejects.toThrow(`Failed to find package index file for bogus-package@1.0.0 (at ${path.join('store-dir', 'index', 'b2', '16-bogus-package@1.0.0.json')}), please consider running 'pnpm install'`) + ).rejects.toThrow(`Failed to find package index file for bogus-package@1.0.0 (at ${path.join('store-dir', 'index', 'b2', '16-bogus-package@1.0.0.v8')}), please consider running 'pnpm install'`) }) }) diff --git a/reviewing/license-scanner/tsconfig.json b/reviewing/license-scanner/tsconfig.json index 8f33d9aec4..eeba0492cf 100644 --- a/reviewing/license-scanner/tsconfig.json +++ b/reviewing/license-scanner/tsconfig.json @@ -15,6 +15,9 @@ { "path": "../../fetching/directory-fetcher" }, + { + "path": "../../fs/v8-file" + }, { "path": "../../lockfile/detect-dep-types" }, diff --git a/store/cafs/src/getFilePathInCafs.ts b/store/cafs/src/getFilePathInCafs.ts index 31f5249c58..b327aecaea 100644 --- a/store/cafs/src/getFilePathInCafs.ts +++ b/store/cafs/src/getFilePathInCafs.ts @@ -39,7 +39,7 @@ export function getIndexFilePathInCafs ( // 1. Validate that the integrity in the lockfile corresponds to the correct package, // which might not be the case after a poorly resolved Git conflict. // 2. Allow the same content to be referenced by different packages or different versions of the same package. - return path.join(storeDir, `index/${path.join(hex.slice(0, 2), hex.slice(2))}-${pkgId.replace(/[\\/:*?"<>|]/g, '+')}.json`) + return path.join(storeDir, `index/${path.join(hex.slice(0, 2), hex.slice(2))}-${pkgId.replace(/[\\/:*?"<>|]/g, '+')}.v8`) } function contentPathFromIntegrity ( @@ -58,6 +58,6 @@ export function contentPathFromHex (fileType: FileType, hex: string): string { case 'nonexec': return p case 'index': - return `${p}-index.json` + return `${p}-index.v8` } } diff --git a/store/package-store/package.json b/store/package-store/package.json index 7af96ee075..8935ab723f 100644 --- a/store/package-store/package.json +++ b/store/package-store/package.json @@ -46,13 +46,13 @@ "dependencies": { "@pnpm/create-cafs-store": "workspace:*", "@pnpm/fetcher-base": "workspace:*", + "@pnpm/fs.v8-file": "workspace:*", "@pnpm/package-requester": "workspace:*", "@pnpm/resolver-base": "workspace:*", "@pnpm/store-controller-types": "workspace:*", "@pnpm/store.cafs": "workspace:*", "@pnpm/types": "workspace:*", "@zkochan/rimraf": "catalog:", - "load-json-file": "catalog:", "ramda": "catalog:", "ssri": "catalog:" }, diff --git a/store/package-store/src/storeController/prune.ts b/store/package-store/src/storeController/prune.ts index e7c61effc7..5dd36e1177 100644 --- a/store/package-store/src/storeController/prune.ts +++ b/store/package-store/src/storeController/prune.ts @@ -1,10 +1,10 @@ import { type Dirent, promises as fs } from 'fs' import util from 'util' import path from 'path' +import { readV8FileStrictAsync } from '@pnpm/fs.v8-file' import { type PackageFilesIndex } from '@pnpm/store.cafs' import { globalInfo, globalWarn } from '@pnpm/logger' import rimraf from '@zkochan/rimraf' -import { loadJsonFile } from 'load-json-file' import ssri from 'ssri' const BIG_ONE = BigInt(1) as unknown @@ -35,7 +35,7 @@ export async function prune ({ cacheDir, storeDir }: PruneOptions, removeAlienFi const subdir = path.join(indexDir, dir) await Promise.all((await fs.readdir(subdir)).map(async (fileName) => { const filePath = path.join(subdir, fileName) - if (fileName.endsWith('.json')) { + if (fileName.endsWith('.v8')) { pkgIndexFiles.push(filePath) } })) @@ -47,7 +47,7 @@ export async function prune ({ cacheDir, storeDir }: PruneOptions, removeAlienFi const subdir = path.join(cafsDir, dir) await Promise.all((await fs.readdir(subdir)).map(async (fileName) => { const filePath = path.join(subdir, fileName) - if (fileName.endsWith('.json')) { + if (fileName.endsWith('.v8')) { pkgIndexFiles.push(filePath) return } @@ -74,7 +74,7 @@ export async function prune ({ cacheDir, storeDir }: PruneOptions, removeAlienFi let pkgCounter = 0 await Promise.all(pkgIndexFiles.map(async (pkgIndexFilePath) => { - const { files: pkgFilesIndex } = await loadJsonFile(pkgIndexFilePath) + const { files: pkgFilesIndex } = await readV8FileStrictAsync(pkgIndexFilePath) if (removedHashes.has(pkgFilesIndex['package.json'].integrity)) { await fs.unlink(pkgIndexFilePath) pkgCounter++ diff --git a/store/package-store/tsconfig.json b/store/package-store/tsconfig.json index 2d8d1656c3..625ebb41ad 100644 --- a/store/package-store/tsconfig.json +++ b/store/package-store/tsconfig.json @@ -15,6 +15,9 @@ { "path": "../../fetching/fetcher-base" }, + { + "path": "../../fs/v8-file" + }, { "path": "../../packages/logger" }, diff --git a/store/plugin-commands-store-inspecting/package.json b/store/plugin-commands-store-inspecting/package.json index 7409e96ce6..8ca637a98b 100644 --- a/store/plugin-commands-store-inspecting/package.json +++ b/store/plugin-commands-store-inspecting/package.json @@ -34,6 +34,7 @@ "@pnpm/client": "workspace:*", "@pnpm/config": "workspace:*", "@pnpm/error": "workspace:*", + "@pnpm/fs.v8-file": "workspace:*", "@pnpm/graceful-fs": "workspace:*", "@pnpm/lockfile.types": "workspace:*", "@pnpm/object.key-sorting": "workspace:*", @@ -42,7 +43,6 @@ "@pnpm/store.cafs": "workspace:*", "@pnpm/types": "workspace:*", "chalk": "catalog:", - "load-json-file": "catalog:", "render-help": "catalog:" }, "devDependencies": { diff --git a/store/plugin-commands-store-inspecting/src/catIndex.ts b/store/plugin-commands-store-inspecting/src/catIndex.ts index 31d91873b9..4ec5bcf517 100644 --- a/store/plugin-commands-store-inspecting/src/catIndex.ts +++ b/store/plugin-commands-store-inspecting/src/catIndex.ts @@ -3,12 +3,12 @@ import { createResolver } from '@pnpm/client' import { type TarballResolution } from '@pnpm/lockfile.types' import { PnpmError } from '@pnpm/error' +import { readV8FileStrictAsync } from '@pnpm/fs.v8-file' import { sortDeepKeys } from '@pnpm/object.key-sorting' import { getStorePath } from '@pnpm/store-path' import { getIndexFilePathInCafs, type PackageFilesIndex } from '@pnpm/store.cafs' import { parseWantedDependency } from '@pnpm/parse-wanted-dependency' -import { loadJsonFile } from 'load-json-file' import renderHelp from 'render-help' export const skipPackageManagerCheck = true @@ -86,7 +86,7 @@ export async function handler (opts: CatIndexCommandOptions, params: string[]): `${alias}@${bareSpecifier}` ) try { - const pkgFilesIndex = await loadJsonFile(filesIndexFile) + const pkgFilesIndex = await readV8FileStrictAsync(filesIndexFile) return JSON.stringify(sortDeepKeys(pkgFilesIndex), null, 2) } catch { throw new PnpmError( diff --git a/store/plugin-commands-store-inspecting/src/findHash.ts b/store/plugin-commands-store-inspecting/src/findHash.ts index a47f07dce5..20bef24bee 100644 --- a/store/plugin-commands-store-inspecting/src/findHash.ts +++ b/store/plugin-commands-store-inspecting/src/findHash.ts @@ -4,10 +4,10 @@ import chalk from 'chalk' import { type Config } from '@pnpm/config' import { PnpmError } from '@pnpm/error' +import { safeReadV8FileSync } from '@pnpm/fs.v8-file' import { getStorePath } from '@pnpm/store-path' import { type PackageFilesIndex } from '@pnpm/store.cafs' -import { loadJsonFileSync } from 'load-json-file' import renderHelp from 'render-help' export const PACKAGE_INFO_CLR = chalk.greenBright @@ -57,14 +57,17 @@ export async function handler (opts: FindHashCommandOptions, params: string[]): for (const { name: dirName } of cafsChildrenDirs) { const dirIndexFiles = fs .readdirSync(`${indexDir}/${dirName}`) - .filter((fileName) => fileName.includes('.json')) + .filter((fileName) => fileName.includes('.v8')) ?.map((fileName) => `${indexDir}/${dirName}/${fileName}`) indexFiles.push(...dirIndexFiles) } for (const filesIndexFile of indexFiles) { - const pkgFilesIndex = loadJsonFileSync(filesIndexFile) + const pkgFilesIndex = safeReadV8FileSync(filesIndexFile) + if (!pkgFilesIndex) { + continue + } for (const [, file] of Object.entries(pkgFilesIndex.files)) { if (file?.integrity === hash) { diff --git a/store/plugin-commands-store-inspecting/test/findHash.ts b/store/plugin-commands-store-inspecting/test/findHash.ts index fcf277e66d..5a490df1a8 100644 --- a/store/plugin-commands-store-inspecting/test/findHash.ts +++ b/store/plugin-commands-store-inspecting/test/findHash.ts @@ -26,8 +26,8 @@ test('print index file path with hash', async () => { storeDir, }, ['sha512-fXs1pWlUdqT2jkeoEJW/+odKZ2NwAyYkWea+plJKZI2xmhRKQi2e+nKGcClyDblgLwCLD912oMaua0+sTwwIrw==']) - expect(output).toBe(`${PACKAGE_INFO_CLR('lodash')}@${PACKAGE_INFO_CLR('4.17.19')} ${INDEX_PATH_CLR('/24/dbddf17111f46417d2fdaa260b1a37f9b3142340e4145efe3f0937d77eb56c-lodash@4.17.19.json')} -${PACKAGE_INFO_CLR('lodash')}@${PACKAGE_INFO_CLR('4.17.20')} ${INDEX_PATH_CLR('/3e/585d15c8a594e20d7de57b362ea81754c011acb2641a19f1b72c8531ea3982-lodash@4.17.20.json')} + expect(output).toBe(`${PACKAGE_INFO_CLR('lodash')}@${PACKAGE_INFO_CLR('4.17.19')} ${INDEX_PATH_CLR('/24/dbddf17111f46417d2fdaa260b1a37f9b3142340e4145efe3f0937d77eb56c-lodash@4.17.19.v8')} +${PACKAGE_INFO_CLR('lodash')}@${PACKAGE_INFO_CLR('4.17.20')} ${INDEX_PATH_CLR('/3e/585d15c8a594e20d7de57b362ea81754c011acb2641a19f1b72c8531ea3982-lodash@4.17.20.v8')} `) } }) @@ -43,10 +43,10 @@ test('print index file path with hash error', async () => { }, }) await findHash.handler(config as findHash.FindHashCommandOptions, ['sha512-fXs1pWlUdqT2j']) - } catch (_err: any) { // eslint-disable-line + } catch (_err: any) { // eslint-disable-line err = _err } - expect(err.code).toBe('ERR_PNPM_INVALID_FILE_HASH') expect(err.message).toBe('No package or index file matching this hash was found.') + expect(err.code).toBe('ERR_PNPM_INVALID_FILE_HASH') }) diff --git a/store/plugin-commands-store-inspecting/tsconfig.json b/store/plugin-commands-store-inspecting/tsconfig.json index 25eb2cba22..c4c212fa65 100644 --- a/store/plugin-commands-store-inspecting/tsconfig.json +++ b/store/plugin-commands-store-inspecting/tsconfig.json @@ -18,6 +18,9 @@ { "path": "../../fs/graceful-fs" }, + { + "path": "../../fs/v8-file" + }, { "path": "../../lockfile/types" }, diff --git a/store/plugin-commands-store/package.json b/store/plugin-commands-store/package.json index e282187359..61c4a9ebd8 100644 --- a/store/plugin-commands-store/package.json +++ b/store/plugin-commands-store/package.json @@ -36,6 +36,7 @@ "@pnpm/config": "workspace:*", "@pnpm/dependency-path": "workspace:*", "@pnpm/error": "workspace:*", + "@pnpm/fs.v8-file": "workspace:*", "@pnpm/get-context": "workspace:*", "@pnpm/lockfile.utils": "workspace:*", "@pnpm/normalize-registries": "workspace:*", @@ -47,7 +48,6 @@ "@pnpm/types": "workspace:*", "archy": "catalog:", "dint": "catalog:", - "load-json-file": "catalog:", "p-filter": "catalog:", "ramda": "catalog:", "render-help": "catalog:" diff --git a/store/plugin-commands-store/src/storeStatus/index.ts b/store/plugin-commands-store/src/storeStatus/index.ts index 39bb041974..50d93f18b9 100644 --- a/store/plugin-commands-store/src/storeStatus/index.ts +++ b/store/plugin-commands-store/src/storeStatus/index.ts @@ -1,4 +1,5 @@ import path from 'path' +import { readV8FileStrictAsync } from '@pnpm/fs.v8-file' import { getIndexFilePathInCafs, type PackageFilesIndex } from '@pnpm/store.cafs' import { getContextForSingleImporter } from '@pnpm/get-context' import { @@ -10,7 +11,6 @@ import { streamParser } from '@pnpm/logger' import * as dp from '@pnpm/dependency-path' import { type DepPath } from '@pnpm/types' import dint from 'dint' -import { loadJsonFile } from 'load-json-file' import pFilter from 'p-filter' import { extendStoreStatusOptions, @@ -52,7 +52,7 @@ export async function storeStatus (maybeOpts: StoreStatusOptions): Promise(pkgIndexFilePath) + const { files } = await readV8FileStrictAsync(pkgIndexFilePath) return (await dint.check(path.join(virtualStoreDir, dp.depPathToFilename(depPath, maybeOpts.virtualStoreDirMaxLength), 'node_modules', name), files)) === false }, { concurrency: 8 }) diff --git a/store/plugin-commands-store/tsconfig.json b/store/plugin-commands-store/tsconfig.json index e59e162869..182cb1eebb 100644 --- a/store/plugin-commands-store/tsconfig.json +++ b/store/plugin-commands-store/tsconfig.json @@ -27,6 +27,9 @@ { "path": "../../exec/plugin-commands-script-runners" }, + { + "path": "../../fs/v8-file" + }, { "path": "../../lockfile/fs" }, diff --git a/store/server/package.json b/store/server/package.json index f88c469ad9..11f8983956 100644 --- a/store/server/package.json +++ b/store/server/package.json @@ -44,15 +44,16 @@ }, "devDependencies": { "@pnpm/client": "workspace:*", + "@pnpm/fs.v8-file": "workspace:*", "@pnpm/logger": "workspace:*", "@pnpm/package-requester": "workspace:*", "@pnpm/package-store": "workspace:*", "@pnpm/server": "workspace:*", + "@pnpm/store.cafs": "workspace:*", "@types/uuid": "catalog:", "@zkochan/rimraf": "catalog:", "get-port": "catalog:", "is-port-reachable": "catalog:", - "load-json-file": "catalog:", "node-fetch": "catalog:", "tempy": "catalog:" }, diff --git a/store/server/test/index.ts b/store/server/test/index.ts index 2a64daa443..19b38ddb4e 100644 --- a/store/server/test/index.ts +++ b/store/server/test/index.ts @@ -1,14 +1,16 @@ /// import fs from 'fs' import path from 'path' +import v8 from 'v8' import getPort from 'get-port' import { createClient } from '@pnpm/client' +import { readV8FileStrictSync } from '@pnpm/fs.v8-file' +import { type PackageFilesIndex } from '@pnpm/store.cafs' import { createPackageStore } from '@pnpm/package-store' import { connectStoreController, createServer } from '@pnpm/server' import { type Registries } from '@pnpm/types' import fetch from 'node-fetch' import { sync as rimraf } from '@zkochan/rimraf' -import { loadJsonFileSync } from 'load-json-file' import { temporaryDirectory } from 'tempy' import isPortReachable from 'is-port-reachable' @@ -172,19 +174,19 @@ test('server upload', async () => { const fakeEngine = 'client-engine' const filesIndexFile = path.join(storeDir, 'fake-pkg@1.0.0.json') - fs.writeFileSync(filesIndexFile, JSON.stringify({ + fs.writeFileSync(filesIndexFile, v8.serialize({ name: 'fake-pkg', version: '1.0.0', files: {}, - }), 'utf8') + })) await storeCtrl.upload(path.join(import.meta.dirname, '__fixtures__/side-effect-fake-dir'), { sideEffectsCacheKey: fakeEngine, filesIndexFile, }) - const cacheIntegrity = loadJsonFileSync(filesIndexFile) // eslint-disable-line @typescript-eslint/no-explicit-any - expect(Object.keys(cacheIntegrity?.['sideEffects'][fakeEngine].added).sort()).toStrictEqual(['side-effect.js', 'side-effect.txt']) + const cacheIntegrity = readV8FileStrictSync(filesIndexFile) + expect(Object.keys(cacheIntegrity.sideEffects![fakeEngine].added!).sort()).toStrictEqual(['side-effect.js', 'side-effect.txt']) await server.close() await storeCtrl.close() diff --git a/store/server/tsconfig.json b/store/server/tsconfig.json index 726082d066..5518f3e5a3 100644 --- a/store/server/tsconfig.json +++ b/store/server/tsconfig.json @@ -9,6 +9,9 @@ "../../__typings__/**/*.d.ts" ], "references": [ + { + "path": "../../fs/v8-file" + }, { "path": "../../network/fetch" }, @@ -24,6 +27,9 @@ { "path": "../../pkg-manager/package-requester" }, + { + "path": "../cafs" + }, { "path": "../package-store" }, diff --git a/worker/package.json b/worker/package.json index cb24e91943..0ea7b3a888 100644 --- a/worker/package.json +++ b/worker/package.json @@ -37,12 +37,12 @@ "@pnpm/error": "workspace:*", "@pnpm/exec.pkg-requires-build": "workspace:*", "@pnpm/fs.hard-link-dir": "workspace:*", + "@pnpm/fs.v8-file": "workspace:*", "@pnpm/graceful-fs": "workspace:*", "@pnpm/store.cafs": "workspace:*", "@pnpm/symlink-dependency": "workspace:*", "@rushstack/worker-pool": "catalog:", "is-windows": "catalog:", - "load-json-file": "catalog:", "p-limit": "catalog:", "shlex": "catalog:" }, diff --git a/worker/src/start.ts b/worker/src/start.ts index c5ee0842c7..7e38875c64 100644 --- a/worker/src/start.ts +++ b/worker/src/start.ts @@ -1,6 +1,8 @@ import crypto from 'crypto' +import v8 from 'v8' import path from 'path' import fs from 'fs' +import { readV8FileStrictSync } from '@pnpm/fs.v8-file' import gfs from '@pnpm/graceful-fs' import { type Cafs, type PackageFiles, type SideEffects, type SideEffectsDiff } from '@pnpm/cafs-types' import { createCafsStore } from '@pnpm/create-cafs-store' @@ -18,7 +20,6 @@ import { } from '@pnpm/store.cafs' import { symlinkDependencySync } from '@pnpm/symlink-dependency' import { type DependencyManifest } from '@pnpm/types' -import { loadJsonFileSync } from 'load-json-file' import { parentPort } from 'worker_threads' import { type AddDirToStoreMessage, @@ -80,7 +81,7 @@ async function handleMessage ( let { storeDir, filesIndexFile, readManifest, verifyStoreIntegrity } = message let pkgFilesIndex: PackageFilesIndex | undefined try { - pkgFilesIndex = loadJsonFileSync(filesIndexFile) + pkgFilesIndex = readV8FileStrictSync(filesIndexFile) } catch { // ignoring. It is fine if the integrity file is not present. Just refetch the package } @@ -211,7 +212,7 @@ function addFilesFromDir ({ dir, storeDir, filesIndexFile, sideEffectsCacheKey, if (sideEffectsCacheKey) { let filesIndex!: PackageFilesIndex try { - filesIndex = loadJsonFileSync(filesIndexFile) + filesIndex = readV8FileStrictSync(filesIndexFile) } catch { // If there is no existing index file, then we cannot store the side effects. return { @@ -230,7 +231,7 @@ function addFilesFromDir ({ dir, storeDir, filesIndexFile, sideEffectsCacheKey, } else { requiresBuild = filesIndex.requiresBuild } - writeJsonFile(filesIndexFile, filesIndex) + writeV8File(filesIndexFile, filesIndex) } else { requiresBuild = writeFilesIndexFile(filesIndexFile, { manifest: manifest ?? {}, files: filesIntegrity }) } @@ -343,19 +344,19 @@ function writeFilesIndexFile ( files, sideEffects, } - writeJsonFile(filesIndexFile, filesIndex) + writeV8File(filesIndexFile, filesIndex) return requiresBuild } -function writeJsonFile (filePath: string, data: unknown): void { +function writeV8File (filePath: string, data: unknown): void { const targetDir = path.dirname(filePath) // TODO: use the API of @pnpm/cafs to write this file // There is actually no need to create the directory in 99% of cases. // So by using cafs API, we'll improve performance. fs.mkdirSync(targetDir, { recursive: true }) - // We remove the "-index.json" from the end of the temp file name + // We remove the "-index.v8" from the end of the temp file name // in order to avoid ENAMETOOLONG errors - const temp = `${filePath.slice(0, -11)}${process.pid}` - gfs.writeFileSync(temp, JSON.stringify(data)) + const temp = `${filePath.slice(0, -9)}${process.pid}` + gfs.writeFileSync(temp, v8.serialize(data)) optimisticRenameOverwrite(temp, filePath) } diff --git a/worker/tsconfig.json b/worker/tsconfig.json index 03da43fc34..356c1446ee 100644 --- a/worker/tsconfig.json +++ b/worker/tsconfig.json @@ -21,6 +21,9 @@ { "path": "../fs/symlink-dependency" }, + { + "path": "../fs/v8-file" + }, { "path": "../packages/error" },