mirror/pnpm
1
0
Fork 0
mirror of https://github.com/pnpm/pnpm.git synced 2025-12-23 23:29:17 -05:00
Code Issues Packages Projects Releases Wiki Activity

perf: use v8 serialize/deserialize instead of JSON (#9971)

Browse Source 
close #9965
This commit is contained in:
Zoltan Kochan
2025-11-06 01:01:06 +01:00
committed by GitHub
parent d392c3d059
commit 9b344c8982
63 changed files with 384 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
.changeset/ripe-friends-dream.md Normal file
View File

@@ -0,0 +1,19 @@
---
"@pnpm/plugin-commands-store-inspecting": major
"@pnpm/package-requester": major
"@pnpm/plugin-commands-rebuild": major
"@pnpm/plugin-commands-store": major
"@pnpm/license-scanner": major
"@pnpm/mount-modules": major
"@pnpm/npm-resolver": major
"@pnpm/headless": major
"@pnpm/package-store": major
"@pnpm/core": major
"@pnpm/cache.commands": major
"@pnpm/server": major
"@pnpm/store.cafs": major
"@pnpm/cache.api": major
"@pnpm/worker": major
---
Switched to from .json file to .v8 files in the global store and cache.

5
.changeset/short-cougars-arrive.md Normal file
View File

@@ -0,0 +1,5 @@
---
"@pnpm/fs.v8-file": major
---
Initial release.

9
.changeset/young-points-dress.md Normal file
View File

@@ -0,0 +1,9 @@
---
"pnpm": major
---
JSON files in the store and cache are replaced with binary files generated with v8.serialize.
Reading and deserializing a binary file generated with v8.serialize appears to be about 23% faster than reading and parsing a JSON file. Other operations are similar in speed. Therefore, we have switched to V8 binary files for storing data in pnpm’s global store and cache. The disadvantage of this approach is that V8 files generated by newer versions of Node.js cannot be deserialized by older versions. In such cases, we ignore the cache.
Related PR: [#9971](https://github.com/pnpm/pnpm/pull/9971).

1
cache/api/package.json vendored
View File

@@ -33,6 +33,7 @@
"dependencies": {
"@pnpm/config": "workspace:*",
"@pnpm/constants": "workspace:*",
"@pnpm/fs.v8-file": "workspace:*",
"@pnpm/npm-resolver": "workspace:*",
"@pnpm/store.cafs": "workspace:*",
"encode-registry": "catalog:",

2
cache/api/src/cacheList.ts vendored
View File

@@ -13,7 +13,7 @@ export async function cacheList (opts: { cacheDir: string, registry?: string, re
export async function findMetadataFiles (opts: { cacheDir: string, registry?: string }, filter: string[]): Promise<string[]> {
const prefix = opts.registry ? `${getRegistryName(opts.registry)}` : '*'
const patterns = filter.length ? filter.map((filter) => `${prefix}/${filter}.json`) : [`${prefix}/**`]
const patterns = filter.length ? filter.map((filter) => `${prefix}/${filter}.v8`) : [`${prefix}/**`]
const metaFiles = await glob(patterns, {
cwd: opts.cacheDir,
expandDirectories: false,

6
cache/api/src/cacheView.ts vendored
View File

@@ -1,6 +1,7 @@
import fs from 'fs'
import path from 'path'
import { glob } from 'tinyglobby'
import { safeReadV8FileSync } from '@pnpm/fs.v8-file'
import { getIndexFilePathInCafs } from '@pnpm/store.cafs'
import { type PackageMeta } from '@pnpm/npm-resolver'
import getRegistryName from 'encode-registry'
@@ -14,13 +15,14 @@ interface CachedVersions {
export async function cacheView (opts: { cacheDir: string, storeDir: string, registry?: string }, packageName: string): Promise<string> {
const prefix = opts.registry ? `${getRegistryName(opts.registry)}` : '*'
const metaFilePaths = (await glob(`${prefix}/${packageName}.json`, {
const metaFilePaths = (await glob(`${prefix}/${packageName}.v8`, {
cwd: opts.cacheDir,
expandDirectories: false,
})).sort()
const metaFilesByPath: Record<string, CachedVersions> = {}
for (const filePath of metaFilePaths) {
const metaObject = JSON.parse(fs.readFileSync(path.join(opts.cacheDir, filePath), 'utf8')) as PackageMeta
const metaObject = safeReadV8FileSync<PackageMeta>(path.join(opts.cacheDir, filePath))
if (!metaObject) continue
const cachedVersions: string[] = []
const nonCachedVersions: string[] = []
for (const [version, manifest] of Object.entries(metaObject.versions)) {

3
cache/api/tsconfig.json vendored
View File

@@ -12,6 +12,9 @@
{
"path": "../../config/config"
},
{
"path": "../../fs/v8-file"
},
{
"path": "../../packages/constants"
},

4
cache/commands/test/cacheDelete.cmd.test.ts vendored
View File

@@ -49,7 +49,7 @@ describe('cache delete', () => {
pnpmHomeDir: storeDir,
}, ['list'])
expect(result).toBe(`localhost+${REGISTRY_MOCK_PORT}/is-negative.json
registry.npmjs.org/is-negative.json`)
expect(result).toBe(`localhost+${REGISTRY_MOCK_PORT}/is-negative.v8
registry.npmjs.org/is-negative.v8`)
})
})

12
cache/commands/test/cacheList.cmd.test.ts vendored
View File

@@ -44,9 +44,9 @@ describe('cache', () => {
pnpmHomeDir: storeDir,
}, ['list'])
expect(result).toBe(`localhost+${REGISTRY_MOCK_PORT}/is-negative.json
registry.npmjs.org/is-negative.json
registry.npmjs.org/is-positive.json`)
expect(result).toBe(`localhost+${REGISTRY_MOCK_PORT}/is-negative.v8
registry.npmjs.org/is-negative.v8
registry.npmjs.org/is-positive.v8`)
})
test('list all metadata from the cache related to the specified registry', async () => {
const result = await cache.handler({
@@ -57,8 +57,8 @@ registry.npmjs.org/is-positive.json`)
pnpmHomeDir: storeDir,
}, ['list'])
expect(result).toBe(`registry.npmjs.org/is-negative.json
registry.npmjs.org/is-positive.json`)
expect(result).toBe(`registry.npmjs.org/is-negative.v8
registry.npmjs.org/is-positive.v8`)
})
test('list all metadata from the cache that matches a pattern', async () => {
const result = await cache.handler({
@@ -67,7 +67,7 @@ registry.npmjs.org/is-positive.json`)
pnpmHomeDir: storeDir,
}, ['list', '*-positive'])
expect(result).toBe('registry.npmjs.org/is-positive.json')
expect(result).toBe('registry.npmjs.org/is-positive.v8')
})
test('list registries', async () => {
const result = await cache.handler({

1
exec/plugin-commands-rebuild/package.json
View File

@@ -74,6 +74,7 @@
"devDependencies": {
"@pnpm/assert-project": "workspace:*",
"@pnpm/crypto.object-hasher": "workspace:*",
"@pnpm/fs.v8-file": "workspace:*",
"@pnpm/logger": "workspace:*",
"@pnpm/plugin-commands-rebuild": "workspace:*",
"@pnpm/prepare": "workspace:*",

25
exec/plugin-commands-rebuild/test/index.ts
View File

@@ -1,7 +1,9 @@
/// <reference path="../../../__typings__/index.d.ts" />
import fs from 'fs'
import path from 'path'
import { getIndexFilePathInCafs } from '@pnpm/store.cafs'
import v8 from 'v8'
import { readV8FileSync } from '@pnpm/fs.v8-file'
import { getIndexFilePathInCafs, type PackageFilesIndex } from '@pnpm/store.cafs'
import { ENGINE_NAME, STORE_VERSION, WANTED_LOCKFILE } from '@pnpm/constants'
import { hashObject } from '@pnpm/crypto.object-hasher'
import { rebuild } from '@pnpm/plugin-commands-rebuild'
@@ -9,7 +11,6 @@ import { prepare } from '@pnpm/prepare'
import { getIntegrity, REGISTRY_MOCK_PORT } from '@pnpm/registry-mock'
import { fixtures } from '@pnpm/test-fixtures'
import execa from 'execa'
import { loadJsonFileSync } from 'load-json-file'
import sinon from 'sinon'
import { DEFAULT_OPTS } from './utils/index.js'
@@ -76,7 +77,7 @@ test('rebuilds dependencies', async () => {
}
const cacheIntegrityPath = getIndexFilePathInCafs(path.join(storeDir, STORE_VERSION), getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0')
const cacheIntegrity = loadJsonFileSync<any>(cacheIntegrityPath) // eslint-disable-line @typescript-eslint/no-explicit-any
const cacheIntegrity = readV8FileSync<PackageFilesIndex>(cacheIntegrityPath)!
expect(cacheIntegrity!.sideEffects).toBeTruthy()
const sideEffectsKey = `${ENGINE_NAME};deps=${hashObject({
id: `@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0:${getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0')}`,
@@ -88,7 +89,7 @@ test('rebuilds dependencies', async () => {
},
})}`
expect(cacheIntegrity).toHaveProperty(['sideEffects', sideEffectsKey, 'added', 'generated-by-postinstall.js'])
delete cacheIntegrity!.sideEffects[sideEffectsKey].added['generated-by-postinstall.js']
delete cacheIntegrity!.sideEffects![sideEffectsKey].added!['generated-by-postinstall.js']
})
test('skipIfHasSideEffectsCache', async () => {
@@ -109,12 +110,20 @@ test('skipIfHasSideEffectsCache', async () => {
])
const cacheIntegrityPath = getIndexFilePathInCafs(path.join(storeDir, STORE_VERSION), getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0')
let cacheIntegrity = loadJsonFileSync<any>(cacheIntegrityPath) // eslint-disable-line @typescript-eslint/no-explicit-any
let cacheIntegrity = readV8FileSync<PackageFilesIndex>(cacheIntegrityPath)!
const sideEffectsKey = `${ENGINE_NAME};deps=${hashObject({ '@pnpm.e2e/hello-world-js-bin@1.0.0': {} })}`
cacheIntegrity.sideEffects = {
[sideEffectsKey]: { added: { foo: 'bar' } },
[sideEffectsKey]: {
added: {
foo: {
integrity: 'bar',
mode: 1,
size: 1,
},
},
},
}
fs.writeFileSync(cacheIntegrityPath, JSON.stringify(cacheIntegrity, null, 2), 'utf8')
fs.writeFileSync(cacheIntegrityPath, v8.serialize(cacheIntegrity))
let modules = project.readModulesManifest()
expect(modules!.pendingBuilds).toStrictEqual([
@@ -136,7 +145,7 @@ test('skipIfHasSideEffectsCache', async () => {
expect(modules).toBeTruthy()
expect(modules!.pendingBuilds).toHaveLength(0)
cacheIntegrity = loadJsonFileSync<any>(cacheIntegrityPath) // eslint-disable-line @typescript-eslint/no-explicit-any
cacheIntegrity = readV8FileSync<PackageFilesIndex>(cacheIntegrityPath)!
expect(cacheIntegrity!.sideEffects).toBeTruthy()
expect(cacheIntegrity).toHaveProperty(['sideEffects', sideEffectsKey, 'added', 'foo'])
})

3
exec/plugin-commands-rebuild/tsconfig.json
View File

@@ -42,6 +42,9 @@
{
"path": "../../deps/graph-sequencer"
},
{
"path": "../../fs/v8-file"
},
{
"path": "../../lockfile/types"
},

17
fs/v8-file/README.md Normal file
View File

@@ -0,0 +1,17 @@
# @pnpm/fs.v8-file
> Reading/writing binary files serialized with v8
[![npm version](https://img.shields.io/npm/v/@pnpm/fs.v8-file)](https://www.npmjs.com/package/@pnpm/fs.v8-file)
Reading and deserializing a binary file generated with v8.serialize appears to be about 23% faster than reading and parsing a JSON file. Other operations are similar in speed. Therefore, we use V8 binary files instead of JSON files to store data in pnpm’s global store and cache. The disadvantage of this approach is that V8 files generated by newer versions of Node.js cannot be deserialized by older versions.
## Installation
```sh
pnpm add @pnpm/fs.v8-file
```
## License
MIT

43
fs/v8-file/package.json Normal file
View File

@@ -0,0 +1,43 @@
{
"name": "@pnpm/fs.v8-file",
"version": "1000.0.0-0",
"description": "Reading/writing binary files serialized with v8",
"keywords": [
"pnpm",
"pnpm10"
],
"license": "MIT",
"funding": "https://opencollective.com/pnpm",
"repository": "https://github.com/pnpm/pnpm/tree/main/fs/v8-file",
"homepage": "https://github.com/pnpm/pnpm/tree/main/fs/v8-file#readme",
"bugs": {
"url": "https://github.com/pnpm/pnpm/issues"
},
"type": "module",
"main": "lib/index.js",
"types": "lib/index.d.ts",
"exports": {
".": "./lib/index.js"
},
"files": [
"lib",
"!*.map"
],
"scripts": {
"lint": "eslint \"src/**/*.ts\"",
"test": "pnpm run compile",
"prepublishOnly": "pnpm run compile",
"compile": "tsc --build && pnpm run lint --fix"
},
"dependencies": {
},
"devDependencies": {
"@pnpm/fs.v8-file": "workspace:*"
},
"engines": {
"node": ">=20.19"
},
"jest": {
"preset": "@pnpm/jest-config"
}
}

31
fs/v8-file/src/index.ts Normal file
View File

@@ -0,0 +1,31 @@
import fs from 'fs'
import v8 from 'v8'
import util from 'util'
export function safeReadV8FileSync <T> (filePath: string): T | undefined {
try {
return readV8FileSync<T>(filePath)
} catch (error) {
if (util.types.isNativeError(error) && 'code' in error && error.code === 'ENOENT') return undefined
throw error
}
}
export function readV8FileSync <T> (filePath: string): T | undefined {
const buffer: Buffer = fs.readFileSync(filePath)
try {
return v8.deserialize(buffer)
} catch {
return undefined
}
}
export function readV8FileStrictSync <T> (filePath: string): T {
const buffer: Buffer = fs.readFileSync(filePath)
return v8.deserialize(buffer)
}
export async function readV8FileStrictAsync <T> (filePath: string): Promise<T> {
const buffer: Buffer = await fs.promises.readFile(filePath)
return v8.deserialize(buffer)
}

12
fs/v8-file/tsconfig.json Normal file
View File

@@ -0,0 +1,12 @@
{
"extends": "@pnpm/tsconfig",
"compilerOptions": {
"outDir": "lib",
"rootDir": "src"
},
"include": [
"src/**/*.ts",
"../../__typings__/**/*.d.ts"
],
"references": []
}

8
fs/v8-file/tsconfig.lint.json Normal file
View File

@@ -0,0 +1,8 @@
{
"extends": "./tsconfig.json",
"include": [
"src/**/*.ts",
"test/**/*.ts",
"../../__typings__/**/*.d.ts"
]
}

2
modules-mounter/daemon/package.json
View File

@@ -38,6 +38,7 @@
"dependencies": {
"@pnpm/config": "workspace:*",
"@pnpm/dependency-path": "workspace:*",
"@pnpm/fs.v8-file": "workspace:*",
"@pnpm/lockfile.fs": "workspace:*",
"@pnpm/lockfile.utils": "workspace:*",
"@pnpm/logger": "workspace:*",
@@ -45,7 +46,6 @@
"@pnpm/store.cafs": "workspace:*",
"@pnpm/types": "workspace:*",
"hyperdrive-schemas": "catalog:",
"load-json-file": "catalog:",
"normalize-path": "catalog:"
},
"peerDependencies": {

4
modules-mounter/daemon/src/createFuseHandlers.ts
View File

@@ -1,5 +1,6 @@
// cspell:ignore ents
import fs from 'fs'
import { readV8FileStrictSync } from '@pnpm/fs.v8-file'
import { getIndexFilePathInCafs, getFilePathByModeInCafs, type PackageFilesIndex } from '@pnpm/store.cafs'
import { type LockfileObject, readWantedLockfile, type PackageSnapshot, type TarballResolution } from '@pnpm/lockfile.fs'
import {
@@ -7,7 +8,6 @@ import {
} from '@pnpm/lockfile.utils'
import { type DepPath } from '@pnpm/types'
import schemas from 'hyperdrive-schemas'
import { loadJsonFileSync } from 'load-json-file'
import Fuse from 'fuse-native'
import * as cafsExplorer from './cafsExplorer.js'
import { makeVirtualNodeModules } from './makeVirtualNodeModules.js'
@@ -185,7 +185,7 @@ export function createFuseHandlersFromLockfile (lockfile: LockfileObject, storeD
pkgSnapshotCache.set(depPath, {
...nameVer,
pkgSnapshot,
index: loadJsonFileSync<PackageFilesIndex>(indexPath), // TODO: maybe make it async?
index: readV8FileStrictSync<PackageFilesIndex>(indexPath), // TODO: maybe make it async?
})
}
return pkgSnapshotCache.get(depPath)

3
modules-mounter/daemon/tsconfig.json
View File

@@ -12,6 +12,9 @@
{
"path": "../../config/config"
},
{
"path": "../../fs/v8-file"
},
{
"path": "../../lockfile/fs"
},

1
pkg-manager/core/package.json
View File

@@ -126,6 +126,7 @@
"@pnpm/assert-project": "workspace:*",
"@pnpm/assert-store": "workspace:*",
"@pnpm/core": "workspace:*",
"@pnpm/fs.v8-file": "workspace:*",
"@pnpm/git-utils": "workspace:*",
"@pnpm/lockfile.types": "workspace:*",
"@pnpm/logger": "workspace:*",

8
pkg-manager/core/test/install/lockfileOnly.ts
View File

@@ -19,9 +19,9 @@ test('install with lockfileOnly = true', async () => {
const { cafsHasNot } = assertStore(opts.storeDir)
cafsHasNot('@pnpm.e2e/pkg-with-1-dep', '100.0.0')
expect(fs.existsSync(path.join(opts.cacheDir, `${ABBREVIATED_META_DIR}/localhost+${REGISTRY_MOCK_PORT}/@pnpm.e2e/pkg-with-1-dep.json`))).toBeTruthy()
expect(fs.existsSync(path.join(opts.cacheDir, `${ABBREVIATED_META_DIR}/localhost+${REGISTRY_MOCK_PORT}/@pnpm.e2e/pkg-with-1-dep.v8`))).toBeTruthy()
cafsHasNot('@pnpm.e2e/dep-of-pkg-with-1-dep', '100.1.0')
expect(fs.existsSync(path.join(opts.cacheDir, `${ABBREVIATED_META_DIR}/localhost+${REGISTRY_MOCK_PORT}/@pnpm.e2e/dep-of-pkg-with-1-dep.json`))).toBeTruthy()
expect(fs.existsSync(path.join(opts.cacheDir, `${ABBREVIATED_META_DIR}/localhost+${REGISTRY_MOCK_PORT}/@pnpm.e2e/dep-of-pkg-with-1-dep.v8`))).toBeTruthy()
project.hasNot('@pnpm.e2e/pkg-with-1-dep')
expect(manifest.dependencies!['@pnpm.e2e/pkg-with-1-dep']).toBeTruthy()
@@ -37,9 +37,9 @@ test('install with lockfileOnly = true', async () => {
await install(manifest, opts)
cafsHasNot('@pnpm.e2e/pkg-with-1-dep', '100.0.0')
expect(fs.existsSync(path.join(opts.cacheDir, `${ABBREVIATED_META_DIR}/localhost+${REGISTRY_MOCK_PORT}/@pnpm.e2e/pkg-with-1-dep.json`))).toBeTruthy()
expect(fs.existsSync(path.join(opts.cacheDir, `${ABBREVIATED_META_DIR}/localhost+${REGISTRY_MOCK_PORT}/@pnpm.e2e/pkg-with-1-dep.v8`))).toBeTruthy()
cafsHasNot('@pnpm.e2e/dep-of-pkg-with-1-dep', '100.1.0')
expect(fs.existsSync(path.join(opts.cacheDir, `${ABBREVIATED_META_DIR}/localhost+${REGISTRY_MOCK_PORT}/@pnpm.e2e/dep-of-pkg-with-1-dep.json`))).toBeTruthy()
expect(fs.existsSync(path.join(opts.cacheDir, `${ABBREVIATED_META_DIR}/localhost+${REGISTRY_MOCK_PORT}/@pnpm.e2e/dep-of-pkg-with-1-dep.v8`))).toBeTruthy()
project.hasNot('@pnpm.e2e/pkg-with-1-dep')
expect(project.readCurrentLockfile()).toBeFalsy()

18
pkg-manager/core/test/install/patch.ts
View File

@@ -5,11 +5,11 @@ import { ENGINE_NAME } from '@pnpm/constants'
import { install } from '@pnpm/core'
import { type IgnoredScriptsLog } from '@pnpm/core-loggers'
import { createHexHashFromFile } from '@pnpm/crypto.hash'
import { readV8FileStrictSync } from '@pnpm/fs.v8-file'
import { prepareEmpty } from '@pnpm/prepare'
import { fixtures } from '@pnpm/test-fixtures'
import { jest } from '@jest/globals'
import { sync as rimraf } from '@zkochan/rimraf'
import { loadJsonFileSync } from 'load-json-file'
import sinon from 'sinon'
import { testDefaults } from '../utils/index.js'
@@ -56,8 +56,8 @@ test('patch package with exact version', async () => {
})
expect(lockfile.snapshots[`is-positive@1.0.0(patch_hash=${patchFileHash})`]).toBeTruthy()
const filesIndexFile = path.join(opts.storeDir, 'index/c7/1ccf199e0fdae37aad13946b937d67bcd35fa111b84d21b3a19439cfdc2812-is-positive@1.0.0.json')
const filesIndex = loadJsonFileSync<PackageFilesIndex>(filesIndexFile)
const filesIndexFile = path.join(opts.storeDir, 'index/c7/1ccf199e0fdae37aad13946b937d67bcd35fa111b84d21b3a19439cfdc2812-is-positive@1.0.0.v8')
const filesIndex = readV8FileStrictSync<PackageFilesIndex>(filesIndexFile)
const sideEffectsKey = `${ENGINE_NAME};patch=${patchFileHash}`
const patchedFileIntegrity = filesIndex.sideEffects?.[sideEffectsKey].added?.['index.js']?.integrity
expect(patchedFileIntegrity).toBeTruthy()
@@ -151,8 +151,8 @@ test('patch package with version range', async () => {
})
expect(lockfile.snapshots[`is-positive@1.0.0(patch_hash=${patchFileHash})`]).toBeTruthy()
const filesIndexFile = path.join(opts.storeDir, 'index/c7/1ccf199e0fdae37aad13946b937d67bcd35fa111b84d21b3a19439cfdc2812-is-positive@1.0.0.json')
const filesIndex = loadJsonFileSync<PackageFilesIndex>(filesIndexFile)
const filesIndexFile = path.join(opts.storeDir, 'index/c7/1ccf199e0fdae37aad13946b937d67bcd35fa111b84d21b3a19439cfdc2812-is-positive@1.0.0.v8')
const filesIndex = readV8FileStrictSync<PackageFilesIndex>(filesIndexFile)
const sideEffectsKey = `${ENGINE_NAME};patch=${patchFileHash}`
const patchedFileIntegrity = filesIndex.sideEffects?.[sideEffectsKey].added?.['index.js']?.integrity
expect(patchedFileIntegrity).toBeTruthy()
@@ -318,8 +318,8 @@ test('patch package when scripts are ignored', async () => {
})
expect(lockfile.snapshots[`is-positive@1.0.0(patch_hash=${patchFileHash})`]).toBeTruthy()
const filesIndexFile = path.join(opts.storeDir, 'index/c7/1ccf199e0fdae37aad13946b937d67bcd35fa111b84d21b3a19439cfdc2812-is-positive@1.0.0.json')
const filesIndex = loadJsonFileSync<PackageFilesIndex>(filesIndexFile)
const filesIndexFile = path.join(opts.storeDir, 'index/c7/1ccf199e0fdae37aad13946b937d67bcd35fa111b84d21b3a19439cfdc2812-is-positive@1.0.0.v8')
const filesIndex = readV8FileStrictSync<PackageFilesIndex>(filesIndexFile)
const sideEffectsKey = `${ENGINE_NAME};patch=${patchFileHash}`
const patchedFileIntegrity = filesIndex.sideEffects?.[sideEffectsKey].added?.['index.js']?.integrity
expect(patchedFileIntegrity).toBeTruthy()
@@ -406,8 +406,8 @@ test('patch package when the package is not in onlyBuiltDependencies list', asyn
})
expect(lockfile.snapshots[`is-positive@1.0.0(patch_hash=${patchFileHash})`]).toBeTruthy()
const filesIndexFile = path.join(opts.storeDir, 'index/c7/1ccf199e0fdae37aad13946b937d67bcd35fa111b84d21b3a19439cfdc2812-is-positive@1.0.0.json')
const filesIndex = loadJsonFileSync<PackageFilesIndex>(filesIndexFile)
const filesIndexFile = path.join(opts.storeDir, 'index/c7/1ccf199e0fdae37aad13946b937d67bcd35fa111b84d21b3a19439cfdc2812-is-positive@1.0.0.v8')
const filesIndex = readV8FileStrictSync<PackageFilesIndex>(filesIndexFile)
const sideEffectsKey = `${ENGINE_NAME};patch=${patchFileHash}`
const patchedFileIntegrity = filesIndex.sideEffects?.[sideEffectsKey].added?.['index.js']?.integrity
expect(patchedFileIntegrity).toBeTruthy()

14
pkg-manager/core/test/install/sideEffects.ts
View File

@@ -1,14 +1,14 @@
import fs from 'fs'
import path from 'path'
import v8 from 'v8'
import { addDependenciesToPackage, install } from '@pnpm/core'
import { hashObject } from '@pnpm/crypto.object-hasher'
import { readV8FileStrictSync } from '@pnpm/fs.v8-file'
import { getIndexFilePathInCafs, getFilePathByModeInCafs, type PackageFilesIndex } from '@pnpm/store.cafs'
import { getIntegrity, REGISTRY_MOCK_PORT } from '@pnpm/registry-mock'
import { prepareEmpty } from '@pnpm/prepare'
import { ENGINE_NAME } from '@pnpm/constants'
import { sync as rimraf } from '@zkochan/rimraf'
import { loadJsonFileSync } from 'load-json-file'
import { writeJsonFileSync } from 'write-json-file'
import { testDefaults } from '../utils/index.js'
const ENGINE_DIR = `${process.platform}-${process.arch}-node-${process.version.split('.')[0]}`
@@ -83,7 +83,7 @@ test('using side effects cache', async () => {
const { updatedManifest: manifest } = await addDependenciesToPackage({}, ['@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0'], opts)
const filesIndexFile = getIndexFilePathInCafs(opts.storeDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0')
const filesIndex = loadJsonFileSync<PackageFilesIndex>(filesIndexFile)
const filesIndex = readV8FileStrictSync<PackageFilesIndex>(filesIndexFile)
expect(filesIndex.sideEffects).toBeTruthy() // files index has side effects
const sideEffectsKey = `${ENGINE_NAME};deps=${hashObject({
id: `@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0:${getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0')}`,
@@ -97,7 +97,7 @@ test('using side effects cache', async () => {
expect(filesIndex.sideEffects).toHaveProperty([sideEffectsKey, 'added', 'generated-by-preinstall.js'])
expect(filesIndex.sideEffects).toHaveProperty([sideEffectsKey, 'added', 'generated-by-postinstall.js'])
delete filesIndex.sideEffects![sideEffectsKey].added?.['generated-by-postinstall.js']
writeJsonFileSync(filesIndexFile, filesIndex)
fs.writeFileSync(filesIndexFile, v8.serialize(filesIndex))
rimraf('node_modules')
rimraf('pnpm-lock.yaml') // to avoid headless install
@@ -164,7 +164,7 @@ test('uploading errors do not interrupt installation', async () => {
expect(fs.existsSync('node_modules/@pnpm.e2e/pre-and-postinstall-scripts-example/generated-by-postinstall.js')).toBeTruthy()
const filesIndexFile = getIndexFilePathInCafs(opts.storeDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0')
const filesIndex = loadJsonFileSync<PackageFilesIndex>(filesIndexFile)
const filesIndex = readV8FileStrictSync<PackageFilesIndex>(filesIndexFile)
expect(filesIndex.sideEffects).toBeFalsy()
})
@@ -181,7 +181,7 @@ test('a postinstall script does not modify the original sources added to the sto
expect(fs.readFileSync('node_modules/@pnpm/postinstall-modifies-source/empty-file.txt', 'utf8')).toContain('hello')
const filesIndexFile = getIndexFilePathInCafs(opts.storeDir, getIntegrity('@pnpm/postinstall-modifies-source', '1.0.0'), '@pnpm/postinstall-modifies-source@1.0.0')
const filesIndex = loadJsonFileSync<PackageFilesIndex>(filesIndexFile)
const filesIndex = readV8FileStrictSync<PackageFilesIndex>(filesIndexFile)
const patchedFileIntegrity = filesIndex.sideEffects?.[`${ENGINE_NAME};deps=${hashObject({
id: `@pnpm/postinstall-modifies-source@1.0.0:${getIntegrity('@pnpm/postinstall-modifies-source', '1.0.0')}`,
deps: {},
@@ -206,7 +206,7 @@ test('a corrupted side-effects cache is ignored', async () => {
const { updatedManifest: manifest } = await addDependenciesToPackage({}, ['@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0'], opts)
const filesIndexFile = getIndexFilePathInCafs(opts.storeDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0')
const filesIndex = loadJsonFileSync<PackageFilesIndex>(filesIndexFile)
const filesIndex = readV8FileStrictSync<PackageFilesIndex>(filesIndexFile)
expect(filesIndex.sideEffects).toBeTruthy() // files index has side effects
const sideEffectsKey = `${ENGINE_NAME};deps=${hashObject({
id: `@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0:${getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0')}`,

3
pkg-manager/core/tsconfig.json
View File

@@ -66,6 +66,9 @@
{
"path": "../../fs/symlink-dependency"
},
{
"path": "../../fs/v8-file"
},
{
"path": "../../hooks/read-package-hook"
},

1
pkg-manager/headless/package.json
View File

@@ -81,6 +81,7 @@
"@jest/globals": "catalog:",
"@pnpm/assert-project": "workspace:*",
"@pnpm/crypto.object-hasher": "workspace:*",
"@pnpm/fs.v8-file": "workspace:*",
"@pnpm/headless": "workspace:*",
"@pnpm/logger": "workspace:*",
"@pnpm/prepare": "workspace:*",

11
pkg-manager/headless/test/index.ts
View File

@@ -1,9 +1,11 @@
/// <reference path="../../../__typings__/index.d.ts" />
import fs from 'fs'
import v8 from 'v8'
import path from 'path'
import { assertProject } from '@pnpm/assert-project'
import { hashObject } from '@pnpm/crypto.object-hasher'
import { getIndexFilePathInCafs } from '@pnpm/store.cafs'
import { readV8FileStrictSync } from '@pnpm/fs.v8-file'
import { getIndexFilePathInCafs, type PackageFilesIndex } from '@pnpm/store.cafs'
import { ENGINE_NAME, WANTED_LOCKFILE } from '@pnpm/constants'
import {
type PackageManifestLog,
@@ -23,7 +25,6 @@ import { jest } from '@jest/globals'
import { sync as rimraf } from '@zkochan/rimraf'
import { loadJsonFileSync } from 'load-json-file'
import sinon from 'sinon'
import { writeJsonFileSync } from 'write-json-file'
import { testDefaults } from './utils/testDefaults.js'
const f = fixtures(import.meta.dirname)
@@ -680,7 +681,7 @@ test.each([['isolated'], ['hoisted']])('using side effects cache with nodeLinker
await headlessInstall(opts)
const cacheIntegrityPath = getIndexFilePathInCafs(opts.storeDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0')
const cacheIntegrity = loadJsonFileSync<any>(cacheIntegrityPath) // eslint-disable-line @typescript-eslint/no-explicit-any
const cacheIntegrity = readV8FileStrictSync<PackageFilesIndex>(cacheIntegrityPath)
expect(cacheIntegrity!.sideEffects).toBeTruthy()
const sideEffectsKey = `${ENGINE_NAME};deps=${hashObject({
id: `@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0:${getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0')}`,
@@ -692,10 +693,10 @@ test.each([['isolated'], ['hoisted']])('using side effects cache with nodeLinker
},
})}`
expect(cacheIntegrity).toHaveProperty(['sideEffects', sideEffectsKey, 'added', 'generated-by-postinstall.js'])
delete cacheIntegrity!.sideEffects[sideEffectsKey].added['generated-by-postinstall.js']
delete cacheIntegrity!.sideEffects![sideEffectsKey].added!['generated-by-postinstall.js']
expect(cacheIntegrity).toHaveProperty(['sideEffects', sideEffectsKey, 'added', 'generated-by-preinstall.js'])
writeJsonFileSync(cacheIntegrityPath, cacheIntegrity)
fs.writeFileSync(cacheIntegrityPath, v8.serialize(cacheIntegrity))
prefix = f.prepare('side-effects')
const opts2 = await testDefaults({

3
pkg-manager/headless/tsconfig.json
View File

@@ -42,6 +42,9 @@
{
"path": "../../fs/symlink-dependency"
},
{
"path": "../../fs/v8-file"
},
{
"path": "../../lockfile/filtering"
},

2
pkg-manager/package-requester/package.json
View File

@@ -64,6 +64,7 @@
"@pnpm/cafs-types": "workspace:*",
"@pnpm/client": "workspace:*",
"@pnpm/create-cafs-store": "workspace:*",
"@pnpm/fs.v8-file": "workspace:*",
"@pnpm/logger": "workspace:*",
"@pnpm/package-requester": "workspace:*",
"@pnpm/registry-mock": "catalog:",
@@ -73,7 +74,6 @@
"@types/semver": "catalog:",
"@types/ssri": "catalog:",
"delay": "catalog:",
"load-json-file": "catalog:",
"nock": "catalog:",
"normalize-path": "catalog:",
"tempy": "catalog:"

4
pkg-manager/package-requester/test/index.ts
View File

@@ -3,6 +3,7 @@ import fs from 'fs'
import path from 'path'
import { type PackageFilesIndex } from '@pnpm/store.cafs'
import { createClient } from '@pnpm/client'
import { readV8FileStrictSync } from '@pnpm/fs.v8-file'
import { streamParser } from '@pnpm/logger'
import { createPackageRequester, type PackageResponse } from '@pnpm/package-requester'
import { createCafsStore } from '@pnpm/create-cafs-store'
@@ -12,7 +13,6 @@ import delay from 'delay'
import { depPathToFilename } from '@pnpm/dependency-path'
import { restartWorkerPool } from '@pnpm/worker'
import { jest } from '@jest/globals'
import { loadJsonFileSync } from 'load-json-file'
import nock from 'nock'
import normalize from 'normalize-path'
import { temporaryDirectory } from 'tempy'
@@ -385,7 +385,7 @@ test('fetchPackageToStore()', async () => {
expect(Object.keys(files.filesIndex).sort()).toStrictEqual(['package.json', 'index.js', 'license', 'readme.md'].sort())
expect(files.resolvedFrom).toBe('remote')
const indexFile = loadJsonFileSync<PackageFilesIndex>(fetchResult.filesIndexFile)
const indexFile = readV8FileStrictSync<PackageFilesIndex>(fetchResult.filesIndexFile)
expect(indexFile).toBeTruthy()
expect(typeof indexFile.files['package.json'].checkedAt).toBeTruthy()

3
pkg-manager/package-requester/tsconfig.json
View File

@@ -24,6 +24,9 @@
{
"path": "../../fs/graceful-fs"
},
{
"path": "../../fs/v8-file"
},
{
"path": "../../packages/core-loggers"
},

81
pnpm-lock.yaml generated
View File

@@ -1254,6 +1254,9 @@ importers:
'@pnpm/constants':
specifier: workspace:*
version: link:../../packages/constants
'@pnpm/fs.v8-file':
specifier: workspace:*
version: link:../../fs/v8-file
'@pnpm/npm-resolver':
specifier: workspace:*
version: link:../../resolving/npm-resolver
@@ -2800,6 +2803,9 @@ importers:
'@pnpm/crypto.object-hasher':
specifier: workspace:*
version: link:../../crypto/object-hasher
'@pnpm/fs.v8-file':
specifier: workspace:*
version: link:../../fs/v8-file
'@pnpm/logger':
specifier: workspace:*
version: link:../../packages/logger
@@ -3462,6 +3468,12 @@ importers:
specifier: workspace:*
version: 'link:'
fs/v8-file:
devDependencies:
'@pnpm/fs.v8-file':
specifier: workspace:*
version: 'link:'
hooks/pnpmfile:
dependencies:
'@pnpm/core-loggers':
@@ -4156,6 +4168,9 @@ importers:
'@pnpm/dependency-path':
specifier: workspace:*
version: link:../../packages/dependency-path
'@pnpm/fs.v8-file':
specifier: workspace:*
version: link:../../fs/v8-file
'@pnpm/lockfile.fs':
specifier: workspace:*
version: link:../../lockfile/fs
@@ -4177,9 +4192,6 @@ importers:
hyperdrive-schemas:
specifier: 'catalog:'
version: 2.0.0
load-json-file:
specifier: 'catalog:'
version: 7.0.1
normalize-path:
specifier: 'catalog:'
version: 3.0.0
@@ -5041,6 +5053,9 @@ importers:
'@pnpm/core':
specifier: workspace:*
version: 'link:'
'@pnpm/fs.v8-file':
specifier: workspace:*
version: link:../../fs/v8-file
'@pnpm/git-utils':
specifier: workspace:*
version: link:../../packages/git-utils
@@ -5320,6 +5335,9 @@ importers:
'@pnpm/crypto.object-hasher':
specifier: workspace:*
version: link:../../crypto/object-hasher
'@pnpm/fs.v8-file':
specifier: workspace:*
version: link:../../fs/v8-file
'@pnpm/headless':
specifier: workspace:*
version: 'link:'
@@ -5686,6 +5704,9 @@ importers:
'@pnpm/create-cafs-store':
specifier: workspace:*
version: link:../../store/create-cafs-store
'@pnpm/fs.v8-file':
specifier: workspace:*
version: link:../../fs/v8-file
'@pnpm/logger':
specifier: workspace:*
version: link:../../packages/logger
@@ -5713,9 +5734,6 @@ importers:
delay:
specifier: 'catalog:'
version: 6.0.0
load-json-file:
specifier: 'catalog:'
version: 7.0.1
nock:
specifier: 'catalog:'
version: 13.3.4
@@ -6465,6 +6483,9 @@ importers:
'@pnpm/find-workspace-dir':
specifier: workspace:*
version: link:../workspace/find-workspace-dir
'@pnpm/fs.v8-file':
specifier: workspace:*
version: link:../fs/v8-file
'@pnpm/lockfile.types':
specifier: workspace:*
version: link:../lockfile/types
@@ -7321,6 +7342,9 @@ importers:
'@pnpm/fetching-types':
specifier: workspace:*
version: link:../../network/fetching-types
'@pnpm/fs.v8-file':
specifier: workspace:*
version: link:../../fs/v8-file
'@pnpm/graceful-fs':
specifier: workspace:*
version: link:../../fs/graceful-fs
@@ -7354,9 +7378,6 @@ importers:
encode-registry:
specifier: 'catalog:'
version: 3.0.1
load-json-file:
specifier: 'catalog:'
version: 7.0.1
lru-cache:
specifier: 'catalog:'
version: 11.1.0
@@ -7418,6 +7439,9 @@ importers:
'@types/ssri':
specifier: 'catalog:'
version: 7.1.5
load-json-file:
specifier: 'catalog:'
version: 7.0.1
nock:
specifier: 'catalog:'
version: 13.3.4
@@ -7526,6 +7550,9 @@ importers:
'@pnpm/error':
specifier: workspace:*
version: link:../../packages/error
'@pnpm/fs.v8-file':
specifier: workspace:*
version: link:../../fs/v8-file
'@pnpm/lockfile.detect-dep-types':
specifier: workspace:*
version: link:../../lockfile/detect-dep-types
@@ -7553,9 +7580,6 @@ importers:
'@pnpm/types':
specifier: workspace:*
version: link:../../packages/types
load-json-file:
specifier: 'catalog:'
version: 7.0.1
p-limit:
specifier: 'catalog:'
version: 7.1.0
@@ -8066,6 +8090,9 @@ importers:
'@pnpm/fetcher-base':
specifier: workspace:*
version: link:../../fetching/fetcher-base
'@pnpm/fs.v8-file':
specifier: workspace:*
version: link:../../fs/v8-file
'@pnpm/package-requester':
specifier: workspace:*
version: link:../../pkg-manager/package-requester
@@ -8087,9 +8114,6 @@ importers:
'@zkochan/rimraf':
specifier: 'catalog:'
version: 3.0.2
load-json-file:
specifier: 'catalog:'
version: 7.0.1
ramda:
specifier: 'catalog:'
version: '@pnpm/ramda@0.28.1'
@@ -8203,6 +8227,9 @@ importers:
'@pnpm/error':
specifier: workspace:*
version: link:../../packages/error
'@pnpm/fs.v8-file':
specifier: workspace:*
version: link:../../fs/v8-file
'@pnpm/get-context':
specifier: workspace:*
version: link:../../pkg-manager/get-context
@@ -8236,9 +8263,6 @@ importers:
dint:
specifier: 'catalog:'
version: 5.1.0
load-json-file:
specifier: 'catalog:'
version: 7.0.1
p-filter:
specifier: 'catalog:'
version: 4.1.0
@@ -8309,6 +8333,9 @@ importers:
'@pnpm/error':
specifier: workspace:*
version: link:../../packages/error
'@pnpm/fs.v8-file':
specifier: workspace:*
version: link:../../fs/v8-file
'@pnpm/graceful-fs':
specifier: workspace:*
version: link:../../fs/graceful-fs
@@ -8333,9 +8360,6 @@ importers:
chalk:
specifier: 'catalog:'
version: 5.6.0
load-json-file:
specifier: 'catalog:'
version: 7.0.1
render-help:
specifier: 'catalog:'
version: 1.0.3
@@ -8377,6 +8401,9 @@ importers:
'@pnpm/client':
specifier: workspace:*
version: link:../../pkg-manager/client
'@pnpm/fs.v8-file':
specifier: workspace:*
version: link:../../fs/v8-file
'@pnpm/logger':
specifier: workspace:*
version: link:../../packages/logger
@@ -8389,6 +8416,9 @@ importers:
'@pnpm/server':
specifier: workspace:*
version: 'link:'
'@pnpm/store.cafs':
specifier: workspace:*
version: link:../cafs
'@types/uuid':
specifier: 'catalog:'
version: 8.3.4
@@ -8401,9 +8431,6 @@ importers:
is-port-reachable:
specifier: 'catalog:'
version: 4.0.0
load-json-file:
specifier: 'catalog:'
version: 7.0.1
node-fetch:
specifier: 'catalog:'
version: 3.3.2
@@ -8647,6 +8674,9 @@ importers:
'@pnpm/fs.hard-link-dir':
specifier: workspace:*
version: link:../fs/hard-link-dir
'@pnpm/fs.v8-file':
specifier: workspace:*
version: link:../fs/v8-file
'@pnpm/graceful-fs':
specifier: workspace:*
version: link:../fs/graceful-fs
@@ -8662,9 +8692,6 @@ importers:
is-windows:
specifier: 'catalog:'
version: 1.0.2
load-json-file:
specifier: 'catalog:'
version: 7.0.1
p-limit:
specifier: 'catalog:'
version: 7.1.0

1
pnpm/package.json
View File

@@ -78,6 +78,7 @@
"devDependencies": {
"@jest/globals": "catalog:",
"@pnpm/assert-project": "workspace:*",
"@pnpm/fs.v8-file": "workspace:*",
"@pnpm/byline": "catalog:",
"@pnpm/cache.commands": "workspace:*",
"@pnpm/cli-meta": "workspace:*",

14
pnpm/test/install/misc.ts
View File

@@ -1,20 +1,20 @@
import fs from 'fs'
import path from 'path'
import v8 from 'v8'
import { STORE_VERSION, WANTED_LOCKFILE } from '@pnpm/constants'
import { readV8FileStrictSync } from '@pnpm/fs.v8-file'
import { type LockfileObject } from '@pnpm/lockfile.types'
import { prepare, prepareEmpty, preparePackages } from '@pnpm/prepare'
import { readPackageJsonFromDir } from '@pnpm/read-package-json'
import { readProjectManifest } from '@pnpm/read-project-manifest'
import { getIntegrity } from '@pnpm/registry-mock'
import { getIndexFilePathInCafs } from '@pnpm/store.cafs'
import { getIndexFilePathInCafs, type PackageFilesIndex } from '@pnpm/store.cafs'
import { writeProjectManifest } from '@pnpm/write-project-manifest'
import { fixtures } from '@pnpm/test-fixtures'
import dirIsCaseSensitive from 'dir-is-case-sensitive'
import { sync as readYamlFile } from 'read-yaml-file'
import { sync as rimraf } from '@zkochan/rimraf'
import isWindows from 'is-windows'
import { loadJsonFileSync } from 'load-json-file'
import { writeJsonFileSync } from 'write-json-file'
import { sync as writeYamlFile } from 'write-yaml-file'
import crossSpawn from 'cross-spawn'
import {
@@ -159,7 +159,7 @@ test("don't fail on case insensitive filesystems when package has 2 files with s
project.has('@pnpm.e2e/with-same-file-in-different-cases')
const { files: integrityFile } = loadJsonFileSync<{ files: object }>(project.getPkgIndexFilePath('@pnpm.e2e/with-same-file-in-different-cases', '1.0.0'))
const { files: integrityFile } = readV8FileStrictSync<PackageFilesIndex>(project.getPkgIndexFilePath('@pnpm.e2e/with-same-file-in-different-cases', '1.0.0'))
const packageFiles = Object.keys(integrityFile).sort()
expect(packageFiles).toStrictEqual(['Foo.js', 'foo.js', 'package.json'])
@@ -453,12 +453,12 @@ test('installation fails when the stored package name and version do not match t
await execPnpm(['add', '@pnpm.e2e/dep-of-pkg-with-1-dep@100.1.0', ...settings])
const cacheIntegrityPath = getIndexFilePathInCafs(path.join(storeDir, STORE_VERSION), getIntegrity('@pnpm.e2e/dep-of-pkg-with-1-dep', '100.1.0'), '@pnpm.e2e/dep-of-pkg-with-1-dep@100.1.0')
const cacheIntegrity = loadJsonFileSync<any>(cacheIntegrityPath) // eslint-disable-line @typescript-eslint/no-explicit-any
const cacheIntegrity = readV8FileStrictSync<PackageFilesIndex>(cacheIntegrityPath)
cacheIntegrity.name = 'foo'
writeJsonFileSync(cacheIntegrityPath, {
fs.writeFileSync(cacheIntegrityPath, v8.serialize({
...cacheIntegrity,
name: 'foo',
})
}))
rimraf('node_modules')
await expect(

3
pnpm/tsconfig.json
View File

@@ -74,6 +74,9 @@
{
"path": "../exec/run-npm"
},
{
"path": "../fs/v8-file"
},
{
"path": "../lockfile/plugin-commands-audit"
},

3
resolving/npm-resolver/package.json
View File

@@ -38,6 +38,7 @@
"@pnpm/crypto.hash": "workspace:*",
"@pnpm/error": "workspace:*",
"@pnpm/fetching-types": "workspace:*",
"@pnpm/fs.v8-file": "workspace:*",
"@pnpm/graceful-fs": "workspace:*",
"@pnpm/pick-registry-for-package": "workspace:*",
"@pnpm/registry.pkg-metadata-filter": "workspace:*",
@@ -49,7 +50,6 @@
"@pnpm/workspace.spec-parser": "workspace:*",
"@zkochan/retry": "catalog:",
"encode-registry": "catalog:",
"load-json-file": "catalog:",
"lru-cache": "catalog:",
"normalize-path": "catalog:",
"p-limit": "catalog:",
@@ -75,6 +75,7 @@
"@types/ramda": "catalog:",
"@types/semver": "catalog:",
"@types/ssri": "catalog:",
"load-json-file": "catalog:",
"nock": "catalog:",
"tempy": "catalog:"
},

13
resolving/npm-resolver/src/pickPackage.ts
View File

@@ -1,12 +1,13 @@
import v8 from 'v8'
import { promises as fs } from 'fs'
import path from 'path'
import { createHexHash } from '@pnpm/crypto.hash'
import { PnpmError } from '@pnpm/error'
import { logger } from '@pnpm/logger'
import { readV8FileStrictAsync } from '@pnpm/fs.v8-file'
import gfs from '@pnpm/graceful-fs'
import { type PackageMeta, type PackageInRegistry } from '@pnpm/registry.types'
import getRegistryName from 'encode-registry'
import { loadJsonFile } from 'load-json-file'
import pLimit, { type LimitFunction } from 'p-limit'
import { fastPathTemp as pathTemp } from 'path-temp'
import { pick } from 'ramda'
@@ -134,7 +135,7 @@ export async function pickPackage (
}
const registryName = getRegistryName(opts.registry)
const pkgMirror = path.join(ctx.cacheDir, ctx.metaDir, registryName, `${encodePkgName(spec.name)}.json`)
const pkgMirror = path.join(ctx.cacheDir, ctx.metaDir, registryName, `${encodePkgName(spec.name)}.v8`)
return runLimited(pkgMirror, async (limit) => {
let metaCachedInStore: PackageMeta | null | undefined
@@ -210,7 +211,7 @@ export async function pickPackage (
ctx.metaCache.set(spec.name, meta)
if (!opts.dryRun) {
// We stringify this meta here to avoid saving any mutations that could happen to the meta object.
const stringifiedMeta = JSON.stringify(meta)
const stringifiedMeta = v8.serialize(meta)
// eslint-disable-next-line @typescript-eslint/no-floating-promises
runLimited(pkgMirror, (limit) => limit(async () => {
try {
@@ -283,15 +284,15 @@ function encodePkgName (pkgName: string): string {
async function loadMeta (pkgMirror: string): Promise<PackageMeta | null> {
try {
return await loadJsonFile<PackageMeta>(pkgMirror)
} catch (err: any) { // eslint-disable-line
return await readV8FileStrictAsync<PackageMeta>(pkgMirror)
} catch {
return null
}
}
const createdDirs = new Set<string>()
async function saveMeta (pkgMirror: string, meta: string): Promise<void> {
async function saveMeta (pkgMirror: string, meta: Buffer): Promise<void> {
const dir = path.dirname(pkgMirror)
if (!createdDirs.has(dir)) {
await fs.mkdir(dir, { recursive: true })

14
resolving/npm-resolver/test/index.ts
View File

@@ -73,7 +73,7 @@ test('resolveFromNpm()', async () => {
// The resolve function does not wait for the package meta cache file to be saved
// so we must delay for a bit in order to read it
const meta = await retryLoadJsonFile<any>(path.join(cacheDir, ABBREVIATED_META_DIR, 'registry.npmjs.org/is-positive.json')) // eslint-disable-line @typescript-eslint/no-explicit-any
const meta = await retryLoadJsonFile<any>(path.join(cacheDir, ABBREVIATED_META_DIR, 'registry.npmjs.org/is-positive.v8')) // eslint-disable-line @typescript-eslint/no-explicit-any
expect(meta.name).toBeTruthy()
expect(meta.versions).toBeTruthy()
expect(meta['dist-tags']).toBeTruthy()
@@ -95,7 +95,7 @@ test('resolveFromNpm() does not save mutated meta to the cache', async () => {
// The resolve function does not wait for the package meta cache file to be saved
// so we must delay for a bit in order to read it
const meta = await retryLoadJsonFile<any>(path.join(cacheDir, ABBREVIATED_META_DIR, 'registry.npmjs.org/is-positive.json')) // eslint-disable-line @typescript-eslint/no-explicit-any
const meta = await retryLoadJsonFile<any>(path.join(cacheDir, ABBREVIATED_META_DIR, 'registry.npmjs.org/is-positive.v8')) // eslint-disable-line @typescript-eslint/no-explicit-any
expect(meta.versions['1.0.0'].version).toBe('1.0.0')
})
@@ -116,7 +116,7 @@ test('resolveFromNpm() should save metadata to a unique file when the package na
// The resolve function does not wait for the package meta cache file to be saved
// so we must delay for a bit in order to read it
const meta = await retryLoadJsonFile<any>(path.join(cacheDir, ABBREVIATED_META_DIR, `registry.npmjs.org/JSON_${createHexHash('JSON')}.json`)) // eslint-disable-line @typescript-eslint/no-explicit-any
const meta = await retryLoadJsonFile<any>(path.join(cacheDir, ABBREVIATED_META_DIR, `registry.npmjs.org/JSON_${createHexHash('JSON')}.v8`)) // eslint-disable-line @typescript-eslint/no-explicit-any
expect(meta.name).toBeTruthy()
expect(meta.versions).toBeTruthy()
expect(meta['dist-tags']).toBeTruthy()
@@ -642,7 +642,7 @@ test('offline resolution fails when package meta not found in the store', async
await expect(resolveFromNpm({ alias: 'is-positive', bareSpecifier: '1.0.0' }, {})).rejects
.toThrow(
new PnpmError('NO_OFFLINE_META', `Failed to resolve is-positive@1.0.0 in package mirror ${path.join(cacheDir, ABBREVIATED_META_DIR, 'registry.npmjs.org/is-positive.json')}`)
new PnpmError('NO_OFFLINE_META', `Failed to resolve is-positive@1.0.0 in package mirror ${path.join(cacheDir, ABBREVIATED_META_DIR, 'registry.npmjs.org/is-positive.v8')}`)
)
})
@@ -947,7 +947,7 @@ test('resolve when tarball URL is requested from the registry', async () => {
// The resolve function does not wait for the package meta cache file to be saved
// so we must delay for a bit in order to read it
const meta = await retryLoadJsonFile<any>(path.join(cacheDir, ABBREVIATED_META_DIR, 'registry.npmjs.org/is-positive.json')) // eslint-disable-line @typescript-eslint/no-explicit-any
const meta = await retryLoadJsonFile<any>(path.join(cacheDir, ABBREVIATED_META_DIR, 'registry.npmjs.org/is-positive.v8')) // eslint-disable-line @typescript-eslint/no-explicit-any
expect(meta.name).toBeTruthy()
expect(meta.versions).toBeTruthy()
expect(meta['dist-tags']).toBeTruthy()
@@ -979,7 +979,7 @@ test('resolve when tarball URL is requested from the registry and alias is not s
// The resolve function does not wait for the package meta cache file to be saved
// so we must delay for a bit in order to read it
const meta = await retryLoadJsonFile<any>(path.join(cacheDir, ABBREVIATED_META_DIR, 'registry.npmjs.org/is-positive.json')) // eslint-disable-line @typescript-eslint/no-explicit-any
const meta = await retryLoadJsonFile<any>(path.join(cacheDir, ABBREVIATED_META_DIR, 'registry.npmjs.org/is-positive.v8')) // eslint-disable-line @typescript-eslint/no-explicit-any
expect(meta.name).toBeTruthy()
expect(meta.versions).toBeTruthy()
expect(meta['dist-tags']).toBeTruthy()
@@ -1732,7 +1732,7 @@ test('resolveFromNpm() should always return the name of the package that is spec
// The resolve function does not wait for the package meta cache file to be saved
// so we must delay for a bit in order to read it
const meta = await retryLoadJsonFile<any>(path.join(cacheDir, ABBREVIATED_META_DIR, 'registry.npmjs.org/is-positive.json')) // eslint-disable-line @typescript-eslint/no-explicit-any
const meta = await retryLoadJsonFile<any>(path.join(cacheDir, ABBREVIATED_META_DIR, 'registry.npmjs.org/is-positive.v8')) // eslint-disable-line @typescript-eslint/no-explicit-any
expect(meta.name).toBeTruthy()
expect(meta.versions).toBeTruthy()
expect(meta['dist-tags']).toBeTruthy()

4
resolving/npm-resolver/test/resolveJsr.test.ts
View File

@@ -70,7 +70,7 @@ test('resolveFromJsr() on jsr', async () => {
// The resolve function does not wait for the package meta cache file to be saved
// so we must delay for a bit in order to read it
const meta = await retryLoadJsonFile<any>(path.join(cacheDir, ABBREVIATED_META_DIR, 'npm.jsr.io/@jsr/rus__greet.json')) // eslint-disable-line @typescript-eslint/no-explicit-any
const meta = await retryLoadJsonFile<any>(path.join(cacheDir, ABBREVIATED_META_DIR, 'npm.jsr.io/@jsr/rus__greet.v8')) // eslint-disable-line @typescript-eslint/no-explicit-any
expect(meta).toMatchObject({
name: expect.any(String),
versions: expect.any(Object),
@@ -114,7 +114,7 @@ test('resolveFromJsr() on jsr with alias renaming', async () => {
// The resolve function does not wait for the package meta cache file to be saved
// so we must delay for a bit in order to read it
const meta = await retryLoadJsonFile<any>(path.join(cacheDir, ABBREVIATED_META_DIR, 'npm.jsr.io/@jsr/rus__greet.json')) // eslint-disable-line @typescript-eslint/no-explicit-any
const meta = await retryLoadJsonFile<any>(path.join(cacheDir, ABBREVIATED_META_DIR, 'npm.jsr.io/@jsr/rus__greet.v8')) // eslint-disable-line @typescript-eslint/no-explicit-any
expect(meta).toMatchObject({
name: expect.any(String),
versions: expect.any(Object),

4
resolving/npm-resolver/test/utils/index.ts
View File

@@ -1,4 +1,4 @@
import { loadJsonFile } from 'load-json-file'
import { readV8FileStrictAsync } from '@pnpm/fs.v8-file'
export async function retryLoadJsonFile<T> (filePath: string): Promise<T> {
let retry = 0
@@ -6,7 +6,7 @@ export async function retryLoadJsonFile<T> (filePath: string): Promise<T> {
while (true) {
await delay(500)
try {
return await loadJsonFile<T>(filePath)
return await readV8FileStrictAsync<T>(filePath)
} catch (err: any) { // eslint-disable-line
if (retry > 2) throw err
retry++

3
resolving/npm-resolver/tsconfig.json
View File

@@ -21,6 +21,9 @@
{
"path": "../../fs/graceful-fs"
},
{
"path": "../../fs/v8-file"
},
{
"path": "../../network/fetch"
},

2
reviewing/license-scanner/package.json
View File

@@ -35,6 +35,7 @@
"@pnpm/dependency-path": "workspace:*",
"@pnpm/directory-fetcher": "workspace:*",
"@pnpm/error": "workspace:*",
"@pnpm/fs.v8-file": "workspace:*",
"@pnpm/lockfile.detect-dep-types": "workspace:*",
"@pnpm/lockfile.fs": "workspace:*",
"@pnpm/lockfile.types": "workspace:*",
@@ -44,7 +45,6 @@
"@pnpm/read-package-json": "workspace:*",
"@pnpm/store.cafs": "workspace:*",
"@pnpm/types": "workspace:*",
"load-json-file": "catalog:",
"p-limit": "catalog:",
"path-absolute": "catalog:",
"ramda": "catalog:",

4
reviewing/license-scanner/src/getPkgInfo.ts
View File

@@ -12,11 +12,11 @@ import {
type PackageFileInfo,
type PackageFilesIndex,
} from '@pnpm/store.cafs'
import { loadJsonFile } from 'load-json-file'
import { PnpmError } from '@pnpm/error'
import { type LicensePackage } from './licenses.js'
import { type DirectoryResolution, type PackageSnapshot, pkgSnapshotToResolution, type Resolution } from '@pnpm/lockfile.utils'
import { fetchFromDir } from '@pnpm/directory-fetcher'
import { readV8FileStrictAsync } from '@pnpm/fs.v8-file'
const limitPkgReads = pLimit(4)
@@ -276,7 +276,7 @@ export async function readPackageIndexFile (
}
try {
const { files } = await loadJsonFile<PackageFilesIndex>(pkgIndexFilePath)
const { files } = await readV8FileStrictAsync<PackageFilesIndex>(pkgIndexFilePath)
return {
local: false,
files,

2
reviewing/license-scanner/test/getPkgInfo.spec.ts
View File

@@ -30,6 +30,6 @@ describe('licences', () => {
virtualStoreDirMaxLength: 120,
}
)
).rejects.toThrow(`Failed to find package index file for bogus-package@1.0.0 (at ${path.join('store-dir', 'index', 'b2', '16-bogus-package@1.0.0.json')}), please consider running 'pnpm install'`)
).rejects.toThrow(`Failed to find package index file for bogus-package@1.0.0 (at ${path.join('store-dir', 'index', 'b2', '16-bogus-package@1.0.0.v8')}), please consider running 'pnpm install'`)
})
})

3
reviewing/license-scanner/tsconfig.json
View File

@@ -15,6 +15,9 @@
{
"path": "../../fetching/directory-fetcher"
},
{
"path": "../../fs/v8-file"
},
{
"path": "../../lockfile/detect-dep-types"
},

4
store/cafs/src/getFilePathInCafs.ts
View File

@@ -39,7 +39,7 @@ export function getIndexFilePathInCafs (
// 1. Validate that the integrity in the lockfile corresponds to the correct package,
// which might not be the case after a poorly resolved Git conflict.
// 2. Allow the same content to be referenced by different packages or different versions of the same package.
return path.join(storeDir, `index/${path.join(hex.slice(0, 2), hex.slice(2))}-${pkgId.replace(/[\\/:*?"<>|]/g, '+')}.json`)
return path.join(storeDir, `index/${path.join(hex.slice(0, 2), hex.slice(2))}-${pkgId.replace(/[\\/:*?"<>|]/g, '+')}.v8`)
}
function contentPathFromIntegrity (
@@ -58,6 +58,6 @@ export function contentPathFromHex (fileType: FileType, hex: string): string {
case 'nonexec':
return p
case 'index':
return `${p}-index.json`
return `${p}-index.v8`
}
}

2
store/package-store/package.json
View File

@@ -46,13 +46,13 @@
"dependencies": {
"@pnpm/create-cafs-store": "workspace:*",
"@pnpm/fetcher-base": "workspace:*",
"@pnpm/fs.v8-file": "workspace:*",
"@pnpm/package-requester": "workspace:*",
"@pnpm/resolver-base": "workspace:*",
"@pnpm/store-controller-types": "workspace:*",
"@pnpm/store.cafs": "workspace:*",
"@pnpm/types": "workspace:*",
"@zkochan/rimraf": "catalog:",
"load-json-file": "catalog:",
"ramda": "catalog:",
"ssri": "catalog:"
},

8
store/package-store/src/storeController/prune.ts
View File

@@ -1,10 +1,10 @@
import { type Dirent, promises as fs } from 'fs'
import util from 'util'
import path from 'path'
import { readV8FileStrictAsync } from '@pnpm/fs.v8-file'
import { type PackageFilesIndex } from '@pnpm/store.cafs'
import { globalInfo, globalWarn } from '@pnpm/logger'
import rimraf from '@zkochan/rimraf'
import { loadJsonFile } from 'load-json-file'
import ssri from 'ssri'
const BIG_ONE = BigInt(1) as unknown
@@ -35,7 +35,7 @@ export async function prune ({ cacheDir, storeDir }: PruneOptions, removeAlienFi
const subdir = path.join(indexDir, dir)
await Promise.all((await fs.readdir(subdir)).map(async (fileName) => {
const filePath = path.join(subdir, fileName)
if (fileName.endsWith('.json')) {
if (fileName.endsWith('.v8')) {
pkgIndexFiles.push(filePath)
}
}))
@@ -47,7 +47,7 @@ export async function prune ({ cacheDir, storeDir }: PruneOptions, removeAlienFi
const subdir = path.join(cafsDir, dir)
await Promise.all((await fs.readdir(subdir)).map(async (fileName) => {
const filePath = path.join(subdir, fileName)
if (fileName.endsWith('.json')) {
if (fileName.endsWith('.v8')) {
pkgIndexFiles.push(filePath)
return
}
@@ -74,7 +74,7 @@ export async function prune ({ cacheDir, storeDir }: PruneOptions, removeAlienFi
let pkgCounter = 0
await Promise.all(pkgIndexFiles.map(async (pkgIndexFilePath) => {
const { files: pkgFilesIndex } = await loadJsonFile<PackageFilesIndex>(pkgIndexFilePath)
const { files: pkgFilesIndex } = await readV8FileStrictAsync<PackageFilesIndex>(pkgIndexFilePath)
if (removedHashes.has(pkgFilesIndex['package.json'].integrity)) {
await fs.unlink(pkgIndexFilePath)
pkgCounter++

3
store/package-store/tsconfig.json
View File

@@ -15,6 +15,9 @@
{
"path": "../../fetching/fetcher-base"
},
{
"path": "../../fs/v8-file"
},
{
"path": "../../packages/logger"
},

2
store/plugin-commands-store-inspecting/package.json
View File

@@ -34,6 +34,7 @@
"@pnpm/client": "workspace:*",
"@pnpm/config": "workspace:*",
"@pnpm/error": "workspace:*",
"@pnpm/fs.v8-file": "workspace:*",
"@pnpm/graceful-fs": "workspace:*",
"@pnpm/lockfile.types": "workspace:*",
"@pnpm/object.key-sorting": "workspace:*",
@@ -42,7 +43,6 @@
"@pnpm/store.cafs": "workspace:*",
"@pnpm/types": "workspace:*",
"chalk": "catalog:",
"load-json-file": "catalog:",
"render-help": "catalog:"
},
"devDependencies": {

4
store/plugin-commands-store-inspecting/src/catIndex.ts
View File

@@ -3,12 +3,12 @@ import { createResolver } from '@pnpm/client'
import { type TarballResolution } from '@pnpm/lockfile.types'
import { PnpmError } from '@pnpm/error'
import { readV8FileStrictAsync } from '@pnpm/fs.v8-file'
import { sortDeepKeys } from '@pnpm/object.key-sorting'
import { getStorePath } from '@pnpm/store-path'
import { getIndexFilePathInCafs, type PackageFilesIndex } from '@pnpm/store.cafs'
import { parseWantedDependency } from '@pnpm/parse-wanted-dependency'
import { loadJsonFile } from 'load-json-file'
import renderHelp from 'render-help'
export const skipPackageManagerCheck = true
@@ -86,7 +86,7 @@ export async function handler (opts: CatIndexCommandOptions, params: string[]):
`${alias}@${bareSpecifier}`
)
try {
const pkgFilesIndex = await loadJsonFile<PackageFilesIndex>(filesIndexFile)
const pkgFilesIndex = await readV8FileStrictAsync<PackageFilesIndex>(filesIndexFile)
return JSON.stringify(sortDeepKeys(pkgFilesIndex), null, 2)
} catch {
throw new PnpmError(

9
store/plugin-commands-store-inspecting/src/findHash.ts
View File

@@ -4,10 +4,10 @@ import chalk from 'chalk'
import { type Config } from '@pnpm/config'
import { PnpmError } from '@pnpm/error'
import { safeReadV8FileSync } from '@pnpm/fs.v8-file'
import { getStorePath } from '@pnpm/store-path'
import { type PackageFilesIndex } from '@pnpm/store.cafs'
import { loadJsonFileSync } from 'load-json-file'
import renderHelp from 'render-help'
export const PACKAGE_INFO_CLR = chalk.greenBright
@@ -57,14 +57,17 @@ export async function handler (opts: FindHashCommandOptions, params: string[]):
for (const { name: dirName } of cafsChildrenDirs) {
const dirIndexFiles = fs
.readdirSync(`${indexDir}/${dirName}`)
.filter((fileName) => fileName.includes('.json'))
.filter((fileName) => fileName.includes('.v8'))
?.map((fileName) => `${indexDir}/${dirName}/${fileName}`)
indexFiles.push(...dirIndexFiles)
}
for (const filesIndexFile of indexFiles) {
const pkgFilesIndex = loadJsonFileSync<PackageFilesIndex>(filesIndexFile)
const pkgFilesIndex = safeReadV8FileSync<PackageFilesIndex>(filesIndexFile)
if (!pkgFilesIndex) {
continue
}
for (const [, file] of Object.entries(pkgFilesIndex.files)) {
if (file?.integrity === hash) {

8
store/plugin-commands-store-inspecting/test/findHash.ts
View File

@@ -26,8 +26,8 @@ test('print index file path with hash', async () => {
storeDir,
}, ['sha512-fXs1pWlUdqT2jkeoEJW/+odKZ2NwAyYkWea+plJKZI2xmhRKQi2e+nKGcClyDblgLwCLD912oMaua0+sTwwIrw=='])
expect(output).toBe(`${PACKAGE_INFO_CLR('lodash')}@${PACKAGE_INFO_CLR('4.17.19')} ${INDEX_PATH_CLR('/24/dbddf17111f46417d2fdaa260b1a37f9b3142340e4145efe3f0937d77eb56c-lodash@4.17.19.json')}
${PACKAGE_INFO_CLR('lodash')}@${PACKAGE_INFO_CLR('4.17.20')} ${INDEX_PATH_CLR('/3e/585d15c8a594e20d7de57b362ea81754c011acb2641a19f1b72c8531ea3982-lodash@4.17.20.json')}
expect(output).toBe(`${PACKAGE_INFO_CLR('lodash')}@${PACKAGE_INFO_CLR('4.17.19')} ${INDEX_PATH_CLR('/24/dbddf17111f46417d2fdaa260b1a37f9b3142340e4145efe3f0937d77eb56c-lodash@4.17.19.v8')}
${PACKAGE_INFO_CLR('lodash')}@${PACKAGE_INFO_CLR('4.17.20')} ${INDEX_PATH_CLR('/3e/585d15c8a594e20d7de57b362ea81754c011acb2641a19f1b72c8531ea3982-lodash@4.17.20.v8')}
`)
}
})
@@ -43,10 +43,10 @@ test('print index file path with hash error', async () => {
},
})
await findHash.handler(config as findHash.FindHashCommandOptions, ['sha512-fXs1pWlUdqT2j'])
} catch (_err: any) { // eslint-disable-line
} catch (_err: any) { // eslint-disable-line
err = _err
}
expect(err.code).toBe('ERR_PNPM_INVALID_FILE_HASH')
expect(err.message).toBe('No package or index file matching this hash was found.')
expect(err.code).toBe('ERR_PNPM_INVALID_FILE_HASH')
})

3
store/plugin-commands-store-inspecting/tsconfig.json
View File

@@ -18,6 +18,9 @@
{
"path": "../../fs/graceful-fs"
},
{
"path": "../../fs/v8-file"
},
{
"path": "../../lockfile/types"
},

2
store/plugin-commands-store/package.json
View File

@@ -36,6 +36,7 @@
"@pnpm/config": "workspace:*",
"@pnpm/dependency-path": "workspace:*",
"@pnpm/error": "workspace:*",
"@pnpm/fs.v8-file": "workspace:*",
"@pnpm/get-context": "workspace:*",
"@pnpm/lockfile.utils": "workspace:*",
"@pnpm/normalize-registries": "workspace:*",
@@ -47,7 +48,6 @@
"@pnpm/types": "workspace:*",
"archy": "catalog:",
"dint": "catalog:",
"load-json-file": "catalog:",
"p-filter": "catalog:",
"ramda": "catalog:",
"render-help": "catalog:"

4
store/plugin-commands-store/src/storeStatus/index.ts
View File

@@ -1,4 +1,5 @@
import path from 'path'
import { readV8FileStrictAsync } from '@pnpm/fs.v8-file'
import { getIndexFilePathInCafs, type PackageFilesIndex } from '@pnpm/store.cafs'
import { getContextForSingleImporter } from '@pnpm/get-context'
import {
@@ -10,7 +11,6 @@ import { streamParser } from '@pnpm/logger'
import * as dp from '@pnpm/dependency-path'
import { type DepPath } from '@pnpm/types'
import dint from 'dint'
import { loadJsonFile } from 'load-json-file'
import pFilter from 'p-filter'
import {
extendStoreStatusOptions,
@@ -52,7 +52,7 @@ export async function storeStatus (maybeOpts: StoreStatusOptions): Promise<strin
const pkgIndexFilePath = integrity
? getIndexFilePathInCafs(storeDir, integrity, id)
: path.join(storeDir, dp.depPathToFilename(id, maybeOpts.virtualStoreDirMaxLength), 'integrity.json')
const { files } = await loadJsonFile<PackageFilesIndex>(pkgIndexFilePath)
const { files } = await readV8FileStrictAsync<PackageFilesIndex>(pkgIndexFilePath)
return (await dint.check(path.join(virtualStoreDir, dp.depPathToFilename(depPath, maybeOpts.virtualStoreDirMaxLength), 'node_modules', name), files)) === false
}, { concurrency: 8 })

3
store/plugin-commands-store/tsconfig.json
View File

@@ -27,6 +27,9 @@
{
"path": "../../exec/plugin-commands-script-runners"
},
{
"path": "../../fs/v8-file"
},
{
"path": "../../lockfile/fs"
},

3
store/server/package.json
View File

@@ -44,15 +44,16 @@
},
"devDependencies": {
"@pnpm/client": "workspace:*",
"@pnpm/fs.v8-file": "workspace:*",
"@pnpm/logger": "workspace:*",
"@pnpm/package-requester": "workspace:*",
"@pnpm/package-store": "workspace:*",
"@pnpm/server": "workspace:*",
"@pnpm/store.cafs": "workspace:*",
"@types/uuid": "catalog:",
"@zkochan/rimraf": "catalog:",
"get-port": "catalog:",
"is-port-reachable": "catalog:",
"load-json-file": "catalog:",
"node-fetch": "catalog:",
"tempy": "catalog:"
},

12
store/server/test/index.ts
View File

@@ -1,14 +1,16 @@
/// <reference path="../../../__typings__/index.d.ts"/>
import fs from 'fs'
import path from 'path'
import v8 from 'v8'
import getPort from 'get-port'
import { createClient } from '@pnpm/client'
import { readV8FileStrictSync } from '@pnpm/fs.v8-file'
import { type PackageFilesIndex } from '@pnpm/store.cafs'
import { createPackageStore } from '@pnpm/package-store'
import { connectStoreController, createServer } from '@pnpm/server'
import { type Registries } from '@pnpm/types'
import fetch from 'node-fetch'
import { sync as rimraf } from '@zkochan/rimraf'
import { loadJsonFileSync } from 'load-json-file'
import { temporaryDirectory } from 'tempy'
import isPortReachable from 'is-port-reachable'
@@ -172,19 +174,19 @@ test('server upload', async () => {
const fakeEngine = 'client-engine'
const filesIndexFile = path.join(storeDir, 'fake-pkg@1.0.0.json')
fs.writeFileSync(filesIndexFile, JSON.stringify({
fs.writeFileSync(filesIndexFile, v8.serialize({
name: 'fake-pkg',
version: '1.0.0',
files: {},
}), 'utf8')
}))
await storeCtrl.upload(path.join(import.meta.dirname, '__fixtures__/side-effect-fake-dir'), {
sideEffectsCacheKey: fakeEngine,
filesIndexFile,
})
const cacheIntegrity = loadJsonFileSync<any>(filesIndexFile) // eslint-disable-line @typescript-eslint/no-explicit-any
expect(Object.keys(cacheIntegrity?.['sideEffects'][fakeEngine].added).sort()).toStrictEqual(['side-effect.js', 'side-effect.txt'])
const cacheIntegrity = readV8FileStrictSync<PackageFilesIndex>(filesIndexFile)
expect(Object.keys(cacheIntegrity.sideEffects![fakeEngine].added!).sort()).toStrictEqual(['side-effect.js', 'side-effect.txt'])
await server.close()
await storeCtrl.close()

6
store/server/tsconfig.json
View File

@@ -9,6 +9,9 @@
"../../__typings__/**/*.d.ts"
],
"references": [
{
"path": "../../fs/v8-file"
},
{
"path": "../../network/fetch"
},
@@ -24,6 +27,9 @@
{
"path": "../../pkg-manager/package-requester"
},
{
"path": "../cafs"
},
{
"path": "../package-store"
},

2
worker/package.json
View File

@@ -37,12 +37,12 @@
"@pnpm/error": "workspace:*",
"@pnpm/exec.pkg-requires-build": "workspace:*",
"@pnpm/fs.hard-link-dir": "workspace:*",
"@pnpm/fs.v8-file": "workspace:*",
"@pnpm/graceful-fs": "workspace:*",
"@pnpm/store.cafs": "workspace:*",
"@pnpm/symlink-dependency": "workspace:*",
"@rushstack/worker-pool": "catalog:",
"is-windows": "catalog:",
"load-json-file": "catalog:",
"p-limit": "catalog:",
"shlex": "catalog:"
},

19
worker/src/start.ts
View File

@@ -1,6 +1,8 @@
import crypto from 'crypto'
import v8 from 'v8'
import path from 'path'
import fs from 'fs'
import { readV8FileStrictSync } from '@pnpm/fs.v8-file'
import gfs from '@pnpm/graceful-fs'
import { type Cafs, type PackageFiles, type SideEffects, type SideEffectsDiff } from '@pnpm/cafs-types'
import { createCafsStore } from '@pnpm/create-cafs-store'
@@ -18,7 +20,6 @@ import {
} from '@pnpm/store.cafs'
import { symlinkDependencySync } from '@pnpm/symlink-dependency'
import { type DependencyManifest } from '@pnpm/types'
import { loadJsonFileSync } from 'load-json-file'
import { parentPort } from 'worker_threads'
import {
type AddDirToStoreMessage,
@@ -80,7 +81,7 @@ async function handleMessage (
let { storeDir, filesIndexFile, readManifest, verifyStoreIntegrity } = message
let pkgFilesIndex: PackageFilesIndex | undefined
try {
pkgFilesIndex = loadJsonFileSync<PackageFilesIndex>(filesIndexFile)
pkgFilesIndex = readV8FileStrictSync(filesIndexFile)
} catch {
// ignoring. It is fine if the integrity file is not present. Just refetch the package
}
@@ -211,7 +212,7 @@ function addFilesFromDir ({ dir, storeDir, filesIndexFile, sideEffectsCacheKey,
if (sideEffectsCacheKey) {
let filesIndex!: PackageFilesIndex
try {
filesIndex = loadJsonFileSync<PackageFilesIndex>(filesIndexFile)
filesIndex = readV8FileStrictSync<PackageFilesIndex>(filesIndexFile)
} catch {
// If there is no existing index file, then we cannot store the side effects.
return {
@@ -230,7 +231,7 @@ function addFilesFromDir ({ dir, storeDir, filesIndexFile, sideEffectsCacheKey,
} else {
requiresBuild = filesIndex.requiresBuild
}
writeJsonFile(filesIndexFile, filesIndex)
writeV8File(filesIndexFile, filesIndex)
} else {
requiresBuild = writeFilesIndexFile(filesIndexFile, { manifest: manifest ?? {}, files: filesIntegrity })
}
@@ -343,19 +344,19 @@ function writeFilesIndexFile (
files,
sideEffects,
}
writeJsonFile(filesIndexFile, filesIndex)
writeV8File(filesIndexFile, filesIndex)
return requiresBuild
}
function writeJsonFile (filePath: string, data: unknown): void {
function writeV8File (filePath: string, data: unknown): void {
const targetDir = path.dirname(filePath)
// TODO: use the API of @pnpm/cafs to write this file
// There is actually no need to create the directory in 99% of cases.
// So by using cafs API, we'll improve performance.
fs.mkdirSync(targetDir, { recursive: true })
// We remove the "-index.json" from the end of the temp file name
// We remove the "-index.v8" from the end of the temp file name
// in order to avoid ENAMETOOLONG errors
const temp = `${filePath.slice(0, -11)}${process.pid}`
gfs.writeFileSync(temp, JSON.stringify(data))
const temp = `${filePath.slice(0, -9)}${process.pid}`
gfs.writeFileSync(temp, v8.serialize(data))
optimisticRenameOverwrite(temp, filePath)
}

3
worker/tsconfig.json
View File

@@ -21,6 +21,9 @@
{
"path": "../fs/symlink-dependency"
},
{
"path": "../fs/v8-file"
},
{
"path": "../packages/error"
},