perf: persist bundled manifest in store index to avoid reading package.json from CAFS (#10473)

close #10461
This commit is contained in:
Zoltan Kochan
2026-02-17 12:03:08 +01:00
committed by GitHub
parent 3846366bb0
commit 56a59df674
28 changed files with 464 additions and 290 deletions

View File

@@ -0,0 +1,6 @@
---
"@pnpm/package-requester": patch
"pnpm": patch
---
Check if a package is installable for non npm-hosted packages (e.g., git or tarball dependencies) after the manifest has been fetched.

View File

@@ -0,0 +1,12 @@
---
"@pnpm/store.cafs": major
"@pnpm/store-controller-types": major
"@pnpm/worker": major
"@pnpm/package-requester": major
"@pnpm/npm-resolver": major
"@pnpm/reviewing.dependencies-hierarchy": major
"@pnpm/build-modules": major
"pnpm": major
---
Store the bundled manifest (name, version, bin, engines, scripts, etc.) directly in the package index file, eliminating the need to read `package.json` from the content-addressable store during resolution and installation. This reduces I/O and speeds up repeat installs [#10473](https://github.com/pnpm/pnpm/pull/10473).

View File

@@ -0,0 +1,6 @@
---
"@pnpm/build-modules": patch
"pnpm": patch
---
Defer patch errors until all patches in a group are applied, so that one failed patch does not prevent other patches from being attempted.

View File

@@ -2,109 +2,117 @@
"name": "bench-project",
"version": "1.0.0",
"dependencies": {
"express": "^4.21.0",
"animate.less": "^2.2.0",
"autoprefixer": "^10.4.17",
"babel-core": "^6.26.3",
"babel-eslint": "^10.1.0",
"babel-loader": "^9.1.3",
"babel-plugin-lodash": "^3.3.4",
"babel-plugin-module-resolver": "^5.0.0",
"babel-plugin-transform-decorators-legacy": "^1.3.5",
"babel-plugin-transform-runtime": "^6.23.0",
"babel-polyfill": "^6.26.0",
"babel-preset-es2015": "^6.24.1",
"babel-preset-react": "^6.24.1",
"babel-preset-react-hmre": "^1.1.1",
"babel-preset-stage-1": "^6.24.1",
"babel-runtime": "^6.26.0",
"clean-webpack-plugin": "^4.0.0",
"core-decorators": "^0.20.0",
"css-loader": "^6.9.0",
"css-mqpacker": "^7.0.0",
"cssnano": "^6.0.3",
"custom-event-polyfill": "^1.0.7",
"draft-js": "^0.11.7",
"ejs": "^3.1.9",
"eslint": "^8.56.0",
"eslint-config-airbnb": "^19.0.4",
"eslint-import-resolver-webpack": "^0.13.8",
"eslint-plugin-import": "^2.29.1",
"eslint-plugin-jsx-a11y": "^6.8.0",
"eslint-plugin-react": "^7.33.2",
"express": "^4.18.2",
"express-http-proxy": "^2.0.0",
"font-awesome": "^4.7.0",
"fready": "^1.0.0",
"glob": "^10.3.10",
"gulp": "^4.0.2",
"gulp-concat": "^2.6.1",
"gulp-csslint": "^1.0.1",
"gulp-cssnano": "^2.1.3",
"gulp-eol": "^0.2.0",
"gulp-less": "^5.0.0",
"gulp-livereload": "^4.0.2",
"gulp-minify-css": "^1.2.4",
"gulp-postcss": "^9.1.0",
"gulp-rename": "^2.0.0",
"gulp-util": "^3.0.8",
"happypack": "^5.0.1",
"highcharts": "^11.3.0",
"highcharts-solid-gauge": "^0.1.7",
"history": "^5.3.0",
"howler": "^2.2.4",
"imports-loader": "^5.0.0",
"jquery": "^3.7.1",
"jquery-ui": "1.13.2",
"js-cookie": "^3.0.5",
"json-loader": "^0.5.7",
"leftpad": "^0.0.1",
"less": "^4.2.0",
"lesshat": "^4.1.0",
"lodash": "^4.17.21",
"axios": "^1.7.0",
"chalk": "^4.1.2",
"debug": "^4.3.4",
"commander": "^12.0.0",
"glob": "^10.3.0",
"minimatch": "^9.0.0",
"semver": "^7.6.0",
"yargs": "^17.7.0",
"inquirer": "^9.2.0",
"ora": "^5.4.1",
"fs-extra": "^11.2.0",
"rimraf": "^5.0.0",
"mkdirp": "^3.0.0",
"cross-spawn": "^7.0.3",
"execa": "^5.1.1",
"which": "^4.0.0",
"dotenv": "^16.4.0",
"uuid": "^9.0.0",
"moment": "^2.30.0",
"dayjs": "^1.11.0",
"date-fns": "^3.0.0",
"fast-glob": "^3.3.0",
"chokidar": "^3.6.0",
"ws": "^8.16.0",
"node-fetch": "^2.7.0",
"form-data": "^4.0.0",
"http-proxy-agent": "^7.0.0",
"https-proxy-agent": "^7.0.0",
"js-yaml": "^4.1.0",
"ini": "^4.1.0",
"toml": "^3.0.0",
"strip-ansi": "^6.0.1",
"wrap-ansi": "^7.0.0",
"string-width": "^4.2.3",
"cli-table3": "^0.6.3",
"figures": "^3.2.0",
"log-symbols": "^4.1.0",
"boxen": "^5.1.2",
"p-limit": "^3.1.0",
"p-map": "^4.0.0",
"p-queue": "^6.6.2",
"retry": "^0.13.1",
"graceful-fs": "^4.2.11",
"jsonfile": "^6.1.0",
"tar": "^7.0.0",
"archiver": "^7.0.0",
"decompress": "^4.2.1",
"mime-types": "^2.1.35",
"content-type": "^1.0.5",
"accepts": "^1.3.8",
"negotiator": "^0.6.3",
"cors": "^2.8.5",
"helmet": "^7.1.0",
"compression": "^1.7.4",
"cookie-parser": "^1.4.6",
"body-parser": "^1.20.2",
"multer": "^1.4.5-lts.1",
"morgan": "^1.10.0",
"winston": "^3.11.0",
"pino": "^8.18.0",
"bunyan": "^1.8.15",
"ajv": "^8.12.0",
"zod": "^3.22.0",
"joi": "^17.12.0",
"yup": "^1.3.0",
"ramda": "^0.29.0",
"underscore": "^1.13.6",
"rxjs": "^7.8.1",
"eventemitter3": "^5.0.1",
"bluebird": "^3.7.2",
"async": "^3.2.5",
"lru-cache": "^10.2.0",
"node-cache": "^5.1.2",
"keyv": "^4.5.4",
"got": "^11.8.6",
"medium-draft": "^0.5.18",
"mobx": "^6.12.0",
"mobx-react": "^9.1.0",
"moment": "^2.30.1",
"moment-range": "^4.0.2",
"moment-timezone": "^0.5.44",
"password-policy": "0.0.3",
"postcss-reporter": "^7.1.0",
"progress": "^2.0.3",
"qs": "^6.11.2",
"raw-loader": "^4.0.2",
"rc-slider": "^10.5.0",
"react": "^18.2.0",
"react-addons-css-transition-group": "^15.6.2",
"react-addons-shallow-compare": "^15.6.3",
"react-dnd": "^16.0.1",
"react-dnd-html5-backend": "^16.0.1",
"react-dom": "^18.2.0",
"react-draft-wysiwyg": "^1.15.0",
"react-dropzone": "^14.2.3",
"react-grid-layout": "^1.4.4",
"react-highcharts": "^16.1.0",
"react-hot-loader": "4.13.1",
"react-input-calendar": "^0.5.4",
"react-lazyload": "^3.2.0",
"react-measure": "^2.5.2",
"react-mixin": "^5.0.0",
"react-responsive": "9.0.2",
"react-responsive-tabs": "^4.4.3",
"react-router": "^6.21.2",
"react-router-dom": "^6.21.2",
"react-select-plus": "1.0.0-rc.3.patch12",
"react-skylight": "^0.5.1",
"react-sortablejs": "^6.1.4",
"react-tappable": "^1.0.4",
"react-tooltip": "5.25.2",
"react-virtualized": "^9.22.5",
"react-waypoint": "^10.3.0",
"sortablejs": "^1.15.2",
"style-loader": "^3.3.4",
"stylelint": "^16.1.0",
"superagent": "^8.1.2",
"cheerio": "^1.0.0-rc.12",
"marked": "^12.0.0",
"highlight.js": "^11.9.0",
"sharp": "^0.33.0",
"jimp": "^0.22.12",
"canvas": "^2.11.2",
"socket.io": "^4.7.0",
"redis": "^4.6.0",
"ioredis": "^5.3.0",
"mongoose": "^8.1.0",
"typeorm": "^0.3.20",
"knex": "^3.1.0",
"pg": "^8.11.0",
"mysql2": "^3.9.0",
"better-sqlite3": "^9.4.0",
"bcrypt": "^5.1.1",
"jsonwebtoken": "^9.0.2",
"passport": "^0.7.0",
"nanoid": "^3.3.7",
"cuid": "^3.0.0",
"shortid": "^2.2.16",
"color": "^4.2.3",
"pluralize": "^8.0.0",
"change-case": "^4.1.2",
"camelcase": "^6.3.0",
"escape-string-regexp": "^4.0.0"
"uglify-js": "^3.17.4",
"uuid": "^9.0.1",
"verge": "^1.10.2",
"webpack-bundle-analyzer": "^4.10.1",
"webpack-hot-middleware": "^2.26.0",
"webpack-notifier": "^1.15.0",
"webpack-split-by-path": "^2.0.0",
"whatwg-fetch": "^3.6.20"
},
"devDependencies": {
"nan-as": "^1.6.1"
}
}
}

View File

@@ -103,7 +103,24 @@ export async function buildModules<T extends string> (
}
)
})
await runGroups(getWorkspaceConcurrency(opts.childConcurrency), groups)
const patchErrors: Error[] = []
const groupsWithPatchErrors = groups.map((group) =>
group.map((task) => async () => {
try {
await task()
} catch (err: unknown) {
if (util.types.isNativeError(err) && 'code' in err && err.code === 'ERR_PNPM_PATCH_FAILED') {
patchErrors.push(err)
} else {
throw err
}
}
})
)
await runGroups(getWorkspaceConcurrency(opts.childConcurrency), groupsWithPatchErrors)
if (patchErrors.length > 0) {
throw patchErrors[0]
}
return { ignoredBuilds }
}
@@ -247,7 +264,7 @@ export async function linkBinsOfDependencies<T extends string> (
const pkgs = await Promise.all(pkgNodes
.map(async (dep) => ({
location: dep.dir,
manifest: (await dep.fetching?.())?.bundledManifest ?? (await safeReadPackageJsonFromDir(dep.dir) as DependencyManifest) ?? {},
manifest: ((await dep.fetching?.())?.bundledManifest ?? (await safeReadPackageJsonFromDir(dep.dir))) as DependencyManifest ?? {},
}))
)

View File

@@ -5,7 +5,7 @@ import {
type BinaryResolution,
} from '@pnpm/resolver-base'
import { type Cafs, type FilesMap } from '@pnpm/cafs-types'
import { type AllowBuild, type DependencyManifest } from '@pnpm/types'
import { type AllowBuild, type BundledManifest, type DependencyManifest } from '@pnpm/types'
export interface PkgNameVersion {
name?: string
@@ -31,7 +31,7 @@ export type FetchFunction<FetcherResolution = Resolution, Options = FetchOptions
export interface FetchResult {
local?: boolean
manifest?: DependencyManifest
manifest?: BundledManifest
filesMap: FilesMap
requiresBuild: boolean
integrity?: string
@@ -46,7 +46,7 @@ export interface GitFetcherOptions {
export interface GitFetcherResult {
filesMap: FilesMap
manifest?: DependencyManifest
manifest?: BundledManifest
requiresBuild: boolean
}

View File

@@ -6,7 +6,7 @@ import { type Cafs, type FilesMap } from '@pnpm/cafs-types'
import { packlist } from '@pnpm/fs.packlist'
import { globalWarn } from '@pnpm/logger'
import { preparePackage } from '@pnpm/prepare-package'
import { type DependencyManifest } from '@pnpm/types'
import { type BundledManifest } from '@pnpm/types'
import { addFilesFromDir } from '@pnpm/worker'
import renameOverwrite from 'rename-overwrite'
import { fastPathTemp as pathTemp } from 'path-temp'
@@ -53,7 +53,7 @@ export function createGitHostedTarballFetcher (fetchRemoteTarball: FetchFunction
interface PrepareGitHostedPkgResult {
filesMap: FilesMap
manifest?: DependencyManifest
manifest?: BundledManifest
ignoredBuild: boolean
}

View File

@@ -183,6 +183,29 @@ export interface PackageManifest extends DependencyManifest {
deprecated?: string
}
/**
* Subset of package.json fields cached in the store index.
* Used for bin linking, build scripts, runtime selection, and dependency resolution.
*/
export type BundledManifest = Pick<
BaseManifest,
| 'bin'
| 'bundledDependencies'
| 'bundleDependencies'
| 'cpu'
| 'dependencies'
| 'directories'
| 'engines'
| 'libc'
| 'name'
| 'optionalDependencies'
| 'os'
| 'peerDependencies'
| 'peerDependenciesMeta'
| 'scripts'
| 'version'
>
export interface SupportedArchitectures {
os?: string[]
cpu?: string[]

View File

@@ -42,12 +42,12 @@
"@pnpm/hooks.types": "workspace:*",
"@pnpm/package-is-installable": "workspace:*",
"@pnpm/pick-fetcher": "workspace:*",
"@pnpm/read-package-json": "workspace:*",
"@pnpm/resolver-base": "workspace:*",
"@pnpm/store-controller-types": "workspace:*",
"@pnpm/store.cafs": "workspace:*",
"@pnpm/types": "workspace:*",
"detect-libc": "catalog:",
"load-json-file": "catalog:",
"p-defer": "catalog:",
"p-limit": "catalog:",
"p-queue": "catalog:",
@@ -75,7 +75,6 @@
"@types/semver": "catalog:",
"@types/ssri": "catalog:",
"delay": "catalog:",
"load-json-file": "catalog:",
"nock": "catalog:",
"normalize-path": "catalog:",
"tempy": "catalog:"

View File

@@ -2,6 +2,7 @@ import { createReadStream, promises as fs } from 'fs'
import path from 'path'
import {
getIndexFilePathInCafs as _getIndexFilePathInCafs,
normalizeBundledManifest,
} from '@pnpm/store.cafs'
import { fetchingProgressLogger, progressLogger } from '@pnpm/core-loggers'
import { pickFetcher } from '@pnpm/pick-fetcher'
@@ -16,7 +17,7 @@ import { type Cafs } from '@pnpm/cafs-types'
import gfs from '@pnpm/graceful-fs'
import { logger } from '@pnpm/logger'
import { packageIsInstallable } from '@pnpm/package-is-installable'
import { readPackageJson } from '@pnpm/read-package-json'
import { loadJsonFile } from 'load-json-file'
import {
type PlatformAssetResolution,
type DirectoryResolution,
@@ -53,7 +54,6 @@ import PQueue from 'p-queue'
import pDefer, { type DeferredPromise } from 'p-defer'
import pShare from 'promise-share'
import { pick } from 'ramda'
import semver from 'semver'
import ssri from 'ssri'
let currentLibc: 'glibc' | 'musl' | undefined | null
@@ -66,29 +66,6 @@ function getLibcFamilySync () {
const TARBALL_INTEGRITY_FILENAME = 'tarball-integrity'
const packageRequestLogger = logger('package-requester')
const pickBundledManifest = pick([
'bin',
'bundledDependencies',
'bundleDependencies',
'cpu',
'dependencies',
'directories',
'engines',
'name',
'optionalDependencies',
'os',
'peerDependencies',
'peerDependenciesMeta',
'scripts',
'version',
])
function normalizeBundledManifest (manifest: DependencyManifest): BundledManifest {
return {
...pickBundledManifest(manifest),
version: semver.clean(manifest.version ?? '0.0.0', { loose: true }) ?? manifest.version,
}
}
export function createPackageRequester (
opts: {
@@ -248,7 +225,7 @@ async function resolveAndFetch (
}
}
const isInstallable = (
let isInstallable: boolean | null | undefined = (
ctx.force === true ||
(
manifest == null
@@ -303,12 +280,26 @@ async function resolveAndFetch (
if (!manifest) {
const fetchedResult = await fetchResult.fetching()
manifest = fetchedResult.bundledManifest
if (fetchedResult.bundledManifest) {
manifest = fetchedResult.bundledManifest as DependencyManifest
} else if (fetchedResult.files.filesMap.has('package.json')) {
manifest = await loadJsonFile<DependencyManifest>(fetchedResult.files.filesMap.get('package.json')!)
}
// Add integrity to resolution if it was computed during fetching (only for TarballResolution)
if (fetchedResult.integrity && !resolution.type && !(resolution as TarballResolution).integrity) {
(resolution as TarballResolution).integrity = fetchedResult.integrity
}
}
// Check installability now that we have the manifest (for git/tarball packages without registry metadata)
if (isInstallable === undefined && manifest != null) {
isInstallable = ctx.force === true || packageIsInstallable(id, manifest, {
engineStrict: ctx.engineStrict,
lockfileDir: options.lockfileDir,
nodeVersion: ctx.nodeVersion,
optional: wantedDependency.optional === true,
supportedArchitectures: options.supportedArchitectures,
})
}
return {
body: {
id,
@@ -537,14 +528,14 @@ function fetchToStore (
) &&
!isLocalPkg
) {
const { verified, files, manifest } = await ctx.readPkgFromCafs(filesIndexFile, {
const { verified, files, bundledManifest } = await ctx.readPkgFromCafs(filesIndexFile, {
readManifest: opts.fetchRawManifest,
expectedPkg: opts.pkg,
})
if (verified) {
fetching.resolve({
files,
bundledManifest: manifest == null ? manifest : normalizeBundledManifest(manifest),
bundledManifest,
})
return
}
@@ -608,7 +599,7 @@ function fetchToStore (
packageImportMethod: (fetchedPackage as DirectoryFetcherResult).packageImportMethod,
requiresBuild: fetchedPackage.requiresBuild,
},
bundledManifest: fetchedPackage.manifest == null ? fetchedPackage.manifest : normalizeBundledManifest(fetchedPackage.manifest),
bundledManifest: fetchedPackage.manifest,
integrity,
})
} catch (err: any) { // eslint-disable-line
@@ -617,8 +608,8 @@ function fetchToStore (
}
}
async function readBundledManifest (pkgJsonPath: string): Promise<BundledManifest> {
return pickBundledManifest(await readPackageJson(pkgJsonPath) as DependencyManifest)
async function readBundledManifest (pkgJsonPath: string): Promise<BundledManifest | undefined> {
return normalizeBundledManifest(await loadJsonFile<DependencyManifest>(pkgJsonPath))
}
async function tarballIsUpToDate (

View File

@@ -473,7 +473,6 @@ test('fetchPackageToStore()', async () => {
{
engines: { node: '>=0.10.0' },
name: 'is-positive',
scripts: { test: 'node test.js' },
version: '1.0.0',
}
)
@@ -668,7 +667,6 @@ test('always return a package manifest in the response', async () => {
{
engines: { node: '>=0.10.0' },
name: 'is-positive',
scripts: { test: 'node test.js' },
version: '1.0.0',
}
)

View File

@@ -45,9 +45,6 @@
{
"path": "../../packages/types"
},
{
"path": "../../pkg-manifest/read-package-json"
},
{
"path": "../../resolving/resolver-base"
},

24
pnpm-lock.yaml generated
View File

@@ -5772,9 +5772,6 @@ importers:
'@pnpm/pick-fetcher':
specifier: workspace:*
version: link:../../fetching/pick-fetcher
'@pnpm/read-package-json':
specifier: workspace:*
version: link:../../pkg-manifest/read-package-json
'@pnpm/resolver-base':
specifier: workspace:*
version: link:../../resolving/resolver-base
@@ -5793,6 +5790,9 @@ importers:
detect-libc:
specifier: 'catalog:'
version: 2.1.2
load-json-file:
specifier: 'catalog:'
version: 7.0.1
p-defer:
specifier: 'catalog:'
version: 4.0.1
@@ -5857,9 +5857,6 @@ importers:
delay:
specifier: 'catalog:'
version: 6.0.0
load-json-file:
specifier: 'catalog:'
version: 7.0.1
nock:
specifier: 'catalog:'
version: 13.3.4
@@ -7707,6 +7704,9 @@ importers:
'@pnpm/util.lex-comparator':
specifier: 'catalog:'
version: 3.0.2
load-json-file:
specifier: 'catalog:'
version: 7.0.1
normalize-path:
specifier: 'catalog:'
version: 3.0.0
@@ -8183,6 +8183,9 @@ importers:
'@pnpm/store-controller-types':
specifier: workspace:*
version: link:../store-controller-types
'@pnpm/types':
specifier: workspace:*
version: link:../../packages/types
'@zkochan/rimraf':
specifier: 'catalog:'
version: 3.0.2
@@ -8198,6 +8201,9 @@ importers:
rename-overwrite:
specifier: 'catalog:'
version: 6.0.3
semver:
specifier: 'catalog:'
version: 7.7.4
strip-bom:
specifier: 'catalog:'
version: 5.0.0
@@ -8211,15 +8217,15 @@ importers:
'@pnpm/test-fixtures':
specifier: workspace:*
version: link:../../__utils__/test-fixtures
'@pnpm/types':
specifier: workspace:*
version: link:../../packages/types
'@types/is-gzip':
specifier: 'catalog:'
version: 2.0.0
'@types/node':
specifier: 'catalog:'
version: 22.19.11
'@types/semver':
specifier: 'catalog:'
version: 7.7.1
symlink-dir:
specifier: 'catalog:'
version: 7.1.0

View File

@@ -507,10 +507,9 @@ test('installation fails when the stored package name and version do not match t
const cacheIntegrityPath = getIndexFilePathInCafs(path.join(storeDir, STORE_VERSION), getIntegrity('@pnpm.e2e/dep-of-pkg-with-1-dep', '100.1.0'), '@pnpm.e2e/dep-of-pkg-with-1-dep@100.1.0')
const cacheIntegrity = readMsgpackFileSync<PackageFilesIndex>(cacheIntegrityPath)
cacheIntegrity.name = 'foo'
writeMsgpackFileSync(cacheIntegrityPath, {
...cacheIntegrity,
name: 'foo',
manifest: { ...cacheIntegrity.manifest, name: 'foo' },
})
rimraf('node_modules')

View File

@@ -192,16 +192,15 @@ export function createNpmResolver (
const request = readPkgFromCafs(
{
storeDir,
verifyStoreIntegrity: true,
verifyStoreIntegrity: false,
},
filesIndexFile,
{
readManifest: true,
expectedPkg: { name: peekOpts.name, version: peekOpts.version },
}
).then(({ manifest, verified }) => {
if (!verified) return undefined
return manifest
).then(({ bundledManifest }) => {
if (!bundledManifest) return undefined
return bundledManifest as DependencyManifest
}).catch(() => undefined)
peekLockerForPeek.set(filesIndexFile, request)
return request

View File

@@ -48,6 +48,7 @@
"@pnpm/store.cafs": "workspace:*",
"@pnpm/types": "workspace:*",
"@pnpm/util.lex-comparator": "catalog:",
"load-json-file": "catalog:",
"normalize-path": "catalog:",
"realpath-missing": "catalog:",
"resolve-link-target": "catalog:",

View File

@@ -1,5 +1,6 @@
import { readMsgpackFileSync } from '@pnpm/fs.msgpack-file'
import { getIndexFilePathInCafs, readManifestFromStore, type PackageFilesIndex } from '@pnpm/store.cafs'
import { loadJsonFileSync } from 'load-json-file'
import { getIndexFilePathInCafs, getFilePathByModeInCafs, type PackageFilesIndex } from '@pnpm/store.cafs'
import { type DependencyManifest } from '@pnpm/types'
/**
@@ -15,8 +16,11 @@ export function readManifestFromCafs (storeDir: string, pkg: {
const pkgId = `${pkg.name}@${pkg.version}`
const indexPath = getIndexFilePathInCafs(storeDir, pkg.integrity, pkgId)
const pkgIndex = readMsgpackFileSync<PackageFilesIndex>(indexPath)
const manifest = readManifestFromStore(storeDir, pkgIndex)
if (manifest) return manifest as DependencyManifest
const pkgJsonEntry = pkgIndex.files.get('package.json')
if (pkgJsonEntry) {
const filePath = getFilePathByModeInCafs(storeDir, pkgJsonEntry.digest, pkgJsonEntry.mode)
return loadJsonFileSync<DependencyManifest>(filePath)
}
} catch {
// Fall through to undefined
}

View File

@@ -36,20 +36,22 @@
"@pnpm/fetcher-base": "workspace:*",
"@pnpm/graceful-fs": "workspace:*",
"@pnpm/store-controller-types": "workspace:*",
"@pnpm/types": "workspace:*",
"@zkochan/rimraf": "catalog:",
"is-gzip": "catalog:",
"is-subdir": "catalog:",
"p-limit": "catalog:",
"rename-overwrite": "catalog:",
"semver": "catalog:",
"strip-bom": "catalog:"
},
"devDependencies": {
"@pnpm/cafs-types": "workspace:*",
"@pnpm/store.cafs": "workspace:*",
"@pnpm/test-fixtures": "workspace:*",
"@pnpm/types": "workspace:*",
"@types/is-gzip": "catalog:",
"@types/node": "catalog:",
"@types/semver": "catalog:",
"symlink-dir": "catalog:",
"tempy": "catalog:"
},

View File

@@ -4,11 +4,9 @@ import util from 'util'
import { PnpmError } from '@pnpm/error'
import { type PackageFiles, type PackageFileInfo, type SideEffects, type FilesMap } from '@pnpm/cafs-types'
import gfs from '@pnpm/graceful-fs'
import { type DependencyManifest } from '@pnpm/types'
import { type BundledManifest } from '@pnpm/types'
import rimraf from '@zkochan/rimraf'
import { getFilePathByModeInCafs } from './getFilePathInCafs.js'
import { parseJsonBufferSync } from './parseJson.js'
import { readManifestFromStore } from './readManifestFromStore.js'
export interface Integrity {
digest: string
@@ -24,36 +22,28 @@ global['verifiedFileIntegrity'] = 0
export interface VerifyResult {
passed: boolean
manifest?: DependencyManifest
filesMap: FilesMap
sideEffectsMaps?: Map<string, { added?: FilesMap, deleted?: string[] }>
}
export interface PackageFilesIndex {
// name and version are nullable for backward compatibility
// the initial specs of pnpm store v3 did not require these fields.
// However, it might be possible that some types of dependencies don't
// have the name/version fields, like the local tarball dependencies.
name?: string
version?: string
manifest?: BundledManifest
requiresBuild?: boolean
algo: string
files: PackageFiles
sideEffects?: SideEffects
}
export function checkPkgFilesIntegrity (
storeDir: string,
pkgIndex: PackageFilesIndex,
readManifest?: boolean
pkgIndex: PackageFilesIndex
): VerifyResult {
// It might make sense to use this cache for all files in the store
// but there's a smaller chance that the same file will be checked twice
// so it's probably not worth the memory (this assumption should be verified)
const verifiedFilesCache = new Set<string>()
const _checkFilesIntegrity = checkFilesIntegrity.bind(null, verifiedFilesCache, storeDir, pkgIndex.algo)
const verified = _checkFilesIntegrity(pkgIndex.files, readManifest)
const verified = _checkFilesIntegrity(pkgIndex.files)
if (!verified.passed) return verified
const sideEffectsMaps = new Map<string, { added?: FilesMap, deleted?: string[] }>()
@@ -88,8 +78,7 @@ export function checkPkgFilesIntegrity (
*/
export function buildFileMapsFromIndex (
storeDir: string,
pkgIndex: PackageFilesIndex,
readManifest?: boolean
pkgIndex: PackageFilesIndex
): VerifyResult {
const filesMap: FilesMap = new Map()
@@ -122,7 +111,6 @@ export function buildFileMapsFromIndex (
return {
passed: true,
manifest: readManifest ? readManifestFromStore(storeDir, pkgIndex) : undefined,
filesMap,
sideEffectsMaps: sideEffectsMaps.size > 0 ? sideEffectsMaps : undefined,
}
@@ -132,11 +120,9 @@ function checkFilesIntegrity (
verifiedFilesCache: Set<string>,
storeDir: string,
algo: string,
files: PackageFiles,
readManifest?: boolean
files: PackageFiles
): VerifyResult {
let allVerified = true
let manifest: DependencyManifest | undefined
const filesMap: FilesMap = new Map()
for (const [f, fstat] of files) {
@@ -146,13 +132,9 @@ function checkFilesIntegrity (
const filename = getFilePathByModeInCafs(storeDir, fstat.digest, fstat.mode)
filesMap.set(f, filename)
const readFile = readManifest && f === 'package.json'
if (!readFile && verifiedFilesCache.has(filename)) continue
const verifyResult = verifyFile(filename, fstat, algo, readFile)
if (readFile) {
manifest = verifyResult.manifest
}
if (verifyResult.passed) {
if (verifiedFilesCache.has(filename)) continue
const passed = verifyFile(filename, fstat, algo)
if (passed) {
verifiedFilesCache.add(filename)
} else {
allVerified = false
@@ -160,7 +142,6 @@ function checkFilesIntegrity (
}
return {
passed: allVerified,
manifest,
filesMap,
}
}
@@ -170,34 +151,26 @@ type FileInfo = Pick<PackageFileInfo, 'size' | 'checkedAt' | 'digest'>
function verifyFile (
filename: string,
fstat: FileInfo,
algorithm: string,
readManifest?: boolean
): Pick<VerifyResult, 'passed' | 'manifest'> {
algorithm: string
): boolean {
const currentFile = checkFile(filename, fstat.checkedAt)
if (currentFile == null) return { passed: false }
if (currentFile == null) return false
if (currentFile.isModified) {
if (currentFile.size !== fstat.size) {
rimraf.sync(filename)
return { passed: false }
}
return verifyFileIntegrity(filename, { digest: fstat.digest, algorithm }, readManifest)
}
if (readManifest) {
return {
passed: true,
manifest: parseJsonBufferSync(gfs.readFileSync(filename)) as DependencyManifest,
return false
}
return verifyFileIntegrity(filename, { digest: fstat.digest, algorithm })
}
// If a file was not edited, we are skipping integrity check.
// We assume that nobody will manually remove a file in the store and create a new one.
return { passed: true }
return true
}
export function verifyFileIntegrity (
filename: string,
integrity: Integrity,
readManifest?: boolean
): Pick<VerifyResult, 'passed' | 'manifest'> {
integrity: Integrity
): boolean {
// @ts-expect-error
global['verifiedFileIntegrity']++
let data: Buffer
@@ -205,7 +178,7 @@ export function verifyFileIntegrity (
data = gfs.readFileSync(filename)
} catch (err: unknown) {
if (util.types.isNativeError(err) && 'code' in err && err.code === 'ENOENT') {
return { passed: false }
return false
}
throw err
}
@@ -214,20 +187,13 @@ export function verifyFileIntegrity (
computedDigest = crypto.hash(integrity.algorithm, data, 'hex')
} catch {
// Invalid algorithm (e.g., corrupted index file) - treat as verification failure
return { passed: false }
return false
}
const passed = computedDigest === integrity.digest
if (!passed) {
gfs.unlinkSync(filename)
return { passed }
}
if (readManifest) {
return {
passed,
manifest: parseJsonBufferSync(data) as DependencyManifest,
}
}
return { passed }
return passed
}
function checkFile (filename: string, checkedAt?: number): { isModified: boolean, size: number } | null {

View File

@@ -17,7 +17,6 @@ import {
type PackageFilesIndex,
type VerifyResult,
} from './checkPkgFilesIntegrity.js'
import { readManifestFromStore } from './readManifestFromStore.js'
import {
getIndexFilePathInCafs,
contentPathFromHex,
@@ -25,14 +24,17 @@ import {
getFilePathByModeInCafs,
modeIsExecutable,
} from './getFilePathInCafs.js'
import { normalizeBundledManifest } from './normalizeBundledManifest.js'
import { optimisticRenameOverwrite, writeBufferToCafs } from './writeBufferToCafs.js'
export const HASH_ALGORITHM = 'sha512'
export { type BundledManifest } from '@pnpm/types'
export { normalizeBundledManifest }
export {
checkPkgFilesIntegrity,
buildFileMapsFromIndex,
readManifestFromStore,
type FileType,
getFilePathByModeInCafs,
getIndexFilePathInCafs,

View File

@@ -0,0 +1,49 @@
import { type BaseManifest, type BundledManifest } from '@pnpm/types'
import semver from 'semver'
const BUNDLED_MANIFEST_FIELDS: Array<keyof BaseManifest> = [
'bin',
'bundledDependencies',
'bundleDependencies',
'cpu',
'dependencies',
'directories',
'engines',
'libc',
'name',
'optionalDependencies',
'os',
'peerDependencies',
'peerDependenciesMeta',
]
const LIFECYCLE_SCRIPTS = ['preinstall', 'install', 'postinstall'] as const
/**
* Picks the subset of manifest fields stored in the package index and normalizes the version.
* Used both when writing the index (worker) and when creating a BundledManifest from a fresh fetch.
*/
export function normalizeBundledManifest (manifest: Partial<BaseManifest>): BundledManifest | undefined {
let result: Record<string, unknown> | undefined
for (const key of BUNDLED_MANIFEST_FIELDS) {
if (manifest[key] != null) {
if (!result) result = {}
result[key] = manifest[key]
}
}
let scripts: Record<string, string> | undefined
if (manifest.scripts) {
for (const key of LIFECYCLE_SCRIPTS) {
if (manifest.scripts[key]) {
if (!scripts) scripts = {}
scripts[key] = manifest.scripts[key]
}
}
}
if (!result && !scripts) return undefined
return {
version: semver.clean(manifest.version ?? '0.0.0', { loose: true }) ?? manifest.version,
...result,
...scripts ? { scripts } : {},
} as BundledManifest
}

View File

@@ -1,14 +0,0 @@
import gfs from '@pnpm/graceful-fs'
import { type PackageManifest } from '@pnpm/types'
import { type PackageFilesIndex } from './checkPkgFilesIntegrity.js'
import { getFilePathByModeInCafs } from './getFilePathInCafs.js'
import { parseJsonBufferSync } from './parseJson.js'
export function readManifestFromStore (storeDir: string, pkgIndex: PackageFilesIndex): PackageManifest | undefined {
const pkg = pkgIndex.files.get('package.json')
if (pkg) {
const fileName = getFilePathByModeInCafs(storeDir, pkg.digest, pkg.mode)
return parseJsonBufferSync(gfs.readFileSync(fileName)) as PackageManifest
}
return undefined
}

View File

@@ -105,5 +105,5 @@ function removeSuffix (filePath: string): string {
function existsSame (filename: string, integrity: Integrity): boolean {
const existingFile = fs.statSync(filename, { throwIfNoEntry: false })
if (!existingFile) return false
return verifyFileIntegrity(filename, integrity).passed
return verifyFileIntegrity(filename, integrity)
}

View File

@@ -0,0 +1,120 @@
import { normalizeBundledManifest } from '../src/normalizeBundledManifest.js'
describe('normalizeBundledManifest', () => {
it('returns undefined for an empty manifest', () => {
expect(normalizeBundledManifest({})).toBeUndefined()
})
it('returns undefined when manifest has only excluded fields', () => {
expect(normalizeBundledManifest({
description: 'a package',
keywords: ['test'],
license: 'MIT',
author: 'test',
repository: 'test/test',
devDependencies: { jest: '^29' },
})).toBeUndefined()
})
it('picks included fields and excludes others', () => {
const result = normalizeBundledManifest({
name: 'foo',
version: '1.0.0',
description: 'should be excluded',
license: 'MIT',
bin: { foo: './bin/foo.js' },
engines: { node: '>=18' },
cpu: ['x64'],
os: ['linux'],
libc: ['glibc'],
dependencies: { bar: '^1.0.0' },
optionalDependencies: { baz: '^2.0.0' },
peerDependencies: { react: '^18' },
peerDependenciesMeta: { react: { optional: true } },
bundledDependencies: ['bar'],
directories: { bin: './bin' },
})
expect(result).toStrictEqual({
name: 'foo',
version: '1.0.0',
bin: { foo: './bin/foo.js' },
engines: { node: '>=18' },
cpu: ['x64'],
os: ['linux'],
libc: ['glibc'],
dependencies: { bar: '^1.0.0' },
optionalDependencies: { baz: '^2.0.0' },
peerDependencies: { react: '^18' },
peerDependenciesMeta: { react: { optional: true } },
bundledDependencies: ['bar'],
directories: { bin: './bin' },
})
// Excluded fields should not be present
expect(result).not.toHaveProperty('description')
expect(result).not.toHaveProperty('license')
})
it('only picks lifecycle scripts, not all scripts', () => {
const result = normalizeBundledManifest({
name: 'foo',
version: '1.0.0',
scripts: {
preinstall: 'echo pre',
install: 'echo install',
postinstall: 'echo post',
test: 'jest',
build: 'tsc',
start: 'node index.js',
prepare: 'tsc',
},
})
expect(result!.scripts).toStrictEqual({
preinstall: 'echo pre',
install: 'echo install',
postinstall: 'echo post',
})
})
it('omits scripts key when no lifecycle scripts exist', () => {
const result = normalizeBundledManifest({
name: 'foo',
version: '1.0.0',
scripts: {
test: 'jest',
build: 'tsc',
},
})
expect(result).not.toHaveProperty('scripts')
})
it('normalizes version with semver.clean', () => {
expect(normalizeBundledManifest({
name: 'foo',
version: ' =v1.2.3 ',
})!.version).toBe('1.2.3')
})
it('keeps version as-is when semver.clean returns null', () => {
expect(normalizeBundledManifest({
name: 'foo',
version: 'not-semver',
})!.version).toBe('not-semver')
})
it('defaults missing version to 0.0.0', () => {
expect(normalizeBundledManifest({
name: 'foo',
})!.version).toBe('0.0.0')
})
it('skips null/undefined fields', () => {
const result = normalizeBundledManifest({
name: 'foo',
version: '1.0.0',
bin: undefined,
engines: undefined,
})
expect(result).not.toHaveProperty('bin')
expect(result).not.toHaveProperty('engines')
})
})

View File

@@ -86,7 +86,7 @@ export async function handler (opts: FindHashCommandOptions, params: string[]):
if (pkgFilesIndex.files) {
for (const file of pkgFilesIndex.files.values()) {
if (file?.digest === hash) {
result.push({ name: pkgFilesIndex.name ?? 'unknown', version: pkgFilesIndex?.version ?? 'unknown', filesIndexFile: filesIndexFile.replace(indexDir, '') })
result.push({ name: pkgFilesIndex.manifest?.name ?? 'unknown', version: pkgFilesIndex.manifest?.version ?? 'unknown', filesIndexFile: filesIndexFile.replace(indexDir, '') })
// a package is only found once.
continue
@@ -99,7 +99,7 @@ export async function handler (opts: FindHashCommandOptions, params: string[]):
if (!added) continue
for (const file of added.values()) {
if (file?.digest === hash) {
result.push({ name: pkgFilesIndex.name ?? 'unknown', version: pkgFilesIndex?.version ?? 'unknown', filesIndexFile: filesIndexFile.replace(indexDir, '') })
result.push({ name: pkgFilesIndex.manifest?.name ?? 'unknown', version: pkgFilesIndex.manifest?.version ?? 'unknown', filesIndexFile: filesIndexFile.replace(indexDir, '') })
// a package is only found once.
continue

View File

@@ -16,8 +16,8 @@ import {
} from '@pnpm/cafs-types'
import {
type AllowBuild,
type BundledManifest,
type SupportedArchitectures,
type DependencyManifest,
type PackageManifest,
type PinnedVersion,
type PackageVersionPolicy,
@@ -27,23 +27,7 @@ import {
export type { PackageFileInfo, PackageFilesResponse, ImportPackageFunction, ImportPackageFunctionAsync, FilesMap }
export * from '@pnpm/resolver-base'
export type BundledManifest = Pick<
DependencyManifest,
| 'bin'
| 'bundledDependencies'
| 'bundleDependencies'
| 'cpu'
| 'dependencies'
| 'directories'
| 'engines'
| 'name'
| 'optionalDependencies'
| 'os'
| 'peerDependencies'
| 'peerDependenciesMeta'
| 'scripts'
| 'version'
>
export type { BundledManifest }
export interface UploadPkgToStoreOpts {
filesIndexFile: string

View File

@@ -6,7 +6,7 @@ import { PnpmError } from '@pnpm/error'
import { execSync } from 'child_process'
import isWindows from 'is-windows'
import { type PackageFilesResponse, type FilesMap } from '@pnpm/cafs-types'
import { type DependencyManifest } from '@pnpm/types'
import { type BundledManifest } from '@pnpm/types'
import pLimit from 'p-limit'
import { globalWarn } from '@pnpm/logger'
import {
@@ -69,7 +69,7 @@ function availableParallelism (): number {
interface AddFilesResult {
filesMap: FilesMap
manifest: DependencyManifest
manifest?: BundledManifest
requiresBuild: boolean
integrity?: string
}
@@ -81,7 +81,7 @@ export async function addFilesFromDir (opts: AddFilesFromDirOptions): Promise<Ad
workerPool = createTarballWorkerPool()
}
const localWorker = await workerPool.checkoutWorkerAsync(true)
return new Promise<{ filesMap: FilesMap, manifest: DependencyManifest, requiresBuild: boolean }>((resolve, reject) => {
return new Promise<AddFilesResult>((resolve, reject) => {
localWorker.once('message', ({ status, error, value }) => {
workerPool!.checkinWorker(localWorker)
if (status === 'error') {
@@ -190,7 +190,7 @@ export interface ReadPkgFromCafsOptions {
export interface ReadPkgFromCafsResult {
verified: boolean
files: PackageFilesResponse
manifest?: DependencyManifest
bundledManifest?: BundledManifest
}
export async function readPkgFromCafs (

View File

@@ -14,13 +14,14 @@ import {
buildFileMapsFromIndex,
createCafs,
HASH_ALGORITHM,
normalizeBundledManifest,
type PackageFilesIndex,
type FilesIndex,
optimisticRenameOverwrite,
type VerifyResult,
} from '@pnpm/store.cafs'
import { symlinkDependencySync } from '@pnpm/symlink-dependency'
import { type DependencyManifest } from '@pnpm/types'
import { type BundledManifest, type DependencyManifest } from '@pnpm/types'
import { parentPort } from 'worker_threads'
import { equalOrSemverEqual } from './equalOrSemverEqual.js'
import {
@@ -78,7 +79,7 @@ async function handleMessage (
break
}
case 'readPkgFromCafs': {
let { storeDir, filesIndexFile, readManifest, verifyStoreIntegrity, expectedPkg, strictStorePkgContentCheck } = message
const { storeDir, filesIndexFile, verifyStoreIntegrity, expectedPkg, strictStorePkgContentCheck } = message
let pkgFilesIndex: PackageFilesIndex | undefined
try {
pkgFilesIndex = readMsgpackFileSync<PackageFilesIndex>(filesIndexFile)
@@ -99,18 +100,18 @@ async function handleMessage (
if (expectedPkg) {
if (
(
pkgFilesIndex.name != null &&
pkgFilesIndex.manifest?.name != null &&
expectedPkg.name != null &&
pkgFilesIndex.name.toLowerCase() !== expectedPkg.name.toLowerCase()
pkgFilesIndex.manifest.name.toLowerCase() !== expectedPkg.name.toLowerCase()
) ||
(
pkgFilesIndex.version != null &&
pkgFilesIndex.manifest?.version != null &&
expectedPkg.version != null &&
!equalOrSemverEqual(pkgFilesIndex.version, expectedPkg.version)
!equalOrSemverEqual(pkgFilesIndex.manifest.version, expectedPkg.version)
)
) {
const msg = 'Package name or version mismatch found while reading from the store.'
const hint = `This means that either the lockfile is broken or the package metadata (name and version) inside the package's package.json file doesn't match the metadata in the registry. Expected package: ${expectedPkg.name}@${expectedPkg.version}. Actual package in the store: ${pkgFilesIndex.name}@${pkgFilesIndex.version}.`
const hint = `This means that either the lockfile is broken or the package metadata (name and version) inside the package's package.json file doesn't match the metadata in the registry. Expected package: ${expectedPkg.name}@${expectedPkg.version}. Actual package in the store: ${pkgFilesIndex.manifest?.name}@${pkgFilesIndex.manifest?.version}.`
if (strictStorePkgContentCheck ?? true) {
throw new PnpmError('UNEXPECTED_PKG_CONTENT_IN_STORE', msg, {
hint: `${hint}\n\nIf you want to ignore this issue, set strictStorePkgContentCheck to false in your configuration`,
@@ -120,24 +121,21 @@ async function handleMessage (
}
}
}
let verifyResult: VerifyResult | undefined
if (pkgFilesIndex.requiresBuild == null) {
readManifest = true
}
// Get file maps and optionally verify
let verifyResult: VerifyResult
if (verifyStoreIntegrity) {
verifyResult = checkPkgFilesIntegrity(storeDir, pkgFilesIndex, readManifest)
verifyResult = checkPkgFilesIntegrity(storeDir, pkgFilesIndex)
} else {
verifyResult = buildFileMapsFromIndex(storeDir, pkgFilesIndex, readManifest)
verifyResult = buildFileMapsFromIndex(storeDir, pkgFilesIndex)
}
const requiresBuild = pkgFilesIndex.requiresBuild ?? pkgRequiresBuild(verifyResult.manifest, verifyResult.filesMap)
const bundledManifest = pkgFilesIndex.manifest
const requiresBuild = pkgFilesIndex.requiresBuild ?? pkgRequiresBuild(bundledManifest, verifyResult.filesMap)
parentPort!.postMessage({
status: 'success',
warnings,
value: {
verified: verifyResult.passed,
manifest: verifyResult.manifest,
bundledManifest,
files: {
filesMap: verifyResult.filesMap,
sideEffectsMaps: verifyResult.sideEffectsMaps,
@@ -196,12 +194,13 @@ function addTarballToStore ({ buffer, storeDir, integrity, filesIndexFile, appen
addManifestToCafs(cafs, filesIndex, appendManifest)
}
const { filesIntegrity, filesMap } = processFilesIndex(filesIndex)
const requiresBuild = writeFilesIndexFile(filesIndexFile, { algo: HASH_ALGORITHM, manifest: manifest ?? {}, files: filesIntegrity })
const bundledManifest = manifest != null ? normalizeBundledManifest(manifest) : undefined
const requiresBuild = writeFilesIndexFile(filesIndexFile, { algo: HASH_ALGORITHM, manifest: bundledManifest, files: filesIntegrity })
return {
status: 'success',
value: {
filesMap,
manifest,
manifest: bundledManifest,
requiresBuild,
integrity: integrity ?? calcIntegrity(buffer),
},
@@ -217,7 +216,7 @@ interface AddFilesFromDirResult {
status: string
value: {
filesMap: FilesMap
manifest?: DependencyManifest
manifest?: BundledManifest
requiresBuild: boolean
}
}
@@ -270,6 +269,7 @@ function addFilesFromDir (
addManifestToCafs(cafs, filesIndex, appendManifest)
}
const { filesIntegrity, filesMap } = processFilesIndex(filesIndex)
const bundledManifest = manifest != null ? normalizeBundledManifest(manifest) : undefined
let requiresBuild: boolean
if (sideEffectsCacheKey) {
let existingFilesIndex!: PackageFilesIndex
@@ -281,7 +281,7 @@ function addFilesFromDir (
status: 'success',
value: {
filesMap,
manifest,
manifest: bundledManifest,
requiresBuild: pkgRequiresBuild(manifest, filesMap),
},
}
@@ -304,9 +304,9 @@ function addFilesFromDir (
}
writeIndexFile(filesIndexFile, existingFilesIndex)
} else {
requiresBuild = writeFilesIndexFile(filesIndexFile, { algo: HASH_ALGORITHM, manifest: manifest ?? {}, files: filesIntegrity })
requiresBuild = writeFilesIndexFile(filesIndexFile, { algo: HASH_ALGORITHM, manifest: bundledManifest, files: filesIntegrity })
}
return { status: 'success', value: { filesMap, manifest, requiresBuild } }
return { status: 'success', value: { filesMap, manifest: bundledManifest, requiresBuild } }
}
function addManifestToCafs (cafs: CafsFunctions, filesIndex: FilesIndex, manifest: DependencyManifest): void {
@@ -414,16 +414,15 @@ function writeFilesIndexFile (
filesIndexFile: string,
{ algo, manifest, files, sideEffects }: {
algo: string
manifest: Partial<DependencyManifest>
manifest?: BundledManifest
files: PackageFiles
sideEffects?: SideEffects
}
): boolean {
const requiresBuild = pkgRequiresBuild(manifest, files)
const filesIndex: PackageFilesIndex = {
name: manifest.name,
version: manifest.version,
requiresBuild,
manifest,
algo,
files,
sideEffects,
@@ -443,4 +442,4 @@ function writeIndexFile (filePath: string, data: PackageFilesIndex): void {
const temp = `${filePath.slice(0, -10)}${process.pid}`
writeMsgpackFileSync(temp, data)
optimisticRenameOverwrite(temp, filePath)
}
}