mirror of
https://github.com/pnpm/pnpm.git
synced 2026-04-27 18:46:18 -04:00
perf: persist bundled manifest in store index to avoid reading package.json from CAFS (#10473)
close #10461
This commit is contained in:
6
.changeset/gentle-waves-rise.md
Normal file
6
.changeset/gentle-waves-rise.md
Normal file
@@ -0,0 +1,6 @@
|
||||
---
|
||||
"@pnpm/package-requester": patch
|
||||
"pnpm": patch
|
||||
---
|
||||
|
||||
Check if a package is installable for non npm-hosted packages (e.g., git or tarball dependencies) after the manifest has been fetched.
|
||||
12
.changeset/quick-foxes-leap.md
Normal file
12
.changeset/quick-foxes-leap.md
Normal file
@@ -0,0 +1,12 @@
|
||||
---
|
||||
"@pnpm/store.cafs": major
|
||||
"@pnpm/store-controller-types": major
|
||||
"@pnpm/worker": major
|
||||
"@pnpm/package-requester": major
|
||||
"@pnpm/npm-resolver": major
|
||||
"@pnpm/reviewing.dependencies-hierarchy": major
|
||||
"@pnpm/build-modules": major
|
||||
"pnpm": major
|
||||
---
|
||||
|
||||
Store the bundled manifest (name, version, bin, engines, scripts, etc.) directly in the package index file, eliminating the need to read `package.json` from the content-addressable store during resolution and installation. This reduces I/O and speeds up repeat installs [#10473](https://github.com/pnpm/pnpm/pull/10473).
|
||||
6
.changeset/warm-clouds-drift.md
Normal file
6
.changeset/warm-clouds-drift.md
Normal file
@@ -0,0 +1,6 @@
|
||||
---
|
||||
"@pnpm/build-modules": patch
|
||||
"pnpm": patch
|
||||
---
|
||||
|
||||
Defer patch errors until all patches in a group are applied, so that one failed patch does not prevent other patches from being attempted.
|
||||
@@ -2,109 +2,117 @@
|
||||
"name": "bench-project",
|
||||
"version": "1.0.0",
|
||||
"dependencies": {
|
||||
"express": "^4.21.0",
|
||||
"animate.less": "^2.2.0",
|
||||
"autoprefixer": "^10.4.17",
|
||||
"babel-core": "^6.26.3",
|
||||
"babel-eslint": "^10.1.0",
|
||||
"babel-loader": "^9.1.3",
|
||||
"babel-plugin-lodash": "^3.3.4",
|
||||
"babel-plugin-module-resolver": "^5.0.0",
|
||||
"babel-plugin-transform-decorators-legacy": "^1.3.5",
|
||||
"babel-plugin-transform-runtime": "^6.23.0",
|
||||
"babel-polyfill": "^6.26.0",
|
||||
"babel-preset-es2015": "^6.24.1",
|
||||
"babel-preset-react": "^6.24.1",
|
||||
"babel-preset-react-hmre": "^1.1.1",
|
||||
"babel-preset-stage-1": "^6.24.1",
|
||||
"babel-runtime": "^6.26.0",
|
||||
"clean-webpack-plugin": "^4.0.0",
|
||||
"core-decorators": "^0.20.0",
|
||||
"css-loader": "^6.9.0",
|
||||
"css-mqpacker": "^7.0.0",
|
||||
"cssnano": "^6.0.3",
|
||||
"custom-event-polyfill": "^1.0.7",
|
||||
"draft-js": "^0.11.7",
|
||||
"ejs": "^3.1.9",
|
||||
"eslint": "^8.56.0",
|
||||
"eslint-config-airbnb": "^19.0.4",
|
||||
"eslint-import-resolver-webpack": "^0.13.8",
|
||||
"eslint-plugin-import": "^2.29.1",
|
||||
"eslint-plugin-jsx-a11y": "^6.8.0",
|
||||
"eslint-plugin-react": "^7.33.2",
|
||||
"express": "^4.18.2",
|
||||
"express-http-proxy": "^2.0.0",
|
||||
"font-awesome": "^4.7.0",
|
||||
"fready": "^1.0.0",
|
||||
"glob": "^10.3.10",
|
||||
"gulp": "^4.0.2",
|
||||
"gulp-concat": "^2.6.1",
|
||||
"gulp-csslint": "^1.0.1",
|
||||
"gulp-cssnano": "^2.1.3",
|
||||
"gulp-eol": "^0.2.0",
|
||||
"gulp-less": "^5.0.0",
|
||||
"gulp-livereload": "^4.0.2",
|
||||
"gulp-minify-css": "^1.2.4",
|
||||
"gulp-postcss": "^9.1.0",
|
||||
"gulp-rename": "^2.0.0",
|
||||
"gulp-util": "^3.0.8",
|
||||
"happypack": "^5.0.1",
|
||||
"highcharts": "^11.3.0",
|
||||
"highcharts-solid-gauge": "^0.1.7",
|
||||
"history": "^5.3.0",
|
||||
"howler": "^2.2.4",
|
||||
"imports-loader": "^5.0.0",
|
||||
"jquery": "^3.7.1",
|
||||
"jquery-ui": "1.13.2",
|
||||
"js-cookie": "^3.0.5",
|
||||
"json-loader": "^0.5.7",
|
||||
"leftpad": "^0.0.1",
|
||||
"less": "^4.2.0",
|
||||
"lesshat": "^4.1.0",
|
||||
"lodash": "^4.17.21",
|
||||
"axios": "^1.7.0",
|
||||
"chalk": "^4.1.2",
|
||||
"debug": "^4.3.4",
|
||||
"commander": "^12.0.0",
|
||||
"glob": "^10.3.0",
|
||||
"minimatch": "^9.0.0",
|
||||
"semver": "^7.6.0",
|
||||
"yargs": "^17.7.0",
|
||||
"inquirer": "^9.2.0",
|
||||
"ora": "^5.4.1",
|
||||
"fs-extra": "^11.2.0",
|
||||
"rimraf": "^5.0.0",
|
||||
"mkdirp": "^3.0.0",
|
||||
"cross-spawn": "^7.0.3",
|
||||
"execa": "^5.1.1",
|
||||
"which": "^4.0.0",
|
||||
"dotenv": "^16.4.0",
|
||||
"uuid": "^9.0.0",
|
||||
"moment": "^2.30.0",
|
||||
"dayjs": "^1.11.0",
|
||||
"date-fns": "^3.0.0",
|
||||
"fast-glob": "^3.3.0",
|
||||
"chokidar": "^3.6.0",
|
||||
"ws": "^8.16.0",
|
||||
"node-fetch": "^2.7.0",
|
||||
"form-data": "^4.0.0",
|
||||
"http-proxy-agent": "^7.0.0",
|
||||
"https-proxy-agent": "^7.0.0",
|
||||
"js-yaml": "^4.1.0",
|
||||
"ini": "^4.1.0",
|
||||
"toml": "^3.0.0",
|
||||
"strip-ansi": "^6.0.1",
|
||||
"wrap-ansi": "^7.0.0",
|
||||
"string-width": "^4.2.3",
|
||||
"cli-table3": "^0.6.3",
|
||||
"figures": "^3.2.0",
|
||||
"log-symbols": "^4.1.0",
|
||||
"boxen": "^5.1.2",
|
||||
"p-limit": "^3.1.0",
|
||||
"p-map": "^4.0.0",
|
||||
"p-queue": "^6.6.2",
|
||||
"retry": "^0.13.1",
|
||||
"graceful-fs": "^4.2.11",
|
||||
"jsonfile": "^6.1.0",
|
||||
"tar": "^7.0.0",
|
||||
"archiver": "^7.0.0",
|
||||
"decompress": "^4.2.1",
|
||||
"mime-types": "^2.1.35",
|
||||
"content-type": "^1.0.5",
|
||||
"accepts": "^1.3.8",
|
||||
"negotiator": "^0.6.3",
|
||||
"cors": "^2.8.5",
|
||||
"helmet": "^7.1.0",
|
||||
"compression": "^1.7.4",
|
||||
"cookie-parser": "^1.4.6",
|
||||
"body-parser": "^1.20.2",
|
||||
"multer": "^1.4.5-lts.1",
|
||||
"morgan": "^1.10.0",
|
||||
"winston": "^3.11.0",
|
||||
"pino": "^8.18.0",
|
||||
"bunyan": "^1.8.15",
|
||||
"ajv": "^8.12.0",
|
||||
"zod": "^3.22.0",
|
||||
"joi": "^17.12.0",
|
||||
"yup": "^1.3.0",
|
||||
"ramda": "^0.29.0",
|
||||
"underscore": "^1.13.6",
|
||||
"rxjs": "^7.8.1",
|
||||
"eventemitter3": "^5.0.1",
|
||||
"bluebird": "^3.7.2",
|
||||
"async": "^3.2.5",
|
||||
"lru-cache": "^10.2.0",
|
||||
"node-cache": "^5.1.2",
|
||||
"keyv": "^4.5.4",
|
||||
"got": "^11.8.6",
|
||||
"medium-draft": "^0.5.18",
|
||||
"mobx": "^6.12.0",
|
||||
"mobx-react": "^9.1.0",
|
||||
"moment": "^2.30.1",
|
||||
"moment-range": "^4.0.2",
|
||||
"moment-timezone": "^0.5.44",
|
||||
"password-policy": "0.0.3",
|
||||
"postcss-reporter": "^7.1.0",
|
||||
"progress": "^2.0.3",
|
||||
"qs": "^6.11.2",
|
||||
"raw-loader": "^4.0.2",
|
||||
"rc-slider": "^10.5.0",
|
||||
"react": "^18.2.0",
|
||||
"react-addons-css-transition-group": "^15.6.2",
|
||||
"react-addons-shallow-compare": "^15.6.3",
|
||||
"react-dnd": "^16.0.1",
|
||||
"react-dnd-html5-backend": "^16.0.1",
|
||||
"react-dom": "^18.2.0",
|
||||
"react-draft-wysiwyg": "^1.15.0",
|
||||
"react-dropzone": "^14.2.3",
|
||||
"react-grid-layout": "^1.4.4",
|
||||
"react-highcharts": "^16.1.0",
|
||||
"react-hot-loader": "4.13.1",
|
||||
"react-input-calendar": "^0.5.4",
|
||||
"react-lazyload": "^3.2.0",
|
||||
"react-measure": "^2.5.2",
|
||||
"react-mixin": "^5.0.0",
|
||||
"react-responsive": "9.0.2",
|
||||
"react-responsive-tabs": "^4.4.3",
|
||||
"react-router": "^6.21.2",
|
||||
"react-router-dom": "^6.21.2",
|
||||
"react-select-plus": "1.0.0-rc.3.patch12",
|
||||
"react-skylight": "^0.5.1",
|
||||
"react-sortablejs": "^6.1.4",
|
||||
"react-tappable": "^1.0.4",
|
||||
"react-tooltip": "5.25.2",
|
||||
"react-virtualized": "^9.22.5",
|
||||
"react-waypoint": "^10.3.0",
|
||||
"sortablejs": "^1.15.2",
|
||||
"style-loader": "^3.3.4",
|
||||
"stylelint": "^16.1.0",
|
||||
"superagent": "^8.1.2",
|
||||
"cheerio": "^1.0.0-rc.12",
|
||||
"marked": "^12.0.0",
|
||||
"highlight.js": "^11.9.0",
|
||||
"sharp": "^0.33.0",
|
||||
"jimp": "^0.22.12",
|
||||
"canvas": "^2.11.2",
|
||||
"socket.io": "^4.7.0",
|
||||
"redis": "^4.6.0",
|
||||
"ioredis": "^5.3.0",
|
||||
"mongoose": "^8.1.0",
|
||||
"typeorm": "^0.3.20",
|
||||
"knex": "^3.1.0",
|
||||
"pg": "^8.11.0",
|
||||
"mysql2": "^3.9.0",
|
||||
"better-sqlite3": "^9.4.0",
|
||||
"bcrypt": "^5.1.1",
|
||||
"jsonwebtoken": "^9.0.2",
|
||||
"passport": "^0.7.0",
|
||||
"nanoid": "^3.3.7",
|
||||
"cuid": "^3.0.0",
|
||||
"shortid": "^2.2.16",
|
||||
"color": "^4.2.3",
|
||||
"pluralize": "^8.0.0",
|
||||
"change-case": "^4.1.2",
|
||||
"camelcase": "^6.3.0",
|
||||
"escape-string-regexp": "^4.0.0"
|
||||
"uglify-js": "^3.17.4",
|
||||
"uuid": "^9.0.1",
|
||||
"verge": "^1.10.2",
|
||||
"webpack-bundle-analyzer": "^4.10.1",
|
||||
"webpack-hot-middleware": "^2.26.0",
|
||||
"webpack-notifier": "^1.15.0",
|
||||
"webpack-split-by-path": "^2.0.0",
|
||||
"whatwg-fetch": "^3.6.20"
|
||||
},
|
||||
"devDependencies": {
|
||||
"nan-as": "^1.6.1"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -103,7 +103,24 @@ export async function buildModules<T extends string> (
|
||||
}
|
||||
)
|
||||
})
|
||||
await runGroups(getWorkspaceConcurrency(opts.childConcurrency), groups)
|
||||
const patchErrors: Error[] = []
|
||||
const groupsWithPatchErrors = groups.map((group) =>
|
||||
group.map((task) => async () => {
|
||||
try {
|
||||
await task()
|
||||
} catch (err: unknown) {
|
||||
if (util.types.isNativeError(err) && 'code' in err && err.code === 'ERR_PNPM_PATCH_FAILED') {
|
||||
patchErrors.push(err)
|
||||
} else {
|
||||
throw err
|
||||
}
|
||||
}
|
||||
})
|
||||
)
|
||||
await runGroups(getWorkspaceConcurrency(opts.childConcurrency), groupsWithPatchErrors)
|
||||
if (patchErrors.length > 0) {
|
||||
throw patchErrors[0]
|
||||
}
|
||||
return { ignoredBuilds }
|
||||
}
|
||||
|
||||
@@ -247,7 +264,7 @@ export async function linkBinsOfDependencies<T extends string> (
|
||||
const pkgs = await Promise.all(pkgNodes
|
||||
.map(async (dep) => ({
|
||||
location: dep.dir,
|
||||
manifest: (await dep.fetching?.())?.bundledManifest ?? (await safeReadPackageJsonFromDir(dep.dir) as DependencyManifest) ?? {},
|
||||
manifest: ((await dep.fetching?.())?.bundledManifest ?? (await safeReadPackageJsonFromDir(dep.dir))) as DependencyManifest ?? {},
|
||||
}))
|
||||
)
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ import {
|
||||
type BinaryResolution,
|
||||
} from '@pnpm/resolver-base'
|
||||
import { type Cafs, type FilesMap } from '@pnpm/cafs-types'
|
||||
import { type AllowBuild, type DependencyManifest } from '@pnpm/types'
|
||||
import { type AllowBuild, type BundledManifest, type DependencyManifest } from '@pnpm/types'
|
||||
|
||||
export interface PkgNameVersion {
|
||||
name?: string
|
||||
@@ -31,7 +31,7 @@ export type FetchFunction<FetcherResolution = Resolution, Options = FetchOptions
|
||||
|
||||
export interface FetchResult {
|
||||
local?: boolean
|
||||
manifest?: DependencyManifest
|
||||
manifest?: BundledManifest
|
||||
filesMap: FilesMap
|
||||
requiresBuild: boolean
|
||||
integrity?: string
|
||||
@@ -46,7 +46,7 @@ export interface GitFetcherOptions {
|
||||
|
||||
export interface GitFetcherResult {
|
||||
filesMap: FilesMap
|
||||
manifest?: DependencyManifest
|
||||
manifest?: BundledManifest
|
||||
requiresBuild: boolean
|
||||
}
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ import { type Cafs, type FilesMap } from '@pnpm/cafs-types'
|
||||
import { packlist } from '@pnpm/fs.packlist'
|
||||
import { globalWarn } from '@pnpm/logger'
|
||||
import { preparePackage } from '@pnpm/prepare-package'
|
||||
import { type DependencyManifest } from '@pnpm/types'
|
||||
import { type BundledManifest } from '@pnpm/types'
|
||||
import { addFilesFromDir } from '@pnpm/worker'
|
||||
import renameOverwrite from 'rename-overwrite'
|
||||
import { fastPathTemp as pathTemp } from 'path-temp'
|
||||
@@ -53,7 +53,7 @@ export function createGitHostedTarballFetcher (fetchRemoteTarball: FetchFunction
|
||||
|
||||
interface PrepareGitHostedPkgResult {
|
||||
filesMap: FilesMap
|
||||
manifest?: DependencyManifest
|
||||
manifest?: BundledManifest
|
||||
ignoredBuild: boolean
|
||||
}
|
||||
|
||||
|
||||
@@ -183,6 +183,29 @@ export interface PackageManifest extends DependencyManifest {
|
||||
deprecated?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Subset of package.json fields cached in the store index.
|
||||
* Used for bin linking, build scripts, runtime selection, and dependency resolution.
|
||||
*/
|
||||
export type BundledManifest = Pick<
|
||||
BaseManifest,
|
||||
| 'bin'
|
||||
| 'bundledDependencies'
|
||||
| 'bundleDependencies'
|
||||
| 'cpu'
|
||||
| 'dependencies'
|
||||
| 'directories'
|
||||
| 'engines'
|
||||
| 'libc'
|
||||
| 'name'
|
||||
| 'optionalDependencies'
|
||||
| 'os'
|
||||
| 'peerDependencies'
|
||||
| 'peerDependenciesMeta'
|
||||
| 'scripts'
|
||||
| 'version'
|
||||
>
|
||||
|
||||
export interface SupportedArchitectures {
|
||||
os?: string[]
|
||||
cpu?: string[]
|
||||
|
||||
@@ -42,12 +42,12 @@
|
||||
"@pnpm/hooks.types": "workspace:*",
|
||||
"@pnpm/package-is-installable": "workspace:*",
|
||||
"@pnpm/pick-fetcher": "workspace:*",
|
||||
"@pnpm/read-package-json": "workspace:*",
|
||||
"@pnpm/resolver-base": "workspace:*",
|
||||
"@pnpm/store-controller-types": "workspace:*",
|
||||
"@pnpm/store.cafs": "workspace:*",
|
||||
"@pnpm/types": "workspace:*",
|
||||
"detect-libc": "catalog:",
|
||||
"load-json-file": "catalog:",
|
||||
"p-defer": "catalog:",
|
||||
"p-limit": "catalog:",
|
||||
"p-queue": "catalog:",
|
||||
@@ -75,7 +75,6 @@
|
||||
"@types/semver": "catalog:",
|
||||
"@types/ssri": "catalog:",
|
||||
"delay": "catalog:",
|
||||
"load-json-file": "catalog:",
|
||||
"nock": "catalog:",
|
||||
"normalize-path": "catalog:",
|
||||
"tempy": "catalog:"
|
||||
|
||||
@@ -2,6 +2,7 @@ import { createReadStream, promises as fs } from 'fs'
|
||||
import path from 'path'
|
||||
import {
|
||||
getIndexFilePathInCafs as _getIndexFilePathInCafs,
|
||||
normalizeBundledManifest,
|
||||
} from '@pnpm/store.cafs'
|
||||
import { fetchingProgressLogger, progressLogger } from '@pnpm/core-loggers'
|
||||
import { pickFetcher } from '@pnpm/pick-fetcher'
|
||||
@@ -16,7 +17,7 @@ import { type Cafs } from '@pnpm/cafs-types'
|
||||
import gfs from '@pnpm/graceful-fs'
|
||||
import { logger } from '@pnpm/logger'
|
||||
import { packageIsInstallable } from '@pnpm/package-is-installable'
|
||||
import { readPackageJson } from '@pnpm/read-package-json'
|
||||
import { loadJsonFile } from 'load-json-file'
|
||||
import {
|
||||
type PlatformAssetResolution,
|
||||
type DirectoryResolution,
|
||||
@@ -53,7 +54,6 @@ import PQueue from 'p-queue'
|
||||
import pDefer, { type DeferredPromise } from 'p-defer'
|
||||
import pShare from 'promise-share'
|
||||
import { pick } from 'ramda'
|
||||
import semver from 'semver'
|
||||
import ssri from 'ssri'
|
||||
|
||||
let currentLibc: 'glibc' | 'musl' | undefined | null
|
||||
@@ -66,29 +66,6 @@ function getLibcFamilySync () {
|
||||
const TARBALL_INTEGRITY_FILENAME = 'tarball-integrity'
|
||||
const packageRequestLogger = logger('package-requester')
|
||||
|
||||
const pickBundledManifest = pick([
|
||||
'bin',
|
||||
'bundledDependencies',
|
||||
'bundleDependencies',
|
||||
'cpu',
|
||||
'dependencies',
|
||||
'directories',
|
||||
'engines',
|
||||
'name',
|
||||
'optionalDependencies',
|
||||
'os',
|
||||
'peerDependencies',
|
||||
'peerDependenciesMeta',
|
||||
'scripts',
|
||||
'version',
|
||||
])
|
||||
|
||||
function normalizeBundledManifest (manifest: DependencyManifest): BundledManifest {
|
||||
return {
|
||||
...pickBundledManifest(manifest),
|
||||
version: semver.clean(manifest.version ?? '0.0.0', { loose: true }) ?? manifest.version,
|
||||
}
|
||||
}
|
||||
|
||||
export function createPackageRequester (
|
||||
opts: {
|
||||
@@ -248,7 +225,7 @@ async function resolveAndFetch (
|
||||
}
|
||||
}
|
||||
|
||||
const isInstallable = (
|
||||
let isInstallable: boolean | null | undefined = (
|
||||
ctx.force === true ||
|
||||
(
|
||||
manifest == null
|
||||
@@ -303,12 +280,26 @@ async function resolveAndFetch (
|
||||
|
||||
if (!manifest) {
|
||||
const fetchedResult = await fetchResult.fetching()
|
||||
manifest = fetchedResult.bundledManifest
|
||||
if (fetchedResult.bundledManifest) {
|
||||
manifest = fetchedResult.bundledManifest as DependencyManifest
|
||||
} else if (fetchedResult.files.filesMap.has('package.json')) {
|
||||
manifest = await loadJsonFile<DependencyManifest>(fetchedResult.files.filesMap.get('package.json')!)
|
||||
}
|
||||
// Add integrity to resolution if it was computed during fetching (only for TarballResolution)
|
||||
if (fetchedResult.integrity && !resolution.type && !(resolution as TarballResolution).integrity) {
|
||||
(resolution as TarballResolution).integrity = fetchedResult.integrity
|
||||
}
|
||||
}
|
||||
// Check installability now that we have the manifest (for git/tarball packages without registry metadata)
|
||||
if (isInstallable === undefined && manifest != null) {
|
||||
isInstallable = ctx.force === true || packageIsInstallable(id, manifest, {
|
||||
engineStrict: ctx.engineStrict,
|
||||
lockfileDir: options.lockfileDir,
|
||||
nodeVersion: ctx.nodeVersion,
|
||||
optional: wantedDependency.optional === true,
|
||||
supportedArchitectures: options.supportedArchitectures,
|
||||
})
|
||||
}
|
||||
return {
|
||||
body: {
|
||||
id,
|
||||
@@ -537,14 +528,14 @@ function fetchToStore (
|
||||
) &&
|
||||
!isLocalPkg
|
||||
) {
|
||||
const { verified, files, manifest } = await ctx.readPkgFromCafs(filesIndexFile, {
|
||||
const { verified, files, bundledManifest } = await ctx.readPkgFromCafs(filesIndexFile, {
|
||||
readManifest: opts.fetchRawManifest,
|
||||
expectedPkg: opts.pkg,
|
||||
})
|
||||
if (verified) {
|
||||
fetching.resolve({
|
||||
files,
|
||||
bundledManifest: manifest == null ? manifest : normalizeBundledManifest(manifest),
|
||||
bundledManifest,
|
||||
})
|
||||
return
|
||||
}
|
||||
@@ -608,7 +599,7 @@ function fetchToStore (
|
||||
packageImportMethod: (fetchedPackage as DirectoryFetcherResult).packageImportMethod,
|
||||
requiresBuild: fetchedPackage.requiresBuild,
|
||||
},
|
||||
bundledManifest: fetchedPackage.manifest == null ? fetchedPackage.manifest : normalizeBundledManifest(fetchedPackage.manifest),
|
||||
bundledManifest: fetchedPackage.manifest,
|
||||
integrity,
|
||||
})
|
||||
} catch (err: any) { // eslint-disable-line
|
||||
@@ -617,8 +608,8 @@ function fetchToStore (
|
||||
}
|
||||
}
|
||||
|
||||
async function readBundledManifest (pkgJsonPath: string): Promise<BundledManifest> {
|
||||
return pickBundledManifest(await readPackageJson(pkgJsonPath) as DependencyManifest)
|
||||
async function readBundledManifest (pkgJsonPath: string): Promise<BundledManifest | undefined> {
|
||||
return normalizeBundledManifest(await loadJsonFile<DependencyManifest>(pkgJsonPath))
|
||||
}
|
||||
|
||||
async function tarballIsUpToDate (
|
||||
|
||||
@@ -473,7 +473,6 @@ test('fetchPackageToStore()', async () => {
|
||||
{
|
||||
engines: { node: '>=0.10.0' },
|
||||
name: 'is-positive',
|
||||
scripts: { test: 'node test.js' },
|
||||
version: '1.0.0',
|
||||
}
|
||||
)
|
||||
@@ -668,7 +667,6 @@ test('always return a package manifest in the response', async () => {
|
||||
{
|
||||
engines: { node: '>=0.10.0' },
|
||||
name: 'is-positive',
|
||||
scripts: { test: 'node test.js' },
|
||||
version: '1.0.0',
|
||||
}
|
||||
)
|
||||
|
||||
@@ -45,9 +45,6 @@
|
||||
{
|
||||
"path": "../../packages/types"
|
||||
},
|
||||
{
|
||||
"path": "../../pkg-manifest/read-package-json"
|
||||
},
|
||||
{
|
||||
"path": "../../resolving/resolver-base"
|
||||
},
|
||||
|
||||
24
pnpm-lock.yaml
generated
24
pnpm-lock.yaml
generated
@@ -5772,9 +5772,6 @@ importers:
|
||||
'@pnpm/pick-fetcher':
|
||||
specifier: workspace:*
|
||||
version: link:../../fetching/pick-fetcher
|
||||
'@pnpm/read-package-json':
|
||||
specifier: workspace:*
|
||||
version: link:../../pkg-manifest/read-package-json
|
||||
'@pnpm/resolver-base':
|
||||
specifier: workspace:*
|
||||
version: link:../../resolving/resolver-base
|
||||
@@ -5793,6 +5790,9 @@ importers:
|
||||
detect-libc:
|
||||
specifier: 'catalog:'
|
||||
version: 2.1.2
|
||||
load-json-file:
|
||||
specifier: 'catalog:'
|
||||
version: 7.0.1
|
||||
p-defer:
|
||||
specifier: 'catalog:'
|
||||
version: 4.0.1
|
||||
@@ -5857,9 +5857,6 @@ importers:
|
||||
delay:
|
||||
specifier: 'catalog:'
|
||||
version: 6.0.0
|
||||
load-json-file:
|
||||
specifier: 'catalog:'
|
||||
version: 7.0.1
|
||||
nock:
|
||||
specifier: 'catalog:'
|
||||
version: 13.3.4
|
||||
@@ -7707,6 +7704,9 @@ importers:
|
||||
'@pnpm/util.lex-comparator':
|
||||
specifier: 'catalog:'
|
||||
version: 3.0.2
|
||||
load-json-file:
|
||||
specifier: 'catalog:'
|
||||
version: 7.0.1
|
||||
normalize-path:
|
||||
specifier: 'catalog:'
|
||||
version: 3.0.0
|
||||
@@ -8183,6 +8183,9 @@ importers:
|
||||
'@pnpm/store-controller-types':
|
||||
specifier: workspace:*
|
||||
version: link:../store-controller-types
|
||||
'@pnpm/types':
|
||||
specifier: workspace:*
|
||||
version: link:../../packages/types
|
||||
'@zkochan/rimraf':
|
||||
specifier: 'catalog:'
|
||||
version: 3.0.2
|
||||
@@ -8198,6 +8201,9 @@ importers:
|
||||
rename-overwrite:
|
||||
specifier: 'catalog:'
|
||||
version: 6.0.3
|
||||
semver:
|
||||
specifier: 'catalog:'
|
||||
version: 7.7.4
|
||||
strip-bom:
|
||||
specifier: 'catalog:'
|
||||
version: 5.0.0
|
||||
@@ -8211,15 +8217,15 @@ importers:
|
||||
'@pnpm/test-fixtures':
|
||||
specifier: workspace:*
|
||||
version: link:../../__utils__/test-fixtures
|
||||
'@pnpm/types':
|
||||
specifier: workspace:*
|
||||
version: link:../../packages/types
|
||||
'@types/is-gzip':
|
||||
specifier: 'catalog:'
|
||||
version: 2.0.0
|
||||
'@types/node':
|
||||
specifier: 'catalog:'
|
||||
version: 22.19.11
|
||||
'@types/semver':
|
||||
specifier: 'catalog:'
|
||||
version: 7.7.1
|
||||
symlink-dir:
|
||||
specifier: 'catalog:'
|
||||
version: 7.1.0
|
||||
|
||||
@@ -507,10 +507,9 @@ test('installation fails when the stored package name and version do not match t
|
||||
|
||||
const cacheIntegrityPath = getIndexFilePathInCafs(path.join(storeDir, STORE_VERSION), getIntegrity('@pnpm.e2e/dep-of-pkg-with-1-dep', '100.1.0'), '@pnpm.e2e/dep-of-pkg-with-1-dep@100.1.0')
|
||||
const cacheIntegrity = readMsgpackFileSync<PackageFilesIndex>(cacheIntegrityPath)
|
||||
cacheIntegrity.name = 'foo'
|
||||
writeMsgpackFileSync(cacheIntegrityPath, {
|
||||
...cacheIntegrity,
|
||||
name: 'foo',
|
||||
manifest: { ...cacheIntegrity.manifest, name: 'foo' },
|
||||
})
|
||||
|
||||
rimraf('node_modules')
|
||||
|
||||
@@ -192,16 +192,15 @@ export function createNpmResolver (
|
||||
const request = readPkgFromCafs(
|
||||
{
|
||||
storeDir,
|
||||
verifyStoreIntegrity: true,
|
||||
verifyStoreIntegrity: false,
|
||||
},
|
||||
filesIndexFile,
|
||||
{
|
||||
readManifest: true,
|
||||
expectedPkg: { name: peekOpts.name, version: peekOpts.version },
|
||||
}
|
||||
).then(({ manifest, verified }) => {
|
||||
if (!verified) return undefined
|
||||
return manifest
|
||||
).then(({ bundledManifest }) => {
|
||||
if (!bundledManifest) return undefined
|
||||
return bundledManifest as DependencyManifest
|
||||
}).catch(() => undefined)
|
||||
peekLockerForPeek.set(filesIndexFile, request)
|
||||
return request
|
||||
|
||||
@@ -48,6 +48,7 @@
|
||||
"@pnpm/store.cafs": "workspace:*",
|
||||
"@pnpm/types": "workspace:*",
|
||||
"@pnpm/util.lex-comparator": "catalog:",
|
||||
"load-json-file": "catalog:",
|
||||
"normalize-path": "catalog:",
|
||||
"realpath-missing": "catalog:",
|
||||
"resolve-link-target": "catalog:",
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { readMsgpackFileSync } from '@pnpm/fs.msgpack-file'
|
||||
import { getIndexFilePathInCafs, readManifestFromStore, type PackageFilesIndex } from '@pnpm/store.cafs'
|
||||
import { loadJsonFileSync } from 'load-json-file'
|
||||
import { getIndexFilePathInCafs, getFilePathByModeInCafs, type PackageFilesIndex } from '@pnpm/store.cafs'
|
||||
import { type DependencyManifest } from '@pnpm/types'
|
||||
|
||||
/**
|
||||
@@ -15,8 +16,11 @@ export function readManifestFromCafs (storeDir: string, pkg: {
|
||||
const pkgId = `${pkg.name}@${pkg.version}`
|
||||
const indexPath = getIndexFilePathInCafs(storeDir, pkg.integrity, pkgId)
|
||||
const pkgIndex = readMsgpackFileSync<PackageFilesIndex>(indexPath)
|
||||
const manifest = readManifestFromStore(storeDir, pkgIndex)
|
||||
if (manifest) return manifest as DependencyManifest
|
||||
const pkgJsonEntry = pkgIndex.files.get('package.json')
|
||||
if (pkgJsonEntry) {
|
||||
const filePath = getFilePathByModeInCafs(storeDir, pkgJsonEntry.digest, pkgJsonEntry.mode)
|
||||
return loadJsonFileSync<DependencyManifest>(filePath)
|
||||
}
|
||||
} catch {
|
||||
// Fall through to undefined
|
||||
}
|
||||
|
||||
@@ -36,20 +36,22 @@
|
||||
"@pnpm/fetcher-base": "workspace:*",
|
||||
"@pnpm/graceful-fs": "workspace:*",
|
||||
"@pnpm/store-controller-types": "workspace:*",
|
||||
"@pnpm/types": "workspace:*",
|
||||
"@zkochan/rimraf": "catalog:",
|
||||
"is-gzip": "catalog:",
|
||||
"is-subdir": "catalog:",
|
||||
"p-limit": "catalog:",
|
||||
"rename-overwrite": "catalog:",
|
||||
"semver": "catalog:",
|
||||
"strip-bom": "catalog:"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@pnpm/cafs-types": "workspace:*",
|
||||
"@pnpm/store.cafs": "workspace:*",
|
||||
"@pnpm/test-fixtures": "workspace:*",
|
||||
"@pnpm/types": "workspace:*",
|
||||
"@types/is-gzip": "catalog:",
|
||||
"@types/node": "catalog:",
|
||||
"@types/semver": "catalog:",
|
||||
"symlink-dir": "catalog:",
|
||||
"tempy": "catalog:"
|
||||
},
|
||||
|
||||
@@ -4,11 +4,9 @@ import util from 'util'
|
||||
import { PnpmError } from '@pnpm/error'
|
||||
import { type PackageFiles, type PackageFileInfo, type SideEffects, type FilesMap } from '@pnpm/cafs-types'
|
||||
import gfs from '@pnpm/graceful-fs'
|
||||
import { type DependencyManifest } from '@pnpm/types'
|
||||
import { type BundledManifest } from '@pnpm/types'
|
||||
import rimraf from '@zkochan/rimraf'
|
||||
import { getFilePathByModeInCafs } from './getFilePathInCafs.js'
|
||||
import { parseJsonBufferSync } from './parseJson.js'
|
||||
import { readManifestFromStore } from './readManifestFromStore.js'
|
||||
|
||||
export interface Integrity {
|
||||
digest: string
|
||||
@@ -24,36 +22,28 @@ global['verifiedFileIntegrity'] = 0
|
||||
|
||||
export interface VerifyResult {
|
||||
passed: boolean
|
||||
manifest?: DependencyManifest
|
||||
filesMap: FilesMap
|
||||
sideEffectsMaps?: Map<string, { added?: FilesMap, deleted?: string[] }>
|
||||
}
|
||||
|
||||
export interface PackageFilesIndex {
|
||||
// name and version are nullable for backward compatibility
|
||||
// the initial specs of pnpm store v3 did not require these fields.
|
||||
// However, it might be possible that some types of dependencies don't
|
||||
// have the name/version fields, like the local tarball dependencies.
|
||||
name?: string
|
||||
version?: string
|
||||
manifest?: BundledManifest
|
||||
requiresBuild?: boolean
|
||||
algo: string
|
||||
|
||||
files: PackageFiles
|
||||
sideEffects?: SideEffects
|
||||
}
|
||||
|
||||
export function checkPkgFilesIntegrity (
|
||||
storeDir: string,
|
||||
pkgIndex: PackageFilesIndex,
|
||||
readManifest?: boolean
|
||||
pkgIndex: PackageFilesIndex
|
||||
): VerifyResult {
|
||||
// It might make sense to use this cache for all files in the store
|
||||
// but there's a smaller chance that the same file will be checked twice
|
||||
// so it's probably not worth the memory (this assumption should be verified)
|
||||
const verifiedFilesCache = new Set<string>()
|
||||
const _checkFilesIntegrity = checkFilesIntegrity.bind(null, verifiedFilesCache, storeDir, pkgIndex.algo)
|
||||
const verified = _checkFilesIntegrity(pkgIndex.files, readManifest)
|
||||
const verified = _checkFilesIntegrity(pkgIndex.files)
|
||||
if (!verified.passed) return verified
|
||||
|
||||
const sideEffectsMaps = new Map<string, { added?: FilesMap, deleted?: string[] }>()
|
||||
@@ -88,8 +78,7 @@ export function checkPkgFilesIntegrity (
|
||||
*/
|
||||
export function buildFileMapsFromIndex (
|
||||
storeDir: string,
|
||||
pkgIndex: PackageFilesIndex,
|
||||
readManifest?: boolean
|
||||
pkgIndex: PackageFilesIndex
|
||||
): VerifyResult {
|
||||
const filesMap: FilesMap = new Map()
|
||||
|
||||
@@ -122,7 +111,6 @@ export function buildFileMapsFromIndex (
|
||||
|
||||
return {
|
||||
passed: true,
|
||||
manifest: readManifest ? readManifestFromStore(storeDir, pkgIndex) : undefined,
|
||||
filesMap,
|
||||
sideEffectsMaps: sideEffectsMaps.size > 0 ? sideEffectsMaps : undefined,
|
||||
}
|
||||
@@ -132,11 +120,9 @@ function checkFilesIntegrity (
|
||||
verifiedFilesCache: Set<string>,
|
||||
storeDir: string,
|
||||
algo: string,
|
||||
files: PackageFiles,
|
||||
readManifest?: boolean
|
||||
files: PackageFiles
|
||||
): VerifyResult {
|
||||
let allVerified = true
|
||||
let manifest: DependencyManifest | undefined
|
||||
const filesMap: FilesMap = new Map()
|
||||
|
||||
for (const [f, fstat] of files) {
|
||||
@@ -146,13 +132,9 @@ function checkFilesIntegrity (
|
||||
const filename = getFilePathByModeInCafs(storeDir, fstat.digest, fstat.mode)
|
||||
filesMap.set(f, filename)
|
||||
|
||||
const readFile = readManifest && f === 'package.json'
|
||||
if (!readFile && verifiedFilesCache.has(filename)) continue
|
||||
const verifyResult = verifyFile(filename, fstat, algo, readFile)
|
||||
if (readFile) {
|
||||
manifest = verifyResult.manifest
|
||||
}
|
||||
if (verifyResult.passed) {
|
||||
if (verifiedFilesCache.has(filename)) continue
|
||||
const passed = verifyFile(filename, fstat, algo)
|
||||
if (passed) {
|
||||
verifiedFilesCache.add(filename)
|
||||
} else {
|
||||
allVerified = false
|
||||
@@ -160,7 +142,6 @@ function checkFilesIntegrity (
|
||||
}
|
||||
return {
|
||||
passed: allVerified,
|
||||
manifest,
|
||||
filesMap,
|
||||
}
|
||||
}
|
||||
@@ -170,34 +151,26 @@ type FileInfo = Pick<PackageFileInfo, 'size' | 'checkedAt' | 'digest'>
|
||||
function verifyFile (
|
||||
filename: string,
|
||||
fstat: FileInfo,
|
||||
algorithm: string,
|
||||
readManifest?: boolean
|
||||
): Pick<VerifyResult, 'passed' | 'manifest'> {
|
||||
algorithm: string
|
||||
): boolean {
|
||||
const currentFile = checkFile(filename, fstat.checkedAt)
|
||||
if (currentFile == null) return { passed: false }
|
||||
if (currentFile == null) return false
|
||||
if (currentFile.isModified) {
|
||||
if (currentFile.size !== fstat.size) {
|
||||
rimraf.sync(filename)
|
||||
return { passed: false }
|
||||
}
|
||||
return verifyFileIntegrity(filename, { digest: fstat.digest, algorithm }, readManifest)
|
||||
}
|
||||
if (readManifest) {
|
||||
return {
|
||||
passed: true,
|
||||
manifest: parseJsonBufferSync(gfs.readFileSync(filename)) as DependencyManifest,
|
||||
return false
|
||||
}
|
||||
return verifyFileIntegrity(filename, { digest: fstat.digest, algorithm })
|
||||
}
|
||||
// If a file was not edited, we are skipping integrity check.
|
||||
// We assume that nobody will manually remove a file in the store and create a new one.
|
||||
return { passed: true }
|
||||
return true
|
||||
}
|
||||
|
||||
export function verifyFileIntegrity (
|
||||
filename: string,
|
||||
integrity: Integrity,
|
||||
readManifest?: boolean
|
||||
): Pick<VerifyResult, 'passed' | 'manifest'> {
|
||||
integrity: Integrity
|
||||
): boolean {
|
||||
// @ts-expect-error
|
||||
global['verifiedFileIntegrity']++
|
||||
let data: Buffer
|
||||
@@ -205,7 +178,7 @@ export function verifyFileIntegrity (
|
||||
data = gfs.readFileSync(filename)
|
||||
} catch (err: unknown) {
|
||||
if (util.types.isNativeError(err) && 'code' in err && err.code === 'ENOENT') {
|
||||
return { passed: false }
|
||||
return false
|
||||
}
|
||||
throw err
|
||||
}
|
||||
@@ -214,20 +187,13 @@ export function verifyFileIntegrity (
|
||||
computedDigest = crypto.hash(integrity.algorithm, data, 'hex')
|
||||
} catch {
|
||||
// Invalid algorithm (e.g., corrupted index file) - treat as verification failure
|
||||
return { passed: false }
|
||||
return false
|
||||
}
|
||||
const passed = computedDigest === integrity.digest
|
||||
if (!passed) {
|
||||
gfs.unlinkSync(filename)
|
||||
return { passed }
|
||||
}
|
||||
if (readManifest) {
|
||||
return {
|
||||
passed,
|
||||
manifest: parseJsonBufferSync(data) as DependencyManifest,
|
||||
}
|
||||
}
|
||||
return { passed }
|
||||
return passed
|
||||
}
|
||||
|
||||
function checkFile (filename: string, checkedAt?: number): { isModified: boolean, size: number } | null {
|
||||
|
||||
@@ -17,7 +17,6 @@ import {
|
||||
type PackageFilesIndex,
|
||||
type VerifyResult,
|
||||
} from './checkPkgFilesIntegrity.js'
|
||||
import { readManifestFromStore } from './readManifestFromStore.js'
|
||||
import {
|
||||
getIndexFilePathInCafs,
|
||||
contentPathFromHex,
|
||||
@@ -25,14 +24,17 @@ import {
|
||||
getFilePathByModeInCafs,
|
||||
modeIsExecutable,
|
||||
} from './getFilePathInCafs.js'
|
||||
import { normalizeBundledManifest } from './normalizeBundledManifest.js'
|
||||
import { optimisticRenameOverwrite, writeBufferToCafs } from './writeBufferToCafs.js'
|
||||
|
||||
export const HASH_ALGORITHM = 'sha512'
|
||||
|
||||
export { type BundledManifest } from '@pnpm/types'
|
||||
export { normalizeBundledManifest }
|
||||
|
||||
export {
|
||||
checkPkgFilesIntegrity,
|
||||
buildFileMapsFromIndex,
|
||||
readManifestFromStore,
|
||||
type FileType,
|
||||
getFilePathByModeInCafs,
|
||||
getIndexFilePathInCafs,
|
||||
|
||||
49
store/cafs/src/normalizeBundledManifest.ts
Normal file
49
store/cafs/src/normalizeBundledManifest.ts
Normal file
@@ -0,0 +1,49 @@
|
||||
import { type BaseManifest, type BundledManifest } from '@pnpm/types'
|
||||
import semver from 'semver'
|
||||
|
||||
const BUNDLED_MANIFEST_FIELDS: Array<keyof BaseManifest> = [
|
||||
'bin',
|
||||
'bundledDependencies',
|
||||
'bundleDependencies',
|
||||
'cpu',
|
||||
'dependencies',
|
||||
'directories',
|
||||
'engines',
|
||||
'libc',
|
||||
'name',
|
||||
'optionalDependencies',
|
||||
'os',
|
||||
'peerDependencies',
|
||||
'peerDependenciesMeta',
|
||||
]
|
||||
|
||||
const LIFECYCLE_SCRIPTS = ['preinstall', 'install', 'postinstall'] as const
|
||||
|
||||
/**
|
||||
* Picks the subset of manifest fields stored in the package index and normalizes the version.
|
||||
* Used both when writing the index (worker) and when creating a BundledManifest from a fresh fetch.
|
||||
*/
|
||||
export function normalizeBundledManifest (manifest: Partial<BaseManifest>): BundledManifest | undefined {
|
||||
let result: Record<string, unknown> | undefined
|
||||
for (const key of BUNDLED_MANIFEST_FIELDS) {
|
||||
if (manifest[key] != null) {
|
||||
if (!result) result = {}
|
||||
result[key] = manifest[key]
|
||||
}
|
||||
}
|
||||
let scripts: Record<string, string> | undefined
|
||||
if (manifest.scripts) {
|
||||
for (const key of LIFECYCLE_SCRIPTS) {
|
||||
if (manifest.scripts[key]) {
|
||||
if (!scripts) scripts = {}
|
||||
scripts[key] = manifest.scripts[key]
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!result && !scripts) return undefined
|
||||
return {
|
||||
version: semver.clean(manifest.version ?? '0.0.0', { loose: true }) ?? manifest.version,
|
||||
...result,
|
||||
...scripts ? { scripts } : {},
|
||||
} as BundledManifest
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
import gfs from '@pnpm/graceful-fs'
|
||||
import { type PackageManifest } from '@pnpm/types'
|
||||
import { type PackageFilesIndex } from './checkPkgFilesIntegrity.js'
|
||||
import { getFilePathByModeInCafs } from './getFilePathInCafs.js'
|
||||
import { parseJsonBufferSync } from './parseJson.js'
|
||||
|
||||
export function readManifestFromStore (storeDir: string, pkgIndex: PackageFilesIndex): PackageManifest | undefined {
|
||||
const pkg = pkgIndex.files.get('package.json')
|
||||
if (pkg) {
|
||||
const fileName = getFilePathByModeInCafs(storeDir, pkg.digest, pkg.mode)
|
||||
return parseJsonBufferSync(gfs.readFileSync(fileName)) as PackageManifest
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
@@ -105,5 +105,5 @@ function removeSuffix (filePath: string): string {
|
||||
function existsSame (filename: string, integrity: Integrity): boolean {
|
||||
const existingFile = fs.statSync(filename, { throwIfNoEntry: false })
|
||||
if (!existingFile) return false
|
||||
return verifyFileIntegrity(filename, integrity).passed
|
||||
return verifyFileIntegrity(filename, integrity)
|
||||
}
|
||||
|
||||
120
store/cafs/test/normalizeBundledManifest.test.ts
Normal file
120
store/cafs/test/normalizeBundledManifest.test.ts
Normal file
@@ -0,0 +1,120 @@
|
||||
import { normalizeBundledManifest } from '../src/normalizeBundledManifest.js'
|
||||
|
||||
describe('normalizeBundledManifest', () => {
|
||||
it('returns undefined for an empty manifest', () => {
|
||||
expect(normalizeBundledManifest({})).toBeUndefined()
|
||||
})
|
||||
|
||||
it('returns undefined when manifest has only excluded fields', () => {
|
||||
expect(normalizeBundledManifest({
|
||||
description: 'a package',
|
||||
keywords: ['test'],
|
||||
license: 'MIT',
|
||||
author: 'test',
|
||||
repository: 'test/test',
|
||||
devDependencies: { jest: '^29' },
|
||||
})).toBeUndefined()
|
||||
})
|
||||
|
||||
it('picks included fields and excludes others', () => {
|
||||
const result = normalizeBundledManifest({
|
||||
name: 'foo',
|
||||
version: '1.0.0',
|
||||
description: 'should be excluded',
|
||||
license: 'MIT',
|
||||
bin: { foo: './bin/foo.js' },
|
||||
engines: { node: '>=18' },
|
||||
cpu: ['x64'],
|
||||
os: ['linux'],
|
||||
libc: ['glibc'],
|
||||
dependencies: { bar: '^1.0.0' },
|
||||
optionalDependencies: { baz: '^2.0.0' },
|
||||
peerDependencies: { react: '^18' },
|
||||
peerDependenciesMeta: { react: { optional: true } },
|
||||
bundledDependencies: ['bar'],
|
||||
directories: { bin: './bin' },
|
||||
})
|
||||
expect(result).toStrictEqual({
|
||||
name: 'foo',
|
||||
version: '1.0.0',
|
||||
bin: { foo: './bin/foo.js' },
|
||||
engines: { node: '>=18' },
|
||||
cpu: ['x64'],
|
||||
os: ['linux'],
|
||||
libc: ['glibc'],
|
||||
dependencies: { bar: '^1.0.0' },
|
||||
optionalDependencies: { baz: '^2.0.0' },
|
||||
peerDependencies: { react: '^18' },
|
||||
peerDependenciesMeta: { react: { optional: true } },
|
||||
bundledDependencies: ['bar'],
|
||||
directories: { bin: './bin' },
|
||||
})
|
||||
// Excluded fields should not be present
|
||||
expect(result).not.toHaveProperty('description')
|
||||
expect(result).not.toHaveProperty('license')
|
||||
})
|
||||
|
||||
it('only picks lifecycle scripts, not all scripts', () => {
|
||||
const result = normalizeBundledManifest({
|
||||
name: 'foo',
|
||||
version: '1.0.0',
|
||||
scripts: {
|
||||
preinstall: 'echo pre',
|
||||
install: 'echo install',
|
||||
postinstall: 'echo post',
|
||||
test: 'jest',
|
||||
build: 'tsc',
|
||||
start: 'node index.js',
|
||||
prepare: 'tsc',
|
||||
},
|
||||
})
|
||||
expect(result!.scripts).toStrictEqual({
|
||||
preinstall: 'echo pre',
|
||||
install: 'echo install',
|
||||
postinstall: 'echo post',
|
||||
})
|
||||
})
|
||||
|
||||
it('omits scripts key when no lifecycle scripts exist', () => {
|
||||
const result = normalizeBundledManifest({
|
||||
name: 'foo',
|
||||
version: '1.0.0',
|
||||
scripts: {
|
||||
test: 'jest',
|
||||
build: 'tsc',
|
||||
},
|
||||
})
|
||||
expect(result).not.toHaveProperty('scripts')
|
||||
})
|
||||
|
||||
it('normalizes version with semver.clean', () => {
|
||||
expect(normalizeBundledManifest({
|
||||
name: 'foo',
|
||||
version: ' =v1.2.3 ',
|
||||
})!.version).toBe('1.2.3')
|
||||
})
|
||||
|
||||
it('keeps version as-is when semver.clean returns null', () => {
|
||||
expect(normalizeBundledManifest({
|
||||
name: 'foo',
|
||||
version: 'not-semver',
|
||||
})!.version).toBe('not-semver')
|
||||
})
|
||||
|
||||
it('defaults missing version to 0.0.0', () => {
|
||||
expect(normalizeBundledManifest({
|
||||
name: 'foo',
|
||||
})!.version).toBe('0.0.0')
|
||||
})
|
||||
|
||||
it('skips null/undefined fields', () => {
|
||||
const result = normalizeBundledManifest({
|
||||
name: 'foo',
|
||||
version: '1.0.0',
|
||||
bin: undefined,
|
||||
engines: undefined,
|
||||
})
|
||||
expect(result).not.toHaveProperty('bin')
|
||||
expect(result).not.toHaveProperty('engines')
|
||||
})
|
||||
})
|
||||
@@ -86,7 +86,7 @@ export async function handler (opts: FindHashCommandOptions, params: string[]):
|
||||
if (pkgFilesIndex.files) {
|
||||
for (const file of pkgFilesIndex.files.values()) {
|
||||
if (file?.digest === hash) {
|
||||
result.push({ name: pkgFilesIndex.name ?? 'unknown', version: pkgFilesIndex?.version ?? 'unknown', filesIndexFile: filesIndexFile.replace(indexDir, '') })
|
||||
result.push({ name: pkgFilesIndex.manifest?.name ?? 'unknown', version: pkgFilesIndex.manifest?.version ?? 'unknown', filesIndexFile: filesIndexFile.replace(indexDir, '') })
|
||||
|
||||
// a package is only found once.
|
||||
continue
|
||||
@@ -99,7 +99,7 @@ export async function handler (opts: FindHashCommandOptions, params: string[]):
|
||||
if (!added) continue
|
||||
for (const file of added.values()) {
|
||||
if (file?.digest === hash) {
|
||||
result.push({ name: pkgFilesIndex.name ?? 'unknown', version: pkgFilesIndex?.version ?? 'unknown', filesIndexFile: filesIndexFile.replace(indexDir, '') })
|
||||
result.push({ name: pkgFilesIndex.manifest?.name ?? 'unknown', version: pkgFilesIndex.manifest?.version ?? 'unknown', filesIndexFile: filesIndexFile.replace(indexDir, '') })
|
||||
|
||||
// a package is only found once.
|
||||
continue
|
||||
|
||||
@@ -16,8 +16,8 @@ import {
|
||||
} from '@pnpm/cafs-types'
|
||||
import {
|
||||
type AllowBuild,
|
||||
type BundledManifest,
|
||||
type SupportedArchitectures,
|
||||
type DependencyManifest,
|
||||
type PackageManifest,
|
||||
type PinnedVersion,
|
||||
type PackageVersionPolicy,
|
||||
@@ -27,23 +27,7 @@ import {
|
||||
export type { PackageFileInfo, PackageFilesResponse, ImportPackageFunction, ImportPackageFunctionAsync, FilesMap }
|
||||
|
||||
export * from '@pnpm/resolver-base'
|
||||
export type BundledManifest = Pick<
|
||||
DependencyManifest,
|
||||
| 'bin'
|
||||
| 'bundledDependencies'
|
||||
| 'bundleDependencies'
|
||||
| 'cpu'
|
||||
| 'dependencies'
|
||||
| 'directories'
|
||||
| 'engines'
|
||||
| 'name'
|
||||
| 'optionalDependencies'
|
||||
| 'os'
|
||||
| 'peerDependencies'
|
||||
| 'peerDependenciesMeta'
|
||||
| 'scripts'
|
||||
| 'version'
|
||||
>
|
||||
export type { BundledManifest }
|
||||
|
||||
export interface UploadPkgToStoreOpts {
|
||||
filesIndexFile: string
|
||||
|
||||
@@ -6,7 +6,7 @@ import { PnpmError } from '@pnpm/error'
|
||||
import { execSync } from 'child_process'
|
||||
import isWindows from 'is-windows'
|
||||
import { type PackageFilesResponse, type FilesMap } from '@pnpm/cafs-types'
|
||||
import { type DependencyManifest } from '@pnpm/types'
|
||||
import { type BundledManifest } from '@pnpm/types'
|
||||
import pLimit from 'p-limit'
|
||||
import { globalWarn } from '@pnpm/logger'
|
||||
import {
|
||||
@@ -69,7 +69,7 @@ function availableParallelism (): number {
|
||||
|
||||
interface AddFilesResult {
|
||||
filesMap: FilesMap
|
||||
manifest: DependencyManifest
|
||||
manifest?: BundledManifest
|
||||
requiresBuild: boolean
|
||||
integrity?: string
|
||||
}
|
||||
@@ -81,7 +81,7 @@ export async function addFilesFromDir (opts: AddFilesFromDirOptions): Promise<Ad
|
||||
workerPool = createTarballWorkerPool()
|
||||
}
|
||||
const localWorker = await workerPool.checkoutWorkerAsync(true)
|
||||
return new Promise<{ filesMap: FilesMap, manifest: DependencyManifest, requiresBuild: boolean }>((resolve, reject) => {
|
||||
return new Promise<AddFilesResult>((resolve, reject) => {
|
||||
localWorker.once('message', ({ status, error, value }) => {
|
||||
workerPool!.checkinWorker(localWorker)
|
||||
if (status === 'error') {
|
||||
@@ -190,7 +190,7 @@ export interface ReadPkgFromCafsOptions {
|
||||
export interface ReadPkgFromCafsResult {
|
||||
verified: boolean
|
||||
files: PackageFilesResponse
|
||||
manifest?: DependencyManifest
|
||||
bundledManifest?: BundledManifest
|
||||
}
|
||||
|
||||
export async function readPkgFromCafs (
|
||||
|
||||
@@ -14,13 +14,14 @@ import {
|
||||
buildFileMapsFromIndex,
|
||||
createCafs,
|
||||
HASH_ALGORITHM,
|
||||
normalizeBundledManifest,
|
||||
type PackageFilesIndex,
|
||||
type FilesIndex,
|
||||
optimisticRenameOverwrite,
|
||||
type VerifyResult,
|
||||
} from '@pnpm/store.cafs'
|
||||
import { symlinkDependencySync } from '@pnpm/symlink-dependency'
|
||||
import { type DependencyManifest } from '@pnpm/types'
|
||||
import { type BundledManifest, type DependencyManifest } from '@pnpm/types'
|
||||
import { parentPort } from 'worker_threads'
|
||||
import { equalOrSemverEqual } from './equalOrSemverEqual.js'
|
||||
import {
|
||||
@@ -78,7 +79,7 @@ async function handleMessage (
|
||||
break
|
||||
}
|
||||
case 'readPkgFromCafs': {
|
||||
let { storeDir, filesIndexFile, readManifest, verifyStoreIntegrity, expectedPkg, strictStorePkgContentCheck } = message
|
||||
const { storeDir, filesIndexFile, verifyStoreIntegrity, expectedPkg, strictStorePkgContentCheck } = message
|
||||
let pkgFilesIndex: PackageFilesIndex | undefined
|
||||
try {
|
||||
pkgFilesIndex = readMsgpackFileSync<PackageFilesIndex>(filesIndexFile)
|
||||
@@ -99,18 +100,18 @@ async function handleMessage (
|
||||
if (expectedPkg) {
|
||||
if (
|
||||
(
|
||||
pkgFilesIndex.name != null &&
|
||||
pkgFilesIndex.manifest?.name != null &&
|
||||
expectedPkg.name != null &&
|
||||
pkgFilesIndex.name.toLowerCase() !== expectedPkg.name.toLowerCase()
|
||||
pkgFilesIndex.manifest.name.toLowerCase() !== expectedPkg.name.toLowerCase()
|
||||
) ||
|
||||
(
|
||||
pkgFilesIndex.version != null &&
|
||||
pkgFilesIndex.manifest?.version != null &&
|
||||
expectedPkg.version != null &&
|
||||
!equalOrSemverEqual(pkgFilesIndex.version, expectedPkg.version)
|
||||
!equalOrSemverEqual(pkgFilesIndex.manifest.version, expectedPkg.version)
|
||||
)
|
||||
) {
|
||||
const msg = 'Package name or version mismatch found while reading from the store.'
|
||||
const hint = `This means that either the lockfile is broken or the package metadata (name and version) inside the package's package.json file doesn't match the metadata in the registry. Expected package: ${expectedPkg.name}@${expectedPkg.version}. Actual package in the store: ${pkgFilesIndex.name}@${pkgFilesIndex.version}.`
|
||||
const hint = `This means that either the lockfile is broken or the package metadata (name and version) inside the package's package.json file doesn't match the metadata in the registry. Expected package: ${expectedPkg.name}@${expectedPkg.version}. Actual package in the store: ${pkgFilesIndex.manifest?.name}@${pkgFilesIndex.manifest?.version}.`
|
||||
if (strictStorePkgContentCheck ?? true) {
|
||||
throw new PnpmError('UNEXPECTED_PKG_CONTENT_IN_STORE', msg, {
|
||||
hint: `${hint}\n\nIf you want to ignore this issue, set strictStorePkgContentCheck to false in your configuration`,
|
||||
@@ -120,24 +121,21 @@ async function handleMessage (
|
||||
}
|
||||
}
|
||||
}
|
||||
let verifyResult: VerifyResult | undefined
|
||||
if (pkgFilesIndex.requiresBuild == null) {
|
||||
readManifest = true
|
||||
}
|
||||
// Get file maps and optionally verify
|
||||
let verifyResult: VerifyResult
|
||||
if (verifyStoreIntegrity) {
|
||||
verifyResult = checkPkgFilesIntegrity(storeDir, pkgFilesIndex, readManifest)
|
||||
verifyResult = checkPkgFilesIntegrity(storeDir, pkgFilesIndex)
|
||||
} else {
|
||||
verifyResult = buildFileMapsFromIndex(storeDir, pkgFilesIndex, readManifest)
|
||||
verifyResult = buildFileMapsFromIndex(storeDir, pkgFilesIndex)
|
||||
}
|
||||
const requiresBuild = pkgFilesIndex.requiresBuild ?? pkgRequiresBuild(verifyResult.manifest, verifyResult.filesMap)
|
||||
const bundledManifest = pkgFilesIndex.manifest
|
||||
const requiresBuild = pkgFilesIndex.requiresBuild ?? pkgRequiresBuild(bundledManifest, verifyResult.filesMap)
|
||||
|
||||
parentPort!.postMessage({
|
||||
status: 'success',
|
||||
warnings,
|
||||
value: {
|
||||
verified: verifyResult.passed,
|
||||
manifest: verifyResult.manifest,
|
||||
bundledManifest,
|
||||
files: {
|
||||
filesMap: verifyResult.filesMap,
|
||||
sideEffectsMaps: verifyResult.sideEffectsMaps,
|
||||
@@ -196,12 +194,13 @@ function addTarballToStore ({ buffer, storeDir, integrity, filesIndexFile, appen
|
||||
addManifestToCafs(cafs, filesIndex, appendManifest)
|
||||
}
|
||||
const { filesIntegrity, filesMap } = processFilesIndex(filesIndex)
|
||||
const requiresBuild = writeFilesIndexFile(filesIndexFile, { algo: HASH_ALGORITHM, manifest: manifest ?? {}, files: filesIntegrity })
|
||||
const bundledManifest = manifest != null ? normalizeBundledManifest(manifest) : undefined
|
||||
const requiresBuild = writeFilesIndexFile(filesIndexFile, { algo: HASH_ALGORITHM, manifest: bundledManifest, files: filesIntegrity })
|
||||
return {
|
||||
status: 'success',
|
||||
value: {
|
||||
filesMap,
|
||||
manifest,
|
||||
manifest: bundledManifest,
|
||||
requiresBuild,
|
||||
integrity: integrity ?? calcIntegrity(buffer),
|
||||
},
|
||||
@@ -217,7 +216,7 @@ interface AddFilesFromDirResult {
|
||||
status: string
|
||||
value: {
|
||||
filesMap: FilesMap
|
||||
manifest?: DependencyManifest
|
||||
manifest?: BundledManifest
|
||||
requiresBuild: boolean
|
||||
}
|
||||
}
|
||||
@@ -270,6 +269,7 @@ function addFilesFromDir (
|
||||
addManifestToCafs(cafs, filesIndex, appendManifest)
|
||||
}
|
||||
const { filesIntegrity, filesMap } = processFilesIndex(filesIndex)
|
||||
const bundledManifest = manifest != null ? normalizeBundledManifest(manifest) : undefined
|
||||
let requiresBuild: boolean
|
||||
if (sideEffectsCacheKey) {
|
||||
let existingFilesIndex!: PackageFilesIndex
|
||||
@@ -281,7 +281,7 @@ function addFilesFromDir (
|
||||
status: 'success',
|
||||
value: {
|
||||
filesMap,
|
||||
manifest,
|
||||
manifest: bundledManifest,
|
||||
requiresBuild: pkgRequiresBuild(manifest, filesMap),
|
||||
},
|
||||
}
|
||||
@@ -304,9 +304,9 @@ function addFilesFromDir (
|
||||
}
|
||||
writeIndexFile(filesIndexFile, existingFilesIndex)
|
||||
} else {
|
||||
requiresBuild = writeFilesIndexFile(filesIndexFile, { algo: HASH_ALGORITHM, manifest: manifest ?? {}, files: filesIntegrity })
|
||||
requiresBuild = writeFilesIndexFile(filesIndexFile, { algo: HASH_ALGORITHM, manifest: bundledManifest, files: filesIntegrity })
|
||||
}
|
||||
return { status: 'success', value: { filesMap, manifest, requiresBuild } }
|
||||
return { status: 'success', value: { filesMap, manifest: bundledManifest, requiresBuild } }
|
||||
}
|
||||
|
||||
function addManifestToCafs (cafs: CafsFunctions, filesIndex: FilesIndex, manifest: DependencyManifest): void {
|
||||
@@ -414,16 +414,15 @@ function writeFilesIndexFile (
|
||||
filesIndexFile: string,
|
||||
{ algo, manifest, files, sideEffects }: {
|
||||
algo: string
|
||||
manifest: Partial<DependencyManifest>
|
||||
manifest?: BundledManifest
|
||||
files: PackageFiles
|
||||
sideEffects?: SideEffects
|
||||
}
|
||||
): boolean {
|
||||
const requiresBuild = pkgRequiresBuild(manifest, files)
|
||||
const filesIndex: PackageFilesIndex = {
|
||||
name: manifest.name,
|
||||
version: manifest.version,
|
||||
requiresBuild,
|
||||
manifest,
|
||||
algo,
|
||||
files,
|
||||
sideEffects,
|
||||
@@ -443,4 +442,4 @@ function writeIndexFile (filePath: string, data: PackageFilesIndex): void {
|
||||
const temp = `${filePath.slice(0, -10)}${process.pid}`
|
||||
writeMsgpackFileSync(temp, data)
|
||||
optimisticRenameOverwrite(temp, filePath)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user