feat!: skip npm, npx, and corepack when installing node runtime (#11325)

## Summary

- pnpm installing a Node.js runtime (`node@runtime:<ver>`, `pnpm env use`, `pnpm runtime set node`) no longer extracts the bundled `npm`, `npx`, and `corepack`. These make up ~2,800 of ~5,800 files in a typical Node.js archive, so skipping them materially reduces hashing, CAS writes, SQLite index inserts, and import/link work.
- Users who still need `npm` can install it as a separate package.

## How

A new optional `ignoreFilePattern` (regex source string, serializable across the worker boundary) threads through `FetchOptions` → `tarball-fetcher` → `@pnpm/worker` → `cafs.addFilesFromTarball`. `cafs.addFilesFromTarball` now accepts a per-call ignore on top of the existing cafs-level `ignoreFile`; the two are combined.

`@pnpm/fetching.binary-fetcher` defines the Node-specific regex and applies it when `opts.pkg.name === 'node'`:

- Tarball path: sets `ignoreFilePattern`.
- Windows zip path: new `ignoreEntry?: RegExp` on `AssetInfo`; `extractZipToTarget` strips the `basename/` prefix and skips matching entries before `zip.extractEntryTo`.

`@pnpm/engine.runtime.node-resolver`'s `getNodeBinsForCurrentOS` drops `npm`/`npx` so pnpm no longer creates shims for bins that no longer exist.

## Breaking change

Shipping in v11. After this lands, `pnpm runtime set node` / `node@runtime:<version>` no longer puts `npm`, `npx`, or `corepack` on `$PATH`. Scripts that call them directly will need to install npm separately.
This commit is contained in:
Zoltan Kochan
2026-04-21 13:44:48 +02:00
committed by GitHub
parent db81c32e0f
commit 421317c31a
20 changed files with 333 additions and 50 deletions

View File

@@ -0,0 +1,11 @@
---
"@pnpm/fetching.binary-fetcher": minor
"@pnpm/fetching.fetcher-base": minor
"@pnpm/fetching.tarball-fetcher": minor
"@pnpm/engine.runtime.node-resolver": major
"@pnpm/store.cafs": minor
"@pnpm/worker": minor
"pnpm": major
---
Installing a Node.js runtime via `node@runtime:<version>` (including `pnpm env use` and `pnpm runtime set node`) no longer extracts the bundled `npm`, `npx`, and `corepack` from the Node.js archive. This cuts roughly half of the files pnpm has to hash, write to the CAS, and link during installation, making runtime installs noticeably faster. Users who still need `npm` can install it as a separate package.

View File

@@ -23,6 +23,12 @@ export { getNodeArtifactAddress, getNodeMirror, parseNodeSpecifier }
export const DEFAULT_NODE_MIRROR_BASE_URL = 'https://nodejs.org/download/release/'
export const UNOFFICIAL_NODE_MIRROR_BASE_URL = 'https://unofficial-builds.nodejs.org/download/release/'
// Node.js archives ship with npm, npx, and corepack. pnpm manages package managers itself,
// so these are excluded from the runtime install — skipping ~2,800 files out of ~5,800 in the
// Node.js tarball. The pattern matches paths *after* the archive's top-level
// `node-vX.Y.Z-<platform>-<arch>/` prefix has been stripped.
export const NODE_EXTRAS_IGNORE_PATTERN = '^(?:(?:lib/)?node_modules/(?:npm|corepack)(?:/|$)|bin/(?:npm|npx|corepack)$|(?:npm|npx|corepack)(?:\\.(?:cmd|ps1))?$)'
export interface NodeRuntimeResolveResult extends ResolveResult {
resolution: VariationsResolution
resolvedVia: 'nodejs.org'
@@ -198,17 +204,9 @@ async function fetchAllVersions (fetch: FetchFromRegistry, nodeMirrorBaseUrl?: s
function getNodeBinsForCurrentOS (platform: string = process.platform): Record<string, string> {
if (platform === 'win32') {
return {
node: 'node.exe',
npm: 'node_modules/npm/bin/npm-cli.js',
npx: 'node_modules/npm/bin/npx-cli.js',
}
}
return {
node: 'bin/node',
npm: 'lib/node_modules/npm/bin/npm-cli.js',
npx: 'lib/node_modules/npm/bin/npx-cli.js',
return { node: 'node.exe' }
}
return { node: 'bin/node' }
}
function filterVersions (versions: NodeVersion[], versionSelector: string): { versions: string[], versionRange: string } {

View File

@@ -1,5 +1,6 @@
import fsPromises from 'node:fs/promises'
import path from 'node:path'
import util from 'node:util'
import { PnpmError } from '@pnpm/error'
import type { BinaryFetcher, FetchFunction, FetchResult } from '@pnpm/fetching.fetcher-base'
@@ -12,12 +13,37 @@ import { renameOverwrite } from 'rename-overwrite'
import ssri from 'ssri'
import { temporaryDirectory } from 'tempy'
export function createBinaryFetcher (ctx: {
export interface CreateBinaryFetcherOptions {
fetch: FetchFromRegistry
fetchFromRemoteTarball: FetchFunction
storeIndex: StoreIndex
offline?: boolean
}): { binary: BinaryFetcher } {
/**
* Per-package-name regex sources (compatible with `new RegExp(pattern)`) matching file
* paths inside the downloaded archive that should be skipped during extraction.
* The lookup key is `pkg.name`. For zip archives, paths are matched relative to the
* archive's top-level directory (i.e. after the `prefix` has been stripped).
*/
archiveFilters?: Record<string, string>
}
export function createBinaryFetcher (ctx: CreateBinaryFetcherOptions): { binary: BinaryFetcher } {
// Snapshot and pre-compile `archiveFilters` at creation time so later mutations to the
// caller's object can't reintroduce invalid patterns, and so zip extraction doesn't
// recompile the regex per fetch. The tarball path still needs the pattern string — it
// crosses the worker thread boundary, where RegExp instances don't survive structured clone.
const archiveFilters = new Map<string, { pattern: string, regex: RegExp }>()
for (const [name, pattern] of Object.entries(ctx.archiveFilters ?? {})) {
try {
archiveFilters.set(name, { pattern, regex: new RegExp(pattern) })
} catch (err: unknown) {
const detail = util.types.isNativeError(err) ? `: ${err.message}` : ''
throw new PnpmError(
'INVALID_ARCHIVE_FILTER',
`Invalid archive filter regex for "${name}"${detail}: ${pattern}`
)
}
}
const fetchBinary: BinaryFetcher = async (cafs, resolution, opts) => {
if (ctx.offline) {
throw new PnpmError('CANNOT_DOWNLOAD_BINARY_OFFLINE', `Cannot download binary "${resolution.url}" because offline mode is enabled.`)
@@ -28,6 +54,7 @@ export function createBinaryFetcher (ctx: {
version: opts.pkg.version!,
bin: resolution.bin,
}
const archiveFilter = opts.pkg.name != null ? archiveFilters.get(opts.pkg.name) : undefined
let fetchResult!: FetchResult
switch (resolution.archive) {
@@ -36,8 +63,9 @@ export function createBinaryFetcher (ctx: {
tarball: resolution.url,
integrity: resolution.integrity,
}, {
appendManifest: manifest,
...opts,
appendManifest: manifest,
ignoreFilePattern: archiveFilter?.pattern ?? opts.ignoreFilePattern,
})
break
}
@@ -47,6 +75,7 @@ export function createBinaryFetcher (ctx: {
url: resolution.url,
integrity: resolution.integrity,
basename: resolution.prefix ?? '',
ignoreEntry: archiveFilter?.regex,
}, tempLocation)
fetchResult = await addFilesFromDir({
storeDir: cafs.storeDir,
@@ -77,6 +106,11 @@ export interface AssetInfo {
url: string
integrity: string
basename: string
/**
* Regex matched against each zip entry's path relative to the archive's top-level basename.
* Matching entries are skipped during extraction.
*/
ignoreEntry?: RegExp
}
/**
@@ -96,7 +130,7 @@ export async function downloadAndUnpackZip (
try {
await downloadWithIntegrityCheck(fetchFromRegistry, assetInfo, tmp)
await extractZipToTarget(tmp, assetInfo.basename, targetDir)
await extractZipToTarget(tmp, assetInfo.basename, targetDir, assetInfo.ignoreEntry)
} finally {
// Clean up temporary file
try {
@@ -144,12 +178,15 @@ async function downloadWithIntegrityCheck (
* @param zipPath - Path to the zip file
* @param basename - Base name of the file (without extension)
* @param targetDir - Directory where contents should be extracted
* @param ignoreEntry - Optional regex matched against the entry path relative to `basename`;
* matching entries are skipped.
* @throws {PnpmError} When extraction fails or path traversal is detected
*/
async function extractZipToTarget (
zipPath: string,
basename: string,
targetDir: string
targetDir: string,
ignoreEntry?: RegExp
): Promise<void> {
const zip = new AdmZip(zipPath)
const nodeDir = basename === '' ? targetDir : path.dirname(targetDir)
@@ -159,10 +196,22 @@ async function extractZipToTarget (
validatePathSecurity(nodeDir, basename)
}
const basenamePrefix = basename === '' ? '' : `${basename}/`
// Normalize `ignoreEntry` to a stateless regex. `.test()` on a `/g` or `/y` regex
// advances `lastIndex` between calls, which would cause inconsistent skips across
// entries in this loop.
const testEntry = toStatelessTester(ignoreEntry)
// Extract each entry with path validation to prevent path traversal attacks
for (const entry of zip.getEntries()) {
const entryPath = entry.entryName
validatePathSecurity(nodeDir, entryPath)
if (testEntry) {
const relative = basenamePrefix && entryPath.startsWith(basenamePrefix)
? entryPath.slice(basenamePrefix.length)
: entryPath
if (testEntry(relative)) continue
}
zip.extractEntryTo(entry, nodeDir, true, true)
}
@@ -170,6 +219,18 @@ async function extractZipToTarget (
await renameOverwrite(extractedDir, targetDir)
}
function toStatelessTester (regex: RegExp | undefined): ((input: string) => boolean) | undefined {
if (!regex) return undefined
// `/g` and `/y` make `RegExp.prototype.test` stateful via `lastIndex`.
// Strip those flags by cloning into a fresh RegExp with only the safe flags.
if (!regex.global && !regex.sticky) {
return (input) => regex.test(input)
}
const safeFlags = regex.flags.replace(/[gy]/g, '')
const clone = new RegExp(regex.source, safeFlags)
return (input) => clone.test(input)
}
/**
* Validates that a path does not escape the base directory via path traversal.
*

View File

@@ -3,7 +3,7 @@ import fs from 'node:fs'
import path from 'node:path'
import { PnpmError } from '@pnpm/error'
import { downloadAndUnpackZip } from '@pnpm/fetching.binary-fetcher'
import { createBinaryFetcher, downloadAndUnpackZip } from '@pnpm/fetching.binary-fetcher'
import AdmZip from 'adm-zip'
import ssri from 'ssri'
import { temporaryDirectory } from 'tempy'
@@ -220,5 +220,149 @@ describe('extractZipToTarget security', () => {
expect(fs.existsSync(path.join(targetDir, 'bin/node'))).toBe(true)
})
it('skips entries matching ignoreEntry regex (basename stripped)', async () => {
const targetDir = temporaryDirectory()
const zip = new AdmZip()
zip.addFile('node-v20.0.0/node.exe', Buffer.from('binary'))
zip.addFile('node-v20.0.0/npm', Buffer.from('npm shim'))
zip.addFile('node-v20.0.0/npm.cmd', Buffer.from('npm cmd'))
zip.addFile('node-v20.0.0/node_modules/npm/package.json', Buffer.from('{}'))
zip.addFile('node-v20.0.0/node_modules/corepack/package.json', Buffer.from('{}'))
zip.addFile('node-v20.0.0/node_modules/keep-me/index.js', Buffer.from('kept'))
const zipBuffer = zip.toBuffer()
const integrity = ssri.fromData(zipBuffer).toString()
const mockFetch = createMockFetch(zipBuffer)
await downloadAndUnpackZip(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
mockFetch as any,
{
url: 'https://example.com/node.zip',
integrity,
basename: 'node-v20.0.0',
ignoreEntry: /^(?:node_modules\/(?:npm|corepack)(?:\/|$)|npm(?:\.cmd)?$)/,
},
targetDir
)
expect(fs.existsSync(path.join(targetDir, 'node.exe'))).toBe(true)
expect(fs.existsSync(path.join(targetDir, 'node_modules/keep-me/index.js'))).toBe(true)
expect(fs.existsSync(path.join(targetDir, 'npm'))).toBe(false)
expect(fs.existsSync(path.join(targetDir, 'npm.cmd'))).toBe(false)
expect(fs.existsSync(path.join(targetDir, 'node_modules/npm'))).toBe(false)
expect(fs.existsSync(path.join(targetDir, 'node_modules/corepack'))).toBe(false)
})
it('skips entries matching ignoreEntry regex when basename is empty', async () => {
const targetDir = temporaryDirectory()
const zip = new AdmZip()
zip.addFile('bin/node', Buffer.from('#!/bin/sh\necho "node"'))
zip.addFile('bin/npm', Buffer.from('npm shim'))
const zipBuffer = zip.toBuffer()
const integrity = ssri.fromData(zipBuffer).toString()
const mockFetch = createMockFetch(zipBuffer)
await downloadAndUnpackZip(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
mockFetch as any,
{
url: 'https://example.com/node.zip',
integrity,
basename: '',
ignoreEntry: /^bin\/npm$/,
},
targetDir
)
expect(fs.existsSync(path.join(targetDir, 'bin/node'))).toBe(true)
expect(fs.existsSync(path.join(targetDir, 'bin/npm'))).toBe(false)
})
it('strips /g /y flags from ignoreEntry so .test() is not stateful across entries', async () => {
const targetDir = temporaryDirectory()
const zip = new AdmZip()
zip.addFile('node-v20.0.0/node.exe', Buffer.from('binary'))
zip.addFile('node-v20.0.0/npm', Buffer.from('npm shim 1'))
zip.addFile('node-v20.0.0/npx', Buffer.from('npx shim 2'))
zip.addFile('node-v20.0.0/corepack', Buffer.from('corepack 3'))
const zipBuffer = zip.toBuffer()
const integrity = ssri.fromData(zipBuffer).toString()
const mockFetch = createMockFetch(zipBuffer)
await downloadAndUnpackZip(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
mockFetch as any,
{
url: 'https://example.com/node.zip',
integrity,
basename: 'node-v20.0.0',
// Deliberately pass a /g regex — a stateful .test() would skip only
// every other matching entry. All three shims must still be dropped.
ignoreEntry: /^(?:npm|npx|corepack)$/g,
},
targetDir
)
expect(fs.existsSync(path.join(targetDir, 'node.exe'))).toBe(true)
expect(fs.existsSync(path.join(targetDir, 'npm'))).toBe(false)
expect(fs.existsSync(path.join(targetDir, 'npx'))).toBe(false)
expect(fs.existsSync(path.join(targetDir, 'corepack'))).toBe(false)
})
})
})
describe('createBinaryFetcher', () => {
it('rejects an invalid archiveFilters regex at creation time', () => {
const noop = (() => {
throw new Error('should not be called')
}) as never
expect(() =>
createBinaryFetcher({
fetch: noop,
fetchFromRemoteTarball: noop,
storeIndex: noop,
archiveFilters: { node: '(' },
})
).toThrow(PnpmError)
expect(() =>
createBinaryFetcher({
fetch: noop,
fetchFromRemoteTarball: noop,
storeIndex: noop,
archiveFilters: { node: '(' },
})
).toThrow(/Invalid archive filter regex for "node"/)
})
it('snapshots archiveFilters so post-creation mutations cannot reintroduce invalid patterns', () => {
const noop = (() => {
throw new Error('should not be called')
}) as never
const filters: Record<string, string> = { node: '^ok$' }
// Must succeed — the pattern is valid at construction time.
expect(() =>
createBinaryFetcher({
fetch: noop,
fetchFromRemoteTarball: noop,
storeIndex: noop,
archiveFilters: filters,
})
).not.toThrow()
// Mutating the caller's object after construction must not affect the fetcher.
// There's no direct read back, but any mutation reaching the fetcher would throw
// on subsequent fetches; the snapshot guarantees it can't.
filters.node = '('
// Reconstructing with the broken pattern fails — demonstrating the original
// fetcher would have failed at construction if it had seen the broken pattern.
expect(() =>
createBinaryFetcher({
fetch: noop,
fetchFromRemoteTarball: noop,
storeIndex: noop,
archiveFilters: filters,
})
).toThrow(/Invalid archive filter regex for "node"/)
})
})

View File

@@ -21,6 +21,11 @@ export interface FetchOptions {
readManifest?: boolean
pkg: PkgNameVersion
appendManifest?: DependencyManifest
/**
* Regex source (compatible with `new RegExp(pattern)`) matching file paths inside the
* downloaded archive that should be skipped during extraction. Honored by tarball fetchers.
*/
ignoreFilePattern?: string
}
export type FetchFunction<FetcherResolution = Resolution, Options = FetchOptions, Result = FetchResult> = (

View File

@@ -99,5 +99,6 @@ async function fetchFromTarball (
filesIndexFile: opts.filesIndexFile,
pkg: opts.pkg,
appendManifest: opts.appendManifest,
ignoreFilePattern: opts.ignoreFilePattern,
})
}

View File

@@ -28,6 +28,7 @@ export function createLocalTarballFetcher (storeIndex: StoreIndex): FetchFunctio
url: tarball,
pkg: opts.pkg,
appendManifest: opts.appendManifest,
ignoreFilePattern: opts.ignoreFilePattern,
})
}

View File

@@ -28,7 +28,7 @@ export type DownloadOptions = {
onProgress?: (downloaded: number) => void
integrity?: string
storeIndex: StoreIndex
} & Pick<FetchOptions, 'pkg' | 'appendManifest' | 'readManifest' | 'filesIndexFile'>
} & Pick<FetchOptions, 'pkg' | 'appendManifest' | 'readManifest' | 'filesIndexFile' | 'ignoreFilePattern'>
export type DownloadFunction = (url: string, opts: DownloadOptions) => Promise<FetchResult>
@@ -208,6 +208,7 @@ export function createDownloader (
url,
pkg: opts.pkg,
appendManifest: opts.appendManifest,
ignoreFilePattern: opts.ignoreFilePattern,
})
}
}

View File

@@ -33,6 +33,7 @@
".test": "cross-env NODE_OPTIONS=\"$NODE_OPTIONS --experimental-vm-modules --disable-warning=ExperimentalWarning --disable-warning=DEP0169\" jest"
},
"dependencies": {
"@pnpm/engine.runtime.node-resolver": "workspace:*",
"@pnpm/fetching.binary-fetcher": "workspace:*",
"@pnpm/fetching.directory-fetcher": "workspace:*",
"@pnpm/fetching.git-fetcher": "workspace:*",

View File

@@ -1,3 +1,4 @@
import { NODE_EXTRAS_IGNORE_PATTERN } from '@pnpm/engine.runtime.node-resolver'
import { createBinaryFetcher } from '@pnpm/fetching.binary-fetcher'
import { createDirectoryFetcher } from '@pnpm/fetching.directory-fetcher'
import type { BinaryFetcher, DirectoryFetcher, GitFetcher } from '@pnpm/fetching.fetcher-base'
@@ -81,6 +82,7 @@ function createFetchers (
fetchFromRemoteTarball: tarballFetchers.remoteTarball,
offline: opts.offline,
storeIndex: opts.storeIndex,
archiveFilters: { node: NODE_EXTRAS_IGNORE_PATTERN },
}),
}
}

View File

@@ -12,6 +12,9 @@
{
"path": "../../core/types"
},
{
"path": "../../engine/runtime/node-resolver"
},
{
"path": "../../fetching/binary-fetcher"
},

View File

@@ -27,8 +27,6 @@ const GLIBC_RESOLUTIONS = [
integrity: 'sha256-13Q/3fXoZxJPVVqR9scpEE/Vx12TgvEChsP7s/0S7wc=',
bin: {
node: 'bin/node',
npm: 'lib/node_modules/npm/bin/npm-cli.js',
npx: 'lib/node_modules/npm/bin/npx-cli.js',
},
},
},
@@ -46,8 +44,6 @@ const GLIBC_RESOLUTIONS = [
integrity: 'sha256-6pbTSc+qZ6qHzuqj5bUskWf3rDAv2NH/Fi0HhencB4U=',
bin: {
node: 'bin/node',
npm: 'lib/node_modules/npm/bin/npm-cli.js',
npx: 'lib/node_modules/npm/bin/npx-cli.js',
},
},
},
@@ -65,8 +61,6 @@ const GLIBC_RESOLUTIONS = [
integrity: 'sha256-Qio4h/9UGPCkVS2Jz5k0arirUbtdOEZguqiLhETSwRE=',
bin: {
node: 'bin/node',
npm: 'lib/node_modules/npm/bin/npm-cli.js',
npx: 'lib/node_modules/npm/bin/npx-cli.js',
},
},
},
@@ -84,8 +78,6 @@ const GLIBC_RESOLUTIONS = [
integrity: 'sha256-HTVHImvn5ZrO7lx9Aan4/BjeZ+AVxaFdjPOFtuAtBis=',
bin: {
node: 'bin/node',
npm: 'lib/node_modules/npm/bin/npm-cli.js',
npx: 'lib/node_modules/npm/bin/npx-cli.js',
},
},
},
@@ -103,8 +95,6 @@ const GLIBC_RESOLUTIONS = [
integrity: 'sha256-0h239Xxc4YKuwrmoPjKVq8N+FzGrtzmV09Vz4EQJl3w=',
bin: {
node: 'bin/node',
npm: 'lib/node_modules/npm/bin/npm-cli.js',
npx: 'lib/node_modules/npm/bin/npx-cli.js',
},
},
},
@@ -122,8 +112,6 @@ const GLIBC_RESOLUTIONS = [
integrity: 'sha256-OwmNzPVtRGu7gIRdNbvsvbdGEoYNFpDzohY4fJnJ1iA=',
bin: {
node: 'bin/node',
npm: 'lib/node_modules/npm/bin/npm-cli.js',
npx: 'lib/node_modules/npm/bin/npx-cli.js',
},
},
},
@@ -141,8 +129,6 @@ const GLIBC_RESOLUTIONS = [
integrity: 'sha256-fsX9rQyBnuoXkA60PB3pSNYgp4OxrJQGLKpDh3ipKzA=',
bin: {
node: 'bin/node',
npm: 'lib/node_modules/npm/bin/npm-cli.js',
npx: 'lib/node_modules/npm/bin/npx-cli.js',
},
},
},
@@ -160,8 +146,6 @@ const GLIBC_RESOLUTIONS = [
integrity: 'sha256-dLsPOoAwfFKUIcPthFF7j1Q4Z3CfQeU81z35nmRCr00=',
bin: {
node: 'bin/node',
npm: 'lib/node_modules/npm/bin/npm-cli.js',
npx: 'lib/node_modules/npm/bin/npx-cli.js',
},
},
},
@@ -179,8 +163,6 @@ const GLIBC_RESOLUTIONS = [
integrity: 'sha256-N2Ehz0a9PAJcXmetrhkK/14l0zoLWPvA2GUtczULOPA=',
bin: {
node: 'node.exe',
npm: 'node_modules/npm/bin/npm-cli.js',
npx: 'node_modules/npm/bin/npx-cli.js',
},
prefix: 'node-v22.0.0-win-arm64',
},
@@ -199,8 +181,6 @@ const GLIBC_RESOLUTIONS = [
integrity: 'sha256-MtY5tH1MCmUf+PjX1BpFQWij1ARb43mF+agQz4zvYXQ=',
bin: {
node: 'node.exe',
npm: 'node_modules/npm/bin/npm-cli.js',
npx: 'node_modules/npm/bin/npx-cli.js',
},
prefix: 'node-v22.0.0-win-x64',
},
@@ -219,8 +199,6 @@ const GLIBC_RESOLUTIONS = [
integrity: 'sha256-4BNPUBcVSjN2csf7zRVOKyx3S0MQkRhWAZINY9DEt9A=',
bin: {
node: 'node.exe',
npm: 'node_modules/npm/bin/npm-cli.js',
npx: 'node_modules/npm/bin/npx-cli.js',
},
prefix: 'node-v22.0.0-win-x86',
},
@@ -234,6 +212,25 @@ test('installing Node.js runtime', async () => {
project.isExecutable('.bin/node')
expect(fs.readlinkSync('node_modules/node')).toContain(path.join('links', '@', 'node', '22.0.0'))
// npm, npx, and corepack are stripped from the Node.js archive; no shims or bundled sources should exist.
const nodeInstallDir = fs.realpathSync(path.resolve('node_modules/node'))
const isWindows = process.platform === 'win32'
const bundledNpmDir = isWindows
? path.join(nodeInstallDir, 'node_modules', 'npm')
: path.join(nodeInstallDir, 'lib', 'node_modules', 'npm')
const bundledCorepackDir = isWindows
? path.join(nodeInstallDir, 'node_modules', 'corepack')
: path.join(nodeInstallDir, 'lib', 'node_modules', 'corepack')
expect(fs.existsSync(bundledNpmDir)).toBe(false)
expect(fs.existsSync(bundledCorepackDir)).toBe(false)
for (const shim of ['npm', 'npx', 'corepack']) {
expect(fs.existsSync(path.resolve('node_modules/.bin', shim))).toBe(false)
expect(fs.existsSync(path.resolve('node_modules/.bin', `${shim}.cmd`))).toBe(false)
if (isWindows) {
expect(fs.existsSync(path.resolve('node_modules/.bin', `${shim}.ps1`))).toBe(false)
}
}
const lockfile = project.readLockfile()
expect(lockfile).toStrictEqual({
settings: {

3
pnpm-lock.yaml generated
View File

@@ -4893,6 +4893,9 @@ importers:
installing/client:
dependencies:
'@pnpm/engine.runtime.node-resolver':
specifier: workspace:*
version: link:../../engine/runtime/node-resolver
'@pnpm/fetching.binary-fetcher':
specifier: workspace:*
version: link:../../fetching/binary-fetcher

View File

@@ -70,7 +70,7 @@ export interface AddToStoreResult {
export interface Cafs {
storeDir: string
addFilesFromDir: (dir: string) => AddToStoreResult
addFilesFromTarball: (buffer: Buffer) => AddToStoreResult
addFilesFromTarball: (buffer: Buffer, readManifest?: boolean, ignore?: (filename: string) => boolean) => AddToStoreResult
addFile: (buffer: Buffer, mode: number) => FileWriteResult
getFilePathByModeInCafs: (digest: string, mode: number) => string
importPackage: ImportPackageFunction

View File

@@ -13,11 +13,10 @@ import { parseTarball } from './parseTarball.js'
export function addFilesFromTarball (
addBufferToCafs: (buffer: Buffer, mode: number) => FileWriteResult,
_ignore: null | ((filename: string) => boolean),
tarballBuffer: Buffer,
readManifest?: boolean
readManifest?: boolean,
ignore?: (filename: string) => boolean
): AddToStoreResult {
const ignore = _ignore ?? (() => false)
// chunkSize 128KB (8x the Node.js default of 16KB) reduces the number of
// internal buffer allocations and copies during decompression. Benchmarks
// showed ~2.3x faster decompress at 128KB. Larger values (256KB+) showed
@@ -36,7 +35,7 @@ export function addFilesFromTarball (
let manifestBuffer: Buffer | undefined
for (const [relativePath, { mode, offset, size }] of files) {
if (ignore(relativePath)) continue
if (ignore?.(relativePath)) continue
const fileBuffer = tarContent.subarray(offset, offset + size)
if (readManifest && relativePath === 'package.json') {

View File

@@ -58,7 +58,7 @@ export interface CreateCafsOpts {
export interface CafsFunctions {
addFilesFromDir: (dirname: string, opts?: { files?: string[], readManifest?: boolean, includeNodeModules?: boolean }) => AddToStoreResult
addFilesFromTarball: (tarballBuffer: Buffer, readManifest?: boolean) => AddToStoreResult
addFilesFromTarball: (tarballBuffer: Buffer, readManifest?: boolean, ignore?: (filename: string) => boolean) => AddToStoreResult
addFile: (buffer: Buffer, mode: number) => FileWriteResult
getFilePathByModeInCafs: (digest: string, mode: number) => string
}
@@ -68,12 +68,22 @@ export function createCafs (storeDir: string, { ignoreFile, cafsLocker }: Create
const addBuffer = addBufferToCafs.bind(null, _writeBufferToCafs)
return {
addFilesFromDir: addFilesFromDir.bind(null, addBuffer),
addFilesFromTarball: addFilesFromTarball.bind(null, addBuffer, ignoreFile ?? null),
addFilesFromTarball: (tarballBuffer, readManifest, callIgnore) =>
addFilesFromTarball(addBuffer, tarballBuffer, readManifest, combineIgnore(ignoreFile, callIgnore)),
addFile: addBuffer,
getFilePathByModeInCafs: getFilePathByModeInCafs.bind(null, storeDir),
}
}
function combineIgnore (
a?: (filename: string) => boolean,
b?: (filename: string) => boolean
): ((filename: string) => boolean) | undefined {
if (!a) return b
if (!b) return a
return (filename) => a(filename) || b(filename)
}
type WriteBufferToCafs = (buffer: Buffer, fileDest: string, mode: number | undefined, integrity: Integrity) => { checkedAt: number, filePath: string }
function addBufferToCafs (

View File

@@ -29,6 +29,25 @@ describe('cafs', () => {
expect(pkgFile!.digest).toBe('f310afae50bb5b74e5c17c5eb6fe426538b9deccd88664fbb66a5717fb6d36d86d4d1f530bb63b58914f9894e81da490e2e39bb99c8e01174e258358b9349b5c')
})
it('addFilesFromTarball honors a per-call ignore predicate', () => {
const dest = temporaryDirectory()
const cafs = createCafs(dest)
const tarball = fs.readFileSync(f.find('node-gyp-6.1.0.tgz'))
const baseline = cafs.addFilesFromTarball(tarball)
const filtered = cafs.addFilesFromTarball(tarball, false, (name) => name === 'package.json')
expect(filtered.filesIndex.has('package.json')).toBe(false)
expect(filtered.filesIndex.size).toBe(baseline.filesIndex.size - 1)
})
it('addFilesFromTarball combines cafs-level ignoreFile with per-call ignore', () => {
const dest = temporaryDirectory()
const cafs = createCafs(dest, { ignoreFile: (name) => name === 'package.json' })
const tarball = fs.readFileSync(f.find('node-gyp-6.1.0.tgz'))
const { filesIndex } = cafs.addFilesFromTarball(tarball, false, (name) => name === 'README.md')
expect(filesIndex.has('package.json')).toBe(false)
expect(filesIndex.has('README.md')).toBe(false)
})
it('replaces an already existing file, if the integrity of it was broken', () => {
const storeDir = temporaryDirectory()
const srcDir = path.join(import.meta.dirname, 'fixtures/one-file')

View File

@@ -154,7 +154,7 @@ If you think that this is the case, then run "pnpm store prune" and rerun the co
}
}
type AddFilesFromTarballOptions = Pick<TarballExtractMessage, 'buffer' | 'storeDir' | 'filesIndexFile' | 'integrity' | 'readManifest' | 'pkg' | 'appendManifest'> & {
type AddFilesFromTarballOptions = Pick<TarballExtractMessage, 'buffer' | 'storeDir' | 'filesIndexFile' | 'integrity' | 'readManifest' | 'pkg' | 'appendManifest' | 'ignoreFilePattern'> & {
storeIndex: StoreIndex
url: string
}
@@ -192,7 +192,8 @@ export async function addFilesFromTarball (opts: AddFilesFromTarballOptions): Pr
readManifest: opts.readManifest,
pkg: opts.pkg,
appendManifest: opts.appendManifest,
})
ignoreFilePattern: opts.ignoreFilePattern,
} satisfies TarballExtractMessage)
})
}

View File

@@ -1,6 +1,7 @@
import crypto from 'node:crypto'
import fs from 'node:fs'
import path from 'node:path'
import util from 'node:util'
import { parentPort } from 'node:worker_threads'
import { pkgRequiresBuild } from '@pnpm/building.pkg-requires-build'
@@ -185,7 +186,7 @@ async function handleMessage (
}
}
function addTarballToStore ({ buffer, storeDir, integrity, filesIndexFile, appendManifest }: TarballExtractMessage) {
function addTarballToStore ({ buffer, storeDir, integrity, filesIndexFile, appendManifest, ignoreFilePattern }: TarballExtractMessage) {
if (integrity) {
const { algorithm, hexDigest } = parseIntegrity(integrity)
const calculatedHash: string = crypto.hash(algorithm, buffer, 'hex')
@@ -205,7 +206,8 @@ function addTarballToStore ({ buffer, storeDir, integrity, filesIndexFile, appen
cafsCache.set(storeDir, createCafs(storeDir))
}
const cafs = cafsCache.get(storeDir)!
let { filesIndex, manifest } = cafs.addFilesFromTarball(buffer, true)
const ignore = ignoreFilePattern ? makeIgnoreFromPattern(ignoreFilePattern) : undefined
let { filesIndex, manifest } = cafs.addFilesFromTarball(buffer, true, ignore)
if (appendManifest && manifest == null) {
manifest = appendManifest
addManifestToCafs(cafs, filesIndex, appendManifest)
@@ -238,6 +240,24 @@ function calcIntegrity (buffer: Buffer): string {
return formatIntegrity('sha512', calculatedHash)
}
function makeIgnoreFromPattern (pattern: string): (filename: string) => boolean {
// `ignoreFilePattern` is a public field on FetchOptions, so callers that don't go
// through the binary-fetcher's validated `archiveFilters` path could still supply a
// bad regex. Convert the SyntaxError into a PnpmError with a stable code so it's
// actionable for users.
let regex: RegExp
try {
regex = new RegExp(pattern)
} catch (err: unknown) {
const detail = util.types.isNativeError(err) ? `: ${err.message}` : ''
throw new PnpmError(
'INVALID_IGNORE_FILE_PATTERN',
`Invalid ignoreFilePattern regex${detail}: ${pattern}`
)
}
return (filename) => regex.test(filename)
}
function packToShared (data: unknown): Uint8Array {
const packed = packForStorage(data)
const shared = new SharedArrayBuffer(packed.byteLength)

View File

@@ -20,6 +20,12 @@ export interface TarballExtractMessage {
readManifest?: boolean
pkg?: PkgNameVersion
appendManifest?: DependencyManifest
/**
* Regex source matching the normalized relative path of files inside the tarball that
* should be skipped. Matching happens after the tar parser strips the top-level directory
* segment — i.e. against the same path form that is written to `filesIndex`.
*/
ignoreFilePattern?: string
}
export interface LinkPkgMessage {