mirror of
https://github.com/pnpm/pnpm.git
synced 2026-05-12 10:11:42 -04:00
feat!: skip npm, npx, and corepack when installing node runtime (#11325)
## Summary - pnpm installing a Node.js runtime (`node@runtime:<ver>`, `pnpm env use`, `pnpm runtime set node`) no longer extracts the bundled `npm`, `npx`, and `corepack`. These make up ~2,800 of ~5,800 files in a typical Node.js archive, so skipping them materially reduces hashing, CAS writes, SQLite index inserts, and import/link work. - Users who still need `npm` can install it as a separate package. ## How A new optional `ignoreFilePattern` (regex source string, serializable across the worker boundary) threads through `FetchOptions` → `tarball-fetcher` → `@pnpm/worker` → `cafs.addFilesFromTarball`. `cafs.addFilesFromTarball` now accepts a per-call ignore on top of the existing cafs-level `ignoreFile`; the two are combined. `@pnpm/fetching.binary-fetcher` defines the Node-specific regex and applies it when `opts.pkg.name === 'node'`: - Tarball path: sets `ignoreFilePattern`. - Windows zip path: new `ignoreEntry?: RegExp` on `AssetInfo`; `extractZipToTarget` strips the `basename/` prefix and skips matching entries before `zip.extractEntryTo`. `@pnpm/engine.runtime.node-resolver`'s `getNodeBinsForCurrentOS` drops `npm`/`npx` so pnpm no longer creates shims for bins that no longer exist. ## Breaking change Shipping in v11. After this lands, `pnpm runtime set node` / `node@runtime:<version>` no longer puts `npm`, `npx`, or `corepack` on `$PATH`. Scripts that call them directly will need to install npm separately.
This commit is contained in:
11
.changeset/node-runtime-without-npm.md
Normal file
11
.changeset/node-runtime-without-npm.md
Normal file
@@ -0,0 +1,11 @@
|
||||
---
|
||||
"@pnpm/fetching.binary-fetcher": minor
|
||||
"@pnpm/fetching.fetcher-base": minor
|
||||
"@pnpm/fetching.tarball-fetcher": minor
|
||||
"@pnpm/engine.runtime.node-resolver": major
|
||||
"@pnpm/store.cafs": minor
|
||||
"@pnpm/worker": minor
|
||||
"pnpm": major
|
||||
---
|
||||
|
||||
Installing a Node.js runtime via `node@runtime:<version>` (including `pnpm env use` and `pnpm runtime set node`) no longer extracts the bundled `npm`, `npx`, and `corepack` from the Node.js archive. This cuts roughly half of the files pnpm has to hash, write to the CAS, and link during installation, making runtime installs noticeably faster. Users who still need `npm` can install it as a separate package.
|
||||
@@ -23,6 +23,12 @@ export { getNodeArtifactAddress, getNodeMirror, parseNodeSpecifier }
|
||||
export const DEFAULT_NODE_MIRROR_BASE_URL = 'https://nodejs.org/download/release/'
|
||||
export const UNOFFICIAL_NODE_MIRROR_BASE_URL = 'https://unofficial-builds.nodejs.org/download/release/'
|
||||
|
||||
// Node.js archives ship with npm, npx, and corepack. pnpm manages package managers itself,
|
||||
// so these are excluded from the runtime install — skipping ~2,800 files out of ~5,800 in the
|
||||
// Node.js tarball. The pattern matches paths *after* the archive's top-level
|
||||
// `node-vX.Y.Z-<platform>-<arch>/` prefix has been stripped.
|
||||
export const NODE_EXTRAS_IGNORE_PATTERN = '^(?:(?:lib/)?node_modules/(?:npm|corepack)(?:/|$)|bin/(?:npm|npx|corepack)$|(?:npm|npx|corepack)(?:\\.(?:cmd|ps1))?$)'
|
||||
|
||||
export interface NodeRuntimeResolveResult extends ResolveResult {
|
||||
resolution: VariationsResolution
|
||||
resolvedVia: 'nodejs.org'
|
||||
@@ -198,17 +204,9 @@ async function fetchAllVersions (fetch: FetchFromRegistry, nodeMirrorBaseUrl?: s
|
||||
|
||||
function getNodeBinsForCurrentOS (platform: string = process.platform): Record<string, string> {
|
||||
if (platform === 'win32') {
|
||||
return {
|
||||
node: 'node.exe',
|
||||
npm: 'node_modules/npm/bin/npm-cli.js',
|
||||
npx: 'node_modules/npm/bin/npx-cli.js',
|
||||
}
|
||||
}
|
||||
return {
|
||||
node: 'bin/node',
|
||||
npm: 'lib/node_modules/npm/bin/npm-cli.js',
|
||||
npx: 'lib/node_modules/npm/bin/npx-cli.js',
|
||||
return { node: 'node.exe' }
|
||||
}
|
||||
return { node: 'bin/node' }
|
||||
}
|
||||
|
||||
function filterVersions (versions: NodeVersion[], versionSelector: string): { versions: string[], versionRange: string } {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import fsPromises from 'node:fs/promises'
|
||||
import path from 'node:path'
|
||||
import util from 'node:util'
|
||||
|
||||
import { PnpmError } from '@pnpm/error'
|
||||
import type { BinaryFetcher, FetchFunction, FetchResult } from '@pnpm/fetching.fetcher-base'
|
||||
@@ -12,12 +13,37 @@ import { renameOverwrite } from 'rename-overwrite'
|
||||
import ssri from 'ssri'
|
||||
import { temporaryDirectory } from 'tempy'
|
||||
|
||||
export function createBinaryFetcher (ctx: {
|
||||
export interface CreateBinaryFetcherOptions {
|
||||
fetch: FetchFromRegistry
|
||||
fetchFromRemoteTarball: FetchFunction
|
||||
storeIndex: StoreIndex
|
||||
offline?: boolean
|
||||
}): { binary: BinaryFetcher } {
|
||||
/**
|
||||
* Per-package-name regex sources (compatible with `new RegExp(pattern)`) matching file
|
||||
* paths inside the downloaded archive that should be skipped during extraction.
|
||||
* The lookup key is `pkg.name`. For zip archives, paths are matched relative to the
|
||||
* archive's top-level directory (i.e. after the `prefix` has been stripped).
|
||||
*/
|
||||
archiveFilters?: Record<string, string>
|
||||
}
|
||||
|
||||
export function createBinaryFetcher (ctx: CreateBinaryFetcherOptions): { binary: BinaryFetcher } {
|
||||
// Snapshot and pre-compile `archiveFilters` at creation time so later mutations to the
|
||||
// caller's object can't reintroduce invalid patterns, and so zip extraction doesn't
|
||||
// recompile the regex per fetch. The tarball path still needs the pattern string — it
|
||||
// crosses the worker thread boundary, where RegExp instances don't survive structured clone.
|
||||
const archiveFilters = new Map<string, { pattern: string, regex: RegExp }>()
|
||||
for (const [name, pattern] of Object.entries(ctx.archiveFilters ?? {})) {
|
||||
try {
|
||||
archiveFilters.set(name, { pattern, regex: new RegExp(pattern) })
|
||||
} catch (err: unknown) {
|
||||
const detail = util.types.isNativeError(err) ? `: ${err.message}` : ''
|
||||
throw new PnpmError(
|
||||
'INVALID_ARCHIVE_FILTER',
|
||||
`Invalid archive filter regex for "${name}"${detail}: ${pattern}`
|
||||
)
|
||||
}
|
||||
}
|
||||
const fetchBinary: BinaryFetcher = async (cafs, resolution, opts) => {
|
||||
if (ctx.offline) {
|
||||
throw new PnpmError('CANNOT_DOWNLOAD_BINARY_OFFLINE', `Cannot download binary "${resolution.url}" because offline mode is enabled.`)
|
||||
@@ -28,6 +54,7 @@ export function createBinaryFetcher (ctx: {
|
||||
version: opts.pkg.version!,
|
||||
bin: resolution.bin,
|
||||
}
|
||||
const archiveFilter = opts.pkg.name != null ? archiveFilters.get(opts.pkg.name) : undefined
|
||||
|
||||
let fetchResult!: FetchResult
|
||||
switch (resolution.archive) {
|
||||
@@ -36,8 +63,9 @@ export function createBinaryFetcher (ctx: {
|
||||
tarball: resolution.url,
|
||||
integrity: resolution.integrity,
|
||||
}, {
|
||||
appendManifest: manifest,
|
||||
...opts,
|
||||
appendManifest: manifest,
|
||||
ignoreFilePattern: archiveFilter?.pattern ?? opts.ignoreFilePattern,
|
||||
})
|
||||
break
|
||||
}
|
||||
@@ -47,6 +75,7 @@ export function createBinaryFetcher (ctx: {
|
||||
url: resolution.url,
|
||||
integrity: resolution.integrity,
|
||||
basename: resolution.prefix ?? '',
|
||||
ignoreEntry: archiveFilter?.regex,
|
||||
}, tempLocation)
|
||||
fetchResult = await addFilesFromDir({
|
||||
storeDir: cafs.storeDir,
|
||||
@@ -77,6 +106,11 @@ export interface AssetInfo {
|
||||
url: string
|
||||
integrity: string
|
||||
basename: string
|
||||
/**
|
||||
* Regex matched against each zip entry's path relative to the archive's top-level basename.
|
||||
* Matching entries are skipped during extraction.
|
||||
*/
|
||||
ignoreEntry?: RegExp
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -96,7 +130,7 @@ export async function downloadAndUnpackZip (
|
||||
|
||||
try {
|
||||
await downloadWithIntegrityCheck(fetchFromRegistry, assetInfo, tmp)
|
||||
await extractZipToTarget(tmp, assetInfo.basename, targetDir)
|
||||
await extractZipToTarget(tmp, assetInfo.basename, targetDir, assetInfo.ignoreEntry)
|
||||
} finally {
|
||||
// Clean up temporary file
|
||||
try {
|
||||
@@ -144,12 +178,15 @@ async function downloadWithIntegrityCheck (
|
||||
* @param zipPath - Path to the zip file
|
||||
* @param basename - Base name of the file (without extension)
|
||||
* @param targetDir - Directory where contents should be extracted
|
||||
* @param ignoreEntry - Optional regex matched against the entry path relative to `basename`;
|
||||
* matching entries are skipped.
|
||||
* @throws {PnpmError} When extraction fails or path traversal is detected
|
||||
*/
|
||||
async function extractZipToTarget (
|
||||
zipPath: string,
|
||||
basename: string,
|
||||
targetDir: string
|
||||
targetDir: string,
|
||||
ignoreEntry?: RegExp
|
||||
): Promise<void> {
|
||||
const zip = new AdmZip(zipPath)
|
||||
const nodeDir = basename === '' ? targetDir : path.dirname(targetDir)
|
||||
@@ -159,10 +196,22 @@ async function extractZipToTarget (
|
||||
validatePathSecurity(nodeDir, basename)
|
||||
}
|
||||
|
||||
const basenamePrefix = basename === '' ? '' : `${basename}/`
|
||||
// Normalize `ignoreEntry` to a stateless regex. `.test()` on a `/g` or `/y` regex
|
||||
// advances `lastIndex` between calls, which would cause inconsistent skips across
|
||||
// entries in this loop.
|
||||
const testEntry = toStatelessTester(ignoreEntry)
|
||||
|
||||
// Extract each entry with path validation to prevent path traversal attacks
|
||||
for (const entry of zip.getEntries()) {
|
||||
const entryPath = entry.entryName
|
||||
validatePathSecurity(nodeDir, entryPath)
|
||||
if (testEntry) {
|
||||
const relative = basenamePrefix && entryPath.startsWith(basenamePrefix)
|
||||
? entryPath.slice(basenamePrefix.length)
|
||||
: entryPath
|
||||
if (testEntry(relative)) continue
|
||||
}
|
||||
zip.extractEntryTo(entry, nodeDir, true, true)
|
||||
}
|
||||
|
||||
@@ -170,6 +219,18 @@ async function extractZipToTarget (
|
||||
await renameOverwrite(extractedDir, targetDir)
|
||||
}
|
||||
|
||||
function toStatelessTester (regex: RegExp | undefined): ((input: string) => boolean) | undefined {
|
||||
if (!regex) return undefined
|
||||
// `/g` and `/y` make `RegExp.prototype.test` stateful via `lastIndex`.
|
||||
// Strip those flags by cloning into a fresh RegExp with only the safe flags.
|
||||
if (!regex.global && !regex.sticky) {
|
||||
return (input) => regex.test(input)
|
||||
}
|
||||
const safeFlags = regex.flags.replace(/[gy]/g, '')
|
||||
const clone = new RegExp(regex.source, safeFlags)
|
||||
return (input) => clone.test(input)
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates that a path does not escape the base directory via path traversal.
|
||||
*
|
||||
|
||||
@@ -3,7 +3,7 @@ import fs from 'node:fs'
|
||||
import path from 'node:path'
|
||||
|
||||
import { PnpmError } from '@pnpm/error'
|
||||
import { downloadAndUnpackZip } from '@pnpm/fetching.binary-fetcher'
|
||||
import { createBinaryFetcher, downloadAndUnpackZip } from '@pnpm/fetching.binary-fetcher'
|
||||
import AdmZip from 'adm-zip'
|
||||
import ssri from 'ssri'
|
||||
import { temporaryDirectory } from 'tempy'
|
||||
@@ -220,5 +220,149 @@ describe('extractZipToTarget security', () => {
|
||||
|
||||
expect(fs.existsSync(path.join(targetDir, 'bin/node'))).toBe(true)
|
||||
})
|
||||
|
||||
it('skips entries matching ignoreEntry regex (basename stripped)', async () => {
|
||||
const targetDir = temporaryDirectory()
|
||||
const zip = new AdmZip()
|
||||
zip.addFile('node-v20.0.0/node.exe', Buffer.from('binary'))
|
||||
zip.addFile('node-v20.0.0/npm', Buffer.from('npm shim'))
|
||||
zip.addFile('node-v20.0.0/npm.cmd', Buffer.from('npm cmd'))
|
||||
zip.addFile('node-v20.0.0/node_modules/npm/package.json', Buffer.from('{}'))
|
||||
zip.addFile('node-v20.0.0/node_modules/corepack/package.json', Buffer.from('{}'))
|
||||
zip.addFile('node-v20.0.0/node_modules/keep-me/index.js', Buffer.from('kept'))
|
||||
const zipBuffer = zip.toBuffer()
|
||||
const integrity = ssri.fromData(zipBuffer).toString()
|
||||
const mockFetch = createMockFetch(zipBuffer)
|
||||
|
||||
await downloadAndUnpackZip(
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
mockFetch as any,
|
||||
{
|
||||
url: 'https://example.com/node.zip',
|
||||
integrity,
|
||||
basename: 'node-v20.0.0',
|
||||
ignoreEntry: /^(?:node_modules\/(?:npm|corepack)(?:\/|$)|npm(?:\.cmd)?$)/,
|
||||
},
|
||||
targetDir
|
||||
)
|
||||
|
||||
expect(fs.existsSync(path.join(targetDir, 'node.exe'))).toBe(true)
|
||||
expect(fs.existsSync(path.join(targetDir, 'node_modules/keep-me/index.js'))).toBe(true)
|
||||
expect(fs.existsSync(path.join(targetDir, 'npm'))).toBe(false)
|
||||
expect(fs.existsSync(path.join(targetDir, 'npm.cmd'))).toBe(false)
|
||||
expect(fs.existsSync(path.join(targetDir, 'node_modules/npm'))).toBe(false)
|
||||
expect(fs.existsSync(path.join(targetDir, 'node_modules/corepack'))).toBe(false)
|
||||
})
|
||||
|
||||
it('skips entries matching ignoreEntry regex when basename is empty', async () => {
|
||||
const targetDir = temporaryDirectory()
|
||||
const zip = new AdmZip()
|
||||
zip.addFile('bin/node', Buffer.from('#!/bin/sh\necho "node"'))
|
||||
zip.addFile('bin/npm', Buffer.from('npm shim'))
|
||||
const zipBuffer = zip.toBuffer()
|
||||
const integrity = ssri.fromData(zipBuffer).toString()
|
||||
const mockFetch = createMockFetch(zipBuffer)
|
||||
|
||||
await downloadAndUnpackZip(
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
mockFetch as any,
|
||||
{
|
||||
url: 'https://example.com/node.zip',
|
||||
integrity,
|
||||
basename: '',
|
||||
ignoreEntry: /^bin\/npm$/,
|
||||
},
|
||||
targetDir
|
||||
)
|
||||
|
||||
expect(fs.existsSync(path.join(targetDir, 'bin/node'))).toBe(true)
|
||||
expect(fs.existsSync(path.join(targetDir, 'bin/npm'))).toBe(false)
|
||||
})
|
||||
|
||||
it('strips /g /y flags from ignoreEntry so .test() is not stateful across entries', async () => {
|
||||
const targetDir = temporaryDirectory()
|
||||
const zip = new AdmZip()
|
||||
zip.addFile('node-v20.0.0/node.exe', Buffer.from('binary'))
|
||||
zip.addFile('node-v20.0.0/npm', Buffer.from('npm shim 1'))
|
||||
zip.addFile('node-v20.0.0/npx', Buffer.from('npx shim 2'))
|
||||
zip.addFile('node-v20.0.0/corepack', Buffer.from('corepack 3'))
|
||||
const zipBuffer = zip.toBuffer()
|
||||
const integrity = ssri.fromData(zipBuffer).toString()
|
||||
const mockFetch = createMockFetch(zipBuffer)
|
||||
|
||||
await downloadAndUnpackZip(
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
mockFetch as any,
|
||||
{
|
||||
url: 'https://example.com/node.zip',
|
||||
integrity,
|
||||
basename: 'node-v20.0.0',
|
||||
// Deliberately pass a /g regex — a stateful .test() would skip only
|
||||
// every other matching entry. All three shims must still be dropped.
|
||||
ignoreEntry: /^(?:npm|npx|corepack)$/g,
|
||||
},
|
||||
targetDir
|
||||
)
|
||||
|
||||
expect(fs.existsSync(path.join(targetDir, 'node.exe'))).toBe(true)
|
||||
expect(fs.existsSync(path.join(targetDir, 'npm'))).toBe(false)
|
||||
expect(fs.existsSync(path.join(targetDir, 'npx'))).toBe(false)
|
||||
expect(fs.existsSync(path.join(targetDir, 'corepack'))).toBe(false)
|
||||
})
|
||||
|
||||
})
|
||||
})
|
||||
|
||||
describe('createBinaryFetcher', () => {
|
||||
it('rejects an invalid archiveFilters regex at creation time', () => {
|
||||
const noop = (() => {
|
||||
throw new Error('should not be called')
|
||||
}) as never
|
||||
expect(() =>
|
||||
createBinaryFetcher({
|
||||
fetch: noop,
|
||||
fetchFromRemoteTarball: noop,
|
||||
storeIndex: noop,
|
||||
archiveFilters: { node: '(' },
|
||||
})
|
||||
).toThrow(PnpmError)
|
||||
expect(() =>
|
||||
createBinaryFetcher({
|
||||
fetch: noop,
|
||||
fetchFromRemoteTarball: noop,
|
||||
storeIndex: noop,
|
||||
archiveFilters: { node: '(' },
|
||||
})
|
||||
).toThrow(/Invalid archive filter regex for "node"/)
|
||||
})
|
||||
|
||||
it('snapshots archiveFilters so post-creation mutations cannot reintroduce invalid patterns', () => {
|
||||
const noop = (() => {
|
||||
throw new Error('should not be called')
|
||||
}) as never
|
||||
const filters: Record<string, string> = { node: '^ok$' }
|
||||
// Must succeed — the pattern is valid at construction time.
|
||||
expect(() =>
|
||||
createBinaryFetcher({
|
||||
fetch: noop,
|
||||
fetchFromRemoteTarball: noop,
|
||||
storeIndex: noop,
|
||||
archiveFilters: filters,
|
||||
})
|
||||
).not.toThrow()
|
||||
// Mutating the caller's object after construction must not affect the fetcher.
|
||||
// There's no direct read back, but any mutation reaching the fetcher would throw
|
||||
// on subsequent fetches; the snapshot guarantees it can't.
|
||||
filters.node = '('
|
||||
// Reconstructing with the broken pattern fails — demonstrating the original
|
||||
// fetcher would have failed at construction if it had seen the broken pattern.
|
||||
expect(() =>
|
||||
createBinaryFetcher({
|
||||
fetch: noop,
|
||||
fetchFromRemoteTarball: noop,
|
||||
storeIndex: noop,
|
||||
archiveFilters: filters,
|
||||
})
|
||||
).toThrow(/Invalid archive filter regex for "node"/)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -21,6 +21,11 @@ export interface FetchOptions {
|
||||
readManifest?: boolean
|
||||
pkg: PkgNameVersion
|
||||
appendManifest?: DependencyManifest
|
||||
/**
|
||||
* Regex source (compatible with `new RegExp(pattern)`) matching file paths inside the
|
||||
* downloaded archive that should be skipped during extraction. Honored by tarball fetchers.
|
||||
*/
|
||||
ignoreFilePattern?: string
|
||||
}
|
||||
|
||||
export type FetchFunction<FetcherResolution = Resolution, Options = FetchOptions, Result = FetchResult> = (
|
||||
|
||||
@@ -99,5 +99,6 @@ async function fetchFromTarball (
|
||||
filesIndexFile: opts.filesIndexFile,
|
||||
pkg: opts.pkg,
|
||||
appendManifest: opts.appendManifest,
|
||||
ignoreFilePattern: opts.ignoreFilePattern,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -28,6 +28,7 @@ export function createLocalTarballFetcher (storeIndex: StoreIndex): FetchFunctio
|
||||
url: tarball,
|
||||
pkg: opts.pkg,
|
||||
appendManifest: opts.appendManifest,
|
||||
ignoreFilePattern: opts.ignoreFilePattern,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -28,7 +28,7 @@ export type DownloadOptions = {
|
||||
onProgress?: (downloaded: number) => void
|
||||
integrity?: string
|
||||
storeIndex: StoreIndex
|
||||
} & Pick<FetchOptions, 'pkg' | 'appendManifest' | 'readManifest' | 'filesIndexFile'>
|
||||
} & Pick<FetchOptions, 'pkg' | 'appendManifest' | 'readManifest' | 'filesIndexFile' | 'ignoreFilePattern'>
|
||||
|
||||
export type DownloadFunction = (url: string, opts: DownloadOptions) => Promise<FetchResult>
|
||||
|
||||
@@ -208,6 +208,7 @@ export function createDownloader (
|
||||
url,
|
||||
pkg: opts.pkg,
|
||||
appendManifest: opts.appendManifest,
|
||||
ignoreFilePattern: opts.ignoreFilePattern,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -33,6 +33,7 @@
|
||||
".test": "cross-env NODE_OPTIONS=\"$NODE_OPTIONS --experimental-vm-modules --disable-warning=ExperimentalWarning --disable-warning=DEP0169\" jest"
|
||||
},
|
||||
"dependencies": {
|
||||
"@pnpm/engine.runtime.node-resolver": "workspace:*",
|
||||
"@pnpm/fetching.binary-fetcher": "workspace:*",
|
||||
"@pnpm/fetching.directory-fetcher": "workspace:*",
|
||||
"@pnpm/fetching.git-fetcher": "workspace:*",
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { NODE_EXTRAS_IGNORE_PATTERN } from '@pnpm/engine.runtime.node-resolver'
|
||||
import { createBinaryFetcher } from '@pnpm/fetching.binary-fetcher'
|
||||
import { createDirectoryFetcher } from '@pnpm/fetching.directory-fetcher'
|
||||
import type { BinaryFetcher, DirectoryFetcher, GitFetcher } from '@pnpm/fetching.fetcher-base'
|
||||
@@ -81,6 +82,7 @@ function createFetchers (
|
||||
fetchFromRemoteTarball: tarballFetchers.remoteTarball,
|
||||
offline: opts.offline,
|
||||
storeIndex: opts.storeIndex,
|
||||
archiveFilters: { node: NODE_EXTRAS_IGNORE_PATTERN },
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,6 +12,9 @@
|
||||
{
|
||||
"path": "../../core/types"
|
||||
},
|
||||
{
|
||||
"path": "../../engine/runtime/node-resolver"
|
||||
},
|
||||
{
|
||||
"path": "../../fetching/binary-fetcher"
|
||||
},
|
||||
|
||||
@@ -27,8 +27,6 @@ const GLIBC_RESOLUTIONS = [
|
||||
integrity: 'sha256-13Q/3fXoZxJPVVqR9scpEE/Vx12TgvEChsP7s/0S7wc=',
|
||||
bin: {
|
||||
node: 'bin/node',
|
||||
npm: 'lib/node_modules/npm/bin/npm-cli.js',
|
||||
npx: 'lib/node_modules/npm/bin/npx-cli.js',
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -46,8 +44,6 @@ const GLIBC_RESOLUTIONS = [
|
||||
integrity: 'sha256-6pbTSc+qZ6qHzuqj5bUskWf3rDAv2NH/Fi0HhencB4U=',
|
||||
bin: {
|
||||
node: 'bin/node',
|
||||
npm: 'lib/node_modules/npm/bin/npm-cli.js',
|
||||
npx: 'lib/node_modules/npm/bin/npx-cli.js',
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -65,8 +61,6 @@ const GLIBC_RESOLUTIONS = [
|
||||
integrity: 'sha256-Qio4h/9UGPCkVS2Jz5k0arirUbtdOEZguqiLhETSwRE=',
|
||||
bin: {
|
||||
node: 'bin/node',
|
||||
npm: 'lib/node_modules/npm/bin/npm-cli.js',
|
||||
npx: 'lib/node_modules/npm/bin/npx-cli.js',
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -84,8 +78,6 @@ const GLIBC_RESOLUTIONS = [
|
||||
integrity: 'sha256-HTVHImvn5ZrO7lx9Aan4/BjeZ+AVxaFdjPOFtuAtBis=',
|
||||
bin: {
|
||||
node: 'bin/node',
|
||||
npm: 'lib/node_modules/npm/bin/npm-cli.js',
|
||||
npx: 'lib/node_modules/npm/bin/npx-cli.js',
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -103,8 +95,6 @@ const GLIBC_RESOLUTIONS = [
|
||||
integrity: 'sha256-0h239Xxc4YKuwrmoPjKVq8N+FzGrtzmV09Vz4EQJl3w=',
|
||||
bin: {
|
||||
node: 'bin/node',
|
||||
npm: 'lib/node_modules/npm/bin/npm-cli.js',
|
||||
npx: 'lib/node_modules/npm/bin/npx-cli.js',
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -122,8 +112,6 @@ const GLIBC_RESOLUTIONS = [
|
||||
integrity: 'sha256-OwmNzPVtRGu7gIRdNbvsvbdGEoYNFpDzohY4fJnJ1iA=',
|
||||
bin: {
|
||||
node: 'bin/node',
|
||||
npm: 'lib/node_modules/npm/bin/npm-cli.js',
|
||||
npx: 'lib/node_modules/npm/bin/npx-cli.js',
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -141,8 +129,6 @@ const GLIBC_RESOLUTIONS = [
|
||||
integrity: 'sha256-fsX9rQyBnuoXkA60PB3pSNYgp4OxrJQGLKpDh3ipKzA=',
|
||||
bin: {
|
||||
node: 'bin/node',
|
||||
npm: 'lib/node_modules/npm/bin/npm-cli.js',
|
||||
npx: 'lib/node_modules/npm/bin/npx-cli.js',
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -160,8 +146,6 @@ const GLIBC_RESOLUTIONS = [
|
||||
integrity: 'sha256-dLsPOoAwfFKUIcPthFF7j1Q4Z3CfQeU81z35nmRCr00=',
|
||||
bin: {
|
||||
node: 'bin/node',
|
||||
npm: 'lib/node_modules/npm/bin/npm-cli.js',
|
||||
npx: 'lib/node_modules/npm/bin/npx-cli.js',
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -179,8 +163,6 @@ const GLIBC_RESOLUTIONS = [
|
||||
integrity: 'sha256-N2Ehz0a9PAJcXmetrhkK/14l0zoLWPvA2GUtczULOPA=',
|
||||
bin: {
|
||||
node: 'node.exe',
|
||||
npm: 'node_modules/npm/bin/npm-cli.js',
|
||||
npx: 'node_modules/npm/bin/npx-cli.js',
|
||||
},
|
||||
prefix: 'node-v22.0.0-win-arm64',
|
||||
},
|
||||
@@ -199,8 +181,6 @@ const GLIBC_RESOLUTIONS = [
|
||||
integrity: 'sha256-MtY5tH1MCmUf+PjX1BpFQWij1ARb43mF+agQz4zvYXQ=',
|
||||
bin: {
|
||||
node: 'node.exe',
|
||||
npm: 'node_modules/npm/bin/npm-cli.js',
|
||||
npx: 'node_modules/npm/bin/npx-cli.js',
|
||||
},
|
||||
prefix: 'node-v22.0.0-win-x64',
|
||||
},
|
||||
@@ -219,8 +199,6 @@ const GLIBC_RESOLUTIONS = [
|
||||
integrity: 'sha256-4BNPUBcVSjN2csf7zRVOKyx3S0MQkRhWAZINY9DEt9A=',
|
||||
bin: {
|
||||
node: 'node.exe',
|
||||
npm: 'node_modules/npm/bin/npm-cli.js',
|
||||
npx: 'node_modules/npm/bin/npx-cli.js',
|
||||
},
|
||||
prefix: 'node-v22.0.0-win-x86',
|
||||
},
|
||||
@@ -234,6 +212,25 @@ test('installing Node.js runtime', async () => {
|
||||
project.isExecutable('.bin/node')
|
||||
expect(fs.readlinkSync('node_modules/node')).toContain(path.join('links', '@', 'node', '22.0.0'))
|
||||
|
||||
// npm, npx, and corepack are stripped from the Node.js archive; no shims or bundled sources should exist.
|
||||
const nodeInstallDir = fs.realpathSync(path.resolve('node_modules/node'))
|
||||
const isWindows = process.platform === 'win32'
|
||||
const bundledNpmDir = isWindows
|
||||
? path.join(nodeInstallDir, 'node_modules', 'npm')
|
||||
: path.join(nodeInstallDir, 'lib', 'node_modules', 'npm')
|
||||
const bundledCorepackDir = isWindows
|
||||
? path.join(nodeInstallDir, 'node_modules', 'corepack')
|
||||
: path.join(nodeInstallDir, 'lib', 'node_modules', 'corepack')
|
||||
expect(fs.existsSync(bundledNpmDir)).toBe(false)
|
||||
expect(fs.existsSync(bundledCorepackDir)).toBe(false)
|
||||
for (const shim of ['npm', 'npx', 'corepack']) {
|
||||
expect(fs.existsSync(path.resolve('node_modules/.bin', shim))).toBe(false)
|
||||
expect(fs.existsSync(path.resolve('node_modules/.bin', `${shim}.cmd`))).toBe(false)
|
||||
if (isWindows) {
|
||||
expect(fs.existsSync(path.resolve('node_modules/.bin', `${shim}.ps1`))).toBe(false)
|
||||
}
|
||||
}
|
||||
|
||||
const lockfile = project.readLockfile()
|
||||
expect(lockfile).toStrictEqual({
|
||||
settings: {
|
||||
|
||||
3
pnpm-lock.yaml
generated
3
pnpm-lock.yaml
generated
@@ -4893,6 +4893,9 @@ importers:
|
||||
|
||||
installing/client:
|
||||
dependencies:
|
||||
'@pnpm/engine.runtime.node-resolver':
|
||||
specifier: workspace:*
|
||||
version: link:../../engine/runtime/node-resolver
|
||||
'@pnpm/fetching.binary-fetcher':
|
||||
specifier: workspace:*
|
||||
version: link:../../fetching/binary-fetcher
|
||||
|
||||
@@ -70,7 +70,7 @@ export interface AddToStoreResult {
|
||||
export interface Cafs {
|
||||
storeDir: string
|
||||
addFilesFromDir: (dir: string) => AddToStoreResult
|
||||
addFilesFromTarball: (buffer: Buffer) => AddToStoreResult
|
||||
addFilesFromTarball: (buffer: Buffer, readManifest?: boolean, ignore?: (filename: string) => boolean) => AddToStoreResult
|
||||
addFile: (buffer: Buffer, mode: number) => FileWriteResult
|
||||
getFilePathByModeInCafs: (digest: string, mode: number) => string
|
||||
importPackage: ImportPackageFunction
|
||||
|
||||
@@ -13,11 +13,10 @@ import { parseTarball } from './parseTarball.js'
|
||||
|
||||
export function addFilesFromTarball (
|
||||
addBufferToCafs: (buffer: Buffer, mode: number) => FileWriteResult,
|
||||
_ignore: null | ((filename: string) => boolean),
|
||||
tarballBuffer: Buffer,
|
||||
readManifest?: boolean
|
||||
readManifest?: boolean,
|
||||
ignore?: (filename: string) => boolean
|
||||
): AddToStoreResult {
|
||||
const ignore = _ignore ?? (() => false)
|
||||
// chunkSize 128KB (8x the Node.js default of 16KB) reduces the number of
|
||||
// internal buffer allocations and copies during decompression. Benchmarks
|
||||
// showed ~2.3x faster decompress at 128KB. Larger values (256KB+) showed
|
||||
@@ -36,7 +35,7 @@ export function addFilesFromTarball (
|
||||
let manifestBuffer: Buffer | undefined
|
||||
|
||||
for (const [relativePath, { mode, offset, size }] of files) {
|
||||
if (ignore(relativePath)) continue
|
||||
if (ignore?.(relativePath)) continue
|
||||
|
||||
const fileBuffer = tarContent.subarray(offset, offset + size)
|
||||
if (readManifest && relativePath === 'package.json') {
|
||||
|
||||
@@ -58,7 +58,7 @@ export interface CreateCafsOpts {
|
||||
|
||||
export interface CafsFunctions {
|
||||
addFilesFromDir: (dirname: string, opts?: { files?: string[], readManifest?: boolean, includeNodeModules?: boolean }) => AddToStoreResult
|
||||
addFilesFromTarball: (tarballBuffer: Buffer, readManifest?: boolean) => AddToStoreResult
|
||||
addFilesFromTarball: (tarballBuffer: Buffer, readManifest?: boolean, ignore?: (filename: string) => boolean) => AddToStoreResult
|
||||
addFile: (buffer: Buffer, mode: number) => FileWriteResult
|
||||
getFilePathByModeInCafs: (digest: string, mode: number) => string
|
||||
}
|
||||
@@ -68,12 +68,22 @@ export function createCafs (storeDir: string, { ignoreFile, cafsLocker }: Create
|
||||
const addBuffer = addBufferToCafs.bind(null, _writeBufferToCafs)
|
||||
return {
|
||||
addFilesFromDir: addFilesFromDir.bind(null, addBuffer),
|
||||
addFilesFromTarball: addFilesFromTarball.bind(null, addBuffer, ignoreFile ?? null),
|
||||
addFilesFromTarball: (tarballBuffer, readManifest, callIgnore) =>
|
||||
addFilesFromTarball(addBuffer, tarballBuffer, readManifest, combineIgnore(ignoreFile, callIgnore)),
|
||||
addFile: addBuffer,
|
||||
getFilePathByModeInCafs: getFilePathByModeInCafs.bind(null, storeDir),
|
||||
}
|
||||
}
|
||||
|
||||
function combineIgnore (
|
||||
a?: (filename: string) => boolean,
|
||||
b?: (filename: string) => boolean
|
||||
): ((filename: string) => boolean) | undefined {
|
||||
if (!a) return b
|
||||
if (!b) return a
|
||||
return (filename) => a(filename) || b(filename)
|
||||
}
|
||||
|
||||
type WriteBufferToCafs = (buffer: Buffer, fileDest: string, mode: number | undefined, integrity: Integrity) => { checkedAt: number, filePath: string }
|
||||
|
||||
function addBufferToCafs (
|
||||
|
||||
@@ -29,6 +29,25 @@ describe('cafs', () => {
|
||||
expect(pkgFile!.digest).toBe('f310afae50bb5b74e5c17c5eb6fe426538b9deccd88664fbb66a5717fb6d36d86d4d1f530bb63b58914f9894e81da490e2e39bb99c8e01174e258358b9349b5c')
|
||||
})
|
||||
|
||||
it('addFilesFromTarball honors a per-call ignore predicate', () => {
|
||||
const dest = temporaryDirectory()
|
||||
const cafs = createCafs(dest)
|
||||
const tarball = fs.readFileSync(f.find('node-gyp-6.1.0.tgz'))
|
||||
const baseline = cafs.addFilesFromTarball(tarball)
|
||||
const filtered = cafs.addFilesFromTarball(tarball, false, (name) => name === 'package.json')
|
||||
expect(filtered.filesIndex.has('package.json')).toBe(false)
|
||||
expect(filtered.filesIndex.size).toBe(baseline.filesIndex.size - 1)
|
||||
})
|
||||
|
||||
it('addFilesFromTarball combines cafs-level ignoreFile with per-call ignore', () => {
|
||||
const dest = temporaryDirectory()
|
||||
const cafs = createCafs(dest, { ignoreFile: (name) => name === 'package.json' })
|
||||
const tarball = fs.readFileSync(f.find('node-gyp-6.1.0.tgz'))
|
||||
const { filesIndex } = cafs.addFilesFromTarball(tarball, false, (name) => name === 'README.md')
|
||||
expect(filesIndex.has('package.json')).toBe(false)
|
||||
expect(filesIndex.has('README.md')).toBe(false)
|
||||
})
|
||||
|
||||
it('replaces an already existing file, if the integrity of it was broken', () => {
|
||||
const storeDir = temporaryDirectory()
|
||||
const srcDir = path.join(import.meta.dirname, 'fixtures/one-file')
|
||||
|
||||
@@ -154,7 +154,7 @@ If you think that this is the case, then run "pnpm store prune" and rerun the co
|
||||
}
|
||||
}
|
||||
|
||||
type AddFilesFromTarballOptions = Pick<TarballExtractMessage, 'buffer' | 'storeDir' | 'filesIndexFile' | 'integrity' | 'readManifest' | 'pkg' | 'appendManifest'> & {
|
||||
type AddFilesFromTarballOptions = Pick<TarballExtractMessage, 'buffer' | 'storeDir' | 'filesIndexFile' | 'integrity' | 'readManifest' | 'pkg' | 'appendManifest' | 'ignoreFilePattern'> & {
|
||||
storeIndex: StoreIndex
|
||||
url: string
|
||||
}
|
||||
@@ -192,7 +192,8 @@ export async function addFilesFromTarball (opts: AddFilesFromTarballOptions): Pr
|
||||
readManifest: opts.readManifest,
|
||||
pkg: opts.pkg,
|
||||
appendManifest: opts.appendManifest,
|
||||
})
|
||||
ignoreFilePattern: opts.ignoreFilePattern,
|
||||
} satisfies TarballExtractMessage)
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import crypto from 'node:crypto'
|
||||
import fs from 'node:fs'
|
||||
import path from 'node:path'
|
||||
import util from 'node:util'
|
||||
import { parentPort } from 'node:worker_threads'
|
||||
|
||||
import { pkgRequiresBuild } from '@pnpm/building.pkg-requires-build'
|
||||
@@ -185,7 +186,7 @@ async function handleMessage (
|
||||
}
|
||||
}
|
||||
|
||||
function addTarballToStore ({ buffer, storeDir, integrity, filesIndexFile, appendManifest }: TarballExtractMessage) {
|
||||
function addTarballToStore ({ buffer, storeDir, integrity, filesIndexFile, appendManifest, ignoreFilePattern }: TarballExtractMessage) {
|
||||
if (integrity) {
|
||||
const { algorithm, hexDigest } = parseIntegrity(integrity)
|
||||
const calculatedHash: string = crypto.hash(algorithm, buffer, 'hex')
|
||||
@@ -205,7 +206,8 @@ function addTarballToStore ({ buffer, storeDir, integrity, filesIndexFile, appen
|
||||
cafsCache.set(storeDir, createCafs(storeDir))
|
||||
}
|
||||
const cafs = cafsCache.get(storeDir)!
|
||||
let { filesIndex, manifest } = cafs.addFilesFromTarball(buffer, true)
|
||||
const ignore = ignoreFilePattern ? makeIgnoreFromPattern(ignoreFilePattern) : undefined
|
||||
let { filesIndex, manifest } = cafs.addFilesFromTarball(buffer, true, ignore)
|
||||
if (appendManifest && manifest == null) {
|
||||
manifest = appendManifest
|
||||
addManifestToCafs(cafs, filesIndex, appendManifest)
|
||||
@@ -238,6 +240,24 @@ function calcIntegrity (buffer: Buffer): string {
|
||||
return formatIntegrity('sha512', calculatedHash)
|
||||
}
|
||||
|
||||
function makeIgnoreFromPattern (pattern: string): (filename: string) => boolean {
|
||||
// `ignoreFilePattern` is a public field on FetchOptions, so callers that don't go
|
||||
// through the binary-fetcher's validated `archiveFilters` path could still supply a
|
||||
// bad regex. Convert the SyntaxError into a PnpmError with a stable code so it's
|
||||
// actionable for users.
|
||||
let regex: RegExp
|
||||
try {
|
||||
regex = new RegExp(pattern)
|
||||
} catch (err: unknown) {
|
||||
const detail = util.types.isNativeError(err) ? `: ${err.message}` : ''
|
||||
throw new PnpmError(
|
||||
'INVALID_IGNORE_FILE_PATTERN',
|
||||
`Invalid ignoreFilePattern regex${detail}: ${pattern}`
|
||||
)
|
||||
}
|
||||
return (filename) => regex.test(filename)
|
||||
}
|
||||
|
||||
function packToShared (data: unknown): Uint8Array {
|
||||
const packed = packForStorage(data)
|
||||
const shared = new SharedArrayBuffer(packed.byteLength)
|
||||
|
||||
@@ -20,6 +20,12 @@ export interface TarballExtractMessage {
|
||||
readManifest?: boolean
|
||||
pkg?: PkgNameVersion
|
||||
appendManifest?: DependencyManifest
|
||||
/**
|
||||
* Regex source matching the normalized relative path of files inside the tarball that
|
||||
* should be skipped. Matching happens after the tar parser strips the top-level directory
|
||||
* segment — i.e. against the same path form that is written to `filesIndex`.
|
||||
*/
|
||||
ignoreFilePattern?: string
|
||||
}
|
||||
|
||||
export interface LinkPkgMessage {
|
||||
|
||||
Reference in New Issue
Block a user