From 3bf5e218a6de10a3e56f5a42d6a4a16a0e19a26a Mon Sep 17 00:00:00 2001 From: Zoltan Kochan Date: Tue, 23 Dec 2025 12:26:59 +0100 Subject: [PATCH] fix: linking commands of engines (#10354) close #10244 --- .changeset/curly-rings-live.md | 5 +++ .changeset/famous-plums-fix.md | 6 ++++ .changeset/lazy-eggs-jam.md | 5 +++ .changeset/long-radios-cross.md | 8 +++++ fetching/binary-fetcher/src/index.ts | 18 ++++++---- fetching/fetcher-base/src/index.ts | 1 + fetching/tarball-fetcher/src/index.ts | 1 + .../src/localTarballFetcher.ts | 1 + .../src/remoteTarballFetcher.ts | 20 ++++------- pkg-manager/core/test/install/nodeRuntime.ts | 3 ++ store/cafs-types/src/index.ts | 1 + store/cafs/src/index.ts | 2 ++ worker/src/index.ts | 6 ++-- worker/src/start.ts | 35 ++++++++++++++++--- worker/src/types.ts | 3 ++ 15 files changed, 89 insertions(+), 26 deletions(-) create mode 100644 .changeset/curly-rings-live.md create mode 100644 .changeset/famous-plums-fix.md create mode 100644 .changeset/lazy-eggs-jam.md create mode 100644 .changeset/long-radios-cross.md diff --git a/.changeset/curly-rings-live.md b/.changeset/curly-rings-live.md new file mode 100644 index 0000000000..2404405c67 --- /dev/null +++ b/.changeset/curly-rings-live.md @@ -0,0 +1,5 @@ +--- +"@pnpm/fetching.binary-fetcher": patch +--- + +Runtime dependencies (node, bun, deno) are now added to the store with a package.json file. diff --git a/.changeset/famous-plums-fix.md b/.changeset/famous-plums-fix.md new file mode 100644 index 0000000000..c8f071f0ce --- /dev/null +++ b/.changeset/famous-plums-fix.md @@ -0,0 +1,6 @@ +--- +"@pnpm/cafs-types": minor +"@pnpm/store.cafs": minor +--- + +Export a new function to add a new file to the CAFS. diff --git a/.changeset/lazy-eggs-jam.md b/.changeset/lazy-eggs-jam.md new file mode 100644 index 0000000000..e5c9d343f1 --- /dev/null +++ b/.changeset/lazy-eggs-jam.md @@ -0,0 +1,5 @@ +--- +"pnpm": patch +--- + +Binaries of runtime engines (Node.js, Deno, Bun) are written to `node_modules/.bin` before lifecycle scripts (install, postinstall, prepare) are executed [#10244](https://github.com/pnpm/pnpm/issues/10244). diff --git a/.changeset/long-radios-cross.md b/.changeset/long-radios-cross.md new file mode 100644 index 0000000000..a05a5ef046 --- /dev/null +++ b/.changeset/long-radios-cross.md @@ -0,0 +1,8 @@ +--- +"@pnpm/tarball-fetcher": minor +"@pnpm/fetcher-base": minor +"@pnpm/worker": minor +"@pnpm/crypto.object-hasher": minor +--- + +Added a way to append a manifest to a package with no package.json file. diff --git a/fetching/binary-fetcher/src/index.ts b/fetching/binary-fetcher/src/index.ts index f910785184..6bf858c3f3 100644 --- a/fetching/binary-fetcher/src/index.ts +++ b/fetching/binary-fetcher/src/index.ts @@ -19,13 +19,23 @@ export function createBinaryFetcher (ctx: { if (ctx.offline) { throw new PnpmError('CANNOT_DOWNLOAD_BINARY_OFFLINE', `Cannot download binary "${resolution.url}" because offline mode is enabled.`) } + + const manifest = { + name: opts.pkg.name!, + version: opts.pkg.version!, + bin: resolution.bin, + } + let fetchResult!: FetchResult switch (resolution.archive) { case 'tarball': { fetchResult = await ctx.fetchFromRemoteTarball(cafs, { tarball: resolution.url, integrity: resolution.integrity, - }, opts) + }, { + appendManifest: manifest, + ...opts, + }) break } case 'zip': { @@ -40,6 +50,7 @@ export function createBinaryFetcher (ctx: { dir: tempLocation, filesIndexFile: opts.filesIndexFile, readManifest: false, + appendManifest: manifest, }) break } @@ -47,11 +58,6 @@ export function createBinaryFetcher (ctx: { throw new PnpmError('NOT_SUPPORTED_ARCHIVE', `The binary fetcher doesn't support archive type ${resolution.archive as string}`) } } - const manifest = { - name: opts.pkg.name!, - version: opts.pkg.version!, - bin: resolution.bin, - } return { ...fetchResult, manifest, diff --git a/fetching/fetcher-base/src/index.ts b/fetching/fetcher-base/src/index.ts index 574c8534cc..0c61aa849f 100644 --- a/fetching/fetcher-base/src/index.ts +++ b/fetching/fetcher-base/src/index.ts @@ -20,6 +20,7 @@ export interface FetchOptions { onProgress?: (downloaded: number) => void readManifest?: boolean pkg: PkgNameVersion + appendManifest?: DependencyManifest } export type FetchFunction = ( diff --git a/fetching/tarball-fetcher/src/index.ts b/fetching/tarball-fetcher/src/index.ts index 57a912630c..518d7166d9 100644 --- a/fetching/tarball-fetcher/src/index.ts +++ b/fetching/tarball-fetcher/src/index.ts @@ -93,5 +93,6 @@ async function fetchFromTarball ( registry: resolution.registry, filesIndexFile: opts.filesIndexFile, pkg: opts.pkg, + appendManifest: opts.appendManifest, }) } diff --git a/fetching/tarball-fetcher/src/localTarballFetcher.ts b/fetching/tarball-fetcher/src/localTarballFetcher.ts index 5edd773fa6..2ca5e828bf 100644 --- a/fetching/tarball-fetcher/src/localTarballFetcher.ts +++ b/fetching/tarball-fetcher/src/localTarballFetcher.ts @@ -24,6 +24,7 @@ export function createLocalTarballFetcher (): FetchFunction { readManifest: opts.readManifest, url: tarball, pkg: opts.pkg, + appendManifest: opts.appendManifest, }) } diff --git a/fetching/tarball-fetcher/src/remoteTarballFetcher.ts b/fetching/tarball-fetcher/src/remoteTarballFetcher.ts index 24e4b05b31..8630f5a979 100644 --- a/fetching/tarball-fetcher/src/remoteTarballFetcher.ts +++ b/fetching/tarball-fetcher/src/remoteTarballFetcher.ts @@ -18,16 +18,16 @@ export interface HttpResponse { body: string } -export type DownloadFunction = (url: string, opts: { +export type DownloadOptions = { getAuthHeaderByURI: (registry: string) => string | undefined cafs: Cafs - readManifest?: boolean registry?: string onStart?: (totalSize: number | null, attempt: number) => void onProgress?: (downloaded: number) => void integrity?: string - filesIndexFile: string -} & Pick) => Promise +} & Pick + +export type DownloadFunction = (url: string, opts: DownloadOptions) => Promise export interface NpmRegistryClient { get: (url: string, getOpts: object, cb: (err: Error, data: object, raw: object, res: HttpResponse) => void) => void @@ -60,16 +60,7 @@ export function createDownloader ( } const fetchMinSpeedKiBps = gotOpts.fetchMinSpeedKiBps ?? 50 // 50 KiB/s - return async function download (url: string, opts: { - getAuthHeaderByURI: (registry: string) => string | undefined - cafs: Cafs - readManifest?: boolean - registry?: string - onStart?: (totalSize: number | null, attempt: number) => void - onProgress?: (downloaded: number) => void - integrity?: string - filesIndexFile: string - } & Pick): Promise { + return async function download (url: string, opts: DownloadOptions): Promise { const authHeaderValue = opts.getAuthHeaderByURI(url) const op = retry.operation(retryOpts) @@ -179,6 +170,7 @@ export function createDownloader ( filesIndexFile: opts.filesIndexFile, url, pkg: opts.pkg, + appendManifest: opts.appendManifest, }) } } diff --git a/pkg-manager/core/test/install/nodeRuntime.ts b/pkg-manager/core/test/install/nodeRuntime.ts index c05b80742e..b455caa480 100644 --- a/pkg-manager/core/test/install/nodeRuntime.ts +++ b/pkg-manager/core/test/install/nodeRuntime.ts @@ -216,6 +216,9 @@ test('installing Node.js runtime', async () => { }, }) + // Verify that package.json is created + expect(fs.existsSync(path.resolve('node_modules/node/package.json'))).toBeTruthy() + rimraf('node_modules') await install(manifest, testDefaults({ frozenLockfile: true }, { offline: true, // We want to verify that Node.js is resolved from cache. diff --git a/store/cafs-types/src/index.ts b/store/cafs-types/src/index.ts index fda6cd49a4..c515047899 100644 --- a/store/cafs-types/src/index.ts +++ b/store/cafs-types/src/index.ts @@ -73,6 +73,7 @@ export interface Cafs { storeDir: string addFilesFromDir: (dir: string) => AddToStoreResult addFilesFromTarball: (buffer: Buffer) => AddToStoreResult + addFile: (buffer: Buffer, mode: number) => FileWriteResult getIndexFilePathInCafs: (integrity: string | IntegrityLike, fileType: FileType) => string getFilePathByModeInCafs: (integrity: string | IntegrityLike, mode: number) => string importPackage: ImportPackageFunction diff --git a/store/cafs/src/index.ts b/store/cafs/src/index.ts index b2d483bc72..008bc0ef27 100644 --- a/store/cafs/src/index.ts +++ b/store/cafs/src/index.ts @@ -43,6 +43,7 @@ export interface CreateCafsOpts { export interface CafsFunctions { addFilesFromDir: (dirname: string, opts?: { files?: string[], readManifest?: boolean }) => AddToStoreResult addFilesFromTarball: (tarballBuffer: Buffer, readManifest?: boolean) => AddToStoreResult + addFile: (buffer: Buffer, mode: number) => FileWriteResult getIndexFilePathInCafs: (integrity: string | ssri.IntegrityLike, fileType: FileType) => string getFilePathByModeInCafs: (integrity: string | ssri.IntegrityLike, mode: number) => string } @@ -53,6 +54,7 @@ export function createCafs (storeDir: string, { ignoreFile, cafsLocker }: Create return { addFilesFromDir: addFilesFromDir.bind(null, addBuffer), addFilesFromTarball: addFilesFromTarball.bind(null, addBuffer, ignoreFile ?? null), + addFile: addBuffer, getIndexFilePathInCafs: getIndexFilePathInCafs.bind(null, storeDir), getFilePathByModeInCafs: getFilePathByModeInCafs.bind(null, storeDir), } diff --git a/worker/src/index.ts b/worker/src/index.ts index 1838308cca..ff346a944c 100644 --- a/worker/src/index.ts +++ b/worker/src/index.ts @@ -73,7 +73,7 @@ interface AddFilesResult { integrity?: string } -type AddFilesFromDirOptions = Pick +type AddFilesFromDirOptions = Pick export async function addFilesFromDir (opts: AddFilesFromDirOptions): Promise { if (!workerPool) { @@ -97,6 +97,7 @@ export async function addFilesFromDir (opts: AddFilesFromDirOptions): Promise & { +type AddFilesFromTarballOptions = Pick & { url: string } @@ -169,6 +170,7 @@ export async function addFilesFromTarball (opts: AddFilesFromTarballOptions): Pr filesIndexFile: opts.filesIndexFile, readManifest: opts.readManifest, pkg: opts.pkg, + appendManifest: opts.appendManifest, }) }) } diff --git a/worker/src/start.ts b/worker/src/start.ts index c2b3885938..14fc168743 100644 --- a/worker/src/start.ts +++ b/worker/src/start.ts @@ -143,7 +143,7 @@ async function handleMessage ( } } -function addTarballToStore ({ buffer, storeDir, integrity, filesIndexFile }: TarballExtractMessage) { +function addTarballToStore ({ buffer, storeDir, integrity, filesIndexFile, appendManifest }: TarballExtractMessage) { if (integrity) { const [, algo, integrityHash] = integrity.match(INTEGRITY_REGEX)! // Compensate for the possibility of non-uniform Base64 padding @@ -166,7 +166,11 @@ function addTarballToStore ({ buffer, storeDir, integrity, filesIndexFile }: Tar cafsCache.set(storeDir, createCafs(storeDir)) } const cafs = cafsCache.get(storeDir)! - const { filesIndex, manifest } = cafs.addFilesFromTarball(buffer, true) + let { filesIndex, manifest } = cafs.addFilesFromTarball(buffer, true) + if (appendManifest && manifest == null) { + manifest = appendManifest + addManifestToCafs(cafs, filesIndex, appendManifest) + } const { filesIntegrity, filesMap } = processFilesIndex(filesIndex) const requiresBuild = writeFilesIndexFile(filesIndexFile, { manifest: manifest ?? {}, files: filesIntegrity }) return { @@ -214,15 +218,28 @@ function initStore ({ storeDir }: InitStoreMessage): { status: string } { return { status: 'success' } } -function addFilesFromDir ({ dir, storeDir, filesIndexFile, sideEffectsCacheKey, files }: AddDirToStoreMessage): AddFilesFromDirResult { +function addFilesFromDir ( + { + appendManifest, + dir, + files, + filesIndexFile, + sideEffectsCacheKey, + storeDir, + }: AddDirToStoreMessage +): AddFilesFromDirResult { if (!cafsCache.has(storeDir)) { cafsCache.set(storeDir, createCafs(storeDir)) } const cafs = cafsCache.get(storeDir)! - const { filesIndex, manifest } = cafs.addFilesFromDir(dir, { + let { filesIndex, manifest } = cafs.addFilesFromDir(dir, { files, readManifest: true, }) + if (appendManifest && manifest == null) { + manifest = appendManifest + addManifestToCafs(cafs, filesIndex, appendManifest) + } const { filesIntegrity, filesMap } = processFilesIndex(filesIndex) let requiresBuild: boolean if (sideEffectsCacheKey) { @@ -254,6 +271,16 @@ function addFilesFromDir ({ dir, storeDir, filesIndexFile, sideEffectsCacheKey, return { status: 'success', value: { filesIndex: filesMap, manifest, requiresBuild } } } +function addManifestToCafs (cafs: CafsFunctions, filesIndex: FilesIndex, manifest: DependencyManifest): void { + const fileBuffer = Buffer.from(JSON.stringify(manifest, null, 2), 'utf8') + const mode = 0o644 + filesIndex.set('package.json', { + mode, + size: fileBuffer.length, + ...cafs.addFile(fileBuffer, mode), + }) +} + function calculateDiff (baseFiles: PackageFiles, sideEffectsFiles: PackageFiles): SideEffectsDiff { const deleted: string[] = [] const added: PackageFiles = new Map() diff --git a/worker/src/types.ts b/worker/src/types.ts index 43a0a87513..4df618bbc6 100644 --- a/worker/src/types.ts +++ b/worker/src/types.ts @@ -1,4 +1,5 @@ import { type PackageFilesResponse } from '@pnpm/cafs-types' +import { type DependencyManifest } from '@pnpm/types' export interface PkgNameVersion { name?: string @@ -18,6 +19,7 @@ export interface TarballExtractMessage { filesIndexFile: string readManifest?: boolean pkg?: PkgNameVersion + appendManifest?: DependencyManifest } export interface LinkPkgMessage { @@ -50,6 +52,7 @@ export interface AddDirToStoreMessage { sideEffectsCacheKey?: string readManifest?: boolean pkg?: PkgNameVersion + appendManifest?: DependencyManifest files?: string[] }