fix: linking commands of engines (#10354)

close #10244

.changeset/curly-rings-live.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/fetching.binary-fetcher": patch
+---
+
+Runtime dependencies (node, bun, deno) are now added to the store with a package.json file.

.changeset/famous-plums-fix.md

@@ -0,0 +1,6 @@
+---
+"@pnpm/cafs-types": minor
+"@pnpm/store.cafs": minor
+---
+
+Export a new function to add a new file to the CAFS.

.changeset/lazy-eggs-jam.md

@@ -0,0 +1,5 @@
+---
+"pnpm": patch
+---
+
+Binaries of runtime engines (Node.js, Deno, Bun) are written to `node_modules/.bin` before lifecycle scripts (install, postinstall, prepare) are executed [#10244](https://github.com/pnpm/pnpm/issues/10244).

.changeset/long-radios-cross.md

@@ -0,0 +1,8 @@
+---
+"@pnpm/tarball-fetcher": minor
+"@pnpm/fetcher-base": minor
+"@pnpm/worker": minor
+"@pnpm/crypto.object-hasher": minor
+---
+
+Added a way to append a manifest to a package with no package.json file.

fetching/binary-fetcher/src/index.ts

@@ -19,13 +19,23 @@ export function createBinaryFetcher (ctx: {
     if (ctx.offline) {
       throw new PnpmError('CANNOT_DOWNLOAD_BINARY_OFFLINE', `Cannot download binary "${resolution.url}" because offline mode is enabled.`)
     }
+
+    const manifest = {
+      name: opts.pkg.name!,
+      version: opts.pkg.version!,
+      bin: resolution.bin,
+    }
+
     let fetchResult!: FetchResult
     switch (resolution.archive) {
     case 'tarball': {
       fetchResult = await ctx.fetchFromRemoteTarball(cafs, {
         tarball: resolution.url,
         integrity: resolution.integrity,
-      }, opts)
+      }, {
+        appendManifest: manifest,
+        ...opts,
+      })
       break
     }
     case 'zip': {
@@ -40,6 +50,7 @@ export function createBinaryFetcher (ctx: {
         dir: tempLocation,
         filesIndexFile: opts.filesIndexFile,
         readManifest: false,
+        appendManifest: manifest,
       })
       break
     }
@@ -47,11 +58,6 @@ export function createBinaryFetcher (ctx: {
       throw new PnpmError('NOT_SUPPORTED_ARCHIVE', `The binary fetcher doesn't support archive type ${resolution.archive as string}`)
     }
     }
-    const manifest = {
-      name: opts.pkg.name!,
-      version: opts.pkg.version!,
-      bin: resolution.bin,
-    }
     return {
       ...fetchResult,
       manifest,

fetching/fetcher-base/src/index.ts

@@ -20,6 +20,7 @@ export interface FetchOptions {
   onProgress?: (downloaded: number) => void
   readManifest?: boolean
   pkg: PkgNameVersion
+  appendManifest?: DependencyManifest
 }
 
 export type FetchFunction<FetcherResolution = Resolution, Options = FetchOptions, Result = FetchResult> = (

fetching/tarball-fetcher/src/index.ts

@@ -93,5 +93,6 @@ async function fetchFromTarball (
     registry: resolution.registry,
     filesIndexFile: opts.filesIndexFile,
     pkg: opts.pkg,
+    appendManifest: opts.appendManifest,
   })
 }

fetching/tarball-fetcher/src/localTarballFetcher.ts

@@ -24,6 +24,7 @@ export function createLocalTarballFetcher (): FetchFunction {
       readManifest: opts.readManifest,
       url: tarball,
       pkg: opts.pkg,
+      appendManifest: opts.appendManifest,
     })
   }
 

fetching/tarball-fetcher/src/remoteTarballFetcher.ts

@@ -18,16 +18,16 @@ export interface HttpResponse {
   body: string
 }
 
-export type DownloadFunction = (url: string, opts: {
+export type DownloadOptions = {
   getAuthHeaderByURI: (registry: string) => string | undefined
   cafs: Cafs
-  readManifest?: boolean
   registry?: string
   onStart?: (totalSize: number | null, attempt: number) => void
   onProgress?: (downloaded: number) => void
   integrity?: string
-  filesIndexFile: string
-} & Pick<FetchOptions, 'pkg'>) => Promise<FetchResult>
+} & Pick<FetchOptions, 'pkg' | 'appendManifest' | 'readManifest' | 'filesIndexFile'>
+
+export type DownloadFunction = (url: string, opts: DownloadOptions) => Promise<FetchResult>
 
 export interface NpmRegistryClient {
   get: (url: string, getOpts: object, cb: (err: Error, data: object, raw: object, res: HttpResponse) => void) => void
@@ -60,16 +60,7 @@ export function createDownloader (
   }
   const fetchMinSpeedKiBps = gotOpts.fetchMinSpeedKiBps ?? 50 // 50 KiB/s
 
-  return async function download (url: string, opts: {
-    getAuthHeaderByURI: (registry: string) => string | undefined
-    cafs: Cafs
-    readManifest?: boolean
-    registry?: string
-    onStart?: (totalSize: number | null, attempt: number) => void
-    onProgress?: (downloaded: number) => void
-    integrity?: string
-    filesIndexFile: string
-  } & Pick<FetchOptions, 'pkg'>): Promise<FetchResult> {
+  return async function download (url: string, opts: DownloadOptions): Promise<FetchResult> {
     const authHeaderValue = opts.getAuthHeaderByURI(url)
 
     const op = retry.operation(retryOpts)
@@ -179,6 +170,7 @@ export function createDownloader (
         filesIndexFile: opts.filesIndexFile,
         url,
         pkg: opts.pkg,
+        appendManifest: opts.appendManifest,
       })
     }
   }

pkg-manager/core/test/install/nodeRuntime.ts

@@ -216,6 +216,9 @@ test('installing Node.js runtime', async () => {
     },
   })
 
+  // Verify that package.json is created
+  expect(fs.existsSync(path.resolve('node_modules/node/package.json'))).toBeTruthy()
+
   rimraf('node_modules')
   await install(manifest, testDefaults({ frozenLockfile: true }, {
     offline: true, // We want to verify that Node.js is resolved from cache.

store/cafs-types/src/index.ts

@@ -73,6 +73,7 @@ export interface Cafs {
   storeDir: string
   addFilesFromDir: (dir: string) => AddToStoreResult
   addFilesFromTarball: (buffer: Buffer) => AddToStoreResult
+  addFile: (buffer: Buffer, mode: number) => FileWriteResult
   getIndexFilePathInCafs: (integrity: string | IntegrityLike, fileType: FileType) => string
   getFilePathByModeInCafs: (integrity: string | IntegrityLike, mode: number) => string
   importPackage: ImportPackageFunction

store/cafs/src/index.ts

@@ -43,6 +43,7 @@ export interface CreateCafsOpts {
 export interface CafsFunctions {
   addFilesFromDir: (dirname: string, opts?: { files?: string[], readManifest?: boolean }) => AddToStoreResult
   addFilesFromTarball: (tarballBuffer: Buffer, readManifest?: boolean) => AddToStoreResult
+  addFile: (buffer: Buffer, mode: number) => FileWriteResult
   getIndexFilePathInCafs: (integrity: string | ssri.IntegrityLike, fileType: FileType) => string
   getFilePathByModeInCafs: (integrity: string | ssri.IntegrityLike, mode: number) => string
 }
@@ -53,6 +54,7 @@ export function createCafs (storeDir: string, { ignoreFile, cafsLocker }: Create
   return {
     addFilesFromDir: addFilesFromDir.bind(null, addBuffer),
     addFilesFromTarball: addFilesFromTarball.bind(null, addBuffer, ignoreFile ?? null),
+    addFile: addBuffer,
     getIndexFilePathInCafs: getIndexFilePathInCafs.bind(null, storeDir),
     getFilePathByModeInCafs: getFilePathByModeInCafs.bind(null, storeDir),
   }

worker/src/index.ts

@@ -73,7 +73,7 @@ interface AddFilesResult {
   integrity?: string
 }
 
-type AddFilesFromDirOptions = Pick<AddDirToStoreMessage, 'storeDir' | 'dir' | 'filesIndexFile' | 'sideEffectsCacheKey' | 'readManifest' | 'pkg' | 'files'>
+type AddFilesFromDirOptions = Pick<AddDirToStoreMessage, 'storeDir' | 'dir' | 'filesIndexFile' | 'sideEffectsCacheKey' | 'readManifest' | 'pkg' | 'files' | 'appendManifest'>
 
 export async function addFilesFromDir (opts: AddFilesFromDirOptions): Promise<AddFilesResult> {
   if (!workerPool) {
@@ -97,6 +97,7 @@ export async function addFilesFromDir (opts: AddFilesFromDirOptions): Promise<Ad
       sideEffectsCacheKey: opts.sideEffectsCacheKey,
       readManifest: opts.readManifest,
       pkg: opts.pkg,
+      appendManifest: opts.appendManifest,
       files: opts.files,
     })
   })
@@ -136,7 +137,7 @@ If you think that this is the case, then run "pnpm store prune" and rerun the co
   }
 }
 
-type AddFilesFromTarballOptions = Pick<TarballExtractMessage, 'buffer' | 'storeDir' | 'filesIndexFile' | 'integrity' | 'readManifest' | 'pkg'> & {
+type AddFilesFromTarballOptions = Pick<TarballExtractMessage, 'buffer' | 'storeDir' | 'filesIndexFile' | 'integrity' | 'readManifest' | 'pkg' | 'appendManifest'> & {
   url: string
 }
 
@@ -169,6 +170,7 @@ export async function addFilesFromTarball (opts: AddFilesFromTarballOptions): Pr
       filesIndexFile: opts.filesIndexFile,
       readManifest: opts.readManifest,
       pkg: opts.pkg,
+      appendManifest: opts.appendManifest,
     })
   })
 }

worker/src/start.ts

@@ -143,7 +143,7 @@ async function handleMessage (
   }
 }
 
-function addTarballToStore ({ buffer, storeDir, integrity, filesIndexFile }: TarballExtractMessage) {
+function addTarballToStore ({ buffer, storeDir, integrity, filesIndexFile, appendManifest }: TarballExtractMessage) {
   if (integrity) {
     const [, algo, integrityHash] = integrity.match(INTEGRITY_REGEX)!
     // Compensate for the possibility of non-uniform Base64 padding
@@ -166,7 +166,11 @@ function addTarballToStore ({ buffer, storeDir, integrity, filesIndexFile }: Tar
     cafsCache.set(storeDir, createCafs(storeDir))
   }
   const cafs = cafsCache.get(storeDir)!
-  const { filesIndex, manifest } = cafs.addFilesFromTarball(buffer, true)
+  let { filesIndex, manifest } = cafs.addFilesFromTarball(buffer, true)
+  if (appendManifest && manifest == null) {
+    manifest = appendManifest
+    addManifestToCafs(cafs, filesIndex, appendManifest)
+  }
   const { filesIntegrity, filesMap } = processFilesIndex(filesIndex)
   const requiresBuild = writeFilesIndexFile(filesIndexFile, { manifest: manifest ?? {}, files: filesIntegrity })
   return {
@@ -214,15 +218,28 @@ function initStore ({ storeDir }: InitStoreMessage): { status: string } {
   return { status: 'success' }
 }
 
-function addFilesFromDir ({ dir, storeDir, filesIndexFile, sideEffectsCacheKey, files }: AddDirToStoreMessage): AddFilesFromDirResult {
+function addFilesFromDir (
+  {
+    appendManifest,
+    dir,
+    files,
+    filesIndexFile,
+    sideEffectsCacheKey,
+    storeDir,
+  }: AddDirToStoreMessage
+): AddFilesFromDirResult {
   if (!cafsCache.has(storeDir)) {
     cafsCache.set(storeDir, createCafs(storeDir))
   }
   const cafs = cafsCache.get(storeDir)!
-  const { filesIndex, manifest } = cafs.addFilesFromDir(dir, {
+  let { filesIndex, manifest } = cafs.addFilesFromDir(dir, {
     files,
     readManifest: true,
   })
+  if (appendManifest && manifest == null) {
+    manifest = appendManifest
+    addManifestToCafs(cafs, filesIndex, appendManifest)
+  }
   const { filesIntegrity, filesMap } = processFilesIndex(filesIndex)
   let requiresBuild: boolean
   if (sideEffectsCacheKey) {
@@ -254,6 +271,16 @@ function addFilesFromDir ({ dir, storeDir, filesIndexFile, sideEffectsCacheKey,
   return { status: 'success', value: { filesIndex: filesMap, manifest, requiresBuild } }
 }
 
+function addManifestToCafs (cafs: CafsFunctions, filesIndex: FilesIndex, manifest: DependencyManifest): void {
+  const fileBuffer = Buffer.from(JSON.stringify(manifest, null, 2), 'utf8')
+  const mode = 0o644
+  filesIndex.set('package.json', {
+    mode,
+    size: fileBuffer.length,
+    ...cafs.addFile(fileBuffer, mode),
+  })
+}
+
 function calculateDiff (baseFiles: PackageFiles, sideEffectsFiles: PackageFiles): SideEffectsDiff {
   const deleted: string[] = []
   const added: PackageFiles = new Map()

worker/src/types.ts

@@ -1,4 +1,5 @@
 import { type PackageFilesResponse } from '@pnpm/cafs-types'
+import { type DependencyManifest } from '@pnpm/types'
 
 export interface PkgNameVersion {
   name?: string
@@ -18,6 +19,7 @@ export interface TarballExtractMessage {
   filesIndexFile: string
   readManifest?: boolean
   pkg?: PkgNameVersion
+  appendManifest?: DependencyManifest
 }
 
 export interface LinkPkgMessage {
@@ -50,6 +52,7 @@ export interface AddDirToStoreMessage {
   sideEffectsCacheKey?: string
   readManifest?: boolean
   pkg?: PkgNameVersion
+  appendManifest?: DependencyManifest
   files?: string[]
 }