perf: always link runtimes from the global virtual store directory (#10233)

.changeset/angry-streets-bow.md

@@ -0,0 +1,11 @@
+---
+"@pnpm/resolve-dependencies": major
+"@pnpm/dependency-path": major
+"@pnpm/calc-dep-state": major
+"@pnpm/headless": major
+"@pnpm/deps.graph-builder": major
+"@pnpm/core": major
+"pnpm": major
+---
+
+Runtime dependencies are always linked from the global virtual store [#10233](https://github.com/pnpm/pnpm/pull/10233).

deps/graph-builder/src/iteratePkgsForVirtualStore.ts

@@ -1,10 +1,13 @@
+import path from 'path'
 import {
   iterateHashedGraphNodes,
   lockfileToDepGraph,
+  calcGraphNodeHash,
   type PkgMeta,
   type DepsGraph,
   type PkgMetaIterator,
   type HashedDepPath,
+  type DepsStateCache,
 } from '@pnpm/calc-dep-state'
 import { type LockfileObject, type PackageSnapshot } from '@pnpm/lockfile.fs'
 import {
@@ -15,35 +18,51 @@ import * as dp from '@pnpm/dependency-path'
 
 interface PkgSnapshotWithLocation {
   pkgMeta: PkgMetaAndSnapshot
-  dirNameInVirtualStore: string
+  dirInVirtualStore: string
 }
 
 export function * iteratePkgsForVirtualStore (lockfile: LockfileObject, opts: {
   enableGlobalVirtualStore?: boolean
   virtualStoreDirMaxLength: number
+  virtualStoreDir: string
+  globalVirtualStoreDir: string
 }): IterableIterator<PkgSnapshotWithLocation> {
   if (opts.enableGlobalVirtualStore) {
     for (const { hash, pkgMeta } of hashDependencyPaths(lockfile)) {
       yield {
-        dirNameInVirtualStore: hash,
+        dirInVirtualStore: path.join(opts.globalVirtualStoreDir, hash),
         pkgMeta,
       }
     }
   } else if (lockfile.packages) {
+    let graphNodeHashOpts: { graph: DepsGraph<DepPath>, cache: DepsStateCache } | undefined
     for (const depPath in lockfile.packages) {
-      if (Object.hasOwn(lockfile.packages, depPath)) {
-        const pkgSnapshot = lockfile.packages[depPath as DepPath]
-        const { name, version } = nameVerFromPkgSnapshot(depPath, pkgSnapshot)
-        yield {
-          pkgMeta: {
-            depPath: depPath as DepPath,
-            pkgIdWithPatchHash: dp.getPkgIdWithPatchHash(depPath as DepPath),
-            name,
-            version,
-            pkgSnapshot,
-          },
-          dirNameInVirtualStore: dp.depPathToFilename(depPath, opts.virtualStoreDirMaxLength),
+      if (!Object.hasOwn(lockfile.packages, depPath)) {
+        continue
+      }
+      const pkgSnapshot = lockfile.packages[depPath as DepPath]
+      const { name, version } = nameVerFromPkgSnapshot(depPath, pkgSnapshot)
+      const pkgMeta = {
+        depPath: depPath as DepPath,
+        pkgIdWithPatchHash: dp.getPkgIdWithPatchHash(depPath as DepPath),
+        name,
+        version,
+        pkgSnapshot,
+      }
+      let dirInVirtualStore!: string
+      if (dp.isRuntimeDepPath(depPath as DepPath)) {
+        graphNodeHashOpts ??= {
+          cache: {},
+          graph: lockfileToDepGraph(lockfile),
         }
+        const hash = calcGraphNodeHash(graphNodeHashOpts, pkgMeta)
+        dirInVirtualStore = path.join(opts.globalVirtualStoreDir, hash)
+      } else {
+        dirInVirtualStore = path.join(opts.virtualStoreDir, dp.depPathToFilename(depPath, opts.virtualStoreDirMaxLength))
+      }
+      yield {
+        dirInVirtualStore,
+        pkgMeta,
       }
     }
   }

deps/graph-builder/src/lockfileToDepGraph.ts

@@ -70,6 +70,7 @@ export interface LockfileToDepGraphOptions {
   skipped: Set<DepPath>
   storeController: StoreController
   storeDir: string
+  globalVirtualStoreDir: string
   virtualStoreDir: string
   supportedArchitectures?: SupportedArchitectures
   virtualStoreDirMaxLength: number
@@ -166,7 +167,7 @@ async function buildGraphFromPackages (
   const promises: Array<Promise<void>> = []
   const pkgSnapshotsWithLocations = iteratePkgsForVirtualStore(lockfile, opts)
 
-  for (const { dirNameInVirtualStore, pkgMeta } of pkgSnapshotsWithLocations) {
+  for (const { dirInVirtualStore, pkgMeta } of pkgSnapshotsWithLocations) {
     promises.push((async () => {
       const { pkgIdWithPatchHash, name: pkgName, version: pkgVersion, depPath, pkgSnapshot } = pkgMeta
       if (opts.skipped.has(depPath)) return
@@ -198,7 +199,7 @@ async function buildGraphFromPackages (
 
       const depIntegrityIsUnchanged = isIntegrityEqual(pkgSnapshot.resolution, currentPackages[depPath]?.resolution)
 
-      const modules = path.join(opts.virtualStoreDir, dirNameInVirtualStore, 'node_modules')
+      const modules = path.join(dirInVirtualStore, 'node_modules')
       const dir = path.join(modules, pkgName)
       locationByDepPath[depPath] = dir
 

packages/calc-dep-state/src/index.ts

@@ -92,28 +92,38 @@ export function * iterateHashedGraphNodes<T extends PkgMeta> (
   graph: DepsGraph<DepPath>,
   pkgMetaIterator: PkgMetaIterator<T>
 ): IterableIterator<HashedDepPath<T>> {
-  const _calcDepGraphHash = calcDepGraphHash.bind(null, graph, {})
+  const _calcGraphNodeHash = calcGraphNodeHash.bind(null, { graph, cache: {} })
   for (const pkgMeta of pkgMetaIterator) {
-    const { name, version, depPath } = pkgMeta
-    const state = {
-      // Unfortunately, we need to include the engine name in the hash,
-      // even though it's only required for packages that are built,
-      // or have dependencies that are built.
-      // We can't know for sure whether a package needs to be built
-      // before it's fetched from the registry.
-      // However, we fetch and write packages to node_modules in random order for performance,
-      // so we can't determine at this stage which dependencies will be built.
-      engine: ENGINE_NAME,
-      deps: _calcDepGraphHash(new Set(), depPath),
-    }
-    const hexDigest = hashObjectWithoutSorting(state, { encoding: 'hex' })
     yield {
-      hash: `${name}/${version}/${hexDigest}`,
+      hash: _calcGraphNodeHash(pkgMeta),
       pkgMeta,
     }
   }
 }
 
+export function calcGraphNodeHash<T extends PkgMeta> (
+  { graph, cache }: {
+    graph: DepsGraph<DepPath>
+    cache: DepsStateCache
+  },
+  pkgMeta: T
+): string {
+  const { name, version, depPath } = pkgMeta
+  const state = {
+    // Unfortunately, we need to include the engine name in the hash,
+    // even though it's only required for packages that are built,
+    // or have dependencies that are built.
+    // We can't know for sure whether a package needs to be built
+    // before it's fetched from the registry.
+    // However, we fetch and write packages to node_modules in random order for performance,
+    // so we can't determine at this stage which dependencies will be built.
+    engine: ENGINE_NAME,
+    deps: calcDepGraphHash(graph, cache, new Set(), depPath),
+  }
+  const hexDigest = hashObjectWithoutSorting(state, { encoding: 'hex' })
+  return `${name}/${version}/${hexDigest}`
+}
+
 export function lockfileToDepGraph (lockfile: LockfileObject): DepsGraph<DepPath> {
   const graph: DepsGraph<DepPath> = {}
   if (lockfile.packages != null) {

packages/dependency-path/src/index.ts

@@ -211,3 +211,9 @@ export function createPeerDepGraphHash (peerIds: PeerId[], maxLength: number = 1
   }
   return `(${dirName})`
 }
+
+const RUNTIME_DEP_PATH_RE = /^(?:node|bun|deno)@runtime:/
+
+export function isRuntimeDepPath (depPath: DepPath): boolean {
+  return RUNTIME_DEP_PATH_RE.test(depPath)
+}

packages/dependency-path/test/index.ts

@@ -6,6 +6,7 @@ import {
   parse,
   refToRelative,
   tryGetPackageId,
+  isRuntimeDepPath,
 } from '@pnpm/dependency-path'
 import { type DepPath } from '@pnpm/types'
 
@@ -139,3 +140,8 @@ test('getPkgIdWithPatchHash', () => {
   // Scoped packages with both patch hash and peer dependencies
   expect(getPkgIdWithPatchHash('@foo/bar@1.0.0(patch_hash=zzzz)(@types/node@18.0.0)' as DepPath)).toBe('@foo/bar@1.0.0(patch_hash=zzzz)')
 })
+
+test('isRuntimeDepPath', () => {
+  expect(isRuntimeDepPath('node@runtime:20.1.0' as DepPath)).toBeTruthy()
+  expect(isRuntimeDepPath('node@20.1.0' as DepPath)).toBeFalsy()
+})

pkg-manager/core/src/getPeerDependencyIssues.ts

@@ -26,7 +26,7 @@ export type ListMissingPeersOptions = Partial<GetContextOptions>
 >
 & Partial<Pick<InstallOptions, 'supportedArchitectures'>>
 & Pick<GetContextOptions, 'autoInstallPeers' | 'excludeLinksFromLockfile' | 'storeDir'>
-& Required<Pick<InstallOptions, 'virtualStoreDirMaxLength' | 'peersSuffixMaxLength'>>
+& Required<Pick<InstallOptions, 'globalVirtualStoreDir' | 'virtualStoreDirMaxLength' | 'peersSuffixMaxLength'>>
 
 export async function getPeerDependencyIssues (
   projects: ProjectOptions[],
@@ -90,6 +90,7 @@ export async function getPeerDependencyIssues (
       saveWorkspaceProtocol: false, // this doesn't matter in our case. We won't write changes to package.json files
       storeController: opts.storeController,
       tag: 'latest',
+      globalVirtualStoreDir: opts.globalVirtualStoreDir,
       virtualStoreDir: ctx.virtualStoreDir,
       virtualStoreDirMaxLength: ctx.virtualStoreDirMaxLength,
       wantedLockfile: ctx.wantedLockfile,

pkg-manager/core/src/install/extendInstallOptions.ts

@@ -116,6 +116,7 @@ export interface StrictInstallOptions {
   workspacePackages?: WorkspacePackages
   pruneStore: boolean
   virtualStoreDir?: string
+  globalVirtualStoreDir: string
   dir: string
   symlink: boolean
   enableModulesDir: boolean
@@ -325,5 +326,8 @@ export function extendOptions (
   if (extendedOpts.enableGlobalVirtualStore && extendedOpts.virtualStoreDir == null) {
     extendedOpts.virtualStoreDir = path.join(extendedOpts.storeDir, 'links')
   }
+  extendedOpts.globalVirtualStoreDir = extendedOpts.enableGlobalVirtualStore
+    ? extendedOpts.virtualStoreDir!
+    : path.join(extendedOpts.storeDir, 'links')
   return extendedOpts
 }

pkg-manager/core/src/install/index.ts

@@ -1186,6 +1186,7 @@ const _installInContext: InstallFunction = async (projects, ctx, opts) => {
       saveWorkspaceProtocol: opts.saveWorkspaceProtocol,
       storeController: opts.storeController,
       tag: opts.tag,
+      globalVirtualStoreDir: opts.globalVirtualStoreDir,
       virtualStoreDir: ctx.virtualStoreDir,
       virtualStoreDirMaxLength: ctx.virtualStoreDirMaxLength,
       wantedLockfile: ctx.wantedLockfile,

pkg-manager/core/test/install/nodeRuntime.ts

@@ -1,4 +1,5 @@
 import fs from 'fs'
+import path from 'path'
 import { LOCKFILE_VERSION, WANTED_LOCKFILE } from '@pnpm/constants'
 import { prepareEmpty } from '@pnpm/prepare'
 import { addDependenciesToPackage, install } from '@pnpm/core'
@@ -183,6 +184,7 @@ test('installing Node.js runtime', async () => {
   const { updatedManifest: manifest } = await addDependenciesToPackage({}, ['node@runtime:22.0.0'], testDefaults({ fastUnpack: false }))
 
   project.isExecutable('.bin/node')
+  expect(fs.readlinkSync('node_modules/node')).toContain(path.join('links', 'node', '22.0.0'))
   expect(project.readLockfile()).toStrictEqual({
     settings: {
       autoInstallPeers: true,
@@ -219,6 +221,7 @@ test('installing Node.js runtime', async () => {
     offline: true, // We want to verify that Node.js is resolved from cache.
   }))
   project.isExecutable('.bin/node')
+  expect(fs.readlinkSync('node_modules/node')).toContain(path.join('links', 'node', '22.0.0'))
 
   await addDependenciesToPackage(manifest, ['@pnpm.e2e/dep-of-pkg-with-1-dep@100.1.0'], testDefaults({ fastUnpack: false }))
   project.has('@pnpm.e2e/dep-of-pkg-with-1-dep')

pkg-manager/headless/src/index.ts

@@ -140,6 +140,7 @@ export interface HeadlessOptions {
   lockfileDir: string
   modulesDir?: string
   enableGlobalVirtualStore?: boolean
+  globalVirtualStoreDir: string
   virtualStoreDir?: string
   virtualStoreDirMaxLength: number
   patchedDependencies?: PatchGroupRecord

pkg-manager/resolve-dependencies/src/index.ts

@@ -4,6 +4,7 @@ import {
   packageManifestLogger,
 } from '@pnpm/core-loggers'
 import { iterateHashedGraphNodes } from '@pnpm/calc-dep-state'
+import { isRuntimeDepPath } from '@pnpm/dependency-path'
 import {
   type LockfileObject,
   type ProjectSnapshot,
@@ -126,6 +127,7 @@ export async function resolveDependencies (
     lockfileOnly: opts.dryRun,
     preferredVersions: opts.preferredVersions,
     virtualStoreDir: opts.virtualStoreDir,
+    globalVirtualStoreDir: opts.globalVirtualStoreDir,
     workspacePackages: opts.workspacePackages,
     noDependencySelectors: importers.every(({ wantedDependencies }) => wantedDependencies.length === 0),
   })
@@ -335,7 +337,7 @@ export async function resolveDependencies (
 
   return {
     dependenciesByProjectId,
-    dependenciesGraph: opts.enableGlobalVirtualStore ? extendGraph(dependenciesGraph, opts.virtualStoreDir) : dependenciesGraph,
+    dependenciesGraph: extendGraph(dependenciesGraph, opts),
     outdatedDependencies,
     linkedDependenciesByProjectId,
     updatedCatalogs,
@@ -457,10 +459,16 @@ async function getTopParents (pkgAliases: string[], modulesDir: string): Promise
     .filter(Boolean) as DependencyManifest[]
 }
 
-function extendGraph (graph: DependenciesGraph, virtualStoreDir: string): DependenciesGraph {
+function extendGraph (
+  graph: DependenciesGraph,
+  opts: {
+    globalVirtualStoreDir: string
+    enableGlobalVirtualStore?: boolean
+  }
+): DependenciesGraph {
   const pkgMetaIter = (function * () {
     for (const depPath in graph) {
-      if (Object.hasOwn(graph, depPath)) {
+      if ((opts.enableGlobalVirtualStore === true || isRuntimeDepPath(depPath as DepPath)) && Object.hasOwn(graph, depPath)) {
         const { name, version, pkgIdWithPatchHash } = graph[depPath as DepPath]
         yield {
           name,
@@ -472,7 +480,7 @@ function extendGraph (graph: DependenciesGraph, virtualStoreDir: string): Depend
     }
   })()
   for (const { pkgMeta: { depPath }, hash } of iterateHashedGraphNodes(graph, pkgMetaIter)) {
-    const modules = path.join(virtualStoreDir, hash, 'node_modules')
+    const modules = path.join(opts.globalVirtualStoreDir, hash, 'node_modules')
     const node = graph[depPath]
     Object.assign(node, {
       modules,

pkg-manager/resolve-dependencies/src/resolveDependencyTree.ts

@@ -131,6 +131,7 @@ export interface ResolveDependenciesOptions {
   storeController: StoreController
   tag: string
   virtualStoreDir: string
+  globalVirtualStoreDir: string
   virtualStoreDirMaxLength: number
   wantedLockfile: LockfileObject
   workspacePackages: WorkspacePackages

pkg-manager/resolve-dependencies/src/safeIsInnerLink.ts

@@ -11,6 +11,7 @@ export async function safeIsInnerLink (
     hideAlienModules: boolean
     projectDir: string
     virtualStoreDir: string
+    globalVirtualStoreDir: string
   }
 ): Promise<true | string> {
   try {
@@ -18,7 +19,12 @@ export async function safeIsInnerLink (
 
     if (link.isInner) return true
 
-    if (isSubdir(opts.virtualStoreDir, link.target)) return true
+    if (
+      isSubdir(opts.virtualStoreDir, link.target) ||
+      opts.globalVirtualStoreDir !== opts.virtualStoreDir && isSubdir(opts.globalVirtualStoreDir, link.target)
+    ) {
+      return true
+    }
 
     return link.target as string
   } catch (err: any) { // eslint-disable-line

pkg-manager/resolve-dependencies/src/toResolveImporter.ts

@@ -25,6 +25,7 @@ export async function toResolveImporter (
     lockfileOnly: boolean
     preferredVersions?: PreferredVersions
     virtualStoreDir: string
+    globalVirtualStoreDir: string
     workspacePackages: WorkspacePackages
     updateToLatest?: boolean
     noDependencySelectors: boolean
@@ -38,6 +39,7 @@ export async function toResolveImporter (
     modulesDir: project.modulesDir,
     projectDir: project.rootDir,
     virtualStoreDir: opts.virtualStoreDir,
+    globalVirtualStoreDir: opts.globalVirtualStoreDir,
     workspacePackages: opts.workspacePackages,
   })
   const defaultUpdateDepth = (project.update === true || (project.updateMatching != null)) ? opts.defaultUpdateDepth : -1
@@ -98,6 +100,7 @@ async function partitionLinkedPackages (
     lockfileOnly: boolean
     modulesDir: string
     virtualStoreDir: string
+    globalVirtualStoreDir: string
     workspacePackages?: WorkspacePackages
   }
 ): Promise<WantedDependency[]> {
@@ -116,6 +119,7 @@ async function partitionLinkedPackages (
       hideAlienModules: !opts.lockfileOnly,
       projectDir: opts.projectDir,
       virtualStoreDir: opts.virtualStoreDir,
+      globalVirtualStoreDir: opts.globalVirtualStoreDir,
     })
     if (isInnerLink === true) {
       nonLinkedDependencies.push(dependency)