feat: global virtual store (#8190)

close #1001

.changeset/loud-drinks-think.md

@@ -0,0 +1,8 @@
+---
+"@pnpm/plugin-commands-patching": patch
+"@pnpm/reviewing.dependencies-hierarchy": patch
+"@pnpm/outdated": patch
+"@pnpm/deps.status": patch
+---
+
+Read the current lockfile from `node_modules/.pnpm/lock.yaml`, when the project uses a global virtual store.

.changeset/solid-regions-call.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/config": major
+---
+
+Don't return a default value for virtualStoreDir.

.changeset/stale-bars-smoke.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/calc-dep-state": major
+---
+
+Renamed `isBuilt` option to `includeSubdepsHash`.

.changeset/two-pots-hope.md

@@ -0,0 +1,17 @@
+---
+"@pnpm/resolve-dependencies": minor
+"@pnpm/headless": minor
+"@pnpm/deps.graph-builder": minor
+"@pnpm/core": minor
+"@pnpm/config": minor
+"@pnpm/get-context": minor
+"pnpm": minor
+---
+
+**Experimental**. Added support for global virtual stores. When the global virtual store is enabled, `node_modules` doesn’t contain regular files, only symlinks to a central virtual store (by default the central store is located at `<store-path>/links`; run `pnpm store path` to find `<store-path>`).
+
+To enable the global virtual store, add `enableGlobalVirtualStore: true` to your root `pnpm-workspace.yaml`.
+
+A global virtual store can make installations significantly faster when a warm cache is present. In CI, however, it will probably slow installations because there is usually no cache.
+
+Related PR: [#8190](https://github.com/pnpm/pnpm/pull/8190).

.changeset/wide-cats-obey.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/crypto.object-hasher": minor
+---
+
+`hashObjectWithoutSorting` can accept options.

.changeset/yummy-walls-try.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/calc-dep-state": minor
+---
+
+Added `iterateHashedGraphNodes`.

config/config/src/Config.ts

@@ -130,6 +130,7 @@ export interface Config extends OptionsFromRootManifest {
   stateDir: string
   storeDir?: string
   virtualStoreDir?: string
+  enableGlobalVirtualStore?: boolean
   verifyStoreIntegrity?: boolean
   maxSockets?: number
   networkConcurrency?: number

config/config/src/index.ts

@@ -193,7 +193,6 @@ export async function getConfig (opts: {
     userconfig: npmDefaults.userconfig,
     'verify-deps-before-run': false,
     'verify-store-integrity': true,
-    'virtual-store-dir': 'node_modules/.pnpm',
     'workspace-concurrency': getDefaultWorkspaceConcurrency(),
     'workspace-prefix': opts.workspaceDir,
     'embed-readme': false,

config/config/src/types.ts

@@ -20,6 +20,7 @@ export const types = Object.assign({
   'disallow-workspace-cycles': Boolean,
   'enable-modules-dir': Boolean,
   'enable-pre-post-scripts': Boolean,
+  'enable-global-virtual-store': Boolean,
   'exclude-links-from-lockfile': Boolean,
   'extend-node-path': Boolean,
   'fetch-timeout': Number,
@@ -113,6 +114,7 @@ export const types = Object.assign({
   'use-stderr': Boolean,
   'verify-deps-before-run': Boolean,
   'verify-store-integrity': Boolean,
+  'global-virtual-store-dir': String,
   'virtual-store-dir': String,
   'virtual-store-dir-max-length': Number,
   'peers-suffix-max-length': Number,

crypto/object-hasher/src/index.ts

@@ -33,7 +33,12 @@ function hashUnknown (object: unknown, options: hash.BaseOptions): string {
   return hash(object, options)
 }
 
-export const hashObjectWithoutSorting = (object: unknown): string => hashUnknown(object, withoutSortingOptions)
+export type HashObjectOptions = Pick<hash.NormalOption, 'encoding'>
+
+export const hashObjectWithoutSorting = (object: unknown, opts?: HashObjectOptions): string => hashUnknown(object, {
+  ...withoutSortingOptions,
+  ...opts,
+})
 export const hashObject = (object: unknown): string => hashUnknown(object, withSortingOptions)
 
 export type PrefixedHash = `sha256-${string}`

deps/graph-builder/package.json

@@ -29,6 +29,7 @@
     "compile": "tsc --build && pnpm run lint --fix"
   },
   "dependencies": {
+    "@pnpm/calc-dep-state": "workspace:*",
     "@pnpm/constants": "workspace:*",
     "@pnpm/core-loggers": "workspace:*",
     "@pnpm/dependency-path": "workspace:*",

deps/graph-builder/src/iteratePkgsForVirtualStore.ts

@@ -0,0 +1,80 @@
+import {
+  iterateHashedGraphNodes,
+  lockfileToDepGraph,
+  type PkgMeta,
+  type DepsGraph,
+  type PkgMetaIterator,
+  type HashedDepPath,
+} from '@pnpm/calc-dep-state'
+import { type LockfileObject, type PackageSnapshot } from '@pnpm/lockfile.fs'
+import {
+  nameVerFromPkgSnapshot,
+} from '@pnpm/lockfile.utils'
+import { type DepPath, type PkgIdWithPatchHash } from '@pnpm/types'
+import * as dp from '@pnpm/dependency-path'
+
+interface PkgSnapshotWithLocation {
+  pkgMeta: PkgMetaAndSnapshot
+  dirNameInVirtualStore: string
+}
+
+export function * iteratePkgsForVirtualStore (lockfile: LockfileObject, opts: {
+  enableGlobalVirtualStore?: boolean
+  virtualStoreDirMaxLength: number
+}): IterableIterator<PkgSnapshotWithLocation> {
+  if (opts.enableGlobalVirtualStore) {
+    for (const { hash, pkgMeta } of hashDependencyPaths(lockfile)) {
+      yield {
+        dirNameInVirtualStore: hash,
+        pkgMeta,
+      }
+    }
+  } else if (lockfile.packages) {
+    for (const depPath in lockfile.packages) {
+      if (Object.prototype.hasOwnProperty.call(lockfile.packages, depPath)) {
+        const pkgSnapshot = lockfile.packages[depPath as DepPath]
+        const { name, version } = nameVerFromPkgSnapshot(depPath, pkgSnapshot)
+        yield {
+          pkgMeta: {
+            depPath: depPath as DepPath,
+            pkgIdWithPatchHash: dp.getPkgIdWithPatchHash(depPath as DepPath),
+            name,
+            version,
+            pkgSnapshot,
+          },
+          dirNameInVirtualStore: dp.depPathToFilename(depPath, opts.virtualStoreDirMaxLength),
+        }
+      }
+    }
+  }
+}
+
+interface PkgMetaAndSnapshot extends PkgMeta {
+  pkgSnapshot: PackageSnapshot
+  pkgIdWithPatchHash: PkgIdWithPatchHash
+}
+
+function hashDependencyPaths (lockfile: LockfileObject): IterableIterator<HashedDepPath<PkgMetaAndSnapshot>> {
+  const graph = lockfileToDepGraph(lockfile)
+  return iterateHashedGraphNodes(graph, iteratePkgMeta(lockfile, graph))
+}
+
+function * iteratePkgMeta (lockfile: LockfileObject, graph: DepsGraph<DepPath>): PkgMetaIterator<PkgMetaAndSnapshot> {
+  if (lockfile.packages == null) {
+    return
+  }
+  for (const depPath in lockfile.packages) {
+    if (!Object.prototype.hasOwnProperty.call(lockfile.packages, depPath)) {
+      continue
+    }
+    const pkgSnapshot = lockfile.packages[depPath as DepPath]
+    const { name, version } = nameVerFromPkgSnapshot(depPath, pkgSnapshot)
+    yield {
+      name,
+      version,
+      depPath: depPath as DepPath,
+      pkgIdWithPatchHash: graph[depPath as DepPath].pkgIdWithPatchHash,
+      pkgSnapshot,
+    }
+  }
+}

deps/graph-builder/src/lockfileToDepGraph.ts

@@ -3,12 +3,8 @@ import { WANTED_LOCKFILE } from '@pnpm/constants'
 import {
   progressLogger,
 } from '@pnpm/core-loggers'
+import { type LockfileObject } from '@pnpm/lockfile.fs'
 import {
-  type LockfileObject,
-  type PackageSnapshot,
-} from '@pnpm/lockfile.fs'
-import {
-  nameVerFromPkgSnapshot,
   packageIdFromSnapshot,
   pkgSnapshotToResolution,
 } from '@pnpm/lockfile.utils'
@@ -27,6 +23,7 @@ import * as dp from '@pnpm/dependency-path'
 import pathExists from 'path-exists'
 import equals from 'ramda/src/equals'
 import isEmpty from 'ramda/src/isEmpty'
+import { iteratePkgsForVirtualStore } from './iteratePkgsForVirtualStore'
 
 const brokenModulesLogger = logger('_broken_node_modules')
 
@@ -55,6 +52,7 @@ export interface DependenciesGraph {
 
 export interface LockfileToDepGraphOptions {
   autoInstallPeers: boolean
+  enableGlobalVirtualStore?: boolean
   engineStrict: boolean
   force: boolean
   importerIds: ProjectId[]
@@ -97,163 +95,172 @@ export async function lockfileToDepGraph (
   currentLockfile: LockfileObject | null,
   opts: LockfileToDepGraphOptions
 ): Promise<LockfileToDepGraphResult> {
-  const currentPackages = currentLockfile?.packages ?? {}
-  const graph: DependenciesGraph = {}
-  const directDependenciesByImporterId: DirectDependenciesByImporterId = {}
-  if (lockfile.packages != null) {
-    const pkgSnapshotByLocation: Record<string, PackageSnapshot> = {}
-    const _getPatchInfo = getPatchInfo.bind(null, opts.patchedDependencies)
-    await Promise.all(
-      (Object.entries(lockfile.packages) as Array<[DepPath, PackageSnapshot]>).map(async ([depPath, pkgSnapshot]) => {
-        if (opts.skipped.has(depPath)) return
-        // TODO: optimize. This info can be already returned by pkgSnapshotToResolution()
-        const { name: pkgName, version: pkgVersion } = nameVerFromPkgSnapshot(depPath, pkgSnapshot)
-        const modules = path.join(opts.virtualStoreDir, dp.depPathToFilename(depPath, opts.virtualStoreDirMaxLength), 'node_modules')
-        const packageId = packageIdFromSnapshot(depPath, pkgSnapshot)
-        const pkgIdWithPatchHash = dp.getPkgIdWithPatchHash(depPath)
+  const {
+    graph,
+    locationByDepPath,
+  } = await buildGraphFromPackages(lockfile, currentLockfile, opts)
 
-        const pkg = {
-          name: pkgName,
-          version: pkgVersion,
-          engines: pkgSnapshot.engines,
-          cpu: pkgSnapshot.cpu,
-          os: pkgSnapshot.os,
-          libc: pkgSnapshot.libc,
-        }
-        if (!opts.force &&
-          packageIsInstallable(packageId, pkg, {
-            engineStrict: opts.engineStrict,
-            lockfileDir: opts.lockfileDir,
-            nodeVersion: opts.nodeVersion,
-            optional: pkgSnapshot.optional === true,
-            supportedArchitectures: opts.supportedArchitectures,
-          }) === false
-        ) {
-          opts.skipped.add(depPath)
-          return
-        }
-        const dir = path.join(modules, pkgName)
-        const depIsPresent = !('directory' in pkgSnapshot.resolution && pkgSnapshot.resolution.directory != null) &&
-          currentPackages[depPath] && equals(currentPackages[depPath].dependencies, lockfile.packages![depPath].dependencies)
-        let dirExists: boolean | undefined
-        if (
-          depIsPresent && isEmpty(currentPackages[depPath].optionalDependencies ?? {}) &&
-          isEmpty(lockfile.packages![depPath].optionalDependencies ?? {})
-        ) {
-          dirExists = await pathExists(dir)
-          if (dirExists) {
-            return
-          }
+  const _getChildrenPaths = getChildrenPaths.bind(null, {
+    force: opts.force,
+    graph,
+    lockfileDir: opts.lockfileDir,
+    registries: opts.registries,
+    sideEffectsCacheRead: opts.sideEffectsCacheRead,
+    skipped: opts.skipped,
+    storeController: opts.storeController,
+    storeDir: opts.storeDir,
+    virtualStoreDir: opts.virtualStoreDir,
+    virtualStoreDirMaxLength: opts.virtualStoreDirMaxLength,
+    locationByDepPath,
+  } satisfies GetChildrenPathsContext)
 
-          brokenModulesLogger.debug({
-            missing: dir,
-          })
-        }
-        let fetchResponse!: Partial<FetchResponse>
-        if (depIsPresent && equals(currentPackages[depPath].optionalDependencies, lockfile.packages![depPath].optionalDependencies)) {
-          if (dirExists ?? await pathExists(dir)) {
-            fetchResponse = {}
-          } else {
-            brokenModulesLogger.debug({
-              missing: dir,
-            })
-          }
-        }
-        if (!fetchResponse) {
-          const resolution = pkgSnapshotToResolution(depPath, pkgSnapshot, opts.registries)
-          progressLogger.debug({
-            packageId,
-            requester: opts.lockfileDir,
-            status: 'resolved',
-          })
-          try {
-            fetchResponse = opts.storeController.fetchPackage({
-              force: false,
-              lockfileDir: opts.lockfileDir,
-              ignoreScripts: opts.ignoreScripts,
-              pkg: {
-                id: packageId,
-                resolution,
-              },
-              expectedPkg: {
-                name: pkgName,
-                version: pkgVersion,
-              },
-            }) as any // eslint-disable-line
-            if (fetchResponse instanceof Promise) fetchResponse = await fetchResponse
-          } catch (err: unknown) {
-            if (pkgSnapshot.optional) return
-            throw err
-          }
-        }
-        graph[dir] = {
-          children: {},
-          pkgIdWithPatchHash,
-          depPath,
-          dir,
-          fetching: fetchResponse.fetching,
-          filesIndexFile: fetchResponse.filesIndexFile,
-          hasBin: pkgSnapshot.hasBin === true,
-          hasBundledDependencies: pkgSnapshot.bundledDependencies != null,
-          modules,
-          name: pkgName,
-          optional: !!pkgSnapshot.optional,
-          optionalDependencies: new Set(Object.keys(pkgSnapshot.optionalDependencies ?? {})),
-          patch: _getPatchInfo(pkgName, pkgVersion),
-        }
-        pkgSnapshotByLocation[dir] = pkgSnapshot
-      })
-    )
-    const ctx = {
-      force: opts.force,
-      graph,
-      lockfileDir: opts.lockfileDir,
-      pkgSnapshotsByDepPaths: lockfile.packages,
-      registries: opts.registries,
-      sideEffectsCacheRead: opts.sideEffectsCacheRead,
-      skipped: opts.skipped,
-      storeController: opts.storeController,
-      storeDir: opts.storeDir,
-      virtualStoreDir: opts.virtualStoreDir,
-      virtualStoreDirMaxLength: opts.virtualStoreDirMaxLength,
-    }
-    for (const [dir, node] of Object.entries(graph)) {
-      const pkgSnapshot = pkgSnapshotByLocation[dir]
-      const allDeps = {
-        ...pkgSnapshot.dependencies,
-        ...(opts.include.optionalDependencies ? pkgSnapshot.optionalDependencies : {}),
-      }
-
-      const peerDeps = pkgSnapshot.peerDependencies ? new Set(Object.keys(pkgSnapshot.peerDependencies)) : null
-      node.children = getChildrenPaths(ctx, allDeps, peerDeps, '.')
-    }
-    for (const importerId of opts.importerIds) {
-      const projectSnapshot = lockfile.importers[importerId]
-      const rootDeps = {
-        ...(opts.include.devDependencies ? projectSnapshot.devDependencies : {}),
-        ...(opts.include.dependencies ? projectSnapshot.dependencies : {}),
-        ...(opts.include.optionalDependencies ? projectSnapshot.optionalDependencies : {}),
-      }
-      directDependenciesByImporterId[importerId] = getChildrenPaths(ctx, rootDeps, null, importerId)
+  for (const node of Object.values(graph)) {
+    const pkgSnapshot = lockfile.packages![node.depPath]
+    const allDeps = {
+      ...pkgSnapshot.dependencies,
+      ...(opts.include.optionalDependencies ? pkgSnapshot.optionalDependencies : {}),
     }
+    const peerDeps = pkgSnapshot.peerDependencies ? new Set(Object.keys(pkgSnapshot.peerDependencies)) : null
+    node.children = _getChildrenPaths(allDeps, peerDeps, '.')
   }
+
+  const directDependenciesByImporterId: DirectDependenciesByImporterId = {}
+  for (const importerId of opts.importerIds) {
+    const projectSnapshot = lockfile.importers[importerId]
+    const rootDeps = {
+      ...(opts.include.devDependencies ? projectSnapshot.devDependencies : {}),
+      ...(opts.include.dependencies ? projectSnapshot.dependencies : {}),
+      ...(opts.include.optionalDependencies ? projectSnapshot.optionalDependencies : {}),
+    }
+    directDependenciesByImporterId[importerId] = _getChildrenPaths(rootDeps, null, importerId)
+  }
+
   return { graph, directDependenciesByImporterId }
 }
 
-function getChildrenPaths (
-  ctx: {
+async function buildGraphFromPackages (
+  lockfile: LockfileObject,
+  currentLockfile: LockfileObject | null,
+  opts: LockfileToDepGraphOptions
+): Promise<{
     graph: DependenciesGraph
-    force: boolean
-    registries: Registries
-    virtualStoreDir: string
-    storeDir: string
-    skipped: Set<DepPath>
-    pkgSnapshotsByDepPaths: Record<DepPath, PackageSnapshot>
-    lockfileDir: string
-    sideEffectsCacheRead: boolean
-    storeController: StoreController
-    virtualStoreDirMaxLength: number
-  },
+    locationByDepPath: Record<string, string>
+  }> {
+  const currentPackages = currentLockfile?.packages ?? {}
+  const graph: DependenciesGraph = {}
+  const locationByDepPath: Record<string, string> = {}
+
+  const _getPatchInfo = getPatchInfo.bind(null, opts.patchedDependencies)
+  const promises: Array<Promise<void>> = []
+  const pkgSnapshotsWithLocations = iteratePkgsForVirtualStore(lockfile, opts)
+
+  for (const { dirNameInVirtualStore, pkgMeta } of pkgSnapshotsWithLocations) {
+    promises.push((async () => {
+      const { pkgIdWithPatchHash, name: pkgName, version: pkgVersion, depPath, pkgSnapshot } = pkgMeta
+      if (opts.skipped.has(depPath)) return
+
+      const pkg = {
+        name: pkgName,
+        version: pkgVersion,
+        engines: pkgSnapshot.engines,
+        cpu: pkgSnapshot.cpu,
+        os: pkgSnapshot.os,
+        libc: pkgSnapshot.libc,
+      }
+
+      const packageId = packageIdFromSnapshot(depPath, pkgSnapshot)
+      if (!opts.force && packageIsInstallable(packageId, pkg, {
+        engineStrict: opts.engineStrict,
+        lockfileDir: opts.lockfileDir,
+        nodeVersion: opts.nodeVersion,
+        optional: pkgSnapshot.optional === true,
+        supportedArchitectures: opts.supportedArchitectures,
+      }) === false) {
+        opts.skipped.add(depPath)
+        return
+      }
+
+      const depIsPresent = !('directory' in pkgSnapshot.resolution && pkgSnapshot.resolution.directory != null) &&
+        currentPackages[depPath] &&
+        equals(currentPackages[depPath].dependencies, pkgSnapshot.dependencies)
+
+      const modules = path.join(opts.virtualStoreDir, dirNameInVirtualStore, 'node_modules')
+      const dir = path.join(modules, pkgName)
+      locationByDepPath[depPath] = dir
+
+      let dirExists: boolean | undefined
+      if (depIsPresent &&
+        isEmpty(currentPackages[depPath].optionalDependencies ?? {}) &&
+        isEmpty(pkgSnapshot.optionalDependencies ?? {})) {
+        dirExists = await pathExists(dir)
+        if (dirExists) return
+        brokenModulesLogger.debug({ missing: dir })
+      }
+
+      let fetchResponse!: Partial<FetchResponse>
+      if (depIsPresent && equals(currentPackages[depPath].optionalDependencies, pkgSnapshot.optionalDependencies)) {
+        if (dirExists ?? await pathExists(dir)) {
+          fetchResponse = {}
+        } else {
+          brokenModulesLogger.debug({ missing: dir })
+        }
+      }
+
+      if (!fetchResponse) {
+        const resolution = pkgSnapshotToResolution(depPath, pkgSnapshot, opts.registries)
+        progressLogger.debug({ packageId, requester: opts.lockfileDir, status: 'resolved' })
+
+        try {
+          fetchResponse = await opts.storeController.fetchPackage({
+            force: false,
+            lockfileDir: opts.lockfileDir,
+            ignoreScripts: opts.ignoreScripts,
+            pkg: { id: packageId, resolution },
+            expectedPkg: { name: pkgName, version: pkgVersion },
+          })
+        } catch (err) {
+          if (pkgSnapshot.optional) return
+          throw err
+        }
+      }
+
+      graph[dir] = {
+        children: {},
+        pkgIdWithPatchHash,
+        depPath,
+        dir,
+        fetching: fetchResponse.fetching,
+        filesIndexFile: fetchResponse.filesIndexFile,
+        hasBin: pkgSnapshot.hasBin === true,
+        hasBundledDependencies: pkgSnapshot.bundledDependencies != null,
+        modules,
+        name: pkgName,
+        optional: !!pkgSnapshot.optional,
+        optionalDependencies: new Set(Object.keys(pkgSnapshot.optionalDependencies ?? {})),
+        patch: _getPatchInfo(pkgName, pkgVersion),
+      }
+    })())
+  }
+  await Promise.all(promises)
+  return { graph, locationByDepPath }
+}
+
+interface GetChildrenPathsContext {
+  graph: DependenciesGraph
+  force: boolean
+  registries: Registries
+  virtualStoreDir: string
+  storeDir: string
+  skipped: Set<DepPath>
+  lockfileDir: string
+  sideEffectsCacheRead: boolean
+  storeController: StoreController
+  locationByDepPath: Record<string, string>
+  virtualStoreDirMaxLength: number
+}
+
+function getChildrenPaths (
+  ctx: GetChildrenPathsContext,
   allDeps: { [alias: string]: string },
   peerDeps: Set<string> | null,
   importerId: string
@@ -266,13 +273,10 @@ function getChildrenPaths (
       continue
     }
     const childRelDepPath = dp.refToRelative(ref, alias)!
-    const childPkgSnapshot = ctx.pkgSnapshotsByDepPaths[childRelDepPath]
-    if (ctx.graph[childRelDepPath]) {
+    if (ctx.locationByDepPath[childRelDepPath]) {
+      children[alias] = ctx.locationByDepPath[childRelDepPath]
+    } else if (ctx.graph[childRelDepPath]) {
       children[alias] = ctx.graph[childRelDepPath].dir
-    } else if (childPkgSnapshot) {
-      if (ctx.skipped.has(childRelDepPath)) continue
-      const pkgName = nameVerFromPkgSnapshot(childRelDepPath, childPkgSnapshot).name
-      children[alias] = path.join(ctx.virtualStoreDir, dp.depPathToFilename(childRelDepPath, ctx.virtualStoreDirMaxLength), 'node_modules', pkgName)
     } else if (ref.indexOf('file:') === 0) {
       children[alias] = path.resolve(ctx.lockfileDir, ref.slice(5))
     } else if (!ctx.skipped.has(childRelDepPath) && ((peerDeps == null) || !peerDeps.has(alias))) {

deps/graph-builder/tsconfig.json

@@ -18,6 +18,9 @@
     {
       "path": "../../lockfile/utils"
     },
+    {
+      "path": "../../packages/calc-dep-state"
+    },
     {
       "path": "../../packages/constants"
     },

deps/status/src/checkDepsStatus.ts

@@ -56,7 +56,6 @@ export type CheckDepsStatusOptions = Pick<Config,
 | 'rootProjectManifest'
 | 'rootProjectManifestDir'
 | 'sharedWorkspaceLockfile'
-| 'virtualStoreDir'
 | 'workspaceDir'
 | 'patchesDir'
 | 'pnpmfile'
@@ -259,8 +258,7 @@ async function _checkDepsStatus (opts: CheckDepsStatusOptions, workspaceState: W
 
       const wantedLockfilePromise = readWantedLockfile(workspaceDir, { ignoreIncompatible: false })
       if (wantedLockfileStats.mtime.valueOf() > workspaceState.lastValidatedTimestamp) {
-        const virtualStoreDir = opts.virtualStoreDir ?? path.join(workspaceDir, 'node_modules', '.pnpm')
-        const currentLockfile = await readCurrentLockfile(virtualStoreDir, { ignoreIncompatible: false })
+        const currentLockfile = await readCurrentLockfile(path.join(workspaceDir, 'node_modules/.pnpm'), { ignoreIncompatible: false })
         const wantedLockfile = (await wantedLockfilePromise) ?? throwLockfileNotFound(workspaceDir)
         assertLockfilesEqual(currentLockfile, wantedLockfile, workspaceDir)
       }
@@ -275,8 +273,7 @@ async function _checkDepsStatus (opts: CheckDepsStatusOptions, workspaceState: W
 
         if (!wantedLockfileStats) return throwLockfileNotFound(wantedLockfileDir)
         if (wantedLockfileStats.mtime.valueOf() > workspaceState.lastValidatedTimestamp) {
-          const virtualStoreDir = opts.virtualStoreDir ?? path.join(wantedLockfileDir, 'node_modules', '.pnpm')
-          const currentLockfile = await readCurrentLockfile(virtualStoreDir, { ignoreIncompatible: false })
+          const currentLockfile = await readCurrentLockfile(path.join(wantedLockfileDir, 'node_modules/.pnpm'), { ignoreIncompatible: false })
           const wantedLockfile = (await wantedLockfilePromise) ?? throwLockfileNotFound(wantedLockfileDir)
           assertLockfilesEqual(currentLockfile, wantedLockfile, wantedLockfileDir)
         }
@@ -358,15 +355,15 @@ async function _checkDepsStatus (opts: CheckDepsStatusOptions, workspaceState: W
   }
 
   if (rootProjectManifest && rootProjectManifestDir) {
-    const virtualStoreDir = path.join(rootProjectManifestDir, 'node_modules', '.pnpm')
-    const currentLockfilePromise = readCurrentLockfile(virtualStoreDir, { ignoreIncompatible: false })
+    const internalPnpmDir = path.join(rootProjectManifestDir, 'node_modules', '.pnpm')
+    const currentLockfilePromise = readCurrentLockfile(internalPnpmDir, { ignoreIncompatible: false })
     const wantedLockfilePromise = readWantedLockfile(rootProjectManifestDir, { ignoreIncompatible: false })
     const [
       currentLockfileStats,
       wantedLockfileStats,
       manifestStats,
     ] = await Promise.all([
-      safeStat(path.join(virtualStoreDir, 'lock.yaml')),
+      safeStat(path.join(internalPnpmDir, 'lock.yaml')),
       safeStat(path.join(rootProjectManifestDir, WANTED_LOCKFILE)),
       statManifestFile(rootProjectManifestDir),
     ])

exec/build-modules/src/index.ts

@@ -164,7 +164,7 @@ async function buildDependency<T extends string> (
       try {
         const sideEffectsCacheKey = calcDepState(depGraph, opts.depsStateCache, depPath, {
           patchFileHash: depNode.patch?.file.hash,
-          isBuilt: hasSideEffects,
+          includeSubdepsHash: hasSideEffects,
         })
         await opts.storeController.upload(depNode.dir, {
           sideEffectsCacheKey,

exec/plugin-commands-rebuild/src/implementation/index.ts

@@ -345,7 +345,7 @@ async function _rebuild (
           const filesIndexFile = getIndexFilePathInCafs(opts.storeDir, resolution.integrity!.toString(), pkgId)
           const pkgFilesIndex = await loadJsonFile<PackageFilesIndex>(filesIndexFile)
           sideEffectsCacheKey = calcDepState(depGraph, depsStateCache, depPath, {
-            isBuilt: true,
+            includeSubdepsHash: true,
           })
           if (pkgFilesIndex.sideEffects?.[sideEffectsCacheKey]) {
             pkgsThatWereRebuilt.add(depPath)
@@ -378,7 +378,7 @@ async function _rebuild (
           try {
             if (!sideEffectsCacheKey) {
               sideEffectsCacheKey = calcDepState(depGraph, depsStateCache, depPath, {
-                isBuilt: true,
+                includeSubdepsHash: true,
               })
             }
             await opts.storeController.upload(pkgRoot, {

lockfile/fs/src/read.ts

@@ -21,14 +21,14 @@ import { getGitBranchLockfileNames } from './gitBranchLockfile'
 import { convertToLockfileObject } from './lockfileFormatConverters'
 
 export async function readCurrentLockfile (
-  virtualStoreDir: string,
+  pnpmInternalDir: string,
   opts: {
     wantedVersions?: string[]
     ignoreIncompatible: boolean
   }
 ): Promise<LockfileObject | null> {
-  const lockfilePath = path.join(virtualStoreDir, 'lock.yaml')
-  return (await _read(lockfilePath, virtualStoreDir, opts)).lockfile
+  const lockfilePath = path.join(pnpmInternalDir, 'lock.yaml')
+  return (await _read(lockfilePath, pnpmInternalDir, opts)).lockfile
 }
 
 export async function readWantedLockfileAndAutofixConflicts (

packages/calc-dep-state/package.json

@@ -36,7 +36,8 @@
     "@pnpm/lockfile.types": "workspace:*",
     "@pnpm/lockfile.utils": "workspace:*",
     "@pnpm/object.key-sorting": "workspace:*",
-    "@pnpm/types": "workspace:*"
+    "@pnpm/types": "workspace:*",
+    "sort-keys": "catalog:"
   },
   "devDependencies": {
     "@pnpm/calc-dep-state": "workspace:*"

packages/calc-dep-state/src/index.ts

@@ -1,8 +1,8 @@
 import { ENGINE_NAME } from '@pnpm/constants'
 import { getPkgIdWithPatchHash, refToRelative } from '@pnpm/dependency-path'
-import { type LockfileObject } from '@pnpm/lockfile.types'
 import { type DepPath, type PkgIdWithPatchHash } from '@pnpm/types'
 import { hashObjectWithoutSorting } from '@pnpm/crypto.object-hasher'
+import { type LockfileObject } from '@pnpm/lockfile.types'
 import { sortDirectKeys } from '@pnpm/object.key-sorting'
 
 export type DepsGraph<T extends string> = Record<T, DepsGraphNode<T>>
@@ -26,11 +26,11 @@ export function calcDepState<T extends string> (
   depPath: string,
   opts: {
     patchFileHash?: string
-    isBuilt: boolean
+    includeSubdepsHash: boolean
   }
 ): string {
   let result = ENGINE_NAME
-  if (opts.isBuilt) {
+  if (opts.includeSubdepsHash) {
     const depStateObj = calcDepStateObj(depPath, depsGraph, cache, new Set())
     result += `;deps=${hashObjectWithoutSorting(depStateObj)}`
   }
@@ -64,6 +64,35 @@ function calcDepStateObj<T extends string> (
   return cache[depPath]
 }
 
+export interface PkgMeta {
+  depPath: DepPath
+  name: string
+  version: string
+}
+
+export type PkgMetaIterator<T extends PkgMeta> = IterableIterator<T>
+
+export interface HashedDepPath<T extends PkgMeta> {
+  pkgMeta: T
+  hash: string
+}
+
+export function * iterateHashedGraphNodes<T extends PkgMeta> (
+  graph: DepsGraph<DepPath>,
+  pkgMetaIterator: PkgMetaIterator<T>
+): IterableIterator<HashedDepPath<T>> {
+  const cache: DepsStateCache = {}
+  for (const pkgMeta of pkgMetaIterator) {
+    const { name, version, depPath } = pkgMeta
+    const state = calcDepState(graph, cache, depPath, { includeSubdepsHash: true })
+    const hexDigest = hashObjectWithoutSorting(state, { encoding: 'hex' })
+    yield {
+      hash: `${name}/${version}/${hexDigest}`,
+      pkgMeta,
+    }
+  }
+}
+
 export function lockfileToDepGraph (lockfile: LockfileObject): DepsGraph<DepPath> {
   const graph: DepsGraph<DepPath> = {}
   if (lockfile.packages != null) {

packages/calc-dep-state/test/index.ts

@@ -20,7 +20,7 @@ const depsGraph = {
 
 test('calcDepState()', () => {
   expect(calcDepState(depsGraph, {}, 'registry/foo@1.0.0', {
-    isBuilt: true,
+    includeSubdepsHash: true,
   })).toBe(`${ENGINE_NAME};deps=${hashObject({
     'bar@1.0.0': { 'foo@1.0.0': {} },
   })}`)
@@ -28,6 +28,6 @@ test('calcDepState()', () => {
 
 test('calcDepState() when scripts are ignored', () => {
   expect(calcDepState(depsGraph, {}, 'registry/foo@1.0.0', {
-    isBuilt: false,
+    includeSubdepsHash: false,
   })).toBe(ENGINE_NAME)
 })

patching/plugin-commands-patching/src/getPatchedDependency.ts

@@ -4,7 +4,6 @@ import { prompt } from 'enquirer'
 import { readCurrentLockfile, type TarballResolution } from '@pnpm/lockfile.fs'
 import { nameVerFromPkgSnapshot } from '@pnpm/lockfile.utils'
 import { PnpmError } from '@pnpm/error'
-import { readModulesManifest } from '@pnpm/modules-yaml'
 import { isGitHostedPkgUrl } from '@pnpm/pick-fetcher'
 import realpathMissing from 'realpath-missing'
 import semver from 'semver'
@@ -81,10 +80,9 @@ export interface LockfileVersionsList {
 
 export async function getVersionsFromLockfile (dep: ParseWantedDependencyResult, opts: GetPatchedDependencyOptions): Promise<LockfileVersionsList> {
   const modulesDir = await realpathMissing(path.join(opts.lockfileDir, opts.modulesDir ?? 'node_modules'))
-  const modules = await readModulesManifest(modulesDir)
-  const lockfile = (modules?.virtualStoreDir && await readCurrentLockfile(modules.virtualStoreDir, {
+  const lockfile = await readCurrentLockfile(path.join(modulesDir, '.pnpm'), {
     ignoreIncompatible: true,
-  })) ?? null
+  }) ?? null
 
   if (!lockfile) {
     throw new PnpmError(

patching/plugin-commands-patching/test/patch.test.ts

@@ -1291,17 +1291,13 @@ describe('patch with custom modules-dir and virtual-store-dir', () => {
     const { allProjects, allProjectsGraph, selectedProjectsGraph } = await filterPackagesFromDir(customModulesDirFixture, [])
     await install.handler({
       ...DEFAULT_OPTS,
-      cacheDir,
-      storeDir,
-      dir: customModulesDirFixture,
+      ...defaultPatchOption,
       lockfileDir: customModulesDirFixture,
       allProjects,
       allProjectsGraph,
       selectedProjectsGraph,
       workspaceDir: customModulesDirFixture,
       saveLockfile: true,
-      modulesDir: 'fake_modules',
-      virtualStoreDir: 'fake_modules/.fake_store',
       confirmModulesPurge: false,
     })
     const output = await patch.handler(defaultPatchOption, ['is-positive@1'])
@@ -1323,8 +1319,6 @@ describe('patch with custom modules-dir and virtual-store-dir', () => {
       allProjects,
       allProjectsGraph,
       selectedProjectsGraph,
-      modulesDir: 'fake_modules',
-      virtualStoreDir: 'fake_modules/.fake_store',
       lockfileDir: customModulesDirFixture,
       workspaceDir: customModulesDirFixture,
       confirmModulesPurge: false,

pkg-manager/core/src/install/extendInstallOptions.ts

@@ -1,3 +1,4 @@
+import path from 'path'
 import { WANTED_LOCKFILE } from '@pnpm/constants'
 import { type Catalogs } from '@pnpm/catalogs.types'
 import { PnpmError } from '@pnpm/error'
@@ -30,6 +31,7 @@ export interface StrictInstallOptions {
   catalogMode: 'strict' | 'prefer' | 'manual'
   frozenLockfile: boolean
   frozenLockfileIfExists: boolean
+  enableGlobalVirtualStore: boolean
   enablePnp: boolean
   extraBinPaths: string[]
   extraEnv: Record<string, string>
@@ -183,6 +185,7 @@ const defaults = (opts: InstallOptions): StrictInstallOptions => {
     confirmModulesPurge: !opts.force,
     depth: 0,
     dedupeInjectedDeps: true,
+    enableGlobalVirtualStore: false,
     enablePnp: false,
     engineStrict: false,
     force: false,
@@ -309,5 +312,8 @@ export function extendOptions (
   }
   extendedOpts.registries = normalizeRegistries(extendedOpts.registries)
   extendedOpts.rawConfig['registry'] = extendedOpts.registries.default
+  if (extendedOpts.enableGlobalVirtualStore && extendedOpts.virtualStoreDir == null) {
+    extendedOpts.virtualStoreDir = path.join(extendedOpts.storeDir, 'links')
+  }
   return extendedOpts
 }

pkg-manager/core/src/install/index.ts

@@ -320,9 +320,10 @@ export async function mutateModules (
     })
   }
 
-  const pruneVirtualStore = ctx.modulesFile?.prunedAt && opts.modulesCacheMaxAge > 0
+  const pruneVirtualStore = !opts.enableGlobalVirtualStore && (ctx.modulesFile?.prunedAt && opts.modulesCacheMaxAge > 0
     ? cacheExpired(ctx.modulesFile.prunedAt, opts.modulesCacheMaxAge)
     : true
+  )
 
   if (!maybeOpts.ignorePackageManifest) {
     for (const { manifest, rootDir } of Object.values(ctx.projects)) {
@@ -800,9 +801,10 @@ Note that in CI environments, this setting is enabled by default.`,
         opts.useLockfile && opts.saveLockfile && opts.mergeGitBranchLockfiles ||
         !upToDateLockfileMajorVersion && !opts.frozenLockfile
       ) {
+        const currentLockfileDir = path.join(ctx.rootModulesDir, '.pnpm')
         await writeLockfiles({
           currentLockfile: ctx.currentLockfile,
-          currentLockfileDir: ctx.virtualStoreDir,
+          currentLockfileDir,
           wantedLockfile: ctx.wantedLockfile,
           wantedLockfileDir: ctx.lockfileDir,
           useGitBranchLockfile: opts.useGitBranchLockfile,
@@ -1139,6 +1141,7 @@ const _installInContext: InstallFunction = async (projects, ctx, opts) => {
       dedupeInjectedDeps: opts.dedupeInjectedDeps,
       dedupePeerDependents: opts.dedupePeerDependents,
       dryRun: opts.lockfileOnly,
+      enableGlobalVirtualStore: opts.enableGlobalVirtualStore,
       engineStrict: opts.engineStrict,
       excludeLinksFromLockfile: opts.excludeLinksFromLockfile,
       force: opts.force,
@@ -1412,11 +1415,12 @@ const _installInContext: InstallFunction = async (projects, ctx, opts) => {
       virtualStoreDir: ctx.virtualStoreDir,
       virtualStoreDirMaxLength: opts.virtualStoreDirMaxLength,
     })
+    const currentLockfileDir = path.join(ctx.rootModulesDir, '.pnpm')
     await Promise.all([
       opts.useLockfile && opts.saveLockfile
         ? writeLockfiles({
           currentLockfile: result.currentLockfile,
-          currentLockfileDir: ctx.virtualStoreDir,
+          currentLockfileDir,
           wantedLockfile: newLockfile,
           wantedLockfileDir: ctx.lockfileDir,
           ...lockfileOpts,

pkg-manager/core/src/install/link.ts

@@ -468,7 +468,7 @@ async function linkAllPkgs (
       if (opts.sideEffectsCacheRead && files.sideEffects && !isEmpty(files.sideEffects)) {
         if (opts?.allowBuild?.(depNode.name) !== false) {
           sideEffectsCacheKey = calcDepState(opts.depGraph, opts.depsStateCache, depNode.depPath, {
-            isBuilt: !opts.ignoreScripts && depNode.requiresBuild,
+            includeSubdepsHash: !opts.ignoreScripts && depNode.requiresBuild, // true when is built
             patchFileHash: depNode.patch?.file.hash,
           })
         }

pkg-manager/core/test/install/globalVirtualStore.ts

@@ -0,0 +1,72 @@
+import fs from 'fs'
+import path from 'path'
+import { prepareEmpty } from '@pnpm/prepare'
+import { install } from '@pnpm/core'
+import { sync as rimraf } from '@zkochan/rimraf'
+import { testDefaults } from '../utils'
+
+test('using a global virtual store', async () => {
+  prepareEmpty()
+  const globalVirtualStoreDir = path.resolve('links')
+  const manifest = {
+    dependencies: {
+      '@pnpm.e2e/pkg-with-1-dep': '100.0.0',
+    },
+  }
+  await install(manifest, testDefaults({
+    enableGlobalVirtualStore: true,
+    virtualStoreDir: globalVirtualStoreDir,
+    privateHoistPattern: '*',
+  }))
+
+  {
+    expect(fs.existsSync(path.resolve('node_modules/.pnpm/node_modules/@pnpm.e2e/dep-of-pkg-with-1-dep')))
+    expect(fs.existsSync(path.resolve('node_modules/.pnpm/lock.yaml'))).toBeTruthy()
+    const files = fs.readdirSync(path.join(globalVirtualStoreDir, '@pnpm.e2e/pkg-with-1-dep/100.0.0'))
+    expect(files.length).toBe(1)
+    expect(fs.existsSync(path.join(globalVirtualStoreDir, '@pnpm.e2e/pkg-with-1-dep/100.0.0', files[0], 'node_modules/@pnpm.e2e/pkg-with-1-dep/package.json'))).toBeTruthy()
+    expect(fs.existsSync(path.join(globalVirtualStoreDir, '@pnpm.e2e/pkg-with-1-dep/100.0.0', files[0], 'node_modules/@pnpm.e2e/dep-of-pkg-with-1-dep/package.json'))).toBeTruthy()
+  }
+
+  rimraf('node_modules')
+  rimraf(globalVirtualStoreDir)
+  await install(manifest, testDefaults({
+    enableGlobalVirtualStore: true,
+    virtualStoreDir: globalVirtualStoreDir,
+    frozenLockfile: true,
+  }))
+
+  {
+    expect(fs.existsSync(path.resolve('node_modules/.pnpm/node_modules/@pnpm.e2e/dep-of-pkg-with-1-dep')))
+    expect(fs.existsSync(path.resolve('node_modules/.pnpm/lock.yaml'))).toBeTruthy()
+    const files = fs.readdirSync(path.join(globalVirtualStoreDir, '@pnpm.e2e/pkg-with-1-dep/100.0.0'))
+    expect(files.length).toBe(1)
+    expect(fs.existsSync(path.join(globalVirtualStoreDir, '@pnpm.e2e/pkg-with-1-dep/100.0.0', files[0], 'node_modules/@pnpm.e2e/pkg-with-1-dep/package.json'))).toBeTruthy()
+    expect(fs.existsSync(path.join(globalVirtualStoreDir, '@pnpm.e2e/pkg-with-1-dep/100.0.0', files[0], 'node_modules/@pnpm.e2e/dep-of-pkg-with-1-dep/package.json'))).toBeTruthy()
+  }
+})
+
+test('modules are correctly updated when using a global virtual store', async () => {
+  prepareEmpty()
+  const globalVirtualStoreDir = path.resolve('links')
+  const manifest = {
+    dependencies: {
+      '@pnpm.e2e/pkg-with-1-dep': '100.0.0',
+      '@pnpm.e2e/peer-c': '1.0.0',
+    },
+  }
+  const opts = testDefaults({
+    enableGlobalVirtualStore: true,
+    virtualStoreDir: globalVirtualStoreDir,
+  })
+  await install(manifest, opts)
+  manifest.dependencies['@pnpm.e2e/peer-c'] = '2.0.0'
+  await install(manifest, opts)
+
+  {
+    expect(fs.existsSync(path.resolve('node_modules/.pnpm/lock.yaml'))).toBeTruthy()
+    const files = fs.readdirSync(path.join(globalVirtualStoreDir, '@pnpm.e2e/peer-c/2.0.0'))
+    expect(files.length).toBe(1)
+    expect(fs.existsSync(path.join(globalVirtualStoreDir, '@pnpm.e2e/peer-c/2.0.0', files[0], 'node_modules/@pnpm.e2e/peer-c/package.json'))).toBeTruthy()
+  }
+})

pkg-manager/get-context/src/index.ts

@@ -88,6 +88,7 @@ export interface GetContextOptions {
   confirmModulesPurge?: boolean
   force: boolean
   frozenLockfile?: boolean
+  enableGlobalVirtualStore?: boolean
   extraBinPaths: string[]
   extendNodePath?: boolean
   lockfileDir: string
@@ -137,13 +138,22 @@ export async function getContext (
   const extraBinPaths = [
     ...opts.extraBinPaths || [],
   ]
-  const hoistedModulesDir = path.join(virtualStoreDir, 'node_modules')
+  const internalPnpmDir = path.join(importersContext.rootModulesDir, '.pnpm')
+  const hoistedModulesDir = path.join(
+    opts.enableGlobalVirtualStore ? internalPnpmDir : virtualStoreDir,
+    'node_modules'
+  )
   if (opts.hoistPattern?.length) {
     extraBinPaths.unshift(path.join(hoistedModulesDir, '.bin'))
   }
   const ctx: PnpmContext = {
     extraBinPaths,
-    extraNodePaths: getExtraNodePaths({ extendNodePath: opts.extendNodePath, nodeLinker: opts.nodeLinker, hoistPattern: importersContext.currentHoistPattern ?? opts.hoistPattern, virtualStoreDir }),
+    extraNodePaths: getExtraNodePaths({
+      extendNodePath: opts.extendNodePath,
+      nodeLinker: opts.nodeLinker,
+      hoistPattern: importersContext.currentHoistPattern ?? opts.hoistPattern,
+      hoistedModulesDir,
+    }),
     hoistedDependencies: importersContext.hoistedDependencies,
     hoistedModulesDir,
     hoistPattern: opts.hoistPattern,
@@ -174,7 +184,7 @@ export async function getContext (
       useLockfile: opts.useLockfile,
       useGitBranchLockfile: opts.useGitBranchLockfile,
       mergeGitBranchLockfiles: opts.mergeGitBranchLockfiles,
-      virtualStoreDir,
+      internalPnpmDir,
     }),
   }
   contextLogger.debug({
@@ -222,6 +232,7 @@ export async function getContextForSingleImporter (
   manifest: ProjectManifest,
   opts: {
     autoInstallPeers: boolean
+    enableGlobalVirtualStore?: boolean
     excludeLinksFromLockfile: boolean
     peersSuffixMaxLength: number
     force: boolean
@@ -282,13 +293,22 @@ export async function getContextForSingleImporter (
   const extraBinPaths = [
     ...opts.extraBinPaths || [],
   ]
-  const hoistedModulesDir = path.join(virtualStoreDir, 'node_modules')
+  const internalPnpmDir = path.join(rootModulesDir, '.pnpm')
+  const hoistedModulesDir = path.join(
+    opts.enableGlobalVirtualStore ? internalPnpmDir : virtualStoreDir,
+    'node_modules'
+  )
   if (opts.hoistPattern?.length) {
     extraBinPaths.unshift(path.join(hoistedModulesDir, '.bin'))
   }
   const ctx: PnpmSingleContext = {
     extraBinPaths,
-    extraNodePaths: getExtraNodePaths({ extendNodePath: opts.extendNodePath, nodeLinker: opts.nodeLinker, hoistPattern: currentHoistPattern ?? opts.hoistPattern, virtualStoreDir }),
+    extraNodePaths: getExtraNodePaths({
+      extendNodePath: opts.extendNodePath,
+      nodeLinker: opts.nodeLinker,
+      hoistPattern: currentHoistPattern ?? opts.hoistPattern,
+      hoistedModulesDir,
+    }),
     hoistedDependencies,
     hoistedModulesDir,
     hoistPattern: opts.hoistPattern,
@@ -321,7 +341,7 @@ export async function getContextForSingleImporter (
       useLockfile: opts.useLockfile,
       useGitBranchLockfile: opts.useGitBranchLockfile,
       mergeGitBranchLockfiles: opts.mergeGitBranchLockfiles,
-      virtualStoreDir,
+      internalPnpmDir,
     }),
   }
   packageManifestLogger.debug({
@@ -338,15 +358,15 @@ export async function getContextForSingleImporter (
 }
 
 function getExtraNodePaths (
-  { extendNodePath = true, hoistPattern, nodeLinker, virtualStoreDir }: {
+  { extendNodePath = true, hoistPattern, nodeLinker, hoistedModulesDir }: {
     extendNodePath?: boolean
     hoistPattern?: string[]
     nodeLinker: 'isolated' | 'hoisted' | 'pnp'
-    virtualStoreDir: string
+    hoistedModulesDir: string
   }
 ): string[] {
   if (extendNodePath && nodeLinker === 'isolated' && hoistPattern?.length) {
-    return [path.join(virtualStoreDir, 'node_modules')]
+    return [hoistedModulesDir]
   }
   return []
 }

pkg-manager/get-context/src/readLockfiles.ts

@@ -41,7 +41,7 @@ export async function readLockfiles (
     useLockfile: boolean
     useGitBranchLockfile?: boolean
     mergeGitBranchLockfiles?: boolean
-    virtualStoreDir: string
+    internalPnpmDir: string
   }
 ): Promise<{
     currentLockfile: LockfileObject
@@ -96,10 +96,10 @@ export async function readLockfiles (
   fileReads.push(
     (async () => {
       try {
-        return await readCurrentLockfile(opts.virtualStoreDir, lockfileOpts)
+        return await readCurrentLockfile(opts.internalPnpmDir, lockfileOpts)
       } catch (err: any) { // eslint-disable-line
         logger.warn({
-          message: `Ignoring broken lockfile at ${opts.virtualStoreDir}: ${err.message as string}`,
+          message: `Ignoring broken lockfile at ${opts.internalPnpmDir}: ${err.message as string}`,
           prefix: opts.lockfileDir,
         })
         return undefined

pkg-manager/headless/src/index.ts

@@ -142,6 +142,7 @@ export interface HeadlessOptions {
   currentHoistedLocations?: Record<string, string[]>
   lockfileDir: string
   modulesDir?: string
+  enableGlobalVirtualStore?: boolean
   virtualStoreDir?: string
   virtualStoreDirMaxLength: number
   patchedDependencies?: PatchGroupRecord
@@ -212,9 +213,13 @@ export async function headlessInstall (opts: HeadlessOptions): Promise<Installat
   const depsStateCache: DepsStateCache = {}
   const relativeModulesDir = opts.modulesDir ?? 'node_modules'
   const rootModulesDir = await realpathMissing(path.join(lockfileDir, relativeModulesDir))
+  const internalPnpmDir = path.join(rootModulesDir, '.pnpm')
+  const currentLockfile = opts.currentLockfile ?? await readCurrentLockfile(internalPnpmDir, { ignoreIncompatible: false })
   const virtualStoreDir = pathAbsolute(opts.virtualStoreDir ?? path.join(relativeModulesDir, '.pnpm'), lockfileDir)
-  const currentLockfile = opts.currentLockfile ?? await readCurrentLockfile(virtualStoreDir, { ignoreIncompatible: false })
-  const hoistedModulesDir = path.join(virtualStoreDir, 'node_modules')
+  const hoistedModulesDir = path.join(
+    opts.enableGlobalVirtualStore ? internalPnpmDir : virtualStoreDir,
+    'node_modules'
+  )
   const publicHoistedModulesDir = rootModulesDir
   const selectedProjects = Object.values(pick(opts.selectedProjectDirs, opts.allProjects))
 
@@ -520,7 +525,7 @@ export async function headlessInstall (opts: HeadlessOptions): Promise<Installat
     }
     const extraBinPaths = [...opts.extraBinPaths ?? []]
     if (opts.hoistPattern != null) {
-      extraBinPaths.unshift(path.join(virtualStoreDir, 'node_modules/.bin'))
+      extraBinPaths.unshift(path.join(hoistedModulesDir, '.bin'))
     }
     let extraEnv: Record<string, string> | undefined = opts.extraEnv
     if (opts.enablePnp) {
@@ -634,17 +639,18 @@ export async function headlessInstall (opts: HeadlessOptions): Promise<Installat
     }, {
       makeModulesDir: Object.keys(filteredLockfile.packages ?? {}).length > 0,
     })
+    const currentLockfileDir = path.join(rootModulesDir, '.pnpm')
     if (opts.useLockfile) {
       // We need to write the wanted lockfile as well.
       // Even though it will only be changed if the workspace will have new projects with no dependencies.
       await writeLockfiles({
         wantedLockfileDir: opts.lockfileDir,
-        currentLockfileDir: virtualStoreDir,
+        currentLockfileDir,
         wantedLockfile,
         currentLockfile: filteredLockfile,
       })
     } else {
-      await writeCurrentLockfile(virtualStoreDir, filteredLockfile)
+      await writeCurrentLockfile(currentLockfileDir, filteredLockfile)
     }
   }
 
@@ -868,7 +874,7 @@ async function linkAllPkgs (
       if (opts.sideEffectsCacheRead && filesResponse.sideEffects && !isEmpty(filesResponse.sideEffects)) {
         if (opts?.allowBuild?.(depNode.name) !== false) {
           sideEffectsCacheKey = calcDepState(opts.depGraph, opts.depsStateCache, depNode.dir, {
-            isBuilt: !opts.ignoreScripts && depNode.requiresBuild,
+            includeSubdepsHash: !opts.ignoreScripts && depNode.requiresBuild, // true when is built
             patchFileHash: depNode.patch?.file.hash,
           })
         }

pkg-manager/headless/src/linkHoistedModules.ts

@@ -117,7 +117,7 @@ async function linkAllPkgsInOrder (
         if (opts.sideEffectsCacheRead && filesResponse.sideEffects && !isEmpty(filesResponse.sideEffects)) {
           if (opts?.allowBuild?.(depNode.name) !== false) {
             sideEffectsCacheKey = _calcDepState(dir, {
-              isBuilt: !opts.ignoreScripts && depNode.requiresBuild,
+              includeSubdepsHash: !opts.ignoreScripts && depNode.requiresBuild, // true when is built
               patchFileHash: depNode.patch?.file.hash,
             })
           }

pkg-manager/resolve-dependencies/package.json

@@ -31,6 +31,7 @@
     "_test": "jest"
   },
   "dependencies": {
+    "@pnpm/calc-dep-state": "workspace:*",
     "@pnpm/catalogs.resolver": "workspace:*",
     "@pnpm/catalogs.types": "workspace:*",
     "@pnpm/constants": "workspace:*",

pkg-manager/resolve-dependencies/src/index.ts

@@ -3,6 +3,7 @@ import { type Catalogs } from '@pnpm/catalogs.types'
 import {
   packageManifestLogger,
 } from '@pnpm/core-loggers'
+import { iterateHashedGraphNodes } from '@pnpm/calc-dep-state'
 import {
   type LockfileObject,
   type ProjectSnapshot,
@@ -22,6 +23,7 @@ import {
   type ProjectManifest,
   type ProjectId,
   type ProjectRootDir,
+  type DepPath,
 } from '@pnpm/types'
 import difference from 'ramda/src/difference'
 import zipWith from 'ramda/src/zipWith'
@@ -117,6 +119,7 @@ export async function resolveDependencies (
     saveWorkspaceProtocol: 'rolling' | boolean
     lockfileIncludeTarballUrl?: boolean
     allowUnusedPatches?: boolean
+    enableGlobalVirtualStore?: boolean
   }
 ): Promise<ResolveDependenciesResult> {
   const _toResolveImporter = toResolveImporter.bind(null, {
@@ -319,7 +322,7 @@ export async function resolveDependencies (
 
   return {
     dependenciesByProjectId,
-    dependenciesGraph,
+    dependenciesGraph: opts.enableGlobalVirtualStore ? extendGraph(dependenciesGraph, opts.virtualStoreDir) : dependenciesGraph,
     outdatedDependencies,
     linkedDependenciesByProjectId,
     updatedCatalogs,
@@ -441,3 +444,28 @@ async function getTopParents (pkgAliases: string[], modulesDir: string): Promise
   }, pkgs, pkgAliases)
     .filter(Boolean) as DependencyManifest[]
 }
+
+function extendGraph (graph: DependenciesGraph, virtualStoreDir: string): DependenciesGraph {
+  const pkgMetaIter = (function * () {
+    for (const depPath in graph) {
+      if (Object.prototype.hasOwnProperty.call(graph, depPath)) {
+        const { name, version, pkgIdWithPatchHash } = graph[depPath as DepPath]
+        yield {
+          name,
+          version,
+          depPath: depPath as DepPath,
+          pkgIdWithPatchHash,
+        }
+      }
+    }
+  })()
+  for (const { pkgMeta: { depPath }, hash } of iterateHashedGraphNodes(graph, pkgMetaIter)) {
+    const modules = path.join(virtualStoreDir, hash, 'node_modules')
+    const node = graph[depPath]
+    Object.assign(node, {
+      modules,
+      dir: path.join(modules, node.name),
+    })
+  }
+  return graph
+}

pkg-manager/resolve-dependencies/tsconfig.json

@@ -30,6 +30,9 @@
     {
       "path": "../../lockfile/utils"
     },
+    {
+      "path": "../../packages/calc-dep-state"
+    },
     {
       "path": "../../packages/constants"
     },

pnpm-lock.yaml

@@ -1914,6 +1914,9 @@ importers:
 
   deps/graph-builder:
     dependencies:
+      '@pnpm/calc-dep-state':
+        specifier: workspace:*
+        version: link:../../packages/calc-dep-state
       '@pnpm/constants':
         specifier: workspace:*
         version: link:../../packages/constants
@@ -3979,6 +3982,9 @@ importers:
       '@pnpm/types':
         specifier: workspace:*
         version: link:../types
+      sort-keys:
+        specifier: 'catalog:'
+        version: 4.2.0
     devDependencies:
       '@pnpm/calc-dep-state':
         specifier: workspace:*
@@ -5722,6 +5728,9 @@ importers:
 
   pkg-manager/resolve-dependencies:
     dependencies:
+      '@pnpm/calc-dep-state':
+        specifier: workspace:*
+        version: link:../../packages/calc-dep-state
       '@pnpm/catalogs.resolver':
         specifier: workspace:*
         version: link:../../catalogs/resolver
@@ -7118,9 +7127,6 @@ importers:
       '@pnpm/matcher':
         specifier: workspace:*
         version: link:../../config/matcher
-      '@pnpm/modules-yaml':
-        specifier: workspace:*
-        version: link:../../pkg-manager/modules-yaml
       '@pnpm/npm-resolver':
         specifier: workspace:*
         version: link:../../resolving/npm-resolver

pnpm/test/install/globalVirtualStore.ts

@@ -0,0 +1,28 @@
+import fs from 'fs'
+import path from 'path'
+import { prepare } from '@pnpm/prepare'
+import { sync as writeYamlFile } from 'write-yaml-file'
+import { execPnpm } from '../utils'
+
+test('using a global virtual store', async () => {
+  prepare({
+    dependencies: {
+      '@pnpm.e2e/pkg-with-1-dep': '100.0.0',
+    },
+  })
+  const storeDir = path.resolve('store')
+  const globalVirtualStoreDir = path.join(storeDir, 'v10/links')
+  writeYamlFile(path.resolve('pnpm-workspace.yaml'), {
+    enableGlobalVirtualStore: true,
+    storeDir,
+    privateHoistPattern: '*',
+  })
+  await execPnpm(['install'])
+
+  expect(fs.existsSync(path.resolve('node_modules/.pnpm/node_modules/@pnpm.e2e/dep-of-pkg-with-1-dep')))
+  expect(fs.existsSync(path.resolve('node_modules/.pnpm/lock.yaml'))).toBeTruthy()
+  const files = fs.readdirSync(path.join(globalVirtualStoreDir, '@pnpm.e2e/pkg-with-1-dep/100.0.0'))
+  expect(files.length).toBe(1)
+  expect(fs.existsSync(path.join(globalVirtualStoreDir, '@pnpm.e2e/pkg-with-1-dep/100.0.0', files[0], 'node_modules/@pnpm.e2e/pkg-with-1-dep/package.json'))).toBeTruthy()
+  expect(fs.existsSync(path.join(globalVirtualStoreDir, '@pnpm.e2e/pkg-with-1-dep/100.0.0', files[0], 'node_modules/@pnpm.e2e/dep-of-pkg-with-1-dep/package.json'))).toBeTruthy()
+})

pnpm/test/install/misc.ts

@@ -391,7 +391,7 @@ test('using a custom virtual-store-dir location', async () => {
   await execPnpm(['install', '--virtual-store-dir=.pnpm'])
 
   expect(fs.existsSync('.pnpm/rimraf@2.5.1/node_modules/rimraf/package.json')).toBeTruthy()
-  expect(fs.existsSync('.pnpm/lock.yaml')).toBeTruthy()
+  expect(fs.existsSync('node_modules/.pnpm/lock.yaml')).toBeTruthy()
   expect(fs.existsSync('.pnpm/node_modules/once/package.json')).toBeTruthy()
 
   rimraf('node_modules')
@@ -400,7 +400,7 @@ test('using a custom virtual-store-dir location', async () => {
   await execPnpm(['install', '--virtual-store-dir=.pnpm', '--frozen-lockfile'])
 
   expect(fs.existsSync('.pnpm/rimraf@2.5.1/node_modules/rimraf/package.json')).toBeTruthy()
-  expect(fs.existsSync('.pnpm/lock.yaml')).toBeTruthy()
+  expect(fs.existsSync('node_modules/.pnpm/lock.yaml')).toBeTruthy()
   expect(fs.existsSync('.pnpm/node_modules/once/package.json')).toBeTruthy()
 })
 

reviewing/dependencies-hierarchy/src/buildDependenciesHierarchy.ts

@@ -53,7 +53,8 @@ export async function buildDependenciesHierarchy (
     ...maybeOpts?.registries,
     ...modules?.registries,
   })
-  const currentLockfile = (modules?.virtualStoreDir && await readCurrentLockfile(modules.virtualStoreDir, { ignoreIncompatible: false })) ?? null
+  const internalPnpmDir = path.join(modulesDir, '.pnpm')
+  const currentLockfile = await readCurrentLockfile(internalPnpmDir, { ignoreIncompatible: false }) ?? null
   const wantedLockfile = await readWantedLockfile(maybeOpts.lockfileDir, { ignoreIncompatible: false })
   if (projectPaths == null) {
     projectPaths = Object.keys(wantedLockfile?.importers ?? {})

reviewing/outdated/package.json

@@ -42,7 +42,6 @@
     "@pnpm/lockfile.utils": "workspace:*",
     "@pnpm/manifest-utils": "workspace:*",
     "@pnpm/matcher": "workspace:*",
-    "@pnpm/modules-yaml": "workspace:*",
     "@pnpm/npm-resolver": "workspace:*",
     "@pnpm/parse-overrides": "workspace:*",
     "@pnpm/pick-registry-for-package": "workspace:*",

reviewing/outdated/src/outdatedDepsOfProjects.ts

@@ -5,7 +5,6 @@ import {
   readWantedLockfile,
 } from '@pnpm/lockfile.fs'
 import { createMatcher } from '@pnpm/matcher'
-import { readModulesManifest } from '@pnpm/modules-yaml'
 import {
   type IncludedDependencies,
   type ProjectManifest,
@@ -33,9 +32,8 @@ export async function outdatedDepsOfProjects (
     ))
   }
   const lockfileDir = opts.lockfileDir ?? opts.dir
-  const modules = await readModulesManifest(path.join(lockfileDir, 'node_modules'))
-  const virtualStoreDir = modules?.virtualStoreDir ?? path.join(lockfileDir, 'node_modules/.pnpm')
-  const currentLockfile = await readCurrentLockfile(virtualStoreDir, { ignoreIncompatible: false })
+  const internalPnpmDir = path.join(path.join(lockfileDir, 'node_modules/.pnpm'))
+  const currentLockfile = await readCurrentLockfile(internalPnpmDir, { ignoreIncompatible: false })
   const wantedLockfile = await readWantedLockfile(lockfileDir, { ignoreIncompatible: false }) ?? currentLockfile
   const getLatestManifest = createManifestGetter({
     ...opts,

reviewing/outdated/tsconfig.json

@@ -51,9 +51,6 @@
     {
       "path": "../../pkg-manager/client"
     },
-    {
-      "path": "../../pkg-manager/modules-yaml"
-    },
     {
       "path": "../../pkg-manifest/manifest-utils"
     },