feat: Create a node modules directory without using symlinks (#4162)

ref #4073

.changeset/fresh-balloons-brake.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/real-hoist": minor
+---
+
+Initial release.

.changeset/lemon-clocks-brake.md

@@ -0,0 +1,7 @@
+---
+"@pnpm/config": minor
+"@pnpm/core": minor
+"@pnpm/headless": minor
+---
+
+nodeLinker may accept a new value: node-modules. node-modules will create a "classic" node_modules folder without using symlinks.

packages/config/src/Config.ts

@@ -73,6 +73,7 @@ export interface Config {
   enablePrePostScripts?: boolean
   useNodeVersion?: string
   useStderr?: boolean
+  nodeLinker?: 'node-modules' | 'pnpm' | 'pnp'
 
   // proxy
   httpProxy?: string

packages/config/src/index.ts

@@ -462,7 +462,7 @@ export default async (
   if (!pnpmConfig.noProxy) {
     pnpmConfig.noProxy = pnpmConfig['noproxy'] ?? getProcessEnv('no_proxy')
   }
-  pnpmConfig.enablePnp = pnpmConfig['nodeLinker'] === 'pnp'
+  pnpmConfig.enablePnp = pnpmConfig.nodeLinker === 'pnp'
   if (!pnpmConfig.userConfig) {
     pnpmConfig.userConfig = npmConfig.sources.user?.data
   }

packages/core/src/install/extendInstallOptions.ts

@@ -47,6 +47,7 @@ export interface StrictInstallOptions {
   engineStrict: boolean
   neverBuiltDependencies: string[]
   nodeExecPath?: string
+  nodeLinker?: 'node-modules' | 'pnpm' | 'pnp'
   nodeVersion: string
   packageExtensions: Record<string, PackageExtension>
   packageManager: {
@@ -126,6 +127,7 @@ const defaults = async (opts: InstallOptions) => {
     lockfileOnly: false,
     neverBuiltDependencies: [] as string[],
     nodeVersion: process.version,
+    nodeLinker: 'pnpm',
     overrides: {},
     ownLifecycleHooksStdio: 'inherit',
     ignorePackageManifest: false,

packages/core/src/install/index.ts

@@ -285,6 +285,7 @@ export async function mutateModules (
             include: opts.include,
             lockfileDir: ctx.lockfileDir,
             modulesDir: opts.modulesDir,
+            nodeLinker: opts.nodeLinker,
             ownLifecycleHooksStdio: opts.ownLifecycleHooksStdio,
             packageManager: opts.packageManager,
             pendingBuilds: ctx.pendingBuilds,

packages/headless/package.json

@@ -84,6 +84,7 @@
     "@pnpm/package-requester": "workspace:15.2.6",
     "@pnpm/read-package-json": "workspace:5.0.9",
     "@pnpm/read-project-manifest": "workspace:2.0.10",
+    "@pnpm/real-hoist": "workspace:0.0.0",
     "@pnpm/store-controller-types": "workspace:11.0.10",
     "@pnpm/symlink-dependency": "workspace:4.0.11",
     "@pnpm/types": "workspace:7.8.0",

packages/headless/src/index.ts

@@ -63,13 +63,26 @@ import omit from 'ramda/src/omit'
 import props from 'ramda/src/props'
 import realpathMissing from 'realpath-missing'
 import lockfileToDepGraph, {
+  DirectDependenciesByImporterId,
+  DepHierarchy,
   DependenciesGraph,
   DependenciesGraphNode,
   LockfileToDepGraphOptions,
 } from './lockfileToDepGraph'
+import lockfileToHoistedDepGraph from './lockfileToHoistedDepGraph'
 
 export type ReporterFunction = (logObj: LogBase) => void
 
+export interface Project {
+  binsDir: string
+  buildIndex: number
+  manifest: ProjectManifest
+  modulesDir: string
+  id: string
+  pruneDirectDependencies?: boolean
+  rootDir: string
+}
+
 export interface HeadlessOptions {
   childConcurrency?: number
   currentLockfile?: Lockfile
@@ -84,15 +97,7 @@ export interface HeadlessOptions {
   ignoreScripts: boolean
   ignorePackageManifest?: boolean
   include: IncludedDependencies
-  projects: Array<{
-    binsDir: string
-    buildIndex: number
-    manifest: ProjectManifest
-    modulesDir: string
-    id: string
-    pruneDirectDependencies?: boolean
-    rootDir: string
-  }>
+  projects: Project[]
   prunedAt?: string
   hoistedDependencies: HoistedDependencies
   hoistPattern?: string[]
@@ -125,6 +130,7 @@ export interface HeadlessOptions {
   pendingBuilds: string[]
   skipped: Set<string>
   enableModulesDir?: boolean
+  nodeLinker?: 'pnpm' | 'node-modules' | 'pnp'
 }
 
 export default async (opts: HeadlessOptions) => {
@@ -220,18 +226,26 @@ export default async (opts: HeadlessOptions) => {
     includeIncompatiblePackages: opts.force,
     lockfileDir,
   })
-  const { directDependenciesByImporterId, graph } = await lockfileToDepGraph(
-    filteredLockfile,
-    opts.force ? null : currentLockfile,
-    {
-      ...opts,
-      importerIds,
-      lockfileDir,
-      skipped,
-      virtualStoreDir,
-      nodeVersion: opts.currentEngine.nodeVersion,
-      pnpmVersion: opts.currentEngine.pnpmVersion,
-    } as LockfileToDepGraphOptions
+  const lockfileToDepGraphOpts = {
+    ...opts,
+    importerIds,
+    lockfileDir,
+    skipped,
+    virtualStoreDir,
+    nodeVersion: opts.currentEngine.nodeVersion,
+    pnpmVersion: opts.currentEngine.pnpmVersion,
+  } as LockfileToDepGraphOptions
+  const { hierarchy, directDependenciesByImporterId, graph, symlinkedDirectDependenciesByImporterId } = await (
+    opts.nodeLinker === 'node-modules'
+      ? lockfileToHoistedDepGraph(
+        filteredLockfile,
+        lockfileToDepGraphOpts
+      )
+      : lockfileToDepGraph(
+        filteredLockfile,
+        opts.force ? null : currentLockfile,
+        lockfileToDepGraphOpts
+      )
   )
   if (opts.enablePnp) {
     const importerNames = fromPairs(
@@ -259,7 +273,26 @@ export default async (opts: HeadlessOptions) => {
   }
 
   let newHoistedDependencies!: HoistedDependencies
-  if (opts.enableModulesDir !== false) {
+  if (opts.nodeLinker === 'node-modules' && hierarchy) {
+    await linkAllPkgsInOrder(opts.storeController, graph, hierarchy, {
+      force: opts.force,
+      lockfileDir: opts.lockfileDir,
+      targetEngine: opts.sideEffectsCacheRead && ENGINE_NAME || undefined,
+    })
+    stageLogger.debug({
+      prefix: lockfileDir,
+      stage: 'importing_done',
+    })
+
+    await symlinkDirectDependencies({
+      directDependenciesByImporterId: symlinkedDirectDependenciesByImporterId!,
+      filteredLockfile,
+      lockfileDir,
+      projects: opts.projects,
+      registries: opts.registries,
+      symlink: opts.symlink,
+    })
+  } else if (opts.enableModulesDir !== false) {
     await Promise.all(depNodes.map(async (depNode) => fs.mkdir(depNode.modules, { recursive: true })))
     await Promise.all([
       opts.symlink === false
@@ -311,26 +344,14 @@ export default async (opts: HeadlessOptions) => {
 
     /** Skip linking and due to no project manifest */
     if (!opts.ignorePackageManifest) {
-      await Promise.all(opts.projects.map(async ({ rootDir, id, manifest, modulesDir }) => {
-        if (opts.symlink !== false) {
-          await linkRootPackages(filteredLockfile, {
-            importerId: id,
-            importerModulesDir: modulesDir,
-            lockfileDir,
-            projectDir: rootDir,
-            projects: opts.projects,
-            registries: opts.registries,
-            rootDependencies: directDependenciesByImporterId[id],
-          })
-        }
-
-        // Even though headless installation will never update the package.json
-        // this needs to be logged because otherwise install summary won't be printed
-        packageManifestLogger.debug({
-          prefix: rootDir,
-          updated: manifest,
-        })
-      }))
+      await symlinkDirectDependencies({
+        directDependenciesByImporterId,
+        filteredLockfile,
+        lockfileDir,
+        projects: opts.projects,
+        registries: opts.registries,
+        symlink: opts.symlink,
+      })
     }
   }
 
@@ -457,6 +478,43 @@ export default async (opts: HeadlessOptions) => {
   }
 }
 
+type SymlinkDirectDependenciesOpts = Pick<HeadlessOptions, 'projects' | 'registries' | 'symlink' | 'lockfileDir'> & {
+  filteredLockfile: Lockfile
+  directDependenciesByImporterId: DirectDependenciesByImporterId
+}
+
+async function symlinkDirectDependencies (
+  {
+    filteredLockfile,
+    directDependenciesByImporterId,
+    lockfileDir,
+    projects,
+    registries,
+    symlink,
+  }: SymlinkDirectDependenciesOpts
+) {
+  await Promise.all(projects.map(async ({ rootDir, id, manifest, modulesDir }) => {
+    if (symlink !== false) {
+      await linkRootPackages(filteredLockfile, {
+        importerId: id,
+        importerModulesDir: modulesDir,
+        lockfileDir,
+        projectDir: rootDir,
+        projects,
+        registries,
+        rootDependencies: directDependenciesByImporterId[id],
+      })
+    }
+
+    // Even though headless installation will never update the package.json
+    // this needs to be logged because otherwise install summary won't be printed
+    packageManifestLogger.debug({
+      prefix: rootDir,
+      updated: manifest,
+    })
+  }))
+}
+
 async function linkBinsOfImporter (
   { manifest, modulesDir, binsDir, rootDir }: {
     binsDir: string
@@ -560,6 +618,46 @@ async function linkRootPackages (
   )
 }
 
+async function linkAllPkgsInOrder (
+  storeController: StoreController,
+  graph: DependenciesGraph,
+  hierarchy: DepHierarchy,
+  opts: {
+    force: boolean
+    lockfileDir: string
+    targetEngine?: string
+  }
+) {
+  await Promise.all(
+    Object.entries(hierarchy).map(async ([dir, deps]) => {
+      const depNode = graph[dir]
+      let filesResponse!: PackageFilesResponse
+      try {
+        filesResponse = await depNode.fetchingFiles()
+      } catch (err: any) { // eslint-disable-line
+        if (depNode.optional) return
+        throw err
+      }
+
+      const { importMethod, isBuilt } = await storeController.importPackage(depNode.dir, {
+        filesResponse,
+        force: opts.force,
+        targetEngine: opts.targetEngine,
+      })
+      if (importMethod) {
+        progressLogger.debug({
+          method: importMethod,
+          requester: opts.lockfileDir,
+          status: 'imported',
+          to: depNode.dir,
+        })
+      }
+      depNode.isBuilt = isBuilt
+      return linkAllPkgsInOrder(storeController, graph, deps, opts)
+    })
+  )
+}
+
 const limitLinking = pLimit(16)
 
 async function linkAllPkgs (

packages/headless/src/lockfileToDepGraph.ts

@@ -28,6 +28,7 @@ import equals from 'ramda/src/equals'
 const brokenModulesLogger = logger('_broken_node_modules')
 
 export interface DependenciesGraphNode {
+  alias?: string // this is populated in HoistedDepGraphOnly
   hasBundledDependencies: boolean
   modules: string
   name: string
@@ -69,9 +70,15 @@ export interface DirectDependenciesByImporterId {
   [importerId: string]: { [alias: string]: string }
 }
 
+export interface DepHierarchy {
+  [depPath: string]: Record<string, DepHierarchy>
+}
+
 export interface LockfileToDepGraphResult {
   directDependenciesByImporterId: DirectDependenciesByImporterId
   graph: DependenciesGraph
+  hierarchy?: DepHierarchy
+  symlinkedDirectDependenciesByImporterId?: DirectDependenciesByImporterId
 }
 
 export default async function lockfileToDepGraph (

packages/headless/src/lockfileToHoistedDepGraph.ts

@@ -0,0 +1,197 @@
+import path from 'path'
+import {
+  progressLogger,
+} from '@pnpm/core-loggers'
+import {
+  Lockfile,
+  ProjectSnapshot,
+} from '@pnpm/lockfile-file'
+import {
+  nameVerFromPkgSnapshot,
+  packageIdFromSnapshot,
+  pkgSnapshotToResolution,
+} from '@pnpm/lockfile-utils'
+import { IncludedDependencies } from '@pnpm/modules-yaml'
+import packageIsInstallable from '@pnpm/package-is-installable'
+import { Registries } from '@pnpm/types'
+import {
+  FetchPackageToStoreFunction,
+  StoreController,
+} from '@pnpm/store-controller-types'
+import hoist, { HoisterResult } from '@pnpm/real-hoist'
+import {
+  DependenciesGraph,
+  DepHierarchy,
+  DirectDependenciesByImporterId,
+  LockfileToDepGraphResult,
+} from './lockfileToDepGraph'
+
+export interface LockfileToHoistedDepGraphOptions {
+  engineStrict: boolean
+  force: boolean
+  importerIds: string[]
+  include: IncludedDependencies
+  lockfileDir: string
+  nodeVersion: string
+  pnpmVersion: string
+  registries: Registries
+  sideEffectsCacheRead: boolean
+  skipped: Set<string>
+  storeController: StoreController
+  storeDir: string
+  virtualStoreDir: string
+}
+
+export default async function lockfileToHoistedDepGraph (
+  lockfile: Lockfile,
+  opts: LockfileToHoistedDepGraphOptions
+): Promise<LockfileToDepGraphResult> {
+  const tree = hoist(lockfile)
+  const graph: DependenciesGraph = {}
+  let hierarchy = await fetchDeps(lockfile, opts, graph, path.join(opts.lockfileDir, 'node_modules'), tree.dependencies)
+  const directDependenciesByImporterId: DirectDependenciesByImporterId = {
+    '.': directDepsMap(Object.keys(hierarchy), graph),
+  }
+  const symlinkedDirectDependenciesByImporterId: DirectDependenciesByImporterId = { '.': {} }
+  for (const rootDep of Array.from(tree.dependencies)) {
+    const reference = Array.from(rootDep.references)[0]
+    if (reference.startsWith('workspace:')) {
+      const importerId = reference.replace('workspace:', '')
+      const nextHierarchy = (await fetchDeps(lockfile, opts, graph, path.join(opts.lockfileDir, importerId, 'node_modules'), rootDep.dependencies))
+      hierarchy = {
+        ...hierarchy,
+        ...nextHierarchy,
+      }
+
+      const importer = lockfile.importers[importerId]
+      const importerDir = path.join(opts.lockfileDir, importerId)
+      symlinkedDirectDependenciesByImporterId[importerId] = pickLinkedDirectDeps(importer, importerDir, opts.include)
+      directDependenciesByImporterId[importerId] = directDepsMap(Object.keys(nextHierarchy), graph)
+    }
+  }
+  return { directDependenciesByImporterId, graph, hierarchy, symlinkedDirectDependenciesByImporterId }
+}
+
+function directDepsMap (directDepDirs: string[], graph: DependenciesGraph): Record<string, string> {
+  const result: Record<string, string> = {}
+  for (const dir of directDepDirs) {
+    result[graph[dir].alias!] = dir
+  }
+  return result
+}
+
+function pickLinkedDirectDeps (
+  importer: ProjectSnapshot,
+  importerDir: string,
+  include: IncludedDependencies
+): Record<string, string> {
+  const rootDeps = {
+    ...(include.devDependencies ? importer.devDependencies : {}),
+    ...(include.dependencies ? importer.dependencies : {}),
+    ...(include.optionalDependencies ? importer.optionalDependencies : {}),
+  }
+  const directDeps = {}
+  for (const [alias, ref] of Object.entries(rootDeps)) {
+    if (ref.startsWith('link:')) {
+      directDeps[alias] = path.resolve(importerDir, ref.substr(5))
+    }
+  }
+  return directDeps
+}
+
+async function fetchDeps (
+  lockfile: Lockfile,
+  opts: LockfileToHoistedDepGraphOptions,
+  graph: DependenciesGraph,
+  modules: string,
+  deps: Set<HoisterResult>
+): Promise<DepHierarchy> {
+  const depHierarchy = {}
+  await Promise.all(Array.from(deps).map(async (dep) => {
+    const depPath = Array.from(dep.references)[0]
+    if (opts.skipped.has(depPath) || depPath.startsWith('workspace:')) return
+    const pkgSnapshot = lockfile.packages![depPath]
+    if (!pkgSnapshot) {
+      // it is a link
+      return
+    }
+    const { name: pkgName, version: pkgVersion } = nameVerFromPkgSnapshot(depPath, pkgSnapshot)
+    const packageId = packageIdFromSnapshot(depPath, pkgSnapshot, opts.registries)
+
+    const pkg = {
+      name: pkgName,
+      version: pkgVersion,
+      engines: pkgSnapshot.engines,
+      cpu: pkgSnapshot.cpu,
+      os: pkgSnapshot.os,
+    }
+    if (!opts.force &&
+      packageIsInstallable(packageId, pkg, {
+        engineStrict: opts.engineStrict,
+        lockfileDir: opts.lockfileDir,
+        nodeVersion: opts.nodeVersion,
+        optional: pkgSnapshot.optional === true,
+        pnpmVersion: opts.pnpmVersion,
+      }) === false
+    ) {
+      opts.skipped.add(depPath)
+      return
+    }
+    const dir = path.join(modules, dep.name)
+    const resolution = pkgSnapshotToResolution(depPath, pkgSnapshot, opts.registries)
+    progressLogger.debug({
+      packageId,
+      requester: opts.lockfileDir,
+      status: 'resolved',
+    })
+    let fetchResponse!: ReturnType<FetchPackageToStoreFunction>
+    try {
+      fetchResponse = opts.storeController.fetchPackage({
+        force: false,
+        lockfileDir: opts.lockfileDir,
+        pkg: {
+          name: pkgName,
+          version: pkgVersion,
+          id: packageId,
+          resolution,
+        },
+      })
+      if (fetchResponse instanceof Promise) fetchResponse = await fetchResponse
+    } catch (err: any) { // eslint-disable-line
+      if (pkgSnapshot.optional) return
+      throw err
+    }
+    fetchResponse.files() // eslint-disable-line
+      .then(({ fromStore }) => {
+        progressLogger.debug({
+          packageId,
+          requester: opts.lockfileDir,
+          status: fromStore
+            ? 'found_in_store'
+            : 'fetched',
+        })
+      })
+      .catch(() => {
+        // ignore
+      })
+    graph[dir] = {
+      alias: dep.name,
+      children: {},
+      depPath,
+      dir,
+      fetchingFiles: fetchResponse.files,
+      filesIndexFile: fetchResponse.filesIndexFile,
+      finishing: fetchResponse.finishing,
+      hasBin: pkgSnapshot.hasBin === true,
+      hasBundledDependencies: pkgSnapshot.bundledDependencies != null,
+      modules,
+      name: pkgName,
+      optional: !!pkgSnapshot.optional,
+      optionalDependencies: new Set(Object.keys(pkgSnapshot.optionalDependencies ?? {})),
+      prepare: pkgSnapshot.prepare === true,
+      requiresBuild: pkgSnapshot.requiresBuild === true,
+    }
+    depHierarchy[dir] = await fetchDeps(lockfile, opts, graph, path.join(dir, 'node_modules'), dep.dependencies)
+  }))
+  return depHierarchy
+}

packages/headless/test/fixtures/has-several-versions-of-same-pkg/package.json

@@ -0,0 +1,9 @@
+{
+  "name": "has-several-versions-of-same-pkg",
+  "version": "1.0.0",
+  "dependencies": {
+    "send": "0.17.2",
+    "has-flag": "1.0.0",
+    "ms": "1.0.0"
+  }
+}

packages/headless/test/fixtures/has-several-versions-of-same-pkg/pnpm-lock.yaml

@@ -0,0 +1,134 @@
+lockfileVersion: 5.3
+
+specifiers:
+  has-flag: 1.0.0
+  ms: 1.0.0
+  send: 0.17.2
+
+dependencies:
+  has-flag: 1.0.0
+  ms: 1.0.0
+  send: 0.17.2
+
+packages:
+
+  /debug/2.6.9:
+    resolution: {integrity: sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==}
+    dependencies:
+      ms: 2.0.0
+    dev: false
+
+  /depd/1.1.2:
+    resolution: {integrity: sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=}
+    engines: {node: '>= 0.6'}
+    dev: false
+
+  /destroy/1.0.4:
+    resolution: {integrity: sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=}
+    dev: false
+
+  /ee-first/1.1.1:
+    resolution: {integrity: sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=}
+    dev: false
+
+  /encodeurl/1.0.2:
+    resolution: {integrity: sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=}
+    engines: {node: '>= 0.8'}
+    dev: false
+
+  /escape-html/1.0.3:
+    resolution: {integrity: sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=}
+    dev: false
+
+  /etag/1.8.1:
+    resolution: {integrity: sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=}
+    engines: {node: '>= 0.6'}
+    dev: false
+
+  /fresh/0.5.2:
+    resolution: {integrity: sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=}
+    engines: {node: '>= 0.6'}
+    dev: false
+
+  /has-flag/1.0.0:
+    resolution: {integrity: sha1-nZ55MWXOAXoA8AQYxD+UKnsdEfo=}
+    engines: {node: '>=0.10.0'}
+    dev: false
+
+  /http-errors/1.8.1:
+    resolution: {integrity: sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==}
+    engines: {node: '>= 0.6'}
+    dependencies:
+      depd: 1.1.2
+      inherits: 2.0.4
+      setprototypeof: 1.2.0
+      statuses: 1.5.0
+      toidentifier: 1.0.1
+    dev: false
+
+  /inherits/2.0.4:
+    resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==}
+    dev: false
+
+  /mime/1.6.0:
+    resolution: {integrity: sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==}
+    engines: {node: '>=4'}
+    hasBin: true
+    dev: false
+
+  /ms/1.0.0:
+    resolution: {integrity: sha1-Wa3NIu3FQ/e1OBhi0xOHsfS8lHM=}
+    dev: false
+
+  /ms/2.0.0:
+    resolution: {integrity: sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=}
+    dev: false
+
+  /ms/2.1.3:
+    resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==}
+    dev: false
+
+  /on-finished/2.3.0:
+    resolution: {integrity: sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=}
+    engines: {node: '>= 0.8'}
+    dependencies:
+      ee-first: 1.1.1
+    dev: false
+
+  /range-parser/1.2.1:
+    resolution: {integrity: sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==}
+    engines: {node: '>= 0.6'}
+    dev: false
+
+  /send/0.17.2:
+    resolution: {integrity: sha512-UJYB6wFSJE3G00nEivR5rgWp8c2xXvJ3OPWPhmuteU0IKj8nKbG3DrjiOmLwpnHGYWAVwA69zmTm++YG0Hmwww==}
+    engines: {node: '>= 0.8.0'}
+    dependencies:
+      debug: 2.6.9
+      depd: 1.1.2
+      destroy: 1.0.4
+      encodeurl: 1.0.2
+      escape-html: 1.0.3
+      etag: 1.8.1
+      fresh: 0.5.2
+      http-errors: 1.8.1
+      mime: 1.6.0
+      ms: 2.1.3
+      on-finished: 2.3.0
+      range-parser: 1.2.1
+      statuses: 1.5.0
+    dev: false
+
+  /setprototypeof/1.2.0:
+    resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==}
+    dev: false
+
+  /statuses/1.5.0:
+    resolution: {integrity: sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=}
+    engines: {node: '>= 0.6'}
+    dev: false
+
+  /toidentifier/1.0.1:
+    resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==}
+    engines: {node: '>=0.6'}
+    dev: false

packages/headless/test/fixtures/pnpm-workspace.yaml

@@ -1,3 +1,4 @@
 packages:
   - "**"
   - "!workspace/**"
+  - "!workspace2/**"

packages/headless/test/fixtures/workspace2/bar/package.json

@@ -0,0 +1,9 @@
+{
+  "name": "bar",
+  "version": "1.0.0",
+  "dependencies": {
+    "express": "2",
+    "foo": "workspace:*",
+    "webpack": "5.65.0"
+  }
+}

packages/headless/test/fixtures/workspace2/foo/package.json

@@ -0,0 +1,9 @@
+{
+  "name": "foo",
+  "version": "1.0.0",
+  "dependencies": {
+    "express": "4.17.2",
+    "webpack": "2"
+  }
+}
+

packages/headless/test/index.ts

@@ -1,5 +1,5 @@
 /// <reference path="../../../typings/index.d.ts" />
-import { promises as fs, writeFileSync } from 'fs'
+import { promises as fs, realpathSync, writeFileSync } from 'fs'
 import path from 'path'
 import assertProject from '@pnpm/assert-project'
 import { ENGINE_NAME, WANTED_LOCKFILE } from '@pnpm/constants'
@@ -818,3 +818,51 @@ test('installing with no modules directory', async () => {
 
   expect(await exists(path.join(prefix, 'node_modules'))).toBeFalsy()
 })
+
+test('installing with node-linker=node-modules', async () => {
+  const prefix = f.prepare('has-several-versions-of-same-pkg')
+
+  await headless(await testDefaults({
+    enableModulesDir: false,
+    lockfileDir: prefix,
+    nodeLinker: 'node-modules',
+  }))
+
+  expect(await exists(path.join(prefix, 'node_modules/ms'))).toBeTruthy()
+  expect(await exists(path.join(prefix, 'node_modules/send/node_modules/ms'))).toBeTruthy()
+})
+
+test('installing in a workspace with node-linker=node-modules', async () => {
+  const prefix = f.prepare('workspace2')
+
+  let { projects } = await readprojectsContext(
+    [
+      {
+        rootDir: path.join(prefix, 'foo'),
+      },
+      {
+        rootDir: path.join(prefix, 'bar'),
+      },
+    ],
+    { lockfileDir: prefix }
+  )
+
+  projects = await Promise.all(
+    projects.map(async (project) => ({ ...project, manifest: await readPackageJsonFromDir(project.rootDir) }))
+  )
+  await headless(await testDefaults({
+    lockfileDir: prefix,
+    nodeLinker: 'node-modules',
+    projects,
+  }))
+
+  expect(realpathSync('bar/node_modules/foo')).toBe(path.resolve('foo'))
+  expect(readPkgVersion(path.join(prefix, 'foo/node_modules/webpack'))).toBe('2.7.0')
+  expect(readPkgVersion(path.join(prefix, 'foo/node_modules/express'))).toBe('4.17.2')
+  expect(readPkgVersion(path.join(prefix, 'node_modules/webpack'))).toBe('5.65.0')
+  expect(readPkgVersion(path.join(prefix, 'node_modules/express'))).toBe('2.5.11')
+})
+
+function readPkgVersion (dir: string): string {
+  return loadJsonFile.sync<{ version: string }>(path.join(dir, 'package.json')).version
+}

packages/headless/tsconfig.json

@@ -81,6 +81,9 @@
     {
       "path": "../read-projects-context"
     },
+    {
+      "path": "../real-hoist"
+    },
     {
       "path": "../store-controller-types"
     },

packages/real-hoist/README.md

@@ -0,0 +1,13 @@
+# @pnpm/real-hoist
+
+> Hoists dependencies in a node_modules created by pnpm
+
+## Installation
+
+```
+pnpm add @pnpm/real-hoist
+```
+
+## License
+
+MIT

packages/real-hoist/jest.config.js

@@ -0,0 +1,6 @@
+const config = require('../../jest.config.js')
+
+module.exports = {
+  ...config,
+  testMatch: ["**/test/index.ts"],
+}

packages/real-hoist/package.json

@@ -0,0 +1,42 @@
+{
+  "name": "@pnpm/real-hoist",
+  "description": "Hoists dependencies in a node_modules created by pnpm",
+  "version": "0.0.0",
+  "bugs": {
+    "url": "https://github.com/pnpm/pnpm/issues"
+  },
+  "main": "lib/index.js",
+  "types": "lib/index.d.ts",
+  "files": [
+    "lib",
+    "!*.map"
+  ],
+  "homepage": "https://github.com/pnpm/pnpm/blob/master/packages/real-hoist#readme",
+  "keywords": [
+    "pnpm6",
+    "pnpm"
+  ],
+  "license": "MIT",
+  "engines": {
+    "node": ">=12.17"
+  },
+  "repository": "https://github.com/pnpm/pnpm/blob/master/packages/real-hoist",
+  "scripts": {
+    "start": "pnpm run tsc -- --watch",
+    "_test": "jest",
+    "test": "pnpm run compile && pnpm run _test",
+    "lint": "eslint src/**/*.ts test/**/*.ts",
+    "prepublishOnly": "pnpm run compile",
+    "compile": "rimraf lib tsconfig.tsbuildinfo && tsc --build && pnpm run lint -- --fix"
+  },
+  "dependencies": {
+    "@pnpm/lockfile-utils": "workspace:3.1.5",
+    "@yarnpkg/nm": "^3.0.1-rc.8",
+    "dependency-path": "workspace:8.0.9"
+  },
+  "funding": "https://opencollective.com/pnpm",
+  "devDependencies": {
+    "@pnpm/lockfile-file": "workspace:4.2.5",
+    "@pnpm/logger": "^4.0.0"
+  }
+}

packages/real-hoist/src/index.ts

@@ -0,0 +1,83 @@
+import {
+  Lockfile,
+  nameVerFromPkgSnapshot,
+} from '@pnpm/lockfile-utils'
+import * as dp from 'dependency-path'
+import { hoist, HoisterTree, HoisterResult } from '@yarnpkg/nm/lib/hoist'
+
+export { HoisterResult }
+
+export default function hoistByLockfile (lockfile: Lockfile): HoisterResult {
+  const nodes = new Map<string, HoisterTree>()
+  const node: HoisterTree = {
+    name: '.',
+    identName: '.',
+    reference: '',
+    peerNames: new Set<string>([]),
+    isWorkspace: true,
+    dependencies: toTree(nodes, lockfile, {
+      ...lockfile.importers['.']?.dependencies,
+      ...lockfile.importers['.']?.devDependencies,
+      ...lockfile.importers['.']?.optionalDependencies,
+    }),
+  }
+  for (const [importerId, importer] of Object.entries(lockfile.importers)) {
+    if (importerId === '.') continue
+    const importerNode: HoisterTree = {
+      name: encodeURIComponent(importerId),
+      identName: encodeURIComponent(importerId),
+      reference: `workspace:${importerId}`,
+      peerNames: new Set<string>([]),
+      isWorkspace: true,
+      dependencies: toTree(nodes, lockfile, {
+        ...importer.dependencies,
+        ...importer.devDependencies,
+        ...importer.optionalDependencies,
+      }),
+    }
+    node.dependencies.add(importerNode)
+  }
+
+  return hoist(node)
+}
+
+function toTree (nodes: Map<string, HoisterTree>, lockfile: Lockfile, deps: Record<string, string>): Set<HoisterTree> {
+  return new Set(Object.entries(deps).map(([alias, ref]) => {
+    const depPath = dp.refToRelative(ref, alias)!
+    if (!depPath) {
+      let node = nodes.get(ref)
+      if (!node) {
+        node = {
+          name: alias,
+          identName: alias,
+          reference: ref,
+          isWorkspace: false,
+          dependencies: new Set(),
+          peerNames: new Set(),
+        }
+        nodes.set(depPath, node)
+      }
+      return node
+    }
+    let node = nodes.get(depPath)
+    if (!node) {
+      // const { name, version, peersSuffix } = nameVerFromPkgSnapshot(depPath, lockfile.packages![depPath])
+      const pkgSnapshot = lockfile.packages![depPath]
+      const pkgName = nameVerFromPkgSnapshot(depPath, pkgSnapshot).name
+      node = {
+        name: alias,
+        identName: pkgName,
+        reference: depPath,
+        isWorkspace: false,
+        dependencies: new Set(),
+        peerNames: new Set([
+          ...Object.keys(pkgSnapshot.peerDependencies ?? {}),
+          ...(pkgSnapshot.transitivePeerDependencies ?? []),
+        ]),
+      }
+      nodes.set(depPath, node)
+      node.dependencies = toTree(nodes, lockfile, { ...pkgSnapshot.dependencies, ...pkgSnapshot.optionalDependencies })
+    }
+    return node
+  }))
+}

packages/real-hoist/test/index.ts

@@ -0,0 +1,8 @@
+import path from 'path'
+import hoist from '@pnpm/real-hoist'
+import { readWantedLockfile } from '@pnpm/lockfile-file'
+
+test('hoist', async () => {
+  const lockfile = await readWantedLockfile(path.join(__dirname, '../../..'), { ignoreIncompatible: true })
+  expect(hoist(lockfile!)).toBeTruthy()
+})

packages/real-hoist/tsconfig.json

@@ -0,0 +1,22 @@
+{
+  "extends": "@pnpm/tsconfig",
+  "compilerOptions": {
+    "outDir": "lib",
+    "rootDir": "src"
+  },
+  "include": [
+    "src/**/*.ts",
+    "../../typings/**/*.d.ts"
+  ],
+  "references": [
+    {
+      "path": "../dependency-path"
+    },
+    {
+      "path": "../lockfile-file"
+    },
+    {
+      "path": "../lockfile-utils"
+    }
+  ]
+}

packages/real-hoist/tsconfig.lint.json

@@ -0,0 +1,8 @@
+{
+  "extends": "./tsconfig.json",
+  "include": [
+    "src/**/*.ts",
+    "test/**/*.ts",
+    "../../typings/**/*.d.ts"
+  ]
+}

pnpm-lock.yaml

@@ -1058,6 +1058,7 @@ importers:
       '@pnpm/read-package-json': workspace:5.0.9
       '@pnpm/read-project-manifest': workspace:2.0.10
       '@pnpm/read-projects-context': workspace:5.0.14
+      '@pnpm/real-hoist': workspace:0.0.0
       '@pnpm/store-controller-types': workspace:11.0.10
       '@pnpm/store-path': ^5.0.0
       '@pnpm/symlink-dependency': workspace:4.0.11
@@ -1099,6 +1100,7 @@ importers:
       '@pnpm/package-requester': link:../package-requester
       '@pnpm/read-package-json': link:../read-package-json
       '@pnpm/read-project-manifest': link:../read-project-manifest
+      '@pnpm/real-hoist': link:../real-hoist
       '@pnpm/store-controller-types': link:../store-controller-types
       '@pnpm/symlink-dependency': link:../symlink-dependency
       '@pnpm/types': link:../types
@@ -3007,6 +3009,23 @@ importers:
       '@pnpm/logger': 4.0.0
       '@pnpm/read-projects-context': 'link:'
 
+  packages/real-hoist:
+    specifiers:
+      '@pnpm/lockfile-file': workspace:4.2.5
+      '@pnpm/lockfile-utils': workspace:3.1.5
+      '@pnpm/logger': ^4.0.0
+      '@pnpm/real-hoist': 'link:'
+      '@yarnpkg/nm': ^3.0.1-rc.8
+      dependency-path: workspace:8.0.9
+    dependencies:
+      '@pnpm/lockfile-utils': link:../lockfile-utils
+      '@yarnpkg/nm': 3.0.1-rc.8
+      dependency-path: link:../dependency-path
+    devDependencies:
+      '@pnpm/lockfile-file': link:../lockfile-file
+      '@pnpm/logger': 4.0.0
+      '@pnpm/real-hoist': 'link:'
+
   packages/remove-bins:
     specifiers:
       '@pnpm/core-loggers': workspace:6.1.2
@@ -5727,6 +5746,14 @@ packages:
     resolution: {integrity: sha512-GpSwvyXOcOOlV70vbnzjj4fW5xW/FdUF6nQEt1ENy7m4ZCczi1+/buVUPAqmGfqznsORNFzUMjctTIp8a9tuCQ==}
     dev: false
 
+  /@yarnpkg/nm/3.0.1-rc.8:
+    resolution: {integrity: sha512-EuYgE1UGieFL3Mm0OIVnOtKcTqTdRZl2jRsr6/ZUxsTj0pkC7rDVeD1qGCfYWS7HjztRnpyBM73rODif33nRHA==}
+    engines: {node: '>=12 <14 || 14.2 - 14.9 || >14.10.0'}
+    dependencies:
+      '@yarnpkg/core': 3.2.0-rc.8
+      '@yarnpkg/fslib': 2.6.1-rc.3
+    dev: false
+
   /@yarnpkg/parsers/2.3.0:
     resolution: {integrity: sha512-qgz0QUgOvnhtF92kaluIhIIKBUHlYlHUBQxqh5v9+sxEQvUeF6G6PKiFlzo3E6O99XwvNEGpVu1xZPoSGyGscQ==}
     engines: {node: '>=10.19.0'}