feat: add hoist-workspace-packages option (#7451)

---------

Co-authored-by: Zoltan Kochan <z@kochan.io>

.changeset/short-hotels-smell.md

@@ -0,0 +1,9 @@
+---
+"@pnpm/headless": minor
+"@pnpm/hoist": minor
+"@pnpm/core": minor
+"@pnpm/config": minor
+"pnpm": minor
+---
+
+A new option added for hoisting packages from the workspace. When `hoist-workspace-packages` is set to `true`, packages from the workspace are symlinked to either `<workspace_root>/node_modules/.pnpm/node_modules` or to `<workspace_root>/node_modules` depending on other hoisting settings (`hoist-pattern` and `public-hoist-pattern`) [#7451](https://github.com/pnpm/pnpm/pull/7451).

config/config/src/Config.ts

@@ -127,6 +127,7 @@ export interface Config {
   packageImportMethod?: 'auto' | 'hardlink' | 'copy' | 'clone' | 'clone-or-copy'
   hoistPattern?: string[]
   publicHoistPattern?: string[] | string
+  hoistWorkspacePackages?: boolean
   useStoreServer?: boolean
   useRunningStoreServer?: boolean
   workspaceConcurrency: number

config/config/src/index.ts

@@ -74,6 +74,7 @@ export const types = Object.assign({
   'git-branch-lockfile': Boolean,
   hoist: Boolean,
   'hoist-pattern': Array,
+  'hoist-workspace-packages': Boolean,
   'ignore-compatibility-db': Boolean,
   'ignore-dep-scripts': Boolean,
   'ignore-pnpmfile': Boolean,
@@ -229,6 +230,7 @@ export async function getConfig (
     'git-branch-lockfile': false,
     hoist: true,
     'hoist-pattern': ['*'],
+    'hoist-workspace-packages': false,
     'ignore-workspace-cycles': false,
     'ignore-workspace-root-check': false,
     'link-workspace-packages': true,

pkg-manager/core/src/install/extendInstallOptions.ts

@@ -140,6 +140,7 @@ export interface StrictInstallOptions {
   disableRelinkLocalDirDeps: boolean
 
   supportedArchitectures?: SupportedArchitectures
+  hoistWorkspacePackages?: boolean
 }
 
 export type InstallOptions =

pkg-manager/core/src/install/index.ts

@@ -895,6 +895,7 @@ type InstallFunction = (
     pruneVirtualStore: boolean
     scriptsOpts: RunLifecycleHooksConcurrentlyOptions
     currentLockfileIsUpToDate: boolean
+    hoistWorkspacePackages?: boolean
   }
 ) => Promise<InstallFunctionResult>
 
@@ -1121,6 +1122,7 @@ const _installInContext: InstallFunction = async (projects, ctx, opts) => {
         virtualStoreDir: ctx.virtualStoreDir,
         wantedLockfile: newLockfile,
         wantedToBeSkippedPackageIds,
+        hoistWorkspacePackages: opts.hoistWorkspacePackages,
       }
     )
     stats = result.stats
@@ -1388,6 +1390,7 @@ const installInContext: InstallFunction = async (projects, ctx, opts) => {
         prunedAt: ctx.modulesFile?.prunedAt,
         wantedLockfile: result.newLockfile,
         useLockfile: opts.useLockfile && ctx.wantedLockfileIsModified,
+        hoistWorkspacePackages: opts.hoistWorkspacePackages,
       })
       return {
         ...result,

pkg-manager/core/src/install/link.ts

@@ -11,7 +11,7 @@ import {
 } from '@pnpm/filter-lockfile'
 import { linkDirectDeps } from '@pnpm/pkg-manager.direct-dep-linker'
 import { type InstallationResultStats } from '@pnpm/headless'
-import { hoist } from '@pnpm/hoist'
+import { hoist, type HoistedWorkspaceProject } from '@pnpm/hoist'
 import { type Lockfile } from '@pnpm/lockfile-file'
 import { logger } from '@pnpm/logger'
 import { prune } from '@pnpm/modules-cleaner'
@@ -74,6 +74,7 @@ export async function linkPackages (
     virtualStoreDir: string
     wantedLockfile: Lockfile
     wantedToBeSkippedPackageIds: Set<string>
+    hoistWorkspacePackages?: boolean
   }
 ): Promise<{
     currentLockfile: Lockfile
@@ -225,6 +226,17 @@ export async function linkPackages (
       publicHoistedModulesDir: opts.rootModulesDir,
       publicHoistPattern: opts.publicHoistPattern ?? [],
       virtualStoreDir: opts.virtualStoreDir,
+      hoistedWorkspacePackages: opts.hoistWorkspacePackages
+        ? projects.reduce((hoistedWorkspacePackages, project) => {
+          if (project.manifest.name) {
+            hoistedWorkspacePackages[project.id] = {
+              dir: project.rootDir,
+              name: project.manifest.name,
+            }
+          }
+          return hoistedWorkspacePackages
+        }, {} as Record<string, HoistedWorkspaceProject>)
+        : undefined,
     })
   } else {
     newHoistedDependencies = opts.hoistedDependencies

pkg-manager/core/test/install/hoist.ts

@@ -803,3 +803,122 @@ test('should not add extra node paths to command shims, when extend-node-path is
   console.log(cmdShim)
   expect(cmdShim).not.toContain('node_modules/.pnpm/node_modules')
 })
+
+test('hoistWorkspacePackages should hoist all workspace projects', async () => {
+  const workspaceRootManifest = {
+    name: 'root',
+
+    dependencies: {
+      '@pnpm.e2e/pkg-with-1-dep': '100.0.0',
+    },
+  }
+  const workspacePackageManifest = {
+    name: 'package',
+    version: '1.0.0',
+
+    dependencies: {
+      '@pnpm.e2e/foobar': '100.0.0',
+    },
+  }
+  const workspacePackageManifest2 = {
+    name: 'package2',
+    version: '1.0.0',
+
+    dependencies: {
+      package: 'workspace:*',
+    },
+  }
+
+  const projects = preparePackages([
+    {
+      location: '.',
+      package: workspaceRootManifest,
+    },
+    {
+      location: 'package',
+      package: workspacePackageManifest,
+    },
+    {
+      location: 'package2',
+      package: workspacePackageManifest2,
+    },
+  ])
+
+  const mutatedProjects: MutatedProject[] = [
+    {
+      mutation: 'install',
+      rootDir: process.cwd(),
+    },
+    {
+      mutation: 'install',
+      rootDir: path.resolve('package'),
+    },
+    {
+      mutation: 'install',
+      rootDir: path.resolve('package2'),
+    },
+  ]
+  const allProjects = [
+    {
+      buildIndex: 0,
+      manifest: workspaceRootManifest,
+      rootDir: process.cwd(),
+    },
+    {
+      buildIndex: 0,
+      manifest: workspacePackageManifest,
+      rootDir: path.resolve('package'),
+    },
+    {
+      buildIndex: 0,
+      manifest: workspacePackageManifest2,
+      rootDir: path.resolve('package2'),
+    },
+  ]
+  const workspacePackages = {
+    [workspacePackageManifest.name]: {
+      [workspacePackageManifest.version]: {
+        dir: path.resolve('package'),
+        manifest: workspacePackageManifest,
+      },
+    },
+    [workspacePackageManifest2.name]: {
+      [workspacePackageManifest2.version]: {
+        dir: path.resolve('package2'),
+        manifest: workspacePackageManifest2,
+      },
+    },
+  }
+  await mutateModules(mutatedProjects, await testDefaults({
+    allProjects,
+    hoistPattern: '*',
+    hoistWorkspacePackages: true,
+    workspacePackages,
+  }))
+
+  await projects['root'].has('@pnpm.e2e/pkg-with-1-dep')
+  await projects['root'].has('.pnpm/node_modules/@pnpm.e2e/dep-of-pkg-with-1-dep')
+  await projects['root'].has('.pnpm/node_modules/@pnpm.e2e/foobar')
+  await projects['root'].has('.pnpm/node_modules/@pnpm.e2e/foo')
+  await projects['root'].has('.pnpm/node_modules/@pnpm.e2e/bar')
+  await projects['root'].has('.pnpm/node_modules/package')
+  await projects['root'].has('.pnpm/node_modules/package2')
+  await projects['root'].hasNot('@pnpm.e2e/foobar')
+  await projects['root'].hasNot('@pnpm.e2e/foo')
+  await projects['root'].hasNot('@pnpm.e2e/bar')
+
+  await projects['package'].has('@pnpm.e2e/foobar')
+  await projects['package'].hasNot('@pnpm.e2e/foo')
+  await projects['package'].hasNot('@pnpm.e2e/bar')
+
+  await rimraf('node_modules')
+  await mutateModules(mutatedProjects, await testDefaults({
+    allProjects,
+    frozenLockfile: true,
+    hoistPattern: '*',
+    hoistWorkspacePackages: true,
+    workspacePackages,
+  }))
+  await projects['root'].has('.pnpm/node_modules/package')
+  await projects['root'].has('.pnpm/node_modules/package2')
+})

pkg-manager/headless/src/index.ts

@@ -18,7 +18,7 @@ import {
   filterLockfileByEngine,
   filterLockfileByImportersAndEngine,
 } from '@pnpm/filter-lockfile'
-import { hoist } from '@pnpm/hoist'
+import { hoist, type HoistedWorkspaceProject } from '@pnpm/hoist'
 import {
   runLifecycleHooksConcurrently,
   makeNodeRequireOption,
@@ -161,6 +161,7 @@ export interface HeadlessOptions {
   useGitBranchLockfile?: boolean
   useLockfile?: boolean
   supportedArchitectures?: SupportedArchitectures
+  hoistWorkspacePackages?: boolean
 }
 
 export interface InstallationResultStats {
@@ -424,6 +425,17 @@ export async function headlessInstall (opts: HeadlessOptions): Promise<Installat
         publicHoistedModulesDir,
         publicHoistPattern: opts.publicHoistPattern ?? [],
         virtualStoreDir,
+        hoistedWorkspacePackages: opts.hoistWorkspacePackages
+          ? Object.values(opts.allProjects).reduce((hoistedWorkspacePackages, project) => {
+            if (project.manifest.name) {
+              hoistedWorkspacePackages[project.id] = {
+                dir: project.rootDir,
+                name: project.manifest.name,
+              }
+            }
+            return hoistedWorkspacePackages
+          }, {} as Record<string, HoistedWorkspaceProject>)
+          : undefined,
       })
     } else {
       newHoistedDependencies = {}

pkg-manager/hoist/src/index.ts

@@ -36,6 +36,7 @@ export async function hoist (opts: HoistOpts) {
     privateHoistedModulesDir: opts.privateHoistedModulesDir,
     publicHoistedModulesDir: opts.publicHoistedModulesDir,
     virtualStoreDir: opts.virtualStoreDir,
+    hoistedWorkspacePackages: opts.hoistedWorkspacePackages,
   })
 
   // Here we only link the bins of the privately hoisted modules.
@@ -59,24 +60,41 @@ export interface GetHoistedDependenciesOpts {
   privateHoistedModulesDir: string
   publicHoistPattern: string[]
   publicHoistedModulesDir: string
+  hoistedWorkspacePackages?: Record<string, HoistedWorkspaceProject>
 }
 
-export function getHoistedDependencies (opts: GetHoistedDependenciesOpts) {
+export interface HoistedWorkspaceProject {
+  name: string
+  dir: string
+}
+
+export function getHoistedDependencies (opts: GetHoistedDependenciesOpts): HoistGraphResult | null {
   if (opts.lockfile.packages == null) return null
 
   const { directDeps, step } = lockfileWalker(
     opts.lockfile,
     opts.importerIds ?? Object.keys(opts.lockfile.importers)
   )
+  // We want to hoist all the workspace packages, not only those that are in the dependencies
+  // of any other workspace packages.
+  // That is why we can't just simply use the lockfile walker to include links to local workspace packages too.
+  // We have to explicitly include all the workspace packages.
+  const hoistedWorkspaceDeps = Object.fromEntries(
+    Object.entries(opts.hoistedWorkspacePackages ?? {})
+      .map(([id, { name }]) => [name, id])
+  )
   const deps = [
     {
-      children: directDeps
-        .reduce((acc, { alias, depPath }) => {
-          if (!acc[alias]) {
-            acc[alias] = depPath
-          }
-          return acc
-        }, {} as Record<string, string>),
+      children: {
+        ...hoistedWorkspaceDeps,
+        ...directDeps
+          .reduce((acc, { alias, depPath }) => {
+            if (!acc[alias]) {
+              acc[alias] = depPath
+            }
+            return acc
+          }, {} as Record<string, string>),
+      },
       depPath: '',
       depth: -1,
     },
@@ -226,21 +244,27 @@ async function symlinkHoistedDependencies (
     privateHoistedModulesDir: string
     publicHoistedModulesDir: string
     virtualStoreDir: string
+    hoistedWorkspacePackages?: Record<string, HoistedWorkspaceProject>
   }
 ) {
   const symlink = symlinkHoistedDependency.bind(null, opts)
   await Promise.all(
     Object.entries(hoistedDependencies)
-      .map(async ([depPath, pkgAliases]) => {
-        const pkgSnapshot = opts.lockfile.packages![depPath]
-        if (!pkgSnapshot) {
-          // This dependency is probably a skipped optional dependency.
-          hoistLogger.debug({ hoistFailedFor: depPath })
-          return
+      .map(async ([hoistedDepId, pkgAliases]) => {
+        const pkgSnapshot = opts.lockfile.packages![hoistedDepId]
+        let depLocation!: string
+        if (pkgSnapshot) {
+          const pkgName = nameVerFromPkgSnapshot(hoistedDepId, pkgSnapshot).name
+          const modules = path.join(opts.virtualStoreDir, dp.depPathToFilename(hoistedDepId), 'node_modules')
+          depLocation = path.join(modules, pkgName as string)
+        } else {
+          if (!opts.lockfile.importers[hoistedDepId]) {
+            // This dependency is probably a skipped optional dependency.
+            hoistLogger.debug({ hoistFailedFor: hoistedDepId })
+            return
+          }
+          depLocation = opts.hoistedWorkspacePackages![hoistedDepId].dir
         }
-        const pkgName = nameVerFromPkgSnapshot(depPath, pkgSnapshot).name
-        const modules = path.join(opts.virtualStoreDir, dp.depPathToFilename(depPath), 'node_modules')
-        const depLocation = path.join(modules, pkgName)
         await Promise.all(Object.entries(pkgAliases).map(async ([pkgAlias, hoistType]) => {
           const targetDir = hoistType === 'public'
             ? opts.publicHoistedModulesDir