feat: dedupe direct dependencies (#5676)

.changeset/five-pets-fail.md

@@ -0,0 +1,7 @@
+---
+"@pnpm/pkg-manager.direct-dep-linker": major
+"@pnpm/core": minor
+"@pnpm/headless": minor
+---
+
+New setting added for deduping direct dependencies: dedupeDirectDeps [#5676](https://github.com/pnpm/pnpm/pull/5676).

pkg-manager/core/src/install/extendInstallOptions.ts

@@ -117,6 +117,7 @@ export interface StrictInstallOptions {
 
   allProjects: ProjectOptions[]
   resolveSymlinksInInjectedDirs: boolean
+  dedupeDirectDeps: boolean
 }
 
 export type InstallOptions =
@@ -201,6 +202,7 @@ const defaults = async (opts: InstallOptions) => {
     enableModulesDir: true,
     modulesCacheMaxAge: 7 * 24 * 60,
     resolveSymlinksInInjectedDirs: false,
+    dedupeDirectDeps: false,
   } as StrictInstallOptions
 }
 

pkg-manager/core/src/install/index.ts

@@ -886,6 +886,7 @@ const _installInContext: InstallFunction = async (projects, ctx, opts) => {
       dependenciesGraph,
       {
         currentLockfile: ctx.currentLockfile,
+        dedupeDirectDeps: opts.dedupeDirectDeps,
         dependenciesByProjectId,
         depsStateCache,
         extraNodePaths: ctx.extraNodePaths,

pkg-manager/core/src/install/link.ts

@@ -45,6 +45,7 @@ export async function linkPackages (
   depGraph: DependenciesGraph,
   opts: {
     currentLockfile: Lockfile
+    dedupeDirectDeps: boolean
     dependenciesByProjectId: {
       [id: string]: { [alias: string]: string }
     }
@@ -160,11 +161,11 @@ export async function linkPackages (
   })
 
   if (opts.symlink) {
-    const projectsToLink = await Promise.all(
+    const projectsToLink = fromPairs(await Promise.all(
       projects.map(async ({ id, manifest, modulesDir, rootDir }) => {
         const deps = opts.dependenciesByProjectId[id]
         const importerFromLockfile = newCurrentLockfile.importers[id]
-        return {
+        return [id, {
           dir: rootDir,
           modulesDir,
           dependencies: await Promise.all([
@@ -199,10 +200,10 @@ export async function linkPackages (
               }
             }),
           ]),
-        }
-      })
+        }]
+      }))
     )
-    await linkDirectDeps(projectsToLink)
+    await linkDirectDeps(projectsToLink, { dedupe: opts.dedupeDirectDeps })
   }
 
   let currentLockfile: Lockfile

pkg-manager/core/test/install/dedupeDirectDeps.ts

@@ -0,0 +1,95 @@
+import fs from 'fs'
+import path from 'path'
+import { preparePackages } from '@pnpm/prepare'
+import { mutateModules, MutatedProject } from '@pnpm/core'
+import { testDefaults } from '../utils'
+
+test('dedupe direct dependencies', async () => {
+  const projects = preparePackages([
+    {
+      location: '',
+      package: { name: 'project-1' },
+    },
+    {
+      location: 'project-2',
+      package: { name: 'project-2' },
+    },
+    {
+      location: 'project-3',
+      package: { name: 'project-3' },
+    },
+  ])
+
+  const importers: MutatedProject[] = [
+    {
+      mutation: 'install',
+      rootDir: process.cwd(),
+    },
+    {
+      mutation: 'install',
+      rootDir: path.resolve('project-2'),
+    },
+    {
+      mutation: 'install',
+      rootDir: path.resolve('project-3'),
+    },
+  ]
+  const allProjects = [
+    {
+      buildIndex: 0,
+      manifest: {
+        name: 'project-1',
+        version: '1.0.0',
+
+        dependencies: {
+          'is-positive': '1.0.0',
+          'is-odd': '1.0.0',
+        },
+      },
+      rootDir: process.cwd(),
+    },
+    {
+      buildIndex: 0,
+      manifest: {
+        name: 'project-2',
+        version: '1.0.0',
+
+        dependencies: {
+          'is-negative': '1.0.0',
+        },
+      },
+      rootDir: path.resolve('project-2'),
+    },
+    {
+      buildIndex: 0,
+      manifest: {
+        name: 'project-3',
+        version: '1.0.0',
+
+        dependencies: {
+          'is-negative': '1.0.0',
+        },
+      },
+      rootDir: path.resolve('project-3'),
+    },
+  ]
+  await mutateModules(importers, await testDefaults({ allProjects, dedupeDirectDeps: true }))
+  await projects['project-2'].has('is-negative')
+  await projects['project-3'].has('is-negative')
+
+  allProjects[0].manifest.dependencies['is-negative'] = '1.0.0'
+  allProjects[1].manifest.dependencies['is-positive'] = '1.0.0'
+  allProjects[1].manifest.dependencies['is-odd'] = '2.0.0'
+  await mutateModules(importers, await testDefaults({ allProjects, dedupeDirectDeps: true }))
+
+  expect(Array.from(fs.readdirSync('node_modules').sort())).toEqual([
+    '.modules.yaml',
+    '.pnpm',
+    'is-negative',
+    'is-odd',
+    'is-positive',
+  ])
+  expect(fs.readdirSync('project-2/node_modules').sort()).toEqual(['is-odd'])
+  await projects['project-3'].hasNot('is-negative')
+  expect(fs.existsSync('project-3/node_modules')).toBeFalsy()
+})

pkg-manager/direct-dep-linker/package.json

@@ -15,7 +15,8 @@
     "@pnpm/logger": "^5.0.0"
   },
   "devDependencies": {
-    "@pnpm/pkg-manager.direct-dep-linker": "workspace:*"
+    "@pnpm/pkg-manager.direct-dep-linker": "workspace:*",
+    "@types/ramda": "0.28.15"
   },
   "homepage": "https://github.com/pnpm/pnpm/blob/main/pkg-manager/direct-dep-linker#readme",
   "keywords": [
@@ -35,7 +36,11 @@
   },
   "dependencies": {
     "@pnpm/core-loggers": "workspace:*",
-    "@pnpm/symlink-dependency": "workspace:*"
+    "@pnpm/read-modules-dir": "workspace:*",
+    "@pnpm/symlink-dependency": "workspace:*",
+    "@zkochan/rimraf": "^2.1.2",
+    "ramda": "npm:@pnpm/ramda@0.28.1",
+    "resolve-link-target": "^2.0.0"
   },
   "funding": "https://opencollective.com/pnpm",
   "exports": {

pkg-manager/direct-dep-linker/src/linkDirectDeps.ts

@@ -1,5 +1,11 @@
+import fs from 'fs'
+import path from 'path'
 import { rootLogger } from '@pnpm/core-loggers'
 import { symlinkDependency, symlinkDirectRootDependency } from '@pnpm/symlink-dependency'
+import omit from 'ramda/src/omit'
+import { readModulesDir } from '@pnpm/read-modules-dir'
+import rimraf from '@zkochan/rimraf'
+import resolveLinkTarget from 'resolve-link-target'
 
 export interface LinkedDirectDep {
   alias: string
@@ -19,37 +25,106 @@ export interface ProjectToLink {
 }
 
 export async function linkDirectDeps (
-  projects: ProjectToLink[]
+  projects: Record<string, ProjectToLink>,
+  opts: {
+    dedupe: boolean
+  }
 ) {
-  await Promise.all(projects.map(async (project) => {
-    await Promise.all(project.dependencies.map(async (dep) => {
-      if (dep.isExternalLink) {
-        await symlinkDirectRootDependency(dep.dir, project.modulesDir, dep.alias, {
-          fromDependenciesField: dep.dependencyType === 'dev' && 'devDependencies' ||
-            dep.dependencyType === 'optional' && 'optionalDependencies' ||
-            'dependencies',
-          linkedPackage: {
-            name: dep.name,
-            version: dep.version,
-          },
-          prefix: project.dir,
+  if (opts.dedupe && projects['.'] && Object.keys(projects).length > 1) {
+    return linkDirectDepsAndDedupe(projects['.'], omit(['.'], projects))
+  }
+  await Promise.all(Object.values(projects).map(linkDirectDepsOfProject))
+}
+
+async function linkDirectDepsAndDedupe (
+  rootProject: ProjectToLink,
+  projects: Record<string, ProjectToLink>
+) {
+  await linkDirectDepsOfProject(rootProject)
+  const pkgsLinkedToRoot = await readLinkedDeps(rootProject.modulesDir)
+  await Promise.all(
+    Object.values(projects).map(async (project) => {
+      const deletedAll = await deletePkgsPresentInRoot(project.modulesDir, pkgsLinkedToRoot)
+      const dependencies = omitDepsFromRoot(project.dependencies, pkgsLinkedToRoot)
+      if (dependencies.length > 0) {
+        await linkDirectDepsOfProject({
+          ...project,
+          dependencies,
         })
         return
       }
-      if ((await symlinkDependency(dep.dir, project.modulesDir, dep.alias)).reused) {
-        return
+      if (deletedAll) {
+        await rimraf(project.modulesDir)
       }
-      rootLogger.debug({
-        added: {
-          dependencyType: dep.dependencyType,
-          id: dep.id,
-          latest: dep.latest,
-          name: dep.alias,
-          realName: dep.name,
+    })
+  )
+}
+
+function omitDepsFromRoot (deps: LinkedDirectDep[], pkgsLinkedToRoot: string[]) {
+  return deps.filter(({ dir }) => !pkgsLinkedToRoot.some(pathsEqual.bind(null, dir)))
+}
+
+function pathsEqual (path1: string, path2: string) {
+  return path.relative(path1, path2) === ''
+}
+
+async function readLinkedDeps (modulesDir: string): Promise<string[]> {
+  const deps = (await readModulesDir(modulesDir)) ?? []
+  return Promise.all(
+    deps.map((alias) => resolveLinkTarget(path.join(modulesDir, alias)))
+  )
+}
+
+async function deletePkgsPresentInRoot (
+  modulesDir: string,
+  pkgsLinkedToRoot: string[]
+): Promise<boolean> {
+  const pkgsLinkedToCurrentProject = await readLinkedDepsWithRealLocations(modulesDir)
+  const pkgsToDelete = pkgsLinkedToCurrentProject
+    .filter(({ linkedFrom }) => pkgsLinkedToRoot.some(pathsEqual.bind(null, linkedFrom)))
+  await Promise.all(pkgsToDelete.map(({ linkedTo }) => fs.promises.unlink(linkedTo)))
+  return pkgsToDelete.length === pkgsLinkedToCurrentProject.length
+}
+
+async function readLinkedDepsWithRealLocations (modulesDir: string) {
+  const deps = (await readModulesDir(modulesDir)) ?? []
+  return Promise.all(deps.map(async (alias) => {
+    const linkedTo = path.join(modulesDir, alias)
+    return {
+      linkedTo,
+      linkedFrom: await resolveLinkTarget(linkedTo),
+    }
+  }))
+}
+
+async function linkDirectDepsOfProject (project: ProjectToLink) {
+  await Promise.all(project.dependencies.map(async (dep) => {
+    if (dep.isExternalLink) {
+      await symlinkDirectRootDependency(dep.dir, project.modulesDir, dep.alias, {
+        fromDependenciesField: dep.dependencyType === 'dev' && 'devDependencies' ||
+          dep.dependencyType === 'optional' && 'optionalDependencies' ||
+          'dependencies',
+        linkedPackage: {
+          name: dep.name,
           version: dep.version,
         },
         prefix: project.dir,
       })
-    }))
+      return
+    }
+    if ((await symlinkDependency(dep.dir, project.modulesDir, dep.alias)).reused) {
+      return
+    }
+    rootLogger.debug({
+      added: {
+        dependencyType: dep.dependencyType,
+        id: dep.id,
+        latest: dep.latest,
+        name: dep.alias,
+        realName: dep.name,
+        version: dep.version,
+      },
+      prefix: project.dir,
+    })
   }))
 }

pkg-manager/direct-dep-linker/tsconfig.json

@@ -9,6 +9,9 @@
     "../../__typings__/**/*.d.ts"
   ],
   "references": [
+    {
+      "path": "../../fs/read-modules-dir"
+    },
     {
       "path": "../../fs/symlink-dependency"
     },

pkg-manager/headless/src/index.ts

@@ -101,6 +101,7 @@ export interface HeadlessOptions {
     nodeVersion: string
     pnpmVersion: string
   }
+  dedupeDirectDeps?: boolean
   enablePnp?: boolean
   engineStrict: boolean
   extraBinPaths?: string[]
@@ -339,6 +340,7 @@ export async function headlessInstall (opts: HeadlessOptions) {
 
     await symlinkDirectDependencies({
       directDependenciesByImporterId: symlinkedDirectDependenciesByImporterId!,
+      dedupe: opts.dedupeDirectDeps,
       filteredLockfile,
       lockfileDir,
       projects: selectedProjects,
@@ -562,6 +564,7 @@ export async function headlessInstall (opts: HeadlessOptions) {
 
 type SymlinkDirectDependenciesOpts = Pick<HeadlessOptions, 'registries' | 'symlink' | 'lockfileDir'> & {
   filteredLockfile: Lockfile
+  dedupe?: boolean
   directDependenciesByImporterId: DirectDependenciesByImporterId
   projects: Project[]
 }
@@ -569,6 +572,7 @@ type SymlinkDirectDependenciesOpts = Pick<HeadlessOptions, 'registries' | 'symli
 async function symlinkDirectDependencies (
   {
     filteredLockfile,
+    dedupe,
     directDependenciesByImporterId,
     lockfileDir,
     projects,
@@ -585,8 +589,8 @@ async function symlinkDirectDependencies (
     })
   })
   if (symlink !== false) {
-    const projectsToLink = await Promise.all(
-      projects.map(async ({ rootDir, id, modulesDir }) => ({
+    const projectsToLink = fromPairs(await Promise.all(
+      projects.map(async ({ rootDir, id, modulesDir }) => ([id, {
         dir: rootDir,
         modulesDir,
         dependencies: await getRootPackagesToLink(filteredLockfile, {
@@ -598,9 +602,9 @@ async function symlinkDirectDependencies (
           registries,
           rootDependencies: directDependenciesByImporterId[id],
         }),
-      }))
-    )
-    await linkDirectDeps(projectsToLink)
+      }]))
+    ))
+    await linkDirectDeps(projectsToLink, { dedupe: Boolean(dedupe) })
   }
 }
 

pnpm-lock.yaml

@@ -2723,13 +2723,28 @@ importers:
       '@pnpm/logger':
         specifier: ^5.0.0
         version: 5.0.0
+      '@pnpm/read-modules-dir':
+        specifier: workspace:*
+        version: link:../../fs/read-modules-dir
       '@pnpm/symlink-dependency':
         specifier: workspace:*
         version: link:../../fs/symlink-dependency
+      '@zkochan/rimraf':
+        specifier: ^2.1.2
+        version: 2.1.2
+      ramda:
+        specifier: npm:@pnpm/ramda@0.28.1
+        version: /@pnpm/ramda/0.28.1
+      resolve-link-target:
+        specifier: ^2.0.0
+        version: 2.0.0
     devDependencies:
       '@pnpm/pkg-manager.direct-dep-linker':
         specifier: workspace:*
         version: 'link:'
+      '@types/ramda':
+        specifier: 0.28.15
+        version: 0.28.15
 
   pkg-manager/get-context:
     dependencies: