diff --git a/.changeset/five-pets-fail.md b/.changeset/five-pets-fail.md new file mode 100644 index 0000000000..103048be4e --- /dev/null +++ b/.changeset/five-pets-fail.md @@ -0,0 +1,7 @@ +--- +"@pnpm/pkg-manager.direct-dep-linker": major +"@pnpm/core": minor +"@pnpm/headless": minor +--- + +New setting added for deduping direct dependencies: dedupeDirectDeps [#5676](https://github.com/pnpm/pnpm/pull/5676). diff --git a/pkg-manager/core/src/install/extendInstallOptions.ts b/pkg-manager/core/src/install/extendInstallOptions.ts index 5cd892739a..412fe61d50 100644 --- a/pkg-manager/core/src/install/extendInstallOptions.ts +++ b/pkg-manager/core/src/install/extendInstallOptions.ts @@ -117,6 +117,7 @@ export interface StrictInstallOptions { allProjects: ProjectOptions[] resolveSymlinksInInjectedDirs: boolean + dedupeDirectDeps: boolean } export type InstallOptions = @@ -201,6 +202,7 @@ const defaults = async (opts: InstallOptions) => { enableModulesDir: true, modulesCacheMaxAge: 7 * 24 * 60, resolveSymlinksInInjectedDirs: false, + dedupeDirectDeps: false, } as StrictInstallOptions } diff --git a/pkg-manager/core/src/install/index.ts b/pkg-manager/core/src/install/index.ts index fb162eb1bd..56938387bc 100644 --- a/pkg-manager/core/src/install/index.ts +++ b/pkg-manager/core/src/install/index.ts @@ -886,6 +886,7 @@ const _installInContext: InstallFunction = async (projects, ctx, opts) => { dependenciesGraph, { currentLockfile: ctx.currentLockfile, + dedupeDirectDeps: opts.dedupeDirectDeps, dependenciesByProjectId, depsStateCache, extraNodePaths: ctx.extraNodePaths, diff --git a/pkg-manager/core/src/install/link.ts b/pkg-manager/core/src/install/link.ts index 1819972c40..63ccf8fdc8 100644 --- a/pkg-manager/core/src/install/link.ts +++ b/pkg-manager/core/src/install/link.ts @@ -45,6 +45,7 @@ export async function linkPackages ( depGraph: DependenciesGraph, opts: { currentLockfile: Lockfile + dedupeDirectDeps: boolean dependenciesByProjectId: { [id: string]: { [alias: string]: string } } @@ -160,11 +161,11 @@ export async function linkPackages ( }) if (opts.symlink) { - const projectsToLink = await Promise.all( + const projectsToLink = fromPairs(await Promise.all( projects.map(async ({ id, manifest, modulesDir, rootDir }) => { const deps = opts.dependenciesByProjectId[id] const importerFromLockfile = newCurrentLockfile.importers[id] - return { + return [id, { dir: rootDir, modulesDir, dependencies: await Promise.all([ @@ -199,10 +200,10 @@ export async function linkPackages ( } }), ]), - } - }) + }] + })) ) - await linkDirectDeps(projectsToLink) + await linkDirectDeps(projectsToLink, { dedupe: opts.dedupeDirectDeps }) } let currentLockfile: Lockfile diff --git a/pkg-manager/core/test/install/dedupeDirectDeps.ts b/pkg-manager/core/test/install/dedupeDirectDeps.ts new file mode 100644 index 0000000000..5ea2a58dc1 --- /dev/null +++ b/pkg-manager/core/test/install/dedupeDirectDeps.ts @@ -0,0 +1,95 @@ +import fs from 'fs' +import path from 'path' +import { preparePackages } from '@pnpm/prepare' +import { mutateModules, MutatedProject } from '@pnpm/core' +import { testDefaults } from '../utils' + +test('dedupe direct dependencies', async () => { + const projects = preparePackages([ + { + location: '', + package: { name: 'project-1' }, + }, + { + location: 'project-2', + package: { name: 'project-2' }, + }, + { + location: 'project-3', + package: { name: 'project-3' }, + }, + ]) + + const importers: MutatedProject[] = [ + { + mutation: 'install', + rootDir: process.cwd(), + }, + { + mutation: 'install', + rootDir: path.resolve('project-2'), + }, + { + mutation: 'install', + rootDir: path.resolve('project-3'), + }, + ] + const allProjects = [ + { + buildIndex: 0, + manifest: { + name: 'project-1', + version: '1.0.0', + + dependencies: { + 'is-positive': '1.0.0', + 'is-odd': '1.0.0', + }, + }, + rootDir: process.cwd(), + }, + { + buildIndex: 0, + manifest: { + name: 'project-2', + version: '1.0.0', + + dependencies: { + 'is-negative': '1.0.0', + }, + }, + rootDir: path.resolve('project-2'), + }, + { + buildIndex: 0, + manifest: { + name: 'project-3', + version: '1.0.0', + + dependencies: { + 'is-negative': '1.0.0', + }, + }, + rootDir: path.resolve('project-3'), + }, + ] + await mutateModules(importers, await testDefaults({ allProjects, dedupeDirectDeps: true })) + await projects['project-2'].has('is-negative') + await projects['project-3'].has('is-negative') + + allProjects[0].manifest.dependencies['is-negative'] = '1.0.0' + allProjects[1].manifest.dependencies['is-positive'] = '1.0.0' + allProjects[1].manifest.dependencies['is-odd'] = '2.0.0' + await mutateModules(importers, await testDefaults({ allProjects, dedupeDirectDeps: true })) + + expect(Array.from(fs.readdirSync('node_modules').sort())).toEqual([ + '.modules.yaml', + '.pnpm', + 'is-negative', + 'is-odd', + 'is-positive', + ]) + expect(fs.readdirSync('project-2/node_modules').sort()).toEqual(['is-odd']) + await projects['project-3'].hasNot('is-negative') + expect(fs.existsSync('project-3/node_modules')).toBeFalsy() +}) diff --git a/pkg-manager/direct-dep-linker/package.json b/pkg-manager/direct-dep-linker/package.json index 4a9f14b69f..bb76c3b1c9 100644 --- a/pkg-manager/direct-dep-linker/package.json +++ b/pkg-manager/direct-dep-linker/package.json @@ -15,7 +15,8 @@ "@pnpm/logger": "^5.0.0" }, "devDependencies": { - "@pnpm/pkg-manager.direct-dep-linker": "workspace:*" + "@pnpm/pkg-manager.direct-dep-linker": "workspace:*", + "@types/ramda": "0.28.15" }, "homepage": "https://github.com/pnpm/pnpm/blob/main/pkg-manager/direct-dep-linker#readme", "keywords": [ @@ -35,7 +36,11 @@ }, "dependencies": { "@pnpm/core-loggers": "workspace:*", - "@pnpm/symlink-dependency": "workspace:*" + "@pnpm/read-modules-dir": "workspace:*", + "@pnpm/symlink-dependency": "workspace:*", + "@zkochan/rimraf": "^2.1.2", + "ramda": "npm:@pnpm/ramda@0.28.1", + "resolve-link-target": "^2.0.0" }, "funding": "https://opencollective.com/pnpm", "exports": { diff --git a/pkg-manager/direct-dep-linker/src/linkDirectDeps.ts b/pkg-manager/direct-dep-linker/src/linkDirectDeps.ts index 81fed66ce0..d289ac6dc7 100644 --- a/pkg-manager/direct-dep-linker/src/linkDirectDeps.ts +++ b/pkg-manager/direct-dep-linker/src/linkDirectDeps.ts @@ -1,5 +1,11 @@ +import fs from 'fs' +import path from 'path' import { rootLogger } from '@pnpm/core-loggers' import { symlinkDependency, symlinkDirectRootDependency } from '@pnpm/symlink-dependency' +import omit from 'ramda/src/omit' +import { readModulesDir } from '@pnpm/read-modules-dir' +import rimraf from '@zkochan/rimraf' +import resolveLinkTarget from 'resolve-link-target' export interface LinkedDirectDep { alias: string @@ -19,37 +25,106 @@ export interface ProjectToLink { } export async function linkDirectDeps ( - projects: ProjectToLink[] + projects: Record, + opts: { + dedupe: boolean + } ) { - await Promise.all(projects.map(async (project) => { - await Promise.all(project.dependencies.map(async (dep) => { - if (dep.isExternalLink) { - await symlinkDirectRootDependency(dep.dir, project.modulesDir, dep.alias, { - fromDependenciesField: dep.dependencyType === 'dev' && 'devDependencies' || - dep.dependencyType === 'optional' && 'optionalDependencies' || - 'dependencies', - linkedPackage: { - name: dep.name, - version: dep.version, - }, - prefix: project.dir, + if (opts.dedupe && projects['.'] && Object.keys(projects).length > 1) { + return linkDirectDepsAndDedupe(projects['.'], omit(['.'], projects)) + } + await Promise.all(Object.values(projects).map(linkDirectDepsOfProject)) +} + +async function linkDirectDepsAndDedupe ( + rootProject: ProjectToLink, + projects: Record +) { + await linkDirectDepsOfProject(rootProject) + const pkgsLinkedToRoot = await readLinkedDeps(rootProject.modulesDir) + await Promise.all( + Object.values(projects).map(async (project) => { + const deletedAll = await deletePkgsPresentInRoot(project.modulesDir, pkgsLinkedToRoot) + const dependencies = omitDepsFromRoot(project.dependencies, pkgsLinkedToRoot) + if (dependencies.length > 0) { + await linkDirectDepsOfProject({ + ...project, + dependencies, }) return } - if ((await symlinkDependency(dep.dir, project.modulesDir, dep.alias)).reused) { - return + if (deletedAll) { + await rimraf(project.modulesDir) } - rootLogger.debug({ - added: { - dependencyType: dep.dependencyType, - id: dep.id, - latest: dep.latest, - name: dep.alias, - realName: dep.name, + }) + ) +} + +function omitDepsFromRoot (deps: LinkedDirectDep[], pkgsLinkedToRoot: string[]) { + return deps.filter(({ dir }) => !pkgsLinkedToRoot.some(pathsEqual.bind(null, dir))) +} + +function pathsEqual (path1: string, path2: string) { + return path.relative(path1, path2) === '' +} + +async function readLinkedDeps (modulesDir: string): Promise { + const deps = (await readModulesDir(modulesDir)) ?? [] + return Promise.all( + deps.map((alias) => resolveLinkTarget(path.join(modulesDir, alias))) + ) +} + +async function deletePkgsPresentInRoot ( + modulesDir: string, + pkgsLinkedToRoot: string[] +): Promise { + const pkgsLinkedToCurrentProject = await readLinkedDepsWithRealLocations(modulesDir) + const pkgsToDelete = pkgsLinkedToCurrentProject + .filter(({ linkedFrom }) => pkgsLinkedToRoot.some(pathsEqual.bind(null, linkedFrom))) + await Promise.all(pkgsToDelete.map(({ linkedTo }) => fs.promises.unlink(linkedTo))) + return pkgsToDelete.length === pkgsLinkedToCurrentProject.length +} + +async function readLinkedDepsWithRealLocations (modulesDir: string) { + const deps = (await readModulesDir(modulesDir)) ?? [] + return Promise.all(deps.map(async (alias) => { + const linkedTo = path.join(modulesDir, alias) + return { + linkedTo, + linkedFrom: await resolveLinkTarget(linkedTo), + } + })) +} + +async function linkDirectDepsOfProject (project: ProjectToLink) { + await Promise.all(project.dependencies.map(async (dep) => { + if (dep.isExternalLink) { + await symlinkDirectRootDependency(dep.dir, project.modulesDir, dep.alias, { + fromDependenciesField: dep.dependencyType === 'dev' && 'devDependencies' || + dep.dependencyType === 'optional' && 'optionalDependencies' || + 'dependencies', + linkedPackage: { + name: dep.name, version: dep.version, }, prefix: project.dir, }) - })) + return + } + if ((await symlinkDependency(dep.dir, project.modulesDir, dep.alias)).reused) { + return + } + rootLogger.debug({ + added: { + dependencyType: dep.dependencyType, + id: dep.id, + latest: dep.latest, + name: dep.alias, + realName: dep.name, + version: dep.version, + }, + prefix: project.dir, + }) })) } diff --git a/pkg-manager/direct-dep-linker/tsconfig.json b/pkg-manager/direct-dep-linker/tsconfig.json index 2b2e3ca812..ef2fa85d8b 100644 --- a/pkg-manager/direct-dep-linker/tsconfig.json +++ b/pkg-manager/direct-dep-linker/tsconfig.json @@ -9,6 +9,9 @@ "../../__typings__/**/*.d.ts" ], "references": [ + { + "path": "../../fs/read-modules-dir" + }, { "path": "../../fs/symlink-dependency" }, diff --git a/pkg-manager/headless/src/index.ts b/pkg-manager/headless/src/index.ts index 9bc9f028a1..6f0d8eff24 100644 --- a/pkg-manager/headless/src/index.ts +++ b/pkg-manager/headless/src/index.ts @@ -101,6 +101,7 @@ export interface HeadlessOptions { nodeVersion: string pnpmVersion: string } + dedupeDirectDeps?: boolean enablePnp?: boolean engineStrict: boolean extraBinPaths?: string[] @@ -339,6 +340,7 @@ export async function headlessInstall (opts: HeadlessOptions) { await symlinkDirectDependencies({ directDependenciesByImporterId: symlinkedDirectDependenciesByImporterId!, + dedupe: opts.dedupeDirectDeps, filteredLockfile, lockfileDir, projects: selectedProjects, @@ -562,6 +564,7 @@ export async function headlessInstall (opts: HeadlessOptions) { type SymlinkDirectDependenciesOpts = Pick & { filteredLockfile: Lockfile + dedupe?: boolean directDependenciesByImporterId: DirectDependenciesByImporterId projects: Project[] } @@ -569,6 +572,7 @@ type SymlinkDirectDependenciesOpts = Pick ({ + const projectsToLink = fromPairs(await Promise.all( + projects.map(async ({ rootDir, id, modulesDir }) => ([id, { dir: rootDir, modulesDir, dependencies: await getRootPackagesToLink(filteredLockfile, { @@ -598,9 +602,9 @@ async function symlinkDirectDependencies ( registries, rootDependencies: directDependenciesByImporterId[id], }), - })) - ) - await linkDirectDeps(projectsToLink) + }])) + )) + await linkDirectDeps(projectsToLink, { dedupe: Boolean(dedupe) }) } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c4ac509d33..ca665da916 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2723,13 +2723,28 @@ importers: '@pnpm/logger': specifier: ^5.0.0 version: 5.0.0 + '@pnpm/read-modules-dir': + specifier: workspace:* + version: link:../../fs/read-modules-dir '@pnpm/symlink-dependency': specifier: workspace:* version: link:../../fs/symlink-dependency + '@zkochan/rimraf': + specifier: ^2.1.2 + version: 2.1.2 + ramda: + specifier: npm:@pnpm/ramda@0.28.1 + version: /@pnpm/ramda/0.28.1 + resolve-link-target: + specifier: ^2.0.0 + version: 2.0.0 devDependencies: '@pnpm/pkg-manager.direct-dep-linker': specifier: workspace:* version: 'link:' + '@types/ramda': + specifier: 0.28.15 + version: 0.28.15 pkg-manager/get-context: dependencies: