feat: add pnpm dedupe command (#5958)

This commit is contained in:
Brandon Cheng
2023-01-23 05:27:42 -05:00
committed by GitHub
parent 1072ec1286
commit e8f6ab6833
23 changed files with 455 additions and 3 deletions

View File

@@ -0,0 +1,7 @@
---
"@pnpm/plugin-commands-installation": minor
"@pnpm/core": minor
pnpm: minor
---
Add a `pnpm dedupe` command that removes dependencies from the lockfile by re-resolving the dependency graph. This work similar to yarn's [`yarn dedupe --strategy highest`](https://yarnpkg.com/cli/dedupe) command.

View File

@@ -0,0 +1 @@
{}

View File

@@ -0,0 +1,6 @@
{
"name": "bar",
"dependencies": {
"ajv": "^6.10.2"
}
}

View File

@@ -0,0 +1,6 @@
{
"name": "foo",
"dependencies": {
"ajv": "^6.12.5"
}
}

View File

@@ -0,0 +1,80 @@
lockfileVersion: 5.4
importers:
.:
specifiers: {}
packages/bar:
specifiers:
ajv: ^6.10.2
dependencies:
ajv: 6.10.2
packages/foo:
specifiers:
ajv: ^6.12.5
dependencies:
ajv: 6.12.6
packages:
/ajv/6.10.2:
resolution: {integrity: sha512-TXtUUEYHuaTEbLZWIKUr5pmBuhDLy+8KYtPYdcV8qC+pOZL+NKqYwvWSRrVXHn+ZmRRAu8vJTAznH7Oag6RVRw==}
dependencies:
fast-deep-equal: 2.0.1
fast-json-stable-stringify: 2.0.0
json-schema-traverse: 0.4.1
uri-js: 4.2.2
dev: false
/ajv/6.12.6:
resolution: {integrity: sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==}
dependencies:
fast-deep-equal: 3.1.3
fast-json-stable-stringify: 2.1.0
json-schema-traverse: 0.4.1
uri-js: 4.4.1
dev: false
/fast-deep-equal/2.0.1:
resolution: {integrity: sha512-bCK/2Z4zLidyB4ReuIsvALH6w31YfAQDmXMqMx6FyfHqvBxtjC0eRumeSu4Bs3XtXwpyIywtSTrVT99BxY1f9w==}
dev: false
/fast-deep-equal/3.1.3:
resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==}
dev: false
/fast-json-stable-stringify/2.0.0:
resolution: {integrity: sha1-1RQsDK7msRifh9OnYREGT4bIu/I=}
dev: false
/fast-json-stable-stringify/2.1.0:
resolution: {integrity: sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==}
dev: false
/json-schema-traverse/0.4.1:
resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==}
dev: false
/punycode/2.1.1:
resolution: {integrity: sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==}
engines: {node: '>=6'}
dev: false
/punycode/2.3.0:
resolution: {integrity: sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==}
engines: {node: '>=6'}
dev: false
/uri-js/4.2.2:
resolution: {integrity: sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==}
dependencies:
punycode: 2.1.1
dev: false
/uri-js/4.4.1:
resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==}
dependencies:
punycode: 2.3.0
dev: false

View File

@@ -0,0 +1,2 @@
packages:
- 'packages/**'

View File

@@ -0,0 +1 @@
{}

View File

@@ -0,0 +1,6 @@
{
"name": "bar",
"dependencies": {
"uri-js": "=4.2.2"
}
}

View File

@@ -0,0 +1,6 @@
{
"name": "foo",
"dependencies": {
"uri-js": "=4.4.1"
}
}

View File

@@ -0,0 +1,42 @@
lockfileVersion: 5.4
importers:
.:
specifiers: {}
packages/bar:
specifiers:
uri-js: '=4.2.2'
dependencies:
uri-js: 4.2.2
packages/foo:
specifiers:
uri-js: '=4.4.1'
dependencies:
uri-js: 4.4.1
packages:
/punycode/2.1.1:
resolution: {integrity: sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==}
engines: {node: '>=6'}
dev: false
/punycode/2.3.0:
resolution: {integrity: sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==}
engines: {node: '>=6'}
dev: false
/uri-js/4.2.2:
resolution: {integrity: sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==}
dependencies:
punycode: 2.1.1
dev: false
/uri-js/4.4.1:
resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==}
dependencies:
punycode: 2.3.0
dev: false

View File

@@ -0,0 +1,2 @@
packages:
- 'packages/**'

View File

@@ -37,6 +37,7 @@ export interface StrictInstallOptions {
linkWorkspacePackagesDepth: number
lockfileOnly: boolean
fixLockfile: boolean
dedupe: boolean
ignoreCompatibilityDb: boolean
ignoreDepScripts: boolean
ignorePackageManifest: boolean

View File

@@ -305,6 +305,7 @@ export async function mutateModules (
!ctx.lockfileHadConflicts &&
!opts.update &&
!opts.fixLockfile &&
!opts.dedupe &&
installsOnly &&
(
frozenLockfile && !opts.lockfileOnly ||
@@ -644,6 +645,27 @@ function forgetResolutionsOfPrevWantedDeps (importer: ProjectSnapshot, wantedDep
}
}
function forgetResolutionsOfAllPrevWantedDeps (wantedLockfile: Lockfile) {
// Similar to the forgetResolutionsOfPrevWantedDeps function above, we can
// delete existing resolutions in importers to make sure they're resolved
// again.
if ((wantedLockfile.importers != null) && !isEmpty(wantedLockfile.importers)) {
wantedLockfile.importers = mapValues(
({ dependencies, devDependencies, optionalDependencies, ...rest }) => rest,
wantedLockfile.importers)
}
// The resolveDependencies function looks at previous PackageSnapshot
// dependencies/optionalDependencies blocks and merges them with new resolved
// deps. Clear the previous PackageSnapshot fields so the newly resolved deps
// are always used.
if ((wantedLockfile.packages != null) && !isEmpty(wantedLockfile.packages)) {
wantedLockfile.packages = mapValues(
({ dependencies, optionalDependencies, ...rest }) => rest,
wantedLockfile.packages)
}
}
export async function addDependenciesToPackage (
manifest: ProjectManifest,
dependencySelectors: string[],
@@ -798,6 +820,15 @@ const _installInContext: InstallFunction = async (projects, ctx, opts) => {
}), ctx.wantedLockfile.packages)
}
if (opts.dedupe) {
// Deleting recorded version resolutions from importers and packages. These
// fields will be regenerated using the preferred versions computed above.
//
// This is a bit different from a "full resolution", which completely
// ignores preferred versions from the lockfile.
forgetResolutionsOfAllPrevWantedDeps(ctx.wantedLockfile)
}
let {
dependenciesGraph,
dependenciesByProjectId,

View File

@@ -46,6 +46,7 @@
"@types/yarnpkg__lockfile": "^1.1.5",
"@types/zkochan__table": "npm:@types/table@6.0.0",
"delay": "^5.0.0",
"jest-diff": "^29.3.1",
"path-name": "^1.0.0",
"proxyquire": "^2.1.3",
"read-yaml-file": "^2.1.0",

View File

@@ -0,0 +1,37 @@
import { docsUrl } from '@pnpm/cli-utils'
import { UNIVERSAL_OPTIONS } from '@pnpm/common-cli-options-help'
import renderHelp from 'render-help'
import * as install from './install'
export const rcOptionsTypes = cliOptionsTypes
export function cliOptionsTypes () {
return {}
}
export const commandNames = ['dedupe']
export function help () {
return renderHelp({
description: 'Perform an install removing older dependencies in the lockfile if a newer version can be used.',
descriptionLists: [
{
title: 'Options',
list: [
...UNIVERSAL_OPTIONS,
],
},
],
url: docsUrl('dedupe'),
usages: ['pnpm dedupe'],
})
}
export async function handler (
opts: install.InstallCommandOptions
) {
return install.handler({
...opts,
dedupe: true,
})
}

View File

@@ -1,4 +1,5 @@
import * as add from './add'
import * as dedupe from './dedupe'
import * as install from './install'
import * as fetch from './fetch'
import * as link from './link'
@@ -8,4 +9,4 @@ import * as unlink from './unlink'
import * as update from './update'
import * as importCommand from './import'
export { add, fetch, install, link, prune, remove, unlink, update, importCommand }
export { add, dedupe, fetch, install, link, prune, remove, unlink, update, importCommand }

View File

@@ -293,6 +293,7 @@ export type InstallCommandOptions = Pick<Config,
pruneDirectDependencies?: boolean
pruneStore?: boolean
recursive?: boolean
dedupe?: boolean
saveLockfile?: boolean
workspace?: boolean
} & Partial<Pick<Config, 'modulesCacheMaxAge' | 'pnpmHomeDir' | 'preferWorkspacePackages'>>

View File

@@ -90,6 +90,7 @@ export type InstallDepsOptions = Pick<Config,
updatePackageManifest?: boolean
useBetaCli?: boolean
recursive?: boolean
dedupe?: boolean
workspace?: boolean
} & Partial<Pick<Config, 'pnpmHomeDir'>>

View File

@@ -0,0 +1,124 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`pnpm dedupe updates old resolutions from importers block and removes old packages 1`] = `
"- Expected
+ Received
@@ -5,7 +5,7 @@
},
"packages/bar": Object {
"dependencies": Object {
- "ajv": "6.10.2",
+ "ajv": "6.12.6",
},
"specifiers": Object {
"ajv": "^6.10.2",
@@ -22,18 +22,6 @@
},
"lockfileVersion": 5.4,
"packages": Object {
- "/ajv/6.10.2": Object {
- "dependencies": Object {
- "fast-deep-equal": "2.0.1",
- "fast-json-stable-stringify": "2.0.0",
- "json-schema-traverse": "0.4.1",
- "uri-js": "4.2.2",
- },
- "dev": false,
- "resolution": Object {
- "integrity": "sha512-TXtUUEYHuaTEbLZWIKUr5pmBuhDLy+8KYtPYdcV8qC+pOZL+NKqYwvWSRrVXHn+ZmRRAu8vJTAznH7Oag6RVRw==",
- },
- },
"/ajv/6.12.6": Object {
"dependencies": Object {
"fast-deep-equal": "3.1.3",
@@ -44,24 +32,12 @@
"dev": false,
"resolution": Object {
"integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
- },
- },
- "/fast-deep-equal/2.0.1": Object {
- "dev": false,
- "resolution": Object {
- "integrity": "sha512-bCK/2Z4zLidyB4ReuIsvALH6w31YfAQDmXMqMx6FyfHqvBxtjC0eRumeSu4Bs3XtXwpyIywtSTrVT99BxY1f9w==",
},
},
"/fast-deep-equal/3.1.3": Object {
"dev": false,
"resolution": Object {
"integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==",
- },
- },
- "/fast-json-stable-stringify/2.0.0": Object {
- "dev": false,
- "resolution": Object {
- "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=",
},
},
"/fast-json-stable-stringify/2.1.0": Object {
@@ -75,16 +51,7 @@
"resolution": Object {
"integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==",
},
- },
- "/punycode/2.1.1": Object {
- "dev": false,
- "engines": Object {
- "node": ">=6",
},
- "resolution": Object {
- "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==",
- },
- },
"/punycode/2.3.0": Object {
"dev": false,
"engines": Object {
@@ -92,15 +59,6 @@
},
"resolution": Object {
"integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==",
- },
- },
- "/uri-js/4.2.2": Object {
- "dependencies": Object {
- "punycode": "2.1.1",
- },
- "dev": false,
- "resolution": Object {
- "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==",
},
},
"/uri-js/4.4.1": Object {"
`;
exports[`pnpm dedupe updates old resolutions from package block 1`] = `
"- Expected
+ Received
@@ -22,15 +22,6 @@
},
"lockfileVersion": 5.4,
"packages": Object {
- "/punycode/2.1.1": Object {
- "dev": false,
- "engines": Object {
- "node": ">=6",
- },
- "resolution": Object {
- "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==",
- },
- },
"/punycode/2.3.0": Object {
"dev": false,
"engines": Object {
@@ -42,7 +33,7 @@
},
"/uri-js/4.2.2": Object {
"dependencies": Object {
- "punycode": "2.1.1",
+ "punycode": "2.3.0",
},
"dev": false,
"resolution": Object {"
`;

View File

@@ -0,0 +1,91 @@
import path from 'path'
import { readProjects } from '@pnpm/filter-workspace-packages'
import { Lockfile } from '@pnpm/lockfile-types'
import { dedupe, install } from '@pnpm/plugin-commands-installation'
import { prepare } from '@pnpm/prepare'
import { fixtures } from '@pnpm/test-fixtures'
import { diff } from 'jest-diff'
import readYamlFile from 'read-yaml-file'
import { DEFAULT_OPTS } from './utils'
const f = fixtures(__dirname)
describe('pnpm dedupe', () => {
test('updates old resolutions from importers block and removes old packages', async () => {
const { originalLockfile, dedupedLockfile } = await testFixture('workspace-with-lockfile-dupes')
// Many old packages should be deleted as result of deduping. See snapshot file for details.
expect(diff(originalLockfile, dedupedLockfile, diffOptsForLockfile)).toMatchSnapshot()
})
test('updates old resolutions from package block', async () => {
const { originalLockfile, dedupedLockfile } = await testFixture('workspace-with-lockfile-subdep-dupes')
// This is a smaller scale test that should just update uri-js@4.2.2 to
// punycode@2.3.0 and remove punycode@2.1.1. See snapshot file for details.
expect(diff(originalLockfile, dedupedLockfile, diffOptsForLockfile)).toMatchSnapshot()
})
})
const noColor = (str: string) => str
const diffOptsForLockfile = {
// Avoid showing common lines to make the snapshot smaller and less noisy.
// https://github.com/facebook/jest/tree/05deb8393c4ad71/packages/jest-diff#example-of-options-to-limit-common-lines
contextLines: 3,
expand: false,
// Remove color from snapshots
// https://github.com/facebook/jest/tree/05deb8393c4ad71/packages/jest-diff#example-of-options-for-no-colors
aColor: noColor,
bColor: noColor,
changeColor: noColor,
commonColor: noColor,
patchColor: noColor,
}
async function testFixture (fixtureName: string) {
const project = prepare(undefined)
f.copy(fixtureName, project.dir())
const { allProjects, selectedProjectsGraph } = await readProjects(project.dir(), [])
const opts = {
...DEFAULT_OPTS,
allProjects,
selectedProjectsGraph,
recursive: true,
dir: project.dir(),
lockfileDir: project.dir(),
workspaceDir: project.dir(),
}
const readProjectLockfile = () => readYamlFile<Lockfile>(path.join(project.dir(), './pnpm-lock.yaml'))
const originalLockfile = await readProjectLockfile()
// Sanity check that this test is set up correctly by ensuring the lockfile is
// unmodified after a regular install.
await install.handler(opts)
expect(await readProjectLockfile()).toEqual(originalLockfile)
// The lockfile fixture has several packages that could be removed after
// re-resolving versions.
await dedupe.handler(opts)
const dedupedLockfile = await readProjectLockfile()
// It should be possible to remove packages from the fixture lockfile.
const originalLockfilePackageNames = Object.keys(originalLockfile.packages ?? {})
const dedupedLockfilePackageNames = Object.keys(dedupedLockfile.packages ?? {})
expect(dedupedLockfilePackageNames.length).toBeLessThan(originalLockfilePackageNames.length)
// The "pnpm dedupe" command should only remove packages when the lockfile is
// up to date. Ensure no new packages/dependencies were added.
expect(originalLockfilePackageNames).toEqual(expect.arrayContaining(dedupedLockfilePackageNames))
// Run pnpm install one last time to ensure the deduped lockfile is in a good
// state. If so, the "pnpm install" command should pass successfully and not
// make any further edits to the lockfile.
await install.handler(opts)
expect(await readProjectLockfile()).toEqual(dedupedLockfile)
return { originalLockfile, dedupedLockfile }
}

4
pnpm-lock.yaml generated
View File

@@ -3584,6 +3584,9 @@ importers:
delay:
specifier: ^5.0.0
version: 5.0.0
jest-diff:
specifier: ^29.3.1
version: 29.3.1
path-name:
specifier: ^1.0.0
version: 1.0.0
@@ -17550,6 +17553,7 @@ time:
/is-subdir@1.2.0: '2021-01-05T16:52:45.485Z'
/is-windows@1.0.2: '2018-02-14T07:36:43.207Z'
/isexe@2.0.0: '2017-03-23T00:53:16.356Z'
/jest-diff@29.3.1: '2022-11-08T22:56:23.491Z'
/jest@29.3.1: '2022-11-08T22:56:40.420Z'
/json-append@1.1.1: '2017-01-07T04:45:27.600Z'
/json5@2.2.3: '2022-12-31T17:11:32.047Z'

View File

@@ -5,7 +5,7 @@ import { config, getCommand, setCommand } from '@pnpm/plugin-commands-config'
import { doctor } from '@pnpm/plugin-commands-doctor'
import { env } from '@pnpm/plugin-commands-env'
import { deploy } from '@pnpm/plugin-commands-deploy'
import { add, fetch, install, link, prune, remove, unlink, update, importCommand } from '@pnpm/plugin-commands-installation'
import { add, dedupe, fetch, install, link, prune, remove, unlink, update, importCommand } from '@pnpm/plugin-commands-installation'
import { list, ll, why } from '@pnpm/plugin-commands-listing'
import { licenses } from '@pnpm/plugin-commands-licenses'
import { outdated } from '@pnpm/plugin-commands-outdated'
@@ -101,6 +101,7 @@ const commands: CommandDefinition[] = [
audit,
bin,
config,
dedupe,
getCommand,
setCommand,
create,

View File

@@ -168,7 +168,7 @@ export async function main (inputArgv: string[]) {
}
if (
(cmd === 'install' || cmd === 'import') &&
(cmd === 'install' || cmd === 'import' || cmd === "dedupe") &&
typeof workspaceDir === 'string'
) {
cliOptions['recursive'] = true