From ef73c19f1523080f72a21f87b4b232c9af353fa7 Mon Sep 17 00:00:00 2001 From: Zoltan Kochan Date: Thu, 16 May 2024 20:55:08 +0200 Subject: [PATCH] perf: reduce memory usage (#8084) ref #8072 --- .changeset/dirty-flies-do.md | 6 + pkg-manager/resolve-dependencies/package.json | 1 - pkg-manager/resolve-dependencies/src/index.ts | 4 +- .../resolve-dependencies/src/nextNodeId.ts | 5 + .../resolve-dependencies/src/nodeIdUtils.ts | 22 --- .../src/parentIdsContainSequence.ts | 8 + .../src/resolveDependencies.ts | 110 ++++++------ .../src/resolveDependencyTree.ts | 56 +++--- .../resolve-dependencies/src/resolvePeers.ts | 161 +++++++++--------- .../src/updateLockfile.ts | 5 +- .../test/nodeIdUtils.test.ts | 5 - .../test/parentIdsContainSequence.test.ts | 5 + pnpm-lock.yaml | 3 - 13 files changed, 189 insertions(+), 202 deletions(-) create mode 100644 .changeset/dirty-flies-do.md create mode 100644 pkg-manager/resolve-dependencies/src/nextNodeId.ts delete mode 100644 pkg-manager/resolve-dependencies/src/nodeIdUtils.ts create mode 100644 pkg-manager/resolve-dependencies/src/parentIdsContainSequence.ts delete mode 100644 pkg-manager/resolve-dependencies/test/nodeIdUtils.test.ts create mode 100644 pkg-manager/resolve-dependencies/test/parentIdsContainSequence.test.ts diff --git a/.changeset/dirty-flies-do.md b/.changeset/dirty-flies-do.md new file mode 100644 index 0000000000..9d7b2e6383 --- /dev/null +++ b/.changeset/dirty-flies-do.md @@ -0,0 +1,6 @@ +--- +"@pnpm/resolve-dependencies": patch +"pnpm": patch +--- + +Decrease memory consumption [#8084](https://github.com/pnpm/pnpm/pull/8084). diff --git a/pkg-manager/resolve-dependencies/package.json b/pkg-manager/resolve-dependencies/package.json index a14a8ddb7f..bb6a2563ba 100644 --- a/pkg-manager/resolve-dependencies/package.json +++ b/pkg-manager/resolve-dependencies/package.json @@ -53,7 +53,6 @@ "graph-cycles": "1.2.1", "is-inner-link": "^4.0.0", "is-subdir": "^1.2.0", - "mem": "^8.1.1", "normalize-path": "^3.0.0", "p-defer": "^3.0.0", "path-exists": "^4.0.0", diff --git a/pkg-manager/resolve-dependencies/src/index.ts b/pkg-manager/resolve-dependencies/src/index.ts index 92d54e3ad2..e311b4dcec 100644 --- a/pkg-manager/resolve-dependencies/src/index.ts +++ b/pkg-manager/resolve-dependencies/src/index.ts @@ -129,7 +129,7 @@ export async function resolveDependencies ( dependenciesTree, outdatedDependencies, resolvedImporters, - resolvedPackagesByDepPath, + resolvedPkgsById, wantedToBeSkippedPackageIds, appliedPatches, time, @@ -301,7 +301,7 @@ export async function resolveDependencies ( // waiting till package requests are finished async function waitTillAllFetchingsFinish (): Promise { - await Promise.all(Object.values(resolvedPackagesByDepPath).map(async ({ fetching }) => { + await Promise.all(Object.values(resolvedPkgsById).map(async ({ fetching }) => { try { await fetching?.() } catch {} diff --git a/pkg-manager/resolve-dependencies/src/nextNodeId.ts b/pkg-manager/resolve-dependencies/src/nextNodeId.ts new file mode 100644 index 0000000000..78e4434c46 --- /dev/null +++ b/pkg-manager/resolve-dependencies/src/nextNodeId.ts @@ -0,0 +1,5 @@ +let nodeIdCounter = 0 + +export function nextNodeId (): string { + return (++nodeIdCounter).toString() +} diff --git a/pkg-manager/resolve-dependencies/src/nodeIdUtils.ts b/pkg-manager/resolve-dependencies/src/nodeIdUtils.ts deleted file mode 100644 index ee46848dd9..0000000000 --- a/pkg-manager/resolve-dependencies/src/nodeIdUtils.ts +++ /dev/null @@ -1,22 +0,0 @@ -export function nodeIdContains (nodeId: string, pkgId: string): boolean { - const pkgIds = splitNodeId(nodeId) - return pkgIds.includes(pkgId) -} - -export function nodeIdContainsSequence (nodeId: string, pkgId1: string, pkgId2: string): boolean { - const pkgIds = splitNodeId(nodeId) - pkgIds.pop() - const pkg1Index = pkgIds.indexOf(pkgId1) - if (pkg1Index === -1) return false - const pkg2Index = pkgIds.lastIndexOf(pkgId2) - return pkg1Index < pkg2Index -} - -export function createNodeId (parentNodeId: string, pkgId: string): string { - // using ">" as a separator because it will never be used inside a package ID - return `${parentNodeId}${pkgId}>` -} - -export function splitNodeId (nodeId: string): string[] { - return nodeId.split('>').slice(1, -1) -} diff --git a/pkg-manager/resolve-dependencies/src/parentIdsContainSequence.ts b/pkg-manager/resolve-dependencies/src/parentIdsContainSequence.ts new file mode 100644 index 0000000000..31ef5802b4 --- /dev/null +++ b/pkg-manager/resolve-dependencies/src/parentIdsContainSequence.ts @@ -0,0 +1,8 @@ +export function parentIdsContainSequence (pkgIds: string[], pkgId1: string, pkgId2: string): boolean { + const pkg1Index = pkgIds.indexOf(pkgId1) + if (pkg1Index === -1 || pkg1Index === pkgIds.length - 1) { + return false + } + const pkg2Index = pkgIds.lastIndexOf(pkgId2) + return pkg1Index < pkg2Index && pkg2Index !== pkgIds.length - 1 +} diff --git a/pkg-manager/resolve-dependencies/src/resolveDependencies.ts b/pkg-manager/resolve-dependencies/src/resolveDependencies.ts index 5e6e04a87f..430b1b65c7 100644 --- a/pkg-manager/resolve-dependencies/src/resolveDependencies.ts +++ b/pkg-manager/resolve-dependencies/src/resolveDependencies.ts @@ -50,12 +50,8 @@ import semver from 'semver' import { encodePkgId } from './encodePkgId' import { getNonDevWantedDependencies, type WantedDependency } from './getNonDevWantedDependencies' import { safeIntersect } from './mergePeers' -import { - createNodeId, - nodeIdContainsSequence, - nodeIdContains, - splitNodeId, -} from './nodeIdUtils' +import { nextNodeId } from './nextNodeId' +import { parentIdsContainSequence } from './parentIdsContainSequence' import { hoistPeers, getHoistableOptionalPeers } from './hoistPeers' import { wantedDepIsLocallyAvailable } from './wantedDepIsLocallyAvailable' import { replaceVersionInPref } from './replaceVersionInPref' @@ -64,13 +60,14 @@ const dependencyResolvedLogger = logger('_dependency_resolved') const omitDepsFields = omit(['dependencies', 'optionalDependencies', 'peerDependencies', 'peerDependenciesMeta']) -export function nodeIdToParents ( - nodeId: string, - resolvedPackagesByDepPath: ResolvedPackagesByDepPath +export function getPkgsInfoFromIds ( + ids: string[], + resolvedPkgsById: ResolvedPkgsById ): Array<{ id: string, name: string, version: string }> { - return splitNodeId(nodeId).slice(1) - .map((depPath) => { - const { id, name, version } = resolvedPackagesByDepPath[depPath] + return ids + .slice(1) + .map((id) => { + const { name, version } = resolvedPkgsById[id] return { id, name, version } }) } @@ -99,7 +96,7 @@ string, DependenciesTreeNode > -export type ResolvedPackagesByDepPath = Record +export type ResolvedPkgsById = Record export interface LinkedDependency { isLinkedDependency: true @@ -120,12 +117,13 @@ export interface PendingNode { resolvedPackage: ResolvedPackage depth: number installable: boolean + parentIds: string[] } -export interface ChildrenByParentDepPath { - [depPath: string]: Array<{ +export interface ChildrenByParentId { + [id: string]: Array<{ alias: string - depPath: string + id: string }> } @@ -142,9 +140,9 @@ export interface ResolutionContext { dryRun: boolean forceFullResolution: boolean ignoreScripts?: boolean - resolvedPackagesByDepPath: ResolvedPackagesByDepPath + resolvedPkgsById: ResolvedPkgsById outdatedDependencies: { [pkgId: string]: string } - childrenByParentDepPath: ChildrenByParentDepPath + childrenByParentId: ChildrenByParentId patchedDependencies?: Record pendingNodes: PendingNode[] wantedLockfile: Lockfile @@ -246,7 +244,7 @@ export interface ResolvedPackage { parentImporterIds: Set } -type ParentPkg = Pick +type ParentPkg = Pick export type ParentPkgAliases = Record @@ -254,6 +252,7 @@ export type UpdateMatchingFunction = (pkgName: string) => boolean interface ResolvedDependenciesOptions { currentDepth: number + parentIds: string[] parentPkg: ParentPkg parentPkgAliases: ParentPkgAliases // If the package has been updated, the dependencies @@ -377,12 +376,14 @@ interface PkgAddressesByImportersWithoutPeers extends PeersResolutionResult { pkgAddresses: Array } +export type ImporterToResolveOptions = Omit + export interface ImporterToResolve { updatePackageManifest: boolean preferredVersions: PreferredVersions parentPkgAliases: ParentPkgAliases wantedDependencies: Array - options: Omit + options: ImporterToResolveOptions } interface ResolveDependenciesOfImportersResult { @@ -456,7 +457,7 @@ async function resolveDependenciesOfImporters ( if (pkgAddress.updated) { ctx.updatedSet.add(pkgAddress.alias) } - const resolvedPackage = ctx.resolvedPackagesByDepPath[pkgAddress.depPath] + const resolvedPackage = ctx.resolvedPkgsById[pkgAddress.pkgId] if (!resolvedPackage) continue // This will happen only with linked dependencies if (!Object.prototype.hasOwnProperty.call(newPreferredVersions, resolvedPackage.name)) { newPreferredVersions[resolvedPackage.name] = { ...importer.preferredVersions[resolvedPackage.name] } @@ -586,7 +587,7 @@ export async function resolveDependencies ( if (pkgAddress.updated) { ctx.updatedSet.add(pkgAddress.alias) } - const resolvedPackage = ctx.resolvedPackagesByDepPath[pkgAddress.depPath] + const resolvedPackage = ctx.resolvedPkgsById[pkgAddress.pkgId] if (!resolvedPackage) continue // This will happen only with linked dependencies if (!Object.prototype.hasOwnProperty.call(newPreferredVersions, resolvedPackage.name)) { newPreferredVersions[resolvedPackage.name] = { ...preferredVersions[resolvedPackage.name] } @@ -741,6 +742,7 @@ async function resolveDependenciesOfDependency ( updateMatching: options.updateMatching, supportedArchitectures: options.supportedArchitectures, updateToLatest: options.updateToLatest, + parentIds: options.parentIds, } const resolveDependencyResult = await resolveDependency(extendedWantedDep.wantedDependency, ctx, resolveDependencyOpts) @@ -773,6 +775,7 @@ async function resolveDependenciesOfDependency ( parentPkg: resolveDependencyResult, dependencyLockfile: extendedWantedDep.infoFromLockfile?.dependencyLockfile, parentDepth: options.currentDepth, + parentIds: [...options.parentIds, resolveDependencyResult.pkgId], updateDepth, prefix: options.prefix, updateMatching: options.updateMatching, @@ -820,6 +823,7 @@ async function resolveChildren ( ctx: ResolutionContext, { parentPkg, + parentIds, dependencyLockfile, parentDepth, updateDepth, @@ -828,6 +832,7 @@ async function resolveChildren ( supportedArchitectures, }: { parentPkg: PkgAddress + parentIds: string[] dependencyLockfile: PackageSnapshot | undefined parentDepth: number updateDepth: number @@ -881,11 +886,12 @@ async function resolveChildren ( updateDepth, updateMatching, supportedArchitectures, + parentIds, } ) - ctx.childrenByParentDepPath[parentPkg.depPath] = pkgAddresses.map((child) => ({ + ctx.childrenByParentId[parentPkg.pkgId] = pkgAddresses.map((child) => ({ alias: child.alias, - depPath: child.depPath, + id: child.pkgId, })) ctx.dependenciesTree.set(parentPkg.nodeId, { children: pkgAddresses.reduce((chn, child) => { @@ -894,7 +900,7 @@ async function resolveChildren ( }, {} as Record), depth: parentDepth, installable: parentPkg.installable, - resolvedPackage: ctx.resolvedPackagesByDepPath[parentPkg.depPath], + resolvedPackage: ctx.resolvedPkgsById[parentPkg.pkgId], }) return resolvingPeers } @@ -1078,6 +1084,7 @@ interface ResolveDependencyOptions { dependencyLockfile?: PackageSnapshot } parentPkg: ParentPkg + parentIds: string[] parentPkgAliases: ParentPkgAliases preferredVersions: PreferredVersions prefix: string @@ -1166,7 +1173,7 @@ async function resolveDependency ( supportedArchitectures: options.supportedArchitectures, onFetchError: (err: any) => { // eslint-disable-line err.prefix = options.prefix - err.pkgsStack = nodeIdToParents(options.parentPkg.nodeId, ctx.resolvedPackagesByDepPath) + err.pkgsStack = getPkgsInfoFromIds(options.parentIds, ctx.resolvedPkgsById) return err }, updateToLatest: options.updateToLatest, @@ -1180,21 +1187,21 @@ async function resolveDependency ( pref: wantedDependency.pref, version: wantedDependency.alias ? wantedDependency.pref : undefined, }, - parents: nodeIdToParents(options.parentPkg.nodeId, ctx.resolvedPackagesByDepPath), + parents: getPkgsInfoFromIds(options.parentIds, ctx.resolvedPkgsById), prefix: options.prefix, reason: 'resolution_failure', }) return null } err.prefix = options.prefix - err.pkgsStack = nodeIdToParents(options.parentPkg.nodeId, ctx.resolvedPackagesByDepPath) + err.pkgsStack = getPkgsInfoFromIds(options.parentIds, ctx.resolvedPkgsById) throw err } dependencyResolvedLogger.debug({ resolution: pkgResponse.body.id, wanted: { - dependentId: options.parentPkg.depPath, + dependentId: options.parentPkg.pkgId, name: wantedDependency.alias, rawSpec: wantedDependency.pref, }, @@ -1289,11 +1296,11 @@ async function resolveDependency ( // because zoo is a new parent package: // foo > bar > qar > zoo > qar if ( - nodeIdContainsSequence( - options.parentPkg.nodeId, - options.parentPkg.depPath, - depPath - ) || depPath === options.parentPkg.depPath + parentIdsContainSequence( + options.parentIds, + options.parentPkg.pkgId, + pkgResponse.body.id + ) || pkgResponse.body.id === options.parentPkg.pkgId ) { return null } @@ -1342,14 +1349,12 @@ async function resolveDependency ( } // In case of leaf dependencies (dependencies that have no prod deps or peer deps), // we only ever need to analyze one leaf dep in a graph, so the nodeId can be short and stateless. - const nodeId = pkgIsLeaf(pkg) - ? `>${depPath}>` - : createNodeId(options.parentPkg.nodeId, depPath) + const nodeId = pkgIsLeaf(pkg) ? pkgResponse.body.id : nextNodeId() const parentIsInstallable = options.parentPkg.installable === undefined || options.parentPkg.installable const installable = parentIsInstallable && pkgResponse.body.isInstallable !== false - const isNew = !ctx.resolvedPackagesByDepPath[depPath] - const parentImporterId = options.parentPkg.nodeId.substring(0, options.parentPkg.nodeId.indexOf('>', 1) + 1) + const isNew = !ctx.resolvedPkgsById[pkgResponse.body.id] + const parentImporterId = options.parentIds[0] let resolveChildren = false const currentIsOptional = wantedDependency.optional || options.parentPkg.optional @@ -1379,7 +1384,7 @@ async function resolveDependency ( // WARN: It is very important to keep this sync // Otherwise, deprecation messages for the same package might get written several times - ctx.resolvedPackagesByDepPath[depPath] = getResolvedPackage({ + ctx.resolvedPkgsById[pkgResponse.body.id] = getResolvedPackage({ allowBuild: ctx.allowBuild, dependencyLockfile: currentPkg.dependencyLockfile, depPath, @@ -1394,17 +1399,17 @@ async function resolveDependency ( optional: currentIsOptional, }) } else { - ctx.resolvedPackagesByDepPath[depPath].prod = ctx.resolvedPackagesByDepPath[depPath].prod || !wantedDependency.dev && !wantedDependency.optional - ctx.resolvedPackagesByDepPath[depPath].dev = ctx.resolvedPackagesByDepPath[depPath].dev || wantedDependency.dev - ctx.resolvedPackagesByDepPath[depPath].optional = ctx.resolvedPackagesByDepPath[depPath].optional && currentIsOptional + ctx.resolvedPkgsById[pkgResponse.body.id].prod = ctx.resolvedPkgsById[pkgResponse.body.id].prod || !wantedDependency.dev && !wantedDependency.optional + ctx.resolvedPkgsById[pkgResponse.body.id].dev = ctx.resolvedPkgsById[pkgResponse.body.id].dev || wantedDependency.dev + ctx.resolvedPkgsById[pkgResponse.body.id].optional = ctx.resolvedPkgsById[pkgResponse.body.id].optional && currentIsOptional if (ctx.hoistPeers) { resolveChildren = !ctx.missingPeersOfChildrenByPkgId[pkgResponse.body.id].missingPeersOfChildren.resolved && - !ctx.resolvedPackagesByDepPath[depPath].parentImporterIds.has(parentImporterId) - ctx.resolvedPackagesByDepPath[depPath].parentImporterIds.add(parentImporterId) + !ctx.resolvedPkgsById[pkgResponse.body.id].parentImporterIds.has(parentImporterId) + ctx.resolvedPkgsById[pkgResponse.body.id].parentImporterIds.add(parentImporterId) } - if (ctx.resolvedPackagesByDepPath[depPath].fetching == null && pkgResponse.fetching != null) { - ctx.resolvedPackagesByDepPath[depPath].fetching = pkgResponse.fetching - ctx.resolvedPackagesByDepPath[depPath].filesIndexFile = pkgResponse.filesIndexFile! + if (ctx.resolvedPkgsById[pkgResponse.body.id].fetching == null && pkgResponse.fetching != null) { + ctx.resolvedPkgsById[pkgResponse.body.id].fetching = pkgResponse.fetching + ctx.resolvedPkgsById[pkgResponse.body.id].filesIndexFile = pkgResponse.filesIndexFile! } if (ctx.dependenciesTree.has(nodeId)) { @@ -1413,9 +1418,10 @@ async function resolveDependency ( ctx.pendingNodes.push({ alias: wantedDependency.alias || pkg.name, depth: options.currentDepth, + parentIds: options.parentIds, installable, nodeId, - resolvedPackage: ctx.resolvedPackagesByDepPath[depPath], + resolvedPackage: ctx.resolvedPkgsById[pkgResponse.body.id], }) } } @@ -1424,9 +1430,9 @@ async function resolveDependency ( ? path.resolve(ctx.lockfileDir, (pkgResponse.body.resolution as DirectoryResolution).directory) : options.prefix let missingPeersOfChildren!: MissingPeersOfChildren | undefined - if (ctx.hoistPeers && !nodeIdContains(options.parentPkg.nodeId, depPath)) { + if (ctx.hoistPeers && !options.parentIds.includes(pkgResponse.body.id)) { if (ctx.missingPeersOfChildrenByPkgId[pkgResponse.body.id]) { - if (!options.parentPkg.nodeId.startsWith(ctx.missingPeersOfChildrenByPkgId[pkgResponse.body.id].parentImporterId)) { + if (options.parentIds[0] !== ctx.missingPeersOfChildrenByPkgId[pkgResponse.body.id].parentImporterId) { missingPeersOfChildren = ctx.missingPeersOfChildrenByPkgId[pkgResponse.body.id].missingPeersOfChildren } } else { @@ -1453,7 +1459,7 @@ async function resolveDependency ( pkgId: pkgResponse.body.id, rootDir, missingPeers: getMissingPeers(pkg), - optional: ctx.resolvedPackagesByDepPath[depPath].optional, + optional: ctx.resolvedPkgsById[pkgResponse.body.id].optional, // Next fields are actually only needed when isNew = true installable, diff --git a/pkg-manager/resolve-dependencies/src/resolveDependencyTree.ts b/pkg-manager/resolve-dependencies/src/resolveDependencyTree.ts index c837b3255c..872b54886a 100644 --- a/pkg-manager/resolve-dependencies/src/resolveDependencyTree.ts +++ b/pkg-manager/resolve-dependencies/src/resolveDependencyTree.ts @@ -11,25 +11,23 @@ import { import partition from 'ramda/src/partition' import zipObj from 'ramda/src/zipObj' import { type WantedDependency } from './getNonDevWantedDependencies' +import { nextNodeId } from './nextNodeId' +import { parentIdsContainSequence } from './parentIdsContainSequence' import { - createNodeId, - nodeIdContainsSequence, -} from './nodeIdUtils' -import { - type ChildrenByParentDepPath, + type ChildrenByParentId, type DependenciesTree, type LinkedDependency, type ImporterToResolve, + type ImporterToResolveOptions, type ParentPkgAliases, type PendingNode, type PkgAddress, resolveRootDependencies, type ResolvedPackage, - type ResolvedPackagesByDepPath, + type ResolvedPkgsById, type ResolutionContext, } from './resolveDependencies' -export * from './nodeIdUtils' export type { LinkedDependency, ResolvedPackage, DependenciesTree, DependenciesTreeNode } from './resolveDependencies' export interface ResolvedImporters { @@ -113,7 +111,7 @@ export interface ResolveDependencyTreeResult { [pkgId: string]: string } resolvedImporters: ResolvedImporters - resolvedPackagesByDepPath: ResolvedPackagesByDepPath + resolvedPkgsById: ResolvedPkgsById wantedToBeSkippedPackageIds: Set appliedPatches: Set time?: Record @@ -130,7 +128,7 @@ export async function resolveDependencyTree ( autoInstallPeersFromHighestMatch: opts.autoInstallPeersFromHighestMatch === true, allowBuild: opts.allowBuild, allowedDeprecatedVersions: opts.allowedDeprecatedVersions, - childrenByParentDepPath: {} as ChildrenByParentDepPath, + childrenByParentId: {} as ChildrenByParentId, currentLockfile: opts.currentLockfile, defaultTag: opts.tag, dependenciesTree: new Map() as DependenciesTree, @@ -149,7 +147,7 @@ export async function resolveDependencyTree ( preferWorkspacePackages: opts.preferWorkspacePackages, readPackageHook: opts.hooks.readPackage, registries: opts.registries, - resolvedPackagesByDepPath: {} as ResolvedPackagesByDepPath, + resolvedPkgsById: {} as ResolvedPkgsById, resolutionMode: opts.resolutionMode, skipped: wantedToBeSkippedPackageIds, storeController: opts.storeController, @@ -170,15 +168,16 @@ export async function resolveDependencyTree ( // We only need to proceed resolving every dependency // if the newly added dependency has peer dependencies. const proceed = importer.id === '.' || importer.hasRemovedDependencies === true || importer.wantedDependencies.some((wantedDep: any) => wantedDep.isNew) // eslint-disable-line @typescript-eslint/no-explicit-any - const resolveOpts = { + const resolveOpts: ImporterToResolveOptions = { currentDepth: 0, parentPkg: { installable: true, nodeId: `>${importer.id}>`, optional: false, - depPath: importer.id, + pkgId: importer.id, rootDir: importer.rootDir, }, + parentIds: [importer.id], proceed, resolvedDependencies: { ...projectSnapshot.dependencies, @@ -206,8 +205,9 @@ export async function resolveDependencyTree ( ctx.pendingNodes.forEach((pendingNode) => { ctx.dependenciesTree.set(pendingNode.nodeId, { - children: () => buildTree(ctx, pendingNode.nodeId, pendingNode.resolvedPackage.id, - ctx.childrenByParentDepPath[pendingNode.resolvedPackage.depPath], pendingNode.depth + 1, pendingNode.installable), + children: () => buildTree(ctx, pendingNode.resolvedPackage.id, + pendingNode.parentIds, + ctx.childrenByParentId[pendingNode.resolvedPackage.id], pendingNode.depth + 1, pendingNode.installable), depth: pendingNode.depth, installable: pendingNode.installable, resolvedPackage: pendingNode.resolvedPackage, @@ -250,7 +250,7 @@ export async function resolveDependencyTree ( dependenciesTree: ctx.dependenciesTree, outdatedDependencies: ctx.outdatedDependencies, resolvedImporters, - resolvedPackagesByDepPath: ctx.resolvedPackagesByDepPath, + resolvedPkgsById: ctx.resolvedPkgsById, wantedToBeSkippedPackageIds, appliedPatches: ctx.appliedPatches, time, @@ -260,40 +260,40 @@ export async function resolveDependencyTree ( function buildTree ( ctx: { - childrenByParentDepPath: ChildrenByParentDepPath + childrenByParentId: ChildrenByParentId dependenciesTree: DependenciesTree - resolvedPackagesByDepPath: ResolvedPackagesByDepPath + resolvedPkgsById: ResolvedPkgsById skipped: Set }, - parentNodeId: string, parentId: string, - children: Array<{ alias: string, depPath: string }>, + parentIds: string[], + children: Array<{ alias: string, id: string }>, depth: number, installable: boolean ): Record { const childrenNodeIds: Record = {} for (const child of children) { - if (child.depPath.startsWith('link:')) { - childrenNodeIds[child.alias] = child.depPath + if (child.id.startsWith('link:')) { + childrenNodeIds[child.alias] = child.id continue } - if (nodeIdContainsSequence(parentNodeId, parentId, child.depPath) || parentId === child.depPath) { + if (parentIdsContainSequence(parentIds, parentId, child.id) || parentId === child.id) { continue } - const childNodeId = createNodeId(parentNodeId, child.depPath) + const childNodeId = nextNodeId() childrenNodeIds[child.alias] = childNodeId - installable = installable || !ctx.skipped.has(child.depPath) + installable = installable || !ctx.skipped.has(child.id) ctx.dependenciesTree.set(childNodeId, { children: () => buildTree(ctx, - childNodeId, - child.depPath, - ctx.childrenByParentDepPath[child.depPath], + child.id, + [...parentIds, child.id], + ctx.childrenByParentId[child.id], depth + 1, installable ), depth, installable, - resolvedPackage: ctx.resolvedPackagesByDepPath[child.depPath], + resolvedPackage: ctx.resolvedPkgsById[child.id], }) } return childrenNodeIds diff --git a/pkg-manager/resolve-dependencies/src/resolvePeers.ts b/pkg-manager/resolve-dependencies/src/resolvePeers.ts index 273865cdfc..b88080250e 100644 --- a/pkg-manager/resolve-dependencies/src/resolvePeers.ts +++ b/pkg-manager/resolve-dependencies/src/resolvePeers.ts @@ -10,11 +10,9 @@ import type { PeerDependencyIssuesByProjects, } from '@pnpm/types' import { depPathToFilename, createPeersDirSuffix, type PeerId } from '@pnpm/dependency-path' -import memoize from 'mem' import mapValues from 'ramda/src/map' import partition from 'ramda/src/partition' import pick from 'ramda/src/pick' -import scan from 'ramda/src/scan' import { type ChildrenMap, type PeerDependencies, @@ -24,7 +22,6 @@ import { } from './resolveDependencies' import { type ResolvedImporters } from './resolveDependencyTree' import { mergePeers } from './mergePeers' -import { createNodeId, splitNodeId } from './nodeIdUtils' import { dedupeInjectedDeps } from './dedupeInjectedDeps' export interface GenericDependenciesGraphNode { @@ -107,10 +104,12 @@ export async function resolvePeers ( // eslint-disable-next-line no-await-in-loop const { finishing } = await resolvePeersOfChildren(directNodeIdsByAlias, pkgsByName, { allPeerDepNames: opts.allPeerDepNames, - getParentPkgs: {}, + parentPkgsOfNode: new Map(), dependenciesTree: opts.dependenciesTree, depGraph, lockfileDir: opts.lockfileDir, + parentNodeIds: [], + parentDepPathsChain: [], pathsByNodeId, pathsByNodeIdPromises, depPathsByPkgId, @@ -271,6 +270,7 @@ function createPkgsByName ( alias, node: dependenciesTree.get(directNodeIdsByAlias[alias])!, nodeId: directNodeIdsByAlias[alias], + parentNodeIds: [], })) ) const _updateParentRefs = updateParentRefs.bind(null, parentRefs) @@ -281,6 +281,7 @@ function createPkgsByName ( depth: 0, version, nodeId: linkedDir, + parentNodeIds: [], } _updateParentRefs(name, pkg) if (alias && alias !== name) { @@ -318,14 +319,17 @@ interface ParentPkgInfo { depth?: number occurrence?: number } -type GetParentPkgs = () => Record + +type ParentPkgsOfNode = Map> async function resolvePeersOfNode ( nodeId: string, parentParentPkgs: ParentRefs, ctx: ResolvePeersContext & { allPeerDepNames: Set - getParentPkgs: Record + parentPkgsOfNode: ParentPkgsOfNode + parentNodeIds: string[] + parentDepPathsChain: string[] dependenciesTree: DependenciesTree depGraph: GenericDependenciesGraph virtualStoreDir: string @@ -352,6 +356,7 @@ async function resolvePeersOfNode ( if (typeof node.children === 'function') { node.children = node.children() } + const parentNodeIds = [...ctx.parentNodeIds, nodeId] const children = node.children let parentPkgs: ParentRefs if (Object.keys(children).length === 0) { @@ -365,6 +370,7 @@ async function resolvePeersOfNode ( alias, node: ctx.dependenciesTree.get(nodeId)!, nodeId, + parentNodeIds, }) } } @@ -398,7 +404,11 @@ async function resolvePeersOfNode ( resolvedPeers: unknownResolvedPeersOfChildren, missingPeers: missingPeersOfChildren, finishing, - } = await resolvePeersOfChildren(children, parentPkgs, ctx) + } = await resolvePeersOfChildren(children, parentPkgs, { + ...ctx, + parentNodeIds, + parentDepPathsChain: ctx.parentDepPathsChain.includes(resolvedPackage.depPath) ? ctx.parentDepPathsChain : [...ctx.parentDepPathsChain, resolvedPackage.depPath], + }) const { resolvedPeers, missingPeers } = Object.keys(resolvedPackage.peerDependencies).length === 0 ? { resolvedPeers: new Map(), missingPeers: new Set() } @@ -411,6 +421,7 @@ async function resolvePeersOfNode ( peerDependencyIssues: ctx.peerDependencyIssues, resolvedPackage, rootDir: ctx.rootDir, + parentNodeIds, }) const allResolvedPeers = unknownResolvedPeersOfChildren @@ -539,7 +550,7 @@ async function resolvePeersOfNode ( ctx.depGraph[depPath] = { ...(node.resolvedPackage as T), children: Object.assign( - getPreviouslyResolvedChildren(nodeId, ctx.dependenciesTree), + getPreviouslyResolvedChildren(ctx, (node.resolvedPackage as T).depPath), children, Object.fromEntries(resolvedPeers.entries()) ), @@ -558,7 +569,7 @@ async function resolvePeersOfNode ( } function findHit (ctx: { - getParentPkgs: Record + parentPkgsOfNode: ParentPkgsOfNode peersCache: PeersCache purePkgs: Set pathsByNodeId: Map @@ -599,12 +610,12 @@ function findHit (ctx: { } function parentPackagesMatch (ctx: { - getParentPkgs: Record + parentPkgsOfNode: ParentPkgsOfNode purePkgs: Set }, cachedNodeId: string, checkedNodeId: string): boolean { - const cachedParentPkgs = ctx.getParentPkgs[cachedNodeId]?.() + const cachedParentPkgs = ctx.parentPkgsOfNode.get(cachedNodeId) if (!cachedParentPkgs) return false - const checkedParentPkgs = ctx.getParentPkgs[checkedNodeId]?.() + const checkedParentPkgs = ctx.parentPkgsOfNode.get(checkedNodeId) if (!checkedParentPkgs) return false if (Object.keys(cachedParentPkgs).length !== Object.keys(checkedParentPkgs).length) return false const maxDepth = Object.values(checkedParentPkgs) @@ -640,27 +651,34 @@ function parentPkgsHaveSingleOccurrence (parentPkgs: Record (nodeId: string, dependenciesTree: DependenciesTree): ChildrenMap { - const parentIds = splitNodeId(nodeId) - const ownId = parentIds.pop() +function getPreviouslyResolvedChildren ( + { + parentNodeIds, + parentDepPathsChain, + dependenciesTree, + }: { + parentNodeIds: string[] + parentDepPathsChain: string[] + dependenciesTree: DependenciesTree + }, + currentDepPath: string +): ChildrenMap { const allChildren: ChildrenMap = {} - if (!ownId || !parentIds.includes(ownId)) return allChildren + if (!currentDepPath || !parentDepPathsChain.includes(currentDepPath)) return allChildren - const nodeIdChunks = parentIds.join('>').split(`>${ownId}>`) - nodeIdChunks.pop() - nodeIdChunks.reduce((accNodeId, part) => { - accNodeId += `>${part}>${ownId}` - const parentNode = dependenciesTree.get(`${accNodeId}>`)! - if (typeof parentNode.children === 'function') { - parentNode.children = parentNode.children() + for (let i = parentNodeIds.length - 1; i >= 0; i--) { + const parentNode = dependenciesTree.get(parentNodeIds[i])! + if ((parentNode.resolvedPackage as T).depPath === currentDepPath) { + if (typeof parentNode.children === 'function') { + parentNode.children = parentNode.children() + } + Object.assign( + allChildren, + parentNode.children + ) } - Object.assign( - allChildren, - parentNode.children - ) - return accNodeId - }, '') + } return allChildren } @@ -671,7 +689,9 @@ async function resolvePeersOfChildren ( parentPkgs: ParentRefs, ctx: ResolvePeersContext & { allPeerDepNames: Set - getParentPkgs: Record + parentPkgsOfNode: ParentPkgsOfNode + parentNodeIds: string[] + parentDepPathsChain: string[] peerDependencyIssues: Pick peersCache: PeersCache virtualStoreDir: string @@ -701,24 +721,21 @@ async function resolvePeersOfChildren ( const calculateDepPaths: CalculateDepPath[] = [] const graph = [] const finishingList: FinishingResolutionPromise[] = [] - const cachedGetParentPkgs = memoize(() => { - const parentDepPaths: Record = {} - for (const [name, parentPkg] of Object.entries(parentPkgs)) { - if (!ctx.allPeerDepNames.has(name)) continue - if (parentPkg.nodeId && !parentPkg.nodeId.startsWith('link:')) { - parentDepPaths[name] = { - depPath: (ctx.dependenciesTree.get(parentPkg.nodeId)!.resolvedPackage as T).depPath, - depth: parentPkg.depth, - occurrence: parentPkg.occurrence, - } - } else { - parentDepPaths[name] = { version: parentPkg.version } + const parentDepPaths: Record = {} + for (const [name, parentPkg] of Object.entries(parentPkgs)) { + if (!ctx.allPeerDepNames.has(name)) continue + if (parentPkg.nodeId && !parentPkg.nodeId.startsWith('link:')) { + parentDepPaths[name] = { + depPath: (ctx.dependenciesTree.get(parentPkg.nodeId)!.resolvedPackage as T).depPath, + depth: parentPkg.depth, + occurrence: parentPkg.occurrence, } + } else { + parentDepPaths[name] = { version: parentPkg.version } } - return parentDepPaths - }) + } for (const childNodeId of nodeIds) { - ctx.getParentPkgs[childNodeId] = cachedGetParentPkgs + ctx.parentPkgsOfNode.set(childNodeId, parentDepPaths) } for (const childNodeId of nodeIds) { const { @@ -765,6 +782,7 @@ function _resolvePeers ( lockfileDir: string nodeId: string parentPkgs: ParentRefs + parentNodeIds: string[] resolvedPackage: T dependenciesTree: DependenciesTree rootDir: string @@ -781,11 +799,7 @@ function _resolvePeers ( if (!resolved) { missingPeers.add(peerName) - const location = getLocationFromNodeIdAndPkg({ - dependenciesTree: ctx.dependenciesTree, - nodeId: ctx.nodeId, - pkg: ctx.resolvedPackage, - }) + const location = getLocationFromParentNodeIds(ctx) if (!ctx.peerDependencyIssues.missing[peerName]) { ctx.peerDependencyIssues.missing[peerName] = [] } @@ -798,19 +812,15 @@ function _resolvePeers ( } if (!semverUtils.satisfiesWithPrereleases(resolved.version, peerVersionRange, true)) { - const location = getLocationFromNodeIdAndPkg({ - dependenciesTree: ctx.dependenciesTree, - nodeId: ctx.nodeId, - pkg: ctx.resolvedPackage, - }) + const location = getLocationFromParentNodeIds(ctx) if (!ctx.peerDependencyIssues.bad[peerName]) { ctx.peerDependencyIssues.bad[peerName] = [] } const peerLocation = resolved.nodeId == null ? [] - : getLocationFromNodeId({ + : getLocationFromParentNodeIds({ dependenciesTree: ctx.dependenciesTree, - nodeId: resolved.nodeId, + parentNodeIds: resolved.parentNodeIds, }).parents ctx.peerDependencyIssues.bad[peerName].push({ foundVersion: resolved.version, @@ -831,41 +841,19 @@ interface Location { parents: ParentPackages } -function getLocationFromNodeIdAndPkg ( +function getLocationFromParentNodeIds ( { dependenciesTree, - nodeId, - pkg, + parentNodeIds, }: { dependenciesTree: DependenciesTree - nodeId: string - pkg: { name: string, version: string } + parentNodeIds: string[] } ): Location { - const { projectId, parents } = getLocationFromNodeId({ dependenciesTree, nodeId }) - parents.push({ name: pkg.name, version: pkg.version }) - return { - projectId, - parents, - } -} - -function getLocationFromNodeId ( - { - dependenciesTree, - nodeId, - }: { - dependenciesTree: DependenciesTree - nodeId: string - } -): Location { - const parts = splitNodeId(nodeId).slice(0, -1) - const parents = scan((prevNodeId, pkgId) => createNodeId(prevNodeId, pkgId), '>', parts) - .slice(2) - + const parents = parentNodeIds .map((nid) => pick(['name', 'version'], dependenciesTree.get(nid)!.resolvedPackage as ResolvedPackage)) return { - projectId: parts[0], + projectId: '.', parents, } } @@ -881,24 +869,27 @@ interface ParentRef { nodeId?: string alias?: string occurrence: number + parentNodeIds: string[] } interface ParentPkgNode { alias: string nodeId: string node: DependenciesTreeNode + parentNodeIds: string[] } function toPkgByName (nodes: Array>): ParentRefs { const pkgsByName: ParentRefs = {} const _updateParentRefs = updateParentRefs.bind(null, pkgsByName) - for (const { alias, node, nodeId } of nodes) { + for (const { alias, node, nodeId, parentNodeIds } of nodes) { const pkg = { alias, depth: node.depth, nodeId, version: node.resolvedPackage.version, occurrence: 0, + parentNodeIds, } _updateParentRefs(alias, pkg) if (alias !== node.resolvedPackage.name) { diff --git a/pkg-manager/resolve-dependencies/src/updateLockfile.ts b/pkg-manager/resolve-dependencies/src/updateLockfile.ts index 83f3d0be13..48fb44131a 100644 --- a/pkg-manager/resolve-dependencies/src/updateLockfile.ts +++ b/pkg-manager/resolve-dependencies/src/updateLockfile.ts @@ -61,8 +61,7 @@ function toLockfileDependency ( } ): PackageSnapshot { const lockfileResolution = toLockfileResolution( - { id: pkg.id, name: pkg.name, version: pkg.version }, - opts.depPath, + { name: pkg.name, version: pkg.version }, pkg.resolution, opts.registry, opts.lockfileIncludeTarballUrl @@ -175,11 +174,9 @@ function updateResolvedDeps ( function toLockfileResolution ( pkg: { - id: string name: string version: string }, - depPath: string, resolution: Resolution, registry: string, lockfileIncludeTarballUrl?: boolean diff --git a/pkg-manager/resolve-dependencies/test/nodeIdUtils.test.ts b/pkg-manager/resolve-dependencies/test/nodeIdUtils.test.ts deleted file mode 100644 index 6cc4ec3a2e..0000000000 --- a/pkg-manager/resolve-dependencies/test/nodeIdUtils.test.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { nodeIdContainsSequence } from '../lib/nodeIdUtils' - -test('nodeIdContainsSequence()', () => { - expect(nodeIdContainsSequence('>.>b>a>c>b>a>', 'a', 'b')).toBeTruthy() -}) diff --git a/pkg-manager/resolve-dependencies/test/parentIdsContainSequence.test.ts b/pkg-manager/resolve-dependencies/test/parentIdsContainSequence.test.ts new file mode 100644 index 0000000000..760c4cefe1 --- /dev/null +++ b/pkg-manager/resolve-dependencies/test/parentIdsContainSequence.test.ts @@ -0,0 +1,5 @@ +import { parentIdsContainSequence } from '../lib/parentIdsContainSequence' + +test('parentIdsContainSequence()', () => { + expect(parentIdsContainSequence(['.', 'b', 'a', 'c', 'b', 'a'], 'a', 'b')).toBeTruthy() +}) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 60376fb7fb..6f9d9d0237 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -4307,9 +4307,6 @@ importers: is-subdir: specifier: ^1.2.0 version: 1.2.0 - mem: - specifier: ^8.1.1 - version: 8.1.1 normalize-path: specifier: ^3.0.0 version: 3.0.0