perf: reduce memory usage (#8084)

ref #8072
This commit is contained in:
Zoltan Kochan
2024-05-16 20:55:08 +02:00
committed by GitHub
parent 34bc8f48e1
commit ef73c19f15
13 changed files with 189 additions and 202 deletions

View File

@@ -0,0 +1,6 @@
---
"@pnpm/resolve-dependencies": patch
"pnpm": patch
---
Decrease memory consumption [#8084](https://github.com/pnpm/pnpm/pull/8084).

View File

@@ -53,7 +53,6 @@
"graph-cycles": "1.2.1",
"is-inner-link": "^4.0.0",
"is-subdir": "^1.2.0",
"mem": "^8.1.1",
"normalize-path": "^3.0.0",
"p-defer": "^3.0.0",
"path-exists": "^4.0.0",

View File

@@ -129,7 +129,7 @@ export async function resolveDependencies (
dependenciesTree,
outdatedDependencies,
resolvedImporters,
resolvedPackagesByDepPath,
resolvedPkgsById,
wantedToBeSkippedPackageIds,
appliedPatches,
time,
@@ -301,7 +301,7 @@ export async function resolveDependencies (
// waiting till package requests are finished
async function waitTillAllFetchingsFinish (): Promise<void> {
await Promise.all(Object.values(resolvedPackagesByDepPath).map(async ({ fetching }) => {
await Promise.all(Object.values(resolvedPkgsById).map(async ({ fetching }) => {
try {
await fetching?.()
} catch {}

View File

@@ -0,0 +1,5 @@
let nodeIdCounter = 0
export function nextNodeId (): string {
return (++nodeIdCounter).toString()
}

View File

@@ -1,22 +0,0 @@
export function nodeIdContains (nodeId: string, pkgId: string): boolean {
const pkgIds = splitNodeId(nodeId)
return pkgIds.includes(pkgId)
}
export function nodeIdContainsSequence (nodeId: string, pkgId1: string, pkgId2: string): boolean {
const pkgIds = splitNodeId(nodeId)
pkgIds.pop()
const pkg1Index = pkgIds.indexOf(pkgId1)
if (pkg1Index === -1) return false
const pkg2Index = pkgIds.lastIndexOf(pkgId2)
return pkg1Index < pkg2Index
}
export function createNodeId (parentNodeId: string, pkgId: string): string {
// using ">" as a separator because it will never be used inside a package ID
return `${parentNodeId}${pkgId}>`
}
export function splitNodeId (nodeId: string): string[] {
return nodeId.split('>').slice(1, -1)
}

View File

@@ -0,0 +1,8 @@
export function parentIdsContainSequence (pkgIds: string[], pkgId1: string, pkgId2: string): boolean {
const pkg1Index = pkgIds.indexOf(pkgId1)
if (pkg1Index === -1 || pkg1Index === pkgIds.length - 1) {
return false
}
const pkg2Index = pkgIds.lastIndexOf(pkgId2)
return pkg1Index < pkg2Index && pkg2Index !== pkgIds.length - 1
}

View File

@@ -50,12 +50,8 @@ import semver from 'semver'
import { encodePkgId } from './encodePkgId'
import { getNonDevWantedDependencies, type WantedDependency } from './getNonDevWantedDependencies'
import { safeIntersect } from './mergePeers'
import {
createNodeId,
nodeIdContainsSequence,
nodeIdContains,
splitNodeId,
} from './nodeIdUtils'
import { nextNodeId } from './nextNodeId'
import { parentIdsContainSequence } from './parentIdsContainSequence'
import { hoistPeers, getHoistableOptionalPeers } from './hoistPeers'
import { wantedDepIsLocallyAvailable } from './wantedDepIsLocallyAvailable'
import { replaceVersionInPref } from './replaceVersionInPref'
@@ -64,13 +60,14 @@ const dependencyResolvedLogger = logger('_dependency_resolved')
const omitDepsFields = omit(['dependencies', 'optionalDependencies', 'peerDependencies', 'peerDependenciesMeta'])
export function nodeIdToParents (
nodeId: string,
resolvedPackagesByDepPath: ResolvedPackagesByDepPath
export function getPkgsInfoFromIds (
ids: string[],
resolvedPkgsById: ResolvedPkgsById
): Array<{ id: string, name: string, version: string }> {
return splitNodeId(nodeId).slice(1)
.map((depPath) => {
const { id, name, version } = resolvedPackagesByDepPath[depPath]
return ids
.slice(1)
.map((id) => {
const { name, version } = resolvedPkgsById[id]
return { id, name, version }
})
}
@@ -99,7 +96,7 @@ string,
DependenciesTreeNode<T>
>
export type ResolvedPackagesByDepPath = Record<string, ResolvedPackage>
export type ResolvedPkgsById = Record<string, ResolvedPackage>
export interface LinkedDependency {
isLinkedDependency: true
@@ -120,12 +117,13 @@ export interface PendingNode {
resolvedPackage: ResolvedPackage
depth: number
installable: boolean
parentIds: string[]
}
export interface ChildrenByParentDepPath {
[depPath: string]: Array<{
export interface ChildrenByParentId {
[id: string]: Array<{
alias: string
depPath: string
id: string
}>
}
@@ -142,9 +140,9 @@ export interface ResolutionContext {
dryRun: boolean
forceFullResolution: boolean
ignoreScripts?: boolean
resolvedPackagesByDepPath: ResolvedPackagesByDepPath
resolvedPkgsById: ResolvedPkgsById
outdatedDependencies: { [pkgId: string]: string }
childrenByParentDepPath: ChildrenByParentDepPath
childrenByParentId: ChildrenByParentId
patchedDependencies?: Record<string, PatchFile>
pendingNodes: PendingNode[]
wantedLockfile: Lockfile
@@ -246,7 +244,7 @@ export interface ResolvedPackage {
parentImporterIds: Set<string>
}
type ParentPkg = Pick<PkgAddress, 'nodeId' | 'installable' | 'depPath' | 'rootDir' | 'optional'>
type ParentPkg = Pick<PkgAddress, 'nodeId' | 'installable' | 'rootDir' | 'optional' | 'pkgId'>
export type ParentPkgAliases = Record<string, PkgAddress | true>
@@ -254,6 +252,7 @@ export type UpdateMatchingFunction = (pkgName: string) => boolean
interface ResolvedDependenciesOptions {
currentDepth: number
parentIds: string[]
parentPkg: ParentPkg
parentPkgAliases: ParentPkgAliases
// If the package has been updated, the dependencies
@@ -377,12 +376,14 @@ interface PkgAddressesByImportersWithoutPeers extends PeersResolutionResult {
pkgAddresses: Array<PkgAddress | LinkedDependency>
}
export type ImporterToResolveOptions = Omit<ResolvedDependenciesOptions, 'parentPkgAliases' | 'publishedBy'>
export interface ImporterToResolve {
updatePackageManifest: boolean
preferredVersions: PreferredVersions
parentPkgAliases: ParentPkgAliases
wantedDependencies: Array<WantedDependency & { updateDepth?: number }>
options: Omit<ResolvedDependenciesOptions, 'parentPkgAliases' | 'publishedBy'>
options: ImporterToResolveOptions
}
interface ResolveDependenciesOfImportersResult {
@@ -456,7 +457,7 @@ async function resolveDependenciesOfImporters (
if (pkgAddress.updated) {
ctx.updatedSet.add(pkgAddress.alias)
}
const resolvedPackage = ctx.resolvedPackagesByDepPath[pkgAddress.depPath]
const resolvedPackage = ctx.resolvedPkgsById[pkgAddress.pkgId]
if (!resolvedPackage) continue // This will happen only with linked dependencies
if (!Object.prototype.hasOwnProperty.call(newPreferredVersions, resolvedPackage.name)) {
newPreferredVersions[resolvedPackage.name] = { ...importer.preferredVersions[resolvedPackage.name] }
@@ -586,7 +587,7 @@ export async function resolveDependencies (
if (pkgAddress.updated) {
ctx.updatedSet.add(pkgAddress.alias)
}
const resolvedPackage = ctx.resolvedPackagesByDepPath[pkgAddress.depPath]
const resolvedPackage = ctx.resolvedPkgsById[pkgAddress.pkgId]
if (!resolvedPackage) continue // This will happen only with linked dependencies
if (!Object.prototype.hasOwnProperty.call(newPreferredVersions, resolvedPackage.name)) {
newPreferredVersions[resolvedPackage.name] = { ...preferredVersions[resolvedPackage.name] }
@@ -741,6 +742,7 @@ async function resolveDependenciesOfDependency (
updateMatching: options.updateMatching,
supportedArchitectures: options.supportedArchitectures,
updateToLatest: options.updateToLatest,
parentIds: options.parentIds,
}
const resolveDependencyResult = await resolveDependency(extendedWantedDep.wantedDependency, ctx, resolveDependencyOpts)
@@ -773,6 +775,7 @@ async function resolveDependenciesOfDependency (
parentPkg: resolveDependencyResult,
dependencyLockfile: extendedWantedDep.infoFromLockfile?.dependencyLockfile,
parentDepth: options.currentDepth,
parentIds: [...options.parentIds, resolveDependencyResult.pkgId],
updateDepth,
prefix: options.prefix,
updateMatching: options.updateMatching,
@@ -820,6 +823,7 @@ async function resolveChildren (
ctx: ResolutionContext,
{
parentPkg,
parentIds,
dependencyLockfile,
parentDepth,
updateDepth,
@@ -828,6 +832,7 @@ async function resolveChildren (
supportedArchitectures,
}: {
parentPkg: PkgAddress
parentIds: string[]
dependencyLockfile: PackageSnapshot | undefined
parentDepth: number
updateDepth: number
@@ -881,11 +886,12 @@ async function resolveChildren (
updateDepth,
updateMatching,
supportedArchitectures,
parentIds,
}
)
ctx.childrenByParentDepPath[parentPkg.depPath] = pkgAddresses.map((child) => ({
ctx.childrenByParentId[parentPkg.pkgId] = pkgAddresses.map((child) => ({
alias: child.alias,
depPath: child.depPath,
id: child.pkgId,
}))
ctx.dependenciesTree.set(parentPkg.nodeId, {
children: pkgAddresses.reduce((chn, child) => {
@@ -894,7 +900,7 @@ async function resolveChildren (
}, {} as Record<string, string>),
depth: parentDepth,
installable: parentPkg.installable,
resolvedPackage: ctx.resolvedPackagesByDepPath[parentPkg.depPath],
resolvedPackage: ctx.resolvedPkgsById[parentPkg.pkgId],
})
return resolvingPeers
}
@@ -1078,6 +1084,7 @@ interface ResolveDependencyOptions {
dependencyLockfile?: PackageSnapshot
}
parentPkg: ParentPkg
parentIds: string[]
parentPkgAliases: ParentPkgAliases
preferredVersions: PreferredVersions
prefix: string
@@ -1166,7 +1173,7 @@ async function resolveDependency (
supportedArchitectures: options.supportedArchitectures,
onFetchError: (err: any) => { // eslint-disable-line
err.prefix = options.prefix
err.pkgsStack = nodeIdToParents(options.parentPkg.nodeId, ctx.resolvedPackagesByDepPath)
err.pkgsStack = getPkgsInfoFromIds(options.parentIds, ctx.resolvedPkgsById)
return err
},
updateToLatest: options.updateToLatest,
@@ -1180,21 +1187,21 @@ async function resolveDependency (
pref: wantedDependency.pref,
version: wantedDependency.alias ? wantedDependency.pref : undefined,
},
parents: nodeIdToParents(options.parentPkg.nodeId, ctx.resolvedPackagesByDepPath),
parents: getPkgsInfoFromIds(options.parentIds, ctx.resolvedPkgsById),
prefix: options.prefix,
reason: 'resolution_failure',
})
return null
}
err.prefix = options.prefix
err.pkgsStack = nodeIdToParents(options.parentPkg.nodeId, ctx.resolvedPackagesByDepPath)
err.pkgsStack = getPkgsInfoFromIds(options.parentIds, ctx.resolvedPkgsById)
throw err
}
dependencyResolvedLogger.debug({
resolution: pkgResponse.body.id,
wanted: {
dependentId: options.parentPkg.depPath,
dependentId: options.parentPkg.pkgId,
name: wantedDependency.alias,
rawSpec: wantedDependency.pref,
},
@@ -1289,11 +1296,11 @@ async function resolveDependency (
// because zoo is a new parent package:
// foo > bar > qar > zoo > qar
if (
nodeIdContainsSequence(
options.parentPkg.nodeId,
options.parentPkg.depPath,
depPath
) || depPath === options.parentPkg.depPath
parentIdsContainSequence(
options.parentIds,
options.parentPkg.pkgId,
pkgResponse.body.id
) || pkgResponse.body.id === options.parentPkg.pkgId
) {
return null
}
@@ -1342,14 +1349,12 @@ async function resolveDependency (
}
// In case of leaf dependencies (dependencies that have no prod deps or peer deps),
// we only ever need to analyze one leaf dep in a graph, so the nodeId can be short and stateless.
const nodeId = pkgIsLeaf(pkg)
? `>${depPath}>`
: createNodeId(options.parentPkg.nodeId, depPath)
const nodeId = pkgIsLeaf(pkg) ? pkgResponse.body.id : nextNodeId()
const parentIsInstallable = options.parentPkg.installable === undefined || options.parentPkg.installable
const installable = parentIsInstallable && pkgResponse.body.isInstallable !== false
const isNew = !ctx.resolvedPackagesByDepPath[depPath]
const parentImporterId = options.parentPkg.nodeId.substring(0, options.parentPkg.nodeId.indexOf('>', 1) + 1)
const isNew = !ctx.resolvedPkgsById[pkgResponse.body.id]
const parentImporterId = options.parentIds[0]
let resolveChildren = false
const currentIsOptional = wantedDependency.optional || options.parentPkg.optional
@@ -1379,7 +1384,7 @@ async function resolveDependency (
// WARN: It is very important to keep this sync
// Otherwise, deprecation messages for the same package might get written several times
ctx.resolvedPackagesByDepPath[depPath] = getResolvedPackage({
ctx.resolvedPkgsById[pkgResponse.body.id] = getResolvedPackage({
allowBuild: ctx.allowBuild,
dependencyLockfile: currentPkg.dependencyLockfile,
depPath,
@@ -1394,17 +1399,17 @@ async function resolveDependency (
optional: currentIsOptional,
})
} else {
ctx.resolvedPackagesByDepPath[depPath].prod = ctx.resolvedPackagesByDepPath[depPath].prod || !wantedDependency.dev && !wantedDependency.optional
ctx.resolvedPackagesByDepPath[depPath].dev = ctx.resolvedPackagesByDepPath[depPath].dev || wantedDependency.dev
ctx.resolvedPackagesByDepPath[depPath].optional = ctx.resolvedPackagesByDepPath[depPath].optional && currentIsOptional
ctx.resolvedPkgsById[pkgResponse.body.id].prod = ctx.resolvedPkgsById[pkgResponse.body.id].prod || !wantedDependency.dev && !wantedDependency.optional
ctx.resolvedPkgsById[pkgResponse.body.id].dev = ctx.resolvedPkgsById[pkgResponse.body.id].dev || wantedDependency.dev
ctx.resolvedPkgsById[pkgResponse.body.id].optional = ctx.resolvedPkgsById[pkgResponse.body.id].optional && currentIsOptional
if (ctx.hoistPeers) {
resolveChildren = !ctx.missingPeersOfChildrenByPkgId[pkgResponse.body.id].missingPeersOfChildren.resolved &&
!ctx.resolvedPackagesByDepPath[depPath].parentImporterIds.has(parentImporterId)
ctx.resolvedPackagesByDepPath[depPath].parentImporterIds.add(parentImporterId)
!ctx.resolvedPkgsById[pkgResponse.body.id].parentImporterIds.has(parentImporterId)
ctx.resolvedPkgsById[pkgResponse.body.id].parentImporterIds.add(parentImporterId)
}
if (ctx.resolvedPackagesByDepPath[depPath].fetching == null && pkgResponse.fetching != null) {
ctx.resolvedPackagesByDepPath[depPath].fetching = pkgResponse.fetching
ctx.resolvedPackagesByDepPath[depPath].filesIndexFile = pkgResponse.filesIndexFile!
if (ctx.resolvedPkgsById[pkgResponse.body.id].fetching == null && pkgResponse.fetching != null) {
ctx.resolvedPkgsById[pkgResponse.body.id].fetching = pkgResponse.fetching
ctx.resolvedPkgsById[pkgResponse.body.id].filesIndexFile = pkgResponse.filesIndexFile!
}
if (ctx.dependenciesTree.has(nodeId)) {
@@ -1413,9 +1418,10 @@ async function resolveDependency (
ctx.pendingNodes.push({
alias: wantedDependency.alias || pkg.name,
depth: options.currentDepth,
parentIds: options.parentIds,
installable,
nodeId,
resolvedPackage: ctx.resolvedPackagesByDepPath[depPath],
resolvedPackage: ctx.resolvedPkgsById[pkgResponse.body.id],
})
}
}
@@ -1424,9 +1430,9 @@ async function resolveDependency (
? path.resolve(ctx.lockfileDir, (pkgResponse.body.resolution as DirectoryResolution).directory)
: options.prefix
let missingPeersOfChildren!: MissingPeersOfChildren | undefined
if (ctx.hoistPeers && !nodeIdContains(options.parentPkg.nodeId, depPath)) {
if (ctx.hoistPeers && !options.parentIds.includes(pkgResponse.body.id)) {
if (ctx.missingPeersOfChildrenByPkgId[pkgResponse.body.id]) {
if (!options.parentPkg.nodeId.startsWith(ctx.missingPeersOfChildrenByPkgId[pkgResponse.body.id].parentImporterId)) {
if (options.parentIds[0] !== ctx.missingPeersOfChildrenByPkgId[pkgResponse.body.id].parentImporterId) {
missingPeersOfChildren = ctx.missingPeersOfChildrenByPkgId[pkgResponse.body.id].missingPeersOfChildren
}
} else {
@@ -1453,7 +1459,7 @@ async function resolveDependency (
pkgId: pkgResponse.body.id,
rootDir,
missingPeers: getMissingPeers(pkg),
optional: ctx.resolvedPackagesByDepPath[depPath].optional,
optional: ctx.resolvedPkgsById[pkgResponse.body.id].optional,
// Next fields are actually only needed when isNew = true
installable,

View File

@@ -11,25 +11,23 @@ import {
import partition from 'ramda/src/partition'
import zipObj from 'ramda/src/zipObj'
import { type WantedDependency } from './getNonDevWantedDependencies'
import { nextNodeId } from './nextNodeId'
import { parentIdsContainSequence } from './parentIdsContainSequence'
import {
createNodeId,
nodeIdContainsSequence,
} from './nodeIdUtils'
import {
type ChildrenByParentDepPath,
type ChildrenByParentId,
type DependenciesTree,
type LinkedDependency,
type ImporterToResolve,
type ImporterToResolveOptions,
type ParentPkgAliases,
type PendingNode,
type PkgAddress,
resolveRootDependencies,
type ResolvedPackage,
type ResolvedPackagesByDepPath,
type ResolvedPkgsById,
type ResolutionContext,
} from './resolveDependencies'
export * from './nodeIdUtils'
export type { LinkedDependency, ResolvedPackage, DependenciesTree, DependenciesTreeNode } from './resolveDependencies'
export interface ResolvedImporters {
@@ -113,7 +111,7 @@ export interface ResolveDependencyTreeResult {
[pkgId: string]: string
}
resolvedImporters: ResolvedImporters
resolvedPackagesByDepPath: ResolvedPackagesByDepPath
resolvedPkgsById: ResolvedPkgsById
wantedToBeSkippedPackageIds: Set<string>
appliedPatches: Set<string>
time?: Record<string, string>
@@ -130,7 +128,7 @@ export async function resolveDependencyTree<T> (
autoInstallPeersFromHighestMatch: opts.autoInstallPeersFromHighestMatch === true,
allowBuild: opts.allowBuild,
allowedDeprecatedVersions: opts.allowedDeprecatedVersions,
childrenByParentDepPath: {} as ChildrenByParentDepPath,
childrenByParentId: {} as ChildrenByParentId,
currentLockfile: opts.currentLockfile,
defaultTag: opts.tag,
dependenciesTree: new Map() as DependenciesTree<ResolvedPackage>,
@@ -149,7 +147,7 @@ export async function resolveDependencyTree<T> (
preferWorkspacePackages: opts.preferWorkspacePackages,
readPackageHook: opts.hooks.readPackage,
registries: opts.registries,
resolvedPackagesByDepPath: {} as ResolvedPackagesByDepPath,
resolvedPkgsById: {} as ResolvedPkgsById,
resolutionMode: opts.resolutionMode,
skipped: wantedToBeSkippedPackageIds,
storeController: opts.storeController,
@@ -170,15 +168,16 @@ export async function resolveDependencyTree<T> (
// We only need to proceed resolving every dependency
// if the newly added dependency has peer dependencies.
const proceed = importer.id === '.' || importer.hasRemovedDependencies === true || importer.wantedDependencies.some((wantedDep: any) => wantedDep.isNew) // eslint-disable-line @typescript-eslint/no-explicit-any
const resolveOpts = {
const resolveOpts: ImporterToResolveOptions = {
currentDepth: 0,
parentPkg: {
installable: true,
nodeId: `>${importer.id}>`,
optional: false,
depPath: importer.id,
pkgId: importer.id,
rootDir: importer.rootDir,
},
parentIds: [importer.id],
proceed,
resolvedDependencies: {
...projectSnapshot.dependencies,
@@ -206,8 +205,9 @@ export async function resolveDependencyTree<T> (
ctx.pendingNodes.forEach((pendingNode) => {
ctx.dependenciesTree.set(pendingNode.nodeId, {
children: () => buildTree(ctx, pendingNode.nodeId, pendingNode.resolvedPackage.id,
ctx.childrenByParentDepPath[pendingNode.resolvedPackage.depPath], pendingNode.depth + 1, pendingNode.installable),
children: () => buildTree(ctx, pendingNode.resolvedPackage.id,
pendingNode.parentIds,
ctx.childrenByParentId[pendingNode.resolvedPackage.id], pendingNode.depth + 1, pendingNode.installable),
depth: pendingNode.depth,
installable: pendingNode.installable,
resolvedPackage: pendingNode.resolvedPackage,
@@ -250,7 +250,7 @@ export async function resolveDependencyTree<T> (
dependenciesTree: ctx.dependenciesTree,
outdatedDependencies: ctx.outdatedDependencies,
resolvedImporters,
resolvedPackagesByDepPath: ctx.resolvedPackagesByDepPath,
resolvedPkgsById: ctx.resolvedPkgsById,
wantedToBeSkippedPackageIds,
appliedPatches: ctx.appliedPatches,
time,
@@ -260,40 +260,40 @@ export async function resolveDependencyTree<T> (
function buildTree (
ctx: {
childrenByParentDepPath: ChildrenByParentDepPath
childrenByParentId: ChildrenByParentId
dependenciesTree: DependenciesTree<ResolvedPackage>
resolvedPackagesByDepPath: ResolvedPackagesByDepPath
resolvedPkgsById: ResolvedPkgsById
skipped: Set<string>
},
parentNodeId: string,
parentId: string,
children: Array<{ alias: string, depPath: string }>,
parentIds: string[],
children: Array<{ alias: string, id: string }>,
depth: number,
installable: boolean
): Record<string, string> {
const childrenNodeIds: Record<string, string> = {}
for (const child of children) {
if (child.depPath.startsWith('link:')) {
childrenNodeIds[child.alias] = child.depPath
if (child.id.startsWith('link:')) {
childrenNodeIds[child.alias] = child.id
continue
}
if (nodeIdContainsSequence(parentNodeId, parentId, child.depPath) || parentId === child.depPath) {
if (parentIdsContainSequence(parentIds, parentId, child.id) || parentId === child.id) {
continue
}
const childNodeId = createNodeId(parentNodeId, child.depPath)
const childNodeId = nextNodeId()
childrenNodeIds[child.alias] = childNodeId
installable = installable || !ctx.skipped.has(child.depPath)
installable = installable || !ctx.skipped.has(child.id)
ctx.dependenciesTree.set(childNodeId, {
children: () => buildTree(ctx,
childNodeId,
child.depPath,
ctx.childrenByParentDepPath[child.depPath],
child.id,
[...parentIds, child.id],
ctx.childrenByParentId[child.id],
depth + 1,
installable
),
depth,
installable,
resolvedPackage: ctx.resolvedPackagesByDepPath[child.depPath],
resolvedPackage: ctx.resolvedPkgsById[child.id],
})
}
return childrenNodeIds

View File

@@ -10,11 +10,9 @@ import type {
PeerDependencyIssuesByProjects,
} from '@pnpm/types'
import { depPathToFilename, createPeersDirSuffix, type PeerId } from '@pnpm/dependency-path'
import memoize from 'mem'
import mapValues from 'ramda/src/map'
import partition from 'ramda/src/partition'
import pick from 'ramda/src/pick'
import scan from 'ramda/src/scan'
import {
type ChildrenMap,
type PeerDependencies,
@@ -24,7 +22,6 @@ import {
} from './resolveDependencies'
import { type ResolvedImporters } from './resolveDependencyTree'
import { mergePeers } from './mergePeers'
import { createNodeId, splitNodeId } from './nodeIdUtils'
import { dedupeInjectedDeps } from './dedupeInjectedDeps'
export interface GenericDependenciesGraphNode {
@@ -107,10 +104,12 @@ export async function resolvePeers<T extends PartialResolvedPackage> (
// eslint-disable-next-line no-await-in-loop
const { finishing } = await resolvePeersOfChildren(directNodeIdsByAlias, pkgsByName, {
allPeerDepNames: opts.allPeerDepNames,
getParentPkgs: {},
parentPkgsOfNode: new Map(),
dependenciesTree: opts.dependenciesTree,
depGraph,
lockfileDir: opts.lockfileDir,
parentNodeIds: [],
parentDepPathsChain: [],
pathsByNodeId,
pathsByNodeIdPromises,
depPathsByPkgId,
@@ -271,6 +270,7 @@ function createPkgsByName<T extends PartialResolvedPackage> (
alias,
node: dependenciesTree.get(directNodeIdsByAlias[alias])!,
nodeId: directNodeIdsByAlias[alias],
parentNodeIds: [],
}))
)
const _updateParentRefs = updateParentRefs.bind(null, parentRefs)
@@ -281,6 +281,7 @@ function createPkgsByName<T extends PartialResolvedPackage> (
depth: 0,
version,
nodeId: linkedDir,
parentNodeIds: [],
}
_updateParentRefs(name, pkg)
if (alias && alias !== name) {
@@ -318,14 +319,17 @@ interface ParentPkgInfo {
depth?: number
occurrence?: number
}
type GetParentPkgs = () => Record<string, ParentPkgInfo>
type ParentPkgsOfNode = Map<string, Record<string, ParentPkgInfo>>
async function resolvePeersOfNode<T extends PartialResolvedPackage> (
nodeId: string,
parentParentPkgs: ParentRefs,
ctx: ResolvePeersContext & {
allPeerDepNames: Set<string>
getParentPkgs: Record<string, GetParentPkgs>
parentPkgsOfNode: ParentPkgsOfNode
parentNodeIds: string[]
parentDepPathsChain: string[]
dependenciesTree: DependenciesTree<T>
depGraph: GenericDependenciesGraph<T>
virtualStoreDir: string
@@ -352,6 +356,7 @@ async function resolvePeersOfNode<T extends PartialResolvedPackage> (
if (typeof node.children === 'function') {
node.children = node.children()
}
const parentNodeIds = [...ctx.parentNodeIds, nodeId]
const children = node.children
let parentPkgs: ParentRefs
if (Object.keys(children).length === 0) {
@@ -365,6 +370,7 @@ async function resolvePeersOfNode<T extends PartialResolvedPackage> (
alias,
node: ctx.dependenciesTree.get(nodeId)!,
nodeId,
parentNodeIds,
})
}
}
@@ -398,7 +404,11 @@ async function resolvePeersOfNode<T extends PartialResolvedPackage> (
resolvedPeers: unknownResolvedPeersOfChildren,
missingPeers: missingPeersOfChildren,
finishing,
} = await resolvePeersOfChildren(children, parentPkgs, ctx)
} = await resolvePeersOfChildren(children, parentPkgs, {
...ctx,
parentNodeIds,
parentDepPathsChain: ctx.parentDepPathsChain.includes(resolvedPackage.depPath) ? ctx.parentDepPathsChain : [...ctx.parentDepPathsChain, resolvedPackage.depPath],
})
const { resolvedPeers, missingPeers } = Object.keys(resolvedPackage.peerDependencies).length === 0
? { resolvedPeers: new Map<string, string>(), missingPeers: new Set<string>() }
@@ -411,6 +421,7 @@ async function resolvePeersOfNode<T extends PartialResolvedPackage> (
peerDependencyIssues: ctx.peerDependencyIssues,
resolvedPackage,
rootDir: ctx.rootDir,
parentNodeIds,
})
const allResolvedPeers = unknownResolvedPeersOfChildren
@@ -539,7 +550,7 @@ async function resolvePeersOfNode<T extends PartialResolvedPackage> (
ctx.depGraph[depPath] = {
...(node.resolvedPackage as T),
children: Object.assign(
getPreviouslyResolvedChildren(nodeId, ctx.dependenciesTree),
getPreviouslyResolvedChildren(ctx, (node.resolvedPackage as T).depPath),
children,
Object.fromEntries(resolvedPeers.entries())
),
@@ -558,7 +569,7 @@ async function resolvePeersOfNode<T extends PartialResolvedPackage> (
}
function findHit<T extends PartialResolvedPackage> (ctx: {
getParentPkgs: Record<string, GetParentPkgs>
parentPkgsOfNode: ParentPkgsOfNode
peersCache: PeersCache
purePkgs: Set<string>
pathsByNodeId: Map<string, string>
@@ -599,12 +610,12 @@ function findHit<T extends PartialResolvedPackage> (ctx: {
}
function parentPackagesMatch (ctx: {
getParentPkgs: Record<string, GetParentPkgs>
parentPkgsOfNode: ParentPkgsOfNode
purePkgs: Set<string>
}, cachedNodeId: string, checkedNodeId: string): boolean {
const cachedParentPkgs = ctx.getParentPkgs[cachedNodeId]?.()
const cachedParentPkgs = ctx.parentPkgsOfNode.get(cachedNodeId)
if (!cachedParentPkgs) return false
const checkedParentPkgs = ctx.getParentPkgs[checkedNodeId]?.()
const checkedParentPkgs = ctx.parentPkgsOfNode.get(checkedNodeId)
if (!checkedParentPkgs) return false
if (Object.keys(cachedParentPkgs).length !== Object.keys(checkedParentPkgs).length) return false
const maxDepth = Object.values(checkedParentPkgs)
@@ -640,27 +651,34 @@ function parentPkgsHaveSingleOccurrence (parentPkgs: Record<string, ParentPkgInf
// from the dependencies of the parent package.
// So we need to merge all the children of all the parent packages with same ID as the resolved package.
// This way we get all the children that were removed, when ending cycles.
function getPreviouslyResolvedChildren<T extends PartialResolvedPackage> (nodeId: string, dependenciesTree: DependenciesTree<T>): ChildrenMap {
const parentIds = splitNodeId(nodeId)
const ownId = parentIds.pop()
function getPreviouslyResolvedChildren<T extends PartialResolvedPackage> (
{
parentNodeIds,
parentDepPathsChain,
dependenciesTree,
}: {
parentNodeIds: string[]
parentDepPathsChain: string[]
dependenciesTree: DependenciesTree<T>
},
currentDepPath: string
): ChildrenMap {
const allChildren: ChildrenMap = {}
if (!ownId || !parentIds.includes(ownId)) return allChildren
if (!currentDepPath || !parentDepPathsChain.includes(currentDepPath)) return allChildren
const nodeIdChunks = parentIds.join('>').split(`>${ownId}>`)
nodeIdChunks.pop()
nodeIdChunks.reduce((accNodeId, part) => {
accNodeId += `>${part}>${ownId}`
const parentNode = dependenciesTree.get(`${accNodeId}>`)!
if (typeof parentNode.children === 'function') {
parentNode.children = parentNode.children()
for (let i = parentNodeIds.length - 1; i >= 0; i--) {
const parentNode = dependenciesTree.get(parentNodeIds[i])!
if ((parentNode.resolvedPackage as T).depPath === currentDepPath) {
if (typeof parentNode.children === 'function') {
parentNode.children = parentNode.children()
}
Object.assign(
allChildren,
parentNode.children
)
}
Object.assign(
allChildren,
parentNode.children
)
return accNodeId
}, '')
}
return allChildren
}
@@ -671,7 +689,9 @@ async function resolvePeersOfChildren<T extends PartialResolvedPackage> (
parentPkgs: ParentRefs,
ctx: ResolvePeersContext & {
allPeerDepNames: Set<string>
getParentPkgs: Record<string, GetParentPkgs>
parentPkgsOfNode: ParentPkgsOfNode
parentNodeIds: string[]
parentDepPathsChain: string[]
peerDependencyIssues: Pick<PeerDependencyIssues, 'bad' | 'missing'>
peersCache: PeersCache
virtualStoreDir: string
@@ -701,24 +721,21 @@ async function resolvePeersOfChildren<T extends PartialResolvedPackage> (
const calculateDepPaths: CalculateDepPath[] = []
const graph = []
const finishingList: FinishingResolutionPromise[] = []
const cachedGetParentPkgs = memoize(() => {
const parentDepPaths: Record<string, ParentPkgInfo> = {}
for (const [name, parentPkg] of Object.entries(parentPkgs)) {
if (!ctx.allPeerDepNames.has(name)) continue
if (parentPkg.nodeId && !parentPkg.nodeId.startsWith('link:')) {
parentDepPaths[name] = {
depPath: (ctx.dependenciesTree.get(parentPkg.nodeId)!.resolvedPackage as T).depPath,
depth: parentPkg.depth,
occurrence: parentPkg.occurrence,
}
} else {
parentDepPaths[name] = { version: parentPkg.version }
const parentDepPaths: Record<string, ParentPkgInfo> = {}
for (const [name, parentPkg] of Object.entries(parentPkgs)) {
if (!ctx.allPeerDepNames.has(name)) continue
if (parentPkg.nodeId && !parentPkg.nodeId.startsWith('link:')) {
parentDepPaths[name] = {
depPath: (ctx.dependenciesTree.get(parentPkg.nodeId)!.resolvedPackage as T).depPath,
depth: parentPkg.depth,
occurrence: parentPkg.occurrence,
}
} else {
parentDepPaths[name] = { version: parentPkg.version }
}
return parentDepPaths
})
}
for (const childNodeId of nodeIds) {
ctx.getParentPkgs[childNodeId] = cachedGetParentPkgs
ctx.parentPkgsOfNode.set(childNodeId, parentDepPaths)
}
for (const childNodeId of nodeIds) {
const {
@@ -765,6 +782,7 @@ function _resolvePeers<T extends PartialResolvedPackage> (
lockfileDir: string
nodeId: string
parentPkgs: ParentRefs
parentNodeIds: string[]
resolvedPackage: T
dependenciesTree: DependenciesTree<T>
rootDir: string
@@ -781,11 +799,7 @@ function _resolvePeers<T extends PartialResolvedPackage> (
if (!resolved) {
missingPeers.add(peerName)
const location = getLocationFromNodeIdAndPkg({
dependenciesTree: ctx.dependenciesTree,
nodeId: ctx.nodeId,
pkg: ctx.resolvedPackage,
})
const location = getLocationFromParentNodeIds(ctx)
if (!ctx.peerDependencyIssues.missing[peerName]) {
ctx.peerDependencyIssues.missing[peerName] = []
}
@@ -798,19 +812,15 @@ function _resolvePeers<T extends PartialResolvedPackage> (
}
if (!semverUtils.satisfiesWithPrereleases(resolved.version, peerVersionRange, true)) {
const location = getLocationFromNodeIdAndPkg({
dependenciesTree: ctx.dependenciesTree,
nodeId: ctx.nodeId,
pkg: ctx.resolvedPackage,
})
const location = getLocationFromParentNodeIds(ctx)
if (!ctx.peerDependencyIssues.bad[peerName]) {
ctx.peerDependencyIssues.bad[peerName] = []
}
const peerLocation = resolved.nodeId == null
? []
: getLocationFromNodeId({
: getLocationFromParentNodeIds({
dependenciesTree: ctx.dependenciesTree,
nodeId: resolved.nodeId,
parentNodeIds: resolved.parentNodeIds,
}).parents
ctx.peerDependencyIssues.bad[peerName].push({
foundVersion: resolved.version,
@@ -831,41 +841,19 @@ interface Location {
parents: ParentPackages
}
function getLocationFromNodeIdAndPkg<T> (
function getLocationFromParentNodeIds<T> (
{
dependenciesTree,
nodeId,
pkg,
parentNodeIds,
}: {
dependenciesTree: DependenciesTree<T>
nodeId: string
pkg: { name: string, version: string }
parentNodeIds: string[]
}
): Location {
const { projectId, parents } = getLocationFromNodeId({ dependenciesTree, nodeId })
parents.push({ name: pkg.name, version: pkg.version })
return {
projectId,
parents,
}
}
function getLocationFromNodeId<T> (
{
dependenciesTree,
nodeId,
}: {
dependenciesTree: DependenciesTree<T>
nodeId: string
}
): Location {
const parts = splitNodeId(nodeId).slice(0, -1)
const parents = scan((prevNodeId, pkgId) => createNodeId(prevNodeId, pkgId), '>', parts)
.slice(2)
const parents = parentNodeIds
.map((nid) => pick(['name', 'version'], dependenciesTree.get(nid)!.resolvedPackage as ResolvedPackage))
return {
projectId: parts[0],
projectId: '.',
parents,
}
}
@@ -881,24 +869,27 @@ interface ParentRef {
nodeId?: string
alias?: string
occurrence: number
parentNodeIds: string[]
}
interface ParentPkgNode<T> {
alias: string
nodeId: string
node: DependenciesTreeNode<T>
parentNodeIds: string[]
}
function toPkgByName<T extends PartialResolvedPackage> (nodes: Array<ParentPkgNode<T>>): ParentRefs {
const pkgsByName: ParentRefs = {}
const _updateParentRefs = updateParentRefs.bind(null, pkgsByName)
for (const { alias, node, nodeId } of nodes) {
for (const { alias, node, nodeId, parentNodeIds } of nodes) {
const pkg = {
alias,
depth: node.depth,
nodeId,
version: node.resolvedPackage.version,
occurrence: 0,
parentNodeIds,
}
_updateParentRefs(alias, pkg)
if (alias !== node.resolvedPackage.name) {

View File

@@ -61,8 +61,7 @@ function toLockfileDependency (
}
): PackageSnapshot {
const lockfileResolution = toLockfileResolution(
{ id: pkg.id, name: pkg.name, version: pkg.version },
opts.depPath,
{ name: pkg.name, version: pkg.version },
pkg.resolution,
opts.registry,
opts.lockfileIncludeTarballUrl
@@ -175,11 +174,9 @@ function updateResolvedDeps (
function toLockfileResolution (
pkg: {
id: string
name: string
version: string
},
depPath: string,
resolution: Resolution,
registry: string,
lockfileIncludeTarballUrl?: boolean

View File

@@ -1,5 +0,0 @@
import { nodeIdContainsSequence } from '../lib/nodeIdUtils'
test('nodeIdContainsSequence()', () => {
expect(nodeIdContainsSequence('>.>b>a>c>b>a>', 'a', 'b')).toBeTruthy()
})

View File

@@ -0,0 +1,5 @@
import { parentIdsContainSequence } from '../lib/parentIdsContainSequence'
test('parentIdsContainSequence()', () => {
expect(parentIdsContainSequence(['.', 'b', 'a', 'c', 'b', 'a'], 'a', 'b')).toBeTruthy()
})

3
pnpm-lock.yaml generated
View File

@@ -4307,9 +4307,6 @@ importers:
is-subdir:
specifier: ^1.2.0
version: 1.2.0
mem:
specifier: ^8.1.1
version: 8.1.1
normalize-path:
specifier: ^3.0.0
version: 3.0.0