refactor: resolve-dependencies

PR #2829
This commit is contained in:
Zoltan Kochan
2020-09-06 00:32:24 +03:00
committed by GitHub
parent 924ea21ad5
commit e2f6b40b1a
26 changed files with 785 additions and 643 deletions

View File

@@ -0,0 +1,5 @@
---
"@pnpm/manifest-utils": minor
---
`getSpecFromPackageManifest()` added.

View File

@@ -0,0 +1,5 @@
---
"@pnpm/manifest-utils": minor
---
`getPref()` added.

View File

@@ -0,0 +1,5 @@
---
"@pnpm/resolve-dependencies": major
---
Breaking changes to the API. `resolveDependencies()` now returns a dependency graph with peer dependencies resolved.

View File

@@ -0,0 +1,5 @@
---
"@pnpm/manifest-utils": minor
---
`updateProjectManifestObject()` added.

View File

@@ -22,12 +22,15 @@
},
"repository": "https://github.com/pnpm/pnpm/blob/master/packages/manifest-utils",
"scripts": {
"test": "pnpm run compile",
"lint": "eslint -c ../../eslint.json src/**/*.ts",
"_test": "cd ../.. && c8 --reporter lcov --reports-dir packages/manifest-utils/coverage ts-node packages/manifest-utils/test --type-check",
"test": "pnpm run compile && pnpm run _test",
"lint": "eslint -c ../../eslint.json src/**/*.ts test/**/*.ts",
"prepublishOnly": "pnpm run compile",
"compile": "rimraf lib tsconfig.tsbuildinfo && tsc --build"
},
"dependencies": {
"@pnpm/core-loggers": "workspace:^4.2.0",
"@pnpm/error": "workspace:^1.3.1",
"@pnpm/types": "workspace:6.2.0"
},
"homepage": "https://github.com/pnpm/pnpm/blob/master/packages/manifest-utils#readme",

View File

@@ -0,0 +1,32 @@
import PnpmError from '@pnpm/error'
export type PinnedVersion = 'major' | 'minor' | 'patch' | 'none'
export const getPrefix = (alias: string, name: string) => alias !== name ? `npm:${name}@` : ''
export function getPref (
alias: string,
name: string,
version: string,
opts: {
pinnedVersion?: PinnedVersion
}
) {
const prefix = getPrefix(alias, name)
return `${prefix}${createVersionSpec(version, opts.pinnedVersion)}`
}
export function createVersionSpec (version: string, pinnedVersion?: PinnedVersion) {
switch (pinnedVersion ?? 'major') {
case 'none':
return '*'
case 'major':
return `^${version}`
case 'minor':
return `~${version}`
case 'patch':
return `${version}`
default:
throw new PnpmError('BAD_PINNED_VERSION', `Cannot pin '${pinnedVersion ?? 'undefined'}'`)
}
}

View File

@@ -3,6 +3,12 @@ import {
IncludedDependencies,
ProjectManifest,
} from '@pnpm/types'
import getSpecFromPackageManifest from './getSpecFromPackageManifest'
export * from './getPref'
export * from './updateProjectManifestObject'
export { getSpecFromPackageManifest }
export function filterDependenciesByType (
manifest: ProjectManifest,

View File

@@ -12,13 +12,10 @@ export interface PackageSpecObject {
saveType?: DependenciesField
}
export default async function save (
export async function updateProjectManifestObject (
prefix: string,
packageManifest: ProjectManifest,
packageSpecs: PackageSpecObject[],
opts?: {
dryRun?: boolean
}
packageSpecs: PackageSpecObject[]
): Promise<ProjectManifest> {
packageSpecs.forEach((packageSpec) => {
if (packageSpec.saveType) {

View File

@@ -1,8 +1,5 @@
import getSpecFromPackageManifest from 'supi/lib/getSpecFromPackageManifest'
import promisifyTape from 'tape-promise'
import tape = require('tape')
const test = promisifyTape(tape)
import { getSpecFromPackageManifest } from '@pnpm/manifest-utils'
import test = require('tape')
test('getSpecFromPackageManifest()', (t) => {
t.equal(

View File

@@ -0,0 +1,2 @@
/// <reference path="../../../typings/index.d.ts" />
import './getSpecFromPackageManifest.test'

View File

@@ -9,6 +9,12 @@
"../../typings/**/*.d.ts"
],
"references": [
{
"path": "../core-loggers"
},
{
"path": "../error"
},
{
"path": "../types"
}

View File

@@ -27,22 +27,30 @@
"compile": "rimraf lib tsconfig.tsbuildinfo && tsc --build"
},
"dependencies": {
"@pnpm/constants": "workspace:^4.0.0",
"@pnpm/core-loggers": "workspace:4.2.0",
"@pnpm/error": "workspace:1.3.1",
"@pnpm/lockfile-types": "workspace:2.0.1",
"@pnpm/lockfile-utils": "workspace:2.0.16",
"@pnpm/manifest-utils": "workspace:^1.0.3",
"@pnpm/npm-resolver": "workspace:10.0.1",
"@pnpm/package-is-installable": "workspace:4.0.14",
"@pnpm/pick-registry-for-package": "workspace:1.0.3",
"@pnpm/pkgid-to-filename": "^3.0.0",
"@pnpm/prune-lockfile": "workspace:^2.0.14",
"@pnpm/read-package-json": "workspace:^3.1.5",
"@pnpm/resolver-base": "workspace:7.0.3",
"@pnpm/store-controller-types": "workspace:8.0.2",
"@pnpm/types": "workspace:6.2.0",
"dependency-path": "workspace:5.0.3",
"encode-registry": "^2.0.2",
"get-npm-tarball-url": "^2.0.1",
"import-from": "^3.0.0",
"path-exists": "^4.0.0",
"ramda": "^0.27.1",
"replace-string": "^3.1.0",
"semver": "^7.3.2"
"semver": "^7.3.2",
"version-selector-type": "^2.0.1"
},
"devDependencies": {
"@pnpm/logger": "^3.2.2",

View File

@@ -3,7 +3,7 @@ import { Registries } from '@pnpm/types'
import { getRegistryByPackageName } from 'dependency-path'
import encodeRegistry = require('encode-registry')
export function depPathToRef (
export default function depPathToRef (
depPath: string,
opts: {
alias: string

View File

@@ -1,238 +1,316 @@
import { Lockfile } from '@pnpm/lockfile-types'
import { PreferredVersions, Resolution, WorkspacePackages } from '@pnpm/resolver-base'
import { StoreController } from '@pnpm/store-controller-types'
import { WANTED_LOCKFILE } from '@pnpm/constants'
import {
ReadPackageHook,
packageManifestLogger,
} from '@pnpm/core-loggers'
import {
Lockfile,
ProjectSnapshot,
} from '@pnpm/lockfile-types'
import {
getAllDependenciesFromManifest,
getSpecFromPackageManifest,
PinnedVersion,
} from '@pnpm/manifest-utils'
import { safeReadPackageFromDir as safeReadPkgFromDir } from '@pnpm/read-package-json'
import {
DEPENDENCIES_FIELDS,
DependencyManifest,
ProjectManifest,
Registries,
} from '@pnpm/types'
import { WantedDependency } from './getNonDevWantedDependencies'
import {
createNodeId,
nodeIdContainsSequence,
} from './nodeIdUtils'
import resolveDependencies, {
ChildrenByParentDepPath,
DependenciesTree,
import depPathToRef from './depPathToRef'
import resolveDependencyTree, {
Importer,
LinkedDependency,
PendingNode,
PkgAddress,
ResolvedPackage,
ResolvedPackagesByDepPath,
} from './resolveDependencies'
ResolveDependenciesOptions,
ResolvedDirectDependency,
} from './resolveDependencyTree'
import resolvePeers, { DependenciesGraph, DependenciesGraphNode } from './resolvePeers'
import updateLockfile from './updateLockfile'
import updateProjectManifest from './updateProjectManifest'
import path = require('path')
import R = require('ramda')
export * from './nodeIdUtils'
export { LinkedDependency, ResolvedPackage, DependenciesTree, DependenciesTreeNode } from './resolveDependencies'
export interface ResolvedDirectDependency {
alias: string
optional: boolean
dev: boolean
resolution: Resolution
pkgId: string
version: string
name: string
normalizedPref?: string
export {
DependenciesGraph,
DependenciesGraphNode,
LinkedDependency,
}
export interface Importer {
export interface ProjectToLink {
binsDir: string
directNodeIdsByAlias: {[alias: string]: string}
id: string
hasRemovedDependencies?: boolean
linkedDependencies: LinkedDependency[]
manifest: ProjectManifest
modulesDir: string
preferredVersions?: PreferredVersions
pruneDirectDependencies: boolean
removePackages?: string[]
rootDir: string
wantedDependencies: Array<WantedDependency & { updateDepth: number }>
topParents: Array<{name: string, version: string}>
}
export type ImporterToResolve = Importer<{
isNew?: boolean
pinnedVersion?: PinnedVersion
raw: string
updateSpec?: boolean
}>
& {
binsDir: string
manifest: ProjectManifest
originalManifest?: ProjectManifest
removePackages?: string[]
pruneDirectDependencies: boolean
updatePackageManifest: boolean
}
export default async function (
importers: Importer[],
opts: {
currentLockfile: Lockfile
dryRun: boolean
engineStrict: boolean
force: boolean
forceFullResolution: boolean
hooks: {
readPackage?: ReadPackageHook
}
nodeVersion: string
registries: Registries
pnpmVersion: string
updateMatching?: (pkgName: string) => boolean
linkWorkspacePackagesDepth?: number
lockfileDir: string
storeController: StoreController
tag: string
virtualStoreDir: string
wantedLockfile: Lockfile
workspacePackages: WorkspacePackages
importers: ImporterToResolve[],
opts: ResolveDependenciesOptions & {
preserveWorkspaceProtocol: boolean
saveWorkspaceProtocol: boolean
strictPeerDependencies: boolean
}
) {
const directDepsByImporterId = {} as {[id: string]: Array<PkgAddress | LinkedDependency>}
const {
dependenciesTree,
outdatedDependencies,
resolvedImporters,
resolvedPackagesByDepPath,
wantedToBeSkippedPackageIds,
} = await resolveDependencyTree(importers, opts)
const wantedToBeSkippedPackageIds = new Set<string>()
const ctx = {
alwaysTryWorkspacePackages: (opts.linkWorkspacePackagesDepth ?? -1) >= 0,
childrenByParentDepPath: {} as ChildrenByParentDepPath,
currentLockfile: opts.currentLockfile,
defaultTag: opts.tag,
dependenciesTree: {} as DependenciesTree,
dryRun: opts.dryRun,
engineStrict: opts.engineStrict,
force: opts.force,
forceFullResolution: opts.forceFullResolution,
linkWorkspacePackagesDepth: opts.linkWorkspacePackagesDepth ?? -1,
lockfileDir: opts.lockfileDir,
nodeVersion: opts.nodeVersion,
outdatedDependencies: {} as {[pkgId: string]: string},
pendingNodes: [] as PendingNode[],
pnpmVersion: opts.pnpmVersion,
readPackageHook: opts.hooks.readPackage,
registries: opts.registries,
resolvedPackagesByDepPath: {} as ResolvedPackagesByDepPath,
skipped: wantedToBeSkippedPackageIds,
storeController: opts.storeController,
updateMatching: opts.updateMatching,
virtualStoreDir: opts.virtualStoreDir,
wantedLockfile: opts.wantedLockfile,
}
const projectsToLink = await Promise.all<ProjectToLink>(importers.map(async (project, index) => {
const resolvedImporter = resolvedImporters[project.id]
let updatedManifest: ProjectManifest | undefined = project.manifest
let updatedOriginalManifest: ProjectManifest | undefined = project.originalManifest
if (project.updatePackageManifest) {
const manifests = await updateProjectManifest(importers[index], {
directDependencies: resolvedImporter.directDependencies,
preserveWorkspaceProtocol: opts.preserveWorkspaceProtocol,
saveWorkspaceProtocol: opts.saveWorkspaceProtocol,
})
updatedManifest = manifests[0]
updatedOriginalManifest = manifests[1]
} else {
packageManifestLogger.debug({
prefix: project.rootDir,
updated: project.manifest,
})
}
await Promise.all(importers.map(async (importer) => {
const projectSnapshot = opts.wantedLockfile.importers[importer.id]
// This array will only contain the dependencies that should be linked in.
// The already linked-in dependencies will not be added.
const linkedDependencies = [] as LinkedDependency[]
const resolveCtx = {
...ctx,
linkedDependencies,
modulesDir: importer.modulesDir,
prefix: importer.rootDir,
if (updatedManifest) {
const projectSnapshot = opts.wantedLockfile.importers[project.id]
opts.wantedLockfile.importers[project.id] = addDirectDependenciesToLockfile(
updatedManifest,
projectSnapshot,
resolvedImporter.linkedDependencies,
resolvedImporter.directDependencies,
opts.registries
)
}
// This may be optimized.
// We only need to proceed resolving every dependency
// if the newly added dependency has peer dependencies.
const proceed = importer.hasRemovedDependencies === true || importer.wantedDependencies.some((wantedDep) => wantedDep['isNew'])
const resolveOpts = {
currentDepth: 0,
parentPkg: {
installable: true,
nodeId: `>${importer.id}>`,
depPath: importer.id,
},
proceed,
resolvedDependencies: {
...projectSnapshot.dependencies,
...projectSnapshot.devDependencies,
...projectSnapshot.optionalDependencies,
},
updateDepth: -1,
workspacePackages: opts.workspacePackages,
const topParents = project.manifest
? await getTopParents(
R.difference(
Object.keys(getAllDependenciesFromManifest(project.manifest)),
resolvedImporter.directDependencies
.filter((dep, index) => project.wantedDependencies[index].isNew === true)
.map(({ alias }) => alias) || []
),
project.modulesDir
)
: []
return {
binsDir: project.binsDir,
directNodeIdsByAlias: resolvedImporter.directNodeIdsByAlias,
id: project.id,
linkedDependencies: resolvedImporter.linkedDependencies,
manifest: updatedOriginalManifest ?? project.originalManifest ?? project.manifest,
modulesDir: project.modulesDir,
pruneDirectDependencies: project.pruneDirectDependencies,
removePackages: project.removePackages,
rootDir: project.rootDir,
topParents,
}
directDepsByImporterId[importer.id] = await resolveDependencies(
resolveCtx,
importer.preferredVersions ?? {},
importer.wantedDependencies,
resolveOpts
)
}))
ctx.pendingNodes.forEach((pendingNode) => {
ctx.dependenciesTree[pendingNode.nodeId] = {
children: () => buildTree(ctx, pendingNode.nodeId, pendingNode.resolvedPackage.id,
ctx.childrenByParentDepPath[pendingNode.resolvedPackage.depPath], pendingNode.depth + 1, pendingNode.installable),
depth: pendingNode.depth,
installable: pendingNode.installable,
resolvedPackage: pendingNode.resolvedPackage,
}
const {
dependenciesGraph,
projectsDirectPathsByAlias,
} = resolvePeers({
dependenciesTree,
lockfileDir: opts.lockfileDir,
projects: projectsToLink,
strictPeerDependencies: opts.strictPeerDependencies,
virtualStoreDir: opts.virtualStoreDir,
})
const resolvedImporters = {} as {
[id: string]: {
directDependencies: ResolvedDirectDependency[]
directNodeIdsByAlias: {
[alias: string]: string
for (const { id } of projectsToLink) {
for (const [alias, depPath] of R.toPairs(projectsDirectPathsByAlias[id])) {
const depNode = dependenciesGraph[depPath]
if (depNode.isPure) continue
const projectSnapshot = opts.wantedLockfile.importers[id]
const ref = depPathToRef(depPath, {
alias,
realName: depNode.name,
registries: opts.registries,
resolution: depNode.resolution,
})
if (projectSnapshot.dependencies?.[alias]) {
projectSnapshot.dependencies[alias] = ref
} else if (projectSnapshot.devDependencies?.[alias]) {
projectSnapshot.devDependencies[alias] = ref
} else if (projectSnapshot.optionalDependencies?.[alias]) {
projectSnapshot.optionalDependencies[alias] = ref
}
linkedDependencies: LinkedDependency[]
}
}
const { newLockfile, pendingRequiresBuilds } = updateLockfile(dependenciesGraph, opts.wantedLockfile, opts.virtualStoreDir, opts.registries) // eslint-disable-line:prefer-const
for (const { id } of importers) {
const directDeps = directDepsByImporterId[id]
const [linkedDependencies, directNonLinkedDeps] = R.partition((dep) => dep.isLinkedDependency === true, directDeps) as [LinkedDependency[], PkgAddress[]]
resolvedImporters[id] = {
directDependencies: directDeps
.map((dep) => {
if (dep.isLinkedDependency === true) {
return dep
}
const resolvedPackage = ctx.dependenciesTree[dep.nodeId].resolvedPackage as ResolvedPackage
return {
alias: dep.alias,
dev: resolvedPackage.dev,
name: resolvedPackage.name,
normalizedPref: dep.normalizedPref,
optional: resolvedPackage.optional,
pkgId: resolvedPackage.id,
resolution: resolvedPackage.resolution,
version: resolvedPackage.version,
}
}),
directNodeIdsByAlias: directNonLinkedDeps
.reduce((acc, dependency) => {
acc[dependency.alias] = dependency.nodeId
return acc
}, {}),
linkedDependencies,
}
}
// waiting till package requests are finished
const waitTillAllFetchingsFinish = () => Promise.all(R.values(resolvedPackagesByDepPath).map(({ finishing }) => finishing()))
return {
dependenciesTree: ctx.dependenciesTree,
outdatedDependencies: ctx.outdatedDependencies,
resolvedImporters,
resolvedPackagesByDepPath: ctx.resolvedPackagesByDepPath,
dependenciesGraph,
finishLockfileUpdates: finishLockfileUpdates.bind(null, dependenciesGraph, pendingRequiresBuilds, newLockfile),
outdatedDependencies,
projectsToLink,
projectsDirectPathsByAlias,
newLockfile,
waitTillAllFetchingsFinish,
wantedToBeSkippedPackageIds,
}
}
function buildTree (
ctx: {
childrenByParentDepPath: ChildrenByParentDepPath
dependenciesTree: DependenciesTree
resolvedPackagesByDepPath: ResolvedPackagesByDepPath
skipped: Set<string>
},
parentNodeId: string,
parentId: string,
children: Array<{alias: string, depPath: string}>,
depth: number,
installable: boolean
function finishLockfileUpdates (
dependenciesGraph: DependenciesGraph,
pendingRequiresBuilds: string[],
newLockfile: Lockfile
) {
const childrenNodeIds = {}
for (const child of children) {
if (child.depPath.startsWith('link:')) {
childrenNodeIds[child.alias] = child.depPath
continue
return Promise.all(pendingRequiresBuilds.map(async (depPath) => {
const depNode = dependenciesGraph[depPath]
if (!depNode.fetchingBundledManifest) {
// This should never ever happen
throw new Error(`Cannot create ${WANTED_LOCKFILE} because raw manifest (aka package.json) wasn't fetched for "${depPath}"`)
}
if (nodeIdContainsSequence(parentNodeId, parentId, child.depPath)) {
continue
const filesResponse = await depNode.fetchingFiles()
// The npm team suggests to always read the package.json for deciding whether the package has lifecycle scripts
const pkgJson = await depNode.fetchingBundledManifest()
depNode.requiresBuild = Boolean(
pkgJson.scripts != null && (
Boolean(pkgJson.scripts.preinstall) ||
Boolean(pkgJson.scripts.install) ||
Boolean(pkgJson.scripts.postinstall)
) ||
filesResponse.filesIndex['binding.gyp'] ||
Object.keys(filesResponse.filesIndex).some((filename) => !!filename.match(/^[.]hooks[\\/]/)) // TODO: optimize this
)
// TODO: try to cover with unit test the case when entry is no longer available in lockfile
// It is an edge that probably happens if the entry is removed during lockfile prune
if (depNode.requiresBuild && newLockfile.packages![depPath]) {
newLockfile.packages![depPath].requiresBuild = true
}
const childNodeId = createNodeId(parentNodeId, child.depPath)
childrenNodeIds[child.alias] = childNodeId
installable = installable && !ctx.skipped.has(child.depPath)
ctx.dependenciesTree[childNodeId] = {
children: () => buildTree(ctx,
childNodeId,
child.depPath,
ctx.childrenByParentDepPath[child.depPath],
depth + 1,
installable
),
depth,
installable,
resolvedPackage: ctx.resolvedPackagesByDepPath[child.depPath],
}))
}
function addDirectDependenciesToLockfile (
newManifest: ProjectManifest,
projectSnapshot: ProjectSnapshot,
linkedPackages: Array<{alias: string}>,
directDependencies: ResolvedDirectDependency[],
registries: Registries
): ProjectSnapshot {
const newProjectSnapshot = {
dependencies: {},
devDependencies: {},
optionalDependencies: {},
specifiers: {},
}
linkedPackages.forEach((linkedPkg) => {
newProjectSnapshot.specifiers[linkedPkg.alias] = getSpecFromPackageManifest(newManifest, linkedPkg.alias)
})
const directDependenciesByAlias = directDependencies.reduce((acc, directDependency) => {
acc[directDependency.alias] = directDependency
return acc
}, {})
const allDeps = Array.from(new Set(Object.keys(getAllDependenciesFromManifest(newManifest))))
for (const alias of allDeps) {
if (directDependenciesByAlias[alias]) {
const dep = directDependenciesByAlias[alias]
const ref = depPathToRef(dep.pkgId, {
alias: dep.alias,
realName: dep.name,
registries,
resolution: dep.resolution,
})
if (dep.dev) {
newProjectSnapshot.devDependencies[dep.alias] = ref
} else if (dep.optional) {
newProjectSnapshot.optionalDependencies[dep.alias] = ref
} else {
newProjectSnapshot.dependencies[dep.alias] = ref
}
newProjectSnapshot.specifiers[dep.alias] = getSpecFromPackageManifest(newManifest, dep.alias)
} else if (projectSnapshot.specifiers[alias]) {
newProjectSnapshot.specifiers[alias] = projectSnapshot.specifiers[alias]
if (projectSnapshot.dependencies?.[alias]) {
newProjectSnapshot.dependencies[alias] = projectSnapshot.dependencies[alias]
} else if (projectSnapshot.optionalDependencies?.[alias]) {
newProjectSnapshot.optionalDependencies[alias] = projectSnapshot.optionalDependencies[alias]
} else if (projectSnapshot.devDependencies?.[alias]) {
newProjectSnapshot.devDependencies[alias] = projectSnapshot.devDependencies[alias]
}
}
}
return childrenNodeIds
alignDependencyTypes(newManifest, newProjectSnapshot)
return newProjectSnapshot
}
function alignDependencyTypes (manifest: ProjectManifest, projectSnapshot: ProjectSnapshot) {
const depTypesOfAliases = getAliasToDependencyTypeMap(manifest)
// Aligning the dependency types in pnpm-lock.yaml
for (const depType of DEPENDENCIES_FIELDS) {
if (!projectSnapshot[depType]) continue
for (const alias of Object.keys(projectSnapshot[depType] ?? {})) {
if (depType === depTypesOfAliases[alias] || !depTypesOfAliases[alias]) continue
projectSnapshot[depTypesOfAliases[alias]][alias] = projectSnapshot[depType]![alias]
delete projectSnapshot[depType]![alias]
}
}
}
function getAliasToDependencyTypeMap (manifest: ProjectManifest) {
const depTypesOfAliases = {}
for (const depType of DEPENDENCIES_FIELDS) {
if (!manifest[depType]) continue
for (const alias of Object.keys(manifest[depType] ?? {})) {
if (!depTypesOfAliases[alias]) {
depTypesOfAliases[alias] = depType
}
}
}
return depTypesOfAliases
}
async function getTopParents (pkgNames: string[], modules: string) {
const pkgs = await Promise.all(
pkgNames.map((pkgName) => path.join(modules, pkgName)).map(safeReadPkgFromDir)
)
return (
pkgs
.filter(Boolean) as DependencyManifest[]
)
.map(({ name, version }: DependencyManifest) => ({ name, version }))
}

View File

@@ -0,0 +1,240 @@
import { Lockfile } from '@pnpm/lockfile-types'
import { PreferredVersions, Resolution, WorkspacePackages } from '@pnpm/resolver-base'
import { StoreController } from '@pnpm/store-controller-types'
import {
ReadPackageHook,
Registries,
} from '@pnpm/types'
import { WantedDependency } from './getNonDevWantedDependencies'
import {
createNodeId,
nodeIdContainsSequence,
} from './nodeIdUtils'
import resolveDependencies, {
ChildrenByParentDepPath,
DependenciesTree,
LinkedDependency,
PendingNode,
PkgAddress,
ResolvedPackage,
ResolvedPackagesByDepPath,
} from './resolveDependencies'
import R = require('ramda')
export * from './nodeIdUtils'
export { LinkedDependency, ResolvedPackage, DependenciesTree, DependenciesTreeNode } from './resolveDependencies'
export interface ResolvedDirectDependency {
alias: string
optional: boolean
dev: boolean
resolution: Resolution
pkgId: string
version: string
name: string
normalizedPref?: string
}
export interface Importer<T> {
id: string
hasRemovedDependencies?: boolean
modulesDir: string
preferredVersions?: PreferredVersions
rootDir: string
wantedDependencies: Array<T & WantedDependency & { updateDepth: number }>
}
export interface ResolveDependenciesOptions {
currentLockfile: Lockfile
dryRun: boolean
engineStrict: boolean
force: boolean
forceFullResolution: boolean
hooks: {
readPackage?: ReadPackageHook
}
nodeVersion: string
registries: Registries
pnpmVersion: string
updateMatching?: (pkgName: string) => boolean
linkWorkspacePackagesDepth?: number
lockfileDir: string
storeController: StoreController
tag: string
virtualStoreDir: string
wantedLockfile: Lockfile
workspacePackages: WorkspacePackages
}
export default async function<T> (
importers: Array<Importer<T>>,
opts: ResolveDependenciesOptions
) {
const directDepsByImporterId = {} as {[id: string]: Array<PkgAddress | LinkedDependency>}
const wantedToBeSkippedPackageIds = new Set<string>()
const ctx = {
alwaysTryWorkspacePackages: (opts.linkWorkspacePackagesDepth ?? -1) >= 0,
childrenByParentDepPath: {} as ChildrenByParentDepPath,
currentLockfile: opts.currentLockfile,
defaultTag: opts.tag,
dependenciesTree: {} as DependenciesTree,
dryRun: opts.dryRun,
engineStrict: opts.engineStrict,
force: opts.force,
forceFullResolution: opts.forceFullResolution,
linkWorkspacePackagesDepth: opts.linkWorkspacePackagesDepth ?? -1,
lockfileDir: opts.lockfileDir,
nodeVersion: opts.nodeVersion,
outdatedDependencies: {} as {[pkgId: string]: string},
pendingNodes: [] as PendingNode[],
pnpmVersion: opts.pnpmVersion,
readPackageHook: opts.hooks.readPackage,
registries: opts.registries,
resolvedPackagesByDepPath: {} as ResolvedPackagesByDepPath,
skipped: wantedToBeSkippedPackageIds,
storeController: opts.storeController,
updateMatching: opts.updateMatching,
virtualStoreDir: opts.virtualStoreDir,
wantedLockfile: opts.wantedLockfile,
}
await Promise.all(importers.map(async (importer) => {
const projectSnapshot = opts.wantedLockfile.importers[importer.id]
// This array will only contain the dependencies that should be linked in.
// The already linked-in dependencies will not be added.
const linkedDependencies = [] as LinkedDependency[]
const resolveCtx = {
...ctx,
linkedDependencies,
modulesDir: importer.modulesDir,
prefix: importer.rootDir,
}
// This may be optimized.
// We only need to proceed resolving every dependency
// if the newly added dependency has peer dependencies.
const proceed = importer.hasRemovedDependencies === true || importer.wantedDependencies.some((wantedDep) => wantedDep['isNew'])
const resolveOpts = {
currentDepth: 0,
parentPkg: {
installable: true,
nodeId: `>${importer.id}>`,
depPath: importer.id,
},
proceed,
resolvedDependencies: {
...projectSnapshot.dependencies,
...projectSnapshot.devDependencies,
...projectSnapshot.optionalDependencies,
},
updateDepth: -1,
workspacePackages: opts.workspacePackages,
}
directDepsByImporterId[importer.id] = await resolveDependencies(
resolveCtx,
importer.preferredVersions ?? {},
importer.wantedDependencies,
resolveOpts
)
}))
ctx.pendingNodes.forEach((pendingNode) => {
ctx.dependenciesTree[pendingNode.nodeId] = {
children: () => buildTree(ctx, pendingNode.nodeId, pendingNode.resolvedPackage.id,
ctx.childrenByParentDepPath[pendingNode.resolvedPackage.depPath], pendingNode.depth + 1, pendingNode.installable),
depth: pendingNode.depth,
installable: pendingNode.installable,
resolvedPackage: pendingNode.resolvedPackage,
}
})
const resolvedImporters = {} as {
[id: string]: {
directDependencies: ResolvedDirectDependency[]
directNodeIdsByAlias: {
[alias: string]: string
}
linkedDependencies: LinkedDependency[]
}
}
for (const { id } of importers) {
const directDeps = directDepsByImporterId[id]
const [linkedDependencies, directNonLinkedDeps] = R.partition((dep) => dep.isLinkedDependency === true, directDeps) as [LinkedDependency[], PkgAddress[]]
resolvedImporters[id] = {
directDependencies: directDeps
.map((dep) => {
if (dep.isLinkedDependency === true) {
return dep
}
const resolvedPackage = ctx.dependenciesTree[dep.nodeId].resolvedPackage as ResolvedPackage
return {
alias: dep.alias,
dev: resolvedPackage.dev,
name: resolvedPackage.name,
normalizedPref: dep.normalizedPref,
optional: resolvedPackage.optional,
pkgId: resolvedPackage.id,
resolution: resolvedPackage.resolution,
version: resolvedPackage.version,
}
}),
directNodeIdsByAlias: directNonLinkedDeps
.reduce((acc, dependency) => {
acc[dependency.alias] = dependency.nodeId
return acc
}, {}),
linkedDependencies,
}
}
return {
dependenciesTree: ctx.dependenciesTree,
outdatedDependencies: ctx.outdatedDependencies,
resolvedImporters,
resolvedPackagesByDepPath: ctx.resolvedPackagesByDepPath,
wantedToBeSkippedPackageIds,
}
}
function buildTree (
ctx: {
childrenByParentDepPath: ChildrenByParentDepPath
dependenciesTree: DependenciesTree
resolvedPackagesByDepPath: ResolvedPackagesByDepPath
skipped: Set<string>
},
parentNodeId: string,
parentId: string,
children: Array<{alias: string, depPath: string}>,
depth: number,
installable: boolean
) {
const childrenNodeIds = {}
for (const child of children) {
if (child.depPath.startsWith('link:')) {
childrenNodeIds[child.alias] = child.depPath
continue
}
if (nodeIdContainsSequence(parentNodeId, parentId, child.depPath)) {
continue
}
const childNodeId = createNodeId(parentNodeId, child.depPath)
childrenNodeIds[child.alias] = childNodeId
installable = installable && !ctx.skipped.has(child.depPath)
ctx.dependenciesTree[childNodeId] = {
children: () => buildTree(ctx,
childNodeId,
child.depPath,
ctx.childrenByParentDepPath[child.depPath],
depth + 1,
installable
),
depth,
installable,
resolvedPackage: ctx.resolvedPackagesByDepPath[child.depPath],
}
}
return childrenNodeIds
}

View File

@@ -1,16 +1,18 @@
import PnpmError from '@pnpm/error'
import logger from '@pnpm/logger'
import pkgIdToFilename from '@pnpm/pkgid-to-filename'
import {
createNodeId,
DependenciesTree,
DependenciesTreeNode,
ResolvedPackage,
splitNodeId,
} from '@pnpm/resolve-dependencies'
import { Resolution } from '@pnpm/resolver-base'
import { PackageFilesResponse } from '@pnpm/store-controller-types'
import { Dependencies, DependencyManifest } from '@pnpm/types'
import {
createNodeId,
splitNodeId,
} from './nodeIdUtils'
import {
DependenciesTree,
DependenciesTreeNode,
ResolvedPackage,
} from './resolveDependencies'
import crypto = require('crypto')
import importFrom = require('import-from')
import path = require('path')
@@ -76,7 +78,7 @@ export default function (
strictPeerDependencies: boolean
}
): {
depGraph: DependenciesGraph
dependenciesGraph: DependenciesGraph
projectsDirectPathsByAlias: {[id: string]: {[alias: string]: string}}
} {
const depGraph: DependenciesGraph = {}
@@ -132,7 +134,7 @@ export default function (
}, {})
}
return {
depGraph,
dependenciesGraph: depGraph,
projectsDirectPathsByAlias,
}
}

View File

@@ -15,7 +15,7 @@ import {
import * as dp from 'dependency-path'
import getNpmTarballUrl from 'get-npm-tarball-url'
import * as R from 'ramda'
import { depPathToRef } from './lockfile'
import depPathToRef from './depPathToRef'
import { DependenciesGraph } from './resolvePeers'
export default function (

View File

@@ -1,12 +1,16 @@
import PnpmError from '@pnpm/error'
import { ResolvedDirectDependency } from '@pnpm/resolve-dependencies'
import {
createVersionSpec,
getPrefix,
PackageSpecObject,
PinnedVersion,
updateProjectManifestObject,
} from '@pnpm/manifest-utils'
import versionSelectorType from 'version-selector-type'
import { ImporterToUpdate } from '../install'
import { PinnedVersion } from '../install/getWantedDependencies'
import save, { PackageSpecObject } from '../save'
import { ResolvedDirectDependency } from './resolveDependencyTree'
import { ImporterToResolve } from '.'
export async function updateProjectManifest (
importer: ImporterToUpdate,
export default async function updateProjectManifest (
importer: ImporterToResolve,
opts: {
directDependencies: ResolvedDirectDependency[]
preserveWorkspaceProtocol: boolean
@@ -35,17 +39,15 @@ export async function updateProjectManifest (
})
}
}
const hookedManifest = await save(
const hookedManifest = await updateProjectManifestObject(
importer.rootDir,
importer.manifest,
specsToUpsert,
{ dryRun: true }
specsToUpsert
)
const originalManifest = importer.originalManifest && await save(
const originalManifest = importer.originalManifest && await updateProjectManifestObject(
importer.rootDir,
importer.originalManifest,
specsToUpsert,
{ dryRun: true }
specsToUpsert
)
return [hookedManifest, originalManifest]
}
@@ -60,7 +62,7 @@ function resolvedDirectDepToSpecObject (
specRaw,
version,
}: ResolvedDirectDependency & { isNew?: Boolean, specRaw: string },
importer: ImporterToUpdate,
importer: ImporterToResolve,
opts: {
pinnedVersion: PinnedVersion
preserveWorkspaceProtocol: boolean
@@ -107,20 +109,6 @@ function resolvedDirectDepToSpecObject (
}
}
const getPrefix = (alias: string, name: string) => alias !== name ? `npm:${name}@` : ''
export default function getPref (
alias: string,
name: string,
version: string,
opts: {
pinnedVersion?: PinnedVersion
}
) {
const prefix = getPrefix(alias, name)
return `${prefix}${createVersionSpec(version, opts.pinnedVersion)}`
}
function getPrefPreferSpecifiedSpec (
opts: {
alias: string
@@ -162,18 +150,3 @@ function getPrefPreferSpecifiedExoticSpec (
}
return `${prefix}${createVersionSpec(opts.version, opts.pinnedVersion)}`
}
function createVersionSpec (version: string, pinnedVersion?: PinnedVersion) {
switch (pinnedVersion ?? 'major') {
case 'none':
return '*'
case 'major':
return `^${version}`
case 'minor':
return `~${version}`
case 'patch':
return `${version}`
default:
throw new PnpmError('BAD_PINNED_VERSION', `Cannot pin '${pinnedVersion ?? 'undefined'}'`)
}
}

View File

@@ -21,6 +21,9 @@
{
"path": "../lockfile-utils"
},
{
"path": "../manifest-utils"
},
{
"path": "../npm-resolver"
},
@@ -30,6 +33,12 @@
{
"path": "../pick-registry-for-package"
},
{
"path": "../prune-lockfile"
},
{
"path": "../read-package-json"
},
{
"path": "../resolver-base"
},

View File

@@ -49,11 +49,8 @@
"@zkochan/npm-package-arg": "^1.0.2",
"@zkochan/rimraf": "^1.0.0",
"dependency-path": "workspace:5.0.3",
"encode-registry": "^2.0.2",
"get-npm-tarball-url": "^2.0.1",
"graceful-fs": "^4.2.4",
"graph-sequencer": "2.0.0",
"import-from": "^3.0.0",
"is-inner-link": "^2.0.2",
"is-subdir": "^1.1.1",
"load-json-file": "^6.2.0",

View File

@@ -5,7 +5,6 @@ import {
WANTED_LOCKFILE,
} from '@pnpm/constants'
import {
packageManifestLogger,
stageLogger,
summaryLogger,
} from '@pnpm/core-loggers'
@@ -26,10 +25,11 @@ import logger, { streamParser } from '@pnpm/logger'
import { getAllDependenciesFromManifest } from '@pnpm/manifest-utils'
import { write as writeModulesYaml } from '@pnpm/modules-yaml'
import readModulesDirs from '@pnpm/read-modules-dir'
import { safeReadPackageFromDir as safeReadPkgFromDir } from '@pnpm/read-package-json'
import { removeBin } from '@pnpm/remove-bins'
import resolveDependencies, {
ResolvedDirectDependency,
DependenciesGraph,
DependenciesGraphNode,
ProjectToLink,
} from '@pnpm/resolve-dependencies'
import {
PreferredVersions,
@@ -37,16 +37,11 @@ import {
} from '@pnpm/resolver-base'
import {
DependenciesField,
DEPENDENCIES_FIELDS,
DependencyManifest,
ProjectManifest,
Registries,
} from '@pnpm/types'
import getSpecFromPackageManifest from '../getSpecFromPackageManifest'
import parseWantedDependencies from '../parseWantedDependencies'
import safeIsInnerLink from '../safeIsInnerLink'
import removeDeps from '../uninstall/removeDeps'
import { updateProjectManifest } from '../utils/getPref'
import allProjectsAreUpToDate from './allProjectsAreUpToDate'
import extendOptions, {
InstallOptions,
@@ -57,12 +52,7 @@ import getWantedDependencies, {
PinnedVersion,
WantedDependency,
} from './getWantedDependencies'
import linkPackages, {
DependenciesGraph,
DependenciesGraphNode,
Project as ProjectToLink,
} from './link'
import { depPathToRef } from './lockfile'
import linkPackages from './link'
import path = require('path')
import rimraf = require('@zkochan/rimraf')
import isInnerLink = require('is-inner-link')
@@ -605,12 +595,15 @@ async function installInContext (
workspacePackages: opts.workspacePackages,
})
const projectsToResolve = await Promise.all(projects.map((project) => _toResolveImporter(project)))
const {
dependenciesTree,
let {
dependenciesGraph,
finishLockfileUpdates,
newLockfile,
outdatedDependencies,
resolvedImporters,
resolvedPackagesByDepPath,
projectsToLink,
projectsDirectPathsByAlias,
wantedToBeSkippedPackageIds,
waitTillAllFetchingsFinish,
} = await resolveDependencies(
projectsToResolve,
{
@@ -624,8 +617,11 @@ async function installInContext (
lockfileDir: opts.lockfileDir,
nodeVersion: opts.nodeVersion,
pnpmVersion: opts.packageManager.name === 'pnpm' ? opts.packageManager.version : '',
preserveWorkspaceProtocol: opts.preserveWorkspaceProtocol,
registries: opts.registries,
saveWorkspaceProtocol: opts.saveWorkspaceProtocol,
storeController: opts.storeController,
strictPeerDependencies: opts.strictPeerDependencies,
tag: opts.tag,
updateMatching: opts.updateMatching,
virtualStoreDir: ctx.virtualStoreDir,
@@ -639,111 +635,61 @@ async function installInContext (
stage: 'resolution_done',
})
const projectsToLink = await Promise.all<ProjectToLink>(projectsToResolve.map(async (project, index) => {
const resolvedImporter = resolvedImporters[project.id]
let updatedManifest: ProjectManifest | undefined = project.manifest
let updatedOriginalManifest: ProjectManifest | undefined = project.originalManifest
if (project.updatePackageManifest) {
const manifests = await updateProjectManifest(projectsToResolve[index], {
directDependencies: resolvedImporter.directDependencies,
preserveWorkspaceProtocol: opts.preserveWorkspaceProtocol,
saveWorkspaceProtocol: opts.saveWorkspaceProtocol,
})
updatedManifest = manifests[0]
updatedOriginalManifest = manifests[1]
} else {
packageManifestLogger.debug({
prefix: project.rootDir,
updated: project.manifest,
})
}
newLockfile = opts.hooks?.afterAllResolved
? opts.hooks?.afterAllResolved(newLockfile)
: newLockfile
if (updatedManifest) {
const projectSnapshot = ctx.wantedLockfile.importers[project.id]
ctx.wantedLockfile.importers[project.id] = addDirectDependenciesToLockfile(
updatedManifest,
projectSnapshot,
resolvedImporter.linkedDependencies,
resolvedImporter.directDependencies,
ctx.registries
)
}
const topParents = project.manifest
? await getTopParents(
R.difference(
Object.keys(getAllDependenciesFromManifest(project.manifest)),
resolvedImporter.directDependencies
.filter((dep, index) => project.wantedDependencies[index].isNew === true)
.map(({ alias }) => alias) || []
),
project.modulesDir
)
: []
return {
binsDir: project.binsDir,
directNodeIdsByAlias: resolvedImporter.directNodeIdsByAlias,
id: project.id,
linkedDependencies: resolvedImporter.linkedDependencies,
manifest: updatedOriginalManifest ?? project.originalManifest ?? project.manifest,
modulesDir: project.modulesDir,
pruneDirectDependencies: project.pruneDirectDependencies,
removePackages: project.removePackages,
rootDir: project.rootDir,
topParents,
}
}))
const result = await linkPackages(
projectsToLink,
dependenciesTree,
{
afterAllResolvedHook: opts.hooks?.afterAllResolved,
currentLockfile: ctx.currentLockfile,
dryRun: opts.lockfileOnly,
force: opts.force,
hoistedDependencies: ctx.hoistedDependencies,
hoistedModulesDir: ctx.hoistedModulesDir,
hoistPattern: ctx.hoistPattern,
include: opts.include,
lockfileDir: opts.lockfileDir,
makePartialCurrentLockfile: opts.makePartialCurrentLockfile,
outdatedDependencies,
pruneStore: opts.pruneStore,
publicHoistPattern: ctx.publicHoistPattern,
registries: ctx.registries,
rootModulesDir: ctx.rootModulesDir,
sideEffectsCacheRead: opts.sideEffectsCacheRead,
skipped: ctx.skipped,
storeController: opts.storeController,
strictPeerDependencies: opts.strictPeerDependencies,
updateLockfileMinorVersion: opts.updateLockfileMinorVersion,
virtualStoreDir: ctx.virtualStoreDir,
wantedLockfile: ctx.wantedLockfile,
wantedToBeSkippedPackageIds,
}
)
ctx.pendingBuilds = ctx.pendingBuilds
.filter((relDepPath) => !result.removedDepPaths.has(relDepPath))
if (opts.ignoreScripts) {
// we can use concat here because we always only append new packages, which are guaranteed to not be there by definition
ctx.pendingBuilds = ctx.pendingBuilds
.concat(
result.newDepPaths
.filter((depPath) => result.depGraph[depPath].requiresBuild)
)
if (opts.updateLockfileMinorVersion) {
newLockfile.lockfileVersion = LOCKFILE_VERSION
}
const lockfileOpts = { forceSharedFormat: opts.forceSharedLockfile }
if (!opts.lockfileOnly) {
// postinstall hooks
if (!opts.ignoreScripts && result.newDepPaths?.length) {
const depPaths = Object.keys(result.depGraph)
const rootNodes = depPaths.filter((depPath) => result.depGraph[depPath].depth === 0)
const result = await linkPackages(
projectsToLink,
dependenciesGraph,
{
currentLockfile: ctx.currentLockfile,
force: opts.force,
hoistedDependencies: ctx.hoistedDependencies,
hoistedModulesDir: ctx.hoistedModulesDir,
hoistPattern: ctx.hoistPattern,
include: opts.include,
lockfileDir: opts.lockfileDir,
makePartialCurrentLockfile: opts.makePartialCurrentLockfile,
outdatedDependencies,
projectsDirectPathsByAlias,
pruneStore: opts.pruneStore,
publicHoistPattern: ctx.publicHoistPattern,
registries: ctx.registries,
rootModulesDir: ctx.rootModulesDir,
sideEffectsCacheRead: opts.sideEffectsCacheRead,
skipped: ctx.skipped,
storeController: opts.storeController,
strictPeerDependencies: opts.strictPeerDependencies,
virtualStoreDir: ctx.virtualStoreDir,
wantedLockfile: newLockfile,
wantedToBeSkippedPackageIds,
}
)
await finishLockfileUpdates()
await buildModules(result.depGraph, rootNodes, {
ctx.pendingBuilds = ctx.pendingBuilds
.filter((relDepPath) => !result.removedDepPaths.has(relDepPath))
if (opts.ignoreScripts) {
// we can use concat here because we always only append new packages, which are guaranteed to not be there by definition
ctx.pendingBuilds = ctx.pendingBuilds
.concat(
result.newDepPaths
.filter((depPath) => dependenciesGraph[depPath].requiresBuild)
)
} else if (result.newDepPaths?.length) {
// postinstall hooks
const depPaths = Object.keys(dependenciesGraph)
const rootNodes = depPaths.filter((depPath) => dependenciesGraph[depPath].depth === 0)
await buildModules(dependenciesGraph, rootNodes, {
childConcurrency: opts.childConcurrency,
depsToBuild: new Set(result.newDepPaths),
extraBinPaths: ctx.extraBinPaths,
@@ -759,8 +705,8 @@ async function installInContext (
}
if (result.newDepPaths?.length) {
const newPkgs = R.props<string, DependenciesGraphNode>(result.newDepPaths, result.depGraph)
await linkAllBins(newPkgs, result.depGraph, {
const newPkgs = R.props<string, DependenciesGraphNode>(result.newDepPaths, dependenciesGraph)
await linkAllBins(newPkgs, dependenciesGraph, {
optional: opts.include.optionalDependencies,
warn: (message: string) => logger.warn({ message, prefix: opts.lockfileDir }),
})
@@ -777,21 +723,13 @@ async function installInContext (
})
}
}))
}
// waiting till package requests are finished
await Promise.all(R.values(resolvedPackagesByDepPath).map(({ finishing }) => finishing()))
const lockfileOpts = { forceSharedFormat: opts.forceSharedLockfile }
if (opts.lockfileOnly) {
await writeWantedLockfile(ctx.lockfileDir, result.wantedLockfile, lockfileOpts)
} else {
await Promise.all([
opts.useLockfile
? writeLockfiles({
currentLockfile: result.currentLockfile,
currentLockfileDir: ctx.virtualStoreDir,
wantedLockfile: result.wantedLockfile,
wantedLockfile: newLockfile,
wantedLockfileDir: ctx.lockfileDir,
...lockfileOpts,
})
@@ -816,8 +754,19 @@ async function installInContext (
})
})(),
])
} else {
await finishLockfileUpdates()
await writeWantedLockfile(ctx.lockfileDir, newLockfile, lockfileOpts)
// This is only needed because otherwise the reporter will hang
stageLogger.debug({
prefix: opts.lockfileDir,
stage: 'importing_done',
})
}
await waitTillAllFetchingsFinish()
summaryLogger.debug({ prefix: opts.lockfileDir })
await opts.storeController.close()
@@ -906,100 +855,3 @@ function linkAllBins (
depNodes.map(depNode => limitLinking(() => linkBinsOfDependencies(depNode, depGraph, opts)))
)
}
function addDirectDependenciesToLockfile (
newManifest: ProjectManifest,
projectSnapshot: ProjectSnapshot,
linkedPackages: Array<{alias: string}>,
directDependencies: ResolvedDirectDependency[],
registries: Registries
): ProjectSnapshot {
const newProjectSnapshot = {
dependencies: {},
devDependencies: {},
optionalDependencies: {},
specifiers: {},
}
linkedPackages.forEach((linkedPkg) => {
newProjectSnapshot.specifiers[linkedPkg.alias] = getSpecFromPackageManifest(newManifest, linkedPkg.alias)
})
const directDependenciesByAlias = directDependencies.reduce((acc, directDependency) => {
acc[directDependency.alias] = directDependency
return acc
}, {})
const allDeps = Array.from(new Set(Object.keys(getAllDependenciesFromManifest(newManifest))))
for (const alias of allDeps) {
if (directDependenciesByAlias[alias]) {
const dep = directDependenciesByAlias[alias]
const ref = depPathToRef(dep.pkgId, {
alias: dep.alias,
realName: dep.name,
registries,
resolution: dep.resolution,
})
if (dep.dev) {
newProjectSnapshot.devDependencies[dep.alias] = ref
} else if (dep.optional) {
newProjectSnapshot.optionalDependencies[dep.alias] = ref
} else {
newProjectSnapshot.dependencies[dep.alias] = ref
}
newProjectSnapshot.specifiers[dep.alias] = getSpecFromPackageManifest(newManifest, dep.alias)
} else if (projectSnapshot.specifiers[alias]) {
newProjectSnapshot.specifiers[alias] = projectSnapshot.specifiers[alias]
if (projectSnapshot.dependencies?.[alias]) {
newProjectSnapshot.dependencies[alias] = projectSnapshot.dependencies[alias]
} else if (projectSnapshot.optionalDependencies?.[alias]) {
newProjectSnapshot.optionalDependencies[alias] = projectSnapshot.optionalDependencies[alias]
} else if (projectSnapshot.devDependencies?.[alias]) {
newProjectSnapshot.devDependencies[alias] = projectSnapshot.devDependencies[alias]
}
}
}
alignDependencyTypes(newManifest, newProjectSnapshot)
return newProjectSnapshot
}
function alignDependencyTypes (manifest: ProjectManifest, projectSnapshot: ProjectSnapshot) {
const depTypesOfAliases = getAliasToDependencyTypeMap(manifest)
// Aligning the dependency types in pnpm-lock.yaml
for (const depType of DEPENDENCIES_FIELDS) {
if (!projectSnapshot[depType]) continue
for (const alias of Object.keys(projectSnapshot[depType] ?? {})) {
if (depType === depTypesOfAliases[alias] || !depTypesOfAliases[alias]) continue
projectSnapshot[depTypesOfAliases[alias]][alias] = projectSnapshot[depType]![alias]
delete projectSnapshot[depType]![alias]
}
}
}
function getAliasToDependencyTypeMap (manifest: ProjectManifest) {
const depTypesOfAliases = {}
for (const depType of DEPENDENCIES_FIELDS) {
if (!manifest[depType]) continue
for (const alias of Object.keys(manifest[depType] ?? {})) {
if (!depTypesOfAliases[alias]) {
depTypesOfAliases[alias] = depType
}
}
}
return depTypesOfAliases
}
async function getTopParents (pkgNames: string[], modules: string) {
const pkgs = await Promise.all(
pkgNames.map((pkgName) => path.join(modules, pkgName)).map(safeReadPkgFromDir)
)
return (
pkgs
.filter(Boolean) as DependencyManifest[]
)
.map(({ name, version }: DependencyManifest) => ({ name, version }))
}

View File

@@ -1,8 +1,4 @@
import {
ENGINE_NAME,
LOCKFILE_VERSION,
WANTED_LOCKFILE,
} from '@pnpm/constants'
import { ENGINE_NAME } from '@pnpm/constants'
import {
rootLogger,
stageLogger,
@@ -16,20 +12,17 @@ import { Lockfile } from '@pnpm/lockfile-file'
import logger from '@pnpm/logger'
import { prune } from '@pnpm/modules-cleaner'
import { IncludedDependencies } from '@pnpm/modules-yaml'
import { DependenciesTree, LinkedDependency } from '@pnpm/resolve-dependencies'
import {
DependenciesGraph,
DependenciesGraphNode,
ProjectToLink,
} from '@pnpm/resolve-dependencies'
import { StoreController } from '@pnpm/store-controller-types'
import symlinkDependency, { symlinkDirectRootDependency } from '@pnpm/symlink-dependency'
import {
HoistedDependencies,
ProjectManifest,
Registries,
} from '@pnpm/types'
import { depPathToRef } from './lockfile'
import resolvePeers, {
DependenciesGraph,
DependenciesGraphNode,
} from './resolvePeers'
import updateLockfile from './updateLockfile'
import path = require('path')
import fs = require('mz/fs')
import pLimit = require('p-limit')
@@ -37,31 +30,11 @@ import R = require('ramda')
const brokenModulesLogger = logger('_broken_node_modules')
export {
DependenciesGraph,
DependenciesGraphNode,
}
export interface Project {
binsDir: string
directNodeIdsByAlias: {[alias: string]: string}
id: string
linkedDependencies: LinkedDependency[]
manifest: ProjectManifest
modulesDir: string
pruneDirectDependencies: boolean
removePackages?: string[]
rootDir: string
topParents: Array<{name: string, version: string}>
}
export default async function linkPackages (
projects: Project[],
dependenciesTree: DependenciesTree,
projects: ProjectToLink[],
depGraph: DependenciesGraph,
opts: {
afterAllResolvedHook?: (lockfile: Lockfile) => Lockfile
currentLockfile: Lockfile
dryRun: boolean
force: boolean
hoistedDependencies: HoistedDependencies
hoistedModulesDir: string
@@ -71,6 +44,9 @@ export default async function linkPackages (
lockfileDir: string
makePartialCurrentLockfile: boolean
outdatedDependencies: {[pkgId: string]: string}
projectsDirectPathsByAlias: {
[id: string]: {[alias: string]: string}
}
pruneStore: boolean
registries: Registries
rootModulesDir: string
@@ -78,59 +54,18 @@ export default async function linkPackages (
skipped: Set<string>
storeController: StoreController
strictPeerDependencies: boolean
// This is only needed till lockfile v4
updateLockfileMinorVersion: boolean
virtualStoreDir: string
wantedLockfile: Lockfile
wantedToBeSkippedPackageIds: Set<string>
}
): Promise<{
currentLockfile: Lockfile
depGraph: DependenciesGraph
newDepPaths: string[]
newHoistedDependencies: HoistedDependencies
removedDepPaths: Set<string>
wantedLockfile: Lockfile
}> {
// TODO: decide what kind of logging should be here.
// The `Creating dependency graph` is not good to report in all cases as
// sometimes node_modules is alread up-to-date
// logger.info(`Creating dependency graph`)
const { depGraph, projectsDirectPathsByAlias } = resolvePeers({
dependenciesTree,
lockfileDir: opts.lockfileDir,
projects,
strictPeerDependencies: opts.strictPeerDependencies,
virtualStoreDir: opts.virtualStoreDir,
})
for (const { id } of projects) {
for (const [alias, depPath] of R.toPairs(projectsDirectPathsByAlias[id])) {
const depNode = depGraph[depPath]
if (depNode.isPure) continue
const projectSnapshot = opts.wantedLockfile.importers[id]
const ref = depPathToRef(depPath, {
alias,
realName: depNode.name,
registries: opts.registries,
resolution: depNode.resolution,
})
if (projectSnapshot.dependencies?.[alias]) {
projectSnapshot.dependencies[alias] = ref
} else if (projectSnapshot.devDependencies?.[alias]) {
projectSnapshot.devDependencies[alias] = ref
} else if (projectSnapshot.optionalDependencies?.[alias]) {
projectSnapshot.optionalDependencies[alias] = ref
}
}
}
const { newLockfile, pendingRequiresBuilds } = updateLockfile(depGraph, opts.wantedLockfile, opts.virtualStoreDir, opts.registries) // eslint-disable-line:prefer-const
const newWantedLockfile = opts.afterAllResolvedHook
? opts.afterAllResolvedHook(newLockfile)
: newLockfile
let depNodes = R.values(depGraph).filter(({ depPath, packageId }) => {
if (newWantedLockfile.packages?.[depPath] && !newWantedLockfile.packages[depPath].optional) {
if (opts.wantedLockfile.packages?.[depPath] && !opts.wantedLockfile.packages[depPath].optional) {
opts.skipped.delete(depPath)
return true
}
@@ -152,7 +87,6 @@ export default async function linkPackages (
}
const removedDepPaths = await prune(projects, {
currentLockfile: opts.currentLockfile,
dryRun: opts.dryRun,
hoistedDependencies: opts.hoistedDependencies,
hoistedModulesDir: (opts.hoistPattern && opts.hoistedModulesDir) ?? undefined,
include: opts.include,
@@ -163,7 +97,7 @@ export default async function linkPackages (
skipped: opts.skipped,
storeController: opts.storeController,
virtualStoreDir: opts.virtualStoreDir,
wantedLockfile: newWantedLockfile,
wantedLockfile: opts.wantedLockfile,
})
stageLogger.debug({
@@ -177,7 +111,7 @@ export default async function linkPackages (
registries: opts.registries,
skipped: opts.skipped,
}
const newCurrentLockfile = filterLockfileByImporters(newWantedLockfile, projectIds, {
const newCurrentLockfile = filterLockfileByImporters(opts.wantedLockfile, projectIds, {
...filterOpts,
failOnMissingDependencies: true,
skipped: new Set(),
@@ -190,7 +124,6 @@ export default async function linkPackages (
newCurrentLockfile,
depGraph,
{
dryRun: opts.dryRun,
force: opts.force,
lockfileDir: opts.lockfileDir,
optional: opts.include.optionalDependencies,
@@ -214,14 +147,13 @@ export default async function linkPackages (
}, {})
await Promise.all(projects.map(({ id, manifest, modulesDir, rootDir }) => {
const directPathsByAlias = projectsDirectPathsByAlias[id]
const directPathsByAlias = opts.projectsDirectPathsByAlias[id]
return Promise.all(
Object.keys(directPathsByAlias)
.map((rootAlias) => ({ rootAlias, depGraphNode: rootDepsByDepPath[directPathsByAlias[rootAlias]] }))
.filter(({ depGraphNode }) => depGraphNode)
.map(async ({ rootAlias, depGraphNode }) => {
if (
!opts.dryRun &&
(await symlinkDependency(depGraphNode.dir, modulesDir, rootAlias)).reused
) return
@@ -242,57 +174,27 @@ export default async function linkPackages (
)
}))
if (opts.updateLockfileMinorVersion) {
newWantedLockfile.lockfileVersion = LOCKFILE_VERSION
}
await Promise.all(pendingRequiresBuilds.map(async (depPath) => {
const depNode = depGraph[depPath]
if (!depNode.fetchingBundledManifest) {
// This should never ever happen
throw new Error(`Cannot create ${WANTED_LOCKFILE} because raw manifest (aka package.json) wasn't fetched for "${depPath}"`)
}
const filesResponse = await depNode.fetchingFiles()
// The npm team suggests to always read the package.json for deciding whether the package has lifecycle scripts
const pkgJson = await depNode.fetchingBundledManifest()
depNode.requiresBuild = Boolean(
pkgJson.scripts != null && (
Boolean(pkgJson.scripts.preinstall) ||
Boolean(pkgJson.scripts.install) ||
Boolean(pkgJson.scripts.postinstall)
) ||
filesResponse.filesIndex['binding.gyp'] ||
Object.keys(filesResponse.filesIndex).some((filename) => !!filename.match(/^[.]hooks[\\/]/)) // TODO: optimize this
)
// TODO: try to cover with unit test the case when entry is no longer available in lockfile
// It is an edge that probably happens if the entry is removed during lockfile prune
if (depNode.requiresBuild && newWantedLockfile.packages![depPath]) {
newWantedLockfile.packages![depPath].requiresBuild = true
}
}))
let currentLockfile: Lockfile
const allImportersIncluded = R.equals(projectIds.sort(), Object.keys(newWantedLockfile.importers).sort())
const allImportersIncluded = R.equals(projectIds.sort(), Object.keys(opts.wantedLockfile.importers).sort())
if (
opts.makePartialCurrentLockfile ||
!allImportersIncluded
) {
const packages = opts.currentLockfile.packages ?? {}
if (newWantedLockfile.packages) {
for (const depPath in newWantedLockfile.packages) { // eslint-disable-line:forin
if (opts.wantedLockfile.packages) {
for (const depPath in opts.wantedLockfile.packages) { // eslint-disable-line:forin
if (depGraph[depPath]) {
packages[depPath] = newWantedLockfile.packages[depPath]
packages[depPath] = opts.wantedLockfile.packages[depPath]
}
}
}
const projects = projectIds.reduce((acc, projectId) => {
acc[projectId] = newWantedLockfile.importers[projectId]
acc[projectId] = opts.wantedLockfile.importers[projectId]
return acc
}, opts.currentLockfile.importers)
currentLockfile = filterLockfileByImporters(
{
...newWantedLockfile,
...opts.wantedLockfile,
importers: projects,
packages,
},
@@ -308,7 +210,7 @@ export default async function linkPackages (
opts.include.optionalDependencies &&
opts.skipped.size === 0
) {
currentLockfile = newWantedLockfile
currentLockfile = opts.wantedLockfile
} else {
currentLockfile = newCurrentLockfile
}
@@ -328,28 +230,24 @@ export default async function linkPackages (
newHoistedDependencies = {}
}
if (!opts.dryRun) {
await Promise.all(
projects.map((project) =>
Promise.all(project.linkedDependencies.map((linkedDependency) => {
const depLocation = resolvePath(project.rootDir, linkedDependency.resolution.directory)
return symlinkDirectRootDependency(depLocation, project.modulesDir, linkedDependency.alias, {
fromDependenciesField: linkedDependency.dev && 'devDependencies' || linkedDependency.optional && 'optionalDependencies' || 'dependencies',
linkedPackage: linkedDependency,
prefix: project.rootDir,
})
}))
)
await Promise.all(
projects.map((project) =>
Promise.all(project.linkedDependencies.map((linkedDependency) => {
const depLocation = resolvePath(project.rootDir, linkedDependency.resolution.directory)
return symlinkDirectRootDependency(depLocation, project.modulesDir, linkedDependency.alias, {
fromDependenciesField: linkedDependency.dev && 'devDependencies' || linkedDependency.optional && 'optionalDependencies' || 'dependencies',
linkedPackage: linkedDependency,
prefix: project.rootDir,
})
}))
)
}
)
return {
currentLockfile,
depGraph,
newDepPaths,
newHoistedDependencies,
removedDepPaths,
wantedLockfile: newWantedLockfile,
}
}
@@ -366,7 +264,6 @@ async function linkNewPackages (
wantedLockfile: Lockfile,
depGraph: DependenciesGraph,
opts: {
dryRun: boolean
force: boolean
optional: boolean
lockfileDir: string
@@ -416,8 +313,6 @@ async function linkNewPackages (
const newDepPaths = Array.from(newDepPathsSet)
if (opts.dryRun) return newDepPaths
const newPkgs = R.props<string, DependenciesGraphNode>(newDepPaths, depGraph)
await Promise.all([

View File

@@ -11,6 +11,13 @@ import {
writeLockfiles,
} from '@pnpm/lockfile-file'
import logger, { streamParser } from '@pnpm/logger'
import {
getPref,
getSpecFromPackageManifest,
guessDependencyType,
PackageSpecObject,
updateProjectManifestObject,
} from '@pnpm/manifest-utils'
import { prune } from '@pnpm/modules-cleaner'
import { pruneSharedLockfile } from '@pnpm/prune-lockfile'
import readProjectManifest from '@pnpm/read-project-manifest'
@@ -21,9 +28,6 @@ import {
DependencyManifest,
ProjectManifest,
} from '@pnpm/types'
import getSpecFromPackageManifest from '../getSpecFromPackageManifest'
import save, { guessDependencyType, PackageSpecObject } from '../save'
import getPref from '../utils/getPref'
import {
extendOptions,
LinkOptions,
@@ -142,7 +146,7 @@ export default async function link (
let newPkg!: ProjectManifest
if (opts.targetDependenciesField) {
newPkg = await save(opts.dir, opts.manifest, specsToUpsert)
newPkg = await updateProjectManifestObject(opts.dir, opts.manifest, specsToUpsert)
for (const { alias } of specsToUpsert) {
updatedWantedLockfile.importers[importerId].specifiers[alias] = getSpecFromPackageManifest(newPkg, alias)
}

View File

@@ -8,7 +8,6 @@ import './link'
import './lockfile'
import './offline'
import './packageImportMethods'
import './parseWantedDependencies.test'
import './prune'
import './uninstall'
import './unlink'

34
pnpm-lock.yaml generated
View File

@@ -1061,10 +1061,14 @@ importers:
tempy: ^0.6.0
packages/manifest-utils:
dependencies:
'@pnpm/core-loggers': 'link:../core-loggers'
'@pnpm/error': 'link:../error'
'@pnpm/types': 'link:../types'
devDependencies:
'@pnpm/manifest-utils': 'link:'
specifiers:
'@pnpm/core-loggers': 'workspace:^4.2.0'
'@pnpm/error': 'workspace:^1.3.1'
'@pnpm/manifest-utils': 'link:'
'@pnpm/types': 'workspace:6.2.0'
packages/matcher:
@@ -2447,37 +2451,49 @@ importers:
is-windows: ^1.0.2
packages/resolve-dependencies:
dependencies:
'@pnpm/constants': 'link:../constants'
'@pnpm/core-loggers': 'link:../core-loggers'
'@pnpm/error': 'link:../error'
'@pnpm/lockfile-types': 'link:../lockfile-types'
'@pnpm/lockfile-utils': 'link:../lockfile-utils'
'@pnpm/manifest-utils': 'link:../manifest-utils'
'@pnpm/npm-resolver': 'link:../npm-resolver'
'@pnpm/package-is-installable': 'link:../package-is-installable'
'@pnpm/pick-registry-for-package': 'link:../pick-registry-for-package'
'@pnpm/pkgid-to-filename': 3.0.0
'@pnpm/prune-lockfile': 'link:../prune-lockfile'
'@pnpm/read-package-json': 'link:../read-package-json'
'@pnpm/resolver-base': 'link:../resolver-base'
'@pnpm/store-controller-types': 'link:../store-controller-types'
'@pnpm/types': 'link:../types'
dependency-path: 'link:../dependency-path'
encode-registry: 2.0.2
get-npm-tarball-url: 2.0.1
import-from: 3.0.0
path-exists: 4.0.0
ramda: 0.27.1
replace-string: 3.1.0
semver: 7.3.2
version-selector-type: 2.0.1
devDependencies:
'@pnpm/logger': 3.2.2
'@pnpm/resolve-dependencies': 'link:'
'@types/ramda': 0.27.14
'@types/semver': 7.3.3
specifiers:
'@pnpm/constants': 'workspace:^4.0.0'
'@pnpm/core-loggers': 'workspace:4.2.0'
'@pnpm/error': 'workspace:1.3.1'
'@pnpm/lockfile-types': 'workspace:2.0.1'
'@pnpm/lockfile-utils': 'workspace:2.0.16'
'@pnpm/logger': ^3.2.2
'@pnpm/manifest-utils': 'workspace:^1.0.3'
'@pnpm/npm-resolver': 'workspace:10.0.1'
'@pnpm/package-is-installable': 'workspace:4.0.14'
'@pnpm/pick-registry-for-package': 'workspace:1.0.3'
'@pnpm/pkgid-to-filename': ^3.0.0
'@pnpm/prune-lockfile': 'workspace:^2.0.14'
'@pnpm/read-package-json': 'workspace:^3.1.5'
'@pnpm/resolve-dependencies': 'link:'
'@pnpm/resolver-base': 'workspace:7.0.3'
'@pnpm/store-controller-types': 'workspace:8.0.2'
@@ -2485,10 +2501,14 @@ importers:
'@types/ramda': ^0.27.14
'@types/semver': ^7.3.3
dependency-path: 'workspace:5.0.3'
encode-registry: ^2.0.2
get-npm-tarball-url: ^2.0.1
import-from: ^3.0.0
path-exists: ^4.0.0
ramda: ^0.27.1
replace-string: ^3.1.0
semver: ^7.3.2
version-selector-type: ^2.0.1
packages/resolve-workspace-range:
dependencies:
semver: 7.3.2
@@ -2644,11 +2664,8 @@ importers:
'@zkochan/npm-package-arg': 1.0.2
'@zkochan/rimraf': 1.0.0
dependency-path: 'link:../dependency-path'
encode-registry: 2.0.2
get-npm-tarball-url: 2.0.1
graceful-fs: 4.2.4
graph-sequencer: 2.0.0
import-from: 3.0.0
is-inner-link: 2.0.2
is-subdir: 1.1.1
load-json-file: 6.2.0
@@ -2763,13 +2780,10 @@ importers:
deep-require-cwd: 1.0.0
dependency-path: 'workspace:5.0.3'
dir-is-case-sensitive: ^1.0.2
encode-registry: ^2.0.2
execa: ^4.0.3
exists-link: 2.0.0
get-npm-tarball-url: ^2.0.1
graceful-fs: ^4.2.4
graph-sequencer: 2.0.0
import-from: ^3.0.0
is-ci: ^2.0.0
is-inner-link: ^2.0.2
is-subdir: ^1.1.1
@@ -12513,6 +12527,14 @@ packages:
'0': node >=0.6.0
resolution:
integrity: sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=
/version-selector-type/2.0.1:
dependencies:
semver: 6.3.0
dev: false
engines:
node: '>=4'
resolution:
integrity: sha512-wglKuX3nE2jJPld8UfSKo8YZeYovkAsErRCzbzEt5hHWp+s1Ai4q97dQI5X7wFXAykeHxvez38HJScuUWPxb6g==
/version-selector-type/3.0.0:
dependencies:
semver: 7.3.2