feat: the installation API should return installation stats (#6490)

This commit is contained in:
Zoltan Kochan
2023-05-02 12:45:29 +03:00
committed by GitHub
parent c6ffb60322
commit 42902ef851
13 changed files with 209 additions and 76 deletions

View File

@@ -0,0 +1,5 @@
---
"@pnpm/core": major
---
Return installation stats. Breaking change to the API.

View File

@@ -0,0 +1,5 @@
---
"@pnpm/pkg-manager.direct-dep-linker": minor
---
Return the amount of linked dependencies.

View File

@@ -0,0 +1,5 @@
---
"@pnpm/headless": minor
---
Return installation stats.

View File

@@ -14,7 +14,7 @@ import {
import { createBase32HashFromFile } from '@pnpm/crypto.base32-hash'
import { PnpmError } from '@pnpm/error'
import { getContext, type PnpmContext } from '@pnpm/get-context'
import { headlessInstall } from '@pnpm/headless'
import { headlessInstall, type InstallationResultStats } from '@pnpm/headless'
import {
makeNodeRequireOption,
runLifecycleHook,
@@ -137,7 +137,7 @@ export async function install (
} & InstallMutationOptions
) {
const rootDir = opts.dir ?? process.cwd()
const projects = await mutateModules(
const { updatedProjects: projects } = await mutateModules(
[
{
mutation: 'install',
@@ -186,7 +186,7 @@ export async function mutateModulesInSingleProject (
},
maybeOpts: Omit<MutateModulesOptions, 'allProjects'> & InstallMutationOptions
): Promise<UpdatedProject> {
const [updatedProject] = await mutateModules(
const result = await mutateModules(
[
{
...project,
@@ -203,13 +203,18 @@ export async function mutateModulesInSingleProject (
}],
}
)
return updatedProject
return result.updatedProjects[0]
}
interface MutateModulesResult {
updatedProjects: UpdatedProject[]
stats: InstallationResultStats
}
export async function mutateModules (
projects: MutatedProject[],
maybeOpts: MutateModulesOptions
): Promise<UpdatedProject[]> {
): Promise<MutateModulesResult> {
const reporter = maybeOpts?.reporter
if ((reporter != null) && typeof reporter === 'function') {
streamParser.on('data', reporter)
@@ -271,9 +276,12 @@ export async function mutateModules (
await cleanGitBranchLockfiles(ctx.lockfileDir)
}
return result
return {
updatedProjects: result.updatedProjects,
stats: result.stats ?? { added: 0, removed: 0, linkedToRoot: 0 },
}
async function _install (): Promise<UpdatedProject[]> {
async function _install (): Promise<{ updatedProjects: UpdatedProject[], stats?: InstallationResultStats }> {
const scriptsOpts: RunLifecycleHooksConcurrentlyOptions = {
extraBinPaths: opts.extraBinPaths,
extraEnv: opts.extraEnv,
@@ -369,7 +377,9 @@ Note that in CI environments, this setting is enabled by default.`,
if (opts.lockfileOnly) {
// The lockfile will only be changed if the workspace will have new projects with no dependencies.
await writeWantedLockfile(ctx.lockfileDir, ctx.wantedLockfile)
return projects.map((mutatedProject) => ctx.projects[mutatedProject.rootDir])
return {
updatedProjects: projects.map((mutatedProject) => ctx.projects[mutatedProject.rootDir]),
}
}
if (!ctx.existsWantedLockfile) {
if (Object.values(ctx.projects).some((project) => pkgHasDependencies(project.manifest))) {
@@ -382,7 +392,7 @@ Note that in CI environments, this setting is enabled by default.`,
logger.info({ message: 'Lockfile is up to date, resolution step is skipped', prefix: opts.lockfileDir })
}
try {
await headlessInstall({
const { stats } = await headlessInstall({
...ctx,
...opts,
currentEngine: {
@@ -409,13 +419,16 @@ Note that in CI environments, this setting is enabled by default.`,
mergeGitBranchLockfiles: opts.mergeGitBranchLockfiles,
})
}
return projects.map((mutatedProject) => {
const project = ctx.projects[mutatedProject.rootDir]
return {
...project,
manifest: project.originalManifest ?? project.manifest,
}
})
return {
updatedProjects: projects.map((mutatedProject) => {
const project = ctx.projects[mutatedProject.rootDir]
return {
...project,
manifest: project.originalManifest ?? project.manifest,
}
}),
stats,
}
} catch (error: any) { // eslint-disable-line
if (
frozenLockfile ||
@@ -492,7 +505,9 @@ Note that in CI environments, this setting is enabled by default.`,
}
}
if (packagesToInstall.length === 0) {
return projects.map((mutatedProject) => ctx.projects[mutatedProject.rootDir])
return {
updatedProjects: projects.map((mutatedProject) => ctx.projects[mutatedProject.rootDir]),
}
}
// TODO: install only those that were unlinked
@@ -524,7 +539,9 @@ Note that in CI environments, this setting is enabled by default.`,
}
}
if (packagesToInstall.length === 0) {
return projects.map((mutatedProject) => ctx.projects[mutatedProject.rootDir])
return {
updatedProjects: projects.map((mutatedProject) => ctx.projects[mutatedProject.rootDir]),
}
}
// TODO: install only those that were unlinked
@@ -611,7 +628,10 @@ Note that in CI environments, this setting is enabled by default.`,
patchedDependencies: patchedDependenciesWithResolvedPath,
})
return result.projects
return {
updatedProjects: result.projects,
stats: result.stats,
}
}
}
@@ -721,7 +741,7 @@ export async function addDependenciesToPackage (
} & InstallMutationOptions
) {
const rootDir = opts.dir ?? process.cwd()
const projects = await mutateModules(
const { updatedProjects: projects } = await mutateModules(
[
{
allowNew: opts.allowNew,
@@ -775,6 +795,7 @@ export interface UpdatedProject {
interface InstallFunctionResult {
newLockfile: Lockfile
projects: UpdatedProject[]
stats?: InstallationResultStats
}
type InstallFunction = (
@@ -980,6 +1001,7 @@ const _installInContext: InstallFunction = async (projects, ctx, opts) => {
useGitBranchLockfile: opts.useGitBranchLockfile,
mergeGitBranchLockfiles: opts.mergeGitBranchLockfiles,
}
let stats: InstallationResultStats | undefined
if (!opts.lockfileOnly && !isInstallationOnlyForLockfileCheck && opts.enableModulesDir) {
const result = await linkPackages(
projects,
@@ -1014,6 +1036,7 @@ const _installInContext: InstallFunction = async (projects, ctx, opts) => {
wantedToBeSkippedPackageIds,
}
)
stats = result.stats
await finishLockfileUpdates()
if (opts.enablePnp) {
const importerNames = Object.fromEntries(
@@ -1255,6 +1278,7 @@ const _installInContext: InstallFunction = async (projects, ctx, opts) => {
peerDependencyIssues: peerDependencyIssuesByProjects[id],
rootDir,
})),
stats,
}
}
@@ -1281,7 +1305,7 @@ const installInContext: InstallFunction = async (projects, ctx, opts) => {
...opts,
lockfileOnly: true,
})
await headlessInstall({
const { stats } = await headlessInstall({
...ctx,
...opts,
currentEngine: {
@@ -1295,7 +1319,10 @@ const installInContext: InstallFunction = async (projects, ctx, opts) => {
wantedLockfile: result.newLockfile,
useLockfile: opts.useLockfile && ctx.wantedLockfileIsModified,
})
return result
return {
...result,
stats,
}
}
return await _installInContext(projects, ctx, opts)
} catch (error: any) { // eslint-disable-line

View File

@@ -10,6 +10,7 @@ import {
filterLockfileByImporters,
} from '@pnpm/filter-lockfile'
import { linkDirectDeps } from '@pnpm/pkg-manager.direct-dep-linker'
import { type InstallationResultStats } from '@pnpm/headless'
import { hoist } from '@pnpm/hoist'
import { type Lockfile } from '@pnpm/lockfile-file'
import { logger } from '@pnpm/logger'
@@ -76,6 +77,7 @@ export async function linkPackages (
newDepPaths: string[]
newHoistedDependencies: HoistedDependencies
removedDepPaths: Set<string>
stats: InstallationResultStats
}> {
let depNodes = Object.values(depGraph).filter(({ depPath, id }) => {
if (((opts.wantedLockfile.packages?.[depPath]) != null) && !opts.wantedLockfile.packages[depPath].optional) {
@@ -131,7 +133,7 @@ export async function linkPackages (
failOnMissingDependencies: true,
skipped: new Set(),
})
const newDepPaths = await linkNewPackages(
const { newDepPaths, added } = await linkNewPackages(
filterLockfileByImporters(opts.currentLockfile, projectIds, {
...filterOpts,
failOnMissingDependencies: false,
@@ -223,6 +225,7 @@ export async function linkPackages (
newHoistedDependencies = opts.hoistedDependencies
}
let linkedToRoot = 0
if (opts.symlink) {
const projectsToLink = Object.fromEntries(await Promise.all(
projects.map(async ({ id, manifest, modulesDir, rootDir }) => {
@@ -266,7 +269,7 @@ export async function linkPackages (
}]
}))
)
await linkDirectDeps(projectsToLink, { dedupe: opts.dedupeDirectDeps })
linkedToRoot = await linkDirectDeps(projectsToLink, { dedupe: opts.dedupeDirectDeps })
}
return {
@@ -274,6 +277,11 @@ export async function linkPackages (
newDepPaths,
newHoistedDependencies,
removedDepPaths,
stats: {
added,
removed: removedDepPaths.size,
linkedToRoot,
},
}
}
@@ -301,7 +309,7 @@ async function linkNewPackages (
storeController: StoreController
virtualStoreDir: string
}
): Promise<string[]> {
): Promise<{ newDepPaths: string[], added: number }> {
const wantedRelDepPaths = difference(Object.keys(wantedLockfile.packages ?? {}), Array.from(opts.skipped))
let newDepPathsSet: Set<string>
@@ -316,8 +324,9 @@ async function linkNewPackages (
newDepPathsSet = await selectNewFromWantedDeps(wantedRelDepPaths, currentLockfile, depGraph)
}
const added = newDepPathsSet.size
statsLogger.debug({
added: newDepPathsSet.size,
added,
prefix: opts.lockfileDir,
})
@@ -338,7 +347,7 @@ async function linkNewPackages (
}
}
if (!newDepPathsSet.size && (existingWithUpdatedDeps.length === 0)) return []
if (!newDepPathsSet.size && (existingWithUpdatedDeps.length === 0)) return { newDepPaths: [], added }
const newDepPaths = Array.from(newDepPathsSet)
@@ -362,7 +371,7 @@ async function linkNewPackages (
}),
])
return newDepPaths
return { newDepPaths, added }
}
async function selectNewFromWantedDeps (

View File

@@ -114,7 +114,7 @@ test('preserve subdeps on update', async () => {
test('adding a new dependency to one of the workspace projects', async () => {
prepareEmpty()
let [{ manifest }] = await mutateModules([
let [{ manifest }] = (await mutateModules([
{
mutation: 'install',
rootDir: path.resolve('project-1'),
@@ -151,7 +151,7 @@ test('adding a new dependency to one of the workspace projects', async () => {
},
],
nodeLinker: 'hoisted',
}))
}))).updatedProjects
manifest = await addDependenciesToPackage(
manifest,
['is-negative@1.0.0'],

View File

@@ -1481,11 +1481,11 @@ test('do not modify the manifest of the injected workpspace project', async () =
},
},
}
const [project1] = await mutateModules(importers, await testDefaults({
const [project1] = (await mutateModules(importers, await testDefaults({
autoInstallPeers: false,
allProjects,
workspacePackages,
}))
}))).updatedProjects
expect(project1.manifest).toStrictEqual({
name: 'project-1',
version: '1.0.0',

View File

@@ -248,7 +248,7 @@ test('dependencies of other importers are not pruned when installing for a subse
},
])
const [{ manifest }] = await mutateModules([
const [{ manifest }] = (await mutateModules([
{
mutation: 'install',
rootDir: path.resolve('project-1'),
@@ -284,7 +284,7 @@ test('dependencies of other importers are not pruned when installing for a subse
rootDir: path.resolve('project-2'),
},
],
}))
}))).updatedProjects
await addDependenciesToPackage(manifest, ['is-positive@2'], await testDefaults({
dir: path.resolve('project-1'),
@@ -357,7 +357,7 @@ test('dependencies of other importers are not pruned when (headless) installing
rootDir: path.resolve('project-2'),
},
]
const [{ manifest }] = await mutateModules(importers, await testDefaults({ allProjects }))
const [{ manifest }] = (await mutateModules(importers, await testDefaults({ allProjects }))).updatedProjects
await addDependenciesToPackage(manifest, ['is-positive@2'], await testDefaults({
dir: path.resolve('project-1'),
@@ -384,7 +384,7 @@ test('dependencies of other importers are not pruned when (headless) installing
test('adding a new dev dependency to project that uses a shared lockfile', async () => {
prepareEmpty()
let [{ manifest }] = await mutateModules([
let [{ manifest }] = (await mutateModules([
{
mutation: 'install',
rootDir: path.resolve('project-1'),
@@ -404,7 +404,7 @@ test('adding a new dev dependency to project that uses a shared lockfile', async
rootDir: path.resolve('project-1'),
},
],
}))
}))).updatedProjects
manifest = await addDependenciesToPackage(manifest, ['is-negative@1.0.0'], await testDefaults({ prefix: path.resolve('project-1'), targetDependenciesField: 'devDependencies' }))
expect(manifest.dependencies).toStrictEqual({ 'is-positive': '1.0.0' })
@@ -922,7 +922,7 @@ test('adding a new dependency with the workspace: protocol and save-workspace-pr
test('update workspace range', async () => {
prepareEmpty()
const updatedImporters = await mutateModules([
const { updatedProjects: updatedImporters } = await mutateModules([
{
dependencySelectors: ['dep1', 'dep2', 'dep3', 'dep4', 'dep5', 'dep6'],
mutation: 'installSome',
@@ -1068,7 +1068,7 @@ test('update workspace range', async () => {
test('update workspace range when save-workspace-protocol is "rolling"', async () => {
prepareEmpty()
const updatedImporters = await mutateModules([
const { updatedProjects: updatedImporters } = await mutateModules([
{
dependencySelectors: ['dep1', 'dep2', 'dep3', 'dep4', 'dep5', 'dep6'],
mutation: 'installSome',

View File

@@ -465,7 +465,7 @@ test('skip optional dependency that does not support the current OS, when doing
},
])
const [{ manifest }] = await mutateModules(
const [{ manifest }] = (await mutateModules(
[
{
mutation: 'install',
@@ -506,7 +506,7 @@ test('skip optional dependency that does not support the current OS, when doing
lockfileDir: process.cwd(),
lockfileOnly: true,
})
)
)).updatedProjects
await mutateModulesInSingleProject({
manifest,

View File

@@ -0,0 +1,57 @@
import { prepareEmpty } from '@pnpm/prepare'
import {
mutateModules,
type MutatedProject,
} from '@pnpm/core'
import rimraf from '@zkochan/rimraf'
import { testDefaults } from '../utils'
test('spec not specified in package.json.dependencies', async () => {
prepareEmpty()
const importers: MutatedProject[] = [
{
mutation: 'install',
rootDir: process.cwd(),
},
]
const allProjects = [
{
buildIndex: 0,
manifest: {
name: 'project-1',
version: '1.0.0',
dependencies: {
'is-positive': '1.0.0',
},
},
rootDir: process.cwd(),
},
]
{
const { stats } = await mutateModules(importers, await testDefaults({ allProjects }))
expect(stats.added).toEqual(1)
expect(stats.removed).toEqual(0)
expect(stats.linkedToRoot).toEqual(1)
}
await rimraf('node_modules')
{
const { stats } = await mutateModules(importers, await testDefaults({ allProjects, frozenLockfile: true }))
expect(stats.added).toEqual(1)
expect(stats.removed).toEqual(0)
expect(stats.linkedToRoot).toEqual(1)
}
{
const { stats } = await mutateModules([
{
mutation: 'uninstallSome',
dependencyNames: ['is-positive'],
rootDir: process.cwd(),
},
], await testDefaults({ allProjects, frozenLockfile: true }))
expect(stats.added).toEqual(0)
expect(stats.removed).toEqual(1)
expect(stats.linkedToRoot).toEqual(0)
}
})

View File

@@ -29,18 +29,19 @@ export async function linkDirectDeps (
opts: {
dedupe: boolean
}
) {
): Promise<number> {
if (opts.dedupe && projects['.'] && Object.keys(projects).length > 1) {
return linkDirectDepsAndDedupe(projects['.'], omit(['.'], projects))
}
await Promise.all(Object.values(projects).map(linkDirectDepsOfProject))
const numberOfLinkedDeps = await Promise.all(Object.values(projects).map(linkDirectDepsOfProject))
return numberOfLinkedDeps.reduce((sum, count) => sum + count, 0)
}
async function linkDirectDepsAndDedupe (
rootProject: ProjectToLink,
projects: Record<string, ProjectToLink>
) {
await linkDirectDepsOfProject(rootProject)
): Promise<number> {
const linkedDeps = await linkDirectDepsOfProject(rootProject)
const pkgsLinkedToRoot = await readLinkedDeps(rootProject.modulesDir)
await Promise.all(
Object.values(projects).map(async (project) => {
@@ -58,6 +59,7 @@ async function linkDirectDepsAndDedupe (
}
})
)
return linkedDeps
}
function omitDepsFromRoot (deps: LinkedDirectDep[], pkgsLinkedToRoot: string[]) {
@@ -106,7 +108,8 @@ async function resolveLinkTargetOrFile (filePath: string) {
}
}
async function linkDirectDepsOfProject (project: ProjectToLink) {
async function linkDirectDepsOfProject (project: ProjectToLink): Promise<number> {
let linkedDeps = 0
await Promise.all(project.dependencies.map(async (dep) => {
if (dep.isExternalLink) {
await symlinkDirectRootDependency(dep.dir, project.modulesDir, dep.alias, {
@@ -135,5 +138,7 @@ async function linkDirectDepsOfProject (project: ProjectToLink) {
},
prefix: project.dir,
})
linkedDeps++
}))
return linkedDeps
}

View File

@@ -157,7 +157,17 @@ export interface HeadlessOptions {
useLockfile?: boolean
}
export async function headlessInstall (opts: HeadlessOptions) {
export interface InstallationResultStats {
added: number
removed: number
linkedToRoot: number
}
export interface InstallationResult {
stats: InstallationResultStats
}
export async function headlessInstall (opts: HeadlessOptions): Promise<InstallationResult> {
const reporter = opts.reporter
if ((reporter != null) && typeof reporter === 'function') {
streamParser.on('data', reporter)
@@ -215,9 +225,10 @@ export async function headlessInstall (opts: HeadlessOptions) {
}
const skipped = opts.skipped || new Set<string>()
let removed = 0
if (opts.nodeLinker !== 'hoisted') {
if (currentLockfile != null && !opts.ignorePackageManifest) {
await prune(
const removedDepPaths = await prune(
selectedProjects,
{
currentLockfile,
@@ -236,6 +247,7 @@ export async function headlessInstall (opts: HeadlessOptions) {
wantedLockfile,
}
)
removed = removedDepPaths.size
} else {
statsLogger.debug({
prefix: lockfileDir,
@@ -337,8 +349,9 @@ export async function headlessInstall (opts: HeadlessOptions) {
}
const depNodes = Object.values(graph)
const added = depNodes.length
statsLogger.debug({
added: depNodes.length,
added,
prefix: lockfileDir,
})
@@ -350,6 +363,7 @@ export async function headlessInstall (opts: HeadlessOptions) {
}
let newHoistedDependencies!: HoistedDependencies
let linkedToRoot = 0
if (opts.nodeLinker === 'hoisted' && hierarchy && prevGraph) {
await linkHoistedModules(opts.storeController, graph, prevGraph, hierarchy, {
depsStateCache,
@@ -364,7 +378,7 @@ export async function headlessInstall (opts: HeadlessOptions) {
stage: 'importing_done',
})
await symlinkDirectDependencies({
linkedToRoot = await symlinkDirectDependencies({
directDependenciesByImporterId: symlinkedDirectDependenciesByImporterId!,
dedupe: Boolean(opts.dedupeDirectDeps),
filteredLockfile,
@@ -433,7 +447,7 @@ export async function headlessInstall (opts: HeadlessOptions) {
/** Skip linking and due to no project manifest */
if (!opts.ignorePackageManifest) {
await symlinkDirectDependencies({
linkedToRoot = await symlinkDirectDependencies({
dedupe: Boolean(opts.dedupeDirectDeps),
directDependenciesByImporterId,
filteredLockfile,
@@ -602,6 +616,13 @@ export async function headlessInstall (opts: HeadlessOptions) {
if ((reporter != null) && typeof reporter === 'function') {
streamParser.removeListener('data', reporter)
}
return {
stats: {
added,
removed,
linkedToRoot,
},
}
}
type SymlinkDirectDependenciesOpts = Pick<HeadlessOptions, 'registries' | 'symlink' | 'lockfileDir'> & {
@@ -621,7 +642,7 @@ async function symlinkDirectDependencies (
registries,
symlink,
}: SymlinkDirectDependenciesOpts
) {
): Promise<number> {
projects.forEach(({ rootDir, manifest }) => {
// Even though headless installation will never update the package.json
// this needs to be logged because otherwise install summary won't be printed
@@ -630,28 +651,27 @@ async function symlinkDirectDependencies (
updated: manifest,
})
})
if (symlink !== false) {
const importerManifestsByImporterId = {} as { [id: string]: ProjectManifest }
for (const { id, manifest } of projects) {
importerManifestsByImporterId[id] = manifest
}
const projectsToLink = Object.fromEntries(await Promise.all(
projects.map(async ({ rootDir, id, modulesDir }) => ([id, {
dir: rootDir,
modulesDir,
dependencies: await getRootPackagesToLink(filteredLockfile, {
importerId: id,
importerModulesDir: modulesDir,
lockfileDir,
projectDir: rootDir,
importerManifestsByImporterId,
registries,
rootDependencies: directDependenciesByImporterId[id],
}),
}]))
))
await linkDirectDeps(projectsToLink, { dedupe: Boolean(dedupe) })
if (symlink === false) return 0
const importerManifestsByImporterId = {} as { [id: string]: ProjectManifest }
for (const { id, manifest } of projects) {
importerManifestsByImporterId[id] = manifest
}
const projectsToLink = Object.fromEntries(await Promise.all(
projects.map(async ({ rootDir, id, modulesDir }) => ([id, {
dir: rootDir,
modulesDir,
dependencies: await getRootPackagesToLink(filteredLockfile, {
importerId: id,
importerModulesDir: modulesDir,
lockfileDir,
projectDir: rootDir,
importerManifestsByImporterId,
registries,
rootDependencies: directDependenciesByImporterId[id],
}),
}]))
))
return linkDirectDeps(projectsToLink, { dedupe: Boolean(dedupe) })
}
async function linkBinsOfImporter (

View File

@@ -277,7 +277,7 @@ export async function recursive (
throw new PnpmError('NO_PACKAGE_IN_DEPENDENCIES',
'None of the specified packages were found in the dependencies of any of the projects.')
}
const mutatedPkgs = await mutateModules(mutatedImporters, {
const { updatedProjects: mutatedPkgs } = await mutateModules(mutatedImporters, {
...installOpts,
storeController: store.ctrl,
})
@@ -340,14 +340,14 @@ export async function recursive (
break
case 'remove':
action = async (manifest: PackageManifest, opts: any) => { // eslint-disable-line @typescript-eslint/no-explicit-any
const [{ manifest: newManifest }] = await mutateModules([
const mutationResult = await mutateModules([
{
dependencyNames: currentInput,
mutation: 'uninstallSome',
rootDir,
},
], opts)
return newManifest
return mutationResult.updatedProjects[0].manifest
}
break
default: