feat: aggregate-output (#4109)

close #4070

.changeset/hip-gorillas-happen.md

@@ -0,0 +1,10 @@
+---
+"@pnpm/common-cli-options-help": minor
+"@pnpm/config": minor
+"@pnpm/default-reporter": minor
+"pnpm": minor
+---
+
+A new option `--aggregate-output` for `append-only` reporter is added. It aggregates lifecycle logs output for each command that is run in parallel, and only prints command logs when command is finished.
+
+Related discussion: [#4070](https://github.com/pnpm/pnpm/discussions/4070).

packages/common-cli-options-help/src/index.ts

@@ -53,6 +53,10 @@ export const UNIVERSAL_OPTIONS = [
     description: 'Stream output from child processes immediately, prefixed with the originating package directory. This allows output from different packages to be interleaved.',
     name: '--stream',
   },
+  {
+    description: 'Aggregate output from child processes that are run in parallel, and only print output when child process is finished. It makes reading large logs after running `pnpm recursive` with `--parallel` or with `--workspace-concurrency` much easier (especially on CI). Only `--reporter=append-only` is supported.',
+    name: '--aggregate-output',
+  },
   {
     description: 'Divert all output to stderr',
     name: '--use-stderr',

packages/config/src/Config.ts

@@ -115,6 +115,7 @@ export interface Config {
   workspaceConcurrency: number
   workspaceDir?: string
   reporter?: string
+  aggregateOutput: boolean
   linkWorkspacePackages: boolean | 'deep'
   preferWorkspacePackages: boolean
   reverse: boolean

packages/config/src/index.ts

@@ -87,6 +87,7 @@ export const types = Object.assign({
   'publish-branch': String,
   'recursive-install': Boolean,
   reporter: String,
+  'aggregate-output': Boolean,
   'save-peer': Boolean,
   'save-workspace-protocol': Boolean,
   'script-shell': String,

packages/default-reporter/src/index.ts

@@ -20,6 +20,7 @@ export default function (
       appendOnly?: boolean
       logLevel?: LogLevel
       streamLifecycleOutput?: boolean
+      aggregateOutput?: boolean
       throttleProgress?: number
       outputMaxWidth?: number
     }
@@ -80,6 +81,7 @@ export function toOutput$ (
       logLevel?: LogLevel
       outputMaxWidth?: number
       streamLifecycleOutput?: boolean
+      aggregateOutput?: boolean
       throttleProgress?: number
     }
     context: {
@@ -219,6 +221,7 @@ export function toOutput$ (
       logLevel: opts.reportingOptions?.logLevel,
       pnpmConfig: opts.context.config,
       streamLifecycleOutput: opts.reportingOptions?.streamLifecycleOutput,
+      aggregateOutput: opts.reportingOptions?.aggregateOutput,
       throttleProgress: opts.reportingOptions?.throttleProgress,
       width: opts.reportingOptions?.outputMaxWidth,
     }

packages/default-reporter/src/reporterForClient/index.ts

@@ -51,6 +51,7 @@ export default function (
     logLevel?: LogLevel
     pnpmConfig?: Config
     streamLifecycleOutput?: boolean
+    aggregateOutput?: boolean
     throttleProgress?: number
     width?: number
   }
@@ -69,6 +70,7 @@ export default function (
     reportPeerDependencyIssues(log$),
     reportLifecycleScripts(log$, {
       appendOnly: opts.appendOnly === true || opts.streamLifecycleOutput,
+      aggregateOutput: opts.aggregateOutput,
       cwd,
       width,
     }),

packages/default-reporter/src/reporterForClient/reportLifecycleScripts.ts

@@ -1,7 +1,7 @@
 import path from 'path'
 import { LifecycleLog } from '@pnpm/core-loggers'
 import * as Rx from 'rxjs'
-import { map } from 'rxjs/operators'
+import { buffer, filter, groupBy, map, mergeAll, mergeMap } from 'rxjs/operators'
 import chalk from 'chalk'
 import prettyTime from 'pretty-ms'
 import stripAnsi from 'strip-ansi'
@@ -28,6 +28,7 @@ export default (
   },
   opts: {
     appendOnly?: boolean
+    aggregateOutput?: boolean
     cwd: string
     width: number
   }
@@ -35,8 +36,13 @@ export default (
   // When the reporter is not append-only, the length of output is limited
   // in order to reduce flickering
   if (opts.appendOnly) {
+    let lifecycle$ = log$.lifecycle
+    if (opts.aggregateOutput) {
+      lifecycle$ = lifecycle$.pipe(aggregateOutput)
+    }
+
     const streamLifecycleOutput = createStreamLifecycleOutput(opts.cwd)
-    return log$.lifecycle.pipe(
+    return lifecycle$.pipe(
       map((log: LifecycleLog) => Rx.of({
         msg: streamLifecycleOutput(log),
       }))
@@ -264,3 +270,18 @@ function formatLine (maxWidth: number, logObj: LifecycleLog) {
 function cutLine (line: string, maxLength: number) {
   return stripAnsi(line).substr(0, maxLength)
 }
+
+function aggregateOutput (source: Rx.Observable<LifecycleLog>) {
+  return source.pipe(
+    groupBy(data => data.depPath),
+    mergeMap(group => {
+      return group.pipe(
+        buffer(
+          group.pipe(filter(msg => 'exitCode' in msg))
+        )
+      )
+    }),
+    map(ar => Rx.from(ar)),
+    mergeAll()
+  )
+};

packages/default-reporter/test/reportingLifecycleScripts.ts

@@ -2,9 +2,10 @@ import path from 'path'
 import { lifecycleLogger } from '@pnpm/core-loggers'
 import { toOutput$ } from '@pnpm/default-reporter'
 import { createStreamParser } from '@pnpm/logger'
-import { map, skip, take } from 'rxjs/operators'
+import { map, skip, take, toArray } from 'rxjs/operators'
 import chalk from 'chalk'
 import normalizeNewline from 'normalize-newline'
+import { firstValueFrom } from 'rxjs'
 
 const hlValue = chalk.cyanBright
 const hlPkgId = chalk['whiteBright']
@@ -258,6 +259,120 @@ ${chalk.blue('packages/qar')} ${INSTALL}: Done`)
   })
 })
 
+test('groups lifecycle output when append-only and aggregate-output are used', async () => {
+  const output$ = toOutput$({
+    context: { argv: ['install'] },
+    reportingOptions: {
+      appendOnly: true,
+      aggregateOutput: true,
+      outputMaxWidth: 79,
+    },
+    streamParser: createStreamParser(),
+  })
+
+  lifecycleLogger.debug({
+    depPath: 'packages/foo',
+    optional: false,
+    script: 'node foo',
+    stage: 'preinstall',
+    wd: 'packages/foo',
+  })
+  lifecycleLogger.debug({
+    depPath: 'packages/foo',
+    line: 'foo 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30',
+    stage: 'preinstall',
+    stdio: 'stdout',
+    wd: 'packages/foo',
+  })
+  lifecycleLogger.debug({
+    depPath: 'packages/foo',
+    exitCode: 1,
+    optional: true,
+    stage: 'preinstall',
+    wd: 'packages/foo',
+  })
+  lifecycleLogger.debug({
+    depPath: 'packages/foo',
+    optional: false,
+    script: 'node foo',
+    stage: 'postinstall',
+    wd: 'packages/foo',
+  })
+  lifecycleLogger.debug({
+    depPath: 'packages/foo',
+    line: 'foo I',
+    stage: 'postinstall',
+    stdio: 'stdout',
+    wd: 'packages/foo',
+  })
+  lifecycleLogger.debug({
+    depPath: 'packages/bar',
+    optional: false,
+    script: 'node bar',
+    stage: 'postinstall',
+    wd: 'packages/bar',
+  })
+  lifecycleLogger.debug({
+    depPath: 'packages/bar',
+    line: 'bar I',
+    stage: 'postinstall',
+    stdio: 'stdout',
+    wd: 'packages/bar',
+  })
+  lifecycleLogger.debug({
+    depPath: 'packages/foo',
+    line: 'foo II',
+    stage: 'postinstall',
+    stdio: 'stdout',
+    wd: 'packages/foo',
+  })
+  lifecycleLogger.debug({
+    depPath: 'packages/foo',
+    line: 'foo III',
+    stage: 'postinstall',
+    stdio: 'stdout',
+    wd: 'packages/foo',
+  })
+  lifecycleLogger.debug({
+    depPath: 'packages/qar',
+    optional: false,
+    script: 'node qar',
+    stage: 'install',
+    wd: 'packages/qar',
+  })
+  lifecycleLogger.debug({
+    depPath: 'packages/qar',
+    exitCode: 0,
+    optional: false,
+    stage: 'install',
+    wd: 'packages/qar',
+  })
+  lifecycleLogger.debug({
+    depPath: 'packages/bar',
+    exitCode: 0,
+    optional: false,
+    stage: 'postinstall',
+    wd: 'packages/bar',
+  })
+
+  await expect(
+    firstValueFrom(
+      output$.pipe(map<string, string>(normalizeNewline), take(8), toArray())
+    )
+  ).resolves.toEqual([
+    `${chalk.cyan('packages/foo')} ${PREINSTALL}$ node foo`,
+    `${chalk.cyan(
+      'packages/foo'
+    )} ${PREINSTALL}: foo 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30`,
+    `${chalk.cyan('packages/foo')} ${PREINSTALL}: Failed`,
+    `${chalk.magenta('packages/qar')} ${INSTALL}$ node qar`,
+    `${chalk.magenta('packages/qar')} ${INSTALL}: Done`,
+    `${chalk.blue('packages/bar')} ${POSTINSTALL}$ node bar`,
+    `${chalk.blue('packages/bar')} ${POSTINSTALL}: bar I`,
+    `${chalk.blue('packages/bar')} ${POSTINSTALL}: Done`,
+  ])
+})
+
 test('groups lifecycle output when streamLifecycleOutput is used', (done) => {
   const output$ = toOutput$({
     context: { argv: ['install'] },

packages/pnpm/src/cmd/index.ts

@@ -38,6 +38,7 @@ export const GLOBAL_OPTIONS = pick([
   'prefix',
   'reporter',
   'stream',
+  'aggregate-output',
   'test-pattern',
   'changed-files-ignore-pattern',
   'use-stderr',

packages/pnpm/src/reporter/index.ts

@@ -38,6 +38,7 @@ export default (
       },
       reportingOptions: {
         appendOnly: true,
+        aggregateOutput: opts.config.aggregateOutput,
         logLevel: opts.config.loglevel as LogLevel,
         throttleProgress: 1000,
       },