feat(rebuild): skipIfHasSideEffectsCache (#6796)

.changeset/rotten-deers-shake.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/plugin-commands-rebuild": minor
+---
+
+A new option added skipIfHasSideEffectsCache for skipping the build for dependencies that already have side effects cache.

exec/plugin-commands-rebuild/src/implementation/extendRebuildOptions.ts

@@ -20,6 +20,7 @@ export interface StrictRebuildOptions {
   sideEffectsCacheWrite: boolean
   scriptsPrependNodePath: boolean | 'warn-only'
   shellEmulator: boolean
+  skipIfHasSideEffectsCache?: boolean
   storeDir: string // TODO: remove this property
   storeController: StoreController
   force: boolean

exec/plugin-commands-rebuild/src/implementation/index.ts

@@ -1,5 +1,5 @@
 import path from 'path'
-import { getFilePathInCafs } from '@pnpm/cafs'
+import { getFilePathInCafs, type PackageFilesIndex } from '@pnpm/cafs'
 import { calcDepState, lockfileToDepGraph, type DepsStateCache } from '@pnpm/calc-dep-state'
 import {
   LAYOUT_VERSION,
@@ -27,6 +27,7 @@ import { createOrConnectStoreController } from '@pnpm/store-connection-manager'
 import { type ProjectManifest } from '@pnpm/types'
 import * as dp from '@pnpm/dependency-path'
 import { hardLinkDir } from '@pnpm/fs.hard-link-dir'
+import loadJsonFile from 'load-json-file'
 import runGroups from 'run-groups'
 import graphSequencer from '@pnpm/graph-sequencer'
 import npa from '@pnpm/npm-package-arg'
@@ -304,6 +305,19 @@ async function _rebuild (
         } else {
           extraBinPaths.push(...binDirsInAllParentDirs(pkgRoot, opts.lockfileDir))
         }
+        const resolution = (pkgSnapshot.resolution as TarballResolution)
+        let sideEffectsCacheKey: string | undefined
+        if (opts.skipIfHasSideEffectsCache && resolution.integrity) {
+          const filesIndexFile = getFilePathInCafs(cafsDir, resolution.integrity!.toString(), 'index')
+          const pkgFilesIndex = await loadJsonFile<PackageFilesIndex>(filesIndexFile)
+          sideEffectsCacheKey = calcDepState(depGraph, depsStateCache, depPath, {
+            isBuilt: true,
+          })
+          if (pkgFilesIndex.sideEffects?.[sideEffectsCacheKey]) {
+            pkgsThatWereRebuilt.add(depPath)
+            return
+          }
+        }
         const hasSideEffects = await runPostinstallHooks({
           depPath,
           extraBinPaths,
@@ -316,12 +330,14 @@ async function _rebuild (
           shellEmulator: opts.shellEmulator,
           unsafePerm: opts.unsafePerm || false,
         })
-        if (hasSideEffects && (opts.sideEffectsCacheWrite ?? true) && (pkgSnapshot.resolution as TarballResolution).integrity) {
-          const filesIndexFile = getFilePathInCafs(cafsDir, (pkgSnapshot.resolution as TarballResolution).integrity!.toString(), 'index')
+        if (hasSideEffects && (opts.sideEffectsCacheWrite ?? true) && resolution.integrity) {
+          const filesIndexFile = getFilePathInCafs(cafsDir, resolution.integrity!.toString(), 'index')
           try {
-            const sideEffectsCacheKey = calcDepState(depGraph, depsStateCache, depPath, {
-              isBuilt: true,
-            })
+            if (!sideEffectsCacheKey) {
+              sideEffectsCacheKey = calcDepState(depGraph, depsStateCache, depPath, {
+                isBuilt: true,
+              })
+            }
             await opts.storeController.upload(pkgRoot, {
               sideEffectsCacheKey,
               filesIndexFile,

exec/plugin-commands-rebuild/src/rebuild.ts

@@ -93,6 +93,7 @@ export async function handler (
     recursive?: boolean
     reporter?: (logObj: LogBase) => void
     pending: boolean
+    skipIfHasSideEffectsCache?: boolean
   },
   params: string[]
 ) {

exec/plugin-commands-rebuild/test/index.ts

@@ -1,4 +1,5 @@
 /// <reference path="../../../__typings__/index.d.ts" />
+import fs from 'fs'
 import path from 'path'
 import { getFilePathInCafs } from '@pnpm/cafs'
 import { ENGINE_NAME, WANTED_LOCKFILE } from '@pnpm/constants'
@@ -82,6 +83,56 @@ test('rebuilds dependencies', async () => {
   delete cacheIntegrity!.sideEffects[sideEffectsKey]['generated-by-postinstall.js']
 })
 
+test('skipIfHasSideEffectsCache', async () => {
+  const project = prepare()
+  const cacheDir = path.resolve('cache')
+  const storeDir = path.resolve('store')
+
+  await execa('node', [
+    pnpmBin,
+    'add',
+    '--save-dev',
+    '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0',
+    `--registry=${REGISTRY}`,
+    `--store-dir=${storeDir}`,
+    '--ignore-scripts',
+    `--cache-dir=${cacheDir}`,
+  ])
+
+  const cafsDir = path.join(storeDir, 'v3/files')
+  const cacheIntegrityPath = getFilePathInCafs(cafsDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), 'index')
+  let cacheIntegrity = await loadJsonFile<any>(cacheIntegrityPath) // eslint-disable-line @typescript-eslint/no-explicit-any
+  const sideEffectsKey = `${ENGINE_NAME}-${JSON.stringify({ '/@pnpm.e2e/hello-world-js-bin/1.0.0': {} })}`
+  cacheIntegrity.sideEffects = {
+    [sideEffectsKey]: { foo: 'bar' },
+  }
+  fs.writeFileSync(cacheIntegrityPath, JSON.stringify(cacheIntegrity, null, 2), 'utf8')
+
+  let modules = await project.readModulesManifest()
+  expect(modules!.pendingBuilds).toStrictEqual([
+    '/@pnpm.e2e/pre-and-postinstall-scripts-example/1.0.0',
+  ])
+
+  const modulesManifest = await project.readModulesManifest()
+  await rebuild.handler({
+    ...DEFAULT_OPTS,
+    cacheDir,
+    dir: process.cwd(),
+    pending: true,
+    registries: modulesManifest!.registries!,
+    skipIfHasSideEffectsCache: true,
+    storeDir,
+  }, [])
+
+  modules = await project.readModulesManifest()
+  expect(modules).toBeTruthy()
+  expect(modules!.pendingBuilds.length).toBe(0)
+
+  cacheIntegrity = await loadJsonFile<any>(cacheIntegrityPath) // eslint-disable-line @typescript-eslint/no-explicit-any
+  expect(cacheIntegrity!.sideEffects).toBeTruthy()
+  expect(cacheIntegrity).toHaveProperty(['sideEffects', sideEffectsKey, 'foo'])
+})
+
 test('rebuild does not fail when a linked package is present', async () => {
   const project = prepare()
   const cacheDir = path.resolve('cache')

jest.setup.js

@@ -1 +1 @@
-jest.retryTimes(1);
+// jest.retryTimes(1);

pnpm/test/recursive/misc.ts

@@ -204,7 +204,6 @@ test('recursive installation of packages in workspace ignores hooks in packages'
     },
   ])
 
-  process.chdir('project-1')
   const pnpmfile = `
     module.exports = { hooks: { readPackage } }
     function readPackage (pkg) {
@@ -213,12 +212,8 @@ test('recursive installation of packages in workspace ignores hooks in packages'
       return pkg
     }
   `
-  await fs.writeFile('.pnpmfile.cjs', pnpmfile, 'utf8')
-
-  process.chdir('../project-2')
-  await fs.writeFile('.pnpmfile.cjs', pnpmfile, 'utf8')
-
-  process.chdir('..')
+  await fs.writeFile('project-1/.pnpmfile.cjs', pnpmfile, 'utf8')
+  await fs.writeFile('project-2/.pnpmfile.cjs', pnpmfile, 'utf8')
   await fs.writeFile('.pnpmfile.cjs', `
     module.exports = { hooks: { readPackage } }
     function readPackage (pkg) {
@@ -230,11 +225,12 @@ test('recursive installation of packages in workspace ignores hooks in packages'
 
   await writeYamlFile('pnpm-workspace.yaml', { packages: ['project-1', 'project-2'] })
 
-  await execPnpm(['recursive', 'install'])
+  await execPnpm(['install'])
 
   const lockfile = await readYamlFile<Lockfile>('pnpm-lock.yaml')
-  expect(lockfile.packages).not.toHaveProperty(['/@pnpm.e2e/dep-of-pkg-with-1-dep@100.1.0'])
-  expect(lockfile.packages).toHaveProperty(['/is-number@1.0.0'])
+  const depPaths = Object.keys(lockfile.packages ?? [])
+  expect(depPaths).not.toContain('/@pnpm.e2e/dep-of-pkg-with-1-dep@100.1.0')
+  expect(depPaths).toContain('/is-number@1.0.0')
   /* eslint-enable @typescript-eslint/no-unnecessary-type-assertion */
 })