fix: rebuild should upload side-effects (#6790)

.changeset/healthy-tomatoes-promise.md

@@ -0,0 +1,6 @@
+---
+"@pnpm/plugin-commands-rebuild": patch
+"pnpm": patch
+---
+
+`pnpm rebuild` should upload the built artifacts to the content-addressable store.

.changeset/real-cougars-move.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/calc-dep-state": minor
+---
+
+Add `lockfileToDepGraph` function.

exec/plugin-commands-rebuild/package.json

@@ -48,6 +48,8 @@
     "write-yaml-file": "^5.0.0"
   },
   "dependencies": {
+    "@pnpm/cafs": "workspace:*",
+    "@pnpm/calc-dep-state": "workspace:*",
     "@pnpm/cli-utils": "workspace:*",
     "@pnpm/common-cli-options-help": "workspace:*",
     "@pnpm/config": "workspace:*",
@@ -60,6 +62,7 @@
     "@pnpm/graph-sequencer": "1.1.1",
     "@pnpm/lifecycle": "workspace:*",
     "@pnpm/link-bins": "workspace:*",
+    "@pnpm/lockfile-types": "workspace:*",
     "@pnpm/lockfile-utils": "workspace:*",
     "@pnpm/lockfile-walker": "workspace:*",
     "@pnpm/modules-yaml": "workspace:*",

exec/plugin-commands-rebuild/src/implementation/extendRebuildOptions.ts

@@ -17,6 +17,7 @@ export interface StrictRebuildOptions {
   preferSymlinkedExecutables?: boolean
   scriptShell?: string
   sideEffectsCacheRead: boolean
+  sideEffectsCacheWrite: boolean
   scriptsPrependNodePath: boolean | 'warn-only'
   shellEmulator: boolean
   storeDir: string // TODO: remove this property

exec/plugin-commands-rebuild/src/implementation/index.ts

@@ -1,4 +1,6 @@
 import path from 'path'
+import { getFilePathInCafs } from '@pnpm/cafs'
+import { calcDepState, lockfileToDepGraph, type DepsStateCache } from '@pnpm/calc-dep-state'
 import {
   LAYOUT_VERSION,
   WANTED_LOCKFILE,
@@ -11,6 +13,7 @@ import {
   runPostinstallHooks,
 } from '@pnpm/lifecycle'
 import { linkBins } from '@pnpm/link-bins'
+import { type TarballResolution } from '@pnpm/lockfile-types'
 import {
   type Lockfile,
   nameVerFromPkgSnapshot,
@@ -239,6 +242,9 @@ async function _rebuild (
   } & Pick<PnpmContext, 'modulesFile'>,
   opts: StrictRebuildOptions
 ) {
+  const depGraph = lockfileToDepGraph(ctx.currentLockfile)
+  const depsStateCache: DepsStateCache = {}
+  const cafsDir = path.join(opts.storeDir, 'files')
   const pkgsThatWereRebuilt = new Set()
   const graph = new Map()
   const pkgSnapshots: PackageSnapshots = ctx.currentLockfile.packages ?? {}
@@ -298,7 +304,7 @@ async function _rebuild (
         } else {
           extraBinPaths.push(...binDirsInAllParentDirs(pkgRoot, opts.lockfileDir))
         }
-        await runPostinstallHooks({
+        const hasSideEffects = await runPostinstallHooks({
           depPath,
           extraBinPaths,
           extraEnv: opts.extraEnv,
@@ -310,6 +316,31 @@ async function _rebuild (
           shellEmulator: opts.shellEmulator,
           unsafePerm: opts.unsafePerm || false,
         })
+        if (hasSideEffects && (opts.sideEffectsCacheWrite ?? true) && (pkgSnapshot.resolution as TarballResolution).integrity) {
+          const filesIndexFile = getFilePathInCafs(cafsDir, (pkgSnapshot.resolution as TarballResolution).integrity!.toString(), 'index')
+          try {
+            const sideEffectsCacheKey = calcDepState(depGraph, depsStateCache, depPath, {
+              isBuilt: true,
+            })
+            await opts.storeController.upload(pkgRoot, {
+              sideEffectsCacheKey,
+              filesIndexFile,
+            })
+          } catch (err: any) { // eslint-disable-line
+            if (err.statusCode === 403) {
+              logger.warn({
+                message: `The store server disabled upload requests, could not upload ${pkgRoot}`,
+                prefix: opts.lockfileDir,
+              })
+            } else {
+              logger.warn({
+                error: err,
+                message: `An error occurred while uploading ${pkgRoot}`,
+                prefix: opts.lockfileDir,
+              })
+            }
+          }
+        }
         pkgsThatWereRebuilt.add(depPath)
       } catch (err: any) { // eslint-disable-line
         if (pkgSnapshot.optional) {

exec/plugin-commands-rebuild/test/index.ts

@@ -1,11 +1,13 @@
 /// <reference path="../../../__typings__/index.d.ts" />
 import path from 'path'
-import { WANTED_LOCKFILE } from '@pnpm/constants'
+import { getFilePathInCafs } from '@pnpm/cafs'
+import { ENGINE_NAME, WANTED_LOCKFILE } from '@pnpm/constants'
 import { rebuild } from '@pnpm/plugin-commands-rebuild'
-import { prepare, prepareEmpty } from '@pnpm/prepare'
-import { REGISTRY_MOCK_PORT } from '@pnpm/registry-mock'
+import { prepare } from '@pnpm/prepare'
+import { getIntegrity, REGISTRY_MOCK_PORT } from '@pnpm/registry-mock'
 import { fixtures } from '@pnpm/test-fixtures'
 import execa from 'execa'
+import loadJsonFile from 'load-json-file'
 import exists from 'path-exists'
 import sinon from 'sinon'
 import { DEFAULT_OPTS } from './utils'
@@ -15,7 +17,7 @@ const pnpmBin = path.join(__dirname, '../../../pnpm/bin/pnpm.cjs')
 const f = fixtures(__dirname)
 
 test('rebuilds dependencies', async () => {
-  const project = prepareEmpty()
+  const project = prepare()
   const cacheDir = path.resolve('cache')
   const storeDir = path.resolve('store')
 
@@ -70,10 +72,18 @@ test('rebuilds dependencies', async () => {
       'postinstall',
     ])
   }
+
+  const cafsDir = path.join(storeDir, 'v3/files')
+  const cacheIntegrityPath = getFilePathInCafs(cafsDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), 'index')
+  const cacheIntegrity = await loadJsonFile<any>(cacheIntegrityPath) // eslint-disable-line @typescript-eslint/no-explicit-any
+  expect(cacheIntegrity!.sideEffects).toBeTruthy()
+  const sideEffectsKey = `${ENGINE_NAME}-${JSON.stringify({ '/@pnpm.e2e/hello-world-js-bin/1.0.0': {} })}`
+  expect(cacheIntegrity).toHaveProperty(['sideEffects', sideEffectsKey, 'generated-by-postinstall.js'])
+  delete cacheIntegrity!.sideEffects[sideEffectsKey]['generated-by-postinstall.js']
 })
 
 test('rebuild does not fail when a linked package is present', async () => {
-  const project = prepareEmpty()
+  const project = prepare()
   const cacheDir = path.resolve('cache')
   const storeDir = path.resolve('store')
   f.copy('local-pkg', path.resolve('..', 'local-pkg'))
@@ -103,7 +113,7 @@ test('rebuild does not fail when a linked package is present', async () => {
 })
 
 test('rebuilds specific dependencies', async () => {
-  const project = prepareEmpty()
+  const project = prepare()
   const cacheDir = path.resolve('cache')
   const storeDir = path.resolve('store')
   await execa('node', [
@@ -139,7 +149,7 @@ test('rebuilds specific dependencies', async () => {
 })
 
 test('rebuild with pending option', async () => {
-  const project = prepareEmpty()
+  const project = prepare()
   const cacheDir = path.resolve('cache')
   const storeDir = path.resolve('store')
   await execa('node', [
@@ -204,7 +214,7 @@ test('rebuild with pending option', async () => {
 })
 
 test('rebuild dependencies in correct order', async () => {
-  const project = prepareEmpty()
+  const project = prepare()
   const cacheDir = path.resolve('cache')
   const storeDir = path.resolve('store')
 
@@ -242,7 +252,7 @@ test('rebuild dependencies in correct order', async () => {
 })
 
 test('rebuild links bins', async () => {
-  const project = prepareEmpty()
+  const project = prepare()
   const cacheDir = path.resolve('cache')
   const storeDir = path.resolve('store')
 

exec/plugin-commands-rebuild/tsconfig.json

@@ -33,12 +33,18 @@
     {
       "path": "../../fs/hard-link-dir"
     },
+    {
+      "path": "../../lockfile/lockfile-types"
+    },
     {
       "path": "../../lockfile/lockfile-utils"
     },
     {
       "path": "../../lockfile/lockfile-walker"
     },
+    {
+      "path": "../../packages/calc-dep-state"
+    },
     {
       "path": "../../packages/constants"
     },
@@ -63,6 +69,9 @@
     {
       "path": "../../pkg-manager/modules-yaml"
     },
+    {
+      "path": "../../store/cafs"
+    },
     {
       "path": "../../store/store-connection-manager"
     },

packages/calc-dep-state/package.json

@@ -30,6 +30,8 @@
   "homepage": "https://github.com/pnpm/pnpm/blob/main/packages/calc-dep-state#readme",
   "dependencies": {
     "@pnpm/constants": "workspace:*",
+    "@pnpm/dependency-path": "workspace:*",
+    "@pnpm/lockfile-types": "workspace:*",
     "sort-keys": "^4.2.0"
   },
   "devDependencies": {

packages/calc-dep-state/src/index.ts

@@ -1,4 +1,6 @@
 import { ENGINE_NAME } from '@pnpm/constants'
+import { refToRelative } from '@pnpm/dependency-path'
+import { type Lockfile } from '@pnpm/lockfile-types'
 import sortKeys from 'sort-keys'
 
 export interface DepsGraph {
@@ -61,3 +63,31 @@ function calcDepStateObj (
   cache[depPath] = sortKeys(state)
   return cache[depPath]
 }
+
+export function lockfileToDepGraph (lockfile: Lockfile): DepsGraph {
+  const graph: DepsGraph = {}
+  if (lockfile.packages != null) {
+    Object.entries(lockfile.packages).map(async ([depPath, pkgSnapshot]) => {
+      const children = lockfileDepsToGraphChildren({
+        ...pkgSnapshot.dependencies,
+        ...pkgSnapshot.optionalDependencies,
+      })
+      graph[depPath] = {
+        children,
+        depPath,
+      }
+    })
+  }
+  return graph
+}
+
+function lockfileDepsToGraphChildren (deps: Record<string, string>): Record<string, string> {
+  const children: Record<string, string> = {}
+  for (const [alias, reference] of Object.entries(deps)) {
+    const depPath = refToRelative(reference, alias)
+    if (depPath) {
+      children[alias] = depPath
+    }
+  }
+  return children
+}

packages/calc-dep-state/test/lockfileToDepGraph.test.ts

@@ -0,0 +1,52 @@
+import { lockfileToDepGraph } from '@pnpm/calc-dep-state'
+
+test('lockfileToDepGraph', () => {
+  expect(lockfileToDepGraph({
+    lockfileVersion: '6.0',
+    importers: {},
+    packages: {
+      '/foo/1.0.0': {
+        dependencies: {
+          bar: '1.0.0',
+        },
+        optionalDependencies: {
+          qar: '1.0.0',
+        },
+        resolution: {
+          integrity: '',
+        },
+      },
+      '/bar/1.0.0': {
+        dependencies: {
+          qar: '1.0.0',
+        },
+        resolution: {
+          integrity: '',
+        },
+      },
+      '/qar/1.0.0': {
+        resolution: {
+          integrity: '',
+        },
+      },
+    },
+  })).toStrictEqual({
+    '/bar/1.0.0': {
+      children: {
+        qar: '/qar/1.0.0',
+      },
+      depPath: '/bar/1.0.0',
+    },
+    '/foo/1.0.0': {
+      children: {
+        bar: '/bar/1.0.0',
+        qar: '/qar/1.0.0',
+      },
+      depPath: '/foo/1.0.0',
+    },
+    '/qar/1.0.0': {
+      children: {},
+      depPath: '/qar/1.0.0',
+    },
+  })
+})

packages/calc-dep-state/tsconfig.json

@@ -9,8 +9,14 @@
     "../../__typings__/**/*.d.ts"
   ],
   "references": [
+    {
+      "path": "../../lockfile/lockfile-types"
+    },
     {
       "path": "../constants"
+    },
+    {
+      "path": "../dependency-path"
     }
   ],
   "composite": true

pnpm-lock.yaml

@@ -1149,6 +1149,12 @@ importers:
 
   exec/plugin-commands-rebuild:
     dependencies:
+      '@pnpm/cafs':
+        specifier: workspace:*
+        version: link:../../store/cafs
+      '@pnpm/calc-dep-state':
+        specifier: workspace:*
+        version: link:../../packages/calc-dep-state
       '@pnpm/cli-utils':
         specifier: workspace:*
         version: link:../../cli/cli-utils
@@ -1185,6 +1191,9 @@ importers:
       '@pnpm/link-bins':
         specifier: workspace:*
         version: link:../../pkg-manager/link-bins
+      '@pnpm/lockfile-types':
+        specifier: workspace:*
+        version: link:../../lockfile/lockfile-types
       '@pnpm/lockfile-utils':
         specifier: workspace:*
         version: link:../../lockfile/lockfile-utils
@@ -2381,6 +2390,12 @@ importers:
       '@pnpm/constants':
         specifier: workspace:*
         version: link:../constants
+      '@pnpm/dependency-path':
+        specifier: workspace:*
+        version: link:../dependency-path
+      '@pnpm/lockfile-types':
+        specifier: workspace:*
+        version: link:../../lockfile/lockfile-types
       sort-keys:
         specifier: ^4.2.0
         version: 4.2.0