feat: side effects cache key should contain the state of the dep (#4238)

.changeset/late-eels-promise.md

@@ -0,0 +1,11 @@
+---
+"@pnpm/core": minor
+"@pnpm/headless": minor
+"@pnpm/plugin-commands-installation": minor
+"pnpm": minor
+---
+
+Side effects cache is not an experimental feature anymore.
+
+Side effects cache is saved separately for packages with different dependencies. So if `foo` has `bar` in the dependencies, then a separate cache will be created each time `foo` is installed with a different version of `bar` [#4238](https://github.com/pnpm/pnpm/pull/4238).
+

.changeset/plenty-readers-camp.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/calc-dep-state": major
+---
+
+Initial release.

.changeset/sour-chairs-laugh.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/build-modules": major
+---
+
+New required option added: `depsStateCache`.

packages/build-modules/package.json

@@ -33,7 +33,7 @@
     "@pnpm/logger": "^4.0.0"
   },
   "dependencies": {
-    "@pnpm/constants": "workspace:5.0.0",
+    "@pnpm/calc-dep-state": "workspace:0.0.0",
     "@pnpm/core-loggers": "workspace:6.1.3",
     "@pnpm/lifecycle": "workspace:12.1.4",
     "@pnpm/link-bins": "workspace:6.2.9",

packages/build-modules/src/index.ts

@@ -1,5 +1,5 @@
 import path from 'path'
-import { ENGINE_NAME } from '@pnpm/constants'
+import { calcDepState, DepsStateCache } from '@pnpm/calc-dep-state'
 import { skippedOptionalDependencyLogger } from '@pnpm/core-loggers'
 import { runPostinstallHooks } from '@pnpm/lifecycle'
 import linkBins, { linkBinsOfPackages } from '@pnpm/link-bins'
@@ -11,12 +11,15 @@ import runGroups from 'run-groups'
 import graphSequencer from 'graph-sequencer'
 import filter from 'ramda/src/filter'
 
+export { DepsStateCache }
+
 export default async (
   depGraph: DependenciesGraph,
   rootDepPaths: string[],
   opts: {
     childConcurrency?: number
     depsToBuild?: Set<string>
+    depsStateCache: DepsStateCache
     extendNodePath?: boolean
     extraBinPaths?: string[]
     extraEnv?: Record<string, string>
@@ -70,6 +73,7 @@ async function buildDependency (
     extendNodePath?: boolean
     extraBinPaths?: string[]
     extraEnv?: Record<string, string>
+    depsStateCache: DepsStateCache
     lockfileDir: string
     optional: boolean
     rawConfig: object
@@ -103,7 +107,7 @@ async function buildDependency (
     if (hasSideEffects && opts.sideEffectsCacheWrite) {
       try {
         await opts.storeController.upload(depNode.dir, {
-          engine: ENGINE_NAME,
+          engine: calcDepState(depPath, depGraph, opts.depsStateCache),
           filesIndexFile: depNode.filesIndexFile,
         })
       } catch (err: any) { // eslint-disable-line
@@ -167,6 +171,7 @@ function getSubgraphToBuild (
 
 export interface DependenciesGraphNode {
   children: {[alias: string]: string}
+  depPath: string
   dir: string
   fetchingBundledManifest?: () => Promise<PackageManifest>
   filesIndexFile: string

packages/build-modules/tsconfig.json

@@ -10,7 +10,7 @@
   ],
   "references": [
     {
-      "path": "../constants"
+      "path": "../calc-dep-state"
     },
     {
       "path": "../core-loggers"

packages/calc-dep-state/README.md

@@ -0,0 +1,17 @@
+# @pnpm/calc-dep-state
+
+> Calculates the state of a dependency
+
+<!--@shields('npm')-->
+[![npm version](https://img.shields.io/npm/v/@pnpm/calc-dep-state.svg)](https://www.npmjs.com/package/@pnpm/calc-dep-state)
+<!--/@-->
+
+## Installation
+
+```sh
+pnpm add @pnpm/calc-dep-state
+```
+
+## License
+
+MIT

packages/calc-dep-state/jest.config.js

@@ -0,0 +1 @@
+module.exports = require('../../jest.config.js')

packages/calc-dep-state/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "@pnpm/calc-dep-state",
+  "version": "0.0.0",
+  "description": "Calculates the state of a dependency",
+  "main": "lib/index.js",
+  "types": "lib/index.d.ts",
+  "files": [
+    "lib",
+    "!*.map"
+  ],
+  "engines": {
+    "node": ">=12.17"
+  },
+  "scripts": {
+    "lint": "eslint src/**/*.ts test/**/*.ts",
+    "_test": "jest",
+    "test": "pnpm run compile && pnpm run _test",
+    "prepublishOnly": "pnpm run compile",
+    "compile": "rimraf lib tsconfig.tsbuildinfo && tsc --build && pnpm run lint -- --fix"
+  },
+  "repository": "https://github.com/pnpm/pnpm/blob/master/packages/calc-dep-state",
+  "keywords": [
+    "pnpm6",
+    "pnpm"
+  ],
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/pnpm/pnpm/issues"
+  },
+  "homepage": "https://github.com/pnpm/pnpm/blob/master/packages/calc-dep-state#readme",
+  "dependencies": {
+    "@pnpm/constants": "workspace:5.0.0",
+    "sort-keys": "^4.2.0"
+  },
+  "devDependencies": {
+    "@pnpm/calc-dep-state": "workspace:0.0.0"
+  },
+  "funding": "https://opencollective.com/pnpm"
+}

packages/calc-dep-state/src/index.ts

@@ -0,0 +1,52 @@
+import { ENGINE_NAME } from '@pnpm/constants'
+import sortKeys from 'sort-keys'
+
+export interface DepsGraph {
+  [depPath: string]: DepsGraphNode
+}
+
+export interface DepsGraphNode {
+  children: {[alias: string]: string}
+  depPath: string
+}
+
+export interface DepsStateCache {
+  [nodeId: string]: DepStateObj
+}
+
+export interface DepStateObj {
+  [depPath: string]: DepStateObj | {}
+}
+
+export function calcDepState (
+  nodeId: string,
+  depsGraph: DepsGraph,
+  cache: DepsStateCache
+): string {
+  const depStateObj = calcDepStateObj(nodeId, depsGraph, cache, new Set())
+  return `${ENGINE_NAME}-${JSON.stringify(depStateObj)}`
+}
+
+function calcDepStateObj (
+  nodeId: string,
+  depsGraph: DepsGraph,
+  cache: DepsStateCache,
+  parents: Set<string>
+): DepStateObj {
+  if (cache[nodeId]) return cache[nodeId]
+  const node = depsGraph[nodeId]
+  if (!node) return {}
+  const nextParents = new Set([...Array.from(parents), node.depPath])
+  const state: DepStateObj = {}
+  for (const childId of Object.values(node.children)) {
+    const child = depsGraph[childId]
+    if (!child) continue
+    if (parents.has(child.depPath)) {
+      state[child.depPath] = {}
+      continue
+    }
+    state[child.depPath] = calcDepStateObj(childId, depsGraph, cache, nextParents)
+  }
+  cache[nodeId] = sortKeys(state)
+  return cache[nodeId]
+}

packages/calc-dep-state/test/index.ts

@@ -0,0 +1,19 @@
+import { calcDepState } from '@pnpm/calc-dep-state'
+import { ENGINE_NAME } from '@pnpm/constants'
+
+test('calcDepState()', () => {
+  expect(calcDepState('/registry/foo/1.0.0', {
+    'registry/foo/1.0.0': {
+      depPath: '/foo/1.0.0',
+      children: {
+        bar: 'registry/bar/1.0.0',
+      },
+    },
+    'registry/bar/1.0.0': {
+      depPath: '/bar/1.0.0',
+      children: {
+        foo: 'registry/foo/1.0.0',
+      },
+    },
+  }, {})).toBe(`${ENGINE_NAME}-{}`)
+})

packages/calc-dep-state/tsconfig.json

@@ -0,0 +1,16 @@
+{
+  "extends": "@pnpm/tsconfig",
+  "compilerOptions": {
+    "outDir": "lib",
+    "rootDir": "src"
+  },
+  "include": [
+    "src/**/*.ts",
+    "../../typings/**/*.d.ts"
+  ],
+  "references": [
+    {
+      "path": "../constants"
+    }
+  ]
+}

packages/calc-dep-state/tsconfig.lint.json

@@ -0,0 +1,8 @@
+{
+  "extends": "./tsconfig.json",
+  "include": [
+    "src/**/*.ts",
+    "test/**/*.ts",
+    "../../typings/**/*.d.ts"
+  ]
+}

packages/core/package.json

@@ -16,6 +16,7 @@
   },
   "dependencies": {
     "@pnpm/build-modules": "workspace:7.2.5",
+    "@pnpm/calc-dep-state": "workspace:0.0.0",
     "@pnpm/constants": "workspace:5.0.0",
     "@pnpm/core-loggers": "workspace:6.1.3",
     "@pnpm/error": "workspace:2.0.0",

packages/core/src/install/index.ts

@@ -1,6 +1,6 @@
 import crypto from 'crypto'
 import path from 'path'
-import buildModules, { linkBinsOfDependencies } from '@pnpm/build-modules'
+import buildModules, { DepsStateCache, linkBinsOfDependencies } from '@pnpm/build-modules'
 import {
   LAYOUT_VERSION,
   LOCKFILE_VERSION,
@@ -753,6 +753,7 @@ const _installInContext: InstallFunction = async (projects, ctx, opts) => {
     newLockfile.lockfileVersion = LOCKFILE_VERSION
   }
 
+  const depsStateCache: DepsStateCache = {}
   const lockfileOpts = { forceSharedFormat: opts.forceSharedLockfile }
   if (!opts.lockfileOnly && opts.enableModulesDir) {
     const result = await linkPackages(
@@ -761,6 +762,7 @@ const _installInContext: InstallFunction = async (projects, ctx, opts) => {
       {
         currentLockfile: ctx.currentLockfile,
         dependenciesByProjectId,
+        depsStateCache,
         force: opts.force,
         extendNodePath: opts.extendNodePath,
         hoistedDependencies: ctx.hoistedDependencies,
@@ -819,6 +821,7 @@ const _installInContext: InstallFunction = async (projects, ctx, opts) => {
       }
       await buildModules(dependenciesGraph, rootNodes, {
         childConcurrency: opts.childConcurrency,
+        depsStateCache,
         depsToBuild: new Set(result.newDepPaths),
         extendNodePath: opts.extendNodePath,
         extraBinPaths: ctx.extraBinPaths,

packages/core/src/install/link.ts

@@ -1,6 +1,6 @@
 import { promises as fs } from 'fs'
 import path from 'path'
-import { ENGINE_NAME } from '@pnpm/constants'
+import { calcDepState, DepsStateCache } from '@pnpm/calc-dep-state'
 import {
   progressLogger,
   rootLogger,
@@ -30,6 +30,7 @@ import pLimit from 'p-limit'
 import pathExists from 'path-exists'
 import fromPairs from 'ramda/src/fromPairs'
 import equals from 'ramda/src/equals'
+import isEmpty from 'ramda/src/isEmpty'
 import difference from 'ramda/src/difference'
 import omit from 'ramda/src/omit'
 import props from 'ramda/src/props'
@@ -46,6 +47,7 @@ export default async function linkPackages (
       [id: string]: {[alias: string]: string}
     }
     force: boolean
+    depsStateCache: DepsStateCache
     extendNodePath?: boolean
     hoistedDependencies: HoistedDependencies
     hoistedModulesDir: string
@@ -137,6 +139,7 @@ export default async function linkPackages (
     depGraph,
     {
       force: opts.force,
+      depsStateCache: opts.depsStateCache,
       lockfileDir: opts.lockfileDir,
       optional: opts.include.optionalDependencies,
       sideEffectsCacheRead: opts.sideEffectsCacheRead,
@@ -281,6 +284,7 @@ async function linkNewPackages (
   wantedLockfile: Lockfile,
   depGraph: DependenciesGraph,
   opts: {
+    depsStateCache: DepsStateCache
     force: boolean
     optional: boolean
     lockfileDir: string
@@ -342,9 +346,11 @@ async function linkNewPackages (
         optional: opts.optional,
       }),
     linkAllPkgs(opts.storeController, newPkgs, {
+      depGraph,
+      depsStateCache: opts.depsStateCache,
       force: opts.force,
       lockfileDir: opts.lockfileDir,
-      targetEngine: opts.sideEffectsCacheRead && ENGINE_NAME || undefined,
+      sideEffectsCacheRead: opts.sideEffectsCacheRead,
     }),
   ])
 
@@ -388,19 +394,25 @@ async function linkAllPkgs (
   storeController: StoreController,
   depNodes: DependenciesGraphNode[],
   opts: {
+    depGraph: DependenciesGraph
+    depsStateCache: DepsStateCache
     force: boolean
     lockfileDir: string
-    targetEngine?: string
+    sideEffectsCacheRead: boolean
   }
 ) {
   return Promise.all(
     depNodes.map(async (depNode) => {
       const filesResponse = await depNode.fetchingFiles()
 
+      let targetEngine: string | undefined
+      if (opts.sideEffectsCacheRead && filesResponse.sideEffects && !isEmpty(filesResponse.sideEffects)) {
+        targetEngine = calcDepState(depNode.depPath, opts.depGraph, opts.depsStateCache)
+      }
       const { importMethod, isBuilt } = await storeController.importPackage(depNode.dir, {
         filesResponse,
         force: opts.force,
-        targetEngine: opts.targetEngine,
+        targetEngine,
       })
       if (importMethod) {
         progressLogger.debug({

packages/core/test/install/sideEffects.ts

@@ -85,9 +85,10 @@ test('using side effects cache', async () => {
   const filesIndexFile = path.join(opts.storeDir, 'files/2e/28a020ed7c488057d208cd705442e275352fcf88a32b32d0d312668308cb87db3a6df9171ce90d501c3de162b2a6dd5cf62ed7ae8c76532f95adfac924b9a8-index.json')
   const filesIndex = await loadJsonFile<PackageFilesIndex>(filesIndexFile)
   expect(filesIndex.sideEffects).toBeTruthy() // files index has side effects
-  expect(filesIndex.sideEffects).toHaveProperty([ENGINE_NAME, 'generated-by-preinstall.js'])
-  expect(filesIndex.sideEffects).toHaveProperty([ENGINE_NAME, 'generated-by-postinstall.js'])
-  delete filesIndex.sideEffects![ENGINE_NAME]['generated-by-postinstall.js']
+  const sideEffectsKey = `${ENGINE_NAME}-${JSON.stringify({ '/hello-world-js-bin/1.0.0': {} })}`
+  expect(filesIndex.sideEffects).toHaveProperty([sideEffectsKey, 'generated-by-preinstall.js'])
+  expect(filesIndex.sideEffects).toHaveProperty([sideEffectsKey, 'generated-by-postinstall.js'])
+  delete filesIndex.sideEffects![sideEffectsKey]['generated-by-postinstall.js']
   await writeJsonFile(filesIndexFile, filesIndex)
 
   await rimraf('node_modules')

packages/core/tsconfig.json

@@ -27,6 +27,9 @@
     {
       "path": "../cafs"
     },
+    {
+      "path": "../calc-dep-state"
+    },
     {
       "path": "../client"
     },

packages/headless/package.json

@@ -68,6 +68,7 @@
   },
   "dependencies": {
     "@pnpm/build-modules": "workspace:7.2.5",
+    "@pnpm/calc-dep-state": "workspace:0.0.0",
     "@pnpm/constants": "workspace:5.0.0",
     "@pnpm/core-loggers": "workspace:6.1.3",
     "@pnpm/error": "workspace:2.0.0",

packages/headless/src/index.ts

@@ -1,8 +1,8 @@
 import { promises as fs } from 'fs'
 import path from 'path'
 import buildModules from '@pnpm/build-modules'
+import { calcDepState, DepsStateCache } from '@pnpm/calc-dep-state'
 import {
-  ENGINE_NAME,
   LAYOUT_VERSION,
   WANTED_LOCKFILE,
 } from '@pnpm/constants'
@@ -59,6 +59,7 @@ import pLimit from 'p-limit'
 import pathAbsolute from 'path-absolute'
 import equals from 'ramda/src/equals'
 import fromPairs from 'ramda/src/fromPairs'
+import isEmpty from 'ramda/src/isEmpty'
 import omit from 'ramda/src/omit'
 import props from 'ramda/src/props'
 import union from 'ramda/src/union'
@@ -147,6 +148,7 @@ export default async (opts: HeadlessOptions) => {
     throw new Error(`Headless installation requires a ${WANTED_LOCKFILE} file`)
   }
 
+  const depsStateCache: DepsStateCache = {}
   const relativeModulesDir = opts.modulesDir ?? 'node_modules'
   const rootModulesDir = await realpathMissing(path.join(lockfileDir, relativeModulesDir))
   const virtualStoreDir = pathAbsolute(opts.virtualStoreDir ?? path.join(relativeModulesDir, '.pnpm'), lockfileDir)
@@ -285,10 +287,11 @@ export default async (opts: HeadlessOptions) => {
   let newHoistedDependencies!: HoistedDependencies
   if (opts.nodeLinker === 'hoisted' && hierarchy && prevGraph) {
     await linkHoistedModules(opts.storeController, graph, prevGraph, hierarchy, {
+      depsStateCache,
       extendNodePath: opts.extendNodePath,
       force: opts.force,
       lockfileDir: opts.lockfileDir,
-      targetEngine: opts.sideEffectsCacheRead && ENGINE_NAME || undefined,
+      sideEffectsCacheRead: opts.sideEffectsCacheRead,
     })
     stageLogger.debug({
       prefix: lockfileDir,
@@ -314,8 +317,10 @@ export default async (opts: HeadlessOptions) => {
         }),
       linkAllPkgs(opts.storeController, depNodes, {
         force: opts.force,
+        depGraph: graph,
+        depsStateCache,
         lockfileDir: opts.lockfileDir,
-        targetEngine: opts.sideEffectsCacheRead && ENGINE_NAME || undefined,
+        sideEffectsCacheRead: opts.sideEffectsCacheRead,
       }),
     ])
 
@@ -407,6 +412,7 @@ export default async (opts: HeadlessOptions) => {
       extraBinPaths,
       extendNodePath: opts.extendNodePath,
       extraEnv,
+      depsStateCache,
       lockfileDir,
       optional: opts.include.optionalDependencies,
       rawConfig: opts.rawConfig,
@@ -636,9 +642,11 @@ async function linkAllPkgs (
   storeController: StoreController,
   depNodes: DependenciesGraphNode[],
   opts: {
+    depGraph: DependenciesGraph
+    depsStateCache: DepsStateCache
     force: boolean
     lockfileDir: string
-    targetEngine?: string
+    sideEffectsCacheRead: boolean
   }
 ) {
   return Promise.all(
@@ -651,10 +659,14 @@ async function linkAllPkgs (
         throw err
       }
 
+      let targetEngine: string | undefined
+      if (opts.sideEffectsCacheRead && filesResponse.sideEffects && !isEmpty(filesResponse.sideEffects)) {
+        targetEngine = calcDepState(depNode.dir, opts.depGraph, opts.depsStateCache)
+      }
       const { importMethod, isBuilt } = await storeController.importPackage(depNode.dir, {
         filesResponse,
         force: opts.force,
-        targetEngine: opts.targetEngine,
+        targetEngine,
       })
       if (importMethod) {
         progressLogger.debug({

packages/headless/src/linkHoistedModules.ts

@@ -1,4 +1,5 @@
 import path from 'path'
+import { calcDepState, DepsStateCache } from '@pnpm/calc-dep-state'
 import {
   progressLogger,
   removalLogger,
@@ -11,6 +12,7 @@ import {
   StoreController,
 } from '@pnpm/store-controller-types'
 import difference from 'ramda/src/difference'
+import isEmpty from 'ramda/src/isEmpty'
 import rimraf from '@zkochan/rimraf'
 import {
   DepHierarchy,
@@ -23,10 +25,11 @@ export default async function linkHoistedModules (
   prevGraph: DependenciesGraph,
   hierarchy: DepHierarchy,
   opts: {
+    depsStateCache: DepsStateCache
     extendNodePath?: boolean
     force: boolean
     lockfileDir: string
-    targetEngine?: string
+    sideEffectsCacheRead: boolean
   }
 ): Promise<void> {
   // TODO: remove nested node modules first
@@ -78,10 +81,11 @@ async function linkAllPkgsInOrder (
   hierarchy: DepHierarchy,
   parentDir: string,
   opts: {
+    depsStateCache: DepsStateCache
     extendNodePath?: boolean
     force: boolean
     lockfileDir: string
-    targetEngine?: string
+    sideEffectsCacheRead: boolean
     warn: (message: string) => void
   }
 ) {
@@ -96,10 +100,14 @@ async function linkAllPkgsInOrder (
         throw err
       }
 
+      let targetEngine: string | undefined
+      if (opts.sideEffectsCacheRead && filesResponse.sideEffects && !isEmpty(filesResponse.sideEffects)) {
+        targetEngine = calcDepState(dir, graph, opts.depsStateCache)
+      }
       const { importMethod, isBuilt } = await storeController.importPackage(depNode.dir, {
         filesResponse,
         force: opts.force || depNode.depPath !== prevGraph[dir]?.depPath,
-        targetEngine: opts.targetEngine,
+        targetEngine,
       })
       if (importMethod) {
         progressLogger.debug({

packages/headless/src/lockfileToHoistedDepGraph.ts

@@ -1,6 +1,7 @@
 import path from 'path'
 import {
   Lockfile,
+  PackageSnapshot,
   ProjectSnapshot,
 } from '@pnpm/lockfile-file'
 import {
@@ -16,6 +17,7 @@ import {
   StoreController,
 } from '@pnpm/store-controller-types'
 import hoist, { HoisterResult } from '@pnpm/real-hoist'
+import * as dp from 'dependency-path'
 import {
   DependenciesGraph,
   DepHierarchy,
@@ -63,8 +65,14 @@ async function _lockfileToHoistedDepGraph (
   const tree = hoist(lockfile)
   const graph: DependenciesGraph = {}
   const modulesDir = path.join(opts.lockfileDir, 'node_modules')
+  const fetchDepsOpts = {
+    ...opts,
+    lockfile,
+    graph,
+    pkgLocationByDepPath: {},
+  }
   const hierarchy = {
-    [opts.lockfileDir]: await fetchDeps(lockfile, opts, graph, modulesDir, tree.dependencies),
+    [opts.lockfileDir]: await fetchDeps(fetchDepsOpts, modulesDir, tree.dependencies),
   }
   const directDependenciesByImporterId: DirectDependenciesByImporterId = {
     '.': directDepsMap(Object.keys(hierarchy[opts.lockfileDir]), graph),
@@ -76,7 +84,7 @@ async function _lockfileToHoistedDepGraph (
       const importerId = reference.replace('workspace:', '')
       const projectDir = path.join(opts.lockfileDir, importerId)
       const modulesDir = path.join(projectDir, 'node_modules')
-      const nextHierarchy = (await fetchDeps(lockfile, opts, graph, modulesDir, rootDep.dependencies))
+      const nextHierarchy = (await fetchDeps(fetchDepsOpts, modulesDir, rootDep.dependencies))
       hierarchy[projectDir] = nextHierarchy
 
       const importer = lockfile.importers[importerId]
@@ -121,9 +129,11 @@ function pickLinkedDirectDeps (
 }
 
 async function fetchDeps (
-  lockfile: Lockfile,
-  opts: LockfileToHoistedDepGraphOptions,
-  graph: DependenciesGraph,
+  opts: {
+    graph: DependenciesGraph
+    lockfile: Lockfile
+    pkgLocationByDepPath: Record<string, string>
+  } & LockfileToHoistedDepGraphOptions,
   modules: string,
   deps: Set<HoisterResult>
 ): Promise<DepHierarchy> {
@@ -131,7 +141,7 @@ async function fetchDeps (
   await Promise.all(Array.from(deps).map(async (dep) => {
     const depPath = Array.from(dep.references)[0]
     if (opts.skipped.has(depPath) || depPath.startsWith('workspace:')) return
-    const pkgSnapshot = lockfile.packages![depPath]
+    const pkgSnapshot = opts.lockfile.packages![depPath]
     if (!pkgSnapshot) {
       // it is a link
       return
@@ -177,7 +187,7 @@ async function fetchDeps (
       if (pkgSnapshot.optional) return
       throw err
     }
-    graph[dir] = {
+    opts.graph[dir] = {
       alias: dep.name,
       children: {},
       depPath,
@@ -194,7 +204,28 @@ async function fetchDeps (
       prepare: pkgSnapshot.prepare === true,
       requiresBuild: pkgSnapshot.requiresBuild === true,
     }
-    depHierarchy[dir] = await fetchDeps(lockfile, opts, graph, path.join(dir, 'node_modules'), dep.dependencies)
+    opts.pkgLocationByDepPath[depPath] = dir
+    depHierarchy[dir] = await fetchDeps(opts, path.join(dir, 'node_modules'), dep.dependencies)
+    opts.graph[dir].children = getChildren(pkgSnapshot, opts.pkgLocationByDepPath, opts)
   }))
   return depHierarchy
 }
+
+function getChildren (
+  pkgSnapshot: PackageSnapshot,
+  pkgLocationByDepPath: Record<string, string>,
+  opts: { include: IncludedDependencies }
+) {
+  const allDeps = {
+    ...pkgSnapshot.dependencies,
+    ...(opts.include.optionalDependencies ? pkgSnapshot.optionalDependencies : {}),
+  }
+  const children = {}
+  for (const [childName, childRef] of Object.entries(allDeps)) {
+    const childDepPath = dp.refToRelative(childRef, childName)
+    if (childDepPath && pkgLocationByDepPath[childDepPath]) {
+      children[childName] = pkgLocationByDepPath[childDepPath]
+    }
+  }
+  return children
+}

packages/headless/test/index.ts

@@ -671,13 +671,14 @@ test('installing with publicHoistPattern=* in a project with external lockfile',
 
 const ENGINE_DIR = `${process.platform}-${process.arch}-node-${process.version.split('.')[0]}`
 
-test('using side effects cache', async () => {
+test.each([['isolated'], ['hoisted']])('using side effects cache with nodeLinker=%s', async (nodeLinker) => {
   let prefix = f.prepare('side-effects')
 
   // Right now, hardlink does not work with side effects, so we specify copy as the packageImportMethod
   // We disable verifyStoreIntegrity because we are going to change the cache
   const opts = await testDefaults({
     lockfileDir: prefix,
+    nodeLinker,
     sideEffectsCacheRead: true,
     sideEffectsCacheWrite: true,
     verifyStoreIntegrity: false,
@@ -687,15 +688,17 @@ test('using side effects cache', async () => {
   const cacheIntegrityPath = path.join(opts.storeDir, 'files/2e/28a020ed7c488057d208cd705442e275352fcf88a32b32d0d312668308cb87db3a6df9171ce90d501c3de162b2a6dd5cf62ed7ae8c76532f95adfac924b9a8-index.json')
   const cacheIntegrity = await loadJsonFile(cacheIntegrityPath)
   expect(cacheIntegrity!['sideEffects']).toBeTruthy()
-  expect(cacheIntegrity).toHaveProperty(['sideEffects', ENGINE_NAME, 'generated-by-postinstall.js'])
-  delete cacheIntegrity!['sideEffects'][ENGINE_NAME]['generated-by-postinstall.js']
+  const sideEffectsKey = `${ENGINE_NAME}-${JSON.stringify({ '/hello-world-js-bin/1.0.0': {} })}`
+  expect(cacheIntegrity).toHaveProperty(['sideEffects', sideEffectsKey, 'generated-by-postinstall.js'])
+  delete cacheIntegrity!['sideEffects'][sideEffectsKey]['generated-by-postinstall.js']
 
-  expect(cacheIntegrity).toHaveProperty(['sideEffects', ENGINE_NAME, 'generated-by-preinstall.js'])
+  expect(cacheIntegrity).toHaveProperty(['sideEffects', sideEffectsKey, 'generated-by-preinstall.js'])
   await writeJsonFile(cacheIntegrityPath, cacheIntegrity)
 
   prefix = f.prepare('side-effects')
   const opts2 = await testDefaults({
     lockfileDir: prefix,
+    nodeLinker,
     sideEffectsCacheRead: true,
     sideEffectsCacheWrite: true,
     storeDir: opts.storeDir,

packages/headless/tsconfig.json

@@ -21,6 +21,9 @@
     {
       "path": "../build-modules"
     },
+    {
+      "path": "../calc-dep-state"
+    },
     {
       "path": "../client"
     },

packages/plugin-commands-installation/src/install.ts

@@ -211,15 +211,6 @@ by any dependencies, so it is an emulation of a flat node_modules',
             description: 'Force reinstall dependencies: refetch packages modified in store, recreate a lockfile and/or modules directory created by a non-compatible version of pnpm',
             name: '--force',
           },
-          ...UNIVERSAL_OPTIONS,
-        ],
-      },
-      OUTPUT_OPTIONS,
-      FILTERING,
-      {
-        title: 'Experimental options',
-
-        list: [
           {
             description: 'Use or cache the results of (pre/post)install hooks',
             name: '--side-effects-cache',
@@ -228,8 +219,11 @@ by any dependencies, so it is an emulation of a flat node_modules',
             description: 'Only use the side effects cache if present, do not create it for new packages',
             name: '--side-effects-cache-readonly',
           },
+          ...UNIVERSAL_OPTIONS,
         ],
       },
+      OUTPUT_OPTIONS,
+      FILTERING,
     ],
     url: docsUrl('install'),
     usages: ['pnpm install [options]'],

pnpm-lock.yaml

@@ -173,7 +173,7 @@ importers:
   packages/build-modules:
     specifiers:
       '@pnpm/build-modules': workspace:7.2.5
-      '@pnpm/constants': workspace:5.0.0
+      '@pnpm/calc-dep-state': workspace:0.0.0
       '@pnpm/core-loggers': workspace:6.1.3
       '@pnpm/lifecycle': workspace:12.1.4
       '@pnpm/link-bins': workspace:6.2.9
@@ -186,7 +186,7 @@ importers:
       ramda: ^0.27.1
       run-groups: ^3.0.1
     dependencies:
-      '@pnpm/constants': link:../constants
+      '@pnpm/calc-dep-state': link:../calc-dep-state
       '@pnpm/core-loggers': link:../core-loggers
       '@pnpm/lifecycle': link:../lifecycle
       '@pnpm/link-bins': link:../link-bins
@@ -244,6 +244,17 @@ importers:
       '@types/tar-stream': 2.2.2
       tempy: 1.0.1
 
+  packages/calc-dep-state:
+    specifiers:
+      '@pnpm/calc-dep-state': workspace:0.0.0
+      '@pnpm/constants': workspace:5.0.0
+      sort-keys: ^4.2.0
+    dependencies:
+      '@pnpm/constants': link:../constants
+      sort-keys: 4.2.0
+    devDependencies:
+      '@pnpm/calc-dep-state': 'link:'
+
   packages/cli-meta:
     specifiers:
       '@pnpm/cli-meta': workspace:2.0.1
@@ -380,6 +391,7 @@ importers:
       '@pnpm/assert-store': workspace:*
       '@pnpm/build-modules': workspace:7.2.5
       '@pnpm/cafs': workspace:3.0.13
+      '@pnpm/calc-dep-state': workspace:0.0.0
       '@pnpm/client': workspace:6.1.1
       '@pnpm/constants': workspace:5.0.0
       '@pnpm/core': workspace:2.3.0
@@ -457,6 +469,7 @@ importers:
       write-yaml-file: ^4.2.0
     dependencies:
       '@pnpm/build-modules': link:../build-modules
+      '@pnpm/calc-dep-state': link:../calc-dep-state
       '@pnpm/constants': link:../constants
       '@pnpm/core-loggers': link:../core-loggers
       '@pnpm/error': link:../error
@@ -1041,6 +1054,7 @@ importers:
     specifiers:
       '@pnpm/assert-project': workspace:*
       '@pnpm/build-modules': workspace:7.2.5
+      '@pnpm/calc-dep-state': workspace:0.0.0
       '@pnpm/client': workspace:6.1.1
       '@pnpm/constants': workspace:5.0.0
       '@pnpm/core-loggers': workspace:6.1.3
@@ -1089,6 +1103,7 @@ importers:
       write-json-file: ^4.3.0
     dependencies:
       '@pnpm/build-modules': link:../build-modules
+      '@pnpm/calc-dep-state': link:../calc-dep-state
       '@pnpm/constants': link:../constants
       '@pnpm/core-loggers': link:../core-loggers
       '@pnpm/error': link:../error