fix: reduce the side-effects cache key length (#7563)

close #5056

.changeset/late-points-lick.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/crypto.object-hasher": major
+---
+
+Initial release.

.changeset/rude-terms-begin.md

@@ -0,0 +1,6 @@
+---
+"@pnpm/calc-dep-state": major
+"pnpm": patch
+---
+
+Reduce the length of the side-effects cache key. Instead of saving a stringified object composed from the dependency versions of the package, use the hash calculated from the said object [#7563](https://github.com/pnpm/pnpm/pull/7563).

crypto/object-hasher/README.md

@@ -0,0 +1,13 @@
+# @pnpm/crypto.object-hasher
+
+> Generate hashes from objects
+
+## Installation
+
+```sh
+pnpm add @pnpm/crypto.object-hasher
+```
+
+## License
+
+MIT

crypto/object-hasher/jest.config.js

@@ -0,0 +1,3 @@
+const config = require('../../jest.config.js');
+
+module.exports = Object.assign({}, config, {});

crypto/object-hasher/package.json

@@ -0,0 +1,42 @@
+{
+  "name": "@pnpm/crypto.object-hasher",
+  "version": "0.0.0",
+  "description": "Generate hashes from objects",
+  "main": "lib/index.js",
+  "types": "lib/index.d.ts",
+  "files": [
+    "lib",
+    "!*.map"
+  ],
+  "scripts": {
+    "lint": "eslint \"src/**/*.ts\" \"test/**/*.ts\"",
+    "_test": "jest",
+    "test": "pnpm run compile && pnpm run _test",
+    "prepublishOnly": "pnpm run compile",
+    "compile": "tsc --build && pnpm run lint --fix"
+  },
+  "repository": "https://github.com/pnpm/pnpm/blob/main/crypto/object-hasher",
+  "keywords": [
+    "pnpm8",
+    "hash",
+    "crypto"
+  ],
+  "engines": {
+    "node": ">=16.14"
+  },
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/pnpm/pnpm/issues"
+  },
+  "homepage": "https://github.com/pnpm/pnpm/blob/main/crypto/object-hasher#readme",
+  "dependencies": {
+    "node-object-hash": "3.0.0"
+  },
+  "devDependencies": {
+    "@pnpm/crypto.object-hasher": "workspace:*"
+  },
+  "funding": "https://opencollective.com/pnpm",
+  "exports": {
+    ".": "./lib/index.js"
+  }
+}

crypto/object-hasher/src/index.ts

@@ -0,0 +1,4 @@
+import { hasher } from 'node-object-hash'
+
+export const hashObjectWithoutSorting = hasher({ sort: false }).hash
+export const hashObject = hasher({}).hash

crypto/object-hasher/test/index.ts

@@ -0,0 +1,13 @@
+import { hashObject, hashObjectWithoutSorting } from '@pnpm/crypto.object-hasher'
+
+describe('hashObject', () => {
+  it('creates a hash', () => {
+    expect(hashObject({ b: 1, a: 2 })).toEqual('c8c943f9321eb7f98834b58391eee848d458c7b35211fc4911cdb1bbd877b74a')
+  })
+})
+
+describe('hashObjectWithoutSorting', () => {
+  it('creates a hash', () => {
+    expect(hashObjectWithoutSorting({ b: 1, a: 2 })).toEqual('c0a68b14aa1886a799c2e7c1289b65ccb79a668881d5b6956f0b185a9ac112d7')
+  })
+})

crypto/object-hasher/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "extends": "@pnpm/tsconfig",
+  "compilerOptions": {
+    "outDir": "lib",
+    "rootDir": "src"
+  },
+  "include": [
+    "src/**/*.ts",
+    "../../__typings__/**/*.d.ts"
+  ],
+  "references": [],
+  "composite": true
+}

crypto/object-hasher/tsconfig.lint.json

@@ -0,0 +1,8 @@
+{
+  "extends": "./tsconfig.json",
+  "include": [
+    "src/**/*.ts",
+    "test/**/*.ts",
+    "../../__typings__/**/*.d.ts"
+  ]
+}

exec/plugin-commands-rebuild/package.json

@@ -31,6 +31,7 @@
   "homepage": "https://github.com/pnpm/pnpm/blob/main/exec/plugin-commands-rebuild#readme",
   "devDependencies": {
     "@pnpm/assert-project": "workspace:*",
+    "@pnpm/crypto.object-hasher": "workspace:*",
     "@pnpm/filter-workspace-packages": "workspace:*",
     "@pnpm/plugin-commands-rebuild": "workspace:*",
     "@pnpm/prepare": "workspace:*",

exec/plugin-commands-rebuild/test/index.ts

@@ -3,6 +3,7 @@ import fs from 'fs'
 import path from 'path'
 import { getFilePathInCafs } from '@pnpm/store.cafs'
 import { ENGINE_NAME, WANTED_LOCKFILE } from '@pnpm/constants'
+import { hashObject } from '@pnpm/crypto.object-hasher'
 import { rebuild } from '@pnpm/plugin-commands-rebuild'
 import { prepare } from '@pnpm/prepare'
 import { getIntegrity, REGISTRY_MOCK_PORT } from '@pnpm/registry-mock'
@@ -78,7 +79,7 @@ test('rebuilds dependencies', async () => {
   const cacheIntegrityPath = getFilePathInCafs(cafsDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), 'index')
   const cacheIntegrity = await loadJsonFile<any>(cacheIntegrityPath) // eslint-disable-line @typescript-eslint/no-explicit-any
   expect(cacheIntegrity!.sideEffects).toBeTruthy()
-  const sideEffectsKey = `${ENGINE_NAME}-${JSON.stringify({ '/@pnpm.e2e/hello-world-js-bin/1.0.0': {} })}`
+  const sideEffectsKey = `${ENGINE_NAME}-${hashObject({ '/@pnpm.e2e/hello-world-js-bin/1.0.0': {} })}`
   expect(cacheIntegrity).toHaveProperty(['sideEffects', sideEffectsKey, 'generated-by-postinstall.js'])
   delete cacheIntegrity!.sideEffects[sideEffectsKey]['generated-by-postinstall.js']
 })
@@ -102,7 +103,7 @@ test('skipIfHasSideEffectsCache', async () => {
   const cafsDir = path.join(storeDir, 'v3/files')
   const cacheIntegrityPath = getFilePathInCafs(cafsDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), 'index')
   let cacheIntegrity = await loadJsonFile<any>(cacheIntegrityPath) // eslint-disable-line @typescript-eslint/no-explicit-any
-  const sideEffectsKey = `${ENGINE_NAME}-${JSON.stringify({ '/@pnpm.e2e/hello-world-js-bin/1.0.0': {} })}`
+  const sideEffectsKey = `${ENGINE_NAME}-${hashObject({ '/@pnpm.e2e/hello-world-js-bin@1.0.0': {} })}`
   cacheIntegrity.sideEffects = {
     [sideEffectsKey]: { foo: 'bar' },
   }
@@ -415,4 +416,4 @@ test('never build neverBuiltDependencies', async () => {
       'node_modules/@pnpm.e2e/pre-and-postinstall-scripts-example/generated-by-preinstall.js'
     )
   ).toBeTruthy()
-})
+})

exec/plugin-commands-rebuild/tsconfig.json

@@ -33,6 +33,9 @@
     {
       "path": "../../config/normalize-registries"
     },
+    {
+      "path": "../../crypto/object-hasher"
+    },
     {
       "path": "../../deps/graph-sequencer"
     },

packages/calc-dep-state/package.json

@@ -30,6 +30,7 @@
   "homepage": "https://github.com/pnpm/pnpm/blob/main/packages/calc-dep-state#readme",
   "dependencies": {
     "@pnpm/constants": "workspace:*",
+    "@pnpm/crypto.object-hasher": "workspace:*",
     "@pnpm/dependency-path": "workspace:*",
     "@pnpm/lockfile-types": "workspace:*",
     "sort-keys": "^4.2.0"

packages/calc-dep-state/src/index.ts

@@ -1,6 +1,7 @@
 import { ENGINE_NAME } from '@pnpm/constants'
 import { refToRelative } from '@pnpm/dependency-path'
 import { type Lockfile } from '@pnpm/lockfile-types'
+import { hashObjectWithoutSorting } from '@pnpm/crypto.object-hasher'
 import sortKeys from 'sort-keys'
 
 export interface DepsGraph {
@@ -32,7 +33,7 @@ export function calcDepState (
   let result = ENGINE_NAME
   if (opts.isBuilt) {
     const depStateObj = calcDepStateObj(depPath, depsGraph, cache, new Set())
-    result += `-${JSON.stringify(depStateObj)}`
+    result += `-${hashObjectWithoutSorting(depStateObj)}`
   }
   if (opts.patchFileHash) {
     result += `-${opts.patchFileHash}`

packages/calc-dep-state/test/index.ts

@@ -1,5 +1,6 @@
 import { calcDepState } from '@pnpm/calc-dep-state'
 import { ENGINE_NAME } from '@pnpm/constants'
+import { hashObject } from '@pnpm/crypto.object-hasher'
 
 const depsGraph = {
   'registry/foo/1.0.0': {
@@ -19,7 +20,7 @@ const depsGraph = {
 test('calcDepState()', () => {
   expect(calcDepState(depsGraph, {}, '/registry/foo/1.0.0', {
     isBuilt: true,
-  })).toBe(`${ENGINE_NAME}-{}`)
+  })).toBe(`${ENGINE_NAME}-${hashObject({})}`)
 })
 
 test('calcDepState() when scripts are ignored', () => {

packages/calc-dep-state/tsconfig.json

@@ -9,6 +9,9 @@
     "../../__typings__/**/*.d.ts"
   ],
   "references": [
+    {
+      "path": "../../crypto/object-hasher"
+    },
     {
       "path": "../../lockfile/lockfile-types"
     },

pkg-manager/core/package.json

@@ -77,6 +77,7 @@
     "@pnpm/assert-store": "workspace:*",
     "@pnpm/client": "workspace:*",
     "@pnpm/core": "workspace:*",
+    "@pnpm/crypto.object-hasher": "workspace:*",
     "@pnpm/git-utils": "workspace:*",
     "@pnpm/lockfile-types": "workspace:*",
     "@pnpm/package-store": "workspace:*",

pkg-manager/core/test/install/sideEffects.ts

@@ -1,6 +1,7 @@
 import { promises as fs, existsSync, readFileSync } from 'fs'
 import path from 'path'
 import { addDependenciesToPackage, install } from '@pnpm/core'
+import { hashObject } from '@pnpm/crypto.object-hasher'
 import { getFilePathInCafs, getFilePathByModeInCafs, type PackageFilesIndex } from '@pnpm/store.cafs'
 import { getIntegrity, REGISTRY_MOCK_PORT } from '@pnpm/registry-mock'
 import { prepareEmpty } from '@pnpm/prepare'
@@ -86,7 +87,7 @@ test('using side effects cache', async () => {
   const filesIndexFile = getFilePathInCafs(cafsDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), 'index')
   const filesIndex = await loadJsonFile<PackageFilesIndex>(filesIndexFile)
   expect(filesIndex.sideEffects).toBeTruthy() // files index has side effects
-  const sideEffectsKey = `${ENGINE_NAME}-${JSON.stringify({ '/@pnpm.e2e/hello-world-js-bin/1.0.0': {} })}`
+  const sideEffectsKey = `${ENGINE_NAME}-${hashObject({ '/@pnpm.e2e/hello-world-js-bin/1.0.0': {} })}`
   expect(filesIndex.sideEffects).toHaveProperty([sideEffectsKey, 'generated-by-preinstall.js'])
   expect(filesIndex.sideEffects).toHaveProperty([sideEffectsKey, 'generated-by-postinstall.js'])
   delete filesIndex.sideEffects![sideEffectsKey]['generated-by-postinstall.js']
@@ -177,7 +178,7 @@ test('a postinstall script does not modify the original sources added to the sto
   const cafsDir = path.join(opts.storeDir, 'files')
   const filesIndexFile = getFilePathInCafs(cafsDir, getIntegrity('@pnpm/postinstall-modifies-source', '1.0.0'), 'index')
   const filesIndex = await loadJsonFile<PackageFilesIndex>(filesIndexFile)
-  const patchedFileIntegrity = filesIndex.sideEffects?.[`${ENGINE_NAME}-{}`]['empty-file.txt']?.integrity
+  const patchedFileIntegrity = filesIndex.sideEffects?.[`${ENGINE_NAME}-${hashObject({})}`]['empty-file.txt']?.integrity
   expect(patchedFileIntegrity).toBeTruthy()
   const originalFileIntegrity = filesIndex.files['empty-file.txt'].integrity
   expect(originalFileIntegrity).toBeTruthy()
@@ -201,7 +202,7 @@ test('a corrupted side-effects cache is ignored', async () => {
   const filesIndexFile = getFilePathInCafs(cafsDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), 'index')
   const filesIndex = await loadJsonFile<PackageFilesIndex>(filesIndexFile)
   expect(filesIndex.sideEffects).toBeTruthy() // files index has side effects
-  const sideEffectsKey = `${ENGINE_NAME}-${JSON.stringify({ '/@pnpm.e2e/hello-world-js-bin/1.0.0': {} })}`
+  const sideEffectsKey = `${ENGINE_NAME}-${hashObject({ '/@pnpm.e2e/hello-world-js-bin/1.0.0': {} })}`
   expect(filesIndex.sideEffects).toHaveProperty([sideEffectsKey, 'generated-by-preinstall.js'])
   const sideEffectFileStat = filesIndex.sideEffects![sideEffectsKey]['generated-by-preinstall.js']
   const sideEffectFile = getFilePathByModeInCafs(cafsDir, sideEffectFileStat.integrity, sideEffectFileStat.mode)

pkg-manager/core/tsconfig.json

@@ -30,6 +30,9 @@
     {
       "path": "../../config/normalize-registries"
     },
+    {
+      "path": "../../crypto/object-hasher"
+    },
     {
       "path": "../../deps/graph-sequencer"
     },

pkg-manager/headless/package.json

@@ -18,6 +18,7 @@
   "devDependencies": {
     "@pnpm/assert-project": "workspace:*",
     "@pnpm/client": "workspace:*",
+    "@pnpm/crypto.object-hasher": "workspace:*",
     "@pnpm/headless": "workspace:*",
     "@pnpm/package-store": "workspace:*",
     "@pnpm/prepare": "workspace:*",

pkg-manager/headless/test/index.ts

@@ -2,6 +2,7 @@
 import { promises as fs, existsSync, realpathSync, writeFileSync } from 'fs'
 import path from 'path'
 import { assertProject } from '@pnpm/assert-project'
+import { hashObject } from '@pnpm/crypto.object-hasher'
 import { getFilePathInCafs } from '@pnpm/store.cafs'
 import { ENGINE_NAME, WANTED_LOCKFILE } from '@pnpm/constants'
 import {
@@ -685,7 +686,7 @@ test.each([['isolated'], ['hoisted']])('using side effects cache with nodeLinker
   const cacheIntegrityPath = getFilePathInCafs(cafsDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), 'index')
   const cacheIntegrity = await loadJsonFile<any>(cacheIntegrityPath) // eslint-disable-line @typescript-eslint/no-explicit-any
   expect(cacheIntegrity!.sideEffects).toBeTruthy()
-  const sideEffectsKey = `${ENGINE_NAME}-${JSON.stringify({ '/@pnpm.e2e/hello-world-js-bin/1.0.0': {} })}`
+  const sideEffectsKey = `${ENGINE_NAME}-${hashObject({ '/@pnpm.e2e/hello-world-js-bin/1.0.0': {} })}`
   expect(cacheIntegrity).toHaveProperty(['sideEffects', sideEffectsKey, 'generated-by-postinstall.js'])
   delete cacheIntegrity!.sideEffects[sideEffectsKey]['generated-by-postinstall.js']
 

pkg-manager/headless/tsconfig.json

@@ -24,6 +24,9 @@
     {
       "path": "../../config/package-is-installable"
     },
+    {
+      "path": "../../crypto/object-hasher"
+    },
     {
       "path": "../../deps/graph-builder"
     },

pnpm-lock.yaml

@@ -809,6 +809,16 @@ importers:
         specifier: 1.3.31
         version: 1.3.31
 
+  crypto/object-hasher:
+    dependencies:
+      node-object-hash:
+        specifier: 3.0.0
+        version: 3.0.0
+    devDependencies:
+      '@pnpm/crypto.object-hasher':
+        specifier: workspace:*
+        version: 'link:'
+
   dedupe/check:
     dependencies:
       '@pnpm/dedupe.types':
@@ -1297,6 +1307,9 @@ importers:
       '@pnpm/assert-project':
         specifier: workspace:*
         version: link:../../__utils__/assert-project
+      '@pnpm/crypto.object-hasher':
+        specifier: workspace:*
+        version: link:../../crypto/object-hasher
       '@pnpm/filter-workspace-packages':
         specifier: workspace:*
         version: link:../../workspace/filter-workspace-packages
@@ -2487,6 +2500,9 @@ importers:
       '@pnpm/constants':
         specifier: workspace:*
         version: link:../constants
+      '@pnpm/crypto.object-hasher':
+        specifier: workspace:*
+        version: link:../../crypto/object-hasher
       '@pnpm/dependency-path':
         specifier: workspace:*
         version: link:../dependency-path
@@ -3113,6 +3129,9 @@ importers:
       '@pnpm/core':
         specifier: workspace:*
         version: 'link:'
+      '@pnpm/crypto.object-hasher':
+        specifier: workspace:*
+        version: link:../../crypto/object-hasher
       '@pnpm/git-utils':
         specifier: workspace:*
         version: link:../../packages/git-utils
@@ -3392,6 +3411,9 @@ importers:
       '@pnpm/client':
         specifier: workspace:*
         version: link:../client
+      '@pnpm/crypto.object-hasher':
+        specifier: workspace:*
+        version: link:../../crypto/object-hasher
       '@pnpm/headless':
         specifier: workspace:*
         version: 'link:'
@@ -9525,7 +9547,7 @@ packages:
     resolution: {integrity: sha512-YmG+oTBCyrAoMIx5g2I9CfyurSpHyoan+9SCj7laaFKseOe3lFEyIVKvwRBQMmSt8uzh+eY5RWeQnoyyOs6AbA==}
     engines: {node: '>=14.15.0'}
     peerDependencies:
-      '@yarnpkg/fslib': ^3.0.0-rc.25
+      '@yarnpkg/fslib': 3.0.0-rc.25
     dependencies:
       '@types/emscripten': 1.39.10
       '@yarnpkg/fslib': 3.0.0-rc.25
@@ -14750,6 +14772,11 @@ packages:
     resolution: {integrity: sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==}
     dev: true
 
+  /node-object-hash@3.0.0:
+    resolution: {integrity: sha512-jLF6tlyletktvSAawuPmH1SReP0YfZQ+tBrDiTCK+Ai7eXPMS9odi5xW/iKC7ZhrWJJ0Z5xYcW/x+1fVMn1Qvw==}
+    engines: {node: '>=16', pnpm: '>=8'}
+    dev: false
+
   /node-releases@2.0.14:
     resolution: {integrity: sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw==}
     dev: true
@@ -17944,16 +17971,5 @@ packages:
     dev: false
 
 time:
-  /@pnpm/node-fetch@1.0.0: '2023-04-19T11:13:43.487Z'
-  /@pnpm/ramda@0.28.1: '2022-08-03T13:56:59.597Z'
-  /@pnpm/which@3.0.1: '2023-05-14T22:08:27.551Z'
-  /@reflink/reflink@0.1.16: '2024-01-02T17:41:22.200Z'
-  /@types/byline@4.2.36: '2023-11-07T00:13:37.410Z'
-  /@types/semver@7.5.3: '2023-09-25T14:19:37.089Z'
-  /@types/table@6.0.0: '2020-09-17T17:56:44.787Z'
-  /@zkochan/hosted-git-info@4.0.2: '2021-09-05T21:33:51.709Z'
-  /@zkochan/js-yaml@0.0.6: '2022-05-10T14:42:39.813Z'
   /fuse-native@2.2.6: '2020-06-03T19:26:36.838Z'
   /node-gyp@9.4.1: '2023-10-27T17:30:56.146Z'
-  /safe-execa@0.1.2: '2022-07-18T01:09:17.517Z'
-  /semver@7.5.4: '2023-07-07T21:10:32.589Z'

pnpm-workspace.yaml

@@ -5,6 +5,7 @@ packages:
   - "!__utils__/build-artifacts"
   - cli/*
   - config/*
+  - crypto/*
   - dedupe/*
   - deps/*
   - env/*