feat!: use SHA256 for hashing side effects cache keys in index files (#8533)

.changeset/funny-wolves-enjoy.md

@@ -0,0 +1,6 @@
+---
+"pnpm": major
+"@pnpm/core": major
+---
+
+Changed the hash stored in the `packageExtensionsChecksum` field of `pnpm-lock.yaml` to SHA256.

.changeset/happy-cheetahs-grab.md

@@ -0,0 +1,5 @@
+---
+"pnpm": major
+---
+
+Use an SHA256 hash for the side effects cache keys.

.changeset/soft-wasps-drum.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/npm-resolver": major
+---
+
+Use SHA256 to encode the package name of a package that has upper case letters in its name.

.changeset/strange-students-whisper.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/crypto.object-hasher": major
+---
+
+Use SHA256 encoded in base64 to hash objects.

crypto/object-hasher/src/index.ts

@@ -3,19 +3,20 @@
 // avoiding "Invalid string length" errors.
 import hash from 'object-hash'
 
-const defaultOptions: hash.BaseOptions = {
+const defaultOptions: hash.NormalOption = {
   respectType: false,
-  algorithm: 'sha1',
+  algorithm: 'sha256',
+  encoding: 'base64',
 }
 
-const withoutSortingOptions: hash.BaseOptions = {
+const withoutSortingOptions: hash.NormalOption = {
   ...defaultOptions,
   unorderedArrays: false,
   unorderedObjects: false,
   unorderedSets: false,
 }
 
-const withSortingOptions: hash.BaseOptions = {
+const withSortingOptions: hash.NormalOption = {
   ...defaultOptions,
   unorderedArrays: true,
   unorderedObjects: true,
@@ -24,8 +25,8 @@ const withSortingOptions: hash.BaseOptions = {
 
 function hashUnknown (object: unknown, options: hash.BaseOptions): string {
   if (object === undefined) {
-    // '0'.repeat(40) to match the length of other returned sha1 hashes.
-    return '0000000000000000000000000000000000000000'
+    // '0'.repeat(44) to match the length of other returned sha1 hashes.
+    return '00000000000000000000000000000000000000000000'
   }
   return hash(object, options)
 }

crypto/object-hasher/test/index.ts

@@ -3,8 +3,8 @@ import { hashObject, hashObjectWithoutSorting } from '@pnpm/crypto.object-hasher
 describe('hashObject', () => {
   const hash = hashObject
   it('creates a hash', () => {
-    expect(hash({ b: 1, a: 2 })).toEqual('e3d3f89836fac144779e57d0e831efd06336036b')
-    expect(hash(undefined)).toEqual('0000000000000000000000000000000000000000')
+    expect(hash({ b: 1, a: 2 })).toEqual('48AVoXIXcTKcnHt8qVKp5vNw4gyOB5VfztHwtYBRcAQ=')
+    expect(hash(undefined)).toEqual('00000000000000000000000000000000000000000000')
   })
   it('sorts', () => {
     expect(hash({ b: 1, a: 2 })).toEqual(hash({ a: 2, b: 1 }))
@@ -15,8 +15,8 @@ describe('hashObject', () => {
 describe('hashObjectWithoutSorting', () => {
   const hash = hashObjectWithoutSorting
   it('creates a hash', () => {
-    expect(hash({ b: 1, a: 2 })).toEqual('dd34c1644a1d52da41808e5c1e6849829ef77999')
-    expect(hash(undefined)).toEqual('0000000000000000000000000000000000000000')
+    expect(hash({ b: 1, a: 2 })).toEqual('mh+rYklpd1DBj/dg6dnG+yd8BQhU2UiUoRMSXjPV1JA=')
+    expect(hash(undefined)).toEqual('00000000000000000000000000000000000000000000')
   })
   it('does not sort', () => {
     expect(hash({ b: 1, a: 2 })).not.toEqual(hash({ a: 2, b: 1 }))

pkg-manager/core/package.json

@@ -24,6 +24,7 @@
     "@pnpm/constants": "workspace:*",
     "@pnpm/core-loggers": "workspace:*",
     "@pnpm/crypto.hash": "workspace:*",
+    "@pnpm/crypto.object-hasher": "workspace:*",
     "@pnpm/dependency-path": "workspace:*",
     "@pnpm/deps.graph-sequencer": "workspace:*",
     "@pnpm/error": "workspace:*",
@@ -72,15 +73,13 @@
     "path-exists": "catalog:",
     "ramda": "catalog:",
     "run-groups": "catalog:",
-    "semver": "catalog:",
-    "sort-keys": "catalog:"
+    "semver": "catalog:"
   },
   "devDependencies": {
     "@pnpm/assert-project": "workspace:*",
     "@pnpm/assert-store": "workspace:*",
     "@pnpm/client": "workspace:*",
     "@pnpm/core": "workspace:*",
-    "@pnpm/crypto.object-hasher": "workspace:*",
     "@pnpm/git-utils": "workspace:*",
     "@pnpm/lockfile.types": "workspace:*",
     "@pnpm/logger": "workspace:*",

pkg-manager/core/src/install/index.test.ts

@@ -1,60 +0,0 @@
-import { createObjectChecksum } from './index'
-
-function assets () {
-  const sorted = {
-    abc: {
-      a: 0,
-      b: [0, 1, 2],
-      c: null,
-    },
-    def: {
-      foo: 'bar',
-      hello: 'world',
-    },
-  } as const
-
-  const unsorted1 = {
-    abc: {
-      b: [0, 1, 2],
-      a: 0,
-      c: null,
-    },
-    def: {
-      hello: 'world',
-      foo: 'bar',
-    },
-  } as const
-
-  const unsorted2 = {
-    def: {
-      foo: 'bar',
-      hello: 'world',
-    },
-    abc: {
-      a: 0,
-      b: [0, 1, 2],
-      c: null,
-    },
-  } as const
-
-  const unsorted3 = {
-    def: {
-      hello: 'world',
-      foo: 'bar',
-    },
-    abc: {
-      b: [0, 1, 2],
-      a: 0,
-      c: null,
-    },
-  } as const
-
-  return { sorted, unsorted1, unsorted2, unsorted3 } as const
-}
-
-test('createObjectChecksum', () => {
-  const { sorted, unsorted1, unsorted2, unsorted3 } = assets()
-  expect(createObjectChecksum(unsorted1)).toBe(createObjectChecksum(sorted))
-  expect(createObjectChecksum(unsorted2)).toBe(createObjectChecksum(sorted))
-  expect(createObjectChecksum(unsorted3)).toBe(createObjectChecksum(sorted))
-})

pkg-manager/core/src/install/index.ts

@@ -1,4 +1,3 @@
-import crypto from 'crypto'
 import path from 'path'
 import { buildModules, type DepsStateCache, linkBinsOfDependencies } from '@pnpm/build-modules'
 import { createAllowBuildFunction } from '@pnpm/builder.policy'
@@ -15,6 +14,7 @@ import {
   summaryLogger,
 } from '@pnpm/core-loggers'
 import { createHexHashFromFile } from '@pnpm/crypto.hash'
+import { hashObject } from '@pnpm/crypto.object-hasher'
 import { PnpmError } from '@pnpm/error'
 import { getContext, type PnpmContext } from '@pnpm/get-context'
 import { headlessInstall, type InstallationResultStats } from '@pnpm/headless'
@@ -78,7 +78,6 @@ import equals from 'ramda/src/equals'
 import isEmpty from 'ramda/src/isEmpty'
 import pipeWith from 'ramda/src/pipeWith'
 import props from 'ramda/src/props'
-import sortKeys from 'sort-keys'
 import { parseWantedDependencies } from '../parseWantedDependencies'
 import { removeDeps } from '../uninstall/removeDeps'
 import {
@@ -328,7 +327,7 @@ export async function mutateModules (
         }
       )
     }
-    const packageExtensionsChecksum = isEmpty(opts.packageExtensions ?? {}) ? undefined : createObjectChecksum(opts.packageExtensions!)
+    const packageExtensionsChecksum = isEmpty(opts.packageExtensions ?? {}) ? undefined : `sha256-${hashObject(opts.packageExtensions!)}`
     const pnpmfileChecksum = await opts.hooks.calculatePnpmfileChecksum?.()
     const patchedDependencies = opts.ignorePackageManifest
       ? ctx.wantedLockfile.patchedDependencies
@@ -818,11 +817,6 @@ function getOutdatedLockfileSetting (
   return null
 }
 
-export function createObjectChecksum (obj: Record<string, unknown>): string {
-  const s = JSON.stringify(sortKeys(obj, { deep: true }))
-  return crypto.createHash('md5').update(s).digest('hex')
-}
-
 function cacheExpired (prunedAt: string, maxAgeInMinutes: number): boolean {
   return ((Date.now() - new Date(prunedAt).valueOf()) / (1000 * 60)) > maxAgeInMinutes
 }

pkg-manager/core/test/install/packageExtensions.ts

@@ -1,12 +1,16 @@
 import { PnpmError } from '@pnpm/error'
 import { prepareEmpty } from '@pnpm/prepare'
 import { addDependenciesToPackage, mutateModulesInSingleProject, install } from '@pnpm/core'
+import { hashObject as _hashObject } from '@pnpm/crypto.object-hasher'
 import { type ProjectRootDir, type PackageExtension, type ProjectManifest } from '@pnpm/types'
-import { createObjectChecksum } from '../../lib/install/index'
 import {
   testDefaults,
 } from '../utils'
 
+function hashObject (obj: Record<string, unknown>): string {
+  return `sha256-${_hashObject(obj)}`
+}
+
 test('manifests are extended with fields specified by packageExtensions', async () => {
   const project = prepareEmpty()
 
@@ -26,7 +30,7 @@ test('manifests are extended with fields specified by packageExtensions', async
   {
     const lockfile = project.readLockfile()
     expect(lockfile.snapshots['is-positive@1.0.0'].dependencies?.['@pnpm.e2e/bar']).toBe('100.1.0')
-    expect(lockfile.packageExtensionsChecksum).toStrictEqual(createObjectChecksum({
+    expect(lockfile.packageExtensionsChecksum).toStrictEqual(hashObject({
       'is-positive': {
         dependencies: {
           '@pnpm.e2e/bar': '100.1.0',
@@ -48,7 +52,7 @@ test('manifests are extended with fields specified by packageExtensions', async
   {
     const lockfile = project.readLockfile()
     expect(lockfile.snapshots['is-positive@1.0.0'].dependencies?.['@pnpm.e2e/foobar']).toBe('100.0.0')
-    expect(lockfile.packageExtensionsChecksum).toStrictEqual(createObjectChecksum({
+    expect(lockfile.packageExtensionsChecksum).toStrictEqual(hashObject({
       'is-positive': {
         dependencies: {
           '@pnpm.e2e/bar': '100.1.0',
@@ -68,7 +72,7 @@ test('manifests are extended with fields specified by packageExtensions', async
 
   {
     const lockfile = project.readLockfile()
-    expect(lockfile.packageExtensionsChecksum).toStrictEqual(createObjectChecksum({
+    expect(lockfile.packageExtensionsChecksum).toStrictEqual(hashObject({
       'is-positive': {
         dependencies: {
           '@pnpm.e2e/bar': '100.1.0',

pnpm-lock.yaml

@@ -3947,6 +3947,9 @@ importers:
       '@pnpm/crypto.hash':
         specifier: workspace:*
         version: link:../../crypto/hash
+      '@pnpm/crypto.object-hasher':
+        specifier: workspace:*
+        version: link:../../crypto/object-hasher
       '@pnpm/dependency-path':
         specifier: workspace:*
         version: link:../../packages/dependency-path
@@ -4097,9 +4100,6 @@ importers:
       semver:
         specifier: 'catalog:'
         version: 7.6.2
-      sort-keys:
-        specifier: 'catalog:'
-        version: 4.2.0
     devDependencies:
       '@pnpm/assert-project':
         specifier: workspace:*
@@ -4113,9 +4113,6 @@ importers:
       '@pnpm/core':
         specifier: workspace:*
         version: 'link:'
-      '@pnpm/crypto.object-hasher':
-        specifier: workspace:*
-        version: link:../../crypto/object-hasher
       '@pnpm/git-utils':
         specifier: workspace:*
         version: link:../../packages/git-utils
@@ -6126,6 +6123,9 @@ importers:
       '@pnpm/core-loggers':
         specifier: workspace:*
         version: link:../../packages/core-loggers
+      '@pnpm/crypto.hash':
+        specifier: workspace:*
+        version: link:../../crypto/hash
       '@pnpm/error':
         specifier: workspace:*
         version: link:../../packages/error

resolving/npm-resolver/package.json

@@ -35,6 +35,7 @@
   },
   "dependencies": {
     "@pnpm/core-loggers": "workspace:*",
+    "@pnpm/crypto.hash": "workspace:*",
     "@pnpm/error": "workspace:*",
     "@pnpm/fetching-types": "workspace:*",
     "@pnpm/graceful-fs": "workspace:*",

resolving/npm-resolver/src/pickPackage.ts

@@ -1,6 +1,6 @@
-import crypto from 'crypto'
 import { promises as fs } from 'fs'
 import path from 'path'
+import { createHexHash } from '@pnpm/crypto.hash'
 import { PnpmError } from '@pnpm/error'
 import { logger } from '@pnpm/logger'
 import gfs from '@pnpm/graceful-fs'
@@ -268,7 +268,7 @@ function clearMeta (pkg: PackageMeta): PackageMeta {
 
 function encodePkgName (pkgName: string): string {
   if (pkgName !== pkgName.toLowerCase()) {
-    return `${pkgName}_${crypto.createHash('md5').update(pkgName).digest('hex')}`
+    return `${pkgName}_${createHexHash(pkgName)}`
   }
   return pkgName
 }

resolving/npm-resolver/test/index.ts

@@ -1,6 +1,7 @@
 /// <reference path="../../../__typings__/index.d.ts"/>
 import fs from 'fs'
 import path from 'path'
+import { createHexHash } from '@pnpm/crypto.hash'
 import { PnpmError } from '@pnpm/error'
 import { createFetchFromRegistry } from '@pnpm/fetch'
 import {
@@ -110,7 +111,7 @@ test('resolveFromNpm() should save metadata to a unique file when the package na
 
   // The resolve function does not wait for the package meta cache file to be saved
   // so we must delay for a bit in order to read it
-  const meta = await retryLoadJsonFile<any>(path.join(cacheDir, 'metadata/registry.npmjs.org/JSON_0ecd11c1d7a287401d148a23bbd7a2f8.json')) // eslint-disable-line @typescript-eslint/no-explicit-any
+  const meta = await retryLoadJsonFile<any>(path.join(cacheDir, `metadata/registry.npmjs.org/JSON_${createHexHash('JSON')}.json`)) // eslint-disable-line @typescript-eslint/no-explicit-any
   expect(meta.name).toBeTruthy()
   expect(meta.versions).toBeTruthy()
   expect(meta['dist-tags']).toBeTruthy()

resolving/npm-resolver/tsconfig.json

@@ -12,6 +12,9 @@
     {
       "path": "../../__utils__/test-fixtures"
     },
+    {
+      "path": "../../crypto/hash"
+    },
     {
       "path": "../../fs/graceful-fs"
     },