fix: take into account the integrities of packages in when calculating the dependency graph hash (#9605)

* fix: take into account the integrities of packages in when calculating the dependency graph hash

* test: fix

* test: fix

* test: fix

* test: fix

* fix: include the package's integirty in the hash as well

* docs: add comment

* perf: hashing deps graph

* fix: deps graph hash

* refactor: calc graph hash

* test: fix

* refactor: calc graph hash

* refactor: rename uniquePkgId to fullPkgId

* docs: add changeset
This commit is contained in:
Zoltan Kochan
2025-06-08 01:05:10 +02:00
committed by GitHub
parent c2799c6d98
commit b3898dbb1e
19 changed files with 141 additions and 72 deletions

View File

@@ -2,4 +2,4 @@
"@pnpm/calc-dep-state": major
---
Renamed `isBuilt` option to `includeSubdepsHash`.
Renamed `isBuilt` option to `includeDepGraphHash`.

View File

@@ -0,0 +1,6 @@
---
"@pnpm/calc-dep-state": major
"pnpm": minor
---
**Semi-breaking.** The keys used for side-effects caches have changed. If you have a side-effects cache generated by a previous version of pnpm, the new version will not use it and will create a new cache instead [#9605](https://github.com/pnpm/pnpm/pull/9605).

View File

@@ -73,7 +73,7 @@ function * iteratePkgMeta (lockfile: LockfileObject, graph: DepsGraph<DepPath>):
name,
version,
depPath: depPath as DepPath,
pkgIdWithPatchHash: graph[depPath as DepPath].pkgIdWithPatchHash,
pkgIdWithPatchHash: graph[depPath as DepPath].pkgIdWithPatchHash ?? dp.getPkgIdWithPatchHash(depPath as DepPath),
pkgSnapshot,
}
}

View File

@@ -3,7 +3,7 @@ import { WANTED_LOCKFILE } from '@pnpm/constants'
import {
progressLogger,
} from '@pnpm/core-loggers'
import { type LockfileObject } from '@pnpm/lockfile.fs'
import { type LockfileResolution, type LockfileObject } from '@pnpm/lockfile.fs'
import {
packageIdFromSnapshot,
pkgSnapshotToResolution,
@@ -44,6 +44,7 @@ export interface DependenciesGraphNode {
hasBin: boolean
filesIndexFile?: string
patch?: PatchInfo
resolution: LockfileResolution
}
export interface DependenciesGraph {
@@ -227,6 +228,7 @@ async function buildGraphFromPackages (
graph[dir] = {
children: {},
pkgIdWithPatchHash,
resolution: pkgSnapshot.resolution,
depPath,
dir,
fetching: fetchResponse.fetching,

View File

@@ -164,7 +164,7 @@ async function buildDependency<T extends string> (
try {
const sideEffectsCacheKey = calcDepState(depGraph, opts.depsStateCache, depPath, {
patchFileHash: depNode.patch?.file.hash,
includeSubdepsHash: hasSideEffects,
includeDepGraphHash: hasSideEffects,
})
await opts.storeController.upload(depNode.dir, {
sideEffectsCacheKey,

View File

@@ -345,7 +345,7 @@ async function _rebuild (
const filesIndexFile = getIndexFilePathInCafs(opts.storeDir, resolution.integrity!.toString(), pkgId)
const pkgFilesIndex = await loadJsonFile<PackageFilesIndex>(filesIndexFile)
sideEffectsCacheKey = calcDepState(depGraph, depsStateCache, depPath, {
includeSubdepsHash: true,
includeDepGraphHash: true,
})
if (pkgFilesIndex.sideEffects?.[sideEffectsCacheKey]) {
pkgsThatWereRebuilt.add(depPath)
@@ -378,7 +378,7 @@ async function _rebuild (
try {
if (!sideEffectsCacheKey) {
sideEffectsCacheKey = calcDepState(depGraph, depsStateCache, depPath, {
includeSubdepsHash: true,
includeDepGraphHash: true,
})
}
await opts.storeController.upload(pkgRoot, {

View File

@@ -77,7 +77,15 @@ test('rebuilds dependencies', async () => {
const cacheIntegrityPath = getIndexFilePathInCafs(path.join(storeDir, STORE_VERSION), getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0')
const cacheIntegrity = loadJsonFile.sync<any>(cacheIntegrityPath) // eslint-disable-line @typescript-eslint/no-explicit-any
expect(cacheIntegrity!.sideEffects).toBeTruthy()
const sideEffectsKey = `${ENGINE_NAME};deps=${hashObject({ '@pnpm.e2e/hello-world-js-bin@1.0.0': {} })}`
const sideEffectsKey = `${ENGINE_NAME};deps=${hashObject({
id: `@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0:${getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0')}`,
deps: {
'@pnpm.e2e/hello-world-js-bin': hashObject({
id: `@pnpm.e2e/hello-world-js-bin@1.0.0:${getIntegrity('@pnpm.e2e/hello-world-js-bin', '1.0.0')}`,
deps: {},
}),
},
})}`
expect(cacheIntegrity).toHaveProperty(['sideEffects', sideEffectsKey, 'added', 'generated-by-postinstall.js'])
delete cacheIntegrity!.sideEffects[sideEffectsKey].added['generated-by-postinstall.js']
})

View File

@@ -35,9 +35,7 @@
"@pnpm/dependency-path": "workspace:*",
"@pnpm/lockfile.types": "workspace:*",
"@pnpm/lockfile.utils": "workspace:*",
"@pnpm/object.key-sorting": "workspace:*",
"@pnpm/types": "workspace:*",
"sort-keys": "catalog:"
"@pnpm/types": "workspace:*"
},
"devDependencies": {
"@pnpm/calc-dep-state": "workspace:*"

View File

@@ -1,23 +1,22 @@
import { ENGINE_NAME } from '@pnpm/constants'
import { getPkgIdWithPatchHash, refToRelative } from '@pnpm/dependency-path'
import { type DepPath, type PkgIdWithPatchHash } from '@pnpm/types'
import { hashObjectWithoutSorting } from '@pnpm/crypto.object-hasher'
import { type LockfileObject } from '@pnpm/lockfile.types'
import { sortDirectKeys } from '@pnpm/object.key-sorting'
import { hashObjectWithoutSorting, hashObject } from '@pnpm/crypto.object-hasher'
import { type LockfileResolution, type LockfileObject } from '@pnpm/lockfile.types'
export type DepsGraph<T extends string> = Record<T, DepsGraphNode<T>>
export interface DepsGraphNode<T extends string> {
children: { [alias: string]: T }
pkgIdWithPatchHash: PkgIdWithPatchHash
pkgIdWithPatchHash?: PkgIdWithPatchHash
resolution?: LockfileResolution
// The full package ID is a unique fingerprint based on the packages
// integrity checksum, patch information, and other resolution data.
fullPkgId?: string
}
export interface DepsStateCache {
[depPath: string]: DepStateObj
}
export interface DepStateObj {
[depPath: string]: DepStateObj
[depPath: string]: string
}
export function calcDepState<T extends string> (
@@ -26,13 +25,13 @@ export function calcDepState<T extends string> (
depPath: string,
opts: {
patchFileHash?: string
includeSubdepsHash: boolean
includeDepGraphHash: boolean
}
): string {
let result = ENGINE_NAME
if (opts.includeSubdepsHash) {
const depStateObj = calcDepStateObj(depPath, depsGraph, cache, new Set())
result += `;deps=${hashObjectWithoutSorting(depStateObj)}`
if (opts.includeDepGraphHash) {
const depGraphHash = calcDepGraphHash(depsGraph, cache, new Set(), depPath)
result += `;deps=${depGraphHash}`
}
if (opts.patchFileHash) {
result += `;patch=${opts.patchFileHash}`
@@ -40,27 +39,31 @@ export function calcDepState<T extends string> (
return result
}
function calcDepStateObj<T extends string> (
depPath: T,
function calcDepGraphHash<T extends string> (
depsGraph: DepsGraph<T>,
cache: DepsStateCache,
parents: Set<PkgIdWithPatchHash>
): DepStateObj {
parents: Set<string>,
depPath: T
): string {
if (cache[depPath]) return cache[depPath]
const node = depsGraph[depPath]
if (!node) return {}
const nextParents = new Set([...Array.from(parents), node.pkgIdWithPatchHash])
const state: DepStateObj = {}
for (const childId of Object.values(node.children)) {
const child = depsGraph[childId]
if (!child) continue
if (parents.has(child.pkgIdWithPatchHash)) {
state[child.pkgIdWithPatchHash] = {}
continue
if (!node) return ''
node.fullPkgId ??= createFullPkgId(node.pkgIdWithPatchHash!, node.resolution!)
const deps: Record<string, string> = {}
if (Object.keys(node.children).length && !parents.has(node.fullPkgId)) {
const nextParents = new Set([...Array.from(parents), node.fullPkgId])
const _calcDepGraphHash = calcDepGraphHash.bind(null, depsGraph, cache, nextParents)
for (const alias in node.children) {
if (Object.prototype.hasOwnProperty.call(node.children, alias)) {
const childId = node.children[alias]
deps[alias] = _calcDepGraphHash(childId)
}
}
state[child.pkgIdWithPatchHash] = calcDepStateObj(childId, depsGraph, cache, nextParents)
}
cache[depPath] = sortDirectKeys(state)
cache[depPath] = hashObject({
id: node.fullPkgId,
deps,
})
return cache[depPath]
}
@@ -81,10 +84,20 @@ export function * iterateHashedGraphNodes<T extends PkgMeta> (
graph: DepsGraph<DepPath>,
pkgMetaIterator: PkgMetaIterator<T>
): IterableIterator<HashedDepPath<T>> {
const cache: DepsStateCache = {}
const _calcDepGraphHash = calcDepGraphHash.bind(null, graph, {})
for (const pkgMeta of pkgMetaIterator) {
const { name, version, depPath } = pkgMeta
const state = calcDepState(graph, cache, depPath, { includeSubdepsHash: true })
const state = {
// Unfortunately, we need to include the engine name in the hash,
// even though it's only required for packages that are built,
// or have dependencies that are built.
// We can't know for sure whether a package needs to be built
// before it's fetched from the registry.
// However, we fetch and write packages to node_modules in random order for performance,
// so we can't determine at this stage which dependencies will be built.
engine: ENGINE_NAME,
deps: _calcDepGraphHash(new Set(), depPath),
}
const hexDigest = hashObjectWithoutSorting(state, { encoding: 'hex' })
yield {
hash: `${name}/${version}/${hexDigest}`,
@@ -103,7 +116,7 @@ export function lockfileToDepGraph (lockfile: LockfileObject): DepsGraph<DepPath
})
graph[depPath as DepPath] = {
children,
pkgIdWithPatchHash: getPkgIdWithPatchHash(depPath as DepPath),
fullPkgId: createFullPkgId(getPkgIdWithPatchHash(depPath as DepPath), pkgSnapshot.resolution),
}
}
}
@@ -120,3 +133,8 @@ function lockfileDepsToGraphChildren (deps: Record<string, string>): Record<stri
}
return children
}
function createFullPkgId (pkgIdWithPatchHash: PkgIdWithPatchHash, resolution: LockfileResolution): string {
const res = 'integrity' in resolution ? resolution.integrity : JSON.stringify(resolution)
return `${pkgIdWithPatchHash}:${res}`
}

View File

@@ -4,30 +4,47 @@ import { hashObject } from '@pnpm/crypto.object-hasher'
import { type PkgIdWithPatchHash } from '@pnpm/types'
const depsGraph = {
'registry/foo@1.0.0': {
'foo@1.0.0': {
pkgIdWithPatchHash: 'foo@1.0.0' as PkgIdWithPatchHash,
resolution: {
integrity: '000',
},
children: {
bar: 'registry/bar@1.0.0',
bar: 'bar@1.0.0',
},
},
'registry/bar@1.0.0': {
'bar@1.0.0': {
pkgIdWithPatchHash: 'bar@1.0.0' as PkgIdWithPatchHash,
resolution: {
integrity: '001',
},
children: {
foo: 'registry/foo@1.0.0',
foo: 'foo@1.0.0',
},
},
}
test('calcDepState()', () => {
expect(calcDepState(depsGraph, {}, 'registry/foo@1.0.0', {
includeSubdepsHash: true,
expect(calcDepState(depsGraph, {}, 'foo@1.0.0', {
includeDepGraphHash: true,
})).toBe(`${ENGINE_NAME};deps=${hashObject({
'bar@1.0.0': { 'foo@1.0.0': {} },
id: 'foo@1.0.0:000',
deps: {
bar: hashObject({
id: 'bar@1.0.0:001',
deps: {
foo: hashObject({
id: 'foo@1.0.0:000',
deps: {},
}),
},
}),
},
})}`)
})
test('calcDepState() when scripts are ignored', () => {
expect(calcDepState(depsGraph, {}, 'registry/foo@1.0.0', {
includeSubdepsHash: false,
expect(calcDepState(depsGraph, {}, 'foo@1.0.0', {
includeDepGraphHash: false,
})).toBe(ENGINE_NAME)
})

View File

@@ -14,7 +14,7 @@ test('lockfileToDepGraph', () => {
qar: '1.0.0',
},
resolution: {
integrity: '',
integrity: '0',
},
},
['bar@1.0.0' as DepPath]: {
@@ -22,12 +22,12 @@ test('lockfileToDepGraph', () => {
qar: '1.0.0',
},
resolution: {
integrity: '',
integrity: '1',
},
},
['qar@1.0.0' as DepPath]: {
resolution: {
integrity: '',
integrity: '2',
},
},
},
@@ -36,18 +36,18 @@ test('lockfileToDepGraph', () => {
children: {
qar: 'qar@1.0.0',
},
pkgIdWithPatchHash: 'bar@1.0.0',
fullPkgId: 'bar@1.0.0:1',
},
'foo@1.0.0': {
children: {
bar: 'bar@1.0.0',
qar: 'qar@1.0.0',
},
pkgIdWithPatchHash: 'foo@1.0.0',
fullPkgId: 'foo@1.0.0:0',
},
'qar@1.0.0': {
children: {},
pkgIdWithPatchHash: 'qar@1.0.0',
fullPkgId: 'qar@1.0.0:2',
},
})
})

View File

@@ -18,9 +18,6 @@
{
"path": "../../lockfile/utils"
},
{
"path": "../../object/key-sorting"
},
{
"path": "../constants"
},

View File

@@ -468,7 +468,7 @@ async function linkAllPkgs (
if (opts.sideEffectsCacheRead && files.sideEffects && !isEmpty(files.sideEffects)) {
if (opts?.allowBuild?.(depNode.name) !== false) {
sideEffectsCacheKey = calcDepState(opts.depGraph, opts.depsStateCache, depNode.depPath, {
includeSubdepsHash: !opts.ignoreScripts && depNode.requiresBuild, // true when is built
includeDepGraphHash: !opts.ignoreScripts && depNode.requiresBuild, // true when is built
patchFileHash: depNode.patch?.file.hash,
})
}

View File

@@ -85,7 +85,15 @@ test('using side effects cache', async () => {
const filesIndexFile = getIndexFilePathInCafs(opts.storeDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0')
const filesIndex = loadJsonFile.sync<PackageFilesIndex>(filesIndexFile)
expect(filesIndex.sideEffects).toBeTruthy() // files index has side effects
const sideEffectsKey = `${ENGINE_NAME};deps=${hashObject({ '@pnpm.e2e/hello-world-js-bin@1.0.0': {} })}`
const sideEffectsKey = `${ENGINE_NAME};deps=${hashObject({
id: `@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0:${getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0')}`,
deps: {
'@pnpm.e2e/hello-world-js-bin': hashObject({
id: `@pnpm.e2e/hello-world-js-bin@1.0.0:${getIntegrity('@pnpm.e2e/hello-world-js-bin', '1.0.0')}`,
deps: {},
}),
},
})}`
expect(filesIndex.sideEffects).toHaveProperty([sideEffectsKey, 'added', 'generated-by-preinstall.js'])
expect(filesIndex.sideEffects).toHaveProperty([sideEffectsKey, 'added', 'generated-by-postinstall.js'])
delete filesIndex.sideEffects![sideEffectsKey].added?.['generated-by-postinstall.js']
@@ -174,7 +182,10 @@ test('a postinstall script does not modify the original sources added to the sto
const filesIndexFile = getIndexFilePathInCafs(opts.storeDir, getIntegrity('@pnpm/postinstall-modifies-source', '1.0.0'), '@pnpm/postinstall-modifies-source@1.0.0')
const filesIndex = loadJsonFile.sync<PackageFilesIndex>(filesIndexFile)
const patchedFileIntegrity = filesIndex.sideEffects?.[`${ENGINE_NAME};deps=${hashObject({})}`].added?.['empty-file.txt']?.integrity
const patchedFileIntegrity = filesIndex.sideEffects?.[`${ENGINE_NAME};deps=${hashObject({
id: `@pnpm/postinstall-modifies-source@1.0.0:${getIntegrity('@pnpm/postinstall-modifies-source', '1.0.0')}`,
deps: {},
})}`].added?.['empty-file.txt']?.integrity
expect(patchedFileIntegrity).toBeTruthy()
const originalFileIntegrity = filesIndex.files['empty-file.txt'].integrity
expect(originalFileIntegrity).toBeTruthy()
@@ -197,7 +208,16 @@ test('a corrupted side-effects cache is ignored', async () => {
const filesIndexFile = getIndexFilePathInCafs(opts.storeDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0')
const filesIndex = loadJsonFile.sync<PackageFilesIndex>(filesIndexFile)
expect(filesIndex.sideEffects).toBeTruthy() // files index has side effects
const sideEffectsKey = `${ENGINE_NAME};deps=${hashObject({ '@pnpm.e2e/hello-world-js-bin@1.0.0': {} })}`
const sideEffectsKey = `${ENGINE_NAME};deps=${hashObject({
id: `@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0:${getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0')}`,
deps: {
'@pnpm.e2e/hello-world-js-bin': hashObject({
id: `@pnpm.e2e/hello-world-js-bin@1.0.0:${getIntegrity('@pnpm.e2e/hello-world-js-bin', '1.0.0')}`,
deps: {},
}),
},
})}`
expect(filesIndex.sideEffects).toHaveProperty([sideEffectsKey, 'added', 'generated-by-preinstall.js'])
const sideEffectFileStat = filesIndex.sideEffects![sideEffectsKey].added!['generated-by-preinstall.js']
const sideEffectFile = getFilePathByModeInCafs(opts.storeDir, sideEffectFileStat.integrity, sideEffectFileStat.mode)

View File

@@ -874,7 +874,7 @@ async function linkAllPkgs (
if (opts.sideEffectsCacheRead && filesResponse.sideEffects && !isEmpty(filesResponse.sideEffects)) {
if (opts?.allowBuild?.(depNode.name) !== false) {
sideEffectsCacheKey = calcDepState(opts.depGraph, opts.depsStateCache, depNode.dir, {
includeSubdepsHash: !opts.ignoreScripts && depNode.requiresBuild, // true when is built
includeDepGraphHash: !opts.ignoreScripts && depNode.requiresBuild, // true when is built
patchFileHash: depNode.patch?.file.hash,
})
}

View File

@@ -117,7 +117,7 @@ async function linkAllPkgsInOrder (
if (opts.sideEffectsCacheRead && filesResponse.sideEffects && !isEmpty(filesResponse.sideEffects)) {
if (opts?.allowBuild?.(depNode.name) !== false) {
sideEffectsCacheKey = _calcDepState(dir, {
includeSubdepsHash: !opts.ignoreScripts && depNode.requiresBuild, // true when is built
includeDepGraphHash: !opts.ignoreScripts && depNode.requiresBuild, // true when is built
patchFileHash: depNode.patch?.file.hash,
})
}

View File

@@ -253,6 +253,7 @@ async function fetchDeps (
optional: !!pkgSnapshot.optional,
optionalDependencies: new Set(Object.keys(pkgSnapshot.optionalDependencies ?? {})),
patch: getPatchInfo(opts.patchedDependencies, pkgName, pkgVersion),
resolution: pkgSnapshot.resolution,
}
if (!opts.pkgLocationsByDepPath[depPath]) {
opts.pkgLocationsByDepPath[depPath] = []

View File

@@ -681,7 +681,15 @@ test.each([['isolated'], ['hoisted']])('using side effects cache with nodeLinker
const cacheIntegrityPath = getIndexFilePathInCafs(opts.storeDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0')
const cacheIntegrity = loadJsonFile.sync<any>(cacheIntegrityPath) // eslint-disable-line @typescript-eslint/no-explicit-any
expect(cacheIntegrity!.sideEffects).toBeTruthy()
const sideEffectsKey = `${ENGINE_NAME};deps=${hashObject({ '@pnpm.e2e/hello-world-js-bin@1.0.0': {} })}`
const sideEffectsKey = `${ENGINE_NAME};deps=${hashObject({
id: `@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0:${getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0')}`,
deps: {
'@pnpm.e2e/hello-world-js-bin': hashObject({
id: `@pnpm.e2e/hello-world-js-bin@1.0.0:${getIntegrity('@pnpm.e2e/hello-world-js-bin', '1.0.0')}`,
deps: {},
}),
},
})}`
expect(cacheIntegrity).toHaveProperty(['sideEffects', sideEffectsKey, 'added', 'generated-by-postinstall.js'])
delete cacheIntegrity!.sideEffects[sideEffectsKey].added['generated-by-postinstall.js']

6
pnpm-lock.yaml generated
View File

@@ -3976,15 +3976,9 @@ importers:
'@pnpm/lockfile.utils':
specifier: workspace:*
version: link:../../lockfile/utils
'@pnpm/object.key-sorting':
specifier: workspace:*
version: link:../../object/key-sorting
'@pnpm/types':
specifier: workspace:*
version: link:../types
sort-keys:
specifier: 'catalog:'
version: 4.2.0
devDependencies:
'@pnpm/calc-dep-state':
specifier: workspace:*