feat: adding the package name into the index file name (#8510)

close #8204
This commit is contained in:
Zoltan Kochan
2024-09-13 12:34:22 +02:00
parent dbec72005a
commit d433cb9c9f
22 changed files with 94 additions and 56 deletions

View File

@@ -0,0 +1,25 @@
---
"@pnpm/plugin-commands-store-inspecting": minor
"@pnpm/package-requester": major
"@pnpm/plugin-commands-rebuild": major
"@pnpm/plugin-commands-store": major
"@pnpm/license-scanner": major
"@pnpm/assert-project": major
"@pnpm/assert-store": major
"@pnpm/mount-modules": minor
"@pnpm/headless": major
"@pnpm/package-store": major
"@pnpm/core": major
"@pnpm/store.cafs": major
"pnpm": major
---
Some registries allow identical content to be published under different package names or versions. To accommodate this, index files in the store are now stored using both the content hash and package identifier.
This approach ensures that we can:
1. Validate that the integrity in the lockfile corresponds to the correct package,
which might not be the case after a poorly resolved Git conflict.
2. Allow the same content to be referenced by different packages or different versions of the same package.
Related PR: [#8510](https://github.com/pnpm/pnpm/pull/8510)
Related issue: [#8204](https://github.com/pnpm/pnpm/issues/8204)

View File

@@ -20,9 +20,9 @@ export interface Project {
hasNot: (pkgName: string, modulesDir?: string) => void hasNot: (pkgName: string, modulesDir?: string) => void
getStorePath: () => string getStorePath: () => string
resolve: (pkgName: string, version?: string, relativePath?: string) => string resolve: (pkgName: string, version?: string, relativePath?: string) => string
getPkgIndexFilePath: (pkgName: string, version?: string) => string getPkgIndexFilePath: (pkgName: string, version: string) => string
cafsHas: (pkgName: string, version?: string) => void cafsHas: (pkgName: string, version: string) => void
cafsHasNot: (pkgName: string, version?: string) => void cafsHasNot: (pkgName: string, version: string) => void
storeHas: (pkgName: string, version?: string) => string storeHas: (pkgName: string, version?: string) => string
storeHasNot: (pkgName: string, version?: string) => void storeHasNot: (pkgName: string, version?: string) => void
isExecutable: (pathToExe: string) => void isExecutable: (pathToExe: string) => void
@@ -48,9 +48,9 @@ export function assertProject (projectPath: string, encodedRegistryName?: string
interface StoreInstance { interface StoreInstance {
storePath: string storePath: string
getPkgIndexFilePath: (pkgName: string, version?: string) => string getPkgIndexFilePath: (pkgName: string, version: string) => string
cafsHas: (pkgName: string, version?: string) => void cafsHas: (pkgName: string, version: string) => void
cafsHasNot: (pkgName: string, version?: string) => void cafsHasNot: (pkgName: string, version: string) => void
storeHas: (pkgName: string, version?: string) => void storeHas: (pkgName: string, version?: string) => void
storeHasNot: (pkgName: string, version?: string) => void storeHasNot: (pkgName: string, version?: string) => void
resolve: (pkgName: string, version?: string, relativePath?: string) => string resolve: (pkgName: string, version?: string, relativePath?: string) => string
@@ -107,15 +107,15 @@ export function assertProject (projectPath: string, encodedRegistryName?: string
const store = getStoreInstance() const store = getStoreInstance()
return store.resolve(pkgName, version, relativePath) return store.resolve(pkgName, version, relativePath)
}, },
getPkgIndexFilePath (pkgName: string, version?: string): string { getPkgIndexFilePath (pkgName: string, version: string): string {
const store = getStoreInstance() const store = getStoreInstance()
return store.getPkgIndexFilePath(pkgName, version) return store.getPkgIndexFilePath(pkgName, version)
}, },
cafsHas (pkgName: string, version?: string) { cafsHas (pkgName: string, version: string) {
const store = getStoreInstance() const store = getStoreInstance()
store.cafsHas(pkgName, version) store.cafsHas(pkgName, version)
}, },
cafsHasNot (pkgName: string, version?: string) { cafsHasNot (pkgName: string, version: string) {
const store = getStoreInstance() const store = getStoreInstance()
store.cafsHasNot(pkgName, version) store.cafsHasNot(pkgName, version)
}, },

View File

@@ -4,9 +4,9 @@ import { getIndexFilePathInCafs } from '@pnpm/store.cafs'
import { getIntegrity, REGISTRY_MOCK_PORT } from '@pnpm/registry-mock' import { getIntegrity, REGISTRY_MOCK_PORT } from '@pnpm/registry-mock'
export interface StoreAssertions { export interface StoreAssertions {
getPkgIndexFilePath: (pkgName: string, version?: string) => string getPkgIndexFilePath: (pkgName: string, version: string) => string
cafsHas: (pkgName: string, version?: string) => void cafsHas: (pkgName: string, version: string) => void
cafsHasNot: (pkgName: string, version?: string) => void cafsHasNot: (pkgName: string, version: string) => void
storeHas: (pkgName: string, version?: string) => void storeHas: (pkgName: string, version?: string) => void
storeHasNot: (pkgName: string, version?: string) => void storeHasNot: (pkgName: string, version?: string) => void
resolve: (pkgName: string, version?: string, relativePath?: string) => string resolve: (pkgName: string, version?: string, relativePath?: string) => string
@@ -22,16 +22,16 @@ export function assertStore (
const notOk = (value: any) => expect(value).toBeFalsy() const notOk = (value: any) => expect(value).toBeFalsy()
const ern = encodedRegistryName ?? `localhost+${REGISTRY_MOCK_PORT}` const ern = encodedRegistryName ?? `localhost+${REGISTRY_MOCK_PORT}`
const store = { const store = {
getPkgIndexFilePath (pkgName: string, version?: string): string { getPkgIndexFilePath (pkgName: string, version: string): string {
const cafsDir = path.join(storePath, 'files') const cafsDir = path.join(storePath, 'files')
const integrity = version ? getIntegrity(pkgName, version) : pkgName const integrity = getIntegrity(pkgName, version)
return getIndexFilePathInCafs(cafsDir, integrity) return getIndexFilePathInCafs(cafsDir, integrity, `${pkgName}@${version}`)
}, },
cafsHas (pkgName: string, version?: string): void { cafsHas (pkgName: string, version: string): void {
const pathToCheck = store.getPkgIndexFilePath(pkgName, version) const pathToCheck = store.getPkgIndexFilePath(pkgName, version)
ok(fs.existsSync(pathToCheck)) ok(fs.existsSync(pathToCheck))
}, },
cafsHasNot (pkgName: string, version?: string): void { cafsHasNot (pkgName: string, version: string): void {
const pathToCheck = store.getPkgIndexFilePath(pkgName, version) const pathToCheck = store.getPkgIndexFilePath(pkgName, version)
notOk(fs.existsSync(pathToCheck)) notOk(fs.existsSync(pathToCheck))
}, },

View File

@@ -315,8 +315,9 @@ async function _rebuild (
} }
const resolution = (pkgSnapshot.resolution as TarballResolution) const resolution = (pkgSnapshot.resolution as TarballResolution)
let sideEffectsCacheKey: string | undefined let sideEffectsCacheKey: string | undefined
const pkgId = `${pkgInfo.name}@${pkgInfo.version}`
if (opts.skipIfHasSideEffectsCache && resolution.integrity) { if (opts.skipIfHasSideEffectsCache && resolution.integrity) {
const filesIndexFile = getIndexFilePathInCafs(cafsDir, resolution.integrity!.toString()) const filesIndexFile = getIndexFilePathInCafs(cafsDir, resolution.integrity!.toString(), pkgId)
const pkgFilesIndex = await loadJsonFile<PackageFilesIndex>(filesIndexFile) const pkgFilesIndex = await loadJsonFile<PackageFilesIndex>(filesIndexFile)
sideEffectsCacheKey = calcDepState(depGraph, depsStateCache, depPath, { sideEffectsCacheKey = calcDepState(depGraph, depsStateCache, depPath, {
isBuilt: true, isBuilt: true,
@@ -341,7 +342,7 @@ async function _rebuild (
}) })
if (hasSideEffects && (opts.sideEffectsCacheWrite ?? true) && resolution.integrity) { if (hasSideEffects && (opts.sideEffectsCacheWrite ?? true) && resolution.integrity) {
builtDepPaths.add(depPath) builtDepPaths.add(depPath)
const filesIndexFile = getIndexFilePathInCafs(cafsDir, resolution.integrity!.toString()) const filesIndexFile = getIndexFilePathInCafs(cafsDir, resolution.integrity!.toString(), pkgId)
try { try {
if (!sideEffectsCacheKey) { if (!sideEffectsCacheKey) {
sideEffectsCacheKey = calcDepState(depGraph, depsStateCache, depPath, { sideEffectsCacheKey = calcDepState(depGraph, depsStateCache, depPath, {

View File

@@ -75,7 +75,7 @@ test('rebuilds dependencies', async () => {
} }
const cafsDir = path.join(storeDir, 'v3/files') const cafsDir = path.join(storeDir, 'v3/files')
const cacheIntegrityPath = getIndexFilePathInCafs(cafsDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0')) const cacheIntegrityPath = getIndexFilePathInCafs(cafsDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0')
const cacheIntegrity = loadJsonFile.sync<any>(cacheIntegrityPath) // eslint-disable-line @typescript-eslint/no-explicit-any const cacheIntegrity = loadJsonFile.sync<any>(cacheIntegrityPath) // eslint-disable-line @typescript-eslint/no-explicit-any
expect(cacheIntegrity!.sideEffects).toBeTruthy() expect(cacheIntegrity!.sideEffects).toBeTruthy()
const sideEffectsKey = `${ENGINE_NAME}-${hashObject({ '@pnpm.e2e/hello-world-js-bin@1.0.0': {} })}` const sideEffectsKey = `${ENGINE_NAME}-${hashObject({ '@pnpm.e2e/hello-world-js-bin@1.0.0': {} })}`
@@ -100,7 +100,7 @@ test('skipIfHasSideEffectsCache', async () => {
]) ])
const cafsDir = path.join(storeDir, 'v3/files') const cafsDir = path.join(storeDir, 'v3/files')
const cacheIntegrityPath = getIndexFilePathInCafs(cafsDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0')) const cacheIntegrityPath = getIndexFilePathInCafs(cafsDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0')
let cacheIntegrity = loadJsonFile.sync<any>(cacheIntegrityPath) // eslint-disable-line @typescript-eslint/no-explicit-any let cacheIntegrity = loadJsonFile.sync<any>(cacheIntegrityPath) // eslint-disable-line @typescript-eslint/no-explicit-any
const sideEffectsKey = `${ENGINE_NAME}-${hashObject({ '@pnpm.e2e/hello-world-js-bin@1.0.0': {} })}` const sideEffectsKey = `${ENGINE_NAME}-${hashObject({ '@pnpm.e2e/hello-world-js-bin@1.0.0': {} })}`
cacheIntegrity.sideEffects = { cacheIntegrity.sideEffects = {

View File

@@ -180,9 +180,10 @@ export function createFuseHandlersFromLockfile (lockfile: Lockfile, cafsDir: str
if (!pkgSnapshotCache.has(depPath)) { if (!pkgSnapshotCache.has(depPath)) {
const pkgSnapshot = lockfile.packages?.[depPath as DepPath] const pkgSnapshot = lockfile.packages?.[depPath as DepPath]
if (pkgSnapshot == null) return undefined if (pkgSnapshot == null) return undefined
const indexPath = getIndexFilePathInCafs(cafsDir, (pkgSnapshot.resolution as TarballResolution).integrity!) const nameVer = nameVerFromPkgSnapshot(depPath, pkgSnapshot)
const indexPath = getIndexFilePathInCafs(cafsDir, (pkgSnapshot.resolution as TarballResolution).integrity!, `${nameVer.name}@${nameVer.version}`)
pkgSnapshotCache.set(depPath, { pkgSnapshotCache.set(depPath, {
...nameVerFromPkgSnapshot(depPath, pkgSnapshot), ...nameVer,
pkgSnapshot, pkgSnapshot,
index: loadJsonFile.sync<PackageFilesIndex>(indexPath), // TODO: maybe make it async? index: loadJsonFile.sync<PackageFilesIndex>(indexPath), // TODO: maybe make it async?
}) })

View File

@@ -34,13 +34,13 @@ test('installation breaks if the lockfile contains the wrong checksum', async ()
manifest, manifest,
mutation: 'install', mutation: 'install',
rootDir: process.cwd() as ProjectRootDir, rootDir: process.cwd() as ProjectRootDir,
}, testDefaults({ frozenLockfile: true }))).rejects.toThrowError(/Package name mismatch found while reading/) }, testDefaults({ frozenLockfile: true }, { retry: { retries: 0 } }))).rejects.toThrowError(/Got unexpected checksum for/)
await mutateModulesInSingleProject({ await mutateModulesInSingleProject({
manifest, manifest,
mutation: 'install', mutation: 'install',
rootDir: process.cwd() as ProjectRootDir, rootDir: process.cwd() as ProjectRootDir,
}, testDefaults()) }, testDefaults({}, { retry: { retries: 0 } }))
expect(project.readLockfile()).toStrictEqual(correctLockfile) expect(project.readLockfile()).toStrictEqual(correctLockfile)
@@ -53,7 +53,7 @@ test('installation breaks if the lockfile contains the wrong checksum', async ()
manifest, manifest,
mutation: 'install', mutation: 'install',
rootDir: process.cwd() as ProjectRootDir, rootDir: process.cwd() as ProjectRootDir,
}, testDefaults({ preferFrozenLockfile: false })) }, testDefaults({ preferFrozenLockfile: false }, { retry: { retries: 0 } }))
expect(project.readLockfile()).toStrictEqual(correctLockfile) expect(project.readLockfile()).toStrictEqual(correctLockfile)
}) })

View File

@@ -42,7 +42,7 @@ test('patch package', async () => {
}) })
expect(lockfile.snapshots[`is-positive@1.0.0(patch_hash=${patchFileHash})`]).toBeTruthy() expect(lockfile.snapshots[`is-positive@1.0.0(patch_hash=${patchFileHash})`]).toBeTruthy()
const filesIndexFile = path.join(opts.storeDir, 'files/c7/1ccf199e0fdae37aad13946b937d67bcd35fa111b84d21b3a19439cfdc2812c5d8da8a735e94c2a1ccb77b4583808ee8405313951e7146ac83ede3671dc292-index.json') const filesIndexFile = path.join(opts.storeDir, 'files/c7/1ccf199e0fdae37aad13946b937d67bcd35fa111b84d21b3a19439cfdc2812-is-positive@1.0.0.json')
const filesIndex = loadJsonFile.sync<PackageFilesIndex>(filesIndexFile) const filesIndex = loadJsonFile.sync<PackageFilesIndex>(filesIndexFile)
const sideEffectsKey = `${ENGINE_NAME}-${patchFileHash}` const sideEffectsKey = `${ENGINE_NAME}-${patchFileHash}`
const patchedFileIntegrity = filesIndex.sideEffects?.[sideEffectsKey]['index.js']?.integrity const patchedFileIntegrity = filesIndex.sideEffects?.[sideEffectsKey]['index.js']?.integrity
@@ -209,7 +209,7 @@ test('patch package when scripts are ignored', async () => {
}) })
expect(lockfile.snapshots[`is-positive@1.0.0(patch_hash=${patchFileHash})`]).toBeTruthy() expect(lockfile.snapshots[`is-positive@1.0.0(patch_hash=${patchFileHash})`]).toBeTruthy()
const filesIndexFile = path.join(opts.storeDir, 'files/c7/1ccf199e0fdae37aad13946b937d67bcd35fa111b84d21b3a19439cfdc2812c5d8da8a735e94c2a1ccb77b4583808ee8405313951e7146ac83ede3671dc292-index.json') const filesIndexFile = path.join(opts.storeDir, 'files/c7/1ccf199e0fdae37aad13946b937d67bcd35fa111b84d21b3a19439cfdc2812-is-positive@1.0.0.json')
const filesIndex = loadJsonFile.sync<PackageFilesIndex>(filesIndexFile) const filesIndex = loadJsonFile.sync<PackageFilesIndex>(filesIndexFile)
const sideEffectsKey = `${ENGINE_NAME}-${patchFileHash}` const sideEffectsKey = `${ENGINE_NAME}-${patchFileHash}`
const patchedFileIntegrity = filesIndex.sideEffects?.[sideEffectsKey]['index.js']?.integrity const patchedFileIntegrity = filesIndex.sideEffects?.[sideEffectsKey]['index.js']?.integrity
@@ -296,7 +296,7 @@ test('patch package when the package is not in onlyBuiltDependencies list', asyn
}) })
expect(lockfile.snapshots[`is-positive@1.0.0(patch_hash=${patchFileHash})`]).toBeTruthy() expect(lockfile.snapshots[`is-positive@1.0.0(patch_hash=${patchFileHash})`]).toBeTruthy()
const filesIndexFile = path.join(opts.storeDir, 'files/c7/1ccf199e0fdae37aad13946b937d67bcd35fa111b84d21b3a19439cfdc2812c5d8da8a735e94c2a1ccb77b4583808ee8405313951e7146ac83ede3671dc292-index.json') const filesIndexFile = path.join(opts.storeDir, 'files/c7/1ccf199e0fdae37aad13946b937d67bcd35fa111b84d21b3a19439cfdc2812-is-positive@1.0.0.json')
const filesIndex = loadJsonFile.sync<PackageFilesIndex>(filesIndexFile) const filesIndex = loadJsonFile.sync<PackageFilesIndex>(filesIndexFile)
const sideEffectsKey = `${ENGINE_NAME}-${patchFileHash}` const sideEffectsKey = `${ENGINE_NAME}-${patchFileHash}`
const patchedFileIntegrity = filesIndex.sideEffects?.[sideEffectsKey]['index.js']?.integrity const patchedFileIntegrity = filesIndex.sideEffects?.[sideEffectsKey]['index.js']?.integrity

View File

@@ -83,7 +83,7 @@ test('using side effects cache', async () => {
const manifest = await addDependenciesToPackage({}, ['@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0'], opts) const manifest = await addDependenciesToPackage({}, ['@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0'], opts)
const cafsDir = path.join(opts.storeDir, 'files') const cafsDir = path.join(opts.storeDir, 'files')
const filesIndexFile = getIndexFilePathInCafs(cafsDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0')) const filesIndexFile = getIndexFilePathInCafs(cafsDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0')
const filesIndex = loadJsonFile.sync<PackageFilesIndex>(filesIndexFile) const filesIndex = loadJsonFile.sync<PackageFilesIndex>(filesIndexFile)
expect(filesIndex.sideEffects).toBeTruthy() // files index has side effects expect(filesIndex.sideEffects).toBeTruthy() // files index has side effects
const sideEffectsKey = `${ENGINE_NAME}-${hashObject({ '@pnpm.e2e/hello-world-js-bin@1.0.0': {} })}` const sideEffectsKey = `${ENGINE_NAME}-${hashObject({ '@pnpm.e2e/hello-world-js-bin@1.0.0': {} })}`
@@ -157,7 +157,7 @@ test('uploading errors do not interrupt installation', async () => {
expect(fs.existsSync('node_modules/@pnpm.e2e/pre-and-postinstall-scripts-example/generated-by-postinstall.js')).toBeTruthy() expect(fs.existsSync('node_modules/@pnpm.e2e/pre-and-postinstall-scripts-example/generated-by-postinstall.js')).toBeTruthy()
const cafsDir = path.join(opts.storeDir, 'files') const cafsDir = path.join(opts.storeDir, 'files')
const filesIndexFile = getIndexFilePathInCafs(cafsDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0')) const filesIndexFile = getIndexFilePathInCafs(cafsDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0')
const filesIndex = loadJsonFile.sync<PackageFilesIndex>(filesIndexFile) const filesIndex = loadJsonFile.sync<PackageFilesIndex>(filesIndexFile)
expect(filesIndex.sideEffects).toBeFalsy() expect(filesIndex.sideEffects).toBeFalsy()
}) })
@@ -175,7 +175,7 @@ test('a postinstall script does not modify the original sources added to the sto
expect(fs.readFileSync('node_modules/@pnpm/postinstall-modifies-source/empty-file.txt', 'utf8')).toContain('hello') expect(fs.readFileSync('node_modules/@pnpm/postinstall-modifies-source/empty-file.txt', 'utf8')).toContain('hello')
const cafsDir = path.join(opts.storeDir, 'files') const cafsDir = path.join(opts.storeDir, 'files')
const filesIndexFile = getIndexFilePathInCafs(cafsDir, getIntegrity('@pnpm/postinstall-modifies-source', '1.0.0')) const filesIndexFile = getIndexFilePathInCafs(cafsDir, getIntegrity('@pnpm/postinstall-modifies-source', '1.0.0'), '@pnpm/postinstall-modifies-source@1.0.0')
const filesIndex = loadJsonFile.sync<PackageFilesIndex>(filesIndexFile) const filesIndex = loadJsonFile.sync<PackageFilesIndex>(filesIndexFile)
const patchedFileIntegrity = filesIndex.sideEffects?.[`${ENGINE_NAME}-${hashObject({})}`]['empty-file.txt']?.integrity const patchedFileIntegrity = filesIndex.sideEffects?.[`${ENGINE_NAME}-${hashObject({})}`]['empty-file.txt']?.integrity
expect(patchedFileIntegrity).toBeTruthy() expect(patchedFileIntegrity).toBeTruthy()
@@ -198,7 +198,7 @@ test('a corrupted side-effects cache is ignored', async () => {
const manifest = await addDependenciesToPackage({}, ['@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0'], opts) const manifest = await addDependenciesToPackage({}, ['@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0'], opts)
const cafsDir = path.join(opts.storeDir, 'files') const cafsDir = path.join(opts.storeDir, 'files')
const filesIndexFile = getIndexFilePathInCafs(cafsDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0')) const filesIndexFile = getIndexFilePathInCafs(cafsDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0')
const filesIndex = loadJsonFile.sync<PackageFilesIndex>(filesIndexFile) const filesIndex = loadJsonFile.sync<PackageFilesIndex>(filesIndexFile)
expect(filesIndex.sideEffects).toBeTruthy() // files index has side effects expect(filesIndex.sideEffects).toBeTruthy() // files index has side effects
const sideEffectsKey = `${ENGINE_NAME}-${hashObject({ '@pnpm.e2e/hello-world-js-bin@1.0.0': {} })}` const sideEffectsKey = `${ENGINE_NAME}-${hashObject({ '@pnpm.e2e/hello-world-js-bin@1.0.0': {} })}`

View File

@@ -678,7 +678,7 @@ test.each([['isolated'], ['hoisted']])('using side effects cache with nodeLinker
await headlessInstall(opts) await headlessInstall(opts)
const cafsDir = path.join(opts.storeDir, 'files') const cafsDir = path.join(opts.storeDir, 'files')
const cacheIntegrityPath = getIndexFilePathInCafs(cafsDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0')) const cacheIntegrityPath = getIndexFilePathInCafs(cafsDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0')
const cacheIntegrity = loadJsonFile.sync<any>(cacheIntegrityPath) // eslint-disable-line @typescript-eslint/no-explicit-any const cacheIntegrity = loadJsonFile.sync<any>(cacheIntegrityPath) // eslint-disable-line @typescript-eslint/no-explicit-any
expect(cacheIntegrity!.sideEffects).toBeTruthy() expect(cacheIntegrity!.sideEffects).toBeTruthy()
const sideEffectsKey = `${ENGINE_NAME}-${hashObject({ '@pnpm.e2e/hello-world-js-bin@1.0.0': {} })}` const sideEffectsKey = `${ENGINE_NAME}-${hashObject({ '@pnpm.e2e/hello-world-js-bin@1.0.0': {} })}`

View File

@@ -308,7 +308,7 @@ interface FetchLock {
function getFilesIndexFilePath ( function getFilesIndexFilePath (
ctx: { ctx: {
getIndexFilePathInCafs: (integrity: string) => string getIndexFilePathInCafs: (integrity: string, pkgId: string) => string
storeDir: string storeDir: string
virtualStoreDirMaxLength: number virtualStoreDirMaxLength: number
}, },
@@ -317,7 +317,7 @@ function getFilesIndexFilePath (
const targetRelative = depPathToFilename(opts.pkg.id, ctx.virtualStoreDirMaxLength) const targetRelative = depPathToFilename(opts.pkg.id, ctx.virtualStoreDirMaxLength)
const target = path.join(ctx.storeDir, targetRelative) const target = path.join(ctx.storeDir, targetRelative)
const filesIndexFile = (opts.pkg.resolution as TarballResolution).integrity const filesIndexFile = (opts.pkg.resolution as TarballResolution).integrity
? ctx.getIndexFilePathInCafs((opts.pkg.resolution as TarballResolution).integrity!) ? ctx.getIndexFilePathInCafs((opts.pkg.resolution as TarballResolution).integrity!, opts.pkg.id)
: path.join(target, opts.ignoreScripts ? 'integrity-not-built.json' : 'integrity.json') : path.join(target, opts.ignoreScripts ? 'integrity-not-built.json' : 'integrity.json')
return { filesIndexFile, target } return { filesIndexFile, target }
} }
@@ -334,7 +334,7 @@ function fetchToStore (
opts: FetchOptions opts: FetchOptions
) => Promise<FetchResult> ) => Promise<FetchResult>
fetchingLocker: Map<string, FetchLock> fetchingLocker: Map<string, FetchLock>
getIndexFilePathInCafs: (integrity: string) => string getIndexFilePathInCafs: (integrity: string, pkgId: string) => string
getFilePathByModeInCafs: (integrity: string, mode: number) => string getFilePathByModeInCafs: (integrity: string, mode: number) => string
requestsQueue: { requestsQueue: {
add: <T>(fn: () => Promise<T>, opts: { priority: number }) => Promise<T> add: <T>(fn: () => Promise<T>, opts: { priority: number }) => Promise<T>

View File

@@ -519,7 +519,7 @@ test('installation fails when the stored package name and version do not match t
await execPnpm(['add', '@pnpm.e2e/dep-of-pkg-with-1-dep@100.1.0', ...settings]) await execPnpm(['add', '@pnpm.e2e/dep-of-pkg-with-1-dep@100.1.0', ...settings])
const cafsDir = path.join(storeDir, 'v3/files') const cafsDir = path.join(storeDir, 'v3/files')
const cacheIntegrityPath = getIndexFilePathInCafs(cafsDir, getIntegrity('@pnpm.e2e/dep-of-pkg-with-1-dep', '100.1.0')) const cacheIntegrityPath = getIndexFilePathInCafs(cafsDir, getIntegrity('@pnpm.e2e/dep-of-pkg-with-1-dep', '100.1.0'), '@pnpm.e2e/dep-of-pkg-with-1-dep@100.1.0')
const cacheIntegrity = loadJsonFile.sync<any>(cacheIntegrityPath) // eslint-disable-line @typescript-eslint/no-explicit-any const cacheIntegrity = loadJsonFile.sync<any>(cacheIntegrityPath) // eslint-disable-line @typescript-eslint/no-explicit-any
cacheIntegrity.name = 'foo' cacheIntegrity.name = 'foo'
writeJsonFile.sync(cacheIntegrityPath, { writeJsonFile.sync(cacheIntegrityPath, {

View File

@@ -257,10 +257,12 @@ export async function readPackageIndexFile (
let pkgIndexFilePath let pkgIndexFilePath
if (isPackageWithIntegrity) { if (isPackageWithIntegrity) {
const parsedId = parse(id)
// Retrieve all the index file of all files included in the package // Retrieve all the index file of all files included in the package
pkgIndexFilePath = getIndexFilePathInCafs( pkgIndexFilePath = getIndexFilePathInCafs(
opts.cafsDir, opts.cafsDir,
packageResolution.integrity as string packageResolution.integrity as string,
`${parsedId.name}@${parsedId.version}`
) )
} else if (!packageResolution.type && packageResolution.tarball) { } else if (!packageResolution.type && packageResolution.tarball) {
const packageDirInStore = depPathToFilename(parse(id).nonSemverVersion ?? id, opts.virtualStoreDirMaxLength) const packageDirInStore = depPathToFilename(parse(id).nonSemverVersion ?? id, opts.virtualStoreDirMaxLength)
@@ -286,7 +288,7 @@ export async function readPackageIndexFile (
if (err.code === 'ENOENT') { if (err.code === 'ENOENT') {
throw new PnpmError( throw new PnpmError(
'MISSING_PACKAGE_INDEX_FILE', 'MISSING_PACKAGE_INDEX_FILE',
`Failed to find package index file for ${id}, please consider running 'pnpm install'` `Failed to find package index file for ${id} (at ${pkgIndexFilePath}), please consider running 'pnpm install'`
) )
} }

View File

@@ -1,3 +1,4 @@
import path from 'path'
import { getPkgInfo } from '../lib/getPkgInfo' import { getPkgInfo } from '../lib/getPkgInfo'
export const DEFAULT_REGISTRIES = { export const DEFAULT_REGISTRIES = {
@@ -11,8 +12,8 @@ describe('licences', () => {
{ {
name: 'bogus-package', name: 'bogus-package',
version: '1.0.0', version: '1.0.0',
id: '/bogus-package@1.0.0', id: 'bogus-package@1.0.0',
depPath: '/bogus-package@1.0.0', depPath: 'bogus-package@1.0.0',
snapshot: { snapshot: {
resolution: { resolution: {
integrity: 'integrity-sha', integrity: 'integrity-sha',
@@ -28,6 +29,6 @@ describe('licences', () => {
virtualStoreDirMaxLength: 120, virtualStoreDirMaxLength: 120,
} }
) )
).rejects.toThrow('Failed to find package index file for /bogus-package@1.0.0, please consider running \'pnpm install\'') ).rejects.toThrow(`Failed to find package index file for bogus-package@1.0.0 (at ${path.join('store-dir', 'files', 'b2', '16-bogus-package@1.0.0.json')}), please consider running 'pnpm install'`)
}) })
}) })

View File

@@ -16,9 +16,17 @@ export function getFilePathByModeInCafs (
export function getIndexFilePathInCafs ( export function getIndexFilePathInCafs (
cafsDir: string, cafsDir: string,
integrity: string | IntegrityLike integrity: string | IntegrityLike,
pkgId: string
): string { ): string {
return path.join(cafsDir, contentPathFromIntegrity(integrity, 'index')) const hex = ssri.parse(integrity, { single: true }).hexDigest().substring(0, 64)
// Some registries allow identical content to be published under different package names or versions.
// To accommodate this, index files are stored using both the content hash and package identifier.
// This approach ensures that we can:
// 1. Validate that the integrity in the lockfile corresponds to the correct package,
// which might not be the case after a poorly resolved Git conflict.
// 2. Allow the same content to be referenced by different packages or different versions of the same package.
return path.join(cafsDir, `${path.join(hex.slice(0, 2), hex.slice(2))}-${pkgId.replace(/[\\/:*?"<>|]/g, '+')}.json`)
} }
function contentPathFromIntegrity ( function contentPathFromIntegrity (

View File

@@ -32,7 +32,7 @@ export async function prune ({ cacheDir, storeDir }: PruneOptions, removeAlienFi
const subdir = path.join(cafsDir, dir) const subdir = path.join(cafsDir, dir)
await Promise.all((await fs.readdir(subdir)).map(async (fileName) => { await Promise.all((await fs.readdir(subdir)).map(async (fileName) => {
const filePath = path.join(subdir, fileName) const filePath = path.join(subdir, fileName)
if (fileName.endsWith('-index.json')) { if (fileName.endsWith('.json')) {
pkgIndexFiles.push(filePath) pkgIndexFiles.push(filePath)
return return
} }

View File

@@ -85,7 +85,8 @@ export async function handler (opts: CatIndexCommandOptions, params: string[]):
const filesIndexFile = getIndexFilePathInCafs( const filesIndexFile = getIndexFilePathInCafs(
cafsDir, cafsDir,
(pkgSnapshot.resolution as TarballResolution).integrity!.toString() (pkgSnapshot.resolution as TarballResolution).integrity!.toString(),
`${alias}@${pref}`
) )
try { try {
const pkgFilesIndex = await loadJsonFile<PackageFilesIndex>(filesIndexFile) const pkgFilesIndex = await loadJsonFile<PackageFilesIndex>(filesIndexFile)

View File

@@ -55,7 +55,7 @@ export async function handler (opts: FindHashCommandOptions, params: string[]):
cafsChildrenDirs.forEach(({ name: dirName }) => { cafsChildrenDirs.forEach(({ name: dirName }) => {
const dirIndexFiles = fs const dirIndexFiles = fs
.readdirSync(`${cafsDir}/${dirName}`) .readdirSync(`${cafsDir}/${dirName}`)
.filter((fileName) => fileName.includes('-index.json')) .filter((fileName) => fileName.includes('.json'))
?.map((fileName) => `${cafsDir}/${dirName}/${fileName}`) ?.map((fileName) => `${cafsDir}/${dirName}/${fileName}`)
indexFiles.push(...dirIndexFiles) indexFiles.push(...dirIndexFiles)

View File

@@ -26,8 +26,8 @@ test('print index file path with hash', async () => {
storeDir, storeDir,
}, ['sha512-fXs1pWlUdqT2jkeoEJW/+odKZ2NwAyYkWea+plJKZI2xmhRKQi2e+nKGcClyDblgLwCLD912oMaua0+sTwwIrw==']) }, ['sha512-fXs1pWlUdqT2jkeoEJW/+odKZ2NwAyYkWea+plJKZI2xmhRKQi2e+nKGcClyDblgLwCLD912oMaua0+sTwwIrw=='])
expect(output).toBe(`${PACKAGE_INFO_CLR('lodash')}@${PACKAGE_INFO_CLR('4.17.19')} ${INDEX_PATH_CLR('/24/dbddf17111f46417d2fdaa260b1a37f9b3142340e4145efe3f0937d77eb56c862d2a1d2901ca16271dc0d6335b0237c2346768a3ec1a3d579018f1fc5f7a0d-index.json')} expect(output).toBe(`${PACKAGE_INFO_CLR('lodash')}@${PACKAGE_INFO_CLR('4.17.19')} ${INDEX_PATH_CLR('/24/dbddf17111f46417d2fdaa260b1a37f9b3142340e4145efe3f0937d77eb56c-lodash@4.17.19.json')}
${PACKAGE_INFO_CLR('lodash')}@${PACKAGE_INFO_CLR('4.17.20')} ${INDEX_PATH_CLR('/3e/585d15c8a594e20d7de57b362ea81754c011acb2641a19f1b72c8531ea39825896bab344ae616a0a5a824cb9a381df0b3cddd534645cf305aba70a93dac698-index.json')} ${PACKAGE_INFO_CLR('lodash')}@${PACKAGE_INFO_CLR('4.17.20')} ${INDEX_PATH_CLR('/3e/585d15c8a594e20d7de57b362ea81754c011acb2641a19f1b72c8531ea3982-lodash@4.17.20.json')}
`) `)
} }
}) })
@@ -49,4 +49,4 @@ test('print index file path with hash error', async () => {
expect(err.code).toBe('ERR_PNPM_INVALID_FILE_HASH') expect(err.code).toBe('ERR_PNPM_INVALID_FILE_HASH')
expect(err.message).toBe('No package or index file matching this hash was found.') expect(err.message).toBe('No package or index file matching this hash was found.')
}) })

View File

@@ -51,7 +51,7 @@ export async function storeStatus (maybeOpts: StoreStatusOptions): Promise<strin
const cafsDir = path.join(storeDir, 'files') const cafsDir = path.join(storeDir, 'files')
const modified = await pFilter(pkgs, async ({ id, integrity, depPath, name }) => { const modified = await pFilter(pkgs, async ({ id, integrity, depPath, name }) => {
const pkgIndexFilePath = integrity const pkgIndexFilePath = integrity
? getIndexFilePathInCafs(cafsDir, integrity) ? getIndexFilePathInCafs(cafsDir, integrity, id)
: path.join(storeDir, dp.depPathToFilename(id, maybeOpts.virtualStoreDirMaxLength), 'integrity.json') : path.join(storeDir, dp.depPathToFilename(id, maybeOpts.virtualStoreDirMaxLength), 'integrity.json')
const { files } = await loadJsonFile<PackageFilesIndex>(pkgIndexFilePath) const { files } = await loadJsonFile<PackageFilesIndex>(pkgIndexFilePath)
return (await dint.check(path.join(virtualStoreDir, dp.depPathToFilename(depPath, maybeOpts.virtualStoreDirMaxLength), 'node_modules', name), files)) === false return (await dint.check(path.join(virtualStoreDir, dp.depPathToFilename(depPath, maybeOpts.virtualStoreDirMaxLength), 'node_modules', name), files)) === false

View File

@@ -28,7 +28,7 @@ test('pnpm store add express@4.16.3', async () => {
}, ['add', 'express@4.16.3']) }, ['add', 'express@4.16.3'])
const { cafsHas } = assertStore(path.join(storeDir, STORE_VERSION)) const { cafsHas } = assertStore(path.join(storeDir, STORE_VERSION))
cafsHas('sha512-CDaOBMB9knI6vx9SpIxEMOJ6VBbC2U/tYNILs0qv1YOZc15K9U2EcF06v10F0JX6IYcWnKYZJwIDJspEHLvUaQ==') cafsHas('express', '4.16.3')
}) })
test('pnpm store add scoped package that uses not the standard registry', async () => { test('pnpm store add scoped package that uses not the standard registry', async () => {

View File

@@ -7,7 +7,6 @@ import { prepare, prepareEmpty } from '@pnpm/prepare'
import { REGISTRY_MOCK_PORT } from '@pnpm/registry-mock' import { REGISTRY_MOCK_PORT } from '@pnpm/registry-mock'
import { sync as rimraf } from '@zkochan/rimraf' import { sync as rimraf } from '@zkochan/rimraf'
import execa from 'execa' import execa from 'execa'
import ssri from 'ssri'
const STORE_VERSION = 'v3' const STORE_VERSION = 'v3'
const REGISTRY = `http://localhost:${REGISTRY_MOCK_PORT}/` const REGISTRY = `http://localhost:${REGISTRY_MOCK_PORT}/`
@@ -98,7 +97,7 @@ test.skip('remove packages that are used by project that no longer exist', async
rimraf('node_modules') rimraf('node_modules')
cafsHas(ssri.fromHex('f0d86377aa15a64c34961f38ac2a9be2b40a1187', 'sha1').toString()) cafsHas('is-negative', '2.1.0')
const reporter = jest.fn() const reporter = jest.fn()
await store.handler({ await store.handler({
@@ -123,7 +122,7 @@ test.skip('remove packages that are used by project that no longer exist', async
}) })
) )
cafsHasNot(ssri.fromHex('f0d86377aa15a64c34961f38ac2a9be2b40a1187', 'sha1').toString()) cafsHasNot('is-negative', '2.1.0')
}) })
test('keep dependencies used by others', async () => { test('keep dependencies used by others', async () => {