feat!: don't duplicate all file entries in the side effects section of the index file (#8636)

This commit is contained in:
Zoltan Kochan
2024-10-14 11:01:23 +02:00
committed by GitHub
parent b33f153b36
commit 099e6af9e1
14 changed files with 121 additions and 51 deletions

View File

@@ -0,0 +1,11 @@
---
"@pnpm/plugin-commands-store-inspecting": major
"@pnpm/headless": major
"@pnpm/core": major
"@pnpm/cafs-types": major
"@pnpm/store.cafs": major
"@pnpm/worker": major
"pnpm": major
---
Changed the structure of the index files in the store to store side effects cache information more efficiently. In the new version, side effects do not list all the files of the package but just the differences [#8636](https://github.com/pnpm/pnpm/pull/8636).

View File

@@ -79,8 +79,8 @@ test('rebuilds dependencies', async () => {
const cacheIntegrity = loadJsonFile.sync<any>(cacheIntegrityPath) // eslint-disable-line @typescript-eslint/no-explicit-any
expect(cacheIntegrity!.sideEffects).toBeTruthy()
const sideEffectsKey = `${ENGINE_NAME}-${hashObject({ '@pnpm.e2e/hello-world-js-bin@1.0.0': {} })}`
expect(cacheIntegrity).toHaveProperty(['sideEffects', sideEffectsKey, 'generated-by-postinstall.js'])
delete cacheIntegrity!.sideEffects[sideEffectsKey]['generated-by-postinstall.js']
expect(cacheIntegrity).toHaveProperty(['sideEffects', sideEffectsKey, 'added', 'generated-by-postinstall.js'])
delete cacheIntegrity!.sideEffects[sideEffectsKey].added['generated-by-postinstall.js']
})
test('skipIfHasSideEffectsCache', async () => {
@@ -104,7 +104,7 @@ test('skipIfHasSideEffectsCache', async () => {
let cacheIntegrity = loadJsonFile.sync<any>(cacheIntegrityPath) // eslint-disable-line @typescript-eslint/no-explicit-any
const sideEffectsKey = `${ENGINE_NAME}-${hashObject({ '@pnpm.e2e/hello-world-js-bin@1.0.0': {} })}`
cacheIntegrity.sideEffects = {
[sideEffectsKey]: { foo: 'bar' },
[sideEffectsKey]: { added: { foo: 'bar' } },
}
fs.writeFileSync(cacheIntegrityPath, JSON.stringify(cacheIntegrity, null, 2), 'utf8')
@@ -130,7 +130,7 @@ test('skipIfHasSideEffectsCache', async () => {
cacheIntegrity = loadJsonFile.sync<any>(cacheIntegrityPath) // eslint-disable-line @typescript-eslint/no-explicit-any
expect(cacheIntegrity!.sideEffects).toBeTruthy()
expect(cacheIntegrity).toHaveProperty(['sideEffects', sideEffectsKey, 'foo'])
expect(cacheIntegrity).toHaveProperty(['sideEffects', sideEffectsKey, 'added', 'foo'])
})
test('rebuild does not fail when a linked package is present', async () => {

View File

@@ -46,7 +46,7 @@ test('patch package', async () => {
const filesIndexFile = path.join(opts.storeDir, 'files/c7/1ccf199e0fdae37aad13946b937d67bcd35fa111b84d21b3a19439cfdc2812-is-positive@1.0.0.json')
const filesIndex = loadJsonFile.sync<PackageFilesIndex>(filesIndexFile)
const sideEffectsKey = `${ENGINE_NAME}-${patchFileHash}`
const patchedFileIntegrity = filesIndex.sideEffects?.[sideEffectsKey]['index.js']?.integrity
const patchedFileIntegrity = filesIndex.sideEffects?.[sideEffectsKey].added?.['index.js']?.integrity
expect(patchedFileIntegrity).toBeTruthy()
const originalFileIntegrity = filesIndex.files['index.js'].integrity
expect(originalFileIntegrity).toBeTruthy()
@@ -213,7 +213,7 @@ test('patch package when scripts are ignored', async () => {
const filesIndexFile = path.join(opts.storeDir, 'files/c7/1ccf199e0fdae37aad13946b937d67bcd35fa111b84d21b3a19439cfdc2812-is-positive@1.0.0.json')
const filesIndex = loadJsonFile.sync<PackageFilesIndex>(filesIndexFile)
const sideEffectsKey = `${ENGINE_NAME}-${patchFileHash}`
const patchedFileIntegrity = filesIndex.sideEffects?.[sideEffectsKey]['index.js']?.integrity
const patchedFileIntegrity = filesIndex.sideEffects?.[sideEffectsKey].added?.['index.js']?.integrity
expect(patchedFileIntegrity).toBeTruthy()
const originalFileIntegrity = filesIndex.files['index.js'].integrity
expect(originalFileIntegrity).toBeTruthy()
@@ -300,7 +300,7 @@ test('patch package when the package is not in onlyBuiltDependencies list', asyn
const filesIndexFile = path.join(opts.storeDir, 'files/c7/1ccf199e0fdae37aad13946b937d67bcd35fa111b84d21b3a19439cfdc2812-is-positive@1.0.0.json')
const filesIndex = loadJsonFile.sync<PackageFilesIndex>(filesIndexFile)
const sideEffectsKey = `${ENGINE_NAME}-${patchFileHash}`
const patchedFileIntegrity = filesIndex.sideEffects?.[sideEffectsKey]['index.js']?.integrity
const patchedFileIntegrity = filesIndex.sideEffects?.[sideEffectsKey].added?.['index.js']?.integrity
expect(patchedFileIntegrity).toBeTruthy()
const originalFileIntegrity = filesIndex.files['index.js'].integrity
expect(originalFileIntegrity).toBeTruthy()

View File

@@ -87,9 +87,9 @@ test('using side effects cache', async () => {
const filesIndex = loadJsonFile.sync<PackageFilesIndex>(filesIndexFile)
expect(filesIndex.sideEffects).toBeTruthy() // files index has side effects
const sideEffectsKey = `${ENGINE_NAME}-${hashObject({ '@pnpm.e2e/hello-world-js-bin@1.0.0': {} })}`
expect(filesIndex.sideEffects).toHaveProperty([sideEffectsKey, 'generated-by-preinstall.js'])
expect(filesIndex.sideEffects).toHaveProperty([sideEffectsKey, 'generated-by-postinstall.js'])
delete filesIndex.sideEffects![sideEffectsKey]['generated-by-postinstall.js']
expect(filesIndex.sideEffects).toHaveProperty([sideEffectsKey, 'added', 'generated-by-preinstall.js'])
expect(filesIndex.sideEffects).toHaveProperty([sideEffectsKey, 'added', 'generated-by-postinstall.js'])
delete filesIndex.sideEffects![sideEffectsKey].added?.['generated-by-postinstall.js']
writeJsonFile.sync(filesIndexFile, filesIndex)
rimraf('node_modules')
@@ -177,7 +177,7 @@ test('a postinstall script does not modify the original sources added to the sto
const cafsDir = path.join(opts.storeDir, 'files')
const filesIndexFile = getIndexFilePathInCafs(cafsDir, getIntegrity('@pnpm/postinstall-modifies-source', '1.0.0'), '@pnpm/postinstall-modifies-source@1.0.0')
const filesIndex = loadJsonFile.sync<PackageFilesIndex>(filesIndexFile)
const patchedFileIntegrity = filesIndex.sideEffects?.[`${ENGINE_NAME}-${hashObject({})}`]['empty-file.txt']?.integrity
const patchedFileIntegrity = filesIndex.sideEffects?.[`${ENGINE_NAME}-${hashObject({})}`].added?.['empty-file.txt']?.integrity
expect(patchedFileIntegrity).toBeTruthy()
const originalFileIntegrity = filesIndex.files['empty-file.txt'].integrity
expect(originalFileIntegrity).toBeTruthy()
@@ -202,8 +202,8 @@ test('a corrupted side-effects cache is ignored', async () => {
const filesIndex = loadJsonFile.sync<PackageFilesIndex>(filesIndexFile)
expect(filesIndex.sideEffects).toBeTruthy() // files index has side effects
const sideEffectsKey = `${ENGINE_NAME}-${hashObject({ '@pnpm.e2e/hello-world-js-bin@1.0.0': {} })}`
expect(filesIndex.sideEffects).toHaveProperty([sideEffectsKey, 'generated-by-preinstall.js'])
const sideEffectFileStat = filesIndex.sideEffects![sideEffectsKey]['generated-by-preinstall.js']
expect(filesIndex.sideEffects).toHaveProperty([sideEffectsKey, 'added', 'generated-by-preinstall.js'])
const sideEffectFileStat = filesIndex.sideEffects![sideEffectsKey].added!['generated-by-preinstall.js']
const sideEffectFile = getFilePathByModeInCafs(cafsDir, sideEffectFileStat.integrity, sideEffectFileStat.mode)
expect(fs.existsSync(sideEffectFile)).toBeTruthy()
rimraf(sideEffectFile) // we remove the side effect file to break the store

View File

@@ -682,10 +682,10 @@ test.each([['isolated'], ['hoisted']])('using side effects cache with nodeLinker
const cacheIntegrity = loadJsonFile.sync<any>(cacheIntegrityPath) // eslint-disable-line @typescript-eslint/no-explicit-any
expect(cacheIntegrity!.sideEffects).toBeTruthy()
const sideEffectsKey = `${ENGINE_NAME}-${hashObject({ '@pnpm.e2e/hello-world-js-bin@1.0.0': {} })}`
expect(cacheIntegrity).toHaveProperty(['sideEffects', sideEffectsKey, 'generated-by-postinstall.js'])
delete cacheIntegrity!.sideEffects[sideEffectsKey]['generated-by-postinstall.js']
expect(cacheIntegrity).toHaveProperty(['sideEffects', sideEffectsKey, 'added', 'generated-by-postinstall.js'])
delete cacheIntegrity!.sideEffects[sideEffectsKey].added['generated-by-postinstall.js']
expect(cacheIntegrity).toHaveProperty(['sideEffects', sideEffectsKey, 'generated-by-preinstall.js'])
expect(cacheIntegrity).toHaveProperty(['sideEffects', sideEffectsKey, 'added', 'generated-by-preinstall.js'])
writeJsonFile.sync(cacheIntegrityPath, cacheIntegrity)
prefix = f.prepare('side-effects')

View File

@@ -8,6 +8,7 @@ import { type PackageManifest, type Registries } from '@pnpm/types'
import {
getFilePathByModeInCafs,
getIndexFilePathInCafs,
type PackageFiles,
type PackageFileInfo,
type PackageFilesIndex,
} from '@pnpm/store.cafs'
@@ -155,7 +156,7 @@ async function parseLicense (
manifest: PackageManifest
files:
| { local: true, files: Record<string, string> }
| { local: false, files: Record<string, PackageFileInfo> }
| { local: false, files: PackageFiles }
},
opts: { cafsDir: string }
): Promise<LicenseInfo> {
@@ -213,7 +214,7 @@ async function readLicenseFileFromCafs (cafsDir: string, { integrity, mode }: Pa
}
export type ReadPackageIndexFileResult =
| { local: false, files: Record<string, PackageFileInfo> }
| { local: false, files: PackageFiles }
| { local: true, files: Record<string, string> }
export interface ReadPackageIndexFileOptions {

View File

@@ -1,6 +1,8 @@
import type { IntegrityLike } from 'ssri'
import type { DependencyManifest } from '@pnpm/types'
export type PackageFiles = Record<string, PackageFileInfo>
export interface PackageFileInfo {
checkedAt?: number // Nullable for backward compatibility
integrity: string
@@ -8,19 +10,26 @@ export interface PackageFileInfo {
size: number
}
export type SideEffects = Record<string, SideEffectsDiff>
export interface SideEffectsDiff {
deleted?: string[]
added?: PackageFiles
}
export type ResolvedFrom = 'store' | 'local-dir' | 'remote'
export type PackageFilesResponse = {
resolvedFrom: ResolvedFrom
packageImportMethod?: 'auto' | 'hardlink' | 'copy' | 'clone' | 'clone-or-copy'
sideEffects?: Record<string, Record<string, PackageFileInfo>>
sideEffects?: SideEffects
requiresBuild: boolean
} & ({
unprocessed?: false
filesIndex: Record<string, string>
} | {
unprocessed: true
filesIndex: Record<string, PackageFileInfo>
filesIndex: PackageFiles
})
export interface ImportPackageOpts {

View File

@@ -48,10 +48,12 @@ export function addFilesFromDir (
if (opts.readManifest && relativePath === 'package.json') {
manifest = parseJsonBufferSync(buffer) as DependencyManifest
}
// Remove the file type information (regular file, directory, etc.) and leave just the permission bits (rwx for owner, group, and others)
const mode = stat.mode & 0o777
filesIndex[relativePath] = {
mode: stat.mode,
mode,
size: stat.size,
...addBuffer(buffer, stat.mode),
...addBuffer(buffer, mode),
}
}
return { manifest, filesIndex }
@@ -78,7 +80,9 @@ function findFiles (
for (const file of files) {
const relativeSubdir = `${relativeDir}${relativeDir ? '/' : ''}${file.name}`
if (file.isDirectory()) {
findFiles(filesList, path.join(dir, file.name), relativeSubdir)
if (relativeDir !== '' || file.name !== 'node_modules') {
findFiles(filesList, path.join(dir, file.name), relativeSubdir)
}
continue
}
const absolutePath = path.join(dir, file.name)

View File

@@ -1,6 +1,6 @@
import fs from 'fs'
import util from 'util'
import type { PackageFileInfo } from '@pnpm/cafs-types'
import { type PackageFiles, type PackageFileInfo, type SideEffects } from '@pnpm/cafs-types'
import gfs from '@pnpm/graceful-fs'
import { type DependencyManifest } from '@pnpm/types'
import rimraf from '@zkochan/rimraf'
@@ -20,8 +20,6 @@ export interface VerifyResult {
manifest?: DependencyManifest
}
export type SideEffects = Record<string, Record<string, PackageFileInfo>>
export interface PackageFilesIndex {
// name and version are nullable for backward compatibility
// the initial specs of pnpm store v3 did not require these fields.
@@ -31,7 +29,7 @@ export interface PackageFilesIndex {
version?: string
requiresBuild?: boolean
files: Record<string, PackageFileInfo>
files: PackageFiles
sideEffects?: SideEffects
}
@@ -51,10 +49,12 @@ export function checkPkgFilesIntegrity (
// We verify all side effects cache. We could optimize it to verify only the side effects cache
// that satisfies the current os/arch/platform.
// However, it likely won't make a big difference.
for (const [sideEffectName, files] of Object.entries(pkgIndex.sideEffects)) {
const { passed } = _checkFilesIntegrity(files)
if (!passed) {
delete pkgIndex.sideEffects![sideEffectName]
for (const [sideEffectName, { added }] of Object.entries(pkgIndex.sideEffects)) {
if (added) {
const { passed } = _checkFilesIntegrity(added)
if (!passed) {
delete pkgIndex.sideEffects![sideEffectName]
}
}
}
}
@@ -64,7 +64,7 @@ export function checkPkgFilesIntegrity (
function checkFilesIntegrity (
verifiedFilesCache: Set<string>,
cafsDir: string,
files: Record<string, PackageFileInfo>,
files: PackageFiles,
readManifest?: boolean
): VerifyResult {
let allVerified = true

View File

@@ -1,11 +1,10 @@
import { type AddToStoreResult, type FileWriteResult, type PackageFileInfo, type FilesIndex } from '@pnpm/cafs-types'
import { type AddToStoreResult, type FileWriteResult, type PackageFiles, type PackageFileInfo, type FilesIndex } from '@pnpm/cafs-types'
import ssri from 'ssri'
import { addFilesFromDir } from './addFilesFromDir'
import { addFilesFromTarball } from './addFilesFromTarball'
import {
checkPkgFilesIntegrity,
type PackageFilesIndex,
type SideEffects,
type VerifyResult,
} from './checkPkgFilesIntegrity'
import { readManifestFromStore } from './readManifestFromStore'
@@ -27,8 +26,8 @@ export {
getFilePathByModeInCafs,
getIndexFilePathInCafs,
type PackageFileInfo,
type PackageFiles,
type PackageFilesIndex,
type SideEffects,
optimisticRenameOverwrite,
type FilesIndex,
type VerifyResult,

View File

@@ -5,14 +5,13 @@ import {
createCafs,
getFilePathByModeInCafs,
} from '@pnpm/store.cafs'
import type { Cafs, PackageFilesResponse } from '@pnpm/cafs-types'
import { type Cafs, type PackageFilesResponse, type PackageFiles, type SideEffectsDiff } from '@pnpm/cafs-types'
import { createIndexedPkgImporter } from '@pnpm/fs.indexed-pkg-importer'
import {
type ImportIndexedPackage,
type ImportIndexedPackageAsync,
type ImportPackageFunction,
type ImportPackageFunctionAsync,
type PackageFileInfo,
} from '@pnpm/store-controller-types'
import memoize from 'mem'
import pathTemp from 'path-temp'
@@ -86,9 +85,9 @@ function getFlatMap (
targetEngine?: string
): { filesMap: Record<string, string>, isBuilt: boolean } {
let isBuilt!: boolean
let filesIndex!: Record<string, PackageFileInfo>
let filesIndex!: PackageFiles
if (targetEngine && ((filesResponse.sideEffects?.[targetEngine]) != null)) {
filesIndex = filesResponse.sideEffects?.[targetEngine]
filesIndex = applySideEffectsDiff(filesResponse.filesIndex as PackageFiles, filesResponse.sideEffects?.[targetEngine])
isBuilt = true
} else if (!filesResponse.unprocessed) {
return {
@@ -103,6 +102,16 @@ function getFlatMap (
return { filesMap, isBuilt }
}
function applySideEffectsDiff (baseFiles: PackageFiles, { added, deleted }: SideEffectsDiff): PackageFiles {
const filesWithSideEffects: PackageFiles = { ...added }
for (const fileName in baseFiles) {
if (!deleted?.includes(fileName) && !filesWithSideEffects[fileName]) {
filesWithSideEffects[fileName] = baseFiles[fileName]
}
}
return filesWithSideEffects
}
export function createCafsStore (
storeDir: string,
opts?: {

View File

@@ -74,8 +74,9 @@ export async function handler (opts: FindHashCommandOptions, params: string[]):
}
if (pkgFilesIndex?.sideEffects) {
for (const [, files] of Object.entries(pkgFilesIndex.sideEffects)) {
for (const [, file] of Object.entries(files)) {
for (const { added } of Object.values(pkgFilesIndex.sideEffects)) {
if (!added) continue
for (const file of Object.values(added)) {
if (file?.integrity === hash) {
result.push({ name: pkgFilesIndex.name ?? 'unknown', version: pkgFilesIndex?.version ?? 'unknown', filesIndexFile: filesIndexFile.replace(cafsDir, '') })

View File

@@ -164,7 +164,13 @@ test('server upload', async () => {
const storeCtrl = await connectStoreController({ remotePrefix, concurrency: 100 })
const fakeEngine = 'client-engine'
const filesIndexFile = path.join(storeDir, 'test.example.com/fake-pkg/1.0.0.json')
const filesIndexFile = path.join(storeDir, 'fake-pkg@1.0.0.json')
fs.writeFileSync(filesIndexFile, JSON.stringify({
name: 'fake-pkg',
version: '1.0.0',
files: {},
}), 'utf8')
await storeCtrl.upload(path.join(__dirname, 'side-effect-fake-dir'), {
sideEffectsCacheKey: fakeEngine,
@@ -172,7 +178,7 @@ test('server upload', async () => {
})
const cacheIntegrity = loadJsonFile.sync<any>(filesIndexFile) // eslint-disable-line @typescript-eslint/no-explicit-any
expect(Object.keys(cacheIntegrity?.['sideEffects'][fakeEngine]).sort()).toStrictEqual(['side-effect.js', 'side-effect.txt'])
expect(Object.keys(cacheIntegrity?.['sideEffects'][fakeEngine].added).sort()).toStrictEqual(['side-effect.js', 'side-effect.txt'])
await server.close()
await storeCtrl.close()

View File

@@ -2,7 +2,7 @@ import path from 'path'
import fs from 'fs'
import gfs from '@pnpm/graceful-fs'
import * as crypto from 'crypto'
import { type Cafs } from '@pnpm/cafs-types'
import { type Cafs, type PackageFiles, type SideEffects, type SideEffectsDiff } from '@pnpm/cafs-types'
import { createCafsStore } from '@pnpm/create-cafs-store'
import { pkgRequiresBuild } from '@pnpm/exec.pkg-requires-build'
import { hardLinkDir } from '@pnpm/fs.hard-link-dir'
@@ -10,9 +10,7 @@ import {
type CafsFunctions,
checkPkgFilesIntegrity,
createCafs,
type PackageFileInfo,
type PackageFilesIndex,
type SideEffects,
type FilesIndex,
optimisticRenameOverwrite,
readManifestFromStore,
@@ -177,10 +175,18 @@ function addFilesFromDir ({ dir, cafsDir, filesIndexFile, sideEffectsCacheKey, f
try {
filesIndex = loadJsonFile<PackageFilesIndex>(filesIndexFile)
} catch {
filesIndex = { name: manifest?.name, version: manifest?.version, files: filesIntegrity }
// If there is no existing index file, then we cannot store the side effects.
return {
status: 'success',
value: {
filesIndex: filesMap,
manifest,
requiresBuild: pkgRequiresBuild(manifest, filesIntegrity),
},
}
}
filesIndex.sideEffects = filesIndex.sideEffects ?? {}
filesIndex.sideEffects[sideEffectsCacheKey] = filesIntegrity
filesIndex.sideEffects[sideEffectsCacheKey] = calculateDiff(filesIndex.files, filesIntegrity)
if (filesIndex.requiresBuild == null) {
requiresBuild = pkgRequiresBuild(manifest, filesIntegrity)
} else {
@@ -193,13 +199,37 @@ function addFilesFromDir ({ dir, cafsDir, filesIndexFile, sideEffectsCacheKey, f
return { status: 'success', value: { filesIndex: filesMap, manifest, requiresBuild } }
}
function calculateDiff (baseFiles: PackageFiles, sideEffectsFiles: PackageFiles): SideEffectsDiff {
const deleted: string[] = []
const added: PackageFiles = {}
for (const file of new Set([...Object.keys(baseFiles), ...Object.keys(sideEffectsFiles)])) {
if (!sideEffectsFiles[file]) {
deleted.push(file)
} else if (
!baseFiles[file] ||
baseFiles[file].integrity !== sideEffectsFiles[file].integrity ||
baseFiles[file].mode !== sideEffectsFiles[file].mode
) {
added[file] = sideEffectsFiles[file]
}
}
const diff: SideEffectsDiff = {}
if (deleted.length > 0) {
diff.deleted = deleted
}
if (Object.keys(added).length > 0) {
diff.added = added
}
return diff
}
interface ProcessFilesIndexResult {
filesIntegrity: Record<string, PackageFileInfo>
filesIntegrity: PackageFiles
filesMap: Record<string, string>
}
function processFilesIndex (filesIndex: FilesIndex): ProcessFilesIndexResult {
const filesIntegrity: Record<string, PackageFileInfo> = {}
const filesIntegrity: PackageFiles = {}
const filesMap: Record<string, string> = {}
for (const [k, { checkedAt, filePath, integrity, mode, size }] of Object.entries(filesIndex)) {
filesIntegrity[k] = {
@@ -263,7 +293,7 @@ function writeFilesIndexFile (
filesIndexFile: string,
{ manifest, files, sideEffects }: {
manifest: Partial<DependencyManifest>
files: Record<string, PackageFileInfo>
files: PackageFiles
sideEffects?: SideEffects
}
): boolean {