Zoltan Kochan
511d68afe8
pnpm/pnpm#12997 stopped caching a build whose output contains symlinks, so packages like dugite rerun their postinstall on every warm install. Record those symlinks in the cache instead and recreate them on restore. A symlink is an `added` entry whose mode carries the S_IFLNK file-type bits (0o120000) and whose CAFS content is the link target. The content is an ordinary non-executable CAFS file, so an older pnpm still verifies the entry. The side-effects cache key gains a `format=2` segment, so an older pnpm never looks up such an entry and never restores a target path as a regular file. Every built package is built once after the upgrade. Only relative targets are recorded. They may climb only at their start, must not go above the package root, and must not name node_modules. Other symlinks keep the previous behaviour: the entry is not cached and the build reruns. On restore, the target is checked against the same rules, and the entry is dropped if a file or link would be written below a link. Windows neither records nor restores symlinks. pnpm v12 does not publish a diff with symlinks as a remote artifact, and it rejects a remote artifact that records one. Remote artifacts keep their input key, and older clients read them. Related to pnpm/pnpm#12859.
210 lines
8.1 KiB
TypeScript
210 lines
8.1 KiB
TypeScript
import fs from 'node:fs'
|
|
import os from 'node:os'
|
|
import path from 'node:path'
|
|
|
|
import { afterAll, expect, test } from '@jest/globals'
|
|
import { PnpmError } from '@pnpm/error'
|
|
import { getFilePathByModeInCafs, isSymlinkMode, type PackageFilesIndex } from '@pnpm/store.cafs'
|
|
import { StoreIndex } from '@pnpm/store.index'
|
|
|
|
import { addFilesFromDir, finishWorkers } from '../lib/index.js'
|
|
|
|
afterAll(() => finishWorkers())
|
|
|
|
test('addFilesFromDir() records whether git preparation was required', async () => {
|
|
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'pnpm-worker-test-'))
|
|
const dir = path.join(tmp, 'pkg')
|
|
fs.mkdirSync(dir)
|
|
fs.writeFileSync(path.join(dir, 'package.json'), JSON.stringify({ name: 'prepared-pkg', version: '1.0.0' }))
|
|
const storeDir = path.join(tmp, 'store')
|
|
const filesIndexFile = 'prepared-pkg'
|
|
const storeIndex = new StoreIndex(storeDir)
|
|
|
|
const result = await addFilesFromDir({
|
|
storeDir,
|
|
dir,
|
|
filesIndexFile,
|
|
requiresPrepare: true,
|
|
storeIndex,
|
|
})
|
|
|
|
expect(result.requiresPrepare).toBe(true)
|
|
expect((storeIndex.get(filesIndexFile) as PackageFilesIndex).requiresPrepare).toBe(true)
|
|
storeIndex.close()
|
|
})
|
|
|
|
test('addFilesFromDir() rejects when committing the index writes throws (e.g. a read-only store index)', async () => {
|
|
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'pnpm-worker-test-'))
|
|
const dir = path.join(tmp, 'pkg')
|
|
fs.mkdirSync(dir)
|
|
fs.writeFileSync(path.join(dir, 'package.json'), JSON.stringify({ name: 'frozen-test-pkg', version: '1.0.0' }))
|
|
const storeDir = path.join(tmp, 'store')
|
|
|
|
const storeIndex = {
|
|
setRawMany () {
|
|
throw new PnpmError('FROZEN_STORE_WRITE', 'Cannot write to the package store because frozenStore is enabled')
|
|
},
|
|
} as unknown as StoreIndex
|
|
|
|
await expect(addFilesFromDir({
|
|
storeDir,
|
|
dir,
|
|
filesIndexFile: path.join(storeDir, 'frozen-test-pkg.json'),
|
|
storeIndex,
|
|
})).rejects.toMatchObject({ code: 'ERR_PNPM_FROZEN_STORE_WRITE' })
|
|
})
|
|
|
|
const testOnPosix = process.platform === 'win32' ? test.skip : test
|
|
|
|
testOnPosix('addFilesFromDir() records the symlinks a build creates in side effects', async () => {
|
|
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'pnpm-worker-test-'))
|
|
const dir = path.join(tmp, 'pkg')
|
|
fs.mkdirSync(dir)
|
|
fs.writeFileSync(path.join(dir, 'package.json'), JSON.stringify({ name: 'symlink-side-effects-pkg', version: '1.0.0' }))
|
|
const storeDir = path.join(tmp, 'store')
|
|
const filesIndexFile = 'symlink-side-effects-pkg'
|
|
const storeIndex = new StoreIndex(storeDir)
|
|
|
|
await addFilesFromDir({ storeDir, dir, filesIndexFile, storeIndex })
|
|
|
|
fs.writeFileSync(path.join(dir, 'generated.js'), 'module.exports = true')
|
|
fs.symlinkSync('generated.js', path.join(dir, 'generated-link.js'))
|
|
fs.mkdirSync(path.join(dir, 'lib'))
|
|
fs.writeFileSync(path.join(dir, 'lib/index.js'), 'module.exports = true')
|
|
fs.mkdirSync(path.join(dir, 'bin'))
|
|
fs.symlinkSync('../lib', path.join(dir, 'bin/lib'))
|
|
|
|
await addFilesFromDir({
|
|
storeDir,
|
|
dir,
|
|
filesIndexFile,
|
|
sideEffectsCacheKey: 'test-engine',
|
|
storeIndex,
|
|
})
|
|
|
|
const added = (storeIndex.get(filesIndexFile) as PackageFilesIndex).sideEffects?.get('test-engine')?.added
|
|
expect(Array.from(added!.keys()).sort()).toStrictEqual(['bin/lib', 'generated-link.js', 'generated.js', 'lib/index.js'])
|
|
for (const [link, target] of [['generated-link.js', 'generated.js'], ['bin/lib', '../lib']]) {
|
|
const info = added!.get(link)!
|
|
expect(isSymlinkMode(info.mode)).toBe(true)
|
|
expect(fs.readFileSync(getFilePathByModeInCafs(storeDir, info.digest, info.mode), 'utf8')).toBe(target)
|
|
}
|
|
storeIndex.close()
|
|
})
|
|
|
|
test('addFilesFromDir() does not cache side effects that contain symlinks it cannot record', async () => {
|
|
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'pnpm-worker-test-'))
|
|
const dir = path.join(tmp, 'pkg')
|
|
fs.mkdirSync(dir)
|
|
fs.writeFileSync(path.join(dir, 'package.json'), JSON.stringify({ name: 'symlink-side-effects-pkg', version: '1.0.0' }))
|
|
const storeDir = path.join(tmp, 'store')
|
|
const filesIndexFile = 'symlink-side-effects-pkg'
|
|
const storeIndex = new StoreIndex(storeDir)
|
|
|
|
await addFilesFromDir({
|
|
storeDir,
|
|
dir,
|
|
filesIndexFile,
|
|
storeIndex,
|
|
})
|
|
|
|
const outsideDir = path.join(tmp, 'outside')
|
|
fs.mkdirSync(outsideDir)
|
|
fs.writeFileSync(path.join(outsideDir, 'index.js'), 'module.exports = true')
|
|
fs.symlinkSync(outsideDir, path.join(dir, 'generated-link'), process.platform === 'win32' ? 'junction' : 'dir')
|
|
|
|
await addFilesFromDir({
|
|
storeDir,
|
|
dir,
|
|
filesIndexFile,
|
|
sideEffectsCacheKey: 'test-engine',
|
|
storeIndex,
|
|
})
|
|
|
|
const filesIndex = storeIndex.get(filesIndexFile) as PackageFilesIndex
|
|
expect(filesIndex.sideEffects).toBeUndefined()
|
|
storeIndex.close()
|
|
})
|
|
|
|
test.each([false, true])('addFilesFromDir() removes stale symlink side effects (other cache entry: %s)', async (keepOtherEntry) => {
|
|
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'pnpm-worker-test-'))
|
|
const dir = path.join(tmp, 'pkg')
|
|
fs.mkdirSync(dir)
|
|
fs.writeFileSync(path.join(dir, 'package.json'), JSON.stringify({ name: 'stale-side-effects-pkg', version: '1.0.0' }))
|
|
const storeDir = path.join(tmp, 'store')
|
|
const filesIndexFile = 'stale-side-effects-pkg'
|
|
const storeIndex = new StoreIndex(storeDir)
|
|
|
|
await addFilesFromDir({ storeDir, dir, filesIndexFile, storeIndex })
|
|
const targetDir = path.join(dir, 'generated')
|
|
const outputDir = path.join(dir, 'generated-link')
|
|
fs.mkdirSync(targetDir)
|
|
fs.mkdirSync(outputDir)
|
|
fs.writeFileSync(path.join(targetDir, 'index.js'), 'module.exports = true')
|
|
fs.writeFileSync(path.join(outputDir, 'index.js'), 'module.exports = true')
|
|
const uploadOptions = { storeDir, dir, filesIndexFile, storeIndex }
|
|
await addFilesFromDir({ ...uploadOptions, sideEffectsCacheKey: 'test-engine' })
|
|
if (keepOtherEntry) {
|
|
await addFilesFromDir({ ...uploadOptions, sideEffectsCacheKey: 'other-engine' })
|
|
}
|
|
const cachedIndex = storeIndex.get(filesIndexFile) as PackageFilesIndex
|
|
expect(cachedIndex.sideEffects?.get('test-engine')?.added?.has('generated-link/index.js')).toBe(true)
|
|
const otherEntry = cachedIndex.sideEffects?.get('other-engine')
|
|
|
|
fs.rmSync(outputDir, { recursive: true })
|
|
const outsideDir = path.join(tmp, 'outside')
|
|
fs.mkdirSync(outsideDir)
|
|
fs.symlinkSync(outsideDir, outputDir, process.platform === 'win32' ? 'junction' : 'dir')
|
|
const result = await addFilesFromDir({
|
|
storeDir,
|
|
dir,
|
|
filesIndexFile,
|
|
sideEffectsCacheKey: 'test-engine',
|
|
storeIndex,
|
|
})
|
|
|
|
const filesIndex = storeIndex.get(filesIndexFile) as PackageFilesIndex
|
|
expect(filesIndex.sideEffects?.get('test-engine')).toBeUndefined()
|
|
expect(result.sideEffects).toBeUndefined()
|
|
if (keepOtherEntry) {
|
|
expect(filesIndex.sideEffects?.size).toBe(1)
|
|
expect(filesIndex.sideEffects?.get('other-engine')).toEqual(otherEntry)
|
|
} else {
|
|
expect(filesIndex.sideEffects).toBeUndefined()
|
|
}
|
|
storeIndex.close()
|
|
})
|
|
|
|
test('addFilesFromDir() ignores the dependency node_modules link when caching side effects', async () => {
|
|
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'pnpm-worker-test-'))
|
|
const dir = path.join(tmp, 'pkg')
|
|
const dependencyDir = path.join(tmp, 'dependency')
|
|
fs.mkdirSync(dir)
|
|
fs.mkdirSync(dependencyDir)
|
|
fs.writeFileSync(path.join(dir, 'package.json'), JSON.stringify({ name: 'regular-side-effects-pkg', version: '1.0.0' }))
|
|
fs.writeFileSync(path.join(dependencyDir, 'index.js'), 'module.exports = true')
|
|
fs.symlinkSync(dependencyDir, path.join(dir, 'node_modules'), process.platform === 'win32' ? 'junction' : 'dir')
|
|
const storeDir = path.join(tmp, 'store')
|
|
const filesIndexFile = 'regular-side-effects-pkg'
|
|
const storeIndex = new StoreIndex(storeDir)
|
|
|
|
await addFilesFromDir({
|
|
storeDir,
|
|
dir,
|
|
filesIndexFile,
|
|
storeIndex,
|
|
})
|
|
fs.writeFileSync(path.join(dir, 'generated.js'), 'module.exports = true')
|
|
await addFilesFromDir({
|
|
storeDir,
|
|
dir,
|
|
filesIndexFile,
|
|
sideEffectsCacheKey: 'test-engine',
|
|
storeIndex,
|
|
})
|
|
|
|
const filesIndex = storeIndex.get(filesIndexFile) as PackageFilesIndex
|
|
expect(filesIndex.sideEffects?.get('test-engine')?.added?.has('generated.js')).toBe(true)
|
|
storeIndex.close()
|
|
})
|