Zoltan Kochan
5f864904ed
An install that had to re-hash store files to verify them now reports
it, in both stacks. Past a second of hashing it owns the time:
The integrity of 1234 files was checked in 2.5s.
Below that, past a thousand files, it drops the claim that the install
was held up — a thousand files is ~75 ms of hashing — and names the
cause instead:
The integrity of 1001 files was checked, because their timestamps
changed since the store recorded them. A backup tool, an antivirus
scan, or a copied store can cause this.
pnpm gated a single message on the file count alone
(`global.verifiedFileIntegrity > 1000`), and pacquet had no such
message at all.
The pnpm side had never actually run. All CAFS verification happens in
worker threads, so the counter was incremented in each worker's own
realm while the report read the main thread's copy, which never left
zero. Each worker now hands its share back with the response it is
answering, and the main thread sums them.
Measuring is free where it matters. Only a file whose mtime moved past
its recorded `checkedAt` is ever hashed, and that path already opens,
reads and SHA-512s the whole file; the clock reads and the tally sit
inside it, after the algorithm and open guards, so nothing is
attributed to hashing that did not hash.
The figures are per install: both stacks keep a process-wide tally and
each install snapshots it at the start and diffs at the end, which is
exact for one install at a time. A recursive command with dedicated
lockfiles runs its per-project loop at workspaceConcurrency, and
overlapping projects there can pick up each other's hashing; scoping
the tally with AsyncLocalStorage would fix that but costs the whole
process about five times on promise-heavy work under Node's
pre-AsyncContextFrame implementation, which pnpm's supported Node 22
and 23 still use.
The tally covers store verification only. pnpm's CAFS writer also
re-verifies a file it finds already present, but pacquet's writer
byte-compares against the buffer it already holds rather than hashing,
so counting it would make the two stacks report different numbers for
the same install.
64 lines
2.5 KiB
TypeScript
64 lines
2.5 KiB
TypeScript
import fs from 'node:fs'
|
|
import os from 'node:os'
|
|
import path from 'node:path'
|
|
|
|
import { afterAll, expect, test } from '@jest/globals'
|
|
import { StoreIndex } from '@pnpm/store.index'
|
|
|
|
import {
|
|
addFilesFromDir,
|
|
finishWorkers,
|
|
readPkgFromCafs,
|
|
verifiedFileIntegritySince,
|
|
verifiedFileIntegritySnapshot,
|
|
} from '../lib/index.js'
|
|
|
|
afterAll(() => finishWorkers())
|
|
|
|
// Verification runs in the workers, so the figures an install reports at
|
|
// the end only exist if every worker's share travels back with its
|
|
// response and is summed on the main thread.
|
|
test('files re-hashed while verifying the store are tallied on the main thread', async () => {
|
|
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'pnpm-verified-file-integrity-'))
|
|
const dir = path.join(tmp, 'pkg')
|
|
fs.mkdirSync(dir)
|
|
fs.writeFileSync(path.join(dir, 'package.json'), JSON.stringify({ name: 'tallied-pkg', version: '1.0.0' }))
|
|
fs.writeFileSync(path.join(dir, 'index.js'), 'module.exports = "tallied"\n')
|
|
const storeDir = path.join(tmp, 'store')
|
|
const storeIndex = new StoreIndex(storeDir)
|
|
const filesIndexFile = path.join(storeDir, 'tallied-pkg.json')
|
|
|
|
await addFilesFromDir({ storeDir, dir, filesIndexFile, storeIndex })
|
|
|
|
// Push the stored files' mtime past the `checkedAt` the index recorded
|
|
// for them, which is what makes a read re-hash their content instead
|
|
// of trusting the index.
|
|
const future = new Date(Date.now() + 60_000)
|
|
for (const file of storeFiles(storeDir)) {
|
|
fs.utimesSync(file, future, future)
|
|
}
|
|
|
|
const baseline = verifiedFileIntegritySnapshot()
|
|
const result = await readPkgFromCafs({ storeDir, verifyStoreIntegrity: true }, filesIndexFile)
|
|
const verified = verifiedFileIntegritySince(baseline)
|
|
|
|
expect(result.verified).toBe(true)
|
|
expect(verified.files).toBeGreaterThan(0)
|
|
expect(verified.ms).toBeGreaterThan(0)
|
|
|
|
// A later install diffs from its own baseline, so the reads above are
|
|
// not charged to it a second time.
|
|
const second = verifiedFileIntegritySnapshot()
|
|
await readPkgFromCafs({ storeDir, verifyStoreIntegrity: false }, filesIndexFile)
|
|
expect(verifiedFileIntegritySince(second)).toEqual({ files: 0, ms: 0 })
|
|
|
|
storeIndex.close()
|
|
})
|
|
|
|
/** Every content file in the store, skipping the SQLite index. */
|
|
function storeFiles (storeDir: string): string[] {
|
|
return fs.readdirSync(storeDir, { recursive: true, withFileTypes: true })
|
|
.filter((entry) => entry.isFile() && !entry.name.startsWith('index.db'))
|
|
.map((entry) => path.join(entry.parentPath, entry.name))
|
|
}
|