fix: print a warning when too many file's integrity is checked (#4688)

.changeset/bright-boxes-crash.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/core": patch
+---
+
+Print a warning when the integrity of more than 1K files is checked in the CAFS.

.changeset/bright-hairs-joke.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/cafs": patch
+---
+
+Track the number of integrity checks.

packages/cafs/src/checkFilesIntegrity.ts

@@ -10,6 +10,12 @@ import { parseJsonBuffer } from './parseJson'
 const limit = pLimit(20)
 const MAX_BULK_SIZE = 1 * 1024 * 1024 // 1MB
 
+// We track how many files were checked during installation.
+// It should be rare that a files content should be checked.
+// If it happens too frequently, something is wrong.
+// Checking a file's integrity is an expensive operation!
+global['verifiedFileIntegrity'] = 0
+
 export interface PackageFilesIndex {
   // name and version are nullable for backward compatibility
   // the initial specs of pnpm store v3 did not require these fields.
@@ -26,13 +32,12 @@ export default async function (
   cafsDir: string,
   pkgIndex: Record<string, PackageFileInfo>,
   manifest?: DeferredManifestPromise
-) {
+): Promise<boolean> {
   let verified = true
   await Promise.all(
-    Object.keys(pkgIndex)
-      .map(async (f) =>
+    Object.entries(pkgIndex)
+      .map(async ([f, fstat]) =>
         limit(async () => {
-          const fstat = pkgIndex[f]
           if (!fstat.integrity) {
             throw new Error(`Integrity checksum is missing for ${f}`)
           }
@@ -59,7 +64,7 @@ async function verifyFile (
   filename: string,
   fstat: FileInfo,
   deferredManifest?: DeferredManifestPromise
-) {
+): Promise<boolean> {
   const currentFile = await checkFile(filename, fstat.checkedAt)
   if (currentFile == null) return false
   if (currentFile.isModified) {
@@ -82,6 +87,7 @@ export async function verifyFileIntegrity (
   expectedFile: FileInfo,
   deferredManifest?: DeferredManifestPromise
 ) {
+  global['verifiedFileIntegrity']++
   try {
     if (expectedFile.size > MAX_BULK_SIZE && (deferredManifest == null)) {
       const ok = Boolean(await ssri.checkStream(gfs.createReadStream(filename), expectedFile.integrity))

packages/core/src/install/index.ts

@@ -28,7 +28,7 @@ import {
 } from '@pnpm/lockfile-file'
 import { writePnpFile } from '@pnpm/lockfile-to-pnp'
 import { extendProjectsWithTargetDirs } from '@pnpm/lockfile-utils'
-import logger, { streamParser } from '@pnpm/logger'
+import logger, { globalWarn, streamParser } from '@pnpm/logger'
 import { getAllDependenciesFromManifest } from '@pnpm/manifest-utils'
 import { write as writeModulesYaml } from '@pnpm/modules-yaml'
 import readModulesDirs from '@pnpm/read-modules-dir'
@@ -193,6 +193,9 @@ export async function mutateModules (
 
   const result = await _install()
 
+  if (global['verifiedFileIntegrity'] > 1000) {
+    globalWarn(`The integrity of ${global['verifiedFileIntegrity']} files was checked. This might have caused installation to take longer.`) // eslint-disable-line
+  }
   if ((reporter != null) && typeof reporter === 'function') {
     streamParser.removeListener('data', reporter)
   }