feat: support frozen installs in projects using local tarball dependencies (#9190)

.changeset/common-pugs-flash.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/crypto.hash": minor
+---
+
+Added a new `getTarballIntegrity` function. This function was moved from `@pnpm/local-resolver` and is used to compute the integrity hash of a local tarball `file:` dependency in the `pnpm-lock.yaml` file.

.changeset/empty-windows-ring.md

@@ -0,0 +1,6 @@
+---
+"@pnpm/lockfile.verification": minor
+pnpm: minor
+---
+
+Projects using a `file:` dependency on a local tarball file (i.e. `.tgz`, `.tar.gz`, `.tar`) will see a performance improvement during installation. Previously, using a `file:` dependency on a tarball caused the lockfile resolution step to always run. The lockfile will now be considered up-to-date if the tarball is unchanged.

crypto/hash/package.json

@@ -32,11 +32,16 @@
     "compile": "tsc --build && pnpm run lint --fix"
   },
   "dependencies": {
-    "@pnpm/crypto.polyfill": "workspace:*"
+    "@pnpm/crypto.polyfill": "workspace:*",
+    "@pnpm/graceful-fs": "workspace:*",
+    "ssri": "catalog:"
   },
   "devDependencies": {
     "@pnpm/crypto.hash": "workspace:*",
-    "@pnpm/prepare": "workspace:*"
+    "@pnpm/prepare": "workspace:*",
+    "@types/ssri": "catalog:",
+    "@types/tar-stream": "catalog:",
+    "tar-stream": "catalog:"
   },
   "engines": {
     "node": ">=18.12"

crypto/hash/src/index.ts

@@ -1,5 +1,7 @@
 import * as crypto from '@pnpm/crypto.polyfill'
 import fs from 'fs'
+import gfs from '@pnpm/graceful-fs'
+import ssri from 'ssri'
 
 export function createShortHash (input: string): string {
   return createHexHash(input).substring(0, 32)
@@ -25,3 +27,7 @@ async function readNormalizedFile (file: string): Promise<string> {
   const content = await fs.promises.readFile(file, 'utf8')
   return content.split('\r\n').join('\n')
 }
+
+export async function getTarballIntegrity (filename: string): Promise<string> {
+  return (await ssri.fromStream(gfs.createReadStream(filename))).toString()
+}

crypto/hash/test/index.ts

@@ -1,7 +1,9 @@
 /// <reference path="../../../__typings__/index.d.ts"/>
 import fs from 'fs'
-import { createShortHash, createHashFromFile } from '@pnpm/crypto.hash'
+import { createShortHash, createHashFromFile, getTarballIntegrity } from '@pnpm/crypto.hash'
 import { tempDir } from '@pnpm/prepare'
+import { pipeline } from 'node:stream/promises'
+import tar from 'tar-stream'
 
 test('createShortHash()', () => {
   expect(createShortHash('AAA')).toEqual('cb1ad2119d8fafb69566510ee712661f')
@@ -13,3 +15,20 @@ test('createHashFromFile normalizes line endings before calculating the hash', a
   fs.writeFileSync('posix-eol.txt', 'a\nb\r\nc')
   expect(await createHashFromFile('win-eol.txt')).toEqual(await createHashFromFile('posix-eol.txt'))
 })
+
+test('getTarballIntegrity creates integrity hash for tarball', async () => {
+  expect.hasAssertions()
+  tempDir()
+
+  const pack = tar.pack()
+  pack.entry({ name: 'package.json', mtime: new Date('1970-01-01T00:00:00.000Z') }, JSON.stringify({
+    name: 'local-tarball',
+    version: '1.0.0',
+  }))
+  pack.finalize()
+
+  await pipeline(pack, fs.createWriteStream('./local-tarball.tar'))
+
+  await expect(getTarballIntegrity('./local-tarball.tar'))
+    .resolves.toEqual('sha512-nQP7gWOhNQ/5HoM/rJmzOgzZt6Wg6k56CyvO/0sMmiS3UkLSmzY5mW8mMrnbspgqpmOW8q/FHyb0YIr4n2A8VQ==')
+})

crypto/hash/tsconfig.json

@@ -12,6 +12,9 @@
     {
       "path": "../../__utils__/prepare"
     },
+    {
+      "path": "../../fs/graceful-fs"
+    },
     {
       "path": "../polyfill"
     }

lockfile/verification/package.json

@@ -32,6 +32,7 @@
   },
   "dependencies": {
     "@pnpm/catalogs.types": "workspace:*",
+    "@pnpm/crypto.hash": "workspace:*",
     "@pnpm/dependency-path": "workspace:*",
     "@pnpm/get-context": "workspace:*",
     "@pnpm/lockfile.types": "workspace:*",
@@ -53,7 +54,9 @@
     "@pnpm/logger": "workspace:*",
     "@pnpm/prepare": "workspace:*",
     "@types/ramda": "catalog:",
-    "@types/semver": "catalog:"
+    "@types/semver": "catalog:",
+    "@types/tar-stream": "catalog:",
+    "tar-stream": "catalog:"
   },
   "engines": {
     "node": ">=18.12"

lockfile/verification/src/allProjectsAreUpToDate.ts

@@ -2,18 +2,16 @@ import { type Catalogs } from '@pnpm/catalogs.types'
 import { type ProjectOptions } from '@pnpm/get-context'
 import {
   type LockfileObject,
-  type ProjectSnapshot,
 } from '@pnpm/lockfile.types'
-import { refIsLocalTarball } from '@pnpm/lockfile.utils'
 import { type WorkspacePackages } from '@pnpm/resolver-base'
 import { DEPENDENCIES_FIELDS, type ProjectId } from '@pnpm/types'
 import pEvery from 'p-every'
-import any from 'ramda/src/any'
 import isEmpty from 'ramda/src/isEmpty'
 import { allCatalogsAreUpToDate } from './allCatalogsAreUpToDate'
 import { getWorkspacePackagesByDirectory } from './getWorkspacePackagesByDirectory'
 import { linkedPackagesAreUpToDate } from './linkedPackagesAreUpToDate'
 import { satisfiesPackageManifest } from './satisfiesPackageManifest'
+import { localTarballDepsAreUpToDate } from './localTarballDepsAreUpToDate'
 
 export async function allProjectsAreUpToDate (
   projects: Array<Pick<ProjectOptions, 'manifest' | 'rootDir'> & { id: ProjectId }>,
@@ -46,24 +44,26 @@ export async function allProjectsAreUpToDate (
     lockfilePackages: opts.wantedLockfile.packages,
     lockfileDir: opts.lockfileDir,
   })
-  return pEvery(projects, (project) => {
+  const _localTarballDepsAreUpToDate = localTarballDepsAreUpToDate.bind(null, {
+    fileIntegrityCache: new Map(),
+    lockfilePackages: opts.wantedLockfile.packages,
+    lockfileDir: opts.lockfileDir,
+  })
+  return pEvery(projects, async (project) => {
     const importer = opts.wantedLockfile.importers[project.id]
     if (importer == null) {
       return DEPENDENCIES_FIELDS.every((depType) => project.manifest[depType] == null || isEmpty(project.manifest[depType]))
     }
+
+    const projectInfo = {
+      dir: project.rootDir,
+      manifest: project.manifest,
+      snapshot: importer,
+    }
+
     return importer != null &&
-      !hasLocalTarballDepsInRoot(importer) &&
       _satisfiesPackageManifest(importer, project.manifest).satisfies &&
-      _linkedPackagesAreUpToDate({
-        dir: project.rootDir,
-        manifest: project.manifest,
-        snapshot: importer,
-      })
+      (await _localTarballDepsAreUpToDate(projectInfo)) &&
+      (_linkedPackagesAreUpToDate(projectInfo))
   })
 }
-
-function hasLocalTarballDepsInRoot (importer: ProjectSnapshot): boolean {
-  return any(refIsLocalTarball, Object.values(importer.dependencies ?? {})) ||
-    any(refIsLocalTarball, Object.values(importer.devDependencies ?? {})) ||
-    any(refIsLocalTarball, Object.values(importer.optionalDependencies ?? {}))
-}

lockfile/verification/src/index.ts

@@ -1,4 +1,5 @@
 export { allProjectsAreUpToDate } from './allProjectsAreUpToDate'
 export { getWorkspacePackagesByDirectory } from './getWorkspacePackagesByDirectory'
+export { localTarballDepsAreUpToDate } from './localTarballDepsAreUpToDate'
 export { linkedPackagesAreUpToDate } from './linkedPackagesAreUpToDate'
 export { satisfiesPackageManifest } from './satisfiesPackageManifest'

lockfile/verification/src/localTarballDepsAreUpToDate.ts

@@ -0,0 +1,90 @@
+import { getTarballIntegrity } from '@pnpm/crypto.hash'
+import { refToRelative } from '@pnpm/dependency-path'
+import {
+  type ProjectSnapshot,
+  type PackageSnapshots,
+  type TarballResolution,
+} from '@pnpm/lockfile.types'
+import { refIsLocalTarball } from '@pnpm/lockfile.utils'
+import { DEPENDENCIES_FIELDS } from '@pnpm/types'
+import path from 'node:path'
+import pEvery from 'p-every'
+
+export interface LocalTarballDepsUpToDateContext {
+  /**
+   * Local cache of local absolute file paths to their integrity. Expected to be
+   * initialized to an empty map by the caller.
+   */
+  readonly fileIntegrityCache: Map<string, Promise<string>>
+  readonly lockfilePackages?: PackageSnapshots
+  readonly lockfileDir: string
+}
+
+/**
+ * Returns false if a local tarball file has been changed on disk since the last
+ * installation recorded by the project snapshot.
+ *
+ * This function only inspects the project's lockfile snapshot. It does not
+ * inspect the current project manifest. The caller of this function is expected
+ * to handle changes to the project manifest that would cause the corresponding
+ * project snapshot to become out of date.
+ */
+export async function localTarballDepsAreUpToDate (
+  {
+    fileIntegrityCache,
+    lockfilePackages,
+    lockfileDir,
+  }: LocalTarballDepsUpToDateContext,
+  project: {
+    snapshot: ProjectSnapshot
+  }
+): Promise<boolean> {
+  return pEvery(DEPENDENCIES_FIELDS, (depField) => {
+    const lockfileDeps = project.snapshot[depField]
+
+    // If the lockfile is missing a snapshot for this project's dependencies, we
+    // can return true. The "satisfiesPackageManifest" logic in
+    // "allProjectsAreUpToDate" will catch mismatches between a project's
+    // manifest and snapshot dependencies size.
+    if (lockfileDeps == null) {
+      return true
+    }
+
+    return pEvery(
+      Object.entries(lockfileDeps),
+      async ([depName, ref]) => {
+        if (!refIsLocalTarball(ref)) {
+          return true
+        }
+
+        const depPath = refToRelative(ref, depName)
+        const packageSnapshot = depPath != null ? lockfilePackages?.[depPath] : null
+
+        // If there's no snapshot for this local tarball yet, the project is out
+        // of date and needs to be resolved. This should only happen with a
+        // broken lockfile.
+        if (packageSnapshot == null) {
+          return false
+        }
+
+        const filePath = path.join(lockfileDir, ref.slice('file:'.length))
+
+        const fileIntegrityPromise = fileIntegrityCache.get(filePath) ?? getTarballIntegrity(filePath)
+        if (!fileIntegrityCache.has(filePath)) {
+          fileIntegrityCache.set(filePath, fileIntegrityPromise)
+        }
+
+        let fileIntegrity: string
+        try {
+          fileIntegrity = await fileIntegrityPromise
+        } catch (err) {
+          // If there was an error reading the tarball, assume the lockfile is
+          // out of date. The full resolution process will emit a clearer error
+          // later during install.
+          return false
+        }
+
+        return (packageSnapshot.resolution as TarballResolution).integrity === fileIntegrity
+      })
+  })
+}

lockfile/verification/test/allProjectsAreUpToDate.test.ts

@@ -1,10 +1,14 @@
 import { LOCKFILE_VERSION } from '@pnpm/constants'
 import { prepareEmpty } from '@pnpm/prepare'
 import { type WorkspacePackages } from '@pnpm/resolver-base'
-import { type DependencyManifest, type ProjectId, type ProjectRootDir } from '@pnpm/types'
+import { type DepPath, type DependencyManifest, type ProjectId, type ProjectRootDir } from '@pnpm/types'
 import { allProjectsAreUpToDate } from '@pnpm/lockfile.verification'
+import { createWriteStream } from 'fs'
 import { writeFile, mkdir } from 'fs/promises'
 import { type LockfileObject } from '@pnpm/lockfile.types'
+import tar from 'tar-stream'
+import { pipeline } from 'stream/promises'
+import { getTarballIntegrity } from '@pnpm/crypto.hash'
 
 const fooManifest = {
   name: 'foo',
@@ -489,6 +493,104 @@ describe('local file dependency', () => {
   })
 })
 
+describe('local tgz file dependency', () => {
+  beforeEach(async () => {
+    prepareEmpty()
+  })
+
+  const projects = [
+    {
+      id: 'bar' as ProjectId,
+      manifest: {
+        dependencies: {
+          'local-tarball': 'file:local-tarball.tar',
+        },
+      },
+      rootDir: 'bar' as ProjectRootDir,
+    },
+    {
+      id: 'foo' as ProjectId,
+      manifest: fooManifest,
+      rootDir: 'foo' as ProjectRootDir,
+    },
+  ]
+
+  const wantedLockfile: LockfileObject = {
+    lockfileVersion: LOCKFILE_VERSION,
+    importers: {
+      ['bar' as ProjectId]: {
+        dependencies: { 'local-tarball': 'file:local-tarball.tar' },
+        specifiers: { 'local-tarball': 'file:local-tarball.tar' },
+      },
+      ['foo' as ProjectId]: {
+        specifiers: {},
+      },
+    },
+    packages: {
+      ['local-tarball@file:local-tarball.tar' as DepPath]: {
+        resolution: {
+          integrity: 'sha512-nQP7gWOhNQ/5HoM/rJmzOgzZt6Wg6k56CyvO/0sMmiS3UkLSmzY5mW8mMrnbspgqpmOW8q/FHyb0YIr4n2A8VQ==',
+          tarball: 'file:local-tarball.tar',
+        },
+        version: '1.0.0',
+      },
+    },
+  }
+
+  const options = {
+    autoInstallPeers: false,
+    catalogs: {},
+    excludeLinksFromLockfile: false,
+    linkWorkspacePackages: true,
+    wantedLockfile,
+    workspacePackages,
+    lockfileDir: process.cwd(),
+  }
+
+  test('allProjectsAreUpToDate(): returns true if local file not changed', async () => {
+    expect.hasAssertions()
+
+    const pack = tar.pack()
+    pack.entry({ name: 'package.json', mtime: new Date('1970-01-01T00:00:00.000Z') }, JSON.stringify({
+      name: 'local-tarball',
+      version: '1.0.0',
+    }))
+    pack.finalize()
+
+    await pipeline(pack, createWriteStream('./local-tarball.tar'))
+
+    // Make the test is set up correctly and the local-tarball.tar created above
+    // has the expected integrity hash.
+    await expect(getTarballIntegrity('./local-tarball.tar')).resolves.toEqual('sha512-nQP7gWOhNQ/5HoM/rJmzOgzZt6Wg6k56CyvO/0sMmiS3UkLSmzY5mW8mMrnbspgqpmOW8q/FHyb0YIr4n2A8VQ==')
+
+    const lockfileDir = process.cwd()
+    expect(await allProjectsAreUpToDate(projects, { ...options, lockfileDir })).toBeTruthy()
+  })
+
+  test('allProjectsAreUpToDate(): returns false if local file has changed', async () => {
+    expect.hasAssertions()
+
+    const pack = tar.pack()
+    pack.entry({ name: 'package.json', mtime: new Date('2000-01-01T00:00:00') }, JSON.stringify({
+      name: 'local-tarball',
+      version: '1.0.0',
+    }))
+    pack.entry({ name: 'newly-added-file.txt' }, 'This file changes the tarball.')
+    pack.finalize()
+    await pipeline(pack, createWriteStream('./local-tarball.tar'))
+
+    const lockfileDir = process.cwd()
+    expect(await allProjectsAreUpToDate(projects, { ...options, lockfileDir })).toBeFalsy()
+  })
+
+  test('allProjectsAreUpToDate(): returns false if local dep does not exist', async () => {
+    expect.hasAssertions()
+
+    const lockfileDir = process.cwd()
+    expect(await allProjectsAreUpToDate(projects, { ...options, lockfileDir })).toBeFalsy()
+  })
+})
+
 test('allProjectsAreUpToDate(): returns true if workspace dependency\'s version type is tag', async () => {
   const projects = [
     {

lockfile/verification/tsconfig.json

@@ -15,6 +15,9 @@
     {
       "path": "../../catalogs/types"
     },
+    {
+      "path": "../../crypto/hash"
+    },
     {
       "path": "../../packages/constants"
     },

pnpm-lock.yaml

@@ -1680,6 +1680,12 @@ importers:
       '@pnpm/crypto.polyfill':
         specifier: workspace:*
         version: link:../polyfill
+      '@pnpm/graceful-fs':
+        specifier: workspace:*
+        version: link:../../fs/graceful-fs
+      ssri:
+        specifier: 'catalog:'
+        version: 10.0.5
     devDependencies:
       '@pnpm/crypto.hash':
         specifier: workspace:*
@@ -1687,6 +1693,15 @@ importers:
       '@pnpm/prepare':
         specifier: workspace:*
         version: link:../../__utils__/prepare
+      '@types/ssri':
+        specifier: 'catalog:'
+        version: 7.1.5
+      '@types/tar-stream':
+        specifier: 'catalog:'
+        version: 2.2.3
+      tar-stream:
+        specifier: 'catalog:'
+        version: 2.2.0
 
   crypto/object-hasher:
     dependencies:
@@ -3585,6 +3600,9 @@ importers:
       '@pnpm/catalogs.types':
         specifier: workspace:*
         version: link:../../catalogs/types
+      '@pnpm/crypto.hash':
+        specifier: workspace:*
+        version: link:../../crypto/hash
       '@pnpm/dependency-path':
         specifier: workspace:*
         version: link:../../packages/dependency-path
@@ -3637,6 +3655,12 @@ importers:
       '@types/semver':
         specifier: 'catalog:'
         version: 7.5.3
+      '@types/tar-stream':
+        specifier: 'catalog:'
+        version: 2.2.3
+      tar-stream:
+        specifier: 'catalog:'
+        version: 2.2.0
 
   lockfile/walker:
     dependencies:
@@ -6542,12 +6566,12 @@ importers:
 
   resolving/local-resolver:
     dependencies:
+      '@pnpm/crypto.hash':
+        specifier: workspace:*
+        version: link:../../crypto/hash
       '@pnpm/error':
         specifier: workspace:*
         version: link:../../packages/error
-      '@pnpm/graceful-fs':
-        specifier: workspace:*
-        version: link:../../fs/graceful-fs
       '@pnpm/read-project-manifest':
         specifier: workspace:*
         version: link:../../pkg-manifest/read-project-manifest
@@ -6560,9 +6584,6 @@ importers:
       normalize-path:
         specifier: 'catalog:'
         version: 3.0.0
-      ssri:
-        specifier: 'catalog:'
-        version: 10.0.5
     devDependencies:
       '@pnpm/local-resolver':
         specifier: workspace:*
@@ -6573,9 +6594,6 @@ importers:
       '@types/normalize-path':
         specifier: 'catalog:'
         version: 3.0.2
-      '@types/ssri':
-        specifier: 'catalog:'
-        version: 7.1.5
 
   resolving/npm-resolver:
     dependencies:

resolving/local-resolver/package.json

@@ -33,13 +33,12 @@
     "compile": "tsc --build && pnpm run lint --fix"
   },
   "dependencies": {
+    "@pnpm/crypto.hash": "workspace:*",
     "@pnpm/error": "workspace:*",
-    "@pnpm/graceful-fs": "workspace:*",
     "@pnpm/read-project-manifest": "workspace:*",
     "@pnpm/resolver-base": "workspace:*",
     "@pnpm/types": "workspace:*",
-    "normalize-path": "catalog:",
-    "ssri": "catalog:"
+    "normalize-path": "catalog:"
   },
   "peerDependencies": {
     "@pnpm/logger": ">=5.1.0 <1001.0.0"
@@ -47,8 +46,7 @@
   "devDependencies": {
     "@pnpm/local-resolver": "workspace:*",
     "@pnpm/logger": "workspace:*",
-    "@types/normalize-path": "catalog:",
-    "@types/ssri": "catalog:"
+    "@types/normalize-path": "catalog:"
   },
   "engines": {
     "node": ">=18.12"

resolving/local-resolver/src/index.ts

@@ -1,7 +1,7 @@
 import { existsSync } from 'fs'
 import path from 'path'
+import { getTarballIntegrity } from '@pnpm/crypto.hash'
 import { PnpmError } from '@pnpm/error'
-import gfs from '@pnpm/graceful-fs'
 import { readProjectManifestOnly } from '@pnpm/read-project-manifest'
 import {
   type DirectoryResolution,
@@ -9,7 +9,6 @@ import {
   type TarballResolution,
 } from '@pnpm/resolver-base'
 import { type DependencyManifest } from '@pnpm/types'
-import ssri from 'ssri'
 import { logger } from '@pnpm/logger'
 import { parsePref, type WantedLocalDependency } from './parsePref'
 
@@ -38,7 +37,7 @@ export async function resolveFromLocal (
       id: spec.id,
       normalizedPref: spec.normalizedPref,
       resolution: {
-        integrity: await getFileIntegrity(spec.fetchSpec),
+        integrity: await getTarballIntegrity(spec.fetchSpec),
         tarball: spec.id,
       },
       resolvedVia: 'local-filesystem',
@@ -93,7 +92,3 @@ export async function resolveFromLocal (
     resolvedVia: 'local-filesystem',
   }
 }
-
-async function getFileIntegrity (filename: string): Promise<string> {
-  return (await ssri.fromStream(gfs.createReadStream(filename))).toString()
-}

resolving/local-resolver/tsconfig.json

@@ -10,7 +10,7 @@
   ],
   "references": [
     {
-      "path": "../../fs/graceful-fs"
+      "path": "../../crypto/hash"
     },
     {
       "path": "../../packages/error"