fix: preserve file: and git-hosted tarball URLs in lockfile (#11410)

Closes #11407

.changeset/preserve-file-tarball-url.md

@@ -0,0 +1,6 @@
+---
+"@pnpm/lockfile.utils": patch
+"pnpm": patch
+---
+
+Fix `ERR_PNPM_FETCH_404` when installing a project whose lockfile depends on a `file:` tarball. The previous behavior dropped the `tarball` field from `file:` and git-hosted resolutions when `lockfile-include-tarball-url=false` (the default), even though those URLs cannot be reconstructed from the package name, version, and registry [#11407](https://github.com/pnpm/pnpm/issues/11407).

lockfile/utils/src/pkgSnapshotToResolution.ts

@@ -1,5 +1,6 @@
 import url from 'node:url'
 
+import * as dp from '@pnpm/deps.path'
 import { isGitHostedPkgUrl } from '@pnpm/fetching.pick-fetcher'
 import type { PackageSnapshot, TarballResolution } from '@pnpm/lockfile.types'
 import type { Resolution } from '@pnpm/resolving.resolver-base'
@@ -20,6 +21,16 @@ export function pkgSnapshotToResolution (
   ) {
     return pkgSnapshot.resolution as Resolution
   }
+  // Recover the tarball field for `file:` snapshots whose resolution lost
+  // its tarball (e.g. lockfiles written by an earlier pnpm 11 version that
+  // dropped the tarball under `lockfile-include-tarball-url=false`).
+  const nonSemverVersion = dp.parse(depPath).nonSemverVersion
+  if (nonSemverVersion?.startsWith('file:')) {
+    return {
+      ...pkgSnapshot.resolution,
+      tarball: nonSemverVersion,
+    } as Resolution
+  }
   const { name, version } = nameVerFromPkgSnapshot(depPath, pkgSnapshot)
   let registry: string = ''
   if (name != null) {

lockfile/utils/src/toLockfileResolution.ts

@@ -1,3 +1,4 @@
+import { isGitHostedPkgUrl } from '@pnpm/fetching.pick-fetcher'
 import type { LockfileResolution } from '@pnpm/lockfile.types'
 import type { Resolution } from '@pnpm/resolving.resolver-base'
 import getNpmTarballUrl from 'get-npm-tarball-url'
@@ -14,10 +15,21 @@ export function toLockfileResolution (
   if (resolution.type !== undefined || !resolution['integrity']) {
     return resolution as LockfileResolution
   }
+  const tarball = resolution['tarball'] as string | undefined
   if (lockfileIncludeTarballUrl) {
     return {
       integrity: resolution['integrity'],
-      tarball: resolution['tarball'],
+      tarball,
+    }
+  }
+  // Tarball URLs that cannot be reconstructed from the package name, version,
+  // and registry must always stay in the lockfile, otherwise the package can
+  // no longer be re-fetched. This covers local `file:` tarballs and tarballs
+  // served by git providers (GitHub, GitLab, Bitbucket).
+  if (tarball != null && (tarball.startsWith('file:') || isGitHostedPkgUrl(tarball))) {
+    return {
+      integrity: resolution['integrity'],
+      tarball,
     }
   }
   if (lockfileIncludeTarballUrl === false) {
@@ -29,11 +41,11 @@ export function toLockfileResolution (
   // For instance, when they are hosted on npm Enterprise. See https://github.com/pnpm/pnpm/issues/867
   // Or in other weird cases, like https://github.com/pnpm/pnpm/issues/1072
   const expectedTarball = getNpmTarballUrl(pkg.name, pkg.version, { registry })
-  const actualTarball = resolution['tarball'].replaceAll('%2f', '/')
+  const actualTarball = tarball!.replaceAll('%2f', '/')
   if (removeProtocol(expectedTarball) !== removeProtocol(actualTarball)) {
     return {
       integrity: resolution['integrity'],
-      tarball: resolution['tarball'],
+      tarball,
     }
   }
   return {

lockfile/utils/test/pkgSnapshotToResolution.ts

@@ -38,4 +38,16 @@ test('pkgSnapshotToResolution()', () => {
   }, { default: 'https://registry.npmjs.org/' })).toEqual({
     tarball: 'https://cdn.sheetjs.com/xlsx-0.18.9/xlsx-0.18.9.tgz',
   })
+
+  // Snapshot for a `file:` dependency whose resolution lacks the tarball
+  // field — the tarball should be recovered from the depPath.
+  expect(pkgSnapshotToResolution('test-package@file:test-package-1.0.0.tgz', {
+    resolution: {
+      integrity: 'sha512-AAAA',
+    },
+    version: '1.0.0',
+  }, { default: 'https://registry.npmjs.org/' })).toEqual({
+    integrity: 'sha512-AAAA',
+    tarball: 'file:test-package-1.0.0.tgz',
+  })
 })

lockfile/utils/test/toLockfileResolution.ts

@@ -0,0 +1,85 @@
+import { expect, test } from '@jest/globals'
+import { toLockfileResolution } from '@pnpm/lockfile.utils'
+
+const REGISTRY = 'https://registry.npmjs.org/'
+
+test('keeps the tarball when lockfileIncludeTarballUrl is true', () => {
+  expect(toLockfileResolution(
+    { name: 'foo', version: '1.0.0' },
+    { integrity: 'sha512-AAAA', tarball: 'https://registry.npmjs.org/foo/-/foo-1.0.0.tgz' },
+    REGISTRY,
+    true
+  )).toEqual({
+    integrity: 'sha512-AAAA',
+    tarball: 'https://registry.npmjs.org/foo/-/foo-1.0.0.tgz',
+  })
+})
+
+test('drops the tarball for standard registry URLs by default', () => {
+  expect(toLockfileResolution(
+    { name: 'foo', version: '1.0.0' },
+    { integrity: 'sha512-AAAA', tarball: 'https://registry.npmjs.org/foo/-/foo-1.0.0.tgz' },
+    REGISTRY
+  )).toEqual({
+    integrity: 'sha512-AAAA',
+  })
+})
+
+test('drops the tarball for standard registry URLs when lockfileIncludeTarballUrl is false', () => {
+  expect(toLockfileResolution(
+    { name: 'foo', version: '1.0.0' },
+    { integrity: 'sha512-AAAA', tarball: 'https://registry.npmjs.org/foo/-/foo-1.0.0.tgz' },
+    REGISTRY,
+    false
+  )).toEqual({
+    integrity: 'sha512-AAAA',
+  })
+})
+
+test('drops the tarball for non-standard registry URLs when lockfileIncludeTarballUrl is false', () => {
+  expect(toLockfileResolution(
+    { name: 'esprima-fb', version: '3001.1.0-dev-harmony-fb' },
+    { integrity: 'sha512-AAAA', tarball: 'https://example.com/esprima-fb/-/esprima-fb-3001.1.0-dev-harmony-fb.tgz' },
+    REGISTRY,
+    false
+  )).toEqual({
+    integrity: 'sha512-AAAA',
+  })
+})
+
+test('keeps file: tarballs even when lockfileIncludeTarballUrl is false', () => {
+  // file: tarballs cannot be reconstructed from name+version+registry, so the
+  // tarball field must remain so the package can be re-fetched on install.
+  expect(toLockfileResolution(
+    { name: 'test-package', version: '1.0.0' },
+    { integrity: 'sha512-AAAA', tarball: 'file:test-package-1.0.0.tgz' },
+    REGISTRY,
+    false
+  )).toEqual({
+    integrity: 'sha512-AAAA',
+    tarball: 'file:test-package-1.0.0.tgz',
+  })
+})
+
+test('keeps file: tarballs even when lockfileIncludeTarballUrl is undefined', () => {
+  expect(toLockfileResolution(
+    { name: 'test-package', version: '1.0.0' },
+    { integrity: 'sha512-AAAA', tarball: 'file:test-package-1.0.0.tgz' },
+    REGISTRY
+  )).toEqual({
+    integrity: 'sha512-AAAA',
+    tarball: 'file:test-package-1.0.0.tgz',
+  })
+})
+
+test('keeps git-hosted tarballs when lockfileIncludeTarballUrl is false', () => {
+  expect(toLockfileResolution(
+    { name: 'foo', version: '1.0.0' },
+    { integrity: 'sha512-AAAA', tarball: 'https://codeload.github.com/foo/bar/tar.gz/abcdef' },
+    REGISTRY,
+    false
+  )).toEqual({
+    integrity: 'sha512-AAAA',
+    tarball: 'https://codeload.github.com/foo/bar/tar.gz/abcdef',
+  })
+})