fix: use tarball URL returned in package metadata (#10431)

close #10254

.changeset/correct-tarball-url.md

@@ -0,0 +1,11 @@
+---
+"@pnpm/config.deps-installer": minor
+"@pnpm/workspace.state": minor
+"@pnpm/types": minor
+"@pnpm/cli-utils": minor
+"pnpm": minor
+---
+
+Fixed installation of config dependencies from private registries.
+
+Added support for object type in `configDependencies` when the tarball URL returned from package metadata differs from the computed URL [#10431](https://github.com/pnpm/pnpm/pull/10431).

cli/cli-utils/src/getConfig.ts

@@ -5,6 +5,7 @@ import { formatWarn } from '@pnpm/default-reporter'
 import { createOrConnectStoreController } from '@pnpm/store-connection-manager'
 import { installConfigDeps } from '@pnpm/config.deps-installer'
 import { requireHooks } from '@pnpm/pnpmfile'
+import { type ConfigDependencies } from '@pnpm/types'
 import { lexCompare } from '@pnpm/util.lex-comparator'
 
 export async function getConfig (
@@ -70,7 +71,7 @@ export async function getConfig (
   return config
 }
 
-function * calcPnpmfilePathsOfPluginDeps (configModulesDir: string, configDependencies: Record<string, string>): Generator<string> {
+function * calcPnpmfilePathsOfPluginDeps (configModulesDir: string, configDependencies: ConfigDependencies): Generator<string> {
   for (const configDepName of Object.keys(configDependencies).sort(lexCompare)) {
     if (isPluginName(configDepName)) {
       yield path.join(configModulesDir, configDepName, 'pnpmfile.cjs')

config/deps-installer/src/index.ts

@@ -1,2 +1,3 @@
 export { installConfigDeps, type InstallConfigDepsOpts } from './installConfigDeps.js'
 export { resolveConfigDeps, type ResolveConfigDepsOpts } from './resolveConfigDeps.js'
+export { normalizeConfigDeps } from './normalizeConfigDeps.js'

config/deps-installer/src/installConfigDeps.ts

@@ -1,13 +1,11 @@
 import path from 'path'
-import getNpmTarballUrl from 'get-npm-tarball-url'
 import { installingConfigDepsLogger } from '@pnpm/core-loggers'
-import { PnpmError } from '@pnpm/error'
-import { pickRegistryForPackage } from '@pnpm/pick-registry-for-package'
 import { readModulesDir } from '@pnpm/read-modules-dir'
 import rimraf from '@zkochan/rimraf'
 import { safeReadPackageJsonFromDir } from '@pnpm/read-package-json'
 import { type StoreController } from '@pnpm/package-store'
-import { type Registries } from '@pnpm/types'
+import { type ConfigDependencies, type Registries } from '@pnpm/types'
+import { normalizeConfigDeps } from './normalizeConfigDeps.js'
 
 export interface InstallConfigDepsOpts {
   registries: Registries
@@ -15,7 +13,7 @@ export interface InstallConfigDepsOpts {
   store: StoreController
 }
 
-export async function installConfigDeps (configDeps: Record<string, string>, opts: InstallConfigDepsOpts): Promise<void> {
+export async function installConfigDeps (configDeps: ConfigDependencies, opts: InstallConfigDepsOpts): Promise<void> {
   const configModulesDir = path.join(opts.rootDir, 'node_modules/.pnpm-config')
   const existingConfigDeps: string[] = await readModulesDir(configModulesDir) ?? []
   await Promise.all(existingConfigDeps.map(async (existingConfigDep) => {
@@ -23,41 +21,28 @@ export async function installConfigDeps (configDeps: Record<string, string>, opt
       await rimraf(path.join(configModulesDir, existingConfigDep))
     }
   }))
-  const installedConfigDeps: Array<{ name: string, version: string }> = []
-  await Promise.all(Object.entries(configDeps).map(async ([pkgName, pkgSpec]) => {
-    const configDepPath = path.join(configModulesDir, pkgName)
-    const sepIndex = pkgSpec.indexOf('+')
-    if (sepIndex === -1) {
-      throw new PnpmError('CONFIG_DEP_NO_INTEGRITY', `Your config dependency called "${pkgName}" at "pnpm.configDependencies" doesn't have an integrity checksum`, {
-        hint: `All config dependencies should have their integrity checksum inlined in the version specifier. For example:
 
-pnpm-workspace.yaml:
-configDependencies:
-  my-config: "1.0.0+sha512-Xg0tn4HcfTijTwfDwYlvVCl43V6h4KyVVX2aEm4qdO/PC6L2YvzLHFdmxhoeSA3eslcE6+ZVXHgWwopXYLNq4Q=="
-`,
-      })
-    }
-    const version = pkgSpec.substring(0, sepIndex)
-    const integrity = pkgSpec.substring(sepIndex + 1)
+  const installedConfigDeps: Array<{ name: string, version: string }> = []
+  const normalizedConfigDeps = normalizeConfigDeps(configDeps, {
+    registries: opts.registries,
+  })
+  await Promise.all(Object.entries(normalizedConfigDeps).map(async ([pkgName, pkg]) => {
+    const configDepPath = path.join(configModulesDir, pkgName)
     if (existingConfigDeps.includes(pkgName)) {
       const configDepPkgJson = await safeReadPackageJsonFromDir(configDepPath)
-      if (configDepPkgJson == null || configDepPkgJson.name !== pkgName || configDepPkgJson.version !== version) {
+      if (configDepPkgJson == null || configDepPkgJson.name !== pkgName || configDepPkgJson.version !== pkg.version) {
         await rimraf(configDepPath)
       } else {
         return
       }
     }
     installingConfigDepsLogger.debug({ status: 'started' })
-    const registry = pickRegistryForPackage(opts.registries, pkgName)
     const { fetching } = await opts.store.fetchPackage({
       force: true,
       lockfileDir: opts.rootDir,
       pkg: {
-        id: `${pkgName}@${version}`,
-        resolution: {
-          tarball: getNpmTarballUrl(pkgName, version, { registry }),
-          integrity,
-        },
+        id: `${pkgName}@${pkg.version}`,
+        resolution: pkg.resolution,
       },
     })
     const { files: filesResponse } = await fetching()
@@ -68,7 +53,7 @@ configDependencies:
     })
     installedConfigDeps.push({
       name: pkgName,
-      version,
+      version: pkg.version,
     })
   }))
   if (installedConfigDeps.length) {

config/deps-installer/src/normalizeConfigDeps.ts

@@ -0,0 +1,65 @@
+import getNpmTarballUrl from 'get-npm-tarball-url'
+import { PnpmError } from '@pnpm/error'
+import { pickRegistryForPackage } from '@pnpm/pick-registry-for-package'
+import { type ConfigDependencies, type Registries } from '@pnpm/types'
+
+interface NormalizeConfigDepsOpts {
+  registries: Registries
+}
+
+type NormalizedConfigDeps = Record<string, {
+  version: string
+  resolution: {
+    integrity: string
+    tarball: string
+  }
+}>
+
+export function normalizeConfigDeps (configDependencies: ConfigDependencies, opts: NormalizeConfigDepsOpts): NormalizedConfigDeps {
+  const deps: NormalizedConfigDeps = {}
+  for (const [pkgName, pkgSpec] of Object.entries(configDependencies)) {
+    const registry = pickRegistryForPackage(opts.registries, pkgName)
+
+    if (typeof pkgSpec === 'object') {
+      const { version, integrity } = parseIntegrity(pkgName, pkgSpec.integrity)
+      deps[pkgName] = {
+        version,
+        resolution: {
+          integrity,
+          tarball: pkgSpec.tarball ? pkgSpec.tarball : getNpmTarballUrl(pkgName, version, { registry }),
+        },
+      }
+      continue
+    }
+
+    if (typeof pkgSpec === 'string') {
+      const { version, integrity } = parseIntegrity(pkgName, pkgSpec)
+      deps[pkgName] = {
+        version,
+        resolution: {
+          integrity,
+          tarball: getNpmTarballUrl(pkgName, version, { registry }),
+        },
+      }
+    }
+  }
+
+  return deps
+}
+
+function parseIntegrity (pkgName: string, pkgSpec: string) {
+  const sepIndex = pkgSpec.indexOf('+')
+  if (sepIndex === -1) {
+    throw new PnpmError('CONFIG_DEP_NO_INTEGRITY', `Your config dependency called "${pkgName}" doesn't have an integrity checksum`, {
+      hint: `Integrity checksum should be inlined in the version specifier. For example:
+
+pnpm-workspace.yaml:
+configDependencies:
+  my-config: "1.0.0+sha512-Xg0tn4HcfTijTwfDwYlvVCl43V6h4KyVVX2aEm4qdO/PC6L2YvzLHFdmxhoeSA3eslcE6+ZVXHgWwopXYLNq4Q=="
+`,
+    })
+  }
+  const version = pkgSpec.substring(0, sepIndex)
+  const integrity = pkgSpec.substring(sepIndex + 1)
+  return { version, integrity }
+}

config/deps-installer/src/resolveConfigDeps.ts

@@ -1,3 +1,4 @@
+import getNpmTarballUrl from 'get-npm-tarball-url'
 import { PnpmError } from '@pnpm/error'
 import { writeSettings } from '@pnpm/config.config-writer'
 import { createFetchFromRegistry, type CreateFetchFromRegistryOptions } from '@pnpm/fetch'
@@ -5,6 +6,7 @@ import { createNpmResolver, type ResolverFactoryOptions } from '@pnpm/npm-resolv
 import { createGetAuthHeaderByURI } from '@pnpm/network.auth-header'
 import { parseWantedDependency } from '@pnpm/parse-wanted-dependency'
 import { type ConfigDependencies } from '@pnpm/types'
+import { pickRegistryForPackage } from '@pnpm/pick-registry-for-package'
 import { installConfigDeps, type InstallConfigDepsOpts } from './installConfigDeps.js'
 
 export type ResolveConfigDepsOpts = CreateFetchFromRegistryOptions & ResolverFactoryOptions & InstallConfigDepsOpts & {
@@ -31,7 +33,19 @@ export async function resolveConfigDeps (configDeps: string[], opts: ResolveConf
     if (resolution?.resolution == null || !('integrity' in resolution?.resolution)) {
       throw new PnpmError('BAD_CONFIG_DEP', `Cannot install ${configDep} as configuration dependency because it has no integrity`)
     }
-    configDependencies[wantedDep.alias] = `${resolution?.manifest?.version}+${resolution.resolution.integrity}`
+    const pkgName = wantedDep.alias
+    const version = resolution.manifest.version
+    const { tarball, integrity } = resolution.resolution
+    const registry = pickRegistryForPackage(opts.registries, pkgName)
+    const defaultTarball = getNpmTarballUrl(pkgName, version, { registry })
+    if (tarball !== defaultTarball && isValidHttpUrl(tarball)) {
+      configDependencies[pkgName] = {
+        tarball,
+        integrity: `${version}+${integrity}`,
+      }
+    } else {
+      configDependencies[pkgName] = `${version}+${integrity}`
+    }
   }))
   await writeSettings({
     ...opts,
@@ -43,3 +57,12 @@ export async function resolveConfigDeps (configDeps: string[], opts: ResolveConf
   })
   await installConfigDeps(configDependencies, opts)
 }
+
+function isValidHttpUrl (url: string): boolean {
+  try {
+    const parsed = new URL(url)
+    return parsed.protocol === 'http:' || parsed.protocol === 'https:'
+  } catch {
+    return false
+  }
+}

config/deps-installer/test/normalizeConfigDeps.test.ts

@@ -0,0 +1,83 @@
+import { normalizeConfigDeps } from '@pnpm/config.deps-installer'
+import { getIntegrity, REGISTRY_MOCK_PORT } from '@pnpm/registry-mock'
+
+const registry = `http://localhost:${REGISTRY_MOCK_PORT}/`
+
+test('normalizes string spec with integrity to structured dependency and computes tarball', () => {
+  const deps = normalizeConfigDeps({
+    '@pnpm.e2e/foo': `100.0.0+${getIntegrity('@pnpm.e2e/foo', '100.0.0')}`,
+  }, {
+    registries: {
+      default: registry,
+    },
+  })
+  expect(deps['@pnpm.e2e/foo']).toStrictEqual({
+    version: '100.0.0',
+    resolution: {
+      integrity: getIntegrity('@pnpm.e2e/foo', '100.0.0'),
+      tarball: `${registry}@pnpm.e2e/foo/-/foo-100.0.0.tgz`,
+    },
+  })
+})
+
+test('keeps provided tarball when resolution.tarball is specified', () => {
+  const customTarball = 'https://custom.example.com/foo-100.0.0.tgz'
+  const deps = normalizeConfigDeps({
+    '@pnpm.e2e/foo': {
+      integrity: `100.0.0+${getIntegrity('@pnpm.e2e/foo', '100.0.0')}`,
+      tarball: customTarball,
+    },
+  }, {
+    registries: {
+      default: registry,
+    },
+  })
+  expect(deps['@pnpm.e2e/foo']).toStrictEqual({
+    version: '100.0.0',
+    resolution: {
+      integrity: getIntegrity('@pnpm.e2e/foo', '100.0.0'),
+      tarball: customTarball,
+    },
+  })
+})
+
+test('computes tarball when not specified in object spec', () => {
+  const deps = normalizeConfigDeps({
+    '@pnpm.e2e/foo': {
+      integrity: `100.0.0+${getIntegrity('@pnpm.e2e/foo', '100.0.0')}`,
+    },
+  }, {
+    registries: {
+      default: registry,
+    },
+  })
+  expect(deps['@pnpm.e2e/foo']).toStrictEqual({
+    version: '100.0.0',
+    resolution: {
+      integrity: getIntegrity('@pnpm.e2e/foo', '100.0.0'),
+      tarball: `${registry}@pnpm.e2e/foo/-/foo-100.0.0.tgz`,
+    },
+  })
+})
+
+test('throws when string spec does not include integrity', () => {
+  expect(() => normalizeConfigDeps({
+    '@pnpm.e2e/foo': '100.0.0',
+  }, {
+    registries: {
+      default: registry,
+    },
+  })).toThrow("doesn't have an integrity checksum")
+})
+
+test('throws when object spec does not include integrity', () => {
+  expect(() => normalizeConfigDeps({
+    '@pnpm.e2e/foo': {
+      integrity: '100.0.0',
+    },
+  }, {
+    registries: {
+      default: registry,
+    },
+  })).toThrow("doesn't have an integrity checksum")
+})

packages/types/src/package.ts

@@ -144,7 +144,12 @@ export interface PeerDependencyRules {
 
 export type AllowedDeprecatedVersions = Record<string, string>
 
-export type ConfigDependencies = Record<string, string>
+type VersionWithIntegrity = string
+
+export type ConfigDependencies = Record<string, VersionWithIntegrity | {
+  tarball?: string
+  integrity: VersionWithIntegrity
+}>
 
 export interface AuditConfig {
   ignoreCves?: string[]

workspace/state/src/createWorkspaceState.ts

@@ -1,4 +1,5 @@
 import pick from 'ramda/src/pick'
+import { type ConfigDependencies } from '@pnpm/types'
 import { type WorkspaceState, type WorkspaceStateSettings, type ProjectsList } from './types.js'
 
 export interface CreateWorkspaceStateOptions {
@@ -6,7 +7,7 @@ export interface CreateWorkspaceStateOptions {
   pnpmfiles: string[]
   filteredInstall: boolean
   settings: WorkspaceStateSettings
-  configDependencies?: Record<string, string>
+  configDependencies?: ConfigDependencies
 }
 
 export const createWorkspaceState = (opts: CreateWorkspaceStateOptions): WorkspaceState => ({

workspace/state/src/types.ts

@@ -1,5 +1,5 @@
 import { type Config } from '@pnpm/config'
-import { type Project, type ProjectRootDir } from '@pnpm/types'
+import { type ConfigDependencies, type Project, type ProjectRootDir } from '@pnpm/types'
 
 export type ProjectsList = Array<Pick<Project, 'rootDir' | 'manifest'>>
 
@@ -11,7 +11,7 @@ export interface WorkspaceState {
   }>
   pnpmfiles: string[]
   filteredInstall: boolean
-  configDependencies?: Record<string, string>
+  configDependencies?: ConfigDependencies
   settings: WorkspaceStateSettings
 }
 

workspace/state/src/updateWorkspaceState.ts

@@ -1,6 +1,7 @@
 import fs from 'fs'
 import path from 'path'
 import { logger } from '@pnpm/logger'
+import { type ConfigDependencies } from '@pnpm/types'
 import { getFilePath } from './filePath.js'
 import { createWorkspaceState } from './createWorkspaceState.js'
 import { type WorkspaceStateSettings, type ProjectsList } from './types.js'
@@ -11,7 +12,7 @@ export interface UpdateWorkspaceStateOptions {
   workspaceDir: string
   pnpmfiles: string[]
   filteredInstall: boolean
-  configDependencies?: Record<string, string>
+  configDependencies?: ConfigDependencies
 }
 
 export async function updateWorkspaceState (opts: UpdateWorkspaceStateOptions): Promise<void> {