fix: make the peers suffix shorter (#8177)

ref #7079
2025-12-23 23:29:17 -05:00 · 2024-06-07 23:37:07 +02:00
parent 08dad01af0
commit 47341e5004
24 changed files with 85 additions and 7 deletions
--- a/.changeset/fluffy-birds-carry.md
+++ b/.changeset/fluffy-birds-carry.md
@@ -0,0 +1,16 @@
+---
+"@pnpm/plugin-commands-patching": minor
+"@pnpm/resolve-dependencies": minor
+"@pnpm/plugin-commands-audit": minor
+"@pnpm/plugin-commands-rebuild": minor
+"@pnpm/plugin-commands-store": minor
+"@pnpm/dependency-path": minor
+"@pnpm/lockfile-types": minor
+"@pnpm/get-context": minor
+"@pnpm/lockfile-file": minor
+"@pnpm/core": minor
+"@pnpm/config": minor
+"pnpm": minor
+---
+
+**Semi-breaking.** Dependency key names in the lockfile are shortened if they are longer than 1000 characters. We don't expect this change to affect many users. This change is required to fix some edge cases in which installation fails with an out-of-memory error or "Invalid string length (RangeError: Invalid string length)" error. The max allowed length of the dependency key can be controlled with the `peers-suffix-max-length` setting [#8177](https://github.com/pnpm/pnpm/pull/8177).
--- a/config/config/src/Config.ts
+++ b/config/config/src/Config.ts
@@ -194,6 +194,7 @@ export interface Config {
  packageManagerStrict?: boolean
  packageManagerStrictVersion?: boolean
  virtualStoreDirMaxLength: number
+  peersSuffixMaxLength?: number
 }

 export interface ConfigWithDeprecatedSettings extends Config {
--- a/config/config/src/index.ts
+++ b/config/config/src/index.ts
@@ -141,6 +141,7 @@ export const types = Object.assign({
  'verify-store-integrity': Boolean,
  'virtual-store-dir': String,
  'virtual-store-dir-max-length': Number,
+  'peers-suffix-max-length': Number,
  'workspace-concurrency': Number,
  'workspace-packages': [String, Array],
  'workspace-root': Boolean,
@@ -275,6 +276,7 @@ export async function getConfig (
    'embed-readme': false,
    'registry-supports-time-field': false,
    'virtual-store-dir-max-length': 120,
+    'peers-suffix-max-length': 1000,
  }

  const { config: npmConfig, warnings, failedToLoadBuiltInConfig } = loadNpmConf(cliOptions, rcOptionsTypes, defaultOptions)
--- a/exec/plugin-commands-rebuild/src/implementation/extendRebuildOptions.ts
+++ b/exec/plugin-commands-rebuild/src/implementation/extendRebuildOptions.ts
@@ -48,6 +48,7 @@ export type StrictRebuildOptions = {
  neverBuiltDependencies?: string[]
  onlyBuiltDependencies?: string[]
  virtualStoreDirMaxLength: number
+  peersSuffixMaxLength: number
 } & Pick<Config, 'sslConfigs'>

 export type RebuildOptions = Partial<StrictRebuildOptions> &
--- a/lockfile/lockfile-file/src/lockfileFormatConverters.ts
+++ b/lockfile/lockfile-file/src/lockfileFormatConverters.ts
@@ -62,6 +62,9 @@ export function convertToLockfileFile (lockfile: Lockfile, opts: NormalizeLockfi
    lockfileVersion: LOCKFILE_VERSION,
    importers: mapValues(lockfile.importers, convertProjectSnapshotToInlineSpecifiersFormat),
  }
+  if (newLockfile.settings?.peersSuffixMaxLength === 1000) {
+    newLockfile.settings = omit(['peersSuffixMaxLength'], newLockfile.settings)
+  }
  return normalizeLockfile(newLockfile, opts)
 }

--- a/lockfile/lockfile-file/src/read.ts
+++ b/lockfile/lockfile-file/src/read.ts
@@ -138,6 +138,7 @@ export function createLockfileObject (
    lockfileVersion: string
    autoInstallPeers: boolean
    excludeLinksFromLockfile: boolean
+    peersSuffixMaxLength: number
  }
 ): Lockfile {
  const importers = importerIds.reduce((acc, importerId) => {
@@ -153,6 +154,7 @@ export function createLockfileObject (
    settings: {
      autoInstallPeers: opts.autoInstallPeers,
      excludeLinksFromLockfile: opts.excludeLinksFromLockfile,
+      peersSuffixMaxLength: opts.peersSuffixMaxLength,
    },
  }
 }
--- a/lockfile/lockfile-types/src/index.ts
+++ b/lockfile/lockfile-types/src/index.ts
@@ -7,6 +7,7 @@ export * from './lockfileFileTypes'
 export interface LockfileSettings {
  autoInstallPeers?: boolean
  excludeLinksFromLockfile?: boolean
+  peersSuffixMaxLength?: number
 }

 export interface Lockfile {
--- a/lockfile/plugin-commands-audit/test/index.ts
+++ b/lockfile/plugin-commands-audit/test/index.ts
@@ -61,6 +61,7 @@ export const DEFAULT_OPTS = {
  useStoreServer: false,
  workspaceConcurrency: 4,
  virtualStoreDirMaxLength: 120,
+  peersSuffixMaxLength: 1000,
 }

 describe('plugin-commands-audit', () => {
--- a/packages/dependency-path/src/index.ts
+++ b/packages/dependency-path/src/index.ts
@@ -191,8 +191,8 @@ function depPathToFilenameUnescaped (depPath: string): string {

 export type PeerId = { name: string, version: string } | string

-export function createPeersDirSuffix (peerIds: PeerId[]): string {
-  const dirName = peerIds.map(
+export function createPeersDirSuffix (peerIds: PeerId[], maxLength: number = 1000): string {
+  let dirName = peerIds.map(
    (peerId) => {
      if (typeof peerId !== 'string') {
        return `${peerId.name}@${peerId.version}`
@@ -203,5 +203,8 @@ export function createPeersDirSuffix (peerIds: PeerId[]): string {
      return peerId
    }
  ).sort().join(')(')
+  if (dirName.length > maxLength) {
+    dirName = createBase32Hash(dirName)
+  }
  return `(${dirName})`
 }
--- a/patching/plugin-commands-patching/test/utils/index.ts
+++ b/patching/plugin-commands-patching/test/utils/index.ts
@@ -54,4 +54,5 @@ export const DEFAULT_OPTS = {
    libc: ['current'],
  },
  virtualStoreDirMaxLength: 120,
+  peersSuffixMaxLength: 1000,
 }
--- a/pkg-manager/core/src/getPeerDependencyIssues.ts
+++ b/pkg-manager/core/src/getPeerDependencyIssues.ts
@@ -21,10 +21,11 @@ export type ListMissingPeersOptions = Partial<GetContextOptions>
 | 'storeController'
 | 'useGitBranchLockfile'
 | 'workspacePackages'
+| 'peersSuffixMaxLength'
 >
 & Partial<Pick<InstallOptions, 'supportedArchitectures'>>
 & Pick<GetContextOptions, 'autoInstallPeers' | 'excludeLinksFromLockfile' | 'storeDir'>
-& Required<Pick<InstallOptions, 'virtualStoreDirMaxLength'>>
+& Required<Pick<InstallOptions, 'virtualStoreDirMaxLength' | 'peersSuffixMaxLength'>>

 export async function getPeerDependencyIssues (
  projects: ProjectOptions[],
@@ -91,6 +92,7 @@ export async function getPeerDependencyIssues (
      wantedLockfile: ctx.wantedLockfile,
      workspacePackages: opts.workspacePackages ?? {},
      supportedArchitectures: opts.supportedArchitectures,
+      peersSuffixMaxLength: opts.peersSuffixMaxLength,
    }
  )

--- a/pkg-manager/core/src/install/extendInstallOptions.ts
+++ b/pkg-manager/core/src/install/extendInstallOptions.ts
@@ -145,6 +145,7 @@ export interface StrictInstallOptions {
  supportedArchitectures?: SupportedArchitectures
  hoistWorkspacePackages?: boolean
  virtualStoreDirMaxLength: number
+  peersSuffixMaxLength: number
 }

 export type InstallOptions =
@@ -239,6 +240,7 @@ const defaults = (opts: InstallOptions): StrictInstallOptions => {
    disallowWorkspaceCycles: false,
    excludeLinksFromLockfile: false,
    virtualStoreDirMaxLength: 120,
+    peersSuffixMaxLength: 1000,
  } as StrictInstallOptions
 }

--- a/pkg-manager/core/src/install/index.ts
+++ b/pkg-manager/core/src/install/index.ts
@@ -341,6 +341,7 @@ export async function mutateModules (
      const outdatedLockfileSettingName = getOutdatedLockfileSetting(ctx.wantedLockfile, {
        autoInstallPeers: opts.autoInstallPeers,
        excludeLinksFromLockfile: opts.excludeLinksFromLockfile,
+        peersSuffixMaxLength: opts.peersSuffixMaxLength,
        overrides: opts.overrides,
        ignoredOptionalDependencies: opts.ignoredOptionalDependencies?.sort(),
        packageExtensionsChecksum,
@@ -361,6 +362,7 @@ export async function mutateModules (
      ctx.wantedLockfile.settings = {
        autoInstallPeers: opts.autoInstallPeers,
        excludeLinksFromLockfile: opts.excludeLinksFromLockfile,
+        peersSuffixMaxLength: opts.peersSuffixMaxLength,
      }
      ctx.wantedLockfile.overrides = opts.overrides
      ctx.wantedLockfile.packageExtensionsChecksum = packageExtensionsChecksum
@@ -371,6 +373,7 @@ export async function mutateModules (
      ctx.wantedLockfile.settings = {
        autoInstallPeers: opts.autoInstallPeers,
        excludeLinksFromLockfile: opts.excludeLinksFromLockfile,
+        peersSuffixMaxLength: opts.peersSuffixMaxLength,
      }
    }
    if (
@@ -724,27 +727,28 @@ type ChangedField =
  | 'ignoredOptionalDependencies'
  | 'settings.autoInstallPeers'
  | 'settings.excludeLinksFromLockfile'
+  | 'settings.peersSuffixMaxLength'
  | 'pnpmfileChecksum'

 function getOutdatedLockfileSetting (
  lockfile: Lockfile,
  {
-    onlyBuiltDependencies,
    overrides,
    packageExtensionsChecksum,
    ignoredOptionalDependencies,
    patchedDependencies,
    autoInstallPeers,
    excludeLinksFromLockfile,
+    peersSuffixMaxLength,
    pnpmfileChecksum,
  }: {
-    onlyBuiltDependencies?: string[]
    overrides?: Record<string, string>
    packageExtensionsChecksum?: string
    patchedDependencies?: Record<string, PatchFile>
    ignoredOptionalDependencies?: string[]
    autoInstallPeers?: boolean
    excludeLinksFromLockfile?: boolean
+    peersSuffixMaxLength?: number
    pnpmfileChecksum?: string
  }
 ): ChangedField | null {
@@ -766,6 +770,12 @@ function getOutdatedLockfileSetting (
  if (lockfile.settings?.excludeLinksFromLockfile != null && lockfile.settings.excludeLinksFromLockfile !== excludeLinksFromLockfile) {
    return 'settings.excludeLinksFromLockfile'
  }
+  if (
+    lockfile.settings?.peersSuffixMaxLength != null && lockfile.settings.peersSuffixMaxLength !== peersSuffixMaxLength ||
+    lockfile.settings?.peersSuffixMaxLength == null && peersSuffixMaxLength !== 1000
+  ) {
+    return 'settings.peersSuffixMaxLength'
+  }
  if (lockfile.pnpmfileChecksum !== pnpmfileChecksum) {
    return 'pnpmfileChecksum'
  }
@@ -1059,6 +1069,7 @@ const _installInContext: InstallFunction = async (projects, ctx, opts) => {
      lockfileIncludeTarballUrl: opts.lockfileIncludeTarballUrl,
      resolvePeersFromWorkspaceRoot: opts.resolvePeersFromWorkspaceRoot,
      supportedArchitectures: opts.supportedArchitectures,
+      peersSuffixMaxLength: opts.peersSuffixMaxLength,
    }
  )
  if (!opts.include.optionalDependencies || !opts.include.devDependencies || !opts.include.dependencies) {
--- a/pkg-manager/core/src/link/options.ts
+++ b/pkg-manager/core/src/link/options.ts
@@ -35,6 +35,7 @@ interface StrictLinkOptions {
  useGitBranchLockfile: boolean
  mergeGitBranchLockfiles: boolean
  virtualStoreDirMaxLength: number
+  peersSuffixMaxLength: number
 }

 export type LinkOptions =
--- a/pkg-manager/core/test/lockfile.ts
+++ b/pkg-manager/core/test/lockfile.ts
@@ -1566,3 +1566,13 @@ test('installation should work with packages that have () in the scope name', as
  const manifest = await addDependenciesToPackage({}, ['@(-.-)/env@0.3.1'], opts)
  await install(manifest, opts)
 })
+
+test('setting a custom peersSuffixMaxLength', async () => {
+  const project = prepareEmpty()
+
+  await addDependenciesToPackage({}, ['@pnpm.e2e/abc@1.0.0'], testDefaults({ peersSuffixMaxLength: 10 }))
+
+  const lockfile = project.readLockfile()
+  expect(lockfile.settings.peersSuffixMaxLength).toBe(10)
+  expect(lockfile.importers['.']?.dependencies?.['@pnpm.e2e/abc']?.version?.length).toBe(33)
+})
--- a/pkg-manager/get-context/src/index.ts
+++ b/pkg-manager/get-context/src/index.ts
@@ -77,6 +77,7 @@ interface HookOptions {
 export interface GetContextOptions {
  autoInstallPeers: boolean
  excludeLinksFromLockfile: boolean
+  peersSuffixMaxLength: number
  allProjects: Array<ProjectOptions & HookOptions>
  confirmModulesPurge?: boolean
  force: boolean
@@ -189,6 +190,7 @@ export async function getContext (
    ...await readLockfiles({
      autoInstallPeers: opts.autoInstallPeers,
      excludeLinksFromLockfile: opts.excludeLinksFromLockfile,
+      peersSuffixMaxLength: opts.peersSuffixMaxLength,
      force: opts.force,
      frozenLockfile: opts.frozenLockfile === true,
      lockfileDir: opts.lockfileDir,
@@ -423,6 +425,7 @@ export async function getContextForSingleImporter (
  opts: {
    autoInstallPeers: boolean
    excludeLinksFromLockfile: boolean
+    peersSuffixMaxLength: number
    force: boolean
    forceNewModules?: boolean
    confirmModulesPurge?: boolean
@@ -540,6 +543,7 @@ export async function getContextForSingleImporter (
    ...await readLockfiles({
      autoInstallPeers: opts.autoInstallPeers,
      excludeLinksFromLockfile: opts.excludeLinksFromLockfile,
+      peersSuffixMaxLength: opts.peersSuffixMaxLength,
      force: opts.force,
      frozenLockfile: false,
      lockfileDir: opts.lockfileDir,
--- a/pkg-manager/get-context/src/readLockfiles.ts
+++ b/pkg-manager/get-context/src/readLockfiles.ts
@@ -30,6 +30,7 @@ export async function readLockfiles (
  opts: {
    autoInstallPeers: boolean
    excludeLinksFromLockfile: boolean
+    peersSuffixMaxLength: number
    force: boolean
    frozenLockfile: boolean
    projects: Array<{
@@ -111,6 +112,7 @@ export async function readLockfiles (
    autoInstallPeers: opts.autoInstallPeers,
    excludeLinksFromLockfile: opts.excludeLinksFromLockfile,
    lockfileVersion: wantedLockfileVersion,
+    peersSuffixMaxLength: opts.peersSuffixMaxLength,
  }
  const importerIds = opts.projects.map((importer) => importer.id)
  const currentLockfile = files[1] ?? createLockfileObject(importerIds, sopts)
--- a/pkg-manager/get-context/test/index.ts
+++ b/pkg-manager/get-context/test/index.ts
@@ -21,6 +21,7 @@ const DEFAULT_OPTIONS: GetContextOptions = {
  },
  storeDir: path.join(__dirname, 'store'),
  virtualStoreDirMaxLength: 120,
+  peersSuffixMaxLength: 1000,
 }

 test('getContext - extendNodePath false', async () => {
--- a/pkg-manager/resolve-dependencies/src/index.ts
+++ b/pkg-manager/resolve-dependencies/src/index.ts
@@ -206,6 +206,7 @@ export async function resolveDependencies (
    virtualStoreDirMaxLength: opts.virtualStoreDirMaxLength,
    resolvePeersFromWorkspaceRoot: Boolean(opts.resolvePeersFromWorkspaceRoot),
    resolvedImporters,
+    peersSuffixMaxLength: opts.peersSuffixMaxLength,
  })

  const linkedDependenciesByProjectId: Record<string, LinkedDependency[]> = {}
--- a/pkg-manager/resolve-dependencies/src/resolveDependencyTree.ts
+++ b/pkg-manager/resolve-dependencies/src/resolveDependencyTree.ts
@@ -102,6 +102,7 @@ export interface ResolveDependenciesOptions {
  workspacePackages: WorkspacePackages
  supportedArchitectures?: SupportedArchitectures
  updateToLatest?: boolean
+  peersSuffixMaxLength: number
 }

 export interface ResolveDependencyTreeResult {
--- a/pkg-manager/resolve-dependencies/src/resolvePeers.ts
+++ b/pkg-manager/resolve-dependencies/src/resolvePeers.ts
@@ -88,6 +88,7 @@ export async function resolvePeers<T extends PartialResolvedPackage> (
    dedupePeerDependents?: boolean
    dedupeInjectedDeps?: boolean
    resolvedImporters: ResolvedImporters
+    peersSuffixMaxLength: number
  }
 ): Promise<{
    dependenciesGraph: GenericDependenciesGraphWithResolvedChildren<T>
@@ -130,6 +131,7 @@ export async function resolvePeers<T extends PartialResolvedPackage> (
      peersCache: new Map(),
      peerDependencyIssues,
      purePkgs: new Set(),
+      peersSuffixMaxLength: opts.peersSuffixMaxLength,
      rootDir,
      virtualStoreDir: opts.virtualStoreDir,
      virtualStoreDirMaxLength: opts.virtualStoreDirMaxLength,
@@ -370,6 +372,7 @@ async function resolvePeersOfNode<T extends PartialResolvedPackage> (
    purePkgs: Set<PkgIdWithPatchHash> // pure packages are those that don't rely on externally resolved peers
    rootDir: string
    lockfileDir: string
+    peersSuffixMaxLength: number
  }
 ): Promise<PeersResolution & { finishing?: FinishingResolutionPromise, calculateDepPath?: CalculateDepPath }> {
  const node = ctx.dependenciesTree.get(nodeId)!
@@ -509,7 +512,7 @@ async function resolvePeersOfNode<T extends PartialResolvedPackage> (
      pendingPeerNodeIds.push(peerNodeId)
    }
    if (pendingPeerNodeIds.length === 0) {
-      const peersDirSuffix = createPeersDirSuffix(peerIds)
+      const peersDirSuffix = createPeersDirSuffix(peerIds, ctx.peersSuffixMaxLength)
      addDepPathToGraph(`${resolvedPackage.pkgIdWithPatchHash}${peersDirSuffix}` as DepPath)
    } else {
      calculateDepPathIfNeeded = calculateDepPath.bind(null, peerIds, pendingPeerNodeIds)
@@ -547,7 +550,7 @@ async function resolvePeersOfNode<T extends PartialResolvedPackage> (
          return ctx.pathsByNodeIdPromises.get(peerNodeId)!.promise
        })
      ),
-    ])
+    ], ctx.peersSuffixMaxLength)
    addDepPathToGraph(`${resolvedPackage.pkgIdWithPatchHash}${peersDirSuffix}` as DepPath)
  }

@@ -732,6 +735,7 @@ async function resolvePeersOfChildren<T extends PartialResolvedPackage> (
    dependenciesTree: DependenciesTree<T>
    rootDir: string
    lockfileDir: string
+    peersSuffixMaxLength: number
  }
 ): Promise<PeersResolution & { finishing: Promise<void> }> {
  const allResolvedPeers = new Map<string, NodeId>()
--- a/pkg-manager/resolve-dependencies/test/dedupeDepPaths.test.ts
+++ b/pkg-manager/resolve-dependencies/test/dedupeDepPaths.test.ts
@@ -101,6 +101,7 @@ test('packages are not deduplicated when versions do not match', async () => {
    virtualStoreDir: '',
    virtualStoreDirMaxLength: 120,
    lockfileDir: '',
+    peersSuffixMaxLength: 1000,
  })

  expect(dependenciesByProjectId.project1.get('foo')).toEqual(dependenciesByProjectId.project2.get('foo'))
--- a/pkg-manager/resolve-dependencies/test/resolvePeers.ts
+++ b/pkg-manager/resolve-dependencies/test/resolvePeers.ts
@@ -105,6 +105,7 @@ test('resolve peer dependencies of cyclic dependencies', async () => {
    virtualStoreDir: '',
    lockfileDir: '',
    virtualStoreDirMaxLength: 120,
+    peersSuffixMaxLength: 1000,
  })
  expect(Object.keys(dependenciesGraph)).toStrictEqual([
    'foo/1.0.0',
@@ -208,6 +209,7 @@ test('when a package is referenced twice in the dependencies graph and one of th
    virtualStoreDir: '',
    virtualStoreDirMaxLength: 120,
    lockfileDir: '',
+    peersSuffixMaxLength: 1000,
  })
  expect(Object.keys(dependenciesGraph).sort()).toStrictEqual([
    'bar/1.0.0',
@@ -389,6 +391,7 @@ describe('peer dependency issues', () => {
      virtualStoreDir: '',
      virtualStoreDirMaxLength: 120,
      lockfileDir: '',
+      peersSuffixMaxLength: 1000,
    })).peerDependencyIssuesByProjects
  })
  it('should find peer dependency conflicts', () => {
@@ -474,6 +477,7 @@ describe('unmet peer dependency issues', () => {
      virtualStoreDir: '',
      virtualStoreDirMaxLength: 120,
      lockfileDir: '',
+      peersSuffixMaxLength: 1000,
    })).peerDependencyIssuesByProjects
  })
  it('should not warn when the found package has prerelease version and the wanted range is *', () => {
@@ -546,6 +550,7 @@ describe('unmet peer dependency issue resolved from subdependency', () => {
      virtualStoreDir: '',
      virtualStoreDirMaxLength: 120,
      lockfileDir: '',
+      peersSuffixMaxLength: 1000,
    })).peerDependencyIssuesByProjects
  })
  it('should return from where the bad peer dependency is resolved', () => {
@@ -647,6 +652,7 @@ test('resolve peer dependencies with npm aliases', async () => {
    virtualStoreDir: '',
    virtualStoreDirMaxLength: 120,
    lockfileDir: '',
+    peersSuffixMaxLength: 1000,
  })
  expect(Object.keys(dependenciesGraph).sort()).toStrictEqual([
    'bar/1.0.0',
--- a/store/plugin-commands-store/src/storeStatus/extendStoreStatusOptions.ts
+++ b/store/plugin-commands-store/src/storeStatus/extendStoreStatusOptions.ts
@@ -21,6 +21,7 @@ export interface StrictStoreStatusOptions {
  optional: boolean
  binsDir: string
  virtualStoreDirMaxLength: number
+  peersSuffixMaxLength: number
 }

 export type StoreStatusOptions = Partial<StrictStoreStatusOptions> &