feat: packageExtensions (#3553)

close #3551

.changeset/dull-crabs-shave.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/lockfile-file": minor
+---
+
+New optional field added to the lockfile: `packageExtensionsChecksum`.

.changeset/mighty-pugs-wave.md

@@ -0,0 +1,5 @@
+---
+"supi": patch
+---
+
+A new optional field supported in the root `package.json` file: `pnpm.packageExtensions`. This new field allows to extend manifests of dependencies during installation.

.changeset/thirty-readers-pull.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/types": minor
+---
+
+New optional field added to the manifest type (`package.json`): `pnpm.packageExtensions.

packages/lockfile-file/src/sortLockfileKeys.ts

@@ -33,6 +33,7 @@ const ROOT_KEYS_ORDER = {
   lockfileVersion: 1,
   neverBuiltDependencies: 2,
   overrides: 3,
+  packageExtensionsChecksum: 4,
   specifiers: 10,
   dependencies: 11,
   optionalDependencies: 12,

packages/lockfile-file/src/write.ts

@@ -123,6 +123,9 @@ export function normalizeLockfile (lockfile: Lockfile, forceSharedFormat: boolea
       lockfileToSave.neverBuiltDependencies = lockfileToSave.neverBuiltDependencies.sort()
     }
   }
+  if (!lockfileToSave.packageExtensionsChecksum) {
+    delete lockfileToSave.packageExtensionsChecksum
+  }
   return lockfileToSave
 }
 

packages/lockfile-types/src/index.ts

@@ -4,6 +4,7 @@ export interface Lockfile {
   packages?: PackageSnapshots
   neverBuiltDependencies?: string[]
   overrides?: Record<string, string>
+  packageExtensionsChecksum?: string
 }
 
 export interface ProjectSnapshot {

packages/supi/src/install/createPackageExtender.ts

@@ -0,0 +1,43 @@
+import { PackageManifest, PackageExtension, ReadPackageHook } from '@pnpm/types'
+import parseWantedDependency from '@pnpm/parse-wanted-dependency'
+import semver from 'semver'
+
+interface PackageExtensionMatch {
+  packageExtension: PackageExtension
+  range: string | undefined
+}
+
+export default function (
+  packageExtensions: Record<string, PackageExtension>
+): ReadPackageHook {
+  const extensionsByPkgName = {} as Record<string, PackageExtensionMatch[]>
+  Object.entries(packageExtensions)
+    .forEach(([selector, packageExtension]) => {
+      const { alias, pref } = parseWantedDependency(selector)
+      if (!extensionsByPkgName[alias!]) {
+        extensionsByPkgName[alias!] = []
+      }
+      extensionsByPkgName[alias!].push({ packageExtension, range: pref })
+    })
+  return extendPkgHook.bind(null, extensionsByPkgName) as ReadPackageHook
+}
+
+function extendPkgHook (extensionsByPkgName: Record<string, PackageExtensionMatch[]>, manifest: PackageManifest) {
+  const extensions = extensionsByPkgName[manifest.name]
+  if (extensions == null) return manifest
+  extendPkg(manifest, extensions)
+  return manifest
+}
+
+function extendPkg (manifest: PackageManifest, extensions: PackageExtensionMatch[]) {
+  for (const { range, packageExtension } of extensions) {
+    if (range != null && !semver.satisfies(manifest.version, range)) continue
+    for (const field of ['dependencies', 'optionalDependencies', 'peerDependencies', 'peerDependenciesMeta']) {
+      if (!packageExtension[field]) continue
+      manifest[field] = {
+        ...packageExtension[field],
+        ...manifest[field],
+      }
+    }
+  }
+}

packages/supi/src/install/createVersionsOverrider.ts

@@ -4,7 +4,10 @@ import parseWantedDependency from '@pnpm/parse-wanted-dependency'
 import normalizePath from 'normalize-path'
 import semver from 'semver'
 
-export default function (overrides: Record<string, string>, rootDir: string): ReadPackageHook {
+export default function (
+  overrides: Record<string, string>,
+  rootDir: string
+): ReadPackageHook {
   const genericVersionOverrides = [] as VersionOverride[]
   const versionOverrides = [] as VersionOverrideWithParent[]
   Object.entries(overrides)

packages/supi/src/install/index.ts

@@ -1,3 +1,4 @@
+import crypto from 'crypto'
 import path from 'path'
 import buildModules, { linkBinsOfDependencies } from '@pnpm/build-modules'
 import {
@@ -42,6 +43,7 @@ import {
 import {
   DependenciesField,
   DependencyManifest,
+  PackageExtension,
   ProjectManifest,
   ReadPackageHook,
 } from '@pnpm/types'
@@ -52,12 +54,14 @@ import pLimit from 'p-limit'
 import fromPairs from 'ramda/src/fromPairs'
 import equals from 'ramda/src/equals'
 import isEmpty from 'ramda/src/isEmpty'
+import pipe from 'ramda/src/pipe'
 import props from 'ramda/src/props'
 import unnest from 'ramda/src/unnest'
 import parseWantedDependencies from '../parseWantedDependencies'
 import safeIsInnerLink from '../safeIsInnerLink'
 import removeDeps from '../uninstall/removeDeps'
 import allProjectsAreUpToDate from './allProjectsAreUpToDate'
+import createPackageExtender from './createPackageExtender'
 import createVersionsOverrider from './createVersionsOverrider'
 import extendOptions, {
   InstallOptions,
@@ -156,15 +160,13 @@ export async function mutateModules (
     ? rootProjectManifest.pnpm?.overrides ?? rootProjectManifest.resolutions
     : undefined
   const neverBuiltDependencies = rootProjectManifest?.pnpm?.neverBuiltDependencies ?? []
-  if (!isEmpty(overrides ?? {})) {
-    const versionsOverrider = createVersionsOverrider(overrides!, opts.lockfileDir)
-    if (opts.hooks.readPackage != null) {
-      const readPackage = opts.hooks.readPackage
-      opts.hooks.readPackage = ((manifest: ProjectManifest, dir?: string) => versionsOverrider(readPackage(manifest, dir), dir)) as ReadPackageHook
-    } else {
-      opts.hooks.readPackage = versionsOverrider
-    }
-  }
+  const packageExtensions = rootProjectManifest?.pnpm?.packageExtensions
+  opts.hooks.readPackage = createReadPackageHook({
+    readPackageHook: opts.hooks.readPackage,
+    overrides,
+    lockfileDir: opts.lockfileDir,
+    packageExtensions,
+  })
   const ctx = await getContext(projects, opts)
   const pruneVirtualStore = ctx.modulesFile?.prunedAt && opts.modulesCacheMaxAge > 0
     ? cacheExpired(ctx.modulesFile.prunedAt, opts.modulesCacheMaxAge)
@@ -187,10 +189,13 @@ export async function mutateModules (
   return result
 
   async function _install (): Promise<Array<{ rootDir: string, manifest: ProjectManifest }>> {
+    const packageExtensionsChecksum = isEmpty(packageExtensions ?? {}) ? undefined : createObjectChecksum(packageExtensions!)
     let needsFullResolution = !equals(ctx.wantedLockfile.overrides ?? {}, overrides ?? {}) ||
-      !equals((ctx.wantedLockfile.neverBuiltDependencies ?? []).sort(), (neverBuiltDependencies ?? []).sort())
+      !equals((ctx.wantedLockfile.neverBuiltDependencies ?? []).sort(), (neverBuiltDependencies ?? []).sort()) ||
+      ctx.wantedLockfile.packageExtensionsChecksum !== packageExtensionsChecksum
     ctx.wantedLockfile.overrides = overrides
     ctx.wantedLockfile.neverBuiltDependencies = neverBuiltDependencies
+    ctx.wantedLockfile.packageExtensionsChecksum = packageExtensionsChecksum
     const frozenLockfile = opts.frozenLockfile ||
       opts.frozenLockfileIfExists && ctx.existsWantedLockfile
     if (
@@ -473,6 +478,41 @@ export async function mutateModules (
   }
 }
 
+export function createObjectChecksum (obj: Object) {
+  const s = JSON.stringify(obj)
+  return crypto.createHash('md5').update(s).digest('hex')
+}
+
+function createReadPackageHook (
+  {
+    lockfileDir,
+    overrides,
+    packageExtensions,
+    readPackageHook,
+  }: {
+    lockfileDir: string
+    overrides?: Record<string, string>
+    packageExtensions?: Record<string, PackageExtension>
+    readPackageHook?: ReadPackageHook
+  }
+) {
+  const hooks: ReadPackageHook[] = []
+  if (!isEmpty(overrides ?? {})) {
+    hooks.push(createVersionsOverrider(overrides!, lockfileDir))
+  }
+  if (!isEmpty(packageExtensions ?? {})) {
+    hooks.push(createPackageExtender(packageExtensions!))
+  }
+  if (hooks.length === 0) {
+    return readPackageHook
+  }
+  const readPackageAndExtend = hooks.length === 1 ? hooks[0] : pipe(hooks[0], hooks[1]) as ReadPackageHook
+  if (readPackageHook != null) {
+    return ((manifest: ProjectManifest, dir?: string) => readPackageAndExtend(readPackageHook(manifest, dir), dir)) as ReadPackageHook
+  }
+  return readPackageAndExtend
+}
+
 function cacheExpired (prunedAt: string, maxAgeInMinutes: number) {
   return ((Date.now() - new Date(prunedAt).valueOf()) / (1000 * 60)) > maxAgeInMinutes
 }

packages/supi/test/install/createPackageExtender.test.ts

@@ -0,0 +1,98 @@
+import createPackageExtender from 'supi/lib/install/createPackageExtender'
+
+const packageExtender = createPackageExtender({
+  foo: {
+    dependencies: {
+      a: '1',
+    },
+    optionalDependencies: {
+      b: '2',
+    },
+    peerDependencies: {
+      c: '3',
+    },
+    peerDependenciesMeta: {
+      c: {
+        optional: true,
+      },
+    },
+  },
+  'bar@1': {
+    dependencies: {
+      d: '1',
+    },
+  },
+  qar: {
+    dependencies: {
+      e: '1',
+      f: '1',
+    },
+  },
+})
+
+test('createPackageExtender() extends the supported fields', () => {
+  expect(
+    packageExtender({
+      name: 'foo',
+      dependencies: {
+        bar: '^1.0.0',
+      },
+    })
+  ).toStrictEqual({
+    name: 'foo',
+    dependencies: {
+      bar: '^1.0.0',
+      a: '1',
+    },
+    optionalDependencies: {
+      b: '2',
+    },
+    peerDependencies: {
+      c: '3',
+    },
+    peerDependenciesMeta: {
+      c: {
+        optional: true,
+      },
+    },
+  })
+})
+
+test('createPackageExtender() does not change packages that should not be extended', () => {
+  const manifest = { name: 'ignore' }
+  expect(packageExtender(manifest)).toStrictEqual(manifest)
+})
+
+test('createPackageExtender() matches by version', () => {
+  expect(
+    packageExtender({
+      name: 'bar',
+      version: '1.0.0',
+    })
+  ).toStrictEqual({
+    name: 'bar',
+    version: '1.0.0',
+    dependencies: {
+      d: '1',
+    },
+  })
+})
+
+test('createPackageExtender() does not override existing fields', () => {
+  expect(
+    packageExtender({
+      name: 'qar',
+      version: '1.0.0',
+      dependencies: {
+        e: '100',
+      },
+    })
+  ).toStrictEqual({
+    name: 'qar',
+    version: '1.0.0',
+    dependencies: {
+      e: '100',
+      f: '1',
+    },
+  })
+})

packages/supi/test/install/packageExtensions.ts

@@ -0,0 +1,102 @@
+import PnpmError from '@pnpm/error'
+import { prepareEmpty } from '@pnpm/prepare'
+import { addDependenciesToPackage, mutateModules } from 'supi'
+import { createObjectChecksum } from 'supi/lib/install/index'
+import {
+  testDefaults,
+} from '../utils'
+
+test('manifests are extended with fields specified by pnpm.packageExtensions', async () => {
+  const project = prepareEmpty()
+
+  const manifest = await addDependenciesToPackage({
+    pnpm: {
+      packageExtensions: {
+        'is-positive': {
+          dependencies: {
+            bar: '100.1.0',
+          },
+        },
+      },
+    },
+  }, ['is-positive@1.0.0'], await testDefaults())
+
+  {
+    const lockfile = await project.readLockfile()
+    expect(lockfile.packages['/is-positive/1.0.0'].dependencies?.['bar']).toBe('100.1.0')
+    expect(lockfile.packageExtensionsChecksum).toStrictEqual(createObjectChecksum({
+      'is-positive': {
+        dependencies: {
+          bar: '100.1.0',
+        },
+      },
+    }))
+    const currentLockfile = await project.readCurrentLockfile()
+    expect(lockfile.packageExtensionsChecksum).toStrictEqual(currentLockfile.packageExtensionsChecksum)
+  }
+
+  // The lockfile is updated if the overrides are changed
+  manifest.pnpm!.packageExtensions!['is-positive'].dependencies!['foobar'] = '100.0.0'
+  await mutateModules([
+    {
+      buildIndex: 0,
+      manifest,
+      mutation: 'install',
+      rootDir: process.cwd(),
+    },
+  ], await testDefaults())
+
+  {
+    const lockfile = await project.readLockfile()
+    expect(lockfile.packages['/is-positive/1.0.0'].dependencies?.['foobar']).toBe('100.0.0')
+    expect(lockfile.packageExtensionsChecksum).toStrictEqual(createObjectChecksum({
+      'is-positive': {
+        dependencies: {
+          bar: '100.1.0',
+          foobar: '100.0.0',
+        },
+      },
+    }))
+    const currentLockfile = await project.readCurrentLockfile()
+    expect(lockfile.packageExtensionsChecksum).toStrictEqual(currentLockfile.packageExtensionsChecksum)
+  }
+
+  await mutateModules([
+    {
+      buildIndex: 0,
+      manifest,
+      mutation: 'install',
+      rootDir: process.cwd(),
+    },
+  ], await testDefaults({ frozenLockfile: true }))
+
+  {
+    const lockfile = await project.readLockfile()
+    expect(lockfile.packageExtensionsChecksum).toStrictEqual(createObjectChecksum({
+      'is-positive': {
+        dependencies: {
+          bar: '100.1.0',
+          foobar: '100.0.0',
+        },
+      },
+    }))
+    const currentLockfile = await project.readCurrentLockfile()
+    expect(lockfile.packageExtensionsChecksum).toStrictEqual(currentLockfile.packageExtensionsChecksum)
+  }
+
+  manifest.pnpm!.packageExtensions!['is-positive'].dependencies!['bar'] = '100.0.1'
+  await expect(
+    mutateModules([
+      {
+        buildIndex: 0,
+        manifest,
+        mutation: 'install',
+        rootDir: process.cwd(),
+      },
+    ], await testDefaults({ frozenLockfile: true }))
+  ).rejects.toThrow(
+    new PnpmError('FROZEN_LOCKFILE_WITH_OUTDATED_LOCKFILE',
+      'Cannot perform a frozen installation because the lockfile needs updates'
+    )
+  )
+})

packages/types/src/package.ts

@@ -85,10 +85,13 @@ interface BaseManifest {
 
 export type DependencyManifest = BaseManifest & Required<Pick<BaseManifest, 'name' | 'version'>>
 
+export type PackageExtension = Pick<BaseManifest, 'dependencies' | 'optionalDependencies' | 'peerDependencies' | 'peerDependenciesMeta'>
+
 export type ProjectManifest = BaseManifest & {
   pnpm?: {
     neverBuiltDependencies?: string[]
     overrides?: Record<string, string>
+    packageExtensions?: Record<string, PackageExtension>
   }
   private?: boolean
   resolutions?: Record<string, string>