feat: add new catalogMode setting (#9552)

* feat: add new `catalogMode` setting

Add new `catalogMode` setting for automatically adding new dependencies to
the default catalog.

Closes pnpm#8876, Closes pnpm#8308

* fix: catalogs don't only store semver ranges and versions

---------

Co-authored-by: Zoltan Kochan <z@kochan.io>

.changeset/wide-ghosts-own.md

@@ -0,0 +1,12 @@
+---
+"@pnpm/core": minor
+"@pnpm/config": minor
+"@pnpm/plugin-commands-installation": minor
+"@pnpm/plugin-commands-patching": minor
+---
+
+A new `catalogMode` setting is available for controlling if and how dependencies are added to the default catalog. It can be configured to several modes:
+
+- `strict`: Only allows dependency versions from the catalog. Adding a dependency outside the catalog's version range will cause an error.
+- `prefer`: Prefers catalog versions, but will fall back to direct dependencies if no compatible version is found.
+- `manual` (default): Does not automatically add dependencies to the catalog.

config/config/src/Config.ts

@@ -149,6 +149,7 @@ export interface Config extends OptionsFromRootManifest {
   workspaceDir?: string
   workspacePackagePatterns?: string[]
   catalogs?: Catalogs
+  catalogMode?: 'strict' | 'prefer' | 'manual'
   reporter?: string
   aggregateOutput: boolean
   linkWorkspacePackages: boolean | 'deep'

config/config/src/index.ts

@@ -121,6 +121,7 @@ export async function getConfig (opts: {
   const defaultOptions: Partial<KebabCaseConfig> | typeof npmTypes.types = {
     'auto-install-peers': true,
     bail: true,
+    'catalog-mode': 'manual',
     color: 'auto',
     'dangerously-allow-all-builds': false,
     'deploy-all-files': false,

config/config/src/types.ts

@@ -4,6 +4,7 @@ export const types = Object.assign({
   'auto-install-peers': Boolean,
   bail: Boolean,
   'cache-dir': String,
+  'catalog-mode': ['strict', 'prefer', 'manual'],
   'child-concurrency': Number,
   'merge-git-branch-lockfiles': Boolean,
   'merge-git-branch-lockfiles-branch-pattern': Array,

patching/plugin-commands-patching/test/patch.test.ts

@@ -1302,6 +1302,7 @@ describe('patch with custom modules-dir and virtual-store-dir', () => {
       saveLockfile: true,
       modulesDir: 'fake_modules',
       virtualStoreDir: 'fake_modules/.fake_store',
+      confirmModulesPurge: false,
     })
     const output = await patch.handler(defaultPatchOption, ['is-positive@1'])
     const patchDir = getPatchDirFromPatchOutput(output)
@@ -1326,6 +1327,7 @@ describe('patch with custom modules-dir and virtual-store-dir', () => {
       virtualStoreDir: 'fake_modules/.fake_store',
       lockfileDir: customModulesDirFixture,
       workspaceDir: customModulesDirFixture,
+      confirmModulesPurge: false,
     }, [patchDir])
     expect(fs.readFileSync(path.join(customModulesDirFixture, 'packages/bar/fake_modules/is-positive/index.js'), 'utf8')).toContain('// test patching')
   })

pkg-manager/core/package.json

@@ -59,6 +59,7 @@
     "@pnpm/builder.policy": "catalog:",
     "@pnpm/calc-dep-state": "workspace:*",
     "@pnpm/catalogs.protocol-parser": "workspace:*",
+    "@pnpm/catalogs.resolver": "workspace:*",
     "@pnpm/catalogs.types": "workspace:*",
     "@pnpm/constants": "workspace:*",
     "@pnpm/core-loggers": "workspace:*",

pkg-manager/core/src/install/checkCompatibility/CatalogVersionMismatchError.ts

@@ -0,0 +1,16 @@
+import { PnpmError } from '@pnpm/error'
+
+export class CatalogVersionMismatchError extends PnpmError {
+  public catalogDep: string
+  public wantedDep: string
+  constructor (
+    opts: {
+      catalogDep: string
+      wantedDep: string
+    }
+  ) {
+    super('CATALOG_VERSION_MISMATCH', 'Wanted dependency outside the version range defined in catalog')
+    this.catalogDep = opts.catalogDep
+    this.wantedDep = opts.wantedDep
+  }
+}

pkg-manager/core/src/install/extendInstallOptions.ts

@@ -27,6 +27,7 @@ export interface StrictInstallOptions {
   autoInstallPeers: boolean
   autoInstallPeersFromHighestMatch: boolean
   catalogs: Catalogs
+  catalogMode: 'strict' | 'prefer' | 'manual'
   frozenLockfile: boolean
   frozenLockfileIfExists: boolean
   enablePnp: boolean
@@ -177,6 +178,7 @@ const defaults = (opts: InstallOptions): StrictInstallOptions => {
     ignorePatchFailures: undefined,
     autoInstallPeers: true,
     autoInstallPeersFromHighestMatch: false,
+    catalogs: {},
     childConcurrency: 5,
     confirmModulesPurge: !opts.force,
     depth: 0,
@@ -237,6 +239,7 @@ const defaults = (opts: InstallOptions): StrictInstallOptions => {
       process.platform === 'cygwin' ||
       !process.setgid ||
       process.getuid?.() !== 0,
+    catalogMode: 'manual',
     useLockfile: true,
     saveLockfile: true,
     useGitBranchLockfile: false,

pkg-manager/core/src/install/index.ts

@@ -2,6 +2,7 @@ import path from 'path'
 import { buildModules, type DepsStateCache, linkBinsOfDependencies } from '@pnpm/build-modules'
 import { createAllowBuildFunction } from '@pnpm/builder.policy'
 import { parseCatalogProtocol } from '@pnpm/catalogs.protocol-parser'
+import { resolveFromCatalog, matchCatalogResolveResult, type CatalogResultMatcher } from '@pnpm/catalogs.resolver'
 import { type Catalogs } from '@pnpm/catalogs.types'
 import {
   LAYOUT_VERSION,
@@ -88,6 +89,8 @@ import { linkPackages } from './link'
 import { reportPeerDependencyIssues } from './reportPeerDependencyIssues'
 import { validateModules } from './validateModules'
 import { isCI } from 'ci-info'
+import semver from 'semver'
+import { CatalogVersionMismatchError } from './checkCompatibility/CatalogVersionMismatchError'
 
 class LockfileConfigMismatchError extends PnpmError {
   constructor (outdatedLockfileSettingName: string) {
@@ -248,6 +251,13 @@ export interface MutateModulesResult {
   ignoredBuilds: string[] | undefined
 }
 
+const pickCatalogSpecifier: CatalogResultMatcher<string | undefined> = {
+  found: (found) =>
+    found.resolution.specifier,
+  misconfiguration: () => undefined,
+  unused: () => undefined,
+}
+
 export async function mutateModules (
   projects: MutatedProject[],
   maybeOpts: MutateModulesOptions
@@ -576,6 +586,37 @@ export async function mutateModules (
         overrides: opts.overrides,
         defaultCatalog: opts.catalogs?.default,
       })
+
+      if (opts.catalogMode !== 'manual') {
+        const catalogBareSpecifier = `catalog:${opts.saveCatalogName == null || opts.saveCatalogName === 'default' ? '' : opts.saveCatalogName}`
+        for (const wantedDep of wantedDeps) {
+          const catalog = resolveFromCatalog(opts.catalogs, { ...wantedDep, bareSpecifier: catalogBareSpecifier })
+          const catalogDepSpecifier = matchCatalogResolveResult(catalog, pickCatalogSpecifier)
+
+          if (
+            !catalogDepSpecifier ||
+            wantedDep.bareSpecifier === catalogBareSpecifier ||
+            semver.validRange(wantedDep.bareSpecifier) &&
+            semver.validRange(catalogDepSpecifier) &&
+            semver.eq(wantedDep.bareSpecifier, catalogDepSpecifier)
+          ) {
+            wantedDep.saveCatalogName = opts.saveCatalogName ?? 'default'
+            continue
+          }
+
+          switch (opts.catalogMode) {
+          case 'strict':
+            throw new CatalogVersionMismatchError({ catalogDep: `${wantedDep.alias}@${catalogDepSpecifier}`, wantedDep: `${wantedDep.alias}@${wantedDep.bareSpecifier}` })
+
+          case 'prefer':
+            logger.warn({
+              message: `Catalog version mismatch for "${wantedDep.alias}": using direct version "${wantedDep.bareSpecifier}" instead of catalog version "${catalogDepSpecifier}".`,
+              prefix: opts.lockfileDir,
+            })
+          }
+        }
+      }
+
       projectsToInstall.push({
         pruneDirectDependencies: false,
         ...project,

pkg-manager/core/test/catalogs.ts

@@ -4,10 +4,17 @@ import { prepareEmpty } from '@pnpm/prepare'
 import { addDistTag } from '@pnpm/registry-mock'
 import { type MutatedProject, mutateModules, type ProjectOptions, type MutateModulesOptions, addDependenciesToPackage } from '@pnpm/core'
 import { type CatalogSnapshots } from '@pnpm/lockfile.types'
+import { logger } from '@pnpm/logger'
 import { sync as loadJsonFile } from 'load-json-file'
 import path from 'path'
 import { testDefaults } from './utils'
 
+jest.mock('@pnpm/logger', () => {
+  const originalModule = jest.requireActual('@pnpm/logger')
+  originalModule.logger.warn = jest.fn()
+  return originalModule
+})
+
 function preparePackagesAndReturnObjects (manifests: Array<ProjectManifest & Required<Pick<ProjectManifest, 'name'>>>) {
   const project = prepareEmpty()
   const lockfileDir = process.cwd()
@@ -1005,6 +1012,189 @@ describe('add', () => {
       'is-positive@2.0.0': expect.any(Object),
     })
   })
+
+  test('adding with catalogMode: strict will add to or use from catalog', async () => {
+    const { options, projects, readLockfile } = preparePackagesAndReturnObjects([{
+      name: 'project1',
+      dependencies: {},
+    }])
+
+    const { updatedManifest } = await addDependenciesToPackage(
+      projects['project1' as ProjectId],
+      ['is-positive@1.0.0'],
+      {
+        ...options,
+        dir: path.join(options.lockfileDir, 'project1'),
+        lockfileOnly: true,
+        allowNew: true,
+        catalogs: {
+          default: {},
+        },
+        catalogMode: 'strict',
+      })
+
+    expect(updatedManifest).toEqual({
+      name: 'project1',
+      dependencies: {
+        'is-positive': 'catalog:',
+      },
+    })
+    expect(readLockfile()).toMatchObject({
+      catalogs: { default: { 'is-positive': { specifier: '1.0.0', version: '1.0.0' } } },
+      importers: { project1: { dependencies: { 'is-positive': { specifier: 'catalog:', version: '1.0.0' } } } },
+      packages: { 'is-positive@1.0.0': expect.any(Object) },
+    })
+  })
+
+  test('adding with catalogMode: prefer will add to or use from catalog', async () => {
+    const { options, projects, readLockfile } = preparePackagesAndReturnObjects([{
+      name: 'project1',
+      dependencies: {},
+    }])
+
+    const { updatedManifest } = await addDependenciesToPackage(
+      projects['project1' as ProjectId],
+      ['is-positive@1.0.0'],
+      {
+        ...options,
+        dir: path.join(options.lockfileDir, 'project1'),
+        lockfileOnly: true,
+        allowNew: true,
+        catalogs: {
+          default: {},
+        },
+        catalogMode: 'prefer',
+      })
+
+    expect(updatedManifest).toEqual({
+      name: 'project1',
+      dependencies: {
+        'is-positive': 'catalog:',
+      },
+    })
+    expect(readLockfile()).toMatchObject({
+      catalogs: { default: { 'is-positive': { specifier: '1.0.0', version: '1.0.0' } } },
+      importers: { project1: { dependencies: { 'is-positive': { specifier: 'catalog:', version: '1.0.0' } } } },
+      packages: { 'is-positive@1.0.0': expect.any(Object) },
+    })
+  })
+
+  test('adding mismatched version with catalogMode: strict will error', async () => {
+    const { options, projects } = preparePackagesAndReturnObjects([{
+      name: 'project1',
+      dependencies: {
+        'is-positive': 'catalog:',
+      },
+    }])
+
+    await expect(addDependenciesToPackage(
+      projects['project1' as ProjectId],
+      ['is-positive@2.0.0'],
+      {
+        ...options,
+        dir: path.join(options.lockfileDir, 'project1'),
+        lockfileOnly: true,
+        allowNew: true,
+        catalogs: {
+          default: {
+            'is-positive': '1.0.0',
+          },
+        },
+        catalogMode: 'strict',
+      })
+    ).rejects.toThrow()
+  })
+
+  test('adding mismatched version with catalogMode: prefer will warn and use direct', async () => {
+    const { options, projects, readLockfile } = preparePackagesAndReturnObjects([{
+      name: 'project1',
+      dependencies: {
+        'is-positive': 'catalog:',
+      },
+    }, {
+      name: 'project2',
+      dependencies: {
+        'is-positive': 'catalog:',
+      },
+    }])
+
+    options.catalogs = {
+      default: {
+        'is-positive': '1.0.0',
+      },
+    }
+    options.lockfileOnly = true
+
+    await mutateModules(installProjects(projects), options)
+
+    expect(options.catalogs).toStrictEqual({
+      default: {
+        'is-positive': '1.0.0',
+      },
+    })
+
+    let installResult = await addDependenciesToPackage(
+      projects['project1' as ProjectId],
+      ['is-positive@2.0.0'],
+      {
+        ...options,
+        dir: path.join(options.lockfileDir, 'project1'),
+        allowNew: true,
+        catalogMode: 'prefer',
+      })
+
+    expect(installResult.updatedManifest).toEqual({
+      name: 'project1',
+      dependencies: {
+        'is-positive': '2.0.0',
+      },
+    })
+
+    expect(logger.warn).toHaveBeenCalled()
+    expect(readLockfile().importers).toStrictEqual(
+      {
+        project1: { dependencies: { 'is-positive': { specifier: '2.0.0', version: '2.0.0' } } },
+        project2: { dependencies: { 'is-positive': { specifier: 'catalog:', version: '1.0.0' } } },
+      }
+    )
+    expect(readLockfile().packages).toMatchObject(
+      { 'is-positive@2.0.0': expect.any(Object), 'is-positive@1.0.0': expect.any(Object) }
+    )
+    expect(options.catalogs).toStrictEqual(
+      { default: { 'is-positive': '1.0.0' } }
+    )
+
+    installResult = await addDependenciesToPackage(
+      projects['project2' as ProjectId],
+      ['is-positive@2.0.0'],
+      {
+        ...options,
+        dir: path.join(options.lockfileDir, 'project2'),
+        allowNew: true,
+        catalogMode: 'prefer',
+      })
+
+    expect(installResult.updatedManifest).toEqual({
+      name: 'project2',
+      dependencies: {
+        'is-positive': '2.0.0',
+      },
+    })
+
+    expect(logger.warn).toHaveBeenCalled()
+    expect(readLockfile().importers).toStrictEqual(
+      {
+        project1: { dependencies: { 'is-positive': { specifier: '2.0.0', version: '2.0.0' } } },
+        project2: { dependencies: { 'is-positive': { specifier: '2.0.0', version: '2.0.0' } } },
+      }
+    )
+    expect(readLockfile().packages).toMatchObject(
+      { 'is-positive@2.0.0': expect.any(Object) }
+    )
+    expect(options.catalogs).toStrictEqual(
+      { default: { 'is-positive': '1.0.0' } }
+    )
+  })
 })
 
 // The 'pnpm update' command should eventually support updates of dependencies

pkg-manager/core/tsconfig.json

@@ -27,6 +27,9 @@
     {
       "path": "../../catalogs/protocol-parser"
     },
+    {
+      "path": "../../catalogs/resolver"
+    },
     {
       "path": "../../catalogs/types"
     },

pkg-manager/plugin-commands-installation/src/installDeps.ts

@@ -53,6 +53,7 @@ export type InstallDepsOptions = Pick<Config,
 | 'bail'
 | 'bin'
 | 'catalogs'
+| 'catalogMode'
 | 'cliOptions'
 | 'dedupePeerDependents'
 | 'depth'

pnpm-lock.yaml

@@ -4499,6 +4499,9 @@ importers:
       '@pnpm/catalogs.protocol-parser':
         specifier: workspace:*
         version: link:../../catalogs/protocol-parser
+      '@pnpm/catalogs.resolver':
+        specifier: workspace:*
+        version: link:../../catalogs/resolver
       '@pnpm/catalogs.types':
         specifier: workspace:*
         version: link:../../catalogs/types