feat(config): load env pnpm_config_* (#10029)

* feat(config): load env `pnpm_config_*`

* fix: `getEnvKeySuffix`

* fix: regex in `isEnvKeySuffix`

* refactor: use regex `+`

* test: parseEnvVars

* feat: read from `opts.env`

* test: load env `pnpm_config_*`

* fix: `types['only-built-dependencies']`

* feat: json and schema (wip)

* feat: json and schema

* test: fix

* feat: override workspace but not CLI

* fix: don't override kebab-case CLI arguments

* test: correct syntax

* docs(changeset): add information

* refactor: replace `if` with `switch`

* docs: remove outdated comment

* fix: eslint

* refactor: use `JSON.stringify`

.changeset/giant-pots-think.md

@@ -0,0 +1,6 @@
+---
+"@pnpm/config": minor
+"pnpm": minor
+---
+
+Load environment variables whose names start with `pnpm_config_` into config. These environment variables override settings from `pnpm-workspace.yaml` but not the CLI arguments.

config/config/package.json

@@ -74,7 +74,8 @@
     "@types/lodash.kebabcase": "catalog:",
     "@types/ramda": "catalog:",
     "@types/which": "catalog:",
-    "symlink-dir": "catalog:"
+    "symlink-dir": "catalog:",
+    "write-yaml-file": "catalog:"
   },
   "engines": {
     "node": ">=20.19"

config/config/src/env.ts

@@ -0,0 +1,190 @@
+import path from 'path'
+import url from 'url'
+import kebabCase from 'lodash.kebabcase'
+import camelcase from 'camelcase'
+
+const PREFIX = 'pnpm_config_'
+
+export type ValueConstructor =
+  | ArrayConstructor
+  | BooleanConstructor
+  | NumberConstructor
+  | StringConstructor
+
+export type ModuleSchema =
+  | typeof path
+  | typeof url
+
+export type ValueSchema = ValueConstructor | ModuleSchema
+
+export type LiteralSchema = string | boolean | null
+
+export type UnionVariant = LiteralSchema | ValueSchema
+
+export type Schema = ValueSchema | UnionVariant[]
+
+export type GetSchema = (key: string) => Schema | undefined
+
+/**
+ * Pair of a camelCase key and a parsed value
+ */
+export interface ConfigPair<Value> {
+  key: string
+  value: Value
+}
+
+/**
+ * Parse all the environment variables whose names start with {@link PREFIX} according to the {@link types} then emit
+ * pairs of camelCase keys and parsed values.
+ */
+export function * parseEnvVars (getSchema: GetSchema, env: NodeJS.ProcessEnv): Generator<ConfigPair<unknown>, void, void> {
+  for (const envKey in env) {
+    const suffix = getEnvKeySuffix(envKey)
+    if (!suffix) continue
+    const envValue = env[envKey]
+    if (envValue == null) continue
+    const schemaKey = kebabCase(suffix)
+    const schema = getSchema(schemaKey)
+    if (schema == null) continue
+    const key = camelcase(suffix)
+    const value = parseValueBySchema(schema, envValue, env as { HOME?: string })
+    yield { key, value }
+  }
+}
+
+function parseValueBySchema (schema: Schema, envVar: string, env: { HOME?: string }): unknown {
+  if (Array.isArray(schema)) {
+    return parseValueByTypeUnion(schema, envVar, env)
+  } else if (typeof schema === 'function') {
+    return parseValueByConstructor(schema, envVar)
+  } else if (schema && typeof schema === 'object') {
+    return parseValueByModule(schema, envVar, env)
+  }
+
+  const _typeGuard: never = schema
+  throw new Error(`Invalid schema: ${JSON.stringify(_typeGuard)}`)
+}
+
+function parseValueByTypeUnion (schema: readonly UnionVariant[], envVar: string, env: { HOME?: string }): unknown {
+  for (const variant of sortUnionVariant(schema)) {
+    let value: unknown
+    switch (typeof variant) {
+    case 'string':
+      value = parseStringLiteral(variant, envVar)
+      break
+    case 'boolean':
+      value = parseBooleanLiteral(variant, envVar)
+      break
+    case 'function':
+      value = parseValueByConstructor(variant, envVar)
+      break
+    case 'object':
+      value = variant === null
+        ? parseNullLiteral(envVar)
+        : parseValueByModule(variant, envVar, env)
+      break
+    default: {
+      const _typeGuard: never = variant
+      throw new Error(`Invalid schema variant: ${JSON.stringify(_typeGuard)}`)
+    }
+    }
+    if (value !== undefined) return value
+  }
+
+  return undefined
+}
+
+function parseStringLiteral<StringLiteral extends string> (schema: StringLiteral, envVar: string): StringLiteral | undefined {
+  return envVar === schema ? schema : undefined
+}
+
+function parseBooleanLiteral<BooleanLiteral extends boolean> (schema: BooleanLiteral, envVar: string): BooleanLiteral | undefined {
+  return schema.toString() === envVar ? schema : undefined
+}
+
+function parseNullLiteral (envVar: string): null | undefined {
+  return envVar === 'null' ? null : undefined
+}
+
+function parseValueByConstructor (schema: ValueConstructor, envVar: string): unknown {
+  if (schema === Array) {
+    const value = tryParseObjectOrArray(envVar)
+    return Array.isArray(value) ? value : undefined
+  }
+
+  if (schema === Boolean) {
+    switch (envVar) {
+    case 'true': return true
+    case 'false': return false
+    default: return undefined
+    }
+  }
+
+  if (schema === Number) {
+    const value = Number(envVar)
+    return isNaN(value) ? undefined : value
+  }
+
+  if (schema === String) {
+    return envVar
+  }
+
+  return undefined
+}
+
+function parseValueByModule (schema: ModuleSchema, envVar: string, env: { HOME?: string }): unknown {
+  if (schema === path) {
+    const homePrefix = /^~[/\\]/
+    if (env.HOME && homePrefix.test(envVar)) {
+      return path.join(env.HOME, envVar.replace(homePrefix, ''))
+    }
+    return envVar
+  }
+
+  if (schema === url) {
+    return new url.URL(envVar).toString()
+  }
+
+  return undefined
+}
+
+/** De-prioritize string parsing to prevent it from shadowing other types */
+function sortUnionVariant (variants: readonly UnionVariant[]): UnionVariant[] {
+  const sorted = variants.filter(variant => variant !== String)
+  if (variants.includes(String)) {
+    sorted.push(String)
+  }
+  return sorted
+}
+
+function tryParseObjectOrArray (envVar: string): object | unknown[] | undefined {
+  let result: unknown
+  try {
+    result = JSON.parse(envVar)
+  } catch {
+    return undefined
+  }
+
+  // typeof array is also 'object'
+  return result == null || typeof result !== 'object'
+    ? undefined
+    : result
+}
+
+/**
+ * Return the suffix if {@link envKey} starts with {@link PREFIX} and is fully lower_snake_case.
+ * Otherwise, return `undefined`.
+ */
+function getEnvKeySuffix (envKey: string): string | undefined {
+  if (!envKey.startsWith(PREFIX)) return undefined
+  const suffix = envKey.slice(PREFIX.length)
+  if (!isEnvKeySuffix(suffix)) return undefined
+  return suffix
+}
+
+/**
+ * A valid env key suffix is lower_snake_case without redundant underscore characters.
+ */
+function isEnvKeySuffix (envKeySuffix: string): boolean {
+  return envKeySuffix.split('_').every(segment => /^[a-z0-9]+$/.test(segment))
+}

config/config/src/index.ts

@@ -2,6 +2,7 @@ import path from 'path'
 import fs from 'fs'
 import os from 'os'
 import { isCI } from 'ci-info'
+import { omit } from 'ramda'
 import { getCatalogsFromWorkspaceManifest } from '@pnpm/catalogs.config'
 import { LAYOUT_VERSION } from '@pnpm/constants'
 import { PnpmError } from '@pnpm/error'
@@ -32,6 +33,7 @@ import {
   type WantedPackageManager,
 } from './Config.js'
 import { getDefaultWorkspaceConcurrency, getWorkspaceConcurrency } from './concurrency.js'
+import { parseEnvVars } from './env.js'
 import { readWorkspaceManifest } from '@pnpm/workspace.read-manifest'
 
 import { types } from './types.js'
@@ -387,6 +389,25 @@ export async function getConfig (opts: {
     }
   }
 
+  // omit some schema that the custom parser can't yet handle
+  const envPnpmTypes = omit([
+    'init-version', // the type is a private function named 'semver'
+    'node-version', // the type is a private function named 'semver'
+    'umask', // the type is a private function named 'Umask'
+    'logstream', // the custom parser doesn't have logic to handle 'Stream' yet
+  ], types)
+
+  for (const { key, value } of parseEnvVars(key => envPnpmTypes[key as keyof typeof envPnpmTypes], env)) {
+    // undefined means that the env key was defined, but its value couldn't be parsed according to the schema
+    // TODO: should we throw some error or print some warning here?
+    if (value === undefined) continue
+
+    if (key in cliOptions || kebabCase(key) in cliOptions) continue
+
+    // @ts-expect-error
+    pnpmConfig[key] = value
+  }
+
   overrideSupportedArchitecturesWithCLI(pnpmConfig, cliOptions)
 
   if (opts.cliOptions['global']) {

config/config/src/types.ts

@@ -73,7 +73,7 @@ export const types = Object.assign({
   noproxy: String,
   'npm-path': String,
   offline: Boolean,
-  'only-built-dependencies': [String],
+  'only-built-dependencies': [String, Array],
   'pack-destination': String,
   'pack-gzip-level': Number,
   'package-import-method': ['auto', 'hardlink', 'clone', 'copy'],

config/config/test/env.test.ts

@@ -0,0 +1,234 @@
+import path from 'path'
+import url from 'url'
+import { type ConfigPair, type GetSchema, type Schema, parseEnvVars } from '../src/env.js'
+
+function assertSchemaKey (key: string): void {
+  const strictlyKebabCase = key
+    .split('-')
+    .every(segment => /^[a-z0-9]+$/.test(segment))
+  if (!strictlyKebabCase) {
+    throw new Error(`Key ${key} is not strictly kebab-case`)
+  }
+}
+
+const schemaGetter = (getSchema: GetSchema): GetSchema => key => {
+  assertSchemaKey(key)
+  return getSchema(key)
+}
+
+const schemaDict = (dict: Record<string, Schema | undefined>): GetSchema => schemaGetter(key => dict[key])
+
+const alwaysSchema = (schema: Schema): GetSchema => schemaGetter(() => schema)
+
+const pairsToObject = <Value> (pairs: Iterable<ConfigPair<Value>>): Record<string, Value> =>
+  Object.fromEntries(Array.from(pairs).map(({ key, value }) => [key, value]))
+
+test('parseEnvVars works with strings', () => {
+  expect(pairsToObject(parseEnvVars(alwaysSchema(String), {
+    HOME: '/home/fake-user',
+    PATH: '/bin:/usr/bin:/usr/local/bin:/home/fake-user/.bin:/home/fake-user/share/local/bin',
+    pnpm_config_abc_def_ghi: 'value of abcDefGhi',
+    pnpm_config_foo: 'value of foo',
+    pnpm_config_bar: 'value of bar',
+    pnpm_config_undefined_somehow: undefined,
+  }))).toStrictEqual({
+    abcDefGhi: 'value of abcDefGhi',
+    foo: 'value of foo',
+    bar: 'value of bar',
+  })
+})
+
+test('parseEnvVars works with numbers', () => {
+  expect(pairsToObject(parseEnvVars(alwaysSchema(Number), {
+    HOME: '/home/fake-user',
+    PATH: '/bin:/usr/bin:/usr/local/bin:/home/fake-user/.bin:/home/fake-user/share/local/bin',
+    pnpm_config_abc_def_ghi: '123',
+    pnpm_config_foo: '456',
+    pnpm_config_bar: '789',
+    pnpm_config_undefined_somehow: undefined,
+  }))).toStrictEqual({
+    abcDefGhi: 123,
+    foo: 456,
+    bar: 789,
+  })
+})
+
+test('parseEnvVars works with booleans', () => {
+  expect(pairsToObject(parseEnvVars(alwaysSchema(Boolean), {
+    HOME: '/home/fake-user',
+    PATH: '/bin:/usr/bin:/usr/local/bin:/home/fake-user/.bin:/home/fake-user/share/local/bin',
+    pnpm_config_foo: 'false',
+    pnpm_config_bar: 'true',
+    pnpm_config_baz: 'not a boolean',
+    pnpm_config_undefined_somehow: undefined,
+  }))).toStrictEqual({
+    foo: false,
+    bar: true,
+    baz: undefined,
+  })
+})
+
+test('parseEnvVars works with arrays', () => {
+  expect(pairsToObject(parseEnvVars(alwaysSchema(Array), {
+    HOME: '/home/fake-user',
+    PATH: '/bin:/usr/bin:/usr/local/bin:/home/fake-user/.bin:/home/fake-user/share/local/bin',
+    pnpm_config_foo: '[0, 1, 2]',
+    pnpm_config_bar: '["a", "b"]',
+    pnpm_config_baz: 'not an array',
+    pnpm_config_undefined_somehow: undefined,
+  }))).toStrictEqual({
+    foo: [0, 1, 2],
+    bar: ['a', 'b'],
+    baz: undefined,
+  })
+})
+
+test('parseEnvVars works with paths', () => {
+  expect(pairsToObject(parseEnvVars(alwaysSchema(path), {
+    HOME: '/home/fake-user',
+    PATH: '/bin:/usr/bin:/usr/local/bin:/home/fake-user/.bin:/home/fake-user/share/local/bin',
+    pnpm_config_foo: 'abc/def/ghi',
+    pnpm_config_bar: '~/abc/def/ghi',
+    pnpm_config_baz: '~\\abc\\def\\ghi',
+    pnpm_config_undefined_somehow: undefined,
+  }))).toStrictEqual({
+    foo: 'abc/def/ghi',
+    bar: path.join('/home/fake-user', 'abc/def/ghi'),
+    baz: path.join('/home/fake-user', 'abc\\def\\ghi'),
+  })
+
+  expect(pairsToObject(parseEnvVars(alwaysSchema(path), {
+    pnpm_config_foo: 'abc/def/ghi',
+    pnpm_config_bar: '~/abc/def/ghi',
+    pnpm_config_baz: '~\\abc\\def\\ghi',
+    pnpm_config_undefined_somehow: undefined,
+  }))).toStrictEqual({
+    foo: 'abc/def/ghi',
+    bar: '~/abc/def/ghi',
+    baz: '~\\abc\\def\\ghi',
+  })
+})
+
+test('parseEnvVars works with URLs', () => {
+  expect(pairsToObject(parseEnvVars(alwaysSchema(url), {
+    HOME: '/home/fake-user',
+    PATH: '/bin:/usr/bin:/usr/local/bin:/home/fake-user/.bin:/home/fake-user/share/local/bin',
+    pnpm_config_foo: 'https://registry.npmjs.com',
+    pnpm_config_bar: 'http://example.org',
+    pnpm_config_baz: 'file:///path/to/some/local/file',
+    pnpm_config_undefined_somehow: undefined,
+  }))).toStrictEqual({
+    foo: 'https://registry.npmjs.com/',
+    bar: 'http://example.org/',
+    baz: 'file:///path/to/some/local/file',
+  })
+})
+
+test('parseEnvVars works with literals', () => {
+  expect(pairsToObject(parseEnvVars(alwaysSchema(['foo', 'bar', true, null]), {
+    HOME: '/home/fake-user',
+    PATH: '/bin:/usr/bin:/usr/local/bin:/home/fake-user/.bin:/home/fake-user/share/local/bin',
+    pnpm_config_a: 'foo',
+    pnpm_config_b: 'bar',
+    pnpm_config_c: 'baz',
+    pnpm_config_d: 'false',
+    pnpm_config_e: 'true',
+    pnpm_config_f: 'null',
+  }))).toStrictEqual({
+    a: 'foo',
+    b: 'bar',
+    c: undefined,
+    d: undefined,
+    e: true,
+    f: null,
+  })
+})
+
+test('parseEnvVars works with union', () => {
+  expect(pairsToObject(parseEnvVars(alwaysSchema(['foo', 'bar', Number, Array]), {
+    HOME: '/home/fake-user',
+    PATH: '/bin:/usr/bin:/usr/local/bin:/home/fake-user/.bin:/home/fake-user/share/local/bin',
+    pnpm_config_a: 'foo',
+    pnpm_config_b: 'bar',
+    pnpm_config_c: 'baz',
+    pnpm_config_d: '123',
+    pnpm_config_e: '456',
+    pnpm_config_f: '[0, 1, "abc"]',
+  }))).toStrictEqual({
+    a: 'foo',
+    b: 'bar',
+    c: undefined,
+    d: 123,
+    e: 456,
+    f: [0, 1, 'abc'],
+  })
+})
+
+test('parseEnvVars prioritizes parsing non-strings', () => {
+  expect(pairsToObject(parseEnvVars(alwaysSchema([String, Number, Array, Boolean]), {
+    HOME: '/home/fake-user',
+    PATH: '/bin:/usr/bin:/usr/local/bin:/home/fake-user/.bin:/home/fake-user/share/local/bin',
+    pnpm_config_a: 'foo',
+    pnpm_config_b: 'bar',
+    pnpm_config_c: 'baz',
+    pnpm_config_d: '123',
+    pnpm_config_e: '456',
+    pnpm_config_f: '[0, 1, "abc"]',
+  }))).toStrictEqual({
+    a: 'foo',
+    b: 'bar',
+    c: 'baz',
+    d: 123,
+    e: 456,
+    f: [0, 1, 'abc'],
+  })
+})
+
+test('parseEnvVars skips undefined schema', () => {
+  expect(pairsToObject(parseEnvVars(schemaDict({
+    foo: String,
+    bar: String,
+  }), {
+    pnpm_config_foo: 'from foo',
+    pnpm_config_bar: 'from bar',
+    pnpm_config_baz: 'from baz',
+  }))).toStrictEqual({
+    foo: 'from foo',
+    bar: 'from bar',
+  })
+})
+
+test('parseEnvVars skips npm_config_*', () => {
+  expect(pairsToObject(parseEnvVars(alwaysSchema(String), {
+    npm_config_abc_def_ghi: 'value of abcDefGhi',
+    npm_config_foo: 'value of foo',
+    npm_config_bar: 'value of bar',
+  }))).toStrictEqual({})
+})
+
+test('parseEnvVars only reads lower snake case keys', () => {
+  expect(pairsToObject(parseEnvVars(alwaysSchema(String), {
+    PNPM_CONFIG_UPPER_SNAKE_CASE_KEY: 'whole key in upper snake case',
+    pnpmConfigCamelCaseKey: 'whole key in snake case',
+    'pnpm-config-kebab-case': 'whole key in kebab case',
+    pnpm_config_UPPER_SNAKE_CASE_SUFFIX: 'suffix in upper snake case',
+    pnpm_config_camelCaseSuffix: 'suffix in camel case',
+    'pnpm_config_kebab-case-suffix': 'suffix in kebab case',
+    pnpm_config_lower_snake_case_key: 'whole key in lower snake case',
+  }))).toStrictEqual({
+    lowerSnakeCaseKey: 'whole key in lower snake case',
+  })
+})
+
+test('parseEnvVars skips keys that contain multiple consecutive underscore characters', () => {
+  expect(pairsToObject(parseEnvVars(alwaysSchema(String), {
+    pnpm_config_foo__bar: 'foo bar',
+    pnpm_config_abc_def__ghi___jkl: 'abc def ghi jkl',
+  }))).toStrictEqual({})
+})
+
+test('parseEnvVars skips keys that end with underscore character', () => {
+  expect(pairsToObject(parseEnvVars(alwaysSchema(String), {
+    pnpm_config_foo_bar_: 'foo bar',
+  }))).toStrictEqual({})
+})

config/config/test/index.ts

@@ -2,6 +2,7 @@
 import fs from 'fs'
 import path from 'path'
 import PATH from 'path-name'
+import { sync as writeYamlFile } from 'write-yaml-file'
 import loadNpmConf from '@pnpm/npm-conf'
 import { prepare, prepareEmpty } from '@pnpm/prepare'
 import { fixtures } from '@pnpm/test-fixtures'
@@ -16,14 +17,19 @@ const { getCurrentBranch } = await import('@pnpm/git-utils')
 
 // To override any local settings,
 // we force the default values of config
-delete process.env.npm_config_depth
 process.env['npm_config_hoist'] = 'true'
-delete process.env.npm_config_registry
-delete process.env.npm_config_virtual_store_dir
-delete process.env.npm_config_shared_workspace_lockfile
-delete process.env.npm_config_side_effects_cache
-delete process.env.npm_config_node_version
-delete process.env.npm_config_fetch_retries
+process.env['pnpm_config_hoist'] = 'true'
+for (const suffix of [
+  'depth',
+  'registry',
+  'virtual_store_dir',
+  'shared_workspace_lockfile',
+  'node_version',
+  'fetch_retries',
+]) {
+  delete process.env[`npm_config_${suffix}`]
+  delete process.env[`pnpm_config_${suffix}`]
+}
 
 const env = {
   PNPM_HOME: import.meta.dirname,
@@ -1125,3 +1131,78 @@ test('when dangerouslyAllowAllBuilds is set to true and neverBuiltDependencies n
   expect(config.neverBuiltDependencies).toStrictEqual([])
   expect(warnings).toStrictEqual(['You have set dangerouslyAllowAllBuilds to true. The dependencies listed in neverBuiltDependencies will run their scripts.'])
 })
+
+test('loads setting from environment variable pnpm_config_*', async () => {
+  prepareEmpty()
+  const { config } = await getConfig({
+    cliOptions: {},
+    env: {
+      pnpm_config_fetch_retries: '100',
+      pnpm_config_hoist_pattern: '["react", "react-dom"]',
+      pnpm_config_use_node_version: '22.0.0',
+      pnpm_config_only_built_dependencies: '["is-number", "is-positive", "is-negative"]',
+    },
+    packageManager: {
+      name: 'pnpm',
+      version: '1.0.0',
+    },
+    workspaceDir: process.cwd(),
+  })
+  expect(config.fetchRetries).toBe(100)
+  expect(config.hoistPattern).toStrictEqual(['react', 'react-dom'])
+  expect(config.useNodeVersion).toBe('22.0.0')
+  expect(config.onlyBuiltDependencies).toStrictEqual(['is-number', 'is-positive', 'is-negative'])
+})
+
+test('environment variable pnpm_config_* should override pnpm-workspace.yaml', async () => {
+  prepareEmpty()
+
+  writeYamlFile('pnpm-workspace.yaml', {
+    useNodeVersion: '20.0.0',
+  })
+
+  async function getConfigValue (env: NodeJS.ProcessEnv): Promise<string | undefined> {
+    const { config } = await getConfig({
+      cliOptions: {},
+      env,
+      packageManager: {
+        name: 'pnpm',
+        version: '1.0.0',
+      },
+      workspaceDir: process.cwd(),
+    })
+    return config.useNodeVersion
+  }
+
+  expect(await getConfigValue({})).toBe('20.0.0')
+  expect(await getConfigValue({
+    pnpm_config_use_node_version: '22.0.0',
+  })).toBe('22.0.0')
+})
+
+test('CLI should override environment variable pnpm_config_*', async () => {
+  prepareEmpty()
+
+  async function getConfigValue (cliOptions: Record<string, unknown>): Promise<string | undefined> {
+    const { config } = await getConfig({
+      cliOptions,
+      env: {
+        pnpm_config_use_node_version: '18.0.0',
+      },
+      packageManager: {
+        name: 'pnpm',
+        version: '1.0.0',
+      },
+      workspaceDir: process.cwd(),
+    })
+    return config.useNodeVersion
+  }
+
+  expect(await getConfigValue({})).toBe('18.0.0')
+  expect(await getConfigValue({
+    useNodeVersion: '22.0.0',
+  })).toBe('22.0.0')
+  expect(await getConfigValue({
+    'use-node-version': '22.0.0',
+  })).toBe('22.0.0')
+})

pnpm-lock.yaml

@@ -1726,6 +1726,9 @@ importers:
       symlink-dir:
         specifier: 'catalog:'
         version: 7.0.0
+      write-yaml-file:
+        specifier: 'catalog:'
+        version: 5.0.0
 
   config/config-writer:
     dependencies: