fix(cmd-shim): extend NODE_PATH with path to hidden hoisted dir (#4513)

Deprecating `extend-node-path` in pnpm v7 has caused issues with "next dev".

This change is bringing back extending `NODE_PATH` in bin command shims. However, only when `node-linker` is set to `isolated` and packages are hoisted to `node_modules/.pnpm/node_modules`. Only `node_modules/.pnpm/node_modules` is added to `NODE_PATH`, so it should not cause too long input errors (as it was sometimes the case in pnpm v6)
This commit is contained in:
Zoltan Kochan
2022-04-01 21:58:46 +03:00
committed by GitHub
parent 998da3037c
commit 8fa95fd868
22 changed files with 125 additions and 33 deletions

View File

@@ -0,0 +1,8 @@
---
"@pnpm/build-modules": minor
"@pnpm/headless": minor
"@pnpm/hoist": minor
"@pnpm/link-bins": minor
---
New option added: `extraNodePaths`.

View File

@@ -0,0 +1,7 @@
---
"@pnpm/plugin-commands-env": minor
"@pnpm/plugin-commands-rebuild": minor
"@pnpm/plugin-commands-store": minor
---
Path `extraNodePaths` to the bins linker.

View File

@@ -0,0 +1,5 @@
---
"@pnpm/get-context": minor
---
`extraNodePaths` added to the context.

View File

@@ -0,0 +1,5 @@
---
"@pnpm/config": minor
---
The default value of `nodeLinker` is set to `isolated`.

View File

@@ -21,6 +21,7 @@ export default async (
depsToBuild?: Set<string>
depsStateCache: DepsStateCache
extraBinPaths?: string[]
extraNodePaths?: string[]
extraEnv?: Record<string, string>
lockfileDir: string
optional: boolean
@@ -70,6 +71,7 @@ async function buildDependency (
depGraph: DependenciesGraph,
opts: {
extraBinPaths?: string[]
extraNodePaths?: string[]
extraEnv?: Record<string, string>
depsStateCache: DepsStateCache
lockfileDir: string
@@ -190,6 +192,7 @@ export async function linkBinsOfDependencies (
depNode: DependenciesGraphNode,
depGraph: DependenciesGraph,
opts: {
extraNodePaths?: string[]
optional: boolean
warn: (message: string) => void
}
@@ -227,11 +230,11 @@ export async function linkBinsOfDependencies (
}))
)
await linkBinsOfPackages(pkgs, binPath)
await linkBinsOfPackages(pkgs, binPath, { extraNodePaths: opts.extraNodePaths })
// link also the bundled dependencies` bins
if (depNode.hasBundledDependencies) {
const bundledModules = path.join(depNode.dir, 'node_modules')
await linkBins(bundledModules, binPath, { warn: opts.warn })
await linkBins(bundledModules, binPath, { extraNodePaths: opts.extraNodePaths, warn: opts.warn })
}
}

View File

@@ -184,6 +184,7 @@ export default async (
'ignore-workspace-root-check': false,
'link-workspace-packages': true,
'modules-cache-max-age': 7 * 24 * 60, // 7 days
'node-linker': 'isolated',
'package-lock': npmDefaults['package-lock'],
pending: false,
'prefer-workspace-packages': false,

View File

@@ -10,6 +10,7 @@ export type ListMissingPeersOptions = Partial<GetContextOptions>
& Pick<InstallOptions, 'hooks'
| 'linkWorkspacePackagesDepth'
| 'nodeVersion'
| 'nodeLinker'
| 'overrides'
| 'packageExtensions'
| 'preferWorkspacePackages'
@@ -29,6 +30,7 @@ export async function getPeerDependencyIssues (
forceSharedLockfile: false,
extraBinPaths: [],
lockfileDir,
nodeLinker: opts.nodeLinker ?? 'isolated',
registries: DEFAULT_REGISTRIES,
useLockfile: true,
...opts,

View File

@@ -51,7 +51,7 @@ export interface StrictInstallOptions {
neverBuiltDependencies?: string[]
onlyBuiltDependencies?: string[]
nodeExecPath?: string
nodeLinker?: 'isolated' | 'hoisted' | 'pnp'
nodeLinker: 'isolated' | 'hoisted' | 'pnp'
nodeVersion: string
packageExtensions: Record<string, PackageExtension>
packageManager: {

View File

@@ -778,6 +778,7 @@ const _installInContext: InstallFunction = async (projects, ctx, opts) => {
currentLockfile: ctx.currentLockfile,
dependenciesByProjectId,
depsStateCache,
extraNodePaths: ctx.extraNodePaths,
force: opts.force,
hoistedDependencies: ctx.hoistedDependencies,
hoistedModulesDir: ctx.hoistedModulesDir,
@@ -838,6 +839,7 @@ const _installInContext: InstallFunction = async (projects, ctx, opts) => {
depsStateCache,
depsToBuild: new Set(result.newDepPaths),
extraBinPaths: ctx.extraBinPaths,
extraNodePaths: ctx.extraNodePaths,
extraEnv,
lockfileDir: ctx.lockfileDir,
optional: opts.include.optionalDependencies,
@@ -857,6 +859,7 @@ const _installInContext: InstallFunction = async (projects, ctx, opts) => {
if (result.newDepPaths?.length) {
const newPkgs = props<string, DependenciesGraphNode>(result.newDepPaths, dependenciesGraph)
await linkAllBins(newPkgs, dependenciesGraph, {
extraNodePaths: ctx.extraNodePaths,
optional: opts.include.optionalDependencies,
warn: binWarn.bind(null, opts.lockfileDir),
})
@@ -876,6 +879,7 @@ const _installInContext: InstallFunction = async (projects, ctx, opts) => {
allowExoticManifests: true,
projectManifest: project.manifest,
nodeExecPathByAlias,
extraNodePaths: ctx.extraNodePaths,
warn: binWarn.bind(null, project.rootDir),
})
} else {
@@ -907,7 +911,10 @@ const _installInContext: InstallFunction = async (projects, ctx, opts) => {
)
)
.filter(({ manifest }) => manifest != null) as Array<{ location: string, manifest: DependencyManifest }>,
project.binsDir
project.binsDir,
{
extraNodePaths: ctx.extraNodePaths,
}
)
}
const projectToInstall = projects[index]
@@ -1067,6 +1074,7 @@ async function linkAllBins (
depNodes: DependenciesGraphNode[],
depGraph: DependenciesGraph,
opts: {
extraNodePaths?: string[]
optional: boolean
warn: (message: string) => void
}

View File

@@ -48,6 +48,7 @@ export default async function linkPackages (
}
force: boolean
depsStateCache: DepsStateCache
extraNodePaths: string[]
hoistedDependencies: HoistedDependencies
hoistedModulesDir: string
hoistPattern?: string[]
@@ -248,6 +249,7 @@ export default async function linkPackages (
packages: omit(Array.from(opts.skipped), currentLockfile.packages),
}
newHoistedDependencies = await hoist({
extraNodePath: opts.extraNodePaths,
lockfile: hoistLockfile,
importerIds: projectIds,
privateHoistedModulesDir: opts.hoistedModulesDir,

View File

@@ -144,7 +144,9 @@ export default async function link (
}
const linkToBin = maybeOpts?.linkToBin ?? path.join(destModules, '.bin')
await linkBinsOfPackages(linkedPkgs.map((p) => ({ manifest: p.manifest, location: p.path })), linkToBin)
await linkBinsOfPackages(linkedPkgs.map((p) => ({ manifest: p.manifest, location: p.path })), linkToBin, {
extraNodePaths: ctx.extraNodePaths,
})
let newPkg!: ProjectManifest
if (opts.targetDependenciesField) {

View File

@@ -14,6 +14,7 @@ interface StrictLinkOptions {
forceSharedLockfile: boolean
useLockfile: boolean
lockfileDir: string
nodeLinker: 'isolated' | 'hoisted' | 'pnp'
pinnedVersion: 'major' | 'minor' | 'patch'
storeController: StoreController
manifest: ProjectManifest
@@ -57,6 +58,7 @@ async function defaults (opts: LinkOptions) {
forceSharedLockfile: false,
hoistPattern: undefined,
lockfileDir: opts.lockfileDir ?? dir,
nodeLinker: 'isolated',
registries: DEFAULT_REGISTRIES,
storeController: opts.storeController,
storeDir: opts.storeDir,

View File

@@ -33,6 +33,7 @@ export interface PnpmContext<T> {
existsCurrentLockfile: boolean
existsWantedLockfile: boolean
extraBinPaths: string[]
extraNodePaths: string[]
lockfileHadConflicts: boolean
hoistedDependencies: HoistedDependencies
include: IncludedDependencies
@@ -73,6 +74,7 @@ export interface GetContextOptions {
extraBinPaths: string[]
lockfileDir: string
modulesDir?: string
nodeLinker: 'isolated' | 'hoisted' | 'pnp'
hooks?: {
readPackage?: ReadPackageHook
}
@@ -145,11 +147,13 @@ export default async function getContext<T> (
if (opts.hoistPattern?.length) {
extraBinPaths.unshift(path.join(hoistedModulesDir, '.bin'))
}
const hoistPattern = importersContext.currentHoistPattern ?? opts.hoistPattern
const ctx: PnpmContext<T> = {
extraBinPaths,
extraNodePaths: getExtraNodePaths({ nodeLinker: opts.nodeLinker, hoistPattern, virtualStoreDir }),
hoistedDependencies: importersContext.hoistedDependencies,
hoistedModulesDir,
hoistPattern: importersContext.currentHoistPattern ?? opts.hoistPattern,
hoistPattern,
include: opts.include ?? importersContext.include,
lockfileDir: opts.lockfileDir,
modulesFile: importersContext.modules,
@@ -332,6 +336,7 @@ export interface PnpmSingleContext {
existsCurrentLockfile: boolean
existsWantedLockfile: boolean
extraBinPaths: string[]
extraNodePaths: string[]
lockfileHadConflicts: boolean
hoistedDependencies: HoistedDependencies
hoistedModulesDir: string
@@ -361,6 +366,7 @@ export async function getContextForSingleImporter (
forceSharedLockfile: boolean
extraBinPaths: string[]
lockfileDir: string
nodeLinker: 'isolated' | 'hoisted' | 'pnp'
modulesDir?: string
hooks?: {
readPackage?: ReadPackageHook
@@ -441,11 +447,13 @@ export async function getContextForSingleImporter (
if (opts.hoistPattern?.length) {
extraBinPaths.unshift(path.join(hoistedModulesDir, '.bin'))
}
const hoistPattern = currentHoistPattern ?? opts.hoistPattern
const ctx: PnpmSingleContext = {
extraBinPaths,
extraNodePaths: getExtraNodePaths({ nodeLinker: opts.nodeLinker, hoistPattern, virtualStoreDir }),
hoistedDependencies,
hoistedModulesDir,
hoistPattern: currentHoistPattern ?? opts.hoistPattern,
hoistPattern,
importerId,
include: opts.include ?? include,
lockfileDir: opts.lockfileDir,
@@ -486,3 +494,16 @@ export async function getContextForSingleImporter (
return ctx
}
function getExtraNodePaths (
{ hoistPattern, nodeLinker, virtualStoreDir }: {
hoistPattern?: string[]
nodeLinker: 'isolated' | 'hoisted' | 'pnp'
virtualStoreDir: string
}
) {
if (nodeLinker === 'isolated' && hoistPattern?.length) {
return [path.join(virtualStoreDir, 'node_modules')]
}
return []
}

View File

@@ -98,6 +98,7 @@ export interface HeadlessOptions {
enablePnp?: boolean
engineStrict: boolean
extraBinPaths?: string[]
extraNodePaths?: string[]
hoistingLimits?: HoistingLimits
ignoreScripts: boolean
ignorePackageManifest?: boolean
@@ -341,6 +342,7 @@ export default async (opts: HeadlessOptions) => {
packages: omit(Array.from(skipped), filteredLockfile.packages),
}
newHoistedDependencies = await hoist({
extraNodePath: opts.extraNodePaths,
lockfile: hoistLockfile,
importerIds,
privateHoistedModulesDir: hoistedModulesDir,
@@ -353,7 +355,11 @@ export default async (opts: HeadlessOptions) => {
newHoistedDependencies = {}
}
await linkAllBins(graph, { optional: opts.include.optionalDependencies, warn })
await linkAllBins(graph, {
extraNodePaths: opts.extraNodePaths,
optional: opts.include.optionalDependencies,
warn,
})
if ((currentLockfile != null) && !equals(importerIds.sort(), Object.keys(filteredLockfile.importers).sort())) {
Object.assign(filteredLockfile.packages, currentLockfile.packages)
@@ -437,7 +443,7 @@ export default async (opts: HeadlessOptions) => {
if (!opts.ignorePackageManifest) {
await Promise.all(opts.projects.map(async (project) => {
if (opts.publicHoistPattern?.length && path.relative(opts.lockfileDir, project.rootDir) === '') {
await linkBinsOfImporter(project)
await linkBinsOfImporter(project, { extraNodePaths: opts.extraNodePaths })
} else {
const directPkgDirs = Object.values(directDependenciesByImporterId[project.id])
await linkBinsOfPackages(
@@ -450,7 +456,10 @@ export default async (opts: HeadlessOptions) => {
)
)
.filter(({ manifest }) => manifest != null) as Array<{ location: string, manifest: DependencyManifest }>,
project.binsDir
project.binsDir,
{
extraNodePaths: opts.extraNodePaths,
}
)
}
}))
@@ -546,10 +555,12 @@ async function linkBinsOfImporter (
manifest: ProjectManifest
modulesDir: string
rootDir: string
}
},
{ extraNodePaths }: { extraNodePaths?: string[] } = {}
) {
const warn = (message: string) => logger.info({ message, prefix: rootDir })
return linkBins(modulesDir, binsDir, {
extraNodePaths,
allowExoticManifests: true,
projectManifest: manifest,
warn,
@@ -695,6 +706,7 @@ async function linkAllPkgs (
async function linkAllBins (
depGraph: DependenciesGraph,
opts: {
extraNodePaths?: string[]
optional: boolean
warn: (message: string) => void
}
@@ -716,7 +728,7 @@ async function linkAllBins (
const pkgSnapshots = props<string, DependenciesGraphNode>(Object.values(childrenToLink), depGraph)
if (pkgSnapshots.includes(undefined as any)) { // eslint-disable-line
await linkBins(depNode.modules, binPath, { warn: opts.warn })
await linkBins(depNode.modules, binPath, { extraNodePaths: opts.extraNodePaths, warn: opts.warn })
} else {
const pkgs = await Promise.all(
pkgSnapshots
@@ -727,13 +739,13 @@ async function linkAllBins (
}))
)
await linkBinsOfPackages(pkgs, binPath)
await linkBinsOfPackages(pkgs, binPath, { extraNodePaths: opts.extraNodePaths })
}
// link also the bundled dependencies` bins
if (depNode.hasBundledDependencies) {
const bundledModules = path.join(depNode.dir, 'node_modules')
await linkBins(bundledModules, binPath, { warn: opts.warn })
await linkBins(bundledModules, binPath, { extraNodePaths: opts.extraNodePaths, warn: opts.warn })
}
}))
)

View File

@@ -16,6 +16,7 @@ const hoistLogger = logger('hoist')
export default async function hoistByLockfile (
opts: {
extraNodePath?: string[]
lockfile: Lockfile
importerIds?: string[]
privateHoistPattern: string[]
@@ -64,7 +65,7 @@ export default async function hoistByLockfile (
// the bins of the project's direct dependencies.
// This is possible because the publicly hoisted modules
// are in the same directory as the regular dependencies.
await linkAllBins(opts.privateHoistedModulesDir)
await linkAllBins(opts.privateHoistedModulesDir, { extraNodePaths: opts.extraNodePath })
return hoistedDependencies
}
@@ -84,14 +85,18 @@ function createGetAliasHoistType (
}
}
async function linkAllBins (modulesDir: string) {
async function linkAllBins (modulesDir: string, opts: { extraNodePaths?: string[] }) {
const bin = path.join(modulesDir, '.bin')
const warn: WarnFunction = (message, code) => {
if (code === 'BINARIES_CONFLICT') return
logger.info({ message, prefix: path.join(modulesDir, '../..') })
}
try {
await linkBins(modulesDir, bin, { allowExoticManifests: true, warn })
await linkBins(modulesDir, bin, {
allowExoticManifests: true,
extraNodePaths: opts.extraNodePaths,
warn,
})
} catch (err: any) { // eslint-disable-line
// Some packages generate their commands with lifecycle hooks.
// At this stage, such commands are not generated yet.

View File

@@ -38,7 +38,7 @@
"@pnpm/read-package-json": "workspace:6.0.0",
"@pnpm/read-project-manifest": "workspace:3.0.0",
"@pnpm/types": "workspace:8.0.0",
"@zkochan/cmd-shim": "^5.2.1",
"@zkochan/cmd-shim": "^5.2.2",
"bin-links": "^2.3.0",
"is-subdir": "^1.1.1",
"is-windows": "^1.0.2",

View File

@@ -34,6 +34,7 @@ export default async (
binsDir: string,
opts: {
allowExoticManifests?: boolean
extraNodePaths?: string[]
nodeExecPathByAlias?: Record<string, string>
projectManifest?: ProjectManifest
warn: WarnFunction
@@ -68,7 +69,7 @@ export default async (
)
const cmdsToLink = directDependencies != null ? preferDirectCmds(allCmds) : allCmds
return linkBins(cmdsToLink, binsDir)
return linkBins(cmdsToLink, binsDir, opts)
}
function preferDirectCmds (allCmds: Array<CommandInfo & { isDirectDependency?: boolean }>) {
@@ -86,7 +87,8 @@ export async function linkBinsOfPackages (
nodeExecPath?: string
location: string
}>,
binsTarget: string
binsTarget: string,
opts: { extraNodePaths?: string[] }
): Promise<string[]> {
if (pkgs.length === 0) return []
@@ -98,7 +100,7 @@ export async function linkBinsOfPackages (
.filter((cmds: Command[]) => cmds.length)
)
return linkBins(allCmds, binsTarget)
return linkBins(allCmds, binsTarget, opts)
}
type CommandInfo = Command & {
@@ -110,7 +112,8 @@ type CommandInfo = Command & {
async function linkBins (
allCmds: CommandInfo[],
binsDir: string
binsDir: string,
opts: { extraNodePaths?: string[] }
): Promise<string[]> {
if (allCmds.length === 0) return [] as string[]
@@ -118,7 +121,7 @@ async function linkBins (
const [cmdsWithOwnName, cmdsWithOtherNames] = partition(({ ownName }) => ownName, allCmds)
const results1 = await pSettle(cmdsWithOwnName.map(async (cmd) => linkBin(cmd, binsDir)))
const results1 = await pSettle(cmdsWithOwnName.map(async (cmd) => linkBin(cmd, binsDir, opts)))
const usedNames = fromPairs(cmdsWithOwnName.map((cmd) => [cmd.name, cmd.name] as KeyValuePair<string, string>))
const results2 = await pSettle(cmdsWithOtherNames.map(async (cmd) => {
@@ -132,7 +135,7 @@ async function linkBins (
return Promise.resolve(undefined)
}
usedNames[cmd.name] = cmd.pkgName
return linkBin(cmd, binsDir)
return linkBin(cmd, binsDir, opts)
}))
// We want to create all commands that we can create before throwing an exception
@@ -190,12 +193,13 @@ async function getPackageBinsFromManifest (manifest: DependencyManifest, pkgDir:
}))
}
async function linkBin (cmd: CommandInfo, binsDir: string) {
async function linkBin (cmd: CommandInfo, binsDir: string, opts?: { extraNodePaths?: string[] }) {
const externalBinPath = path.join(binsDir, cmd.name)
try {
await cmdShim(cmd.path, externalBinPath, {
createPwshFile: cmd.makePowerShellShim,
nodePath: opts?.extraNodePaths,
nodeExecPath: cmd.nodeExecPath,
})
} catch (err: any) { // eslint-disable-line

View File

@@ -38,7 +38,7 @@
"@pnpm/package-store": "workspace:13.0.0",
"@pnpm/store-path": "^5.0.0",
"@pnpm/tarball-fetcher": "workspace:10.0.0",
"@zkochan/cmd-shim": "^5.2.1",
"@zkochan/cmd-shim": "^5.2.2",
"adm-zip": "^0.5.5",
"load-json-file": "^6.2.0",
"rename-overwrite": "^4.0.2",

View File

@@ -10,6 +10,7 @@ export interface StrictRebuildOptions {
childConcurrency: number
extraBinPaths: string[]
lockfileDir: string
nodeLinker: 'isolated' | 'hoisted' | 'pnp'
scriptShell?: string
sideEffectsCacheRead: boolean
scriptsPrependNodePath: boolean | 'warn-only'
@@ -53,6 +54,7 @@ const defaults = async (opts: RebuildOptions) => {
force: false,
forceSharedLockfile: false,
lockfileDir,
nodeLinker: 'isolated',
optional: true,
packageManager,
pending: false,

View File

@@ -228,6 +228,7 @@ async function _rebuild (
currentLockfile: Lockfile
projects: Array<{ id: string, rootDir: string }>
extraBinPaths: string[]
extraNodePaths: string[]
},
opts: StrictRebuildOptions
) {
@@ -273,7 +274,7 @@ async function _rebuild (
try {
const modules = path.join(ctx.virtualStoreDir, dp.depPathToFilename(depPath), 'node_modules')
const binPath = path.join(pkgRoot, 'node_modules', '.bin')
await linkBins(modules, binPath, { warn })
await linkBins(modules, binPath, { extraNodePaths: ctx.extraNodePaths, warn })
await runPostinstallHooks({
depPath,
extraBinPaths: ctx.extraBinPaths,

View File

@@ -9,6 +9,7 @@ export interface StrictStoreStatusOptions {
storeDir: string
force: boolean
forceSharedLockfile: boolean
nodeLinker: 'isolated' | 'hoisted' | 'pnp'
useLockfile: boolean
registries: Registries
shamefullyHoist: boolean
@@ -32,6 +33,7 @@ const defaults = async (opts: StoreStatusOptions) => {
force: false,
forceSharedLockfile: false,
lockfileDir,
nodeLinker: 'isolated',
registries: DEFAULT_REGISTRIES,
shamefullyHoist: false,
storeDir: opts.storeDir,

12
pnpm-lock.yaml generated
View File

@@ -1227,7 +1227,7 @@ importers:
'@types/node': ^14.17.32
'@types/normalize-path': ^3.0.0
'@types/ramda': 0.27.39
'@zkochan/cmd-shim': ^5.2.1
'@zkochan/cmd-shim': ^5.2.2
bin-links: ^2.3.0
cmd-extension: ^1.0.2
is-subdir: ^1.1.1
@@ -1245,7 +1245,7 @@ importers:
'@pnpm/read-package-json': link:../read-package-json
'@pnpm/read-project-manifest': link:../read-project-manifest
'@pnpm/types': link:../types
'@zkochan/cmd-shim': 5.2.1
'@zkochan/cmd-shim': 5.2.2
bin-links: 2.3.0
is-subdir: 1.2.0
is-windows: 1.0.2
@@ -2065,7 +2065,7 @@ importers:
'@pnpm/tarball-fetcher': workspace:10.0.0
'@types/adm-zip': ^0.4.34
'@types/semver': ^7.3.4
'@zkochan/cmd-shim': ^5.2.1
'@zkochan/cmd-shim': ^5.2.2
adm-zip: ^0.5.5
execa: npm:safe-execa@^0.1.1
load-json-file: ^6.2.0
@@ -2087,7 +2087,7 @@ importers:
'@pnpm/package-store': link:../package-store
'@pnpm/store-path': 5.0.0
'@pnpm/tarball-fetcher': link:../tarball-fetcher
'@zkochan/cmd-shim': 5.2.1
'@zkochan/cmd-shim': 5.2.2
adm-zip: 0.5.9
load-json-file: 6.2.0
rename-overwrite: 4.0.2
@@ -5966,8 +5966,8 @@ packages:
tslib: 1.14.1
dev: false
/@zkochan/cmd-shim/5.2.1:
resolution: {integrity: sha512-oBPLTj/T1t488X1hVv99HbX7AATpApeue/OWWtD37PYxcJN4YBEpFyf86KlYb/51JrqyPe9Rv7z3/3T3p9AERg==}
/@zkochan/cmd-shim/5.2.2:
resolution: {integrity: sha512-uNWpBESHNlerKPs34liL43S14y1j3G39dpSf/wzkyP+axOzqvQTr4i+Nz/4shyS5FIL4fTi/ejHCDMT0ZneNWQ==}
engines: {node: '>=10.13'}
dependencies:
cmd-extension: 1.0.2