diff --git a/.changeset/angry-streets-bow.md b/.changeset/angry-streets-bow.md new file mode 100644 index 0000000000..2369bf4561 --- /dev/null +++ b/.changeset/angry-streets-bow.md @@ -0,0 +1,11 @@ +--- +"@pnpm/resolve-dependencies": major +"@pnpm/dependency-path": major +"@pnpm/calc-dep-state": major +"@pnpm/headless": major +"@pnpm/deps.graph-builder": major +"@pnpm/core": major +"pnpm": major +--- + +Runtime dependencies are always linked from the global virtual store [#10233](https://github.com/pnpm/pnpm/pull/10233). diff --git a/deps/graph-builder/src/iteratePkgsForVirtualStore.ts b/deps/graph-builder/src/iteratePkgsForVirtualStore.ts index e9a40c06ca..b02f97cc17 100644 --- a/deps/graph-builder/src/iteratePkgsForVirtualStore.ts +++ b/deps/graph-builder/src/iteratePkgsForVirtualStore.ts @@ -1,10 +1,13 @@ +import path from 'path' import { iterateHashedGraphNodes, lockfileToDepGraph, + calcGraphNodeHash, type PkgMeta, type DepsGraph, type PkgMetaIterator, type HashedDepPath, + type DepsStateCache, } from '@pnpm/calc-dep-state' import { type LockfileObject, type PackageSnapshot } from '@pnpm/lockfile.fs' import { @@ -15,35 +18,51 @@ import * as dp from '@pnpm/dependency-path' interface PkgSnapshotWithLocation { pkgMeta: PkgMetaAndSnapshot - dirNameInVirtualStore: string + dirInVirtualStore: string } export function * iteratePkgsForVirtualStore (lockfile: LockfileObject, opts: { enableGlobalVirtualStore?: boolean virtualStoreDirMaxLength: number + virtualStoreDir: string + globalVirtualStoreDir: string }): IterableIterator { if (opts.enableGlobalVirtualStore) { for (const { hash, pkgMeta } of hashDependencyPaths(lockfile)) { yield { - dirNameInVirtualStore: hash, + dirInVirtualStore: path.join(opts.globalVirtualStoreDir, hash), pkgMeta, } } } else if (lockfile.packages) { + let graphNodeHashOpts: { graph: DepsGraph, cache: DepsStateCache } | undefined for (const depPath in lockfile.packages) { - if (Object.hasOwn(lockfile.packages, depPath)) { - const pkgSnapshot = lockfile.packages[depPath as DepPath] - const { name, version } = nameVerFromPkgSnapshot(depPath, pkgSnapshot) - yield { - pkgMeta: { - depPath: depPath as DepPath, - pkgIdWithPatchHash: dp.getPkgIdWithPatchHash(depPath as DepPath), - name, - version, - pkgSnapshot, - }, - dirNameInVirtualStore: dp.depPathToFilename(depPath, opts.virtualStoreDirMaxLength), + if (!Object.hasOwn(lockfile.packages, depPath)) { + continue + } + const pkgSnapshot = lockfile.packages[depPath as DepPath] + const { name, version } = nameVerFromPkgSnapshot(depPath, pkgSnapshot) + const pkgMeta = { + depPath: depPath as DepPath, + pkgIdWithPatchHash: dp.getPkgIdWithPatchHash(depPath as DepPath), + name, + version, + pkgSnapshot, + } + let dirInVirtualStore!: string + if (dp.isRuntimeDepPath(depPath as DepPath)) { + graphNodeHashOpts ??= { + cache: {}, + graph: lockfileToDepGraph(lockfile), } + const hash = calcGraphNodeHash(graphNodeHashOpts, pkgMeta) + dirInVirtualStore = path.join(opts.globalVirtualStoreDir, hash) + } else { + dirInVirtualStore = path.join(opts.virtualStoreDir, dp.depPathToFilename(depPath, opts.virtualStoreDirMaxLength)) + } + yield { + dirInVirtualStore, + pkgMeta, } } } diff --git a/deps/graph-builder/src/lockfileToDepGraph.ts b/deps/graph-builder/src/lockfileToDepGraph.ts index bca5a51ff2..0da3e14136 100644 --- a/deps/graph-builder/src/lockfileToDepGraph.ts +++ b/deps/graph-builder/src/lockfileToDepGraph.ts @@ -70,6 +70,7 @@ export interface LockfileToDepGraphOptions { skipped: Set storeController: StoreController storeDir: string + globalVirtualStoreDir: string virtualStoreDir: string supportedArchitectures?: SupportedArchitectures virtualStoreDirMaxLength: number @@ -166,7 +167,7 @@ async function buildGraphFromPackages ( const promises: Array> = [] const pkgSnapshotsWithLocations = iteratePkgsForVirtualStore(lockfile, opts) - for (const { dirNameInVirtualStore, pkgMeta } of pkgSnapshotsWithLocations) { + for (const { dirInVirtualStore, pkgMeta } of pkgSnapshotsWithLocations) { promises.push((async () => { const { pkgIdWithPatchHash, name: pkgName, version: pkgVersion, depPath, pkgSnapshot } = pkgMeta if (opts.skipped.has(depPath)) return @@ -198,7 +199,7 @@ async function buildGraphFromPackages ( const depIntegrityIsUnchanged = isIntegrityEqual(pkgSnapshot.resolution, currentPackages[depPath]?.resolution) - const modules = path.join(opts.virtualStoreDir, dirNameInVirtualStore, 'node_modules') + const modules = path.join(dirInVirtualStore, 'node_modules') const dir = path.join(modules, pkgName) locationByDepPath[depPath] = dir diff --git a/packages/calc-dep-state/src/index.ts b/packages/calc-dep-state/src/index.ts index 80b17f4ef6..0a5ba6d16f 100644 --- a/packages/calc-dep-state/src/index.ts +++ b/packages/calc-dep-state/src/index.ts @@ -92,28 +92,38 @@ export function * iterateHashedGraphNodes ( graph: DepsGraph, pkgMetaIterator: PkgMetaIterator ): IterableIterator> { - const _calcDepGraphHash = calcDepGraphHash.bind(null, graph, {}) + const _calcGraphNodeHash = calcGraphNodeHash.bind(null, { graph, cache: {} }) for (const pkgMeta of pkgMetaIterator) { - const { name, version, depPath } = pkgMeta - const state = { - // Unfortunately, we need to include the engine name in the hash, - // even though it's only required for packages that are built, - // or have dependencies that are built. - // We can't know for sure whether a package needs to be built - // before it's fetched from the registry. - // However, we fetch and write packages to node_modules in random order for performance, - // so we can't determine at this stage which dependencies will be built. - engine: ENGINE_NAME, - deps: _calcDepGraphHash(new Set(), depPath), - } - const hexDigest = hashObjectWithoutSorting(state, { encoding: 'hex' }) yield { - hash: `${name}/${version}/${hexDigest}`, + hash: _calcGraphNodeHash(pkgMeta), pkgMeta, } } } +export function calcGraphNodeHash ( + { graph, cache }: { + graph: DepsGraph + cache: DepsStateCache + }, + pkgMeta: T +): string { + const { name, version, depPath } = pkgMeta + const state = { + // Unfortunately, we need to include the engine name in the hash, + // even though it's only required for packages that are built, + // or have dependencies that are built. + // We can't know for sure whether a package needs to be built + // before it's fetched from the registry. + // However, we fetch and write packages to node_modules in random order for performance, + // so we can't determine at this stage which dependencies will be built. + engine: ENGINE_NAME, + deps: calcDepGraphHash(graph, cache, new Set(), depPath), + } + const hexDigest = hashObjectWithoutSorting(state, { encoding: 'hex' }) + return `${name}/${version}/${hexDigest}` +} + export function lockfileToDepGraph (lockfile: LockfileObject): DepsGraph { const graph: DepsGraph = {} if (lockfile.packages != null) { diff --git a/packages/dependency-path/src/index.ts b/packages/dependency-path/src/index.ts index d6e7c23d07..2d307f96cf 100644 --- a/packages/dependency-path/src/index.ts +++ b/packages/dependency-path/src/index.ts @@ -211,3 +211,9 @@ export function createPeerDepGraphHash (peerIds: PeerId[], maxLength: number = 1 } return `(${dirName})` } + +const RUNTIME_DEP_PATH_RE = /^(?:node|bun|deno)@runtime:/ + +export function isRuntimeDepPath (depPath: DepPath): boolean { + return RUNTIME_DEP_PATH_RE.test(depPath) +} diff --git a/packages/dependency-path/test/index.ts b/packages/dependency-path/test/index.ts index babfddea89..e05259ab0b 100644 --- a/packages/dependency-path/test/index.ts +++ b/packages/dependency-path/test/index.ts @@ -6,6 +6,7 @@ import { parse, refToRelative, tryGetPackageId, + isRuntimeDepPath, } from '@pnpm/dependency-path' import { type DepPath } from '@pnpm/types' @@ -139,3 +140,8 @@ test('getPkgIdWithPatchHash', () => { // Scoped packages with both patch hash and peer dependencies expect(getPkgIdWithPatchHash('@foo/bar@1.0.0(patch_hash=zzzz)(@types/node@18.0.0)' as DepPath)).toBe('@foo/bar@1.0.0(patch_hash=zzzz)') }) + +test('isRuntimeDepPath', () => { + expect(isRuntimeDepPath('node@runtime:20.1.0' as DepPath)).toBeTruthy() + expect(isRuntimeDepPath('node@20.1.0' as DepPath)).toBeFalsy() +}) diff --git a/pkg-manager/core/src/getPeerDependencyIssues.ts b/pkg-manager/core/src/getPeerDependencyIssues.ts index 14b2e05ca4..4719039188 100644 --- a/pkg-manager/core/src/getPeerDependencyIssues.ts +++ b/pkg-manager/core/src/getPeerDependencyIssues.ts @@ -26,7 +26,7 @@ export type ListMissingPeersOptions = Partial > & Partial> & Pick -& Required> +& Required> export async function getPeerDependencyIssues ( projects: ProjectOptions[], @@ -90,6 +90,7 @@ export async function getPeerDependencyIssues ( saveWorkspaceProtocol: false, // this doesn't matter in our case. We won't write changes to package.json files storeController: opts.storeController, tag: 'latest', + globalVirtualStoreDir: opts.globalVirtualStoreDir, virtualStoreDir: ctx.virtualStoreDir, virtualStoreDirMaxLength: ctx.virtualStoreDirMaxLength, wantedLockfile: ctx.wantedLockfile, diff --git a/pkg-manager/core/src/install/extendInstallOptions.ts b/pkg-manager/core/src/install/extendInstallOptions.ts index 6772c82431..db179bf1ea 100644 --- a/pkg-manager/core/src/install/extendInstallOptions.ts +++ b/pkg-manager/core/src/install/extendInstallOptions.ts @@ -116,6 +116,7 @@ export interface StrictInstallOptions { workspacePackages?: WorkspacePackages pruneStore: boolean virtualStoreDir?: string + globalVirtualStoreDir: string dir: string symlink: boolean enableModulesDir: boolean @@ -325,5 +326,8 @@ export function extendOptions ( if (extendedOpts.enableGlobalVirtualStore && extendedOpts.virtualStoreDir == null) { extendedOpts.virtualStoreDir = path.join(extendedOpts.storeDir, 'links') } + extendedOpts.globalVirtualStoreDir = extendedOpts.enableGlobalVirtualStore + ? extendedOpts.virtualStoreDir! + : path.join(extendedOpts.storeDir, 'links') return extendedOpts } diff --git a/pkg-manager/core/src/install/index.ts b/pkg-manager/core/src/install/index.ts index e3bc66b54c..f88781cf3d 100644 --- a/pkg-manager/core/src/install/index.ts +++ b/pkg-manager/core/src/install/index.ts @@ -1186,6 +1186,7 @@ const _installInContext: InstallFunction = async (projects, ctx, opts) => { saveWorkspaceProtocol: opts.saveWorkspaceProtocol, storeController: opts.storeController, tag: opts.tag, + globalVirtualStoreDir: opts.globalVirtualStoreDir, virtualStoreDir: ctx.virtualStoreDir, virtualStoreDirMaxLength: ctx.virtualStoreDirMaxLength, wantedLockfile: ctx.wantedLockfile, diff --git a/pkg-manager/core/test/install/nodeRuntime.ts b/pkg-manager/core/test/install/nodeRuntime.ts index 4ee527558f..c05b80742e 100644 --- a/pkg-manager/core/test/install/nodeRuntime.ts +++ b/pkg-manager/core/test/install/nodeRuntime.ts @@ -1,4 +1,5 @@ import fs from 'fs' +import path from 'path' import { LOCKFILE_VERSION, WANTED_LOCKFILE } from '@pnpm/constants' import { prepareEmpty } from '@pnpm/prepare' import { addDependenciesToPackage, install } from '@pnpm/core' @@ -183,6 +184,7 @@ test('installing Node.js runtime', async () => { const { updatedManifest: manifest } = await addDependenciesToPackage({}, ['node@runtime:22.0.0'], testDefaults({ fastUnpack: false })) project.isExecutable('.bin/node') + expect(fs.readlinkSync('node_modules/node')).toContain(path.join('links', 'node', '22.0.0')) expect(project.readLockfile()).toStrictEqual({ settings: { autoInstallPeers: true, @@ -219,6 +221,7 @@ test('installing Node.js runtime', async () => { offline: true, // We want to verify that Node.js is resolved from cache. })) project.isExecutable('.bin/node') + expect(fs.readlinkSync('node_modules/node')).toContain(path.join('links', 'node', '22.0.0')) await addDependenciesToPackage(manifest, ['@pnpm.e2e/dep-of-pkg-with-1-dep@100.1.0'], testDefaults({ fastUnpack: false })) project.has('@pnpm.e2e/dep-of-pkg-with-1-dep') diff --git a/pkg-manager/headless/src/index.ts b/pkg-manager/headless/src/index.ts index dc2302ac41..6ea529e891 100644 --- a/pkg-manager/headless/src/index.ts +++ b/pkg-manager/headless/src/index.ts @@ -140,6 +140,7 @@ export interface HeadlessOptions { lockfileDir: string modulesDir?: string enableGlobalVirtualStore?: boolean + globalVirtualStoreDir: string virtualStoreDir?: string virtualStoreDirMaxLength: number patchedDependencies?: PatchGroupRecord diff --git a/pkg-manager/resolve-dependencies/src/index.ts b/pkg-manager/resolve-dependencies/src/index.ts index 18a0a02670..ae0e4ea775 100644 --- a/pkg-manager/resolve-dependencies/src/index.ts +++ b/pkg-manager/resolve-dependencies/src/index.ts @@ -4,6 +4,7 @@ import { packageManifestLogger, } from '@pnpm/core-loggers' import { iterateHashedGraphNodes } from '@pnpm/calc-dep-state' +import { isRuntimeDepPath } from '@pnpm/dependency-path' import { type LockfileObject, type ProjectSnapshot, @@ -126,6 +127,7 @@ export async function resolveDependencies ( lockfileOnly: opts.dryRun, preferredVersions: opts.preferredVersions, virtualStoreDir: opts.virtualStoreDir, + globalVirtualStoreDir: opts.globalVirtualStoreDir, workspacePackages: opts.workspacePackages, noDependencySelectors: importers.every(({ wantedDependencies }) => wantedDependencies.length === 0), }) @@ -335,7 +337,7 @@ export async function resolveDependencies ( return { dependenciesByProjectId, - dependenciesGraph: opts.enableGlobalVirtualStore ? extendGraph(dependenciesGraph, opts.virtualStoreDir) : dependenciesGraph, + dependenciesGraph: extendGraph(dependenciesGraph, opts), outdatedDependencies, linkedDependenciesByProjectId, updatedCatalogs, @@ -457,10 +459,16 @@ async function getTopParents (pkgAliases: string[], modulesDir: string): Promise .filter(Boolean) as DependencyManifest[] } -function extendGraph (graph: DependenciesGraph, virtualStoreDir: string): DependenciesGraph { +function extendGraph ( + graph: DependenciesGraph, + opts: { + globalVirtualStoreDir: string + enableGlobalVirtualStore?: boolean + } +): DependenciesGraph { const pkgMetaIter = (function * () { for (const depPath in graph) { - if (Object.hasOwn(graph, depPath)) { + if ((opts.enableGlobalVirtualStore === true || isRuntimeDepPath(depPath as DepPath)) && Object.hasOwn(graph, depPath)) { const { name, version, pkgIdWithPatchHash } = graph[depPath as DepPath] yield { name, @@ -472,7 +480,7 @@ function extendGraph (graph: DependenciesGraph, virtualStoreDir: string): Depend } })() for (const { pkgMeta: { depPath }, hash } of iterateHashedGraphNodes(graph, pkgMetaIter)) { - const modules = path.join(virtualStoreDir, hash, 'node_modules') + const modules = path.join(opts.globalVirtualStoreDir, hash, 'node_modules') const node = graph[depPath] Object.assign(node, { modules, diff --git a/pkg-manager/resolve-dependencies/src/resolveDependencyTree.ts b/pkg-manager/resolve-dependencies/src/resolveDependencyTree.ts index 93b2459298..490c4559b0 100644 --- a/pkg-manager/resolve-dependencies/src/resolveDependencyTree.ts +++ b/pkg-manager/resolve-dependencies/src/resolveDependencyTree.ts @@ -131,6 +131,7 @@ export interface ResolveDependenciesOptions { storeController: StoreController tag: string virtualStoreDir: string + globalVirtualStoreDir: string virtualStoreDirMaxLength: number wantedLockfile: LockfileObject workspacePackages: WorkspacePackages diff --git a/pkg-manager/resolve-dependencies/src/safeIsInnerLink.ts b/pkg-manager/resolve-dependencies/src/safeIsInnerLink.ts index f8c18d9aeb..b07c9fd729 100644 --- a/pkg-manager/resolve-dependencies/src/safeIsInnerLink.ts +++ b/pkg-manager/resolve-dependencies/src/safeIsInnerLink.ts @@ -11,6 +11,7 @@ export async function safeIsInnerLink ( hideAlienModules: boolean projectDir: string virtualStoreDir: string + globalVirtualStoreDir: string } ): Promise { try { @@ -18,7 +19,12 @@ export async function safeIsInnerLink ( if (link.isInner) return true - if (isSubdir(opts.virtualStoreDir, link.target)) return true + if ( + isSubdir(opts.virtualStoreDir, link.target) || + opts.globalVirtualStoreDir !== opts.virtualStoreDir && isSubdir(opts.globalVirtualStoreDir, link.target) + ) { + return true + } return link.target as string } catch (err: any) { // eslint-disable-line diff --git a/pkg-manager/resolve-dependencies/src/toResolveImporter.ts b/pkg-manager/resolve-dependencies/src/toResolveImporter.ts index e59e24b0f0..d1c7c2e0c0 100644 --- a/pkg-manager/resolve-dependencies/src/toResolveImporter.ts +++ b/pkg-manager/resolve-dependencies/src/toResolveImporter.ts @@ -25,6 +25,7 @@ export async function toResolveImporter ( lockfileOnly: boolean preferredVersions?: PreferredVersions virtualStoreDir: string + globalVirtualStoreDir: string workspacePackages: WorkspacePackages updateToLatest?: boolean noDependencySelectors: boolean @@ -38,6 +39,7 @@ export async function toResolveImporter ( modulesDir: project.modulesDir, projectDir: project.rootDir, virtualStoreDir: opts.virtualStoreDir, + globalVirtualStoreDir: opts.globalVirtualStoreDir, workspacePackages: opts.workspacePackages, }) const defaultUpdateDepth = (project.update === true || (project.updateMatching != null)) ? opts.defaultUpdateDepth : -1 @@ -98,6 +100,7 @@ async function partitionLinkedPackages ( lockfileOnly: boolean modulesDir: string virtualStoreDir: string + globalVirtualStoreDir: string workspacePackages?: WorkspacePackages } ): Promise { @@ -116,6 +119,7 @@ async function partitionLinkedPackages ( hideAlienModules: !opts.lockfileOnly, projectDir: opts.projectDir, virtualStoreDir: opts.virtualStoreDir, + globalVirtualStoreDir: opts.globalVirtualStoreDir, }) if (isInnerLink === true) { nonLinkedDependencies.push(dependency)