diff --git a/.changeset/virtual-store-only.md b/.changeset/virtual-store-only.md new file mode 100644 index 0000000000..e089f8a016 --- /dev/null +++ b/.changeset/virtual-store-only.md @@ -0,0 +1,9 @@ +--- +"@pnpm/config": patch +"@pnpm/core": minor +"@pnpm/headless": minor +"@pnpm/plugin-commands-installation": minor +"pnpm": minor +--- + +Added a new setting `virtualStoreOnly` that populates the virtual store without creating importer symlinks, hoisting, bin links, or running lifecycle scripts. This is useful for pre-populating a store (e.g., in Nix builds) without creating unnecessary project-level artifacts. `pnpm fetch` now uses this mode internally [#10840](https://github.com/pnpm/pnpm/issues/10840). diff --git a/config/config/src/Config.ts b/config/config/src/Config.ts index 549e734e23..c0d9286879 100644 --- a/config/config/src/Config.ts +++ b/config/config/src/Config.ts @@ -131,6 +131,7 @@ export interface Config extends AuthInfo, OptionsFromRootManifest { stateDir: string storeDir?: string virtualStoreDir?: string + virtualStoreOnly?: boolean enableGlobalVirtualStore?: boolean verifyStoreIntegrity?: boolean maxSockets?: number diff --git a/config/config/src/configFileKey.ts b/config/config/src/configFileKey.ts index 0d4fd8551b..6707242d13 100644 --- a/config/config/src/configFileKey.ts +++ b/config/config/src/configFileKey.ts @@ -139,6 +139,7 @@ export const excludedPnpmKeys = [ 'global-virtual-store-dir', 'virtual-store-dir', 'virtual-store-dir-max-length', + 'virtual-store-only', 'peers-suffix-max-length', 'workspace-concurrency', 'workspace-packages', diff --git a/config/config/src/index.ts b/config/config/src/index.ts index a3f9463630..4fa499270b 100644 --- a/config/config/src/index.ts +++ b/config/config/src/index.ts @@ -229,6 +229,7 @@ export async function getConfig (opts: { 'embed-readme': false, 'registry-supports-time-field': false, 'virtual-store-dir-max-length': isWindows() ? 60 : 120, + 'virtual-store-only': false, 'peers-suffix-max-length': 1000, } diff --git a/config/config/src/types.ts b/config/config/src/types.ts index 2b53ed824a..695275a46b 100644 --- a/config/config/src/types.ts +++ b/config/config/src/types.ts @@ -123,6 +123,7 @@ export const pnpmTypes = { 'verify-store-integrity': Boolean, 'global-virtual-store-dir': String, 'virtual-store-dir': String, + 'virtual-store-only': Boolean, 'virtual-store-dir-max-length': Number, 'peers-suffix-max-length': Number, 'workspace-concurrency': Number, diff --git a/pkg-manager/core/src/install/extendInstallOptions.ts b/pkg-manager/core/src/install/extendInstallOptions.ts index ed491536d8..f9bde57401 100644 --- a/pkg-manager/core/src/install/extendInstallOptions.ts +++ b/pkg-manager/core/src/install/extendInstallOptions.ts @@ -125,6 +125,7 @@ export interface StrictInstallOptions { dir: string symlink: boolean enableModulesDir: boolean + virtualStoreOnly: boolean modulesCacheMaxAge: number peerDependencyRules: PeerDependencyRules allowedDeprecatedVersions: AllowedDeprecatedVersions @@ -267,6 +268,7 @@ const defaults = (opts: InstallOptions): StrictInstallOptions => { userAgent: `${packageManager.name}/${packageManager.version} npm/? node/${process.version} ${process.platform} ${process.arch}`, verifyStoreIntegrity: true, enableModulesDir: true, + virtualStoreOnly: false, modulesCacheMaxAge: 7 * 24 * 60, resolveSymlinksInInjectedDirs: false, dedupeDirectDeps: true, @@ -313,6 +315,16 @@ export function extendOptions ( packageExtensions: extendedOpts.packageExtensions, ignoredOptionalDependencies: extendedOpts.ignoredOptionalDependencies, }) + if (extendedOpts.virtualStoreOnly && !extendedOpts.enableModulesDir && !extendedOpts.enableGlobalVirtualStore) { + throw new PnpmError('CONFIG_CONFLICT_VIRTUAL_STORE_ONLY_WITH_NO_MODULES_DIR', + 'Cannot use virtualStoreOnly when enableModulesDir is false (the standard virtual store requires node_modules/.pnpm)') + } + if (extendedOpts.virtualStoreOnly) { + // Ensure .modules.yaml records empty hoist patterns so a subsequent + // normal install knows hoisting must be redone from scratch. + extendedOpts.hoistPattern = [] + extendedOpts.publicHoistPattern = [] + } if (extendedOpts.lockfileOnly) { extendedOpts.ignoreScripts = true if (!extendedOpts.useLockfile) { diff --git a/pkg-manager/core/src/install/index.ts b/pkg-manager/core/src/install/index.ts index a273ec60a3..b6f6a12e99 100644 --- a/pkg-manager/core/src/install/index.ts +++ b/pkg-manager/core/src/install/index.ts @@ -1375,6 +1375,7 @@ const _installInContext: InstallFunction = async (projects, ctx, opts) => { wantedLockfile: newLockfile, wantedToBeSkippedPackageIds, hoistWorkspacePackages: opts.hoistWorkspacePackages, + virtualStoreOnly: opts.virtualStoreOnly, } ) stats = result.stats @@ -1451,7 +1452,7 @@ const _installInContext: InstallFunction = async (projects, ctx, opts) => { const binWarn = (prefix: string, message: string) => { logger.info({ message, prefix }) } - if (result.newDepPaths?.length) { + if (result.newDepPaths?.length && !opts.virtualStoreOnly) { const newPkgs = props(result.newDepPaths, dependenciesGraph) await linkAllBins(newPkgs, dependenciesGraph, { extraNodePaths: ctx.extraNodePaths, @@ -1460,7 +1461,7 @@ const _installInContext: InstallFunction = async (projects, ctx, opts) => { }) } - await Promise.all(projects.map(async (project, index) => { + if (!opts.virtualStoreOnly) await Promise.all(projects.map(async (project, index) => { let linkedPackages!: string[] if (ctx.publicHoistPattern?.length && path.relative(project.rootDir, opts.lockfileDir) === '') { linkedPackages = await linkBins(project.modulesDir, project.binsDir, { @@ -1559,7 +1560,7 @@ const _installInContext: InstallFunction = async (projects, ctx, opts) => { }) })(), ]) - if (!opts.ignoreScripts) { + if (!opts.ignoreScripts && !opts.virtualStoreOnly) { if (opts.enablePnp) { opts.scriptsOpts.extraEnv = { ...opts.scriptsOpts.extraEnv, diff --git a/pkg-manager/core/src/install/link.ts b/pkg-manager/core/src/install/link.ts index 1c2b002a26..53917bd737 100644 --- a/pkg-manager/core/src/install/link.ts +++ b/pkg-manager/core/src/install/link.ts @@ -73,6 +73,7 @@ export interface LinkPackagesOptions { wantedLockfile: LockfileObject wantedToBeSkippedPackageIds: Set hoistWorkspacePackages?: boolean + virtualStoreOnly: boolean } export interface LinkPackagesResult { @@ -210,7 +211,7 @@ export async function linkPackages (projects: ImporterToUpdate[], depGraph: Depe } let newHoistedDependencies!: HoistedDependencies - if (opts.hoistPattern == null && opts.publicHoistPattern == null) { + if (opts.virtualStoreOnly || (opts.hoistPattern == null && opts.publicHoistPattern == null)) { newHoistedDependencies = {} } else if (newDepPaths.length > 0 || removedDepPaths.size > 0) { newHoistedDependencies = { @@ -250,7 +251,7 @@ export async function linkPackages (projects: ImporterToUpdate[], depGraph: Depe } let linkedToRoot = 0 - if (opts.symlink) { + if (opts.symlink && !opts.virtualStoreOnly) { const projectsToLink = Object.fromEntries(await Promise.all( projects.map(async ({ id, manifest, modulesDir, rootDir }) => { const deps = opts.dependenciesByProjectId[id] diff --git a/pkg-manager/core/test/install/globalVirtualStore.ts b/pkg-manager/core/test/install/globalVirtualStore.ts index 211563e29e..5cd9d3b23b 100644 --- a/pkg-manager/core/test/install/globalVirtualStore.ts +++ b/pkg-manager/core/test/install/globalVirtualStore.ts @@ -4,7 +4,7 @@ import path from 'node:path' import { assertProject } from '@pnpm/assert-project' import { install, type MutatedProject, mutateModules, type ProjectOptions } from '@pnpm/core' import { prepareEmpty, preparePackages } from '@pnpm/prepare' -import { getIntegrity } from '@pnpm/registry-mock' +import { addDistTag, getIntegrity } from '@pnpm/registry-mock' import type { PackageFilesIndex } from '@pnpm/store.cafs' import { StoreIndex, storeIndexKey } from '@pnpm/store.index' import type { ProjectRootDir } from '@pnpm/types' @@ -486,3 +486,194 @@ test('injected local packages work with global virtual store', async () => { expect(injectedDepLocation).toContain('links') expect(fs.existsSync(path.join(injectedDepLocation!, 'foo.js'))).toBeTruthy() }) + +test('virtualStoreOnly populates standard virtual store without importer symlinks', async () => { + await addDistTag({ package: '@pnpm.e2e/dep-of-pkg-with-1-dep', version: '100.1.0', distTag: 'latest' }) + prepareEmpty() + const manifest = { + dependencies: { + '@pnpm.e2e/pkg-with-1-dep': '100.0.0', + }, + } + await install(manifest, testDefaults({ + virtualStoreOnly: true, + })) + + // Standard virtual store should be populated + expect(fs.existsSync(path.resolve('node_modules/.pnpm/@pnpm.e2e+pkg-with-1-dep@100.0.0/node_modules/@pnpm.e2e/pkg-with-1-dep/package.json'))).toBeTruthy() + expect(fs.existsSync(path.resolve('node_modules/.pnpm/@pnpm.e2e+dep-of-pkg-with-1-dep@100.1.0/node_modules/@pnpm.e2e/dep-of-pkg-with-1-dep/package.json'))).toBeTruthy() + + // Importer-level symlinks should NOT exist + expect(fs.existsSync(path.resolve('node_modules/@pnpm.e2e/pkg-with-1-dep'))).toBeFalsy() +}) + +test('virtualStoreOnly with enableModulesDir=false throws config error (standard virtual store)', async () => { + prepareEmpty() + await expect( + install({}, testDefaults({ + virtualStoreOnly: true, + enableModulesDir: false, + })) + ).rejects.toMatchObject({ + code: 'ERR_PNPM_CONFIG_CONFLICT_VIRTUAL_STORE_ONLY_WITH_NO_MODULES_DIR', + }) +}) + +test('virtualStoreOnly with enableModulesDir=false works when GVS is enabled', async () => { + prepareEmpty() + const globalVirtualStoreDir = path.resolve('gvs-no-modules') + const manifest = { + dependencies: { + '@pnpm.e2e/pkg-with-1-dep': '100.0.0', + }, + } + // First install to generate lockfile (with modules dir enabled) + await install(manifest, testDefaults({ + enableGlobalVirtualStore: true, + virtualStoreDir: globalVirtualStoreDir, + })) + + rimrafSync('node_modules') + rimrafSync(globalVirtualStoreDir) + + // Now install with virtualStoreOnly + enableModulesDir=false + GVS — should NOT throw + await install(manifest, testDefaults({ + enableGlobalVirtualStore: true, + virtualStoreDir: globalVirtualStoreDir, + virtualStoreOnly: true, + enableModulesDir: false, + frozenLockfile: true, + })) + + // GVS should be populated + const pkgDir = path.join(globalVirtualStoreDir, '@pnpm.e2e/pkg-with-1-dep/100.0.0') + expect(fs.existsSync(pkgDir)).toBeTruthy() + const hashes = fs.readdirSync(pkgDir) + expect(hashes).toHaveLength(1) + expect(fs.existsSync(path.join(pkgDir, hashes[0], 'node_modules/@pnpm.e2e/pkg-with-1-dep/package.json'))).toBeTruthy() +}) + +test('virtualStoreOnly with GVS populates global virtual store without importer links', async () => { + prepareEmpty() + const globalVirtualStoreDir = path.resolve('gvs') + const manifest = { + dependencies: { + '@pnpm.e2e/pkg-with-1-dep': '100.0.0', + }, + } + await install(manifest, testDefaults({ + enableGlobalVirtualStore: true, + virtualStoreDir: globalVirtualStoreDir, + virtualStoreOnly: true, + })) + + // GVS should be populated + const pkgDir = path.join(globalVirtualStoreDir, '@pnpm.e2e/pkg-with-1-dep/100.0.0') + expect(fs.existsSync(pkgDir)).toBeTruthy() + const hashes = fs.readdirSync(pkgDir) + expect(hashes).toHaveLength(1) + expect(fs.existsSync(path.join(pkgDir, hashes[0], 'node_modules/@pnpm.e2e/pkg-with-1-dep/package.json'))).toBeTruthy() + expect(fs.existsSync(path.join(pkgDir, hashes[0], 'node_modules/@pnpm.e2e/dep-of-pkg-with-1-dep/package.json'))).toBeTruthy() + + // Importer-level links should NOT exist + expect(fs.existsSync(path.resolve('node_modules/@pnpm.e2e/pkg-with-1-dep'))).toBeFalsy() + // No hoisted deps + expect(fs.existsSync(path.resolve('node_modules/.pnpm/node_modules/@pnpm.e2e/dep-of-pkg-with-1-dep'))).toBeFalsy() + // No bin links + expect(fs.existsSync(path.resolve('node_modules/.bin'))).toBeFalsy() +}) + +test('virtualStoreOnly with frozenLockfile populates virtual store without importer symlinks', async () => { + prepareEmpty() + const globalVirtualStoreDir = path.resolve('gvs-frozen') + const manifest = { + dependencies: { + '@pnpm.e2e/pkg-with-1-dep': '100.0.0', + }, + } + // First install to generate lockfile + await install(manifest, testDefaults({ + enableGlobalVirtualStore: true, + virtualStoreDir: globalVirtualStoreDir, + })) + + // Remove node_modules and GVS, then reinstall with frozenLockfile + virtualStoreOnly + rimrafSync('node_modules') + rimrafSync(globalVirtualStoreDir) + + await install(manifest, testDefaults({ + enableGlobalVirtualStore: true, + virtualStoreDir: globalVirtualStoreDir, + virtualStoreOnly: true, + frozenLockfile: true, + })) + + // GVS should be populated + const pkgDir = path.join(globalVirtualStoreDir, '@pnpm.e2e/pkg-with-1-dep/100.0.0') + expect(fs.existsSync(pkgDir)).toBeTruthy() + const hashes = fs.readdirSync(pkgDir) + expect(hashes).toHaveLength(1) + expect(fs.existsSync(path.join(pkgDir, hashes[0], 'node_modules/@pnpm.e2e/pkg-with-1-dep/package.json'))).toBeTruthy() + // Transitive dependency should also be in GVS + expect(fs.existsSync(path.join(pkgDir, hashes[0], 'node_modules/@pnpm.e2e/dep-of-pkg-with-1-dep/package.json'))).toBeTruthy() + + // Importer-level symlinks should NOT exist + expect(fs.existsSync(path.resolve('node_modules/@pnpm.e2e/pkg-with-1-dep'))).toBeFalsy() + // No hoisted deps + expect(fs.existsSync(path.resolve('node_modules/.pnpm/node_modules/@pnpm.e2e/dep-of-pkg-with-1-dep'))).toBeFalsy() + // No bin links + expect(fs.existsSync(path.resolve('node_modules/.bin'))).toBeFalsy() +}) + +test('virtualStoreOnly with frozenLockfile populates standard virtual store without importer symlinks', async () => { + await addDistTag({ package: '@pnpm.e2e/dep-of-pkg-with-1-dep', version: '100.1.0', distTag: 'latest' }) + prepareEmpty() + const manifest = { + dependencies: { + '@pnpm.e2e/pkg-with-1-dep': '100.0.0', + }, + } + // First install to generate lockfile + await install(manifest, testDefaults()) + + // Remove node_modules, then reinstall with frozenLockfile + virtualStoreOnly + rimrafSync('node_modules') + + await install(manifest, testDefaults({ + virtualStoreOnly: true, + frozenLockfile: true, + })) + + // Standard virtual store should be populated + expect(fs.existsSync(path.resolve('node_modules/.pnpm/@pnpm.e2e+pkg-with-1-dep@100.0.0/node_modules/@pnpm.e2e/pkg-with-1-dep/package.json'))).toBeTruthy() + expect(fs.existsSync(path.resolve('node_modules/.pnpm/@pnpm.e2e+dep-of-pkg-with-1-dep@100.1.0/node_modules/@pnpm.e2e/dep-of-pkg-with-1-dep/package.json'))).toBeTruthy() + + // Importer-level symlinks should NOT exist + expect(fs.existsSync(path.resolve('node_modules/@pnpm.e2e/pkg-with-1-dep'))).toBeFalsy() + // No hoisted deps + expect(fs.existsSync(path.resolve('node_modules/.pnpm/node_modules/@pnpm.e2e/dep-of-pkg-with-1-dep'))).toBeFalsy() + // No bin links + expect(fs.existsSync(path.resolve('node_modules/.bin'))).toBeFalsy() +}) + +test('virtualStoreOnly suppresses hoisting even with explicit hoistPattern', async () => { + prepareEmpty() + const manifest = { + dependencies: { + '@pnpm.e2e/pkg-with-1-dep': '100.0.0', + }, + } + await install(manifest, testDefaults({ + virtualStoreOnly: true, + hoistPattern: ['*'], + publicHoistPattern: ['*'], + })) + + // Virtual store should be populated + expect(fs.existsSync(path.resolve('node_modules/.pnpm/@pnpm.e2e+pkg-with-1-dep@100.0.0/node_modules/@pnpm.e2e/pkg-with-1-dep/package.json'))).toBeTruthy() + + // No hoisted packages (despite hoistPattern: ['*']) + expect(fs.existsSync(path.resolve('node_modules/.pnpm/node_modules/@pnpm.e2e/dep-of-pkg-with-1-dep'))).toBeFalsy() + // No importer-level symlinks + expect(fs.existsSync(path.resolve('node_modules/@pnpm.e2e/pkg-with-1-dep'))).toBeFalsy() +}) diff --git a/pkg-manager/headless/src/index.ts b/pkg-manager/headless/src/index.ts index ff1eb59c69..29ede9d26e 100644 --- a/pkg-manager/headless/src/index.ts +++ b/pkg-manager/headless/src/index.ts @@ -23,6 +23,7 @@ import { lockfileToDepGraph, type LockfileToDepGraphOptions, } from '@pnpm/deps.graph-builder' +import { PnpmError } from '@pnpm/error' import { hoist, type HoistedWorkspaceProject } from '@pnpm/hoist' import { makeNodeRequireOption, @@ -176,6 +177,7 @@ export interface HeadlessOptions { resolveSymlinksInInjectedDirs?: boolean skipped: Set enableModulesDir?: boolean + virtualStoreOnly?: boolean nodeLinker?: 'isolated' | 'hoisted' | 'pnp' useGitBranchLockfile?: boolean useLockfile?: boolean @@ -242,6 +244,12 @@ export async function headlessInstall (opts: HeadlessOptions): Promise() const filterOpts = { include: opts.include, @@ -391,34 +399,38 @@ export async function headlessInstall (opts: HeadlessOptions): Promise fs.mkdir(depNode.modules, { recursive: true }))) + linkedToRoot = await symlinkDirectDependencies({ + directDependenciesByImporterId: symlinkedDirectDependenciesByImporterId!, + dedupe: Boolean(opts.dedupeDirectDeps), + filteredLockfile, + lockfileDir, + projects: selectedProjects, + registries: opts.registries, + symlink: opts.symlink, + }) + } + } else if (opts.enableModulesDir !== false || opts.enableGlobalVirtualStore) { + if (opts.enableModulesDir !== false) { + await Promise.all(depNodes.map(async (depNode) => fs.mkdir(depNode.modules, { recursive: true }))) + } await Promise.all([ - opts.symlink === false + opts.symlink === false || opts.enableModulesDir === false ? Promise.resolve() : linkAllModules(depNodes, { optional: opts.include.optionalDependencies, @@ -442,7 +454,7 @@ export async function headlessInstall (opts: HeadlessOptions): Promise { - if (opts.nodeLinker === 'hoisted' || opts.publicHoistPattern?.length && path.relative(opts.lockfileDir, project.rootDir) === '') { - await linkBinsOfImporter(project, { - extraNodePaths: opts.extraNodePaths, - preferSymlinkedExecutables: opts.preferSymlinkedExecutables, - }) - } else { - let directPkgDirs: string[] - if (project.id === '.') { - directPkgDirs = Object.values(directDependenciesByImporterId[project.id]) - } else { - directPkgDirs = [] - for (const [alias, dir] of Object.entries(directDependenciesByImporterId[project.id])) { - if (rootProjectDeps[alias] !== dir) { - directPkgDirs.push(dir) - } - } - } - await linkBinsOfPackages( - ( - await Promise.all( - directPkgDirs.map(async (dir) => ({ - location: dir, - manifest: await safeReadProjectManifestOnly(dir), - })) - ) - ) - .filter(({ manifest }) => manifest != null) as Array<{ location: string, manifest: DependencyManifest }>, - project.binsDir, - { + if (!skipPostImportLinking) { + const rootProjectDeps = !opts.dedupeDirectDeps ? {} : (directDependenciesByImporterId['.'] ?? {}) + /** Skip linking and due to no project manifest */ + if (!opts.ignorePackageManifest) { + await Promise.all(selectedProjects.map(async (project) => { + if (opts.nodeLinker === 'hoisted' || opts.publicHoistPattern?.length && path.relative(opts.lockfileDir, project.rootDir) === '') { + await linkBinsOfImporter(project, { extraNodePaths: opts.extraNodePaths, preferSymlinkedExecutables: opts.preferSymlinkedExecutables, + }) + } else { + let directPkgDirs: string[] + if (project.id === '.') { + directPkgDirs = Object.values(directDependenciesByImporterId[project.id]) + } else { + directPkgDirs = [] + for (const [alias, dir] of Object.entries(directDependenciesByImporterId[project.id])) { + if (rootProjectDeps[alias] !== dir) { + directPkgDirs.push(dir) + } + } } - ) - } - })) + await linkBinsOfPackages( + ( + await Promise.all( + directPkgDirs.map(async (dir) => ({ + location: dir, + manifest: await safeReadProjectManifestOnly(dir), + })) + ) + ) + .filter(({ manifest }) => manifest != null) as Array<{ location: string, manifest: DependencyManifest }>, + project.binsDir, + { + extraNodePaths: opts.extraNodePaths, + preferSymlinkedExecutables: opts.preferSymlinkedExecutables, + } + ) + } + })) + } } const injectedDeps: Record = {} for (const project of projectsToBeBuilt) { @@ -671,7 +687,7 @@ export async function headlessInstall (opts: HeadlessOptions): Promise { // to let the subsequent install know that hoisting should be performed. hoistPattern: [], publicHoistPattern: [], + // virtualStoreOnly skips post-import linking (symlinks, bins, hoisting) + // even if ignorePackageManifest handling changes in the future. + virtualStoreOnly: true, + // Ensure fetch can populate the virtual store even when the user has + // enable-modules-dir=false in their config — fetch always needs node_modules/.pnpm + // (unless GVS is active, in which case enableModulesDir doesn't matter). + enableModulesDir: true, } as InstallOptions) } diff --git a/pkg-manager/plugin-commands-installation/src/install.ts b/pkg-manager/plugin-commands-installation/src/install.ts index 86be60251c..a73ad12e1f 100644 --- a/pkg-manager/plugin-commands-installation/src/install.ts +++ b/pkg-manager/plugin-commands-installation/src/install.ts @@ -73,6 +73,7 @@ export function rcOptionsTypes (): Record { 'unsafe-perm', 'verify-store-integrity', 'virtual-store-dir', + 'virtual-store-only', ], allTypes) } diff --git a/pkg-manager/plugin-commands-installation/src/installDeps.ts b/pkg-manager/plugin-commands-installation/src/installDeps.ts index 80d0c3a95f..4930cd78ed 100644 --- a/pkg-manager/plugin-commands-installation/src/installDeps.ts +++ b/pkg-manager/plugin-commands-installation/src/installDeps.ts @@ -69,6 +69,7 @@ export type InstallDepsOptions = Pick