diff --git a/.changeset/config-deps-global-virtual-store.md b/.changeset/config-deps-global-virtual-store.md new file mode 100644 index 0000000000..8a64afd81b --- /dev/null +++ b/.changeset/config-deps-global-virtual-store.md @@ -0,0 +1,6 @@ +--- +"@pnpm/config.deps-installer": minor +"pnpm": minor +--- + +Config dependencies are now installed into the global virtual store (`{storeDir}/links/`) and symlinked into `node_modules/.pnpm-config/`. This allows config dependencies to be shared across projects that use the same store, avoiding redundant fetches and imports. diff --git a/cli/cli-utils/src/getConfig.ts b/cli/cli-utils/src/getConfig.ts index e60e10e521..bb5fda597f 100644 --- a/cli/cli-utils/src/getConfig.ts +++ b/cli/cli-utils/src/getConfig.ts @@ -36,6 +36,7 @@ export async function getConfig ( registries: config.registries, rootDir: config.lockfileDir ?? config.rootProjectManifestDir, store: store.ctrl, + storeDir: store.dir, }) } if (!config.ignorePnpmfile) { diff --git a/config/deps-installer/package.json b/config/deps-installer/package.json index 10819735a8..5a0904be9e 100644 --- a/config/deps-installer/package.json +++ b/config/deps-installer/package.json @@ -33,6 +33,7 @@ "compile": "tsgo --build && pnpm run lint --fix" }, "dependencies": { + "@pnpm/calc-dep-state": "workspace:*", "@pnpm/config.config-writer": "workspace:*", "@pnpm/core-loggers": "workspace:*", "@pnpm/error": "workspace:*", @@ -46,7 +47,8 @@ "@pnpm/read-package-json": "workspace:*", "@pnpm/types": "workspace:*", "@zkochan/rimraf": "catalog:", - "get-npm-tarball-url": "catalog:" + "get-npm-tarball-url": "catalog:", + "symlink-dir": "catalog:" }, "peerDependencies": { "@pnpm/logger": "catalog:" diff --git a/config/deps-installer/src/installConfigDeps.ts b/config/deps-installer/src/installConfigDeps.ts index 958ffc6b15..1b87c8f336 100644 --- a/config/deps-installer/src/installConfigDeps.ts +++ b/config/deps-installer/src/installConfigDeps.ts @@ -1,19 +1,24 @@ +import fs from 'fs' import path from 'path' +import { calcLeafGlobalVirtualStorePath } from '@pnpm/calc-dep-state' import { installingConfigDepsLogger } from '@pnpm/core-loggers' import { readModulesDir } from '@pnpm/read-modules-dir' import rimraf from '@zkochan/rimraf' import { safeReadPackageJsonFromDir } from '@pnpm/read-package-json' import type { StoreController } from '@pnpm/package-store' import type { ConfigDependencies, Registries } from '@pnpm/types' +import symlinkDir from 'symlink-dir' import { normalizeConfigDeps } from './normalizeConfigDeps.js' export interface InstallConfigDepsOpts { registries: Registries rootDir: string store: StoreController + storeDir: string } export async function installConfigDeps (configDeps: ConfigDependencies, opts: InstallConfigDepsOpts): Promise { + const globalVirtualStoreDir = path.join(opts.storeDir, 'links') const configModulesDir = path.join(opts.rootDir, 'node_modules/.pnpm-config') const existingConfigDeps: string[] = await readModulesDir(configModulesDir) ?? [] await Promise.all(existingConfigDeps.map(async (existingConfigDep) => { @@ -28,29 +33,37 @@ export async function installConfigDeps (configDeps: ConfigDependencies, opts: I }) await Promise.all(Object.entries(normalizedConfigDeps).map(async ([pkgName, pkg]) => { const configDepPath = path.join(configModulesDir, pkgName) - if (existingConfigDeps.includes(pkgName)) { - const configDepPkgJson = await safeReadPackageJsonFromDir(configDepPath) - if (configDepPkgJson == null || configDepPkgJson.name !== pkgName || configDepPkgJson.version !== pkg.version) { - await rimraf(configDepPath) - } else { - return - } + const existingPkgJson = existingConfigDeps.includes(pkgName) + ? await safeReadPackageJsonFromDir(configDepPath) + : null + if (existingPkgJson != null && existingPkgJson.name === pkgName && existingPkgJson.version === pkg.version) { + return } installingConfigDepsLogger.debug({ status: 'started' }) - const { fetching } = await opts.store.fetchPackage({ - force: true, - lockfileDir: opts.rootDir, - pkg: { - id: `${pkgName}@${pkg.version}`, - resolution: pkg.resolution, - }, - }) - const { files: filesResponse } = await fetching() - await opts.store.importPackage(configDepPath, { - force: true, - requiresBuild: false, - filesResponse, - }) + const fullPkgId = `${pkgName}@${pkg.version}:${pkg.resolution.integrity}` + const relPath = calcLeafGlobalVirtualStorePath(fullPkgId, pkgName, pkg.version) + const pkgDirInGlobalVirtualStore = path.join(globalVirtualStoreDir, relPath, 'node_modules', pkgName) + if (!fs.existsSync(path.join(pkgDirInGlobalVirtualStore, 'package.json'))) { + const { fetching } = await opts.store.fetchPackage({ + force: true, + lockfileDir: opts.rootDir, + pkg: { + id: `${pkgName}@${pkg.version}`, + resolution: pkg.resolution, + }, + }) + const { files: filesResponse } = await fetching() + await opts.store.importPackage(pkgDirInGlobalVirtualStore, { + force: true, + requiresBuild: false, + filesResponse, + }) + } + if (existingConfigDeps.includes(pkgName)) { + await rimraf(configDepPath) + } + await fs.promises.mkdir(path.dirname(configDepPath), { recursive: true }) + await symlinkDir(pkgDirInGlobalVirtualStore, configDepPath) installedConfigDeps.push({ name: pkgName, version: pkg.version, diff --git a/config/deps-installer/test/installConfigDeps.ts b/config/deps-installer/test/installConfigDeps.ts index 34b3cbe87b..c0b13c9b98 100644 --- a/config/deps-installer/test/installConfigDeps.ts +++ b/config/deps-installer/test/installConfigDeps.ts @@ -9,7 +9,7 @@ const registry = `http://localhost:${REGISTRY_MOCK_PORT}/` test('configuration dependency is installed', async () => { prepareEmpty() - const { storeController } = createTempStore() + const { storeController, storeDir } = createTempStore() let configDeps: Record = { '@pnpm.e2e/foo': `100.0.0+${getIntegrity('@pnpm.e2e/foo', '100.0.0')}`, @@ -20,12 +20,15 @@ test('configuration dependency is installed', async () => { }, rootDir: process.cwd(), store: storeController, + storeDir, }) { const configDepManifest = loadJsonFileSync<{ name: string, version: string }>('node_modules/.pnpm-config/@pnpm.e2e/foo/package.json') expect(configDepManifest.name).toBe('@pnpm.e2e/foo') expect(configDepManifest.version).toBe('100.0.0') + // The local path should be a symlink to the global virtual store + expect(fs.lstatSync('node_modules/.pnpm-config/@pnpm.e2e/foo').isSymbolicLink()).toBe(true) } // Dependency is updated @@ -37,6 +40,7 @@ test('configuration dependency is installed', async () => { }, rootDir: process.cwd(), store: storeController, + storeDir, }) { @@ -54,6 +58,7 @@ test('configuration dependency is installed', async () => { }, rootDir: process.cwd(), store: storeController, + storeDir, }) expect(fs.existsSync('node_modules/.pnpm-config/@pnpm.e2e/foo/package.json')).toBeFalsy() @@ -61,7 +66,7 @@ test('configuration dependency is installed', async () => { test('installation fails if the checksum of the config dependency is invalid', async () => { prepareEmpty() - const { storeController } = createTempStore({ + const { storeController, storeDir } = createTempStore({ clientOptions: { retry: { retries: 0, @@ -78,12 +83,13 @@ test('installation fails if the checksum of the config dependency is invalid', a }, rootDir: process.cwd(), store: storeController, + storeDir, })).rejects.toThrow('Got unexpected checksum for') }) test('installation fails if the config dependency does not have a checksum', async () => { prepareEmpty() - const { storeController } = createTempStore({ + const { storeController, storeDir } = createTempStore({ clientOptions: { retry: { retries: 0, @@ -100,5 +106,6 @@ test('installation fails if the config dependency does not have a checksum', asy }, rootDir: process.cwd(), store: storeController, + storeDir, })).rejects.toThrow("doesn't have an integrity checksum") }) diff --git a/config/deps-installer/test/resolveConfigDeps.test.ts b/config/deps-installer/test/resolveConfigDeps.test.ts index 1e8895e98a..928f34e33a 100644 --- a/config/deps-installer/test/resolveConfigDeps.test.ts +++ b/config/deps-installer/test/resolveConfigDeps.test.ts @@ -9,7 +9,7 @@ const registry = `http://localhost:${REGISTRY_MOCK_PORT}/` test('configuration dependency is resolved', async () => { prepareEmpty() - const { storeController } = createTempStore() + const { storeController, storeDir } = createTempStore() await resolveConfigDeps(['@pnpm.e2e/foo@100.0.0'], { registries: { @@ -19,7 +19,7 @@ test('configuration dependency is resolved', async () => { cacheDir: path.resolve('cache'), userConfig: {}, store: storeController, - storeDir: '.store', + storeDir, }) const workspaceManifest = readYamlFile<{ configDependencies: Record }>('pnpm-workspace.yaml') diff --git a/config/deps-installer/tsconfig.json b/config/deps-installer/tsconfig.json index caa113dfe4..28e39985d9 100644 --- a/config/deps-installer/tsconfig.json +++ b/config/deps-installer/tsconfig.json @@ -21,6 +21,9 @@ { "path": "../../network/fetch" }, + { + "path": "../../packages/calc-dep-state" + }, { "path": "../../packages/core-loggers" }, diff --git a/packages/calc-dep-state/src/index.ts b/packages/calc-dep-state/src/index.ts index b503856e71..b413bbf6df 100644 --- a/packages/calc-dep-state/src/index.ts +++ b/packages/calc-dep-state/src/index.ts @@ -133,14 +133,22 @@ export function calcGraphNodeHash ( // so they survive Node.js upgrades and architecture changes. const includeEngine = builtDepPaths === undefined || transitivelyRequiresBuild(graph, builtDepPaths, buildRequiredCache ??= {}, depPath, new Set()) - const state = { - engine: includeEngine ? ENGINE_NAME : null, - deps: calcDepGraphHash(graph, cache, new Set(), depPath), - } - const hexDigest = hashObjectWithoutSorting(state, { encoding: 'hex' }) - // Use @/ prefix for unscoped packages to maintain uniform 4-level directory depth - // Scoped: @scope/pkg/version/hash - // Unscoped: @/pkg/version/hash + const engine = includeEngine ? ENGINE_NAME : null + const deps = calcDepGraphHash(graph, cache, new Set(), depPath) + const hexDigest = hashObjectWithoutSorting({ engine, deps }, { encoding: 'hex' }) + return formatGlobalVirtualStorePath(name, version, hexDigest) +} + +export function calcLeafGlobalVirtualStorePath (fullPkgId: string, name: string, version: string): string { + const depsHash = hashObject({ id: fullPkgId, deps: {} }) + const hexDigest = hashObjectWithoutSorting({ engine: null, deps: depsHash }, { encoding: 'hex' }) + return formatGlobalVirtualStorePath(name, version, hexDigest) +} + +// Use @/ prefix for unscoped packages to maintain uniform 4-level directory depth +// Scoped: @scope/pkg/version/hash +// Unscoped: @/pkg/version/hash +function formatGlobalVirtualStorePath (name: string, version: string, hexDigest: string): string { const prefix = name.startsWith('@') ? '' : '@/' return `${prefix}${name}/${version}/${hexDigest}` } diff --git a/pkg-manager/plugin-commands-installation/src/add.ts b/pkg-manager/plugin-commands-installation/src/add.ts index 9cc3930a30..5beb042855 100644 --- a/pkg-manager/plugin-commands-installation/src/add.ts +++ b/pkg-manager/plugin-commands-installation/src/add.ts @@ -223,6 +223,7 @@ export async function handler ( await resolveConfigDeps(params, { ...opts, store: store.ctrl, + storeDir: store.dir, rootDir: opts.workspaceDir ?? opts.rootProjectManifestDir, }) return diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f9dc33f6d9..afd407c044 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1753,6 +1753,9 @@ importers: config/deps-installer: dependencies: + '@pnpm/calc-dep-state': + specifier: workspace:* + version: link:../../packages/calc-dep-state '@pnpm/config.config-writer': specifier: workspace:* version: link:../config-writer @@ -1798,6 +1801,9 @@ importers: get-npm-tarball-url: specifier: 'catalog:' version: 2.1.0 + symlink-dir: + specifier: 'catalog:' + version: 7.1.0 devDependencies: '@pnpm/config.deps-installer': specifier: workspace:* diff --git a/pnpm/test/configurationalDependencies.test.ts b/pnpm/test/configurationalDependencies.test.ts index 6f854630a8..45a6b4c144 100644 --- a/pnpm/test/configurationalDependencies.test.ts +++ b/pnpm/test/configurationalDependencies.test.ts @@ -35,7 +35,8 @@ test('patch from configuration dependency is applied via updateConfig hook', asy expect(fs.existsSync('node_modules/@pnpm.e2e/foo/index.js')).toBeTruthy() const lockfile = project.readLockfile() - expect(lockfile.patchedDependencies['@pnpm.e2e/foo'].path).toBe('node_modules/.pnpm-config/@pnpm.e2e/has-patch-for-foo/@pnpm.e2e__foo@100.0.0.patch') + // The patch path goes through the global virtual store since config deps are symlinked there + expect(lockfile.patchedDependencies['@pnpm.e2e/foo'].path).toContain('@pnpm.e2e/has-patch-for-foo/@pnpm.e2e__foo@100.0.0.patch') }) test('catalog applied by configurational dependency hook', async () => {