From 9eddabb32b1d484cc0e0d9ae4c0b1ea9a2eb247a Mon Sep 17 00:00:00 2001 From: Zoltan Kochan Date: Fri, 26 Dec 2025 01:35:35 +0100 Subject: [PATCH] feat: enhance `store prune` to clean global virtual store (#10360) * feat: enhance `store prune` to clean global virtual store `pnpm store prune` will now clean the global virtual store via a new project registry and mark-and-sweep garbage collection. * test: add store prune test for transitive dependency preservation * refactor: extract global virtual store pruning logic to a new file * fix: improve symlink handling in global virtual store pruning * fix: optimize removal of unreachable packages in global virtual store * fix: refine project registry error handling Throw `PnpmError` for inaccessible projects and specifically clean up stale symlinks for `ENOENT` errors. * test: create virtual store with install command * refactor: standardize global virtual store directory structure by placing unscoped packages under an `@` scope. * test: update store prune tests to use `toContain` and `not.toContain` assertions` * fix: linting issues * Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * fix: implemented CR suggestions * fix: revert not needed change * fix: use `is-subdir` to accurately determine if symlink targets are within the global virtual store. * revert: changes in package.json files * test: add `--config.ci=false` to store prune tests --- .changeset/global-virtual-store-prune.md | 9 + .changeset/prune-global-virtual-store.md | 14 + pkg-manager/get-context/package.json | 1 + pkg-manager/get-context/src/index.ts | 7 + pkg-manager/get-context/tsconfig.json | 3 + pnpm-lock.yaml | 18 ++ store/package-store/package.json | 6 +- store/package-store/src/index.ts | 1 + .../src/storeController/projectRegistry.ts | 94 ++++++ .../src/storeController/prune.ts | 12 +- .../pruneGlobalVirtualStore.ts | 297 ++++++++++++++++++ store/package-store/test/projectRegistry.ts | 110 +++++++ store/package-store/tsconfig.json | 6 + store/plugin-commands-store/package.json | 3 +- .../plugin-commands-store/test/storePrune.ts | 295 +++++++++++++++++ store/plugin-commands-store/tsconfig.json | 3 + 16 files changed, 876 insertions(+), 3 deletions(-) create mode 100644 .changeset/global-virtual-store-prune.md create mode 100644 .changeset/prune-global-virtual-store.md create mode 100644 store/package-store/src/storeController/projectRegistry.ts create mode 100644 store/package-store/src/storeController/pruneGlobalVirtualStore.ts create mode 100644 store/package-store/test/projectRegistry.ts diff --git a/.changeset/global-virtual-store-prune.md b/.changeset/global-virtual-store-prune.md new file mode 100644 index 0000000000..6c7f467ddb --- /dev/null +++ b/.changeset/global-virtual-store-prune.md @@ -0,0 +1,9 @@ +--- +"@pnpm/package-store": minor +"@pnpm/get-context": minor +"pnpm": minor +--- + +Added project registry for global virtual store prune support. + +Projects using the store are now registered via symlinks in `{storeDir}/v10/projects/`. This enables `pnpm store prune` to track which packages are still in use by active projects and safely remove unused packages from the global virtual store. diff --git a/.changeset/prune-global-virtual-store.md b/.changeset/prune-global-virtual-store.md new file mode 100644 index 0000000000..7ea14e24cd --- /dev/null +++ b/.changeset/prune-global-virtual-store.md @@ -0,0 +1,14 @@ +--- +"@pnpm/package-store": minor +"pnpm": minor +--- + +Added mark-and-sweep garbage collection for global virtual store. + +`pnpm store prune` now removes unused packages from the global virtual store's `links/` directory. The algorithm: + +1. Scans all registered projects for symlinks pointing to the store +2. Walks transitive dependencies to mark reachable packages +3. Removes any package directories not marked as reachable + +This includes support for workspace monorepos - all `node_modules` directories within a project (including those in workspace packages) are scanned. diff --git a/pkg-manager/get-context/package.json b/pkg-manager/get-context/package.json index 88367ecd0e..560c197246 100644 --- a/pkg-manager/get-context/package.json +++ b/pkg-manager/get-context/package.json @@ -36,6 +36,7 @@ "@pnpm/core-loggers": "workspace:*", "@pnpm/lockfile.fs": "workspace:*", "@pnpm/modules-yaml": "workspace:*", + "@pnpm/package-store": "workspace:*", "@pnpm/read-projects-context": "workspace:*", "@pnpm/resolver-base": "workspace:*", "@pnpm/types": "workspace:*", diff --git a/pkg-manager/get-context/src/index.ts b/pkg-manager/get-context/src/index.ts index 7adf2c19ca..08095fc56c 100644 --- a/pkg-manager/get-context/src/index.ts +++ b/pkg-manager/get-context/src/index.ts @@ -21,6 +21,7 @@ import { } from '@pnpm/types' import pathAbsolute from 'path-absolute' import { clone } from 'ramda' +import { registerProject } from '@pnpm/package-store' import { readLockfiles } from './readLockfiles.js' /** @@ -123,6 +124,9 @@ export async function getContext ( await fs.mkdir(opts.storeDir, { recursive: true }) + // Register this project for store prune tracking + await registerProject(opts.storeDir, opts.lockfileDir) + for (const project of opts.allProjects) { packageManifestLogger.debug({ initial: project.manifest, @@ -293,6 +297,9 @@ export async function getContextForSingleImporter ( const virtualStoreDir = pathAbsolute(opts.virtualStoreDir ?? 'node_modules/.pnpm', opts.lockfileDir) await fs.mkdir(storeDir, { recursive: true }) + + // Register this project for store prune tracking + await registerProject(storeDir, opts.lockfileDir) const extraBinPaths = [ ...opts.extraBinPaths || [], ] diff --git a/pkg-manager/get-context/tsconfig.json b/pkg-manager/get-context/tsconfig.json index 7566b8f34e..c4a345013c 100644 --- a/pkg-manager/get-context/tsconfig.json +++ b/pkg-manager/get-context/tsconfig.json @@ -27,6 +27,9 @@ { "path": "../../resolving/resolver-base" }, + { + "path": "../../store/package-store" + }, { "path": "../modules-yaml" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2c691c780a..2ad4d26289 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -5302,6 +5302,9 @@ importers: '@pnpm/modules-yaml': specifier: workspace:* version: link:../modules-yaml + '@pnpm/package-store': + specifier: workspace:* + version: link:../../store/package-store '@pnpm/read-projects-context': specifier: workspace:* version: link:../read-projects-context @@ -8230,6 +8233,12 @@ importers: '@pnpm/create-cafs-store': specifier: workspace:* version: link:../create-cafs-store + '@pnpm/crypto.hash': + specifier: workspace:* + version: link:../../crypto/hash + '@pnpm/error': + specifier: workspace:* + version: link:../../packages/error '@pnpm/fetcher-base': specifier: workspace:* version: link:../../fetching/fetcher-base @@ -8260,12 +8269,18 @@ importers: '@zkochan/rimraf': specifier: 'catalog:' version: 3.0.2 + is-subdir: + specifier: 'catalog:' + version: 1.2.0 ramda: specifier: 'catalog:' version: '@pnpm/ramda@0.28.1' ssri: specifier: 'catalog:' version: 12.0.0 + symlink-dir: + specifier: 'catalog:' + version: 7.0.0 devDependencies: '@pnpm/client': specifier: workspace:* @@ -8434,6 +8449,9 @@ importers: '@pnpm/logger': specifier: workspace:* version: link:../../packages/logger + '@pnpm/package-store': + specifier: workspace:* + version: link:../package-store '@pnpm/plugin-commands-script-runners': specifier: workspace:* version: link:../../exec/plugin-commands-script-runners diff --git a/store/package-store/package.json b/store/package-store/package.json index 9098b1ab0c..94e14f012f 100644 --- a/store/package-store/package.json +++ b/store/package-store/package.json @@ -45,6 +45,8 @@ }, "dependencies": { "@pnpm/create-cafs-store": "workspace:*", + "@pnpm/crypto.hash": "workspace:*", + "@pnpm/error": "workspace:*", "@pnpm/fetcher-base": "workspace:*", "@pnpm/fs.v8-file": "workspace:*", "@pnpm/hooks.types": "workspace:*", @@ -54,8 +56,10 @@ "@pnpm/store.cafs": "workspace:*", "@pnpm/types": "workspace:*", "@zkochan/rimraf": "catalog:", + "is-subdir": "catalog:", "ramda": "catalog:", - "ssri": "catalog:" + "ssri": "catalog:", + "symlink-dir": "catalog:" }, "peerDependencies": { "@pnpm/logger": "catalog:", diff --git a/store/package-store/src/index.ts b/store/package-store/src/index.ts index 3158d38123..822def99bb 100644 --- a/store/package-store/src/index.ts +++ b/store/package-store/src/index.ts @@ -1,3 +1,4 @@ export { createPackageStore, type CafsLocker, type CreatePackageStoreOptions } from './storeController/index.js' +export { registerProject, getRegisteredProjects } from './storeController/projectRegistry.js' export * from '@pnpm/store-controller-types' diff --git a/store/package-store/src/storeController/projectRegistry.ts b/store/package-store/src/storeController/projectRegistry.ts new file mode 100644 index 0000000000..f5b57f8aa6 --- /dev/null +++ b/store/package-store/src/storeController/projectRegistry.ts @@ -0,0 +1,94 @@ +import { type Dirent, promises as fs } from 'fs' +import util from 'util' +import path from 'path' +import { createShortHash } from '@pnpm/crypto.hash' +import { PnpmError } from '@pnpm/error' +import { globalInfo } from '@pnpm/logger' +import symlinkDir from 'symlink-dir' + +const PROJECTS_DIR = 'projects' + +export function getProjectsRegistryDir (storeDir: string): string { + return path.join(storeDir, PROJECTS_DIR) +} + +/** + * Register a project as using the store. + * Creates a symlink in {storeDir}/projects/{hash} → {projectDir} + */ +export async function registerProject (storeDir: string, projectDir: string): Promise { + const registryDir = getProjectsRegistryDir(storeDir) + await fs.mkdir(registryDir, { recursive: true }) + const linkPath = path.join(registryDir, createShortHash(projectDir)) + // symlink-dir handles the case where the symlink already exists + await symlinkDir(projectDir, linkPath) +} + +/** + * Get all registered projects that use the global virtual store. + * Cleans up stale entries (projects that no longer exist). + */ +export async function getRegisteredProjects (storeDir: string): Promise { + const registryDir = getProjectsRegistryDir(storeDir) + let entries: Dirent[] + try { + entries = await fs.readdir(registryDir, { withFileTypes: true }) + } catch (err: unknown) { + if (util.types.isNativeError(err) && 'code' in err && err.code === 'ENOENT') { + return [] + } + throw err + } + + const projects: string[] = [] + await Promise.all(entries.map(async (entry) => { + if (entry.name.startsWith('.')) return + // We expect only symlinks (or junctions on Windows) in the registry + if (!entry.isSymbolicLink()) return + const linkPath = path.join(registryDir, entry.name) + + // Read the symlink target - if this fails, it's an invalid entry + let target: string + try { + target = await fs.readlink(linkPath) + } catch (err: unknown) { + // If the file is not a symlink (EINVAL) or doesn't exist (ENOENT), ignore it + if (util.types.isNativeError(err) && 'code' in err && (err.code === 'ENOENT' || err.code === 'EINVAL')) { + return + } + // For permission errors etc, inform the user + const message = util.types.isNativeError(err) ? err.message : String(err) + throw new PnpmError('PROJECT_REGISTRY_ENTRY_INACCESSIBLE', + `Cannot read project registry entry "${linkPath}": ${message}`, + { + hint: `To remove this project from the registry, delete the file at:\n ${linkPath}`, + } + ) + } + + const absoluteTarget = path.isAbsolute(target) ? target : path.resolve(path.dirname(linkPath), target) + + // Check if project still exists + try { + await fs.stat(absoluteTarget) + projects.push(absoluteTarget) + } catch (err: unknown) { + // Only clean up if project directory no longer exists + if (util.types.isNativeError(err) && 'code' in err && err.code === 'ENOENT') { + await fs.unlink(linkPath) + globalInfo(`Removed stale project registry entry: ${absoluteTarget}`) + return + } + // Can't access project - throw error to prevent incorrect pruning + const message = util.types.isNativeError(err) ? err.message : String(err) + throw new PnpmError('PROJECT_INACCESSIBLE', + `Cannot access registered project "${absoluteTarget}": ${message}`, + { + hint: `To remove this project from the registry, delete the symlink at:\n ${linkPath}`, + } + ) + } + })) + + return projects +} diff --git a/store/package-store/src/storeController/prune.ts b/store/package-store/src/storeController/prune.ts index 086f349e12..2d14a7e5af 100644 --- a/store/package-store/src/storeController/prune.ts +++ b/store/package-store/src/storeController/prune.ts @@ -6,6 +6,7 @@ import { type PackageFilesIndex } from '@pnpm/store.cafs' import { globalInfo, globalWarn } from '@pnpm/logger' import rimraf from '@zkochan/rimraf' import ssri from 'ssri' +import { pruneGlobalVirtualStore } from './pruneGlobalVirtualStore.js' const BIG_ONE = BigInt(1) as unknown @@ -15,7 +16,12 @@ export interface PruneOptions { } export async function prune ({ cacheDir, storeDir }: PruneOptions, removeAlienFiles?: boolean): Promise { - const cafsDir = path.join(storeDir, 'files') + // 1. First, prune the global virtual store + // This must happen BEFORE pruning the CAS, because removing packages from + // the virtual store will reduce hard link counts on files in the CAS + await pruneGlobalVirtualStore(storeDir) + + // 2. Clean up metadata cache const metadataDirs = await getSubdirsSafely(cacheDir) await Promise.all(metadataDirs.map(async (metadataDir) => { if (!metadataDir.startsWith('metadata')) return @@ -29,6 +35,9 @@ export async function prune ({ cacheDir, storeDir }: PruneOptions, removeAlienFi })) await rimraf(path.join(storeDir, 'tmp')) globalInfo('Removed all cached metadata files') + + // 3. Prune the content-addressable store (CAS) + const cafsDir = path.join(storeDir, 'files') const pkgIndexFiles = [] as string[] const indexDir = path.join(storeDir, 'index') await Promise.all((await getSubdirsSafely(indexDir)).map(async (dir) => { @@ -72,6 +81,7 @@ export async function prune ({ cacheDir, storeDir }: PruneOptions, removeAlienFi })) globalInfo(`Removed ${fileCounter} file${fileCounter === 1 ? '' : 's'}`) + // 4. Clean up orphaned package index files let pkgCounter = 0 await Promise.all(pkgIndexFiles.map(async (pkgIndexFilePath) => { const { files: pkgFilesIndex } = await readV8FileStrictAsync(pkgIndexFilePath) diff --git a/store/package-store/src/storeController/pruneGlobalVirtualStore.ts b/store/package-store/src/storeController/pruneGlobalVirtualStore.ts new file mode 100644 index 0000000000..a8abc60bf3 --- /dev/null +++ b/store/package-store/src/storeController/pruneGlobalVirtualStore.ts @@ -0,0 +1,297 @@ +import { type Dirent, promises as fs } from 'fs' +import util from 'util' +import path from 'path' +import crypto from 'crypto' +import { globalInfo } from '@pnpm/logger' +import rimraf from '@zkochan/rimraf' +import isSubdir from 'is-subdir' +import { getRegisteredProjects } from './projectRegistry.js' + +const LINKS_DIR = 'links' + +/** + * Prune unused packages from the global virtual store using mark-and-sweep: + * 1. Get all registered projects + * 2. Find all node_modules directories in each project (including workspace packages) + * 3. Walk symlinks from each node_modules to mark reachable packages + * 4. Remove any package directories that weren't marked as reachable + */ +export async function pruneGlobalVirtualStore (storeDir: string): Promise { + const linksDir = path.join(storeDir, LINKS_DIR) + if (!await pathExists(linksDir)) { + return + } + + const projects = await getRegisteredProjects(storeDir) + if (projects.length === 0) { + globalInfo('No registered projects for global virtual store') + return + } + + globalInfo(`Checking ${projects.length} registered project(s) for global virtual store usage`) + + // Mark phase: collect all reachable package directories + const reachable = new Set() + const visited = new Set() // Track visited directories to prevent infinite loops + + // For each project, find all node_modules directories (root + workspace packages) + await Promise.all( + projects.map(async (projectDir) => { + const nodeModulesDirs = await findAllNodeModulesDirs(projectDir) + await Promise.all( + nodeModulesDirs.map((modulesDir) => + walkSymlinksToStore(modulesDir, linksDir, reachable, visited) + ) + ) + }) + ) + + // Sweep phase: remove unreachable packages + const unreachableCount = await removeUnreachablePackages(linksDir, reachable) + if (unreachableCount > 0) { + globalInfo(`Removed ${unreachableCount} package${unreachableCount === 1 ? '' : 's'} from global virtual store`) + } else { + globalInfo('No unused packages found in global virtual store') + } +} + +/** + * Find all node_modules directories within a project, including those + * in workspace packages. Does not descend into node_modules directories. + */ +async function findAllNodeModulesDirs (projectDir: string): Promise { + const nodeModulesDirs: string[] = [] + + async function scan (dir: string): Promise { + let entries: Dirent[] + try { + entries = await fs.readdir(dir, { withFileTypes: true }) + } catch { + return + } + + const subdirs: string[] = [] + for (const entry of entries) { + if (!entry.isDirectory()) continue + + const entryPath = path.join(dir, entry.name) + + if (entry.name === 'node_modules') { + nodeModulesDirs.push(entryPath) + // Don't descend into node_modules + } else if (!entry.name.startsWith('.')) { + // Collect directories to descend into (workspace packages, etc.) + // Skip hidden directories like .git, .pnpm + subdirs.push(entryPath) + } + } + + // Scan subdirectories concurrently + await Promise.all(subdirs.map((subdir) => scan(subdir))) + } + + await scan(projectDir) + return nodeModulesDirs +} + +/** + * Recursively walk symlinks from a directory, marking any that point + * into the global virtual store's links directory. + */ +async function walkSymlinksToStore ( + dir: string, + linksDir: string, + reachable: Set, + visited: Set +): Promise { + // Prevent infinite loops from circular symlinks + const dirHash = await getRealPathHash(dir) + if (visited.has(dirHash)) { + return + } + visited.add(dirHash) + + let entries: Dirent[] + try { + entries = await fs.readdir(dir, { withFileTypes: true }) + } catch { + return + } + + await Promise.all( + entries.map(async (entry) => { + const entryPath = path.join(dir, entry.name) + + if (entry.isSymbolicLink()) { + try { + const target = await fs.readlink(entryPath) + const absoluteTarget = path.isAbsolute(target) + ? target + : path.resolve(dir, target) + + // Check if this symlink points into the global virtual store + if (isSubdir(linksDir, absoluteTarget)) { + // Mark the package directory as reachable + // The path structure is: + // - Scoped: {linksDir}/{scope}/{pkgName}/{version}/{hash}/node_modules/{pkgName} + // - Unscoped: {linksDir}/@/{pkgName}/{version}/{hash}/node_modules/{pkgName} + // We want to mark the {hash} directory + const relPath = path.relative(linksDir, absoluteTarget) + const parts = relPath.split(path.sep) + // Find the hash directory (the one containing node_modules) + const nodeModulesIdx = parts.indexOf('node_modules') + if (nodeModulesIdx !== -1) { + // Store relative path like "@scope/pkg-a/1.0.0/hash123" or "@/pkg-a/1.0.0/hash123" + const relativePath = parts.slice(0, nodeModulesIdx).join(path.sep) + reachable.add(relativePath) + // Also walk into the package's node_modules for transitive deps + const pkgNodeModules = path.join(linksDir, relativePath, 'node_modules') + await walkSymlinksToStore(pkgNodeModules, linksDir, reachable, visited) + } + } + } catch { + // Ignore broken symlinks + } + } else if (entry.isDirectory() && entry.name !== '.pnpm') { + // Recurse into directories (but not .pnpm which is the local virtual store) + await walkSymlinksToStore(entryPath, linksDir, reachable, visited) + } + }) + ) +} + +/** + * Resolve symlinks and return a hash of the real path (for cycle detection) + */ +async function getRealPathHash (p: string): Promise { + let realPath: string + try { + realPath = await fs.realpath(p) + } catch { + realPath = p + } + // Create a compact hash for in-memory use (base64url is shorter than hex that we use for file name hashes) + return crypto.createHash('sha256').update(realPath).digest('base64url') +} + +/** + * Remove package directories from the global virtual store that are not in the reachable set. + * Returns the count of removed packages. + * + * Directory structure is uniform 4-level: + * - Scoped: {linksDir}/{scope}/{pkgName}/{version}/{hash}/ + * - Unscoped: {linksDir}/@/{pkgName}/{version}/{hash}/ + */ +async function removeUnreachablePackages ( + linksDir: string, + reachable: Set +): Promise { + // First level is always a scope (either @scope or @ for unscoped packages) + const scopes = await getSubdirsSafely(linksDir) + let count = 0 + + await Promise.all( + scopes.map(async (scope) => { + const scopePath = path.join(linksDir, scope) + const pkgNames = await getSubdirsSafely(scopePath) + let removedPkgs = 0 + + await Promise.all( + pkgNames.map(async (pkgName) => { + const pkgDir = path.join(scopePath, pkgName) + const removedVersions = await removeUnreachableVersions( + pkgDir, + path.join(scope, pkgName), + reachable + ) + count += removedVersions.count + if (removedVersions.allRemoved) { + // Remove the package directory when all its versions are removed + await rimraf(pkgDir) + removedPkgs++ + } + }) + ) + + // If we removed all packages in scope, remove the scope directory + if (removedPkgs === pkgNames.length && pkgNames.length > 0) { + await rimraf(scopePath) + } + }) + ) + + return count +} + +/** + * Remove unreachable versions and hashes for a package. + * Returns the count of removed packages and whether all versions were removed. + */ +async function removeUnreachableVersions ( + pkgDir: string, + pkgPath: string, // relative path like "@/is-positive" or "@pnpm.e2e/romeo" + reachable: Set +): Promise<{ count: number, allRemoved: boolean }> { + const versions = await getSubdirsSafely(pkgDir) + let count = 0 + let removedVersions = 0 + + await Promise.all( + versions.map(async (version) => { + const versionDir = path.join(pkgDir, version) + const hashes = await getSubdirsSafely(versionDir) + + // Remove unreachable hash directories + let removedHashes = 0 + await Promise.all( + hashes.map(async (hash) => { + const relativePath = path.join(pkgPath, version, hash) + if (!reachable.has(relativePath)) { + await rimraf(path.join(versionDir, hash)) + removedHashes++ + count++ + } + }) + ) + + // If we removed all hashes, remove the version directory + if (removedHashes === hashes.length && hashes.length > 0) { + await rimraf(versionDir) + removedVersions++ + } + }) + ) + + return { + count, + allRemoved: removedVersions === versions.length && versions.length > 0, + } +} + +async function pathExists (p: string): Promise { + try { + await fs.stat(p) + return true + } catch { + return false + } +} + +async function getSubdirsSafely (dir: string): Promise { + let entries: Dirent[] + try { + entries = await fs.readdir(dir, { withFileTypes: true }) as Dirent[] + } catch (err: unknown) { + if (util.types.isNativeError(err) && 'code' in err && err.code === 'ENOENT') { + return [] + } + throw err + } + const subdirs: string[] = [] + for (const entry of entries) { + if (entry.isDirectory()) { + subdirs.push(entry.name) + } + } + return subdirs +} diff --git a/store/package-store/test/projectRegistry.ts b/store/package-store/test/projectRegistry.ts new file mode 100644 index 0000000000..5edc700772 --- /dev/null +++ b/store/package-store/test/projectRegistry.ts @@ -0,0 +1,110 @@ +/// +import { promises as fs } from 'fs' +import path from 'path' +import { registerProject, getRegisteredProjects } from '@pnpm/package-store' +import { temporaryDirectory } from 'tempy' + +describe('projectRegistry', () => { + describe('registerProject()', () => { + it('creates a symlink in the projects directory', async () => { + const storeDir = temporaryDirectory() + const projectDir = temporaryDirectory() + + await registerProject(storeDir, projectDir) + + // Check that projects directory was created + const projectsDir = path.join(storeDir, 'projects') + const entries = await fs.readdir(projectsDir) + expect(entries).toHaveLength(1) + + // Check that the symlink points to the project + const linkPath = path.join(projectsDir, entries[0]) + const target = await fs.readlink(linkPath) + expect(path.resolve(path.dirname(linkPath), target)).toBe(projectDir) + }) + + it('is idempotent - registering the same project twice works', async () => { + const storeDir = temporaryDirectory() + const projectDir = temporaryDirectory() + + await registerProject(storeDir, projectDir) + await registerProject(storeDir, projectDir) + + const projectsDir = path.join(storeDir, 'projects') + const entries = await fs.readdir(projectsDir) + expect(entries).toHaveLength(1) + }) + + it('registers multiple projects with different hashes', async () => { + const storeDir = temporaryDirectory() + const projectDir1 = temporaryDirectory() + const projectDir2 = temporaryDirectory() + + await registerProject(storeDir, projectDir1) + await registerProject(storeDir, projectDir2) + + const projectsDir = path.join(storeDir, 'projects') + const entries = await fs.readdir(projectsDir) + expect(entries).toHaveLength(2) + }) + }) + + describe('getRegisteredProjects()', () => { + it('returns empty array when no projects are registered', async () => { + const storeDir = temporaryDirectory() + + const projects = await getRegisteredProjects(storeDir) + expect(projects).toEqual([]) + }) + + it('returns registered project paths', async () => { + const storeDir = temporaryDirectory() + const projectDir1 = temporaryDirectory() + const projectDir2 = temporaryDirectory() + + await registerProject(storeDir, projectDir1) + await registerProject(storeDir, projectDir2) + + const projects = await getRegisteredProjects(storeDir) + expect(projects.sort()).toEqual([projectDir1, projectDir2].sort()) + }) + + it('cleans up stale entries for deleted projects', async () => { + const storeDir = temporaryDirectory() + const projectDir = temporaryDirectory() + + await registerProject(storeDir, projectDir) + + // Verify project is registered + let projects = await getRegisteredProjects(storeDir) + expect(projects).toEqual([projectDir]) + + // Delete the project directory + await fs.rm(projectDir, { recursive: true }) + + // getRegisteredProjects should clean up stale entry + projects = await getRegisteredProjects(storeDir) + expect(projects).toEqual([]) + + // Verify the symlink was removed + const projectsDir = path.join(storeDir, 'projects') + const entries = await fs.readdir(projectsDir) + expect(entries).toEqual([]) + }) + + it('handles mix of valid and stale entries', async () => { + const storeDir = temporaryDirectory() + const validProject = temporaryDirectory() + const staleProject = temporaryDirectory() + + await registerProject(storeDir, validProject) + await registerProject(storeDir, staleProject) + + // Delete one project + await fs.rm(staleProject, { recursive: true }) + + const projects = await getRegisteredProjects(storeDir) + expect(projects).toEqual([validProject]) + }) + }) +}) diff --git a/store/package-store/tsconfig.json b/store/package-store/tsconfig.json index 6fa8186037..bb57720d81 100644 --- a/store/package-store/tsconfig.json +++ b/store/package-store/tsconfig.json @@ -12,6 +12,9 @@ { "path": "../../__utils__/prepare" }, + { + "path": "../../crypto/hash" + }, { "path": "../../fetching/fetcher-base" }, @@ -21,6 +24,9 @@ { "path": "../../hooks/types" }, + { + "path": "../../packages/error" + }, { "path": "../../packages/logger" }, diff --git a/store/plugin-commands-store/package.json b/store/plugin-commands-store/package.json index fe7fbb8ccf..abe60075cc 100644 --- a/store/plugin-commands-store/package.json +++ b/store/plugin-commands-store/package.json @@ -61,6 +61,7 @@ "@pnpm/constants": "workspace:*", "@pnpm/lockfile.fs": "workspace:*", "@pnpm/logger": "workspace:*", + "@pnpm/package-store": "workspace:*", "@pnpm/plugin-commands-script-runners": "workspace:*", "@pnpm/plugin-commands-store": "workspace:*", "@pnpm/prepare": "workspace:*", @@ -79,4 +80,4 @@ "jest": { "preset": "@pnpm/jest-config/with-registry" } -} +} \ No newline at end of file diff --git a/store/plugin-commands-store/test/storePrune.ts b/store/plugin-commands-store/test/storePrune.ts index 129a0143e5..814de3bec2 100644 --- a/store/plugin-commands-store/test/storePrune.ts +++ b/store/plugin-commands-store/test/storePrune.ts @@ -419,3 +419,298 @@ test('prune removes cache directories that outlives dlx-cache-max-age', async () .sort() ) }) + +describe('global virtual store prune', () => { + test('prune removes unreferenced packages from global virtual store', async () => { + // Create project that installs a package with global virtual store enabled + prepare({ + dependencies: { + 'is-positive': '1.0.0', + }, + }) + const cacheDir = path.resolve('cache') + const storeDir = path.resolve('store') + + // Install with global virtual store enabled + await execa('node', [ + pnpmBin, + 'install', + `--store-dir=${storeDir}`, + `--cache-dir=${cacheDir}`, + `--registry=${REGISTRY}`, + '--config.enableGlobalVirtualStore=true', + '--config.ci=false', // This is needed because enableGlobalVirtualStore is set to fails in CI + ]) + + // Verify the links directory was created + const linksDir = path.join(storeDir, STORE_VERSION, 'links') + expect(fs.existsSync(linksDir)).toBe(true) + + // Remove the dependency from package.json and reinstall + fs.writeFileSync('package.json', JSON.stringify({ dependencies: {} })) + await execa('node', [ + pnpmBin, + 'install', + `--store-dir=${storeDir}`, + `--cache-dir=${cacheDir}`, + `--registry=${REGISTRY}`, + '--config.enableGlobalVirtualStore=true', + '--config.ci=false', + ]) + + // Run prune - should remove the now-unreferenced package + await store.handler({ + cacheDir, + dir: process.cwd(), + pnpmHomeDir: '', + rawConfig: { + registry: REGISTRY, + }, + registries: { default: REGISTRY }, + storeDir: path.join(storeDir, STORE_VERSION), + userConfig: {}, + dlxCacheMaxAge: Infinity, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, + }, ['prune']) + + // Verify: is-positive should no longer exist in links/@/ directory + const unscopedDir = path.join(linksDir, '@') + const entries = fs.existsSync(unscopedDir) ? fs.readdirSync(unscopedDir) : [] + expect(entries).not.toContain('is-positive') + }) + + test('prune keeps packages that are referenced by multiple projects', async () => { + const storeDir = path.resolve('shared-store') + const cacheDir = path.resolve('cache') + + // Create first project with is-positive + const project1Dir = path.resolve('project1') + fs.mkdirSync(project1Dir, { recursive: true }) + fs.writeFileSync(path.join(project1Dir, 'package.json'), JSON.stringify({ + dependencies: { 'is-positive': '1.0.0' }, + })) + + await execa('node', [ + pnpmBin, + 'install', + `--store-dir=${storeDir}`, + `--cache-dir=${cacheDir}`, + `--registry=${REGISTRY}`, + '--config.enableGlobalVirtualStore=true', + '--config.ci=false', + ], { cwd: project1Dir }) + + // Create second project with the same dependency + const project2Dir = path.resolve('project2') + fs.mkdirSync(project2Dir, { recursive: true }) + fs.writeFileSync(path.join(project2Dir, 'package.json'), JSON.stringify({ + dependencies: { 'is-positive': '1.0.0' }, + })) + + await execa('node', [ + pnpmBin, + 'install', + `--store-dir=${storeDir}`, + `--cache-dir=${cacheDir}`, + `--registry=${REGISTRY}`, + '--config.enableGlobalVirtualStore=true', + '--config.ci=false', + ], { cwd: project2Dir }) + + // Delete project1 + rimraf(project1Dir) + + // Verify package still exists in links/@/ directory + const linksDir = path.join(storeDir, STORE_VERSION, 'links') + const unscopedDir = path.join(linksDir, '@') + const beforePrune = fs.readdirSync(unscopedDir) + expect(beforePrune).toContain('is-positive') + + // Run prune + await store.handler({ + cacheDir, + dir: process.cwd(), + pnpmHomeDir: '', + rawConfig: { + registry: REGISTRY, + }, + registries: { default: REGISTRY }, + storeDir: path.join(storeDir, STORE_VERSION), + userConfig: {}, + dlxCacheMaxAge: Infinity, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, + }, ['prune']) + + // Package should still exist because project2 references it + const afterPrune = fs.readdirSync(unscopedDir) + expect(afterPrune).toContain('is-positive') + + rimraf(project2Dir) + }) + + test('prune removes packages when project using them is deleted', async () => { + const storeDir = path.resolve('orphan-store') + const cacheDir = path.resolve('cache') + + // Create first project with is-positive + const project1Dir = path.resolve('orphan-project1') + fs.mkdirSync(project1Dir, { recursive: true }) + fs.writeFileSync(path.join(project1Dir, 'package.json'), JSON.stringify({ + dependencies: { 'is-positive': '1.0.0' }, + })) + + await execa('node', [ + pnpmBin, + 'install', + `--store-dir=${storeDir}`, + `--cache-dir=${cacheDir}`, + `--registry=${REGISTRY}`, + '--config.enableGlobalVirtualStore=true', + '--config.ci=false', + ], { cwd: project1Dir }) + + // Create second project with a different package (so it stays) + const project2Dir = path.resolve('orphan-project2') + fs.mkdirSync(project2Dir, { recursive: true }) + fs.writeFileSync(path.join(project2Dir, 'package.json'), JSON.stringify({ + dependencies: { 'is-negative': '1.0.0' }, + })) + + await execa('node', [ + pnpmBin, + 'install', + `--store-dir=${storeDir}`, + `--cache-dir=${cacheDir}`, + `--registry=${REGISTRY}`, + '--config.enableGlobalVirtualStore=true', + '--config.ci=false', + ], { cwd: project2Dir }) + + // Verify both packages exist in links/@/ directory + const linksDir = path.join(storeDir, STORE_VERSION, 'links') + const unscopedDir = path.join(linksDir, '@') + expect(fs.existsSync(unscopedDir)).toBe(true) + const beforePrune = fs.readdirSync(unscopedDir) + expect(beforePrune).toContain('is-positive') + expect(beforePrune).toContain('is-negative') + + // Delete project1 (which uses is-positive) + rimraf(project1Dir) + + // Run prune + await store.handler({ + cacheDir, + dir: process.cwd(), + pnpmHomeDir: '', + rawConfig: { + registry: REGISTRY, + }, + registries: { default: REGISTRY }, + storeDir: path.join(storeDir, STORE_VERSION), + userConfig: {}, + dlxCacheMaxAge: Infinity, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, + }, ['prune']) + + // is-positive should be removed since project1 was deleted + const afterPrune = fs.readdirSync(unscopedDir) + expect(afterPrune).not.toContain('is-positive') + // is-negative should remain since project2 still exists + expect(afterPrune).toContain('is-negative') + + rimraf(project2Dir) + }) + + test('prune preserves transitive dependencies and removes isolated ones', async () => { + // Create project with three packages: + // - @pnpm.e2e/pkg-with-1-dep has transitive dep @pnpm.e2e/dep-of-pkg-with-1-dep + // - @pnpm.e2e/romeo has transitive dep @pnpm.e2e/romeo-dep + // - is-positive has no transitive deps + prepare({ + dependencies: { + '@pnpm.e2e/pkg-with-1-dep': '100.0.0', + '@pnpm.e2e/romeo': '1.0.0', + 'is-positive': '1.0.0', + }, + }) + + // Store should be OUTSIDE the project directory to avoid findAllNodeModulesDirs + // scanning the store's internal node_modules + const storeDir = path.resolve('..', 'transitive-store') + const cacheDir = path.resolve('..', 'cache') + + await execa('node', [ + pnpmBin, + 'install', + `--store-dir=${storeDir}`, + `--cache-dir=${cacheDir}`, + `--registry=${REGISTRY}`, + '--config.enableGlobalVirtualStore=true', + '--config.ci=false', + ]) + + // Verify all packages exist in links directory + const linksDir = path.join(storeDir, STORE_VERSION, 'links') + + // Scoped packages are in links/@pnpm.e2e/pkg-name/ + const scopeDir = path.join(linksDir, '@pnpm.e2e') + const scopedPkgs = fs.readdirSync(scopeDir) + expect(scopedPkgs).toContain('pkg-with-1-dep') + expect(scopedPkgs).toContain('dep-of-pkg-with-1-dep') + expect(scopedPkgs).toContain('romeo') + expect(scopedPkgs).toContain('romeo-dep') + // Unscoped packages are in links/@/pkg-name/ (uniform 4-level depth) + const unscopedDir = path.join(linksDir, '@') + const unscopedPkgs = fs.readdirSync(unscopedDir) + expect(unscopedPkgs).toContain('is-positive') + + // Remove @pnpm.e2e/pkg-with-1-dep, keeping romeo and is-positive + fs.writeFileSync('package.json', JSON.stringify({ + dependencies: { + '@pnpm.e2e/romeo': '1.0.0', + 'is-positive': '1.0.0', + }, + })) + await execa('node', [ + pnpmBin, + 'install', + `--store-dir=${storeDir}`, + `--cache-dir=${cacheDir}`, + `--registry=${REGISTRY}`, + '--config.enableGlobalVirtualStore=true', + '--config.ci=false', + ]) + + // Run prune + await store.handler({ + cacheDir, + dir: process.cwd(), + pnpmHomeDir: '', + rawConfig: { + registry: REGISTRY, + }, + registries: { default: REGISTRY }, + storeDir: path.join(storeDir, STORE_VERSION), + userConfig: {}, + dlxCacheMaxAge: Infinity, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, + }, ['prune']) + + // Verify: + // - pkg-with-1-dep and its transitive dep-of-pkg-with-1-dep should be removed + // - romeo and its transitive romeo-dep should still exist + // - is-positive should still exist + const afterPruneScopes = fs.readdirSync(linksDir) + expect(afterPruneScopes).toContain('@') // unscoped packages scope + const unscopedAfterPrune = fs.readdirSync(unscopedDir) + expect(unscopedAfterPrune).toContain('is-positive') + + const scopedPkgsAfter = fs.readdirSync(scopeDir) + // pkg-with-1-dep and its transitive dep should be removed + expect(scopedPkgsAfter).not.toEqual(expect.arrayContaining([expect.stringContaining('pkg-with-1-dep')])) + expect(scopedPkgsAfter).not.toEqual(expect.arrayContaining([expect.stringContaining('dep-of-pkg-with-1-dep')])) + // romeo and its transitive dep should be preserved + expect(scopedPkgsAfter).toContain('romeo') + expect(scopedPkgsAfter).toContain('romeo-dep') + }) +}) diff --git a/store/plugin-commands-store/tsconfig.json b/store/plugin-commands-store/tsconfig.json index 182cb1eebb..798515dbe2 100644 --- a/store/plugin-commands-store/tsconfig.json +++ b/store/plugin-commands-store/tsconfig.json @@ -60,6 +60,9 @@ { "path": "../cafs" }, + { + "path": "../package-store" + }, { "path": "../store-connection-manager" },