mirror of
https://github.com/pnpm/pnpm.git
synced 2026-04-27 18:46:18 -04:00
feat: enhance store prune to clean global virtual store (#10360)
* feat: enhance `store prune` to clean global virtual store `pnpm store prune` will now clean the global virtual store via a new project registry and mark-and-sweep garbage collection. * test: add store prune test for transitive dependency preservation * refactor: extract global virtual store pruning logic to a new file * fix: improve symlink handling in global virtual store pruning * fix: optimize removal of unreachable packages in global virtual store * fix: refine project registry error handling Throw `PnpmError` for inaccessible projects and specifically clean up stale symlinks for `ENOENT` errors. * test: create virtual store with install command * refactor: standardize global virtual store directory structure by placing unscoped packages under an `@` scope. * test: update store prune tests to use `toContain` and `not.toContain` assertions` * fix: linting issues * Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * fix: implemented CR suggestions * fix: revert not needed change * fix: use `is-subdir` to accurately determine if symlink targets are within the global virtual store. * revert: changes in package.json files * test: add `--config.ci=false` to store prune tests
This commit is contained in:
9
.changeset/global-virtual-store-prune.md
Normal file
9
.changeset/global-virtual-store-prune.md
Normal file
@@ -0,0 +1,9 @@
|
||||
---
|
||||
"@pnpm/package-store": minor
|
||||
"@pnpm/get-context": minor
|
||||
"pnpm": minor
|
||||
---
|
||||
|
||||
Added project registry for global virtual store prune support.
|
||||
|
||||
Projects using the store are now registered via symlinks in `{storeDir}/v10/projects/`. This enables `pnpm store prune` to track which packages are still in use by active projects and safely remove unused packages from the global virtual store.
|
||||
14
.changeset/prune-global-virtual-store.md
Normal file
14
.changeset/prune-global-virtual-store.md
Normal file
@@ -0,0 +1,14 @@
|
||||
---
|
||||
"@pnpm/package-store": minor
|
||||
"pnpm": minor
|
||||
---
|
||||
|
||||
Added mark-and-sweep garbage collection for global virtual store.
|
||||
|
||||
`pnpm store prune` now removes unused packages from the global virtual store's `links/` directory. The algorithm:
|
||||
|
||||
1. Scans all registered projects for symlinks pointing to the store
|
||||
2. Walks transitive dependencies to mark reachable packages
|
||||
3. Removes any package directories not marked as reachable
|
||||
|
||||
This includes support for workspace monorepos - all `node_modules` directories within a project (including those in workspace packages) are scanned.
|
||||
@@ -36,6 +36,7 @@
|
||||
"@pnpm/core-loggers": "workspace:*",
|
||||
"@pnpm/lockfile.fs": "workspace:*",
|
||||
"@pnpm/modules-yaml": "workspace:*",
|
||||
"@pnpm/package-store": "workspace:*",
|
||||
"@pnpm/read-projects-context": "workspace:*",
|
||||
"@pnpm/resolver-base": "workspace:*",
|
||||
"@pnpm/types": "workspace:*",
|
||||
|
||||
@@ -21,6 +21,7 @@ import {
|
||||
} from '@pnpm/types'
|
||||
import pathAbsolute from 'path-absolute'
|
||||
import { clone } from 'ramda'
|
||||
import { registerProject } from '@pnpm/package-store'
|
||||
import { readLockfiles } from './readLockfiles.js'
|
||||
|
||||
/**
|
||||
@@ -123,6 +124,9 @@ export async function getContext (
|
||||
|
||||
await fs.mkdir(opts.storeDir, { recursive: true })
|
||||
|
||||
// Register this project for store prune tracking
|
||||
await registerProject(opts.storeDir, opts.lockfileDir)
|
||||
|
||||
for (const project of opts.allProjects) {
|
||||
packageManifestLogger.debug({
|
||||
initial: project.manifest,
|
||||
@@ -293,6 +297,9 @@ export async function getContextForSingleImporter (
|
||||
const virtualStoreDir = pathAbsolute(opts.virtualStoreDir ?? 'node_modules/.pnpm', opts.lockfileDir)
|
||||
|
||||
await fs.mkdir(storeDir, { recursive: true })
|
||||
|
||||
// Register this project for store prune tracking
|
||||
await registerProject(storeDir, opts.lockfileDir)
|
||||
const extraBinPaths = [
|
||||
...opts.extraBinPaths || [],
|
||||
]
|
||||
|
||||
@@ -27,6 +27,9 @@
|
||||
{
|
||||
"path": "../../resolving/resolver-base"
|
||||
},
|
||||
{
|
||||
"path": "../../store/package-store"
|
||||
},
|
||||
{
|
||||
"path": "../modules-yaml"
|
||||
},
|
||||
|
||||
18
pnpm-lock.yaml
generated
18
pnpm-lock.yaml
generated
@@ -5302,6 +5302,9 @@ importers:
|
||||
'@pnpm/modules-yaml':
|
||||
specifier: workspace:*
|
||||
version: link:../modules-yaml
|
||||
'@pnpm/package-store':
|
||||
specifier: workspace:*
|
||||
version: link:../../store/package-store
|
||||
'@pnpm/read-projects-context':
|
||||
specifier: workspace:*
|
||||
version: link:../read-projects-context
|
||||
@@ -8230,6 +8233,12 @@ importers:
|
||||
'@pnpm/create-cafs-store':
|
||||
specifier: workspace:*
|
||||
version: link:../create-cafs-store
|
||||
'@pnpm/crypto.hash':
|
||||
specifier: workspace:*
|
||||
version: link:../../crypto/hash
|
||||
'@pnpm/error':
|
||||
specifier: workspace:*
|
||||
version: link:../../packages/error
|
||||
'@pnpm/fetcher-base':
|
||||
specifier: workspace:*
|
||||
version: link:../../fetching/fetcher-base
|
||||
@@ -8260,12 +8269,18 @@ importers:
|
||||
'@zkochan/rimraf':
|
||||
specifier: 'catalog:'
|
||||
version: 3.0.2
|
||||
is-subdir:
|
||||
specifier: 'catalog:'
|
||||
version: 1.2.0
|
||||
ramda:
|
||||
specifier: 'catalog:'
|
||||
version: '@pnpm/ramda@0.28.1'
|
||||
ssri:
|
||||
specifier: 'catalog:'
|
||||
version: 12.0.0
|
||||
symlink-dir:
|
||||
specifier: 'catalog:'
|
||||
version: 7.0.0
|
||||
devDependencies:
|
||||
'@pnpm/client':
|
||||
specifier: workspace:*
|
||||
@@ -8434,6 +8449,9 @@ importers:
|
||||
'@pnpm/logger':
|
||||
specifier: workspace:*
|
||||
version: link:../../packages/logger
|
||||
'@pnpm/package-store':
|
||||
specifier: workspace:*
|
||||
version: link:../package-store
|
||||
'@pnpm/plugin-commands-script-runners':
|
||||
specifier: workspace:*
|
||||
version: link:../../exec/plugin-commands-script-runners
|
||||
|
||||
@@ -45,6 +45,8 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@pnpm/create-cafs-store": "workspace:*",
|
||||
"@pnpm/crypto.hash": "workspace:*",
|
||||
"@pnpm/error": "workspace:*",
|
||||
"@pnpm/fetcher-base": "workspace:*",
|
||||
"@pnpm/fs.v8-file": "workspace:*",
|
||||
"@pnpm/hooks.types": "workspace:*",
|
||||
@@ -54,8 +56,10 @@
|
||||
"@pnpm/store.cafs": "workspace:*",
|
||||
"@pnpm/types": "workspace:*",
|
||||
"@zkochan/rimraf": "catalog:",
|
||||
"is-subdir": "catalog:",
|
||||
"ramda": "catalog:",
|
||||
"ssri": "catalog:"
|
||||
"ssri": "catalog:",
|
||||
"symlink-dir": "catalog:"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@pnpm/logger": "catalog:",
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
export { createPackageStore, type CafsLocker, type CreatePackageStoreOptions } from './storeController/index.js'
|
||||
export { registerProject, getRegisteredProjects } from './storeController/projectRegistry.js'
|
||||
|
||||
export * from '@pnpm/store-controller-types'
|
||||
|
||||
94
store/package-store/src/storeController/projectRegistry.ts
Normal file
94
store/package-store/src/storeController/projectRegistry.ts
Normal file
@@ -0,0 +1,94 @@
|
||||
import { type Dirent, promises as fs } from 'fs'
|
||||
import util from 'util'
|
||||
import path from 'path'
|
||||
import { createShortHash } from '@pnpm/crypto.hash'
|
||||
import { PnpmError } from '@pnpm/error'
|
||||
import { globalInfo } from '@pnpm/logger'
|
||||
import symlinkDir from 'symlink-dir'
|
||||
|
||||
const PROJECTS_DIR = 'projects'
|
||||
|
||||
export function getProjectsRegistryDir (storeDir: string): string {
|
||||
return path.join(storeDir, PROJECTS_DIR)
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a project as using the store.
|
||||
* Creates a symlink in {storeDir}/projects/{hash} → {projectDir}
|
||||
*/
|
||||
export async function registerProject (storeDir: string, projectDir: string): Promise<void> {
|
||||
const registryDir = getProjectsRegistryDir(storeDir)
|
||||
await fs.mkdir(registryDir, { recursive: true })
|
||||
const linkPath = path.join(registryDir, createShortHash(projectDir))
|
||||
// symlink-dir handles the case where the symlink already exists
|
||||
await symlinkDir(projectDir, linkPath)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all registered projects that use the global virtual store.
|
||||
* Cleans up stale entries (projects that no longer exist).
|
||||
*/
|
||||
export async function getRegisteredProjects (storeDir: string): Promise<string[]> {
|
||||
const registryDir = getProjectsRegistryDir(storeDir)
|
||||
let entries: Dirent[]
|
||||
try {
|
||||
entries = await fs.readdir(registryDir, { withFileTypes: true })
|
||||
} catch (err: unknown) {
|
||||
if (util.types.isNativeError(err) && 'code' in err && err.code === 'ENOENT') {
|
||||
return []
|
||||
}
|
||||
throw err
|
||||
}
|
||||
|
||||
const projects: string[] = []
|
||||
await Promise.all(entries.map(async (entry) => {
|
||||
if (entry.name.startsWith('.')) return
|
||||
// We expect only symlinks (or junctions on Windows) in the registry
|
||||
if (!entry.isSymbolicLink()) return
|
||||
const linkPath = path.join(registryDir, entry.name)
|
||||
|
||||
// Read the symlink target - if this fails, it's an invalid entry
|
||||
let target: string
|
||||
try {
|
||||
target = await fs.readlink(linkPath)
|
||||
} catch (err: unknown) {
|
||||
// If the file is not a symlink (EINVAL) or doesn't exist (ENOENT), ignore it
|
||||
if (util.types.isNativeError(err) && 'code' in err && (err.code === 'ENOENT' || err.code === 'EINVAL')) {
|
||||
return
|
||||
}
|
||||
// For permission errors etc, inform the user
|
||||
const message = util.types.isNativeError(err) ? err.message : String(err)
|
||||
throw new PnpmError('PROJECT_REGISTRY_ENTRY_INACCESSIBLE',
|
||||
`Cannot read project registry entry "${linkPath}": ${message}`,
|
||||
{
|
||||
hint: `To remove this project from the registry, delete the file at:\n ${linkPath}`,
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
const absoluteTarget = path.isAbsolute(target) ? target : path.resolve(path.dirname(linkPath), target)
|
||||
|
||||
// Check if project still exists
|
||||
try {
|
||||
await fs.stat(absoluteTarget)
|
||||
projects.push(absoluteTarget)
|
||||
} catch (err: unknown) {
|
||||
// Only clean up if project directory no longer exists
|
||||
if (util.types.isNativeError(err) && 'code' in err && err.code === 'ENOENT') {
|
||||
await fs.unlink(linkPath)
|
||||
globalInfo(`Removed stale project registry entry: ${absoluteTarget}`)
|
||||
return
|
||||
}
|
||||
// Can't access project - throw error to prevent incorrect pruning
|
||||
const message = util.types.isNativeError(err) ? err.message : String(err)
|
||||
throw new PnpmError('PROJECT_INACCESSIBLE',
|
||||
`Cannot access registered project "${absoluteTarget}": ${message}`,
|
||||
{
|
||||
hint: `To remove this project from the registry, delete the symlink at:\n ${linkPath}`,
|
||||
}
|
||||
)
|
||||
}
|
||||
}))
|
||||
|
||||
return projects
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import { type PackageFilesIndex } from '@pnpm/store.cafs'
|
||||
import { globalInfo, globalWarn } from '@pnpm/logger'
|
||||
import rimraf from '@zkochan/rimraf'
|
||||
import ssri from 'ssri'
|
||||
import { pruneGlobalVirtualStore } from './pruneGlobalVirtualStore.js'
|
||||
|
||||
const BIG_ONE = BigInt(1) as unknown
|
||||
|
||||
@@ -15,7 +16,12 @@ export interface PruneOptions {
|
||||
}
|
||||
|
||||
export async function prune ({ cacheDir, storeDir }: PruneOptions, removeAlienFiles?: boolean): Promise<void> {
|
||||
const cafsDir = path.join(storeDir, 'files')
|
||||
// 1. First, prune the global virtual store
|
||||
// This must happen BEFORE pruning the CAS, because removing packages from
|
||||
// the virtual store will reduce hard link counts on files in the CAS
|
||||
await pruneGlobalVirtualStore(storeDir)
|
||||
|
||||
// 2. Clean up metadata cache
|
||||
const metadataDirs = await getSubdirsSafely(cacheDir)
|
||||
await Promise.all(metadataDirs.map(async (metadataDir) => {
|
||||
if (!metadataDir.startsWith('metadata')) return
|
||||
@@ -29,6 +35,9 @@ export async function prune ({ cacheDir, storeDir }: PruneOptions, removeAlienFi
|
||||
}))
|
||||
await rimraf(path.join(storeDir, 'tmp'))
|
||||
globalInfo('Removed all cached metadata files')
|
||||
|
||||
// 3. Prune the content-addressable store (CAS)
|
||||
const cafsDir = path.join(storeDir, 'files')
|
||||
const pkgIndexFiles = [] as string[]
|
||||
const indexDir = path.join(storeDir, 'index')
|
||||
await Promise.all((await getSubdirsSafely(indexDir)).map(async (dir) => {
|
||||
@@ -72,6 +81,7 @@ export async function prune ({ cacheDir, storeDir }: PruneOptions, removeAlienFi
|
||||
}))
|
||||
globalInfo(`Removed ${fileCounter} file${fileCounter === 1 ? '' : 's'}`)
|
||||
|
||||
// 4. Clean up orphaned package index files
|
||||
let pkgCounter = 0
|
||||
await Promise.all(pkgIndexFiles.map(async (pkgIndexFilePath) => {
|
||||
const { files: pkgFilesIndex } = await readV8FileStrictAsync<PackageFilesIndex>(pkgIndexFilePath)
|
||||
|
||||
@@ -0,0 +1,297 @@
|
||||
import { type Dirent, promises as fs } from 'fs'
|
||||
import util from 'util'
|
||||
import path from 'path'
|
||||
import crypto from 'crypto'
|
||||
import { globalInfo } from '@pnpm/logger'
|
||||
import rimraf from '@zkochan/rimraf'
|
||||
import isSubdir from 'is-subdir'
|
||||
import { getRegisteredProjects } from './projectRegistry.js'
|
||||
|
||||
const LINKS_DIR = 'links'
|
||||
|
||||
/**
|
||||
* Prune unused packages from the global virtual store using mark-and-sweep:
|
||||
* 1. Get all registered projects
|
||||
* 2. Find all node_modules directories in each project (including workspace packages)
|
||||
* 3. Walk symlinks from each node_modules to mark reachable packages
|
||||
* 4. Remove any package directories that weren't marked as reachable
|
||||
*/
|
||||
export async function pruneGlobalVirtualStore (storeDir: string): Promise<void> {
|
||||
const linksDir = path.join(storeDir, LINKS_DIR)
|
||||
if (!await pathExists(linksDir)) {
|
||||
return
|
||||
}
|
||||
|
||||
const projects = await getRegisteredProjects(storeDir)
|
||||
if (projects.length === 0) {
|
||||
globalInfo('No registered projects for global virtual store')
|
||||
return
|
||||
}
|
||||
|
||||
globalInfo(`Checking ${projects.length} registered project(s) for global virtual store usage`)
|
||||
|
||||
// Mark phase: collect all reachable package directories
|
||||
const reachable = new Set<string>()
|
||||
const visited = new Set<string>() // Track visited directories to prevent infinite loops
|
||||
|
||||
// For each project, find all node_modules directories (root + workspace packages)
|
||||
await Promise.all(
|
||||
projects.map(async (projectDir) => {
|
||||
const nodeModulesDirs = await findAllNodeModulesDirs(projectDir)
|
||||
await Promise.all(
|
||||
nodeModulesDirs.map((modulesDir) =>
|
||||
walkSymlinksToStore(modulesDir, linksDir, reachable, visited)
|
||||
)
|
||||
)
|
||||
})
|
||||
)
|
||||
|
||||
// Sweep phase: remove unreachable packages
|
||||
const unreachableCount = await removeUnreachablePackages(linksDir, reachable)
|
||||
if (unreachableCount > 0) {
|
||||
globalInfo(`Removed ${unreachableCount} package${unreachableCount === 1 ? '' : 's'} from global virtual store`)
|
||||
} else {
|
||||
globalInfo('No unused packages found in global virtual store')
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Find all node_modules directories within a project, including those
|
||||
* in workspace packages. Does not descend into node_modules directories.
|
||||
*/
|
||||
async function findAllNodeModulesDirs (projectDir: string): Promise<string[]> {
|
||||
const nodeModulesDirs: string[] = []
|
||||
|
||||
async function scan (dir: string): Promise<void> {
|
||||
let entries: Dirent[]
|
||||
try {
|
||||
entries = await fs.readdir(dir, { withFileTypes: true })
|
||||
} catch {
|
||||
return
|
||||
}
|
||||
|
||||
const subdirs: string[] = []
|
||||
for (const entry of entries) {
|
||||
if (!entry.isDirectory()) continue
|
||||
|
||||
const entryPath = path.join(dir, entry.name)
|
||||
|
||||
if (entry.name === 'node_modules') {
|
||||
nodeModulesDirs.push(entryPath)
|
||||
// Don't descend into node_modules
|
||||
} else if (!entry.name.startsWith('.')) {
|
||||
// Collect directories to descend into (workspace packages, etc.)
|
||||
// Skip hidden directories like .git, .pnpm
|
||||
subdirs.push(entryPath)
|
||||
}
|
||||
}
|
||||
|
||||
// Scan subdirectories concurrently
|
||||
await Promise.all(subdirs.map((subdir) => scan(subdir)))
|
||||
}
|
||||
|
||||
await scan(projectDir)
|
||||
return nodeModulesDirs
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursively walk symlinks from a directory, marking any that point
|
||||
* into the global virtual store's links directory.
|
||||
*/
|
||||
async function walkSymlinksToStore (
|
||||
dir: string,
|
||||
linksDir: string,
|
||||
reachable: Set<string>,
|
||||
visited: Set<string>
|
||||
): Promise<void> {
|
||||
// Prevent infinite loops from circular symlinks
|
||||
const dirHash = await getRealPathHash(dir)
|
||||
if (visited.has(dirHash)) {
|
||||
return
|
||||
}
|
||||
visited.add(dirHash)
|
||||
|
||||
let entries: Dirent[]
|
||||
try {
|
||||
entries = await fs.readdir(dir, { withFileTypes: true })
|
||||
} catch {
|
||||
return
|
||||
}
|
||||
|
||||
await Promise.all(
|
||||
entries.map(async (entry) => {
|
||||
const entryPath = path.join(dir, entry.name)
|
||||
|
||||
if (entry.isSymbolicLink()) {
|
||||
try {
|
||||
const target = await fs.readlink(entryPath)
|
||||
const absoluteTarget = path.isAbsolute(target)
|
||||
? target
|
||||
: path.resolve(dir, target)
|
||||
|
||||
// Check if this symlink points into the global virtual store
|
||||
if (isSubdir(linksDir, absoluteTarget)) {
|
||||
// Mark the package directory as reachable
|
||||
// The path structure is:
|
||||
// - Scoped: {linksDir}/{scope}/{pkgName}/{version}/{hash}/node_modules/{pkgName}
|
||||
// - Unscoped: {linksDir}/@/{pkgName}/{version}/{hash}/node_modules/{pkgName}
|
||||
// We want to mark the {hash} directory
|
||||
const relPath = path.relative(linksDir, absoluteTarget)
|
||||
const parts = relPath.split(path.sep)
|
||||
// Find the hash directory (the one containing node_modules)
|
||||
const nodeModulesIdx = parts.indexOf('node_modules')
|
||||
if (nodeModulesIdx !== -1) {
|
||||
// Store relative path like "@scope/pkg-a/1.0.0/hash123" or "@/pkg-a/1.0.0/hash123"
|
||||
const relativePath = parts.slice(0, nodeModulesIdx).join(path.sep)
|
||||
reachable.add(relativePath)
|
||||
// Also walk into the package's node_modules for transitive deps
|
||||
const pkgNodeModules = path.join(linksDir, relativePath, 'node_modules')
|
||||
await walkSymlinksToStore(pkgNodeModules, linksDir, reachable, visited)
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Ignore broken symlinks
|
||||
}
|
||||
} else if (entry.isDirectory() && entry.name !== '.pnpm') {
|
||||
// Recurse into directories (but not .pnpm which is the local virtual store)
|
||||
await walkSymlinksToStore(entryPath, linksDir, reachable, visited)
|
||||
}
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve symlinks and return a hash of the real path (for cycle detection)
|
||||
*/
|
||||
async function getRealPathHash (p: string): Promise<string> {
|
||||
let realPath: string
|
||||
try {
|
||||
realPath = await fs.realpath(p)
|
||||
} catch {
|
||||
realPath = p
|
||||
}
|
||||
// Create a compact hash for in-memory use (base64url is shorter than hex that we use for file name hashes)
|
||||
return crypto.createHash('sha256').update(realPath).digest('base64url')
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove package directories from the global virtual store that are not in the reachable set.
|
||||
* Returns the count of removed packages.
|
||||
*
|
||||
* Directory structure is uniform 4-level:
|
||||
* - Scoped: {linksDir}/{scope}/{pkgName}/{version}/{hash}/
|
||||
* - Unscoped: {linksDir}/@/{pkgName}/{version}/{hash}/
|
||||
*/
|
||||
async function removeUnreachablePackages (
|
||||
linksDir: string,
|
||||
reachable: Set<string>
|
||||
): Promise<number> {
|
||||
// First level is always a scope (either @scope or @ for unscoped packages)
|
||||
const scopes = await getSubdirsSafely(linksDir)
|
||||
let count = 0
|
||||
|
||||
await Promise.all(
|
||||
scopes.map(async (scope) => {
|
||||
const scopePath = path.join(linksDir, scope)
|
||||
const pkgNames = await getSubdirsSafely(scopePath)
|
||||
let removedPkgs = 0
|
||||
|
||||
await Promise.all(
|
||||
pkgNames.map(async (pkgName) => {
|
||||
const pkgDir = path.join(scopePath, pkgName)
|
||||
const removedVersions = await removeUnreachableVersions(
|
||||
pkgDir,
|
||||
path.join(scope, pkgName),
|
||||
reachable
|
||||
)
|
||||
count += removedVersions.count
|
||||
if (removedVersions.allRemoved) {
|
||||
// Remove the package directory when all its versions are removed
|
||||
await rimraf(pkgDir)
|
||||
removedPkgs++
|
||||
}
|
||||
})
|
||||
)
|
||||
|
||||
// If we removed all packages in scope, remove the scope directory
|
||||
if (removedPkgs === pkgNames.length && pkgNames.length > 0) {
|
||||
await rimraf(scopePath)
|
||||
}
|
||||
})
|
||||
)
|
||||
|
||||
return count
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove unreachable versions and hashes for a package.
|
||||
* Returns the count of removed packages and whether all versions were removed.
|
||||
*/
|
||||
async function removeUnreachableVersions (
|
||||
pkgDir: string,
|
||||
pkgPath: string, // relative path like "@/is-positive" or "@pnpm.e2e/romeo"
|
||||
reachable: Set<string>
|
||||
): Promise<{ count: number, allRemoved: boolean }> {
|
||||
const versions = await getSubdirsSafely(pkgDir)
|
||||
let count = 0
|
||||
let removedVersions = 0
|
||||
|
||||
await Promise.all(
|
||||
versions.map(async (version) => {
|
||||
const versionDir = path.join(pkgDir, version)
|
||||
const hashes = await getSubdirsSafely(versionDir)
|
||||
|
||||
// Remove unreachable hash directories
|
||||
let removedHashes = 0
|
||||
await Promise.all(
|
||||
hashes.map(async (hash) => {
|
||||
const relativePath = path.join(pkgPath, version, hash)
|
||||
if (!reachable.has(relativePath)) {
|
||||
await rimraf(path.join(versionDir, hash))
|
||||
removedHashes++
|
||||
count++
|
||||
}
|
||||
})
|
||||
)
|
||||
|
||||
// If we removed all hashes, remove the version directory
|
||||
if (removedHashes === hashes.length && hashes.length > 0) {
|
||||
await rimraf(versionDir)
|
||||
removedVersions++
|
||||
}
|
||||
})
|
||||
)
|
||||
|
||||
return {
|
||||
count,
|
||||
allRemoved: removedVersions === versions.length && versions.length > 0,
|
||||
}
|
||||
}
|
||||
|
||||
async function pathExists (p: string): Promise<boolean> {
|
||||
try {
|
||||
await fs.stat(p)
|
||||
return true
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
async function getSubdirsSafely (dir: string): Promise<string[]> {
|
||||
let entries: Dirent[]
|
||||
try {
|
||||
entries = await fs.readdir(dir, { withFileTypes: true }) as Dirent[]
|
||||
} catch (err: unknown) {
|
||||
if (util.types.isNativeError(err) && 'code' in err && err.code === 'ENOENT') {
|
||||
return []
|
||||
}
|
||||
throw err
|
||||
}
|
||||
const subdirs: string[] = []
|
||||
for (const entry of entries) {
|
||||
if (entry.isDirectory()) {
|
||||
subdirs.push(entry.name)
|
||||
}
|
||||
}
|
||||
return subdirs
|
||||
}
|
||||
110
store/package-store/test/projectRegistry.ts
Normal file
110
store/package-store/test/projectRegistry.ts
Normal file
@@ -0,0 +1,110 @@
|
||||
/// <reference path="../../../__typings__/index.d.ts"/>
|
||||
import { promises as fs } from 'fs'
|
||||
import path from 'path'
|
||||
import { registerProject, getRegisteredProjects } from '@pnpm/package-store'
|
||||
import { temporaryDirectory } from 'tempy'
|
||||
|
||||
describe('projectRegistry', () => {
|
||||
describe('registerProject()', () => {
|
||||
it('creates a symlink in the projects directory', async () => {
|
||||
const storeDir = temporaryDirectory()
|
||||
const projectDir = temporaryDirectory()
|
||||
|
||||
await registerProject(storeDir, projectDir)
|
||||
|
||||
// Check that projects directory was created
|
||||
const projectsDir = path.join(storeDir, 'projects')
|
||||
const entries = await fs.readdir(projectsDir)
|
||||
expect(entries).toHaveLength(1)
|
||||
|
||||
// Check that the symlink points to the project
|
||||
const linkPath = path.join(projectsDir, entries[0])
|
||||
const target = await fs.readlink(linkPath)
|
||||
expect(path.resolve(path.dirname(linkPath), target)).toBe(projectDir)
|
||||
})
|
||||
|
||||
it('is idempotent - registering the same project twice works', async () => {
|
||||
const storeDir = temporaryDirectory()
|
||||
const projectDir = temporaryDirectory()
|
||||
|
||||
await registerProject(storeDir, projectDir)
|
||||
await registerProject(storeDir, projectDir)
|
||||
|
||||
const projectsDir = path.join(storeDir, 'projects')
|
||||
const entries = await fs.readdir(projectsDir)
|
||||
expect(entries).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('registers multiple projects with different hashes', async () => {
|
||||
const storeDir = temporaryDirectory()
|
||||
const projectDir1 = temporaryDirectory()
|
||||
const projectDir2 = temporaryDirectory()
|
||||
|
||||
await registerProject(storeDir, projectDir1)
|
||||
await registerProject(storeDir, projectDir2)
|
||||
|
||||
const projectsDir = path.join(storeDir, 'projects')
|
||||
const entries = await fs.readdir(projectsDir)
|
||||
expect(entries).toHaveLength(2)
|
||||
})
|
||||
})
|
||||
|
||||
describe('getRegisteredProjects()', () => {
|
||||
it('returns empty array when no projects are registered', async () => {
|
||||
const storeDir = temporaryDirectory()
|
||||
|
||||
const projects = await getRegisteredProjects(storeDir)
|
||||
expect(projects).toEqual([])
|
||||
})
|
||||
|
||||
it('returns registered project paths', async () => {
|
||||
const storeDir = temporaryDirectory()
|
||||
const projectDir1 = temporaryDirectory()
|
||||
const projectDir2 = temporaryDirectory()
|
||||
|
||||
await registerProject(storeDir, projectDir1)
|
||||
await registerProject(storeDir, projectDir2)
|
||||
|
||||
const projects = await getRegisteredProjects(storeDir)
|
||||
expect(projects.sort()).toEqual([projectDir1, projectDir2].sort())
|
||||
})
|
||||
|
||||
it('cleans up stale entries for deleted projects', async () => {
|
||||
const storeDir = temporaryDirectory()
|
||||
const projectDir = temporaryDirectory()
|
||||
|
||||
await registerProject(storeDir, projectDir)
|
||||
|
||||
// Verify project is registered
|
||||
let projects = await getRegisteredProjects(storeDir)
|
||||
expect(projects).toEqual([projectDir])
|
||||
|
||||
// Delete the project directory
|
||||
await fs.rm(projectDir, { recursive: true })
|
||||
|
||||
// getRegisteredProjects should clean up stale entry
|
||||
projects = await getRegisteredProjects(storeDir)
|
||||
expect(projects).toEqual([])
|
||||
|
||||
// Verify the symlink was removed
|
||||
const projectsDir = path.join(storeDir, 'projects')
|
||||
const entries = await fs.readdir(projectsDir)
|
||||
expect(entries).toEqual([])
|
||||
})
|
||||
|
||||
it('handles mix of valid and stale entries', async () => {
|
||||
const storeDir = temporaryDirectory()
|
||||
const validProject = temporaryDirectory()
|
||||
const staleProject = temporaryDirectory()
|
||||
|
||||
await registerProject(storeDir, validProject)
|
||||
await registerProject(storeDir, staleProject)
|
||||
|
||||
// Delete one project
|
||||
await fs.rm(staleProject, { recursive: true })
|
||||
|
||||
const projects = await getRegisteredProjects(storeDir)
|
||||
expect(projects).toEqual([validProject])
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -12,6 +12,9 @@
|
||||
{
|
||||
"path": "../../__utils__/prepare"
|
||||
},
|
||||
{
|
||||
"path": "../../crypto/hash"
|
||||
},
|
||||
{
|
||||
"path": "../../fetching/fetcher-base"
|
||||
},
|
||||
@@ -21,6 +24,9 @@
|
||||
{
|
||||
"path": "../../hooks/types"
|
||||
},
|
||||
{
|
||||
"path": "../../packages/error"
|
||||
},
|
||||
{
|
||||
"path": "../../packages/logger"
|
||||
},
|
||||
|
||||
@@ -61,6 +61,7 @@
|
||||
"@pnpm/constants": "workspace:*",
|
||||
"@pnpm/lockfile.fs": "workspace:*",
|
||||
"@pnpm/logger": "workspace:*",
|
||||
"@pnpm/package-store": "workspace:*",
|
||||
"@pnpm/plugin-commands-script-runners": "workspace:*",
|
||||
"@pnpm/plugin-commands-store": "workspace:*",
|
||||
"@pnpm/prepare": "workspace:*",
|
||||
@@ -79,4 +80,4 @@
|
||||
"jest": {
|
||||
"preset": "@pnpm/jest-config/with-registry"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -419,3 +419,298 @@ test('prune removes cache directories that outlives dlx-cache-max-age', async ()
|
||||
.sort()
|
||||
)
|
||||
})
|
||||
|
||||
describe('global virtual store prune', () => {
|
||||
test('prune removes unreferenced packages from global virtual store', async () => {
|
||||
// Create project that installs a package with global virtual store enabled
|
||||
prepare({
|
||||
dependencies: {
|
||||
'is-positive': '1.0.0',
|
||||
},
|
||||
})
|
||||
const cacheDir = path.resolve('cache')
|
||||
const storeDir = path.resolve('store')
|
||||
|
||||
// Install with global virtual store enabled
|
||||
await execa('node', [
|
||||
pnpmBin,
|
||||
'install',
|
||||
`--store-dir=${storeDir}`,
|
||||
`--cache-dir=${cacheDir}`,
|
||||
`--registry=${REGISTRY}`,
|
||||
'--config.enableGlobalVirtualStore=true',
|
||||
'--config.ci=false', // This is needed because enableGlobalVirtualStore is set to fails in CI
|
||||
])
|
||||
|
||||
// Verify the links directory was created
|
||||
const linksDir = path.join(storeDir, STORE_VERSION, 'links')
|
||||
expect(fs.existsSync(linksDir)).toBe(true)
|
||||
|
||||
// Remove the dependency from package.json and reinstall
|
||||
fs.writeFileSync('package.json', JSON.stringify({ dependencies: {} }))
|
||||
await execa('node', [
|
||||
pnpmBin,
|
||||
'install',
|
||||
`--store-dir=${storeDir}`,
|
||||
`--cache-dir=${cacheDir}`,
|
||||
`--registry=${REGISTRY}`,
|
||||
'--config.enableGlobalVirtualStore=true',
|
||||
'--config.ci=false',
|
||||
])
|
||||
|
||||
// Run prune - should remove the now-unreferenced package
|
||||
await store.handler({
|
||||
cacheDir,
|
||||
dir: process.cwd(),
|
||||
pnpmHomeDir: '',
|
||||
rawConfig: {
|
||||
registry: REGISTRY,
|
||||
},
|
||||
registries: { default: REGISTRY },
|
||||
storeDir: path.join(storeDir, STORE_VERSION),
|
||||
userConfig: {},
|
||||
dlxCacheMaxAge: Infinity,
|
||||
virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120,
|
||||
}, ['prune'])
|
||||
|
||||
// Verify: is-positive should no longer exist in links/@/ directory
|
||||
const unscopedDir = path.join(linksDir, '@')
|
||||
const entries = fs.existsSync(unscopedDir) ? fs.readdirSync(unscopedDir) : []
|
||||
expect(entries).not.toContain('is-positive')
|
||||
})
|
||||
|
||||
test('prune keeps packages that are referenced by multiple projects', async () => {
|
||||
const storeDir = path.resolve('shared-store')
|
||||
const cacheDir = path.resolve('cache')
|
||||
|
||||
// Create first project with is-positive
|
||||
const project1Dir = path.resolve('project1')
|
||||
fs.mkdirSync(project1Dir, { recursive: true })
|
||||
fs.writeFileSync(path.join(project1Dir, 'package.json'), JSON.stringify({
|
||||
dependencies: { 'is-positive': '1.0.0' },
|
||||
}))
|
||||
|
||||
await execa('node', [
|
||||
pnpmBin,
|
||||
'install',
|
||||
`--store-dir=${storeDir}`,
|
||||
`--cache-dir=${cacheDir}`,
|
||||
`--registry=${REGISTRY}`,
|
||||
'--config.enableGlobalVirtualStore=true',
|
||||
'--config.ci=false',
|
||||
], { cwd: project1Dir })
|
||||
|
||||
// Create second project with the same dependency
|
||||
const project2Dir = path.resolve('project2')
|
||||
fs.mkdirSync(project2Dir, { recursive: true })
|
||||
fs.writeFileSync(path.join(project2Dir, 'package.json'), JSON.stringify({
|
||||
dependencies: { 'is-positive': '1.0.0' },
|
||||
}))
|
||||
|
||||
await execa('node', [
|
||||
pnpmBin,
|
||||
'install',
|
||||
`--store-dir=${storeDir}`,
|
||||
`--cache-dir=${cacheDir}`,
|
||||
`--registry=${REGISTRY}`,
|
||||
'--config.enableGlobalVirtualStore=true',
|
||||
'--config.ci=false',
|
||||
], { cwd: project2Dir })
|
||||
|
||||
// Delete project1
|
||||
rimraf(project1Dir)
|
||||
|
||||
// Verify package still exists in links/@/ directory
|
||||
const linksDir = path.join(storeDir, STORE_VERSION, 'links')
|
||||
const unscopedDir = path.join(linksDir, '@')
|
||||
const beforePrune = fs.readdirSync(unscopedDir)
|
||||
expect(beforePrune).toContain('is-positive')
|
||||
|
||||
// Run prune
|
||||
await store.handler({
|
||||
cacheDir,
|
||||
dir: process.cwd(),
|
||||
pnpmHomeDir: '',
|
||||
rawConfig: {
|
||||
registry: REGISTRY,
|
||||
},
|
||||
registries: { default: REGISTRY },
|
||||
storeDir: path.join(storeDir, STORE_VERSION),
|
||||
userConfig: {},
|
||||
dlxCacheMaxAge: Infinity,
|
||||
virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120,
|
||||
}, ['prune'])
|
||||
|
||||
// Package should still exist because project2 references it
|
||||
const afterPrune = fs.readdirSync(unscopedDir)
|
||||
expect(afterPrune).toContain('is-positive')
|
||||
|
||||
rimraf(project2Dir)
|
||||
})
|
||||
|
||||
test('prune removes packages when project using them is deleted', async () => {
|
||||
const storeDir = path.resolve('orphan-store')
|
||||
const cacheDir = path.resolve('cache')
|
||||
|
||||
// Create first project with is-positive
|
||||
const project1Dir = path.resolve('orphan-project1')
|
||||
fs.mkdirSync(project1Dir, { recursive: true })
|
||||
fs.writeFileSync(path.join(project1Dir, 'package.json'), JSON.stringify({
|
||||
dependencies: { 'is-positive': '1.0.0' },
|
||||
}))
|
||||
|
||||
await execa('node', [
|
||||
pnpmBin,
|
||||
'install',
|
||||
`--store-dir=${storeDir}`,
|
||||
`--cache-dir=${cacheDir}`,
|
||||
`--registry=${REGISTRY}`,
|
||||
'--config.enableGlobalVirtualStore=true',
|
||||
'--config.ci=false',
|
||||
], { cwd: project1Dir })
|
||||
|
||||
// Create second project with a different package (so it stays)
|
||||
const project2Dir = path.resolve('orphan-project2')
|
||||
fs.mkdirSync(project2Dir, { recursive: true })
|
||||
fs.writeFileSync(path.join(project2Dir, 'package.json'), JSON.stringify({
|
||||
dependencies: { 'is-negative': '1.0.0' },
|
||||
}))
|
||||
|
||||
await execa('node', [
|
||||
pnpmBin,
|
||||
'install',
|
||||
`--store-dir=${storeDir}`,
|
||||
`--cache-dir=${cacheDir}`,
|
||||
`--registry=${REGISTRY}`,
|
||||
'--config.enableGlobalVirtualStore=true',
|
||||
'--config.ci=false',
|
||||
], { cwd: project2Dir })
|
||||
|
||||
// Verify both packages exist in links/@/ directory
|
||||
const linksDir = path.join(storeDir, STORE_VERSION, 'links')
|
||||
const unscopedDir = path.join(linksDir, '@')
|
||||
expect(fs.existsSync(unscopedDir)).toBe(true)
|
||||
const beforePrune = fs.readdirSync(unscopedDir)
|
||||
expect(beforePrune).toContain('is-positive')
|
||||
expect(beforePrune).toContain('is-negative')
|
||||
|
||||
// Delete project1 (which uses is-positive)
|
||||
rimraf(project1Dir)
|
||||
|
||||
// Run prune
|
||||
await store.handler({
|
||||
cacheDir,
|
||||
dir: process.cwd(),
|
||||
pnpmHomeDir: '',
|
||||
rawConfig: {
|
||||
registry: REGISTRY,
|
||||
},
|
||||
registries: { default: REGISTRY },
|
||||
storeDir: path.join(storeDir, STORE_VERSION),
|
||||
userConfig: {},
|
||||
dlxCacheMaxAge: Infinity,
|
||||
virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120,
|
||||
}, ['prune'])
|
||||
|
||||
// is-positive should be removed since project1 was deleted
|
||||
const afterPrune = fs.readdirSync(unscopedDir)
|
||||
expect(afterPrune).not.toContain('is-positive')
|
||||
// is-negative should remain since project2 still exists
|
||||
expect(afterPrune).toContain('is-negative')
|
||||
|
||||
rimraf(project2Dir)
|
||||
})
|
||||
|
||||
test('prune preserves transitive dependencies and removes isolated ones', async () => {
|
||||
// Create project with three packages:
|
||||
// - @pnpm.e2e/pkg-with-1-dep has transitive dep @pnpm.e2e/dep-of-pkg-with-1-dep
|
||||
// - @pnpm.e2e/romeo has transitive dep @pnpm.e2e/romeo-dep
|
||||
// - is-positive has no transitive deps
|
||||
prepare({
|
||||
dependencies: {
|
||||
'@pnpm.e2e/pkg-with-1-dep': '100.0.0',
|
||||
'@pnpm.e2e/romeo': '1.0.0',
|
||||
'is-positive': '1.0.0',
|
||||
},
|
||||
})
|
||||
|
||||
// Store should be OUTSIDE the project directory to avoid findAllNodeModulesDirs
|
||||
// scanning the store's internal node_modules
|
||||
const storeDir = path.resolve('..', 'transitive-store')
|
||||
const cacheDir = path.resolve('..', 'cache')
|
||||
|
||||
await execa('node', [
|
||||
pnpmBin,
|
||||
'install',
|
||||
`--store-dir=${storeDir}`,
|
||||
`--cache-dir=${cacheDir}`,
|
||||
`--registry=${REGISTRY}`,
|
||||
'--config.enableGlobalVirtualStore=true',
|
||||
'--config.ci=false',
|
||||
])
|
||||
|
||||
// Verify all packages exist in links directory
|
||||
const linksDir = path.join(storeDir, STORE_VERSION, 'links')
|
||||
|
||||
// Scoped packages are in links/@pnpm.e2e/pkg-name/
|
||||
const scopeDir = path.join(linksDir, '@pnpm.e2e')
|
||||
const scopedPkgs = fs.readdirSync(scopeDir)
|
||||
expect(scopedPkgs).toContain('pkg-with-1-dep')
|
||||
expect(scopedPkgs).toContain('dep-of-pkg-with-1-dep')
|
||||
expect(scopedPkgs).toContain('romeo')
|
||||
expect(scopedPkgs).toContain('romeo-dep')
|
||||
// Unscoped packages are in links/@/pkg-name/ (uniform 4-level depth)
|
||||
const unscopedDir = path.join(linksDir, '@')
|
||||
const unscopedPkgs = fs.readdirSync(unscopedDir)
|
||||
expect(unscopedPkgs).toContain('is-positive')
|
||||
|
||||
// Remove @pnpm.e2e/pkg-with-1-dep, keeping romeo and is-positive
|
||||
fs.writeFileSync('package.json', JSON.stringify({
|
||||
dependencies: {
|
||||
'@pnpm.e2e/romeo': '1.0.0',
|
||||
'is-positive': '1.0.0',
|
||||
},
|
||||
}))
|
||||
await execa('node', [
|
||||
pnpmBin,
|
||||
'install',
|
||||
`--store-dir=${storeDir}`,
|
||||
`--cache-dir=${cacheDir}`,
|
||||
`--registry=${REGISTRY}`,
|
||||
'--config.enableGlobalVirtualStore=true',
|
||||
'--config.ci=false',
|
||||
])
|
||||
|
||||
// Run prune
|
||||
await store.handler({
|
||||
cacheDir,
|
||||
dir: process.cwd(),
|
||||
pnpmHomeDir: '',
|
||||
rawConfig: {
|
||||
registry: REGISTRY,
|
||||
},
|
||||
registries: { default: REGISTRY },
|
||||
storeDir: path.join(storeDir, STORE_VERSION),
|
||||
userConfig: {},
|
||||
dlxCacheMaxAge: Infinity,
|
||||
virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120,
|
||||
}, ['prune'])
|
||||
|
||||
// Verify:
|
||||
// - pkg-with-1-dep and its transitive dep-of-pkg-with-1-dep should be removed
|
||||
// - romeo and its transitive romeo-dep should still exist
|
||||
// - is-positive should still exist
|
||||
const afterPruneScopes = fs.readdirSync(linksDir)
|
||||
expect(afterPruneScopes).toContain('@') // unscoped packages scope
|
||||
const unscopedAfterPrune = fs.readdirSync(unscopedDir)
|
||||
expect(unscopedAfterPrune).toContain('is-positive')
|
||||
|
||||
const scopedPkgsAfter = fs.readdirSync(scopeDir)
|
||||
// pkg-with-1-dep and its transitive dep should be removed
|
||||
expect(scopedPkgsAfter).not.toEqual(expect.arrayContaining([expect.stringContaining('pkg-with-1-dep')]))
|
||||
expect(scopedPkgsAfter).not.toEqual(expect.arrayContaining([expect.stringContaining('dep-of-pkg-with-1-dep')]))
|
||||
// romeo and its transitive dep should be preserved
|
||||
expect(scopedPkgsAfter).toContain('romeo')
|
||||
expect(scopedPkgsAfter).toContain('romeo-dep')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -60,6 +60,9 @@
|
||||
{
|
||||
"path": "../cafs"
|
||||
},
|
||||
{
|
||||
"path": "../package-store"
|
||||
},
|
||||
{
|
||||
"path": "../store-connection-manager"
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user