mirror of
https://github.com/pnpm/pnpm.git
synced 2026-05-12 18:49:41 -04:00
## Summary
Adds an opt-in **pnpm agent** server that resolves dependencies server-side and streams only the files missing from the client's content-addressable store.
- **`@pnpm/agent.server`** — multi-process HTTP server (Node.js `cluster`) with SQLite-backed metadata and file caches
- **`@pnpm/agent.client`** — streams an NDJSON response, dispatches worker threads to fetch files while the server is still resolving
- **New config**: `agent` in `pnpm-workspace.yaml` (opt-in)
## How it works
1. Client reads integrity hashes from its local store index
2. Sends `POST /v1/install` with dependencies + store integrities
3. Server resolves the dependency tree using pnpm's `install({ lockfileOnly: true })`, with a SQLite-backed `PackageMetaCache` for fast repeat resolution
4. As each package resolves, a wrapped `storeController.requestPackage` looks up its files and immediately streams digests the client is missing (NDJSON `D` lines)
5. Client reads the stream line by line; digest batches fill up and dispatch worker threads to `POST /v1/files` — file downloads overlap with server-side resolution
6. After resolution, server sends index entries (`I` lines) and lockfile (`L` line)
7. Client writes index entries to store, then runs headless install with a wrapped `fetchPackage` that calls `readPkgFromCafs` with `verifyStoreIntegrity: false` (files are trusted from the agent)
8. `/v1/files` response is gzip-streamed (274MB → ~80MB) — server pipes through `createGzip`, worker pipes through `createGunzip`, parsing and writing files to CAFS as data arrives
## Performance
1351-package project, cold local store, warm server (localhost):
| Scenario | Time |
|----------|------|
| Vanilla pnpm install (cold OS cache) | ~48s |
| Vanilla pnpm install (warm OS cache) | ~34s |
| With pnpm agent (consistent) | **~33s** |
### Key optimizations
1. **SQLite metadata cache** — server-side resolution drops from ~3.4s to ~0.9s
2. **SQLite file store** — consistent read performance regardless of OS file cache state
3. **Streaming `/v1/install`** — file digests stream during resolution, downloads start before resolution finishes
4. **Gzip-streamed `/v1/files`** — whole-stream gzip (274MB → ~80MB), significant savings on remote servers
5. **Worker-thread streaming HTTP** — workers pipe gzip → parse → write to CAFS as data arrives, no buffering
6. **No rehashing** — server-provided digests used directly, skipping 33K SHA-512 computations
7. **No re-verification** — wrapped `fetchPackage` calls `readPkgFromCafs` with `verifyStoreIntegrity: false`
8. **Direct `writeFileSync` with `wx`** — no stat + temp + rename
9. **Pre-packed msgpack** — server sends raw store index buffers, client writes directly to SQLite
10. **WAL checkpoint** — ensures store index entries written by agent are visible to headless install's worker threads
## Usage
Start the server:
```bash
node agent/server/lib/bin.js
```
Configure in `pnpm-workspace.yaml`:
```yaml
agent: http://localhost:4873
```
300 lines
8.5 KiB
TypeScript
300 lines
8.5 KiB
TypeScript
import type { Catalogs } from '@pnpm/catalogs.types'
|
|
import type { Hooks } from '@pnpm/hooks.pnpmfile'
|
|
import type {
|
|
EngineDependency,
|
|
Finder,
|
|
Project,
|
|
ProjectManifest,
|
|
ProjectsGraph,
|
|
Registries,
|
|
RegistryConfig,
|
|
TrustPolicy,
|
|
} from '@pnpm/types'
|
|
|
|
import type { OptionsFromRootManifest } from './getOptionsFromRootManifest.js'
|
|
|
|
export type UniversalOptions = Pick<Config, 'color' | 'dir' | 'authConfig'>
|
|
|
|
|
|
export type VerifyDepsBeforeRun = 'install' | 'warn' | 'error' | 'prompt' | false
|
|
|
|
/**
|
|
* Runtime state, workspace context, and CLI metadata.
|
|
* These fields are NOT user-facing settings — they are computed at startup
|
|
* or populated later by the CLI harness (e.g. workspace filtering, hook loading).
|
|
*/
|
|
export interface ConfigContext {
|
|
// -- Runtime state --
|
|
hooks?: Hooks
|
|
finders?: Record<string, Finder>
|
|
|
|
// -- Workspace context --
|
|
allProjects?: Project[]
|
|
selectedProjectsGraph?: ProjectsGraph
|
|
allProjectsGraph?: ProjectsGraph
|
|
rootProjectManifest?: ProjectManifest
|
|
rootProjectManifestDir: string
|
|
|
|
// -- CLI metadata --
|
|
cliOptions: Record<string, any> // eslint-disable-line
|
|
/** Keys explicitly set from workspace yaml, CLI, or env vars (not defaults). */
|
|
explicitlySetKeys: Set<string>
|
|
packageManager: {
|
|
name: string
|
|
version: string
|
|
}
|
|
wantedPackageManager?: WantedPackageManager
|
|
}
|
|
|
|
/**
|
|
* The package manager requested by the root project's manifest.
|
|
* Extends {@link EngineDependency} with the source of the declaration so that
|
|
* callers can treat the legacy `packageManager` field and
|
|
* `devEngines.packageManager` differently (e.g. only the latter persists
|
|
* resolved pnpm integrity info to `pnpm-lock.yaml`).
|
|
*/
|
|
export interface WantedPackageManager extends EngineDependency {
|
|
fromDevEngines?: boolean
|
|
}
|
|
|
|
/**
|
|
* User-facing settings + auth/network config.
|
|
* Does NOT include runtime state — see {@link ConfigContext} for that.
|
|
*/
|
|
export interface Config extends OptionsFromRootManifest {
|
|
allowNew: boolean
|
|
autoConfirmAllPrompts?: boolean
|
|
autoInstallPeers?: boolean
|
|
bail: boolean
|
|
color: 'always' | 'auto' | 'never'
|
|
useBetaCli: boolean
|
|
excludeLinksFromLockfile: boolean
|
|
extraBinPaths: string[]
|
|
extraEnv: Record<string, string>
|
|
failIfNoMatch: boolean
|
|
filter: string[]
|
|
filterProd: string[]
|
|
authConfig: Record<string, any>, // eslint-disable-line
|
|
dryRun?: boolean // This option might be not supported ever
|
|
global?: boolean
|
|
dir: string
|
|
bin: string
|
|
verifyDepsBeforeRun?: VerifyDepsBeforeRun
|
|
ignoreScripts?: boolean
|
|
ignoreCompatibilityDb?: boolean
|
|
includeWorkspaceRoot?: boolean
|
|
optimisticRepeatInstall?: boolean
|
|
save?: boolean
|
|
saveProd?: boolean
|
|
saveDev?: boolean
|
|
saveOptional?: boolean
|
|
savePeer?: boolean
|
|
saveCatalogName?: string
|
|
saveWorkspaceProtocol?: boolean | 'rolling'
|
|
lockfileIncludeTarballUrl?: boolean
|
|
scriptShell?: string
|
|
stream?: boolean
|
|
pnpmExecPath: string
|
|
pnpmHomeDir: string
|
|
production?: boolean
|
|
fetchRetries?: number
|
|
fetchRetryFactor?: number
|
|
fetchRetryMintimeout?: number
|
|
fetchRetryMaxtimeout?: number
|
|
fetchTimeout?: number
|
|
saveExact?: boolean
|
|
savePrefix?: string
|
|
shellEmulator?: boolean
|
|
scriptsPrependNodePath?: boolean | 'warn-only'
|
|
force?: boolean
|
|
depth?: number
|
|
engineStrict?: boolean
|
|
nodeVersion?: string
|
|
nodeDownloadMirrors?: Record<string, string>
|
|
offline?: boolean
|
|
registry?: string
|
|
optional?: boolean
|
|
unsafePerm?: boolean
|
|
loglevel?: 'silent' | 'error' | 'warn' | 'info' | 'debug'
|
|
frozenLockfile?: boolean
|
|
preferFrozenLockfile?: boolean
|
|
only?: 'prod' | 'production' | 'dev' | 'development'
|
|
preferOffline?: boolean
|
|
sideEffectsCache?: boolean // for backward compatibility
|
|
sideEffectsCacheReadonly?: boolean // for backward compatibility
|
|
sideEffectsCacheRead?: boolean
|
|
sideEffectsCacheWrite?: boolean
|
|
shamefullyHoist?: boolean
|
|
dev?: boolean
|
|
ignoreCurrentSpecifiers?: boolean
|
|
recursive?: boolean
|
|
enablePrePostScripts?: boolean
|
|
useStderr?: boolean
|
|
nodeLinker?: 'hoisted' | 'isolated' | 'pnp'
|
|
preferSymlinkedExecutables?: boolean
|
|
resolutionMode?: 'highest' | 'time-based' | 'lowest-direct'
|
|
registrySupportsTimeField?: boolean
|
|
resolvePeersFromWorkspaceRoot?: boolean
|
|
deployAllFiles?: boolean
|
|
forceLegacyDeploy?: boolean
|
|
reporterHidePrefix?: boolean
|
|
|
|
// proxy
|
|
httpProxy?: string
|
|
httpsProxy?: string
|
|
localAddress?: string
|
|
noProxy?: string | boolean
|
|
|
|
// ssl
|
|
cert?: string | string[]
|
|
key?: string
|
|
ca?: string | string[]
|
|
strictSsl?: boolean
|
|
|
|
userAgent?: string
|
|
tag?: string
|
|
updateNotifier?: boolean
|
|
|
|
// pnpm specific configs
|
|
cacheDir: string
|
|
configDir: string
|
|
stateDir: string
|
|
storeDir?: string
|
|
virtualStoreDir?: string
|
|
virtualStoreOnly?: boolean
|
|
enableGlobalVirtualStore?: boolean
|
|
verifyStoreIntegrity?: boolean
|
|
maxSockets?: number
|
|
networkConcurrency?: number
|
|
fetchingConcurrency?: number
|
|
lockfileOnly?: boolean // like npm's --package-lock-only
|
|
childConcurrency?: number
|
|
ignorePnpmfile?: boolean
|
|
pnpmfile: string[] | string
|
|
tryLoadDefaultPnpmfile?: boolean
|
|
packageImportMethod?: 'auto' | 'hardlink' | 'copy' | 'clone' | 'clone-or-copy'
|
|
hoistPattern?: string[]
|
|
publicHoistPattern?: string[] | string
|
|
hoistWorkspacePackages?: boolean
|
|
useStoreServer?: boolean
|
|
useRunningStoreServer?: boolean
|
|
workspaceConcurrency: number
|
|
workspaceDir?: string
|
|
workspacePackagePatterns?: string[]
|
|
catalogs?: Catalogs
|
|
catalogMode?: 'strict' | 'prefer' | 'manual'
|
|
cleanupUnusedCatalogs?: boolean
|
|
reporter?: string
|
|
aggregateOutput: boolean
|
|
linkWorkspacePackages: boolean | 'deep'
|
|
injectWorkspacePackages?: boolean
|
|
preferWorkspacePackages: boolean
|
|
reverse: boolean
|
|
sort: boolean
|
|
strictPeerDependencies: boolean
|
|
lockfileDir?: string
|
|
modulesDir?: string
|
|
sharedWorkspaceLockfile?: boolean
|
|
useLockfile: boolean
|
|
useGitBranchLockfile: boolean
|
|
mergeGitBranchLockfiles?: boolean
|
|
mergeGitBranchLockfilesBranchPattern?: string[]
|
|
globalPnpmfile?: string
|
|
npmPath?: string
|
|
gitChecks?: boolean
|
|
publishBranch?: string
|
|
recursiveInstall?: boolean
|
|
symlink: boolean
|
|
enablePnp?: boolean
|
|
enableModulesDir: boolean
|
|
modulesCacheMaxAge: number
|
|
dlxCacheMaxAge: number
|
|
embedReadme?: boolean
|
|
gitShallowHosts?: string[]
|
|
legacyDirFiltering?: boolean
|
|
allowBuilds?: Record<string, boolean | string>
|
|
dedupePeerDependents?: boolean
|
|
dedupePeers?: boolean
|
|
patchesDir?: string
|
|
ignoreWorkspaceCycles?: boolean
|
|
disallowWorkspaceCycles?: boolean
|
|
packGzipLevel?: number
|
|
blockExoticSubdeps?: boolean
|
|
|
|
agent?: string
|
|
|
|
registries: Registries
|
|
configByUri: Record<string, RegistryConfig>
|
|
ignoreWorkspaceRootCheck: boolean
|
|
workspaceRoot: boolean
|
|
|
|
testPattern?: string[]
|
|
changedFilesIgnorePattern?: string[]
|
|
userConfig: Record<string, string>
|
|
|
|
hoist: boolean
|
|
packageLock: boolean
|
|
pending: boolean
|
|
userconfig: string
|
|
npmrcAuthFile?: string
|
|
workspacePrefix?: string
|
|
dedupeDirectDeps?: boolean
|
|
extendNodePath?: boolean
|
|
gitBranchLockfile?: boolean
|
|
globalBinDir?: string
|
|
globalDir?: string
|
|
globalPkgDir: string
|
|
lockfile?: boolean
|
|
dedupeInjectedDeps?: boolean
|
|
nodeOptions?: string
|
|
pmOnFail?: 'download' | 'error' | 'warn' | 'ignore'
|
|
runtimeOnFail?: 'download' | 'error' | 'warn' | 'ignore'
|
|
virtualStoreDirMaxLength: number
|
|
peersSuffixMaxLength?: number
|
|
strictStorePkgContentCheck: boolean
|
|
strictDepBuilds: boolean
|
|
syncInjectedDepsAfterScripts?: string[]
|
|
initPackageManager: boolean
|
|
initType: 'commonjs' | 'module'
|
|
dangerouslyAllowAllBuilds: boolean
|
|
ci: boolean
|
|
preserveAbsolutePaths?: boolean
|
|
minimumReleaseAge?: number
|
|
minimumReleaseAgeExclude?: string[]
|
|
minimumReleaseAgeIgnoreMissingTime?: boolean
|
|
minimumReleaseAgeStrict?: boolean
|
|
fetchWarnTimeoutMs?: number
|
|
fetchMinSpeedKiBps?: number
|
|
trustPolicy?: TrustPolicy
|
|
trustPolicyExclude?: string[]
|
|
trustPolicyIgnoreAfter?: number
|
|
auditLevel?: 'info' | 'low' | 'moderate' | 'high' | 'critical'
|
|
|
|
packageConfigs?: ProjectConfigSet
|
|
}
|
|
|
|
export interface ConfigWithDeprecatedSettings extends Config {
|
|
globalPrefix?: string
|
|
proxy?: string
|
|
}
|
|
|
|
export const PROJECT_CONFIG_FIELDS = [
|
|
'hoist',
|
|
'modulesDir',
|
|
'overrides',
|
|
'saveExact',
|
|
'savePrefix',
|
|
] as const satisfies Array<keyof Config>
|
|
|
|
export type ProjectConfig = Partial<Pick<Config, typeof PROJECT_CONFIG_FIELDS[number] | 'hoistPattern'>>
|
|
|
|
/** Simple map from project names to {@link ProjectConfig} */
|
|
export type ProjectConfigRecord = Record<string, ProjectConfig>
|
|
|
|
/** Map multiple project names to a shared {@link ProjectConfig} */
|
|
export type ProjectConfigMultiMatch = { match: string[] } & ProjectConfig
|
|
|
|
export type ProjectConfigSet =
|
|
| ProjectConfigRecord
|
|
| ProjectConfigMultiMatch[]
|