mirror/pnpm
1
0
Fork 0
mirror of https://github.com/pnpm/pnpm.git synced 2026-04-27 02:20:19 -04:00
Code Issues Packages Projects Releases Wiki Activity
Files
fcdd50aaa773bc30f90bba767ca6c374646bf87e
pnpm/pnpm-workspace.yaml
Zoltan Kochan ccc606ed15 feat: pnpm agent — server-side resolution for faster installs (#11251) 
## Summary

Adds an opt-in **pnpm agent** server that resolves dependencies server-side and streams only the files missing from the client's content-addressable store.

- **`@pnpm/agent.server`** — multi-process HTTP server (Node.js `cluster`) with SQLite-backed metadata and file caches
- **`@pnpm/agent.client`** — streams an NDJSON response, dispatches worker threads to fetch files while the server is still resolving
- **New config**: `agent` in `pnpm-workspace.yaml` (opt-in)

## How it works

1. Client reads integrity hashes from its local store index
2. Sends `POST /v1/install` with dependencies + store integrities
3. Server resolves the dependency tree using pnpm's `install({ lockfileOnly: true })`, with a SQLite-backed `PackageMetaCache` for fast repeat resolution
4. As each package resolves, a wrapped `storeController.requestPackage` looks up its files and immediately streams digests the client is missing (NDJSON `D` lines)
5. Client reads the stream line by line; digest batches fill up and dispatch worker threads to `POST /v1/files` — file downloads overlap with server-side resolution
6. After resolution, server sends index entries (`I` lines) and lockfile (`L` line)
7. Client writes index entries to store, then runs headless install with a wrapped `fetchPackage` that calls `readPkgFromCafs` with `verifyStoreIntegrity: false` (files are trusted from the agent)
8. `/v1/files` response is gzip-streamed (274MB → ~80MB) — server pipes through `createGzip`, worker pipes through `createGunzip`, parsing and writing files to CAFS as data arrives

## Performance

1351-package project, cold local store, warm server (localhost):

| Scenario | Time |
|----------|------|
| Vanilla pnpm install (cold OS cache) | ~48s |
| Vanilla pnpm install (warm OS cache) | ~34s |
| With pnpm agent (consistent) | **~33s** |

### Key optimizations

1. **SQLite metadata cache** — server-side resolution drops from ~3.4s to ~0.9s
2. **SQLite file store** — consistent read performance regardless of OS file cache state
3. **Streaming `/v1/install`** — file digests stream during resolution, downloads start before resolution finishes
4. **Gzip-streamed `/v1/files`** — whole-stream gzip (274MB → ~80MB), significant savings on remote servers
5. **Worker-thread streaming HTTP** — workers pipe gzip → parse → write to CAFS as data arrives, no buffering
6. **No rehashing** — server-provided digests used directly, skipping 33K SHA-512 computations
7. **No re-verification** — wrapped `fetchPackage` calls `readPkgFromCafs` with `verifyStoreIntegrity: false`
8. **Direct `writeFileSync` with `wx`** — no stat + temp + rename
9. **Pre-packed msgpack** — server sends raw store index buffers, client writes directly to SQLite
10. **WAL checkpoint** — ensures store index entries written by agent are visible to headless install's worker threads

## Usage

Start the server:
```bash
node agent/server/lib/bin.js
```

Configure in `pnpm-workspace.yaml`:
```yaml
agent: http://localhost:4873
```
2026-04-20 11:56:46 +02:00

486 lines
11 KiB
YAML
Raw Blame History

packages:
- .meta-updater
- __typecheck__
- __typings__
- __utils__/*
- '!__utils__/build-artifacts'
- auth/*
- building/*
- cache/*
- catalogs/*
- cli/*
- config/*
- crypto/*
- installing/dedupe/*
- deps/*
- deps/compliance/*
- deps/inspection/*
- engine/*
- engine/pm/*
- engine/runtime/*
- exec/*
- fetching/*
- fs/*
- global/*
- hooks/*
- lockfile/*
- network/*
- modules-mounter/*
- object/*
- core/*
- bins/*
- installing/*
- installing/linking/*
- pkg-manifest/*
- patching/*
- pnpm
- pnpm/dev
- testing/*
- worker
- pnpm/artifacts/*
- agent/*
- registry-access/*
- releasing/*
- resolving/*
- resolving/registry/*
- shell/*
- store/*
- text/*
- workspace/*
- yaml/*
- '!**/example/**'
- '!**/test/**'
- '!resolving/local-resolver/example-package/**'
allowBuilds:
core-js: false
esbuild: true
fuse-native: true
ghooks: true
msgpackr-extract: true
unrs-resolver: true
auditConfig:
ignoreGhsas:
- GHSA-2g4f-4pwh-qvx6
- GHSA-76c9-3jph-rj3q
- GHSA-ffrw-9mx8-89p8
- GHSA-mh29-5h37-fv8m
catalog:
'@babel/core': ^7.28.3
'@babel/plugin-transform-explicit-resource-management': ^7.27.1
'@changesets/cli': ^2.29.8
'@commitlint/cli': ^20.5.0
'@commitlint/config-conventional': ^20.5.0
'@commitlint/prompt-cli': ^20.5.0
'@cyclonedx/cyclonedx-library': 10.0.0
'@eslint/js': ^10.0.1
'@jest/globals': 30.3.0
'@npm/types': ^2.1.0
'@pnpm/byline': ^1.0.0
'@pnpm/colorize-semver-diff': ^1.0.1
'@pnpm/config.env-replace': ^3.0.2
'@pnpm/config.nerf-dart': ^1.0.0
'@pnpm/exec': ^2.0.0
'@pnpm/log.group': 3.0.2
'@pnpm/logger': '>=1001.0.0 <1002.0.0'
'@pnpm/meta-updater': 2.0.6
'@pnpm/nopt': ^0.3.1
'@pnpm/npm-lifecycle': 1100.0.0-1
'@pnpm/npm-package-arg': ^2.0.0
'@pnpm/os.env.path-extender': ^3.0.0
'@pnpm/patch-package': 0.0.1
'@pnpm/registry-mock': 6.0.0
'@pnpm/semver-diff': ^1.1.0
'@pnpm/tabtab': ^0.5.4
'@pnpm/tgz-fixtures': 0.0.0
'@pnpm/util.lex-comparator': ^3.0.2
'@reflink/reflink': 0.1.19
'@rushstack/worker-pool': 0.7.7
'@stylistic/eslint-plugin': ^5.10.0
'@types/adm-zip': ^0.5.7
'@types/archy': 0.0.36
'@types/cross-spawn': ^6.0.6
'@types/fs-extra': ^11.0.4
'@types/graceful-fs': ^4.1.9
'@types/hosted-git-info': ^3.0.5
'@types/ini': 4.1.1
'@types/is-gzip': 2.0.2
'@types/is-windows': ^1.0.2
'@types/isexe': 2.0.4
'@types/jest': ^30.0.0
'@types/js-yaml': ^4.0.9
'@types/libnpmpublish': ^9.0.1
'@types/lodash.kebabcase': 4.1.9
'@types/lodash.throttle': 4.1.9
'@types/micromatch': ^4.0.9
'@types/node': ^22.19.15
'@types/normalize-package-data': ^2.4.4
'@types/normalize-path': ^3.0.2
'@types/object-hash': 3.0.6
'@types/parse-json': ^4.0.2
'@types/picomatch': ^4.0.2
'@types/pnpm__byline': npm:@types/byline@^4.2.36
'@types/proxyquire': ^1.3.31
'@types/qrcode-terminal': ^0.12.2
'@types/ramda': 0.31.1
'@types/retry': ^0.12.5
'@types/semver': 7.7.1
'@types/ssri': ^7.1.5
'@types/tar': ^7.0.87
'@types/tar-stream': ^3.1.4
'@types/touch': ^3.1.5
'@types/validate-npm-package-name': ^4.0.2
'@types/which': ^3.0.4
'@types/write-file-atomic': ^4.0.3
'@types/yarnpkg__lockfile': ^1.1.9
'@types/zkochan__table': npm:@types/table@6.0.0
'@typescript-eslint/utils': ^8.57.1
# Newer tsgo dev builds have a regression where @types/node can't be resolved,
# causing all node built-in types (path, url, stream, process, etc.) to fail.
'@typescript/native-preview': 7.0.0-dev.20260216.1
'@yarnpkg/core': 4.5.0
'@yarnpkg/extensions': 2.0.6
'@yarnpkg/lockfile': ^1.1.0
'@yarnpkg/nm': 4.0.7
'@yarnpkg/parsers': 3.0.3
'@yarnpkg/pnp': ^4.0.8
'@zkochan/cmd-shim': ^9.0.0
'@zkochan/retry': ^0.2.0
'@zkochan/rimraf': ^4.0.0
'@zkochan/table': ^2.0.1
adm-zip: ^0.5.16
ansi-diff: ^1.2.0
archy: ^1.0.0
better-path-resolve: 2.0.0
bin-links: ^6.0.0
bole: ^5.0.17
boxen: npm:@zkochan/boxen@5.1.2
c8: ^11.0.0
camelcase: ^9.0.0
camelcase-keys: ^10.0.1
can-link: ^3.0.0
can-write-to-dir: ^2.0.0
chalk: ^5.6.0
ci-info: ^4.3.0
cli-truncate: ^5.2.0
cmd-extension: ^2.0.0
comver-to-semver: ^2.0.0
concurrently: 9.2.1
cross-env: ^10.1.0
cross-spawn: ^7.0.6
cspell: 9.7.0
deep-require-cwd: 1.0.0
delay: ^7.0.0
detect-indent: 7.0.2
detect-libc: ^2.0.3
didyoumean2: ^7.0.4
dint: ^5.1.0
dir-is-case-sensitive: ^3.0.0
encode-registry: ^3.0.1
enquirer: ^2.4.1
esbuild: ^0.27.4
escape-string-regexp: ^5.0.0
eslint: ^10.0.3
eslint-plugin-import-x: ^4.16.2
eslint-plugin-jest: ^29.12.1
eslint-plugin-n: ^17.23.2
eslint-plugin-promise: ^7.2.1
eslint-plugin-regexp: ^3.1.0
eslint-plugin-simple-import-sort: ^12.1.1
execa: npm:safe-execa@0.3.0
exists-link: 2.0.0
fast-deep-equal: ^3.1.3
fast-glob: ^3.3.3
find-up: ^8.0.0
fs-extra: ^11.3.1
fuse-native: ^2.2.6
get-npm-tarball-url: ^2.1.0
get-port: ^7.1.0
ghooks: 2.0.4
graceful-fs: ^4.2.11
graceful-git: ^5.0.0
graph-cycles: 3.0.0
hosted-git-info: npm:@pnpm/hosted-git-info@1.0.0
https-proxy-server-express: 0.1.2
husky: ^9.1.7
hyperdrive-schemas: ^2.0.0
ini: 6.0.0
is-gzip: 2.0.0
is-inner-link: ^5.0.0
is-subdir: ^2.0.0
is-windows: ^1.0.2
isexe: 4.0.0
jest: ^30.3.0
jest-diff: ^30.3.0
js-yaml: npm:@zkochan/js-yaml@0.0.11
json5: ^2.2.3
keyv: 5.6.0
lcov-result-merger: ^5.0.1
libnpmpublish: ^11.1.3
load-json-file: ^7.0.1
lodash.kebabcase: ^4.1.1
lodash.throttle: 4.1.1
loud-rejection: ^2.2.0
lru-cache: ^11.2.7
make-empty-dir: ^4.0.0
mdast-util-to-string: ^4.0.0
memoize: ^10.2.0
micromatch: ^4.0.8
# msgpackr 1.11.9 has broken type definitions (uses Iterable/Iterator without
# required type arguments), incompatible with TypeScript 5.9.
msgpackr: 1.11.8
nm-prune: ^5.0.0
nock: 13.3.4
normalize-newline: 5.0.0
normalize-package-data: ^8.0.0
normalize-path: ^3.0.0
normalize-registry-url: 2.0.1
npm-packlist: 10.0.4
object-hash: 3.0.0
open: ^7.4.2
p-defer: ^4.0.1
p-every: ^2.0.0
p-filter: ^4.1.0
p-limit: ^7.1.0
p-map-values: ^0.1.0
p-memoize: 8.0.0
p-queue: ^9.1.0
parse-json: ^8.3.0
parse-npm-tarball-url: ^4.0.0
path-absolute: ^2.0.0
path-exists: ^5.0.0
path-name: ^1.0.0
path-temp: ^3.0.0
pidtree: ^0.6.0
preferred-pm: ^5.0.0
pretty-bytes: ^7.1.0
pretty-ms: ^9.2.0
promise-share: ^2.0.0
proxyquire: ^2.1.3
ps-list: ^9.0.0
qrcode-terminal: ^0.12.0
ramda: npm:@pnpm/ramda@0.28.1
read-ini-file: 5.0.0
read-yaml-file: ^3.0.0
realpath-missing: ^2.0.0
remark-parse: ^11.0.0
remark-stringify: ^11.0.0
rename-overwrite: ^7.0.1
render-help: ^2.0.0
resolve-link-target: ^3.0.0
rimraf: ^6.1.2
root-link-target: ^4.0.0
run-groups: ^5.0.0
rxjs: ^7.8.2
safe-buffer: 5.2.1
safe-execa: ^0.3.0
safe-promise-defer: ^2.0.0
sanitize-filename: ^1.6.3
semver: ^7.7.2
semver-range-intersect: ^0.3.1
semver-utils: ^1.1.4
shlex: ^3.0.0
shx: ^0.4.0
socks: ^2.8.1
sort-keys: ^6.0.0
split-cmd: ^1.1.0
split2: ^4.2.0
ssri: 13.0.1
stacktracey: ^2.2.0
string-length: ^7.0.1
strip-bom: ^5.0.0
strip-comments-strings: 1.2.0
symlink-dir: ^10.0.1
tar: ^7.5.10
tar-stream: ^3.1.7
# tempy >=3.1.0 uses temp-dir 3.x which has an async fs.realpath() in its
# module init. When esbuild bundles this into its __esm lazy-init pattern,
# the async operation deadlocks and the pnpm binary hangs on startup.
tempy: 3.0.0
terminal-link: ^5.0.0
tinyglobby: ^0.2.14
touch: 3.1.1
tree-kill: ^1.2.2
ts-jest-resolver: 2.0.1
typescript: 5.9.3
typescript-eslint: ^8.57.1
undici: ^7.2.0
unified: ^11.0.5
validate-npm-package-name: 7.0.2
verdaccio: 6.3.2
version-selector-type: ^3.0.0
vfile: ^6.0.0
which: npm:@pnpm/which@^3.0.1
write-file-atomic: ^7.0.0
write-ini-file: 5.0.0
write-json-file: ^7.0.0
write-json5-file: ^4.0.0
write-package: 7.2.0
write-yaml-file: ^6.0.0
yaml: ^2.8.3
yaml-tag: 1.1.0
catalogMode: strict
cleanupUnusedCatalogs: true
enableGlobalVirtualStore: true
enablePrePostScripts: false
engineStrict: true
gitChecks: false
hoistPattern:
- jest-runner
minimumReleaseAge: 1440 # At least a day
minimumReleaseAgeExclude:
- '@pnpm/*'
- '@rushstack/worker-pool@0.7.7'
- '@zkochan/*'
- '@zkochan/cmd-shim@9.0.0'
- better-path-resolve
- body-parser@2.2.1
- can-link
- can-write-to-dir
- cmd-extension
- comver-to-semver
- dir-is-case-sensitive
- express@4.22.1
- glob@11.1.0
- graceful-git
- handlebars@4.7.9
- is-inner-link
- is-subdir
- jws@3.2.3
- lodash@4.17.23
- make-empty-dir
- normalize-registry-url
- p-map-values
- parse-npm-tarball-url@4.0.0
- path-absolute
- path-temp
- pnpm
- preferred-pm
- promise-share
- qs@6.14.2
- read-ini-file
- read-json5-file
- read-yaml-file
- realpath-missing
- rename-overwrite
- render-help
- resolve-link-target
- root-link-target
- run-groups
- safe-execa
- safe-promise-defer
- symlink-dir
- tar@7.5.10
- which-pm
- which-pm-runs
- write-ini-file
- write-json5-file
- write-yaml-file
nodeVersion: 22.13.0
optimisticRepeatInstall: true
overrides:
'@cypress/request@3.0.9>qs': ^6.14.1
'@yarnpkg/fslib@2': '3'
ajv@>=7.0.0-alpha.0 <8.18.0: '>=8.18.0'
body-parser@<2.2.1: '^2.2.1'
brace-expansion@<5.0.5: '>=5.0.5'
clipanion: 3.2.0-rc.6
cookie@<0.7.0: '>=0.7.0'
cross-spawn@<7.0.5: '>=7.0.5'
debug@<3.1.0: '>=3.1.0'
diff@<8.0.3: '^8.0.3'
express@<4.22.1: ^4.22.1
follow-redirects@<=1.15.5: '>=1.15.6'
glob-parent@<5.1.2: '>=5.1.2'
glob@>=10.3.7 <=11.0.3: '^11.1.0'
handlebars@>=4.0.0 <4.7.9: '>=4.7.9'
hosted-git-info@1: 'catalog:'
http-proxy-middleware@<2.0.7: ^2.0.7
istanbul-reports: npm:@zkochan/istanbul-reports
js-yaml@<3.14.2: '^3.14.2'
js-yaml@^4.0.0: 'catalog:'
json5@<2.2.2: 'catalog:'
jsonwebtoken@<=8.5.1: '>=9.0.0'
jws@<3.2.3: '^3.2.3'
lodash@<=4.17.23: '>=4.18.0'
lodash@>=4.0.0 <=4.17.22: '^4.17.23'
lodash@>=4.0.0 <=4.17.23: '>=4.18.0'
minimatch@>=7.0.0 <7.4.7: '^7.4.7'
minimatch@>=9.0.0 <10.0.0: '>=10.2.4'
nopt@5: npm:@pnpm/nopt@^0.2.1
on-headers@<1.1.0: '>=1.1.0'
path-to-regexp@<0.1.13: ^0.1.13
path-to-regexp@>=4.0.0 <6.3.0: '>=6.3.0'
path-to-regexp@>=7.0.0 <8.0.0: '>=8.0.0'
postman-request>qs: ^6.14.1
request: npm:postman-request@2.88.1-postman.40
semver@<7.5.2: 'catalog:'
send@<0.19.0: ^0.19.0
serve-static@<1.16.0: ^1.16.0
socks@2: ^2.8.1
tar@<=7.5.9: '>=7.5.10'
tmp@<=0.2.3: '>=0.2.4'
tough-cookie@<4.1.3: '>=4.1.3'
validator@<13.15.22: '>=13.15.22'
yaml@<2.2.2: '>=2.2.2'
packageExtensions:
'@babel/parser':
peerDependencies:
'@babel/types': '*'
'@verdaccio/auth':
peerDependencies:
express: '*'
jest-circus:
dependencies:
slash: '3'
remark-parse:
dependencies:
vfile: ^6.0.0
peerDependencies:
unified: '*'
remark-stringify:
dependencies:
vfile: ^6.0.0
peerDependencies:
unified: '*'
patchedDependencies:
graceful-fs@4.2.11: __patches__/graceful-fs@4.2.11.patch
patchesDir: __patches__
publishBranch: main
resolutionMode: lowest-direct
savePrefix: ''
sharedWorkspaceLockfile: true
trustPolicy: no-downgrade
trustPolicyExclude:
- '@yarnpkg/libzip@3.2.2'
- rxjs@7.8.2
- undici-types@6.21.0
- '@pnpm/*'
trustPolicyIgnoreAfter: 10080 # 1 week
verifyDepsBeforeRun: install
Reference in New Issue View Git Blame Copy Permalink