From b7f0f215820a7ed803ea6e6cebf2ae92a63b1ed4 Mon Sep 17 00:00:00 2001 From: Zoltan Kochan Date: Fri, 6 Mar 2026 12:59:04 +0100 Subject: [PATCH] feat: use SQLite for storing package index in the content-addressable store (#10827) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Replace individual `.mpk` (MessagePack) files under `$STORE/index/` with a single SQLite database at `$STORE/index.db` using Node.js 22's built-in `node:sqlite` module. This reduces filesystem syscall overhead and improves space efficiency for small metadata entries. Closes #10826 ## Design ### New package: `@pnpm/store.index` A new `StoreIndex` class wraps a SQLite database with a simple key-value API (`get`, `set`, `delete`, `has`, `entries`). Data is serialized with msgpackr and stored as BLOBs. The table uses `WITHOUT ROWID` for compact storage. Key design decisions: - **WAL mode** enables concurrent reads from workers while the main process writes. - **`busy_timeout=5000`** plus a retry loop with `Atomics.wait`-based `sleepSync` handles `SQLITE_BUSY` errors from concurrent access. - **Performance PRAGMAs**: `synchronous=NORMAL`, `mmap_size=512MB`, `cache_size=32MB`, `temp_store=MEMORY`, `wal_autocheckpoint=10000`. - **Write batching**: `queueWrites()` batches pre-packed entries from tarball extraction and flushes them in a single transaction on `process.nextTick`. `setRawMany()` writes immediate batches (e.g. from `addFilesFromDir`). - **Lifecycle**: `close()` auto-flushes pending writes, runs `PRAGMA optimize`, and closes the DB. A `process.on('exit')` handler ensures cleanup even on unexpected exits. - **`VACUUM` after `deleteMany`** (used by `pnpm store prune`) to reclaim disk space. ### Key format Keys are `integrity\tpkgId` (tab-separated). Git-hosted packages use `pkgId\tbuilt` or `pkgId\tnot-built`. ### Shared StoreIndex instance A single `StoreIndex` instance is threaded through the entire install lifecycle — from `createNewStoreController` through the fetcher chain, package requester, license scanner, SBOM collector, and dependencies hierarchy. This replaces the previous pattern of each component creating its own file-based index access. ### Worker architecture Index writes are performed in the main process, not in worker threads. Workers send pre-packed `{ key, buffer }` pairs back to the main process via `postMessage`, where they are batched and flushed to SQLite. This avoids SQLite write contention between threads. ### SQLite ExperimentalWarning suppression `node:sqlite` emits an `ExperimentalWarning` on first load. This is suppressed via a `process.emitWarning` override injected through esbuild's `banner` option, which runs on line 1 of both `dist/pnpm.mjs` and `dist/worker.js` — before any module that loads `node:sqlite`. ### No migration from `.mpk` files Old `.mpk` index files are not migrated. Packages missing from the new SQLite index are re-fetched on demand (the same behavior as a fresh store). ## Changed packages 121 files changed across these areas: - **`store/index/`** — New `@pnpm/store.index` package - **`worker/`** — Write batching moved from worker module into `StoreIndex` class; workers send pre-packed buffers to main process - **`store/package-store/`** — StoreIndex creation and lifecycle management - **`store/cafs/`** — Removed `getFilePathInCafs` index-file utilities (no longer needed) - **`store/pkg-finder/`** — Reads from StoreIndex instead of `.mpk` files - **`store/plugin-commands-store/`** — `store status` uses StoreIndex - **`store/plugin-commands-store-inspecting/`** — `cat-index` and `find-hash` use StoreIndex - **`fetching/tarball-fetcher/`** — Threads StoreIndex through fetchers; git-hosted fetcher flushes before reading - **`fetching/git-fetcher/`, `binary-fetcher/`, `pick-fetcher/`** — Accept StoreIndex parameter - **`pkg-manager/`** — `client`, `core`, `headless`, `package-requester` thread StoreIndex - **`reviewing/`** — `license-scanner`, `sbom`, `dependencies-hierarchy` accept StoreIndex - **`cache/api/`** — Cache view uses StoreIndex - **`pnpm/bundle.ts`** — esbuild banner for ExperimentalWarning suppression ## Test plan - [x] `pnpm --filter @pnpm/store.index test` — Unit tests for StoreIndex CRUD and batching - [x] `pnpm --filter @pnpm/package-store test` — Store controller lifecycle - [x] `pnpm --filter @pnpm/package-requester test` — Package requester reads from SQLite index - [x] `pnpm --filter @pnpm/tarball-fetcher test` — Tarball and git-hosted fetcher writes - [x] `pnpm --filter @pnpm/headless test` — Headless install - [x] `pnpm --filter @pnpm/core test` — Core install, side effects, patching - [x] `pnpm --filter @pnpm/plugin-commands-rebuild test` — Rebuild reads from index - [x] `pnpm --filter @pnpm/license-scanner test` — License scanning - [x] e2e tests pass 🤖 Generated with [Claude Code](https://claude.com/claude-code) --- .changeset/sqlite-store-index.md | 14 + __utils__/assert-store/package.json | 3 +- __utils__/assert-store/src/index.ts | 18 +- __utils__/assert-store/tsconfig.json | 3 + .../jest-config/with-registry/jest-preset.js | 4 + cache/api/package.json | 1 + cache/api/src/cacheView.ts | 62 +- cache/api/tsconfig.json | 3 + cspell.json | 5 + env/node.fetcher/package.json | 1 + env/node.fetcher/src/index.ts | 3 + env/node.fetcher/test/node.test.ts | 21 +- env/node.fetcher/tsconfig.json | 3 + exec/plugin-commands-rebuild/package.json | 2 +- .../src/implementation/index.ts | 16 +- exec/plugin-commands-rebuild/test/index.ts | 25 +- exec/plugin-commands-rebuild/tsconfig.json | 6 +- fetching/binary-fetcher/package.json | 1 + fetching/binary-fetcher/src/index.ts | 3 + fetching/binary-fetcher/tsconfig.json | 3 + fetching/git-fetcher/package.json | 1 + fetching/git-fetcher/src/index.ts | 3 + fetching/git-fetcher/test/index.ts | 36 +- fetching/git-fetcher/tsconfig.json | 3 + fetching/pick-fetcher/package.json | 1 + fetching/pick-fetcher/test/customFetch.ts | 13 +- fetching/pick-fetcher/tsconfig.json | 3 + fetching/tarball-fetcher/package.json | 5 +- .../src/gitHostedTarballFetcher.ts | 30 +- fetching/tarball-fetcher/src/index.ts | 7 +- .../src/localTarballFetcher.ts | 4 +- .../src/remoteTarballFetcher.ts | 3 + fetching/tarball-fetcher/test/fetch.ts | 14 + fetching/tarball-fetcher/tsconfig.json | 3 + modules-mounter/daemon/package.json | 2 +- .../daemon/src/createFuseHandlers.ts | 13 +- modules-mounter/daemon/tsconfig.json | 6 +- pkg-manager/client/package.json | 1 + pkg-manager/client/src/index.ts | 7 +- pkg-manager/client/test/index.ts | 9 + pkg-manager/client/tsconfig.json | 3 + pkg-manager/core/package.json | 1 + pkg-manager/core/test/install/patch.ts | 32 +- pkg-manager/core/test/install/sideEffects.ts | 49 +- pkg-manager/core/tsconfig.json | 3 + pkg-manager/headless/package.json | 2 +- pkg-manager/headless/src/index.ts | 2 - pkg-manager/headless/test/index.ts | 17 +- pkg-manager/headless/tsconfig.json | 6 +- pkg-manager/package-requester/package.json | 1 + .../package-requester/src/packageRequester.ts | 13 +- pkg-manager/package-requester/test/index.ts | 63 +- pkg-manager/package-requester/tsconfig.json | 3 + .../plugin-commands-installation/package.json | 2 + .../test/fetch.ts | 6 + .../tsconfig.json | 6 + pnpm-lock.yaml | 617 ++++++++++++------ pnpm/bundle.ts | 7 +- pnpm/package.json | 1 + pnpm/test/install/misc.ts | 30 +- pnpm/tsconfig.json | 3 + resolving/npm-resolver/package.json | 1 + resolving/npm-resolver/src/index.ts | 4 +- resolving/npm-resolver/tsconfig.json | 3 + reviewing/dependencies-hierarchy/package.json | 2 +- .../src/buildDependenciesTree.ts | 7 +- .../src/buildDependentsTree.ts | 5 + .../dependencies-hierarchy/src/getPkgInfo.ts | 6 +- .../dependencies-hierarchy/src/getTree.ts | 2 + .../src/readManifestFromCafs.ts | 11 +- .../dependencies-hierarchy/tsconfig.json | 6 +- reviewing/license-scanner/package.json | 1 + reviewing/license-scanner/src/getPkgInfo.ts | 3 + .../src/lockfileToLicenseNodeTree.ts | 12 +- .../license-scanner/test/getPkgInfo.spec.ts | 22 +- .../license-scanner/test/licenses.spec.ts | 14 +- reviewing/license-scanner/tsconfig.json | 3 + .../outdated/src/createManifestGetter.ts | 2 +- reviewing/sbom/package.json | 1 + reviewing/sbom/src/collectComponents.ts | 8 +- reviewing/sbom/src/getPkgMetadata.ts | 2 + reviewing/sbom/tsconfig.json | 3 + store/cafs-types/src/index.ts | 1 - store/cafs/package.json | 1 - store/cafs/src/getFilePathInCafs.ts | 17 - store/cafs/src/index.ts | 4 - store/cafs/tsconfig.json | 3 - store/index/README.md | 29 + store/index/package.json | 48 ++ store/index/src/index.ts | 267 ++++++++ store/index/test/index.ts | 46 ++ store/index/test/tsconfig.json | 18 + store/index/tsconfig.json | 12 + store/index/tsconfig.lint.json | 8 + store/package-store/package.json | 2 +- .../src/storeController/index.ts | 9 +- .../src/storeController/prune.ts | 32 +- store/package-store/test/index.ts | 7 + store/package-store/tsconfig.json | 6 +- store/pkg-finder/package.json | 4 +- store/pkg-finder/src/index.ts | 28 +- store/pkg-finder/tsconfig.json | 6 +- .../package.json | 2 +- .../src/catIndex.ts | 23 +- .../src/findHash.ts | 71 +- .../test/findHash.ts | 11 +- .../tsconfig.json | 6 +- store/plugin-commands-store/package.json | 2 +- .../src/storeStatus/index.ts | 48 +- .../plugin-commands-store/test/storeStatus.ts | 13 +- store/plugin-commands-store/tsconfig.json | 6 +- store/store-connection-manager/package.json | 1 + .../src/createNewStoreController.ts | 6 +- store/store-connection-manager/tsconfig.json | 3 + testing/temp-store/package.json | 3 +- testing/temp-store/src/index.ts | 4 + testing/temp-store/tsconfig.json | 3 + worker/package.json | 2 +- worker/src/index.ts | 24 +- worker/src/start.ts | 136 ++-- worker/tsconfig.json | 6 +- 121 files changed, 1642 insertions(+), 622 deletions(-) create mode 100644 .changeset/sqlite-store-index.md create mode 100644 store/index/README.md create mode 100644 store/index/package.json create mode 100644 store/index/src/index.ts create mode 100644 store/index/test/index.ts create mode 100644 store/index/test/tsconfig.json create mode 100644 store/index/tsconfig.json create mode 100644 store/index/tsconfig.lint.json diff --git a/.changeset/sqlite-store-index.md b/.changeset/sqlite-store-index.md new file mode 100644 index 0000000000..537d00fa11 --- /dev/null +++ b/.changeset/sqlite-store-index.md @@ -0,0 +1,14 @@ +--- +"@pnpm/store.index": minor +"@pnpm/store.cafs": minor +"@pnpm/worker": minor +"@pnpm/plugin-commands-store-inspecting": minor +"@pnpm/plugin-commands-store": minor +"@pnpm/package-store": minor +"@pnpm/store.pkg-finder": minor +"@pnpm/reviewing.dependencies-hierarchy": minor +"@pnpm/plugin-commands-rebuild": minor +"pnpm": minor +--- + +Use SQLite for storing package index in the content-addressable store. Instead of individual `.mpk` files under `$STORE/index/`, package metadata is now stored in a single SQLite database at `$STORE/index.db`. This reduces filesystem syscall overhead, improves space efficiency for small metadata entries, and enables concurrent access via SQLite's WAL mode. Packages missing from the new index are re-fetched on demand [#10826](https://github.com/pnpm/pnpm/issues/10826). diff --git a/__utils__/assert-store/package.json b/__utils__/assert-store/package.json index f4b693baf9..dac2be8ee5 100644 --- a/__utils__/assert-store/package.json +++ b/__utils__/assert-store/package.json @@ -33,7 +33,8 @@ }, "dependencies": { "@pnpm/registry-mock": "catalog:", - "@pnpm/store.cafs": "workspace:*" + "@pnpm/store.cafs": "workspace:*", + "@pnpm/store.index": "workspace:*" }, "devDependencies": { "@pnpm/assert-store": "workspace:*", diff --git a/__utils__/assert-store/src/index.ts b/__utils__/assert-store/src/index.ts index b8b1b49b42..ed60deeed5 100644 --- a/__utils__/assert-store/src/index.ts +++ b/__utils__/assert-store/src/index.ts @@ -1,6 +1,6 @@ import fs from 'fs' import path from 'path' -import { getIndexFilePathInCafs } from '@pnpm/store.cafs' +import { StoreIndex, storeIndexKey } from '@pnpm/store.index' import { getIntegrity, REGISTRY_MOCK_PORT } from '@pnpm/registry-mock' export interface StoreAssertions { @@ -24,15 +24,25 @@ export function assertStore ( const store = { getPkgIndexFilePath (pkgName: string, version: string): string { const integrity = getIntegrity(pkgName, version) - return getIndexFilePathInCafs(storePath, integrity, `${pkgName}@${version}`) + return storeIndexKey(integrity, `${pkgName}@${version}`) }, cafsHas (pkgName: string, version: string): void { const pathToCheck = store.getPkgIndexFilePath(pkgName, version) - ok(fs.existsSync(pathToCheck)) + const storeIndex = new StoreIndex(storePath) + try { + ok(storeIndex.get(pathToCheck) != null) + } finally { + storeIndex.close() + } }, cafsHasNot (pkgName: string, version: string): void { const pathToCheck = store.getPkgIndexFilePath(pkgName, version) - notOk(fs.existsSync(pathToCheck)) + const storeIndex = new StoreIndex(storePath) + try { + notOk(storeIndex.get(pathToCheck) != null) + } finally { + storeIndex.close() + } }, storeHas (pkgName: string, version?: string): void { const pathToCheck = store.resolve(pkgName, version) diff --git a/__utils__/assert-store/tsconfig.json b/__utils__/assert-store/tsconfig.json index aa2e467e0d..76c3e8ce19 100644 --- a/__utils__/assert-store/tsconfig.json +++ b/__utils__/assert-store/tsconfig.json @@ -14,6 +14,9 @@ }, { "path": "../../store/cafs" + }, + { + "path": "../../store/index" } ] } diff --git a/__utils__/jest-config/with-registry/jest-preset.js b/__utils__/jest-config/with-registry/jest-preset.js index 52ec2fbd1a..e08e981fc5 100644 --- a/__utils__/jest-config/with-registry/jest-preset.js +++ b/__utils__/jest-config/with-registry/jest-preset.js @@ -7,6 +7,10 @@ export default { // Unfortunately, this means that if two such tests will run at the same time, // they may break each other. maxWorkers: 1, + // Force Jest to exit after globalTeardown completes. The Verdaccio server + // and lifecycle child-processes spawned during tests may leave ref'd handles + // that prevent the process from exiting on its own. + forceExit: true, globalSetup: path.join(import.meta.dirname, 'globalSetup.js'), globalTeardown: path.join(import.meta.dirname, 'globalTeardown.js'), } diff --git a/cache/api/package.json b/cache/api/package.json index 879d235e05..e0cfaf50ec 100644 --- a/cache/api/package.json +++ b/cache/api/package.json @@ -36,6 +36,7 @@ "@pnpm/fs.msgpack-file": "workspace:*", "@pnpm/npm-resolver": "workspace:*", "@pnpm/store.cafs": "workspace:*", + "@pnpm/store.index": "workspace:*", "encode-registry": "catalog:", "tinyglobby": "catalog:" }, diff --git a/cache/api/src/cacheView.ts b/cache/api/src/cacheView.ts index c35868435e..cff08b29ed 100644 --- a/cache/api/src/cacheView.ts +++ b/cache/api/src/cacheView.ts @@ -1,8 +1,7 @@ -import fs from 'fs' import path from 'path' import { glob } from 'tinyglobby' import { readMsgpackFileSync } from '@pnpm/fs.msgpack-file' -import { getIndexFilePathInCafs } from '@pnpm/store.cafs' +import { StoreIndex, storeIndexKey } from '@pnpm/store.index' import { type PackageMeta } from '@pnpm/npm-resolver' import getRegistryName from 'encode-registry' @@ -20,35 +19,40 @@ export async function cacheView (opts: { cacheDir: string, storeDir: string, reg expandDirectories: false, })).sort() const metaFilesByPath: Record = {} - for (const filePath of metaFilePaths) { - let metaObject: PackageMeta | null - try { - metaObject = readMsgpackFileSync(path.join(opts.cacheDir, filePath)) - } catch { - continue - } - if (!metaObject) continue - const cachedVersions: string[] = [] - const nonCachedVersions: string[] = [] - for (const [version, manifest] of Object.entries(metaObject.versions)) { - if (!manifest.dist.integrity) continue - const indexFilePath = getIndexFilePathInCafs(opts.storeDir, manifest.dist.integrity, `${manifest.name}@${manifest.version}`) - if (fs.existsSync(indexFilePath)) { - cachedVersions.push(version) - } else { - nonCachedVersions.push(version) + const storeIndex = new StoreIndex(opts.storeDir) + try { + for (const filePath of metaFilePaths) { + let metaObject: PackageMeta | null + try { + metaObject = readMsgpackFileSync(path.join(opts.cacheDir, filePath)) + } catch { + continue + } + if (!metaObject) continue + const cachedVersions: string[] = [] + const nonCachedVersions: string[] = [] + for (const [version, manifest] of Object.entries(metaObject.versions)) { + if (!manifest.dist.integrity) continue + const key = storeIndexKey(manifest.dist.integrity, `${manifest.name}@${manifest.version}`) + if (storeIndex.has(key)) { + cachedVersions.push(version) + } else { + nonCachedVersions.push(version) + } + } + let registryName = filePath + while (path.dirname(registryName) !== '.') { + registryName = path.dirname(registryName) + } + metaFilesByPath[registryName.replaceAll('+', ':')] = { + cachedVersions, + nonCachedVersions, + cachedAt: metaObject.cachedAt ? new Date(metaObject.cachedAt).toString() : undefined, + distTags: metaObject['dist-tags'], } } - let registryName = filePath - while (path.dirname(registryName) !== '.') { - registryName = path.dirname(registryName) - } - metaFilesByPath[registryName.replaceAll('+', ':')] = { - cachedVersions, - nonCachedVersions, - cachedAt: metaObject.cachedAt ? new Date(metaObject.cachedAt).toString() : undefined, - distTags: metaObject['dist-tags'], - } + } finally { + storeIndex.close() } return JSON.stringify(metaFilesByPath, null, 2) } diff --git a/cache/api/tsconfig.json b/cache/api/tsconfig.json index d06e16d991..ba0eb7270f 100644 --- a/cache/api/tsconfig.json +++ b/cache/api/tsconfig.json @@ -26,6 +26,9 @@ }, { "path": "../../store/cafs" + }, + { + "path": "../../store/index" } ] } diff --git a/cspell.json b/cspell.json index 4eda7cfb6d..96caf93c2a 100644 --- a/cspell.json +++ b/cspell.json @@ -9,6 +9,7 @@ "archy", "argumentless", "armv", + "autocheckpoint", "autocompleting", "autofix", "autofixed", @@ -76,6 +77,7 @@ "enten", "eperm", "epipe", + "errcode", "etamponi", "exdev", "execa", @@ -155,6 +157,7 @@ "metafile", "millis", "mintimeout", + "mmap", "monorepolint", "moonrepo", "mountpoint", @@ -297,6 +300,8 @@ "supercede", "syml", "syncer", + "syscall", + "syscalls", "szia", "tabtab", "taffydb", diff --git a/env/node.fetcher/package.json b/env/node.fetcher/package.json index ce3d636941..ad0ba03328 100644 --- a/env/node.fetcher/package.json +++ b/env/node.fetcher/package.json @@ -38,6 +38,7 @@ "@pnpm/fetching-types": "workspace:*", "@pnpm/fetching.binary-fetcher": "workspace:*", "@pnpm/node.resolver": "workspace:*", + "@pnpm/store.index": "workspace:*", "@pnpm/tarball-fetcher": "workspace:*", "detect-libc": "catalog:" }, diff --git a/env/node.fetcher/src/index.ts b/env/node.fetcher/src/index.ts index d69e76ef8d..aa969c659d 100644 --- a/env/node.fetcher/src/index.ts +++ b/env/node.fetcher/src/index.ts @@ -6,6 +6,7 @@ import { } from '@pnpm/fetching-types' import { createCafsStore } from '@pnpm/create-cafs-store' import { type Cafs } from '@pnpm/cafs-types' +import { type StoreIndex } from '@pnpm/store.index' import { createTarballFetcher } from '@pnpm/tarball-fetcher' import { getNodeArtifactAddress, @@ -17,6 +18,7 @@ import { isNonGlibcLinux } from 'detect-libc' export interface FetchNodeOptionsToDir { storeDir: string + storeIndex: StoreIndex fetchTimeout?: number nodeMirrorBaseUrl?: string retry?: RetryTimeoutOptions @@ -167,6 +169,7 @@ async function downloadAndUnpackTarballToDir ( const fetchers = createTarballFetcher(fetch, getAuthHeader, { retry: opts.retry, timeout: opts.fetchTimeout, + storeIndex: opts.storeIndex, // These are not needed for fetching Node.js rawConfig: {}, unsafePerm: false, diff --git a/env/node.fetcher/test/node.test.ts b/env/node.fetcher/test/node.test.ts index 1b15daafb4..1598cd36d0 100644 --- a/env/node.fetcher/test/node.test.ts +++ b/env/node.fetcher/test/node.test.ts @@ -3,6 +3,7 @@ import { Response } from 'node-fetch' import path from 'path' import { Readable } from 'stream' import type { FetchNodeOptionsToDir as FetchNodeOptions } from '@pnpm/node.fetcher' +import { StoreIndex } from '@pnpm/store.index' import { tempDir } from '@pnpm/prepare' import { jest } from '@jest/globals' @@ -38,6 +39,11 @@ const fetchMock = jest.fn(async (url: string) => { return new Response(Readable.from(Buffer.alloc(0))) }) +const storeIndexes: StoreIndex[] = [] +afterAll(() => { + for (const si of storeIndexes) si.close() +}) + beforeEach(() => { jest.mocked(isNonGlibcLinux).mockReturnValue(Promise.resolve(false)) fetchMock.mockClear() @@ -47,9 +53,13 @@ test.skip('install Node using a custom node mirror', async () => { tempDir() const nodeMirrorBaseUrl = 'https://pnpm-node-mirror-test.localhost/download/release/' + const storeDir = path.resolve('store') + const storeIndex = new StoreIndex(storeDir) + storeIndexes.push(storeIndex) const opts: FetchNodeOptions = { nodeMirrorBaseUrl, - storeDir: path.resolve('store'), + storeDir, + storeIndex, } await fetchNode(fetchMock, '16.4.0', path.resolve('node'), opts) @@ -62,8 +72,12 @@ test.skip('install Node using a custom node mirror', async () => { test.skip('install Node using the default node mirror', async () => { tempDir() + const storeDir = path.resolve('store') + const storeIndex = new StoreIndex(storeDir) + storeIndexes.push(storeIndex) const opts: FetchNodeOptions = { - storeDir: path.resolve('store'), + storeDir, + storeIndex, } await fetchNode(fetchMock, '16.4.0', path.resolve('node'), opts) @@ -81,9 +95,12 @@ test('auto-detects musl on non-glibc Linux and uses unofficial-builds mirror', a // The function will throw because the downloaded tarball content won't match // the fake sha256 we put in the SHASUMS256.txt mock, but all fetch calls are // recorded before the integrity check, so we can assert the correct URLs. + const storeIndex = new StoreIndex(path.resolve('store')) + storeIndexes.push(storeIndex) await expect( fetchNode(fetchMock, '22.0.0', path.resolve('node'), { storeDir: path.resolve('store'), + storeIndex, platform: 'linux', arch: 'x64', retry: { retries: 0 }, diff --git a/env/node.fetcher/tsconfig.json b/env/node.fetcher/tsconfig.json index 5b4756dd1e..74bdbaeeab 100644 --- a/env/node.fetcher/tsconfig.json +++ b/env/node.fetcher/tsconfig.json @@ -30,6 +30,9 @@ { "path": "../../store/create-cafs-store" }, + { + "path": "../../store/index" + }, { "path": "../node.resolver" } diff --git a/exec/plugin-commands-rebuild/package.json b/exec/plugin-commands-rebuild/package.json index 11a033732d..f958fac61f 100644 --- a/exec/plugin-commands-rebuild/package.json +++ b/exec/plugin-commands-rebuild/package.json @@ -43,7 +43,6 @@ "@pnpm/deps.graph-sequencer": "workspace:*", "@pnpm/error": "workspace:*", "@pnpm/exec.pkg-requires-build": "workspace:*", - "@pnpm/fs.msgpack-file": "workspace:*", "@pnpm/get-context": "workspace:*", "@pnpm/lifecycle": "workspace:*", "@pnpm/link-bins": "workspace:*", @@ -58,6 +57,7 @@ "@pnpm/store-connection-manager": "workspace:*", "@pnpm/store-controller-types": "workspace:*", "@pnpm/store.cafs": "workspace:*", + "@pnpm/store.index": "workspace:*", "@pnpm/types": "workspace:*", "@pnpm/workspace.find-packages": "workspace:*", "load-json-file": "catalog:", diff --git a/exec/plugin-commands-rebuild/src/implementation/index.ts b/exec/plugin-commands-rebuild/src/implementation/index.ts index 422e82a8d7..44da30c133 100644 --- a/exec/plugin-commands-rebuild/src/implementation/index.ts +++ b/exec/plugin-commands-rebuild/src/implementation/index.ts @@ -1,7 +1,8 @@ import assert from 'assert' import path from 'path' import util from 'util' -import { getIndexFilePathInCafs, type PackageFilesIndex } from '@pnpm/store.cafs' +import { type PackageFilesIndex } from '@pnpm/store.cafs' +import { storeIndexKey } from '@pnpm/store.index' import { calcDepState, lockfileToDepGraph, type DepsStateCache } from '@pnpm/calc-dep-state' import { LAYOUT_VERSION, @@ -36,7 +37,7 @@ import { import { createAllowBuildFunction } from '@pnpm/builder.policy' import { pkgRequiresBuild } from '@pnpm/exec.pkg-requires-build' import * as dp from '@pnpm/dependency-path' -import { readMsgpackFile } from '@pnpm/fs.msgpack-file' +import { StoreIndex } from '@pnpm/store.index' import { safeReadPackageJsonFromDir } from '@pnpm/read-package-json' import { hardLinkDir } from '@pnpm/worker' import { runGroups } from 'run-groups' @@ -328,6 +329,7 @@ async function _rebuild ( return false } const builtDepPaths = new Set() + const storeIndex = opts.skipIfHasSideEffectsCache ? new StoreIndex(opts.storeDir) : undefined const groups = chunks.map((chunk) => chunk.filter((depPath) => ctx.pkgsToRebuild.has(depPath) && !ctx.skipped.has(depPath)).map((depPath) => async () => { @@ -356,11 +358,8 @@ async function _rebuild ( let sideEffectsCacheKey: string | undefined const pkgId = `${pkgInfo.name}@${pkgInfo.version}` if (opts.skipIfHasSideEffectsCache && resolution.integrity) { - const filesIndexFile = getIndexFilePathInCafs(opts.storeDir, resolution.integrity!.toString(), pkgId) - let pkgFilesIndex: PackageFilesIndex | undefined - try { - pkgFilesIndex = await readMsgpackFile(filesIndexFile) - } catch {} + const filesIndexFile = storeIndexKey(resolution.integrity!.toString(), pkgId) + const pkgFilesIndex = storeIndex!.get(filesIndexFile) as PackageFilesIndex | undefined if (pkgFilesIndex) { sideEffectsCacheKey = calcDepState(depGraph, depsStateCache, depPath, { includeDepGraphHash: true, @@ -393,7 +392,7 @@ async function _rebuild ( }) if (hasSideEffects && (opts.sideEffectsCacheWrite ?? true) && resolution.integrity) { builtDepPaths.add(depPath) - const filesIndexFile = getIndexFilePathInCafs(opts.storeDir, resolution.integrity!.toString(), pkgId) + const filesIndexFile = storeIndexKey(resolution.integrity!.toString(), pkgId) try { if (!sideEffectsCacheKey) { sideEffectsCacheKey = calcDepState(depGraph, depsStateCache, depPath, { @@ -439,6 +438,7 @@ async function _rebuild ( )) await runGroups(opts.childConcurrency || 5, groups) + storeIndex?.close() if (builtDepPaths.size > 0) { // It may be optimized because some bins were already linked before running lifecycle scripts diff --git a/exec/plugin-commands-rebuild/test/index.ts b/exec/plugin-commands-rebuild/test/index.ts index 7f08e1b4e0..7a48974bc4 100644 --- a/exec/plugin-commands-rebuild/test/index.ts +++ b/exec/plugin-commands-rebuild/test/index.ts @@ -1,8 +1,8 @@ /// import fs from 'fs' import path from 'path' -import { readMsgpackFileSync, writeMsgpackFileSync } from '@pnpm/fs.msgpack-file' -import { getIndexFilePathInCafs, type PackageFilesIndex } from '@pnpm/store.cafs' +import { type PackageFilesIndex } from '@pnpm/store.cafs' +import { StoreIndex, storeIndexKey } from '@pnpm/store.index' import { ENGINE_NAME, STORE_VERSION, WANTED_LOCKFILE } from '@pnpm/constants' import { hashObject } from '@pnpm/crypto.object-hasher' import { rebuild } from '@pnpm/plugin-commands-rebuild' @@ -17,6 +17,11 @@ const REGISTRY = `http://localhost:${REGISTRY_MOCK_PORT}/` const pnpmBin = path.join(import.meta.dirname, '../../../pnpm/bin/pnpm.mjs') const f = fixtures(import.meta.dirname) +const storeIndexes: StoreIndex[] = [] +afterAll(() => { + for (const si of storeIndexes) si.close() +}) + test('rebuilds dependencies', async () => { const project = prepare() const cacheDir = path.resolve('cache') @@ -76,8 +81,10 @@ test('rebuilds dependencies', async () => { ]) } - const cacheIntegrityPath = getIndexFilePathInCafs(path.join(storeDir, STORE_VERSION), getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0') - const cacheIntegrity = readMsgpackFileSync(cacheIntegrityPath)! + const cacheIntegrityPath = storeIndexKey(getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0') + const storeIndex1 = new StoreIndex(path.join(storeDir, STORE_VERSION)) + storeIndexes.push(storeIndex1) + const cacheIntegrity = storeIndex1.get(cacheIntegrityPath) as PackageFilesIndex expect(cacheIntegrity!.sideEffects).toBeTruthy() const sideEffectsKey = `${ENGINE_NAME};deps=${hashObject({ id: `@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0:${getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0')}`, @@ -109,8 +116,10 @@ test('skipIfHasSideEffectsCache', async () => { '--config.enableGlobalVirtualStore=false', ]) - const cacheIntegrityPath = getIndexFilePathInCafs(path.join(storeDir, STORE_VERSION), getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0') - let cacheIntegrity = readMsgpackFileSync(cacheIntegrityPath)! + const cacheIntegrityPath = storeIndexKey(getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0') + const storeIndex = new StoreIndex(path.join(storeDir, STORE_VERSION)) + storeIndexes.push(storeIndex) + let cacheIntegrity = storeIndex.get(cacheIntegrityPath) as PackageFilesIndex const sideEffectsKey = `${ENGINE_NAME};deps=${hashObject({ '@pnpm.e2e/hello-world-js-bin@1.0.0': {} })}` cacheIntegrity.sideEffects = new Map([ [sideEffectsKey, { @@ -123,7 +132,7 @@ test('skipIfHasSideEffectsCache', async () => { ]), }], ]) - writeMsgpackFileSync(cacheIntegrityPath, cacheIntegrity) + storeIndex.set(cacheIntegrityPath, cacheIntegrity) let modules = project.readModulesManifest() expect(modules!.pendingBuilds).toStrictEqual([ @@ -146,7 +155,7 @@ test('skipIfHasSideEffectsCache', async () => { expect(modules).toBeTruthy() expect(modules!.pendingBuilds).toHaveLength(0) - cacheIntegrity = readMsgpackFileSync(cacheIntegrityPath)! + cacheIntegrity = storeIndex.get(cacheIntegrityPath) as PackageFilesIndex expect(cacheIntegrity!.sideEffects).toBeTruthy() expect(cacheIntegrity!.sideEffects!.get(sideEffectsKey)?.added?.has('foo')).toBeTruthy() }) diff --git a/exec/plugin-commands-rebuild/tsconfig.json b/exec/plugin-commands-rebuild/tsconfig.json index c46ea37a1a..3fbc1bfc4d 100644 --- a/exec/plugin-commands-rebuild/tsconfig.json +++ b/exec/plugin-commands-rebuild/tsconfig.json @@ -42,9 +42,6 @@ { "path": "../../deps/graph-sequencer" }, - { - "path": "../../fs/msgpack-file" - }, { "path": "../../lockfile/types" }, @@ -90,6 +87,9 @@ { "path": "../../store/cafs" }, + { + "path": "../../store/index" + }, { "path": "../../store/store-connection-manager" }, diff --git a/fetching/binary-fetcher/package.json b/fetching/binary-fetcher/package.json index 8b492c3fe5..e6b17055e7 100644 --- a/fetching/binary-fetcher/package.json +++ b/fetching/binary-fetcher/package.json @@ -34,6 +34,7 @@ "@pnpm/error": "workspace:*", "@pnpm/fetcher-base": "workspace:*", "@pnpm/fetching-types": "workspace:*", + "@pnpm/store.index": "workspace:*", "adm-zip": "catalog:", "is-subdir": "catalog:", "rename-overwrite": "catalog:", diff --git a/fetching/binary-fetcher/src/index.ts b/fetching/binary-fetcher/src/index.ts index 8ba15e455b..e76d2386ce 100644 --- a/fetching/binary-fetcher/src/index.ts +++ b/fetching/binary-fetcher/src/index.ts @@ -3,6 +3,7 @@ import fsPromises from 'fs/promises' import { PnpmError } from '@pnpm/error' import { type FetchFromRegistry } from '@pnpm/fetching-types' import { type BinaryFetcher, type FetchFunction, type FetchResult } from '@pnpm/fetcher-base' +import { type StoreIndex } from '@pnpm/store.index' import { addFilesFromDir } from '@pnpm/worker' import AdmZip from 'adm-zip' import isSubdir from 'is-subdir' @@ -14,6 +15,7 @@ export function createBinaryFetcher (ctx: { fetch: FetchFromRegistry fetchFromRemoteTarball: FetchFunction rawConfig: Record + storeIndex: StoreIndex offline?: boolean }): { binary: BinaryFetcher } { const fetchBinary: BinaryFetcher = async (cafs, resolution, opts) => { @@ -48,6 +50,7 @@ export function createBinaryFetcher (ctx: { }, tempLocation) fetchResult = await addFilesFromDir({ storeDir: cafs.storeDir, + storeIndex: ctx.storeIndex, dir: tempLocation, filesIndexFile: opts.filesIndexFile, readManifest: false, diff --git a/fetching/binary-fetcher/tsconfig.json b/fetching/binary-fetcher/tsconfig.json index 525bbaac89..dce2268f3a 100644 --- a/fetching/binary-fetcher/tsconfig.json +++ b/fetching/binary-fetcher/tsconfig.json @@ -15,6 +15,9 @@ { "path": "../../packages/error" }, + { + "path": "../../store/index" + }, { "path": "../../worker" }, diff --git a/fetching/git-fetcher/package.json b/fetching/git-fetcher/package.json index fa3fdf2f57..0984ba5c3d 100644 --- a/fetching/git-fetcher/package.json +++ b/fetching/git-fetcher/package.json @@ -36,6 +36,7 @@ "@pnpm/fetcher-base": "workspace:*", "@pnpm/fs.packlist": "workspace:*", "@pnpm/prepare-package": "workspace:*", + "@pnpm/store.index": "workspace:*", "@zkochan/rimraf": "catalog:", "execa": "catalog:" }, diff --git a/fetching/git-fetcher/src/index.ts b/fetching/git-fetcher/src/index.ts index 7928362e30..a33fef40e5 100644 --- a/fetching/git-fetcher/src/index.ts +++ b/fetching/git-fetcher/src/index.ts @@ -5,6 +5,7 @@ import type { GitFetcher } from '@pnpm/fetcher-base' import { packlist } from '@pnpm/fs.packlist' import { globalWarn } from '@pnpm/logger' import { preparePackage } from '@pnpm/prepare-package' +import { type StoreIndex } from '@pnpm/store.index' import { addFilesFromDir } from '@pnpm/worker' import { PnpmError } from '@pnpm/error' import rimraf from '@zkochan/rimraf' @@ -14,6 +15,7 @@ import { URL } from 'url' export interface CreateGitFetcherOptions { gitShallowHosts?: string[] rawConfig: Record + storeIndex: StoreIndex unsafePerm?: boolean ignoreScripts?: boolean } @@ -61,6 +63,7 @@ export function createGitFetcher (createOpts: CreateGitFetcherOptions): { git: G // the linking of files to the store is in progress. return addFilesFromDir({ storeDir: cafs.storeDir, + storeIndex: createOpts.storeIndex, dir: pkgDir, files, filesIndexFile: opts.filesIndexFile, diff --git a/fetching/git-fetcher/test/index.ts b/fetching/git-fetcher/test/index.ts index 6cad676be5..f837d49408 100644 --- a/fetching/git-fetcher/test/index.ts +++ b/fetching/git-fetcher/test/index.ts @@ -1,6 +1,7 @@ /// import path from 'path' import { createCafsStore } from '@pnpm/create-cafs-store' +import { StoreIndex } from '@pnpm/store.index' import { jest } from '@jest/globals' import { temporaryDirectory } from 'tempy' import { lexCompare } from '@pnpm/util.lex-comparator' @@ -29,6 +30,17 @@ const { globalWarn } = await import('@pnpm/logger') const { default: execa } = await import('execa') const { createGitFetcher } = await import('@pnpm/git-fetcher') +const storeIndexes: StoreIndex[] = [] +afterAll(() => { + for (const si of storeIndexes) si.close() +}) + +function createStoreIndex (storeDir: string): StoreIndex { + const si = new StoreIndex(storeDir) + storeIndexes.push(si) + return si +} + beforeEach(() => { jest.mocked(execa).mockClear() jest.mocked(globalWarn).mockClear() @@ -36,7 +48,7 @@ beforeEach(() => { test('fetch', async () => { const storeDir = temporaryDirectory() - const fetch = createGitFetcher({ rawConfig: {} }).git + const fetch = createGitFetcher({ rawConfig: {}, storeIndex: createStoreIndex(storeDir) }).git const { filesMap, manifest } = await fetch( createCafsStore(storeDir), { @@ -55,7 +67,7 @@ test('fetch', async () => { test('fetch a package from Git sub folder', async () => { const storeDir = temporaryDirectory() - const fetch = createGitFetcher({ rawConfig: {} }).git + const fetch = createGitFetcher({ rawConfig: {}, storeIndex: createStoreIndex(storeDir) }).git const { filesMap } = await fetch( createCafsStore(storeDir), { @@ -73,7 +85,7 @@ test('fetch a package from Git sub folder', async () => { test('prevent directory traversal attack when using Git sub folder', async () => { const storeDir = temporaryDirectory() - const fetch = createGitFetcher({ rawConfig: {} }).git + const fetch = createGitFetcher({ rawConfig: {}, storeIndex: createStoreIndex(storeDir) }).git const repo = 'https://github.com/RexSkz/test-git-subfolder-fetch.git' const pkgDir = '../../etc' await expect( @@ -94,7 +106,7 @@ test('prevent directory traversal attack when using Git sub folder', async () => test('prevent directory traversal attack when using Git sub folder #2', async () => { const storeDir = temporaryDirectory() - const fetch = createGitFetcher({ rawConfig: {} }).git + const fetch = createGitFetcher({ rawConfig: {}, storeIndex: createStoreIndex(storeDir) }).git const repo = 'https://github.com/RexSkz/test-git-subfolder-fetch.git' const pkgDir = 'not/exists' await expect( @@ -117,6 +129,7 @@ test('fetch a package from Git that has a prepare script', async () => { const storeDir = temporaryDirectory() const fetch = createGitFetcher({ rawConfig: {}, + storeIndex: createStoreIndex(storeDir), }).git const { filesMap } = await fetch( createCafsStore(storeDir), @@ -136,7 +149,7 @@ test('fetch a package from Git that has a prepare script', async () => { // Test case for https://github.com/pnpm/pnpm/issues/1866 test('fetch a package without a package.json', async () => { const storeDir = temporaryDirectory() - const fetch = createGitFetcher({ rawConfig: {} }).git + const fetch = createGitFetcher({ rawConfig: {}, storeIndex: createStoreIndex(storeDir) }).git const { filesMap } = await fetch( createCafsStore(storeDir), { @@ -155,7 +168,7 @@ test('fetch a package without a package.json', async () => { // Covers the regression reported in https://github.com/pnpm/pnpm/issues/4064 test('fetch a big repository', async () => { const storeDir = temporaryDirectory() - const fetch = createGitFetcher({ rawConfig: {} }).git + const fetch = createGitFetcher({ rawConfig: {}, storeIndex: createStoreIndex(storeDir) }).git const { filesMap } = await fetch(createCafsStore(storeDir), { commit: 'a65fbf5a90f53c9d72fed4daaca59da50f074355', @@ -169,7 +182,7 @@ test('fetch a big repository', async () => { test('still able to shallow fetch for allowed hosts', async () => { const storeDir = temporaryDirectory() - const fetch = createGitFetcher({ gitShallowHosts: ['github.com'], rawConfig: {} }).git + const fetch = createGitFetcher({ gitShallowHosts: ['github.com'], rawConfig: {}, storeIndex: createStoreIndex(storeDir) }).git const resolution = { commit: 'c9b30e71d704cd30fa71f2edd1ecc7dcc4985493', repo: 'https://github.com/kevva/is-positive.git', @@ -200,6 +213,7 @@ test('fail when preparing a git-hosted package', async () => { const storeDir = temporaryDirectory() const fetch = createGitFetcher({ rawConfig: {}, + storeIndex: createStoreIndex(storeDir), }).git await expect( fetch(createCafsStore(storeDir), @@ -218,6 +232,7 @@ test('fail when preparing a git-hosted package with a partial commit', async () const storeDir = temporaryDirectory() const fetch = createGitFetcher({ rawConfig: {}, + storeIndex: createStoreIndex(storeDir), }).git await expect( fetch(createCafsStore(storeDir), @@ -233,7 +248,7 @@ test('fail when preparing a git-hosted package with a partial commit', async () test('do not build the package when scripts are ignored', async () => { const storeDir = temporaryDirectory() - const fetch = createGitFetcher({ ignoreScripts: true, rawConfig: {} }).git + const fetch = createGitFetcher({ ignoreScripts: true, rawConfig: {}, storeIndex: createStoreIndex(storeDir) }).git const { filesMap } = await fetch(createCafsStore(storeDir), { commit: '55416a9c468806a935636c0ad0371a14a64df8c9', @@ -249,7 +264,7 @@ test('do not build the package when scripts are ignored', async () => { test('block git package with prepare script', async () => { const storeDir = temporaryDirectory() - const fetch = createGitFetcher({ rawConfig: {} }).git + const fetch = createGitFetcher({ rawConfig: {}, storeIndex: createStoreIndex(storeDir) }).git const repo = 'https://github.com/pnpm-e2e/prepare-script-works.git' await expect( fetch(createCafsStore(storeDir), @@ -268,6 +283,7 @@ test('allow git package with prepare script', async () => { const storeDir = temporaryDirectory() const fetch = createGitFetcher({ rawConfig: {}, + storeIndex: createStoreIndex(storeDir), }).git // This should succeed without throwing because the package is in the allowlist const { filesMap } = await fetch(createCafsStore(storeDir), @@ -290,7 +306,7 @@ function prefixGitArgs (): string[] { test('fetch only the included files', async () => { const storeDir = temporaryDirectory() - const fetch = createGitFetcher({ rawConfig: {} }).git + const fetch = createGitFetcher({ rawConfig: {}, storeIndex: createStoreIndex(storeDir) }).git const { filesMap } = await fetch( createCafsStore(storeDir), { diff --git a/fetching/git-fetcher/tsconfig.json b/fetching/git-fetcher/tsconfig.json index ae5b50e25e..66b09274af 100644 --- a/fetching/git-fetcher/tsconfig.json +++ b/fetching/git-fetcher/tsconfig.json @@ -30,6 +30,9 @@ { "path": "../../store/create-cafs-store" }, + { + "path": "../../store/index" + }, { "path": "../../worker" }, diff --git a/fetching/pick-fetcher/package.json b/fetching/pick-fetcher/package.json index 30f361a5b1..9343b8a5c2 100644 --- a/fetching/pick-fetcher/package.json +++ b/fetching/pick-fetcher/package.json @@ -42,6 +42,7 @@ "@pnpm/create-cafs-store": "workspace:*", "@pnpm/fetch": "workspace:*", "@pnpm/pick-fetcher": "workspace:*", + "@pnpm/store.index": "workspace:*", "@pnpm/tarball-fetcher": "workspace:*", "@pnpm/test-fixtures": "workspace:*", "nock": "catalog:", diff --git a/fetching/pick-fetcher/test/customFetch.ts b/fetching/pick-fetcher/test/customFetch.ts index 69345d92c7..baa4c668ab 100644 --- a/fetching/pick-fetcher/test/customFetch.ts +++ b/fetching/pick-fetcher/test/customFetch.ts @@ -3,6 +3,7 @@ import { jest } from '@jest/globals' import { createTarballFetcher } from '@pnpm/tarball-fetcher' import { createFetchFromRegistry } from '@pnpm/fetch' import { createCafsStore } from '@pnpm/create-cafs-store' +import { StoreIndex } from '@pnpm/store.index' import { fixtures } from '@pnpm/test-fixtures' import { temporaryDirectory } from 'tempy' import path from 'path' @@ -13,6 +14,10 @@ import type { AtomicResolution } from '@pnpm/resolver-base' import type { CustomFetcher } from '@pnpm/hooks.types' const f = fixtures(import.meta.dirname) +const storeIndex = new StoreIndex(temporaryDirectory()) +afterAll(() => { + storeIndex.close() +}) // Test helpers to reduce type casting function createMockFetchers (partial: Partial = {}): Fetchers { @@ -280,7 +285,7 @@ describe('custom fetcher implementation examples', () => { const tarballFetchers = createTarballFetcher( fetchFromRegistry, () => undefined, - { rawConfig: {} } + { rawConfig: {}, storeIndex } ) // Custom fetcher that maps custom URLs to tarballs @@ -328,7 +333,7 @@ describe('custom fetcher implementation examples', () => { const tarballFetchers = createTarballFetcher( fetchFromRegistry, () => undefined, - { rawConfig: {} } + { rawConfig: {}, storeIndex } ) // Custom fetcher that maps custom local paths to tarballs @@ -379,7 +384,7 @@ describe('custom fetcher implementation examples', () => { const tarballFetchers = createTarballFetcher( fetchFromRegistry, () => undefined, - { rawConfig: {} } + { rawConfig: {}, storeIndex } ) // Custom fetcher that transforms custom resolution to tarball URL @@ -428,7 +433,7 @@ describe('custom fetcher implementation examples', () => { const tarballFetchers = createTarballFetcher( fetchFromRegistry, () => undefined, - { rawConfig: {}, ignoreScripts: true } + { rawConfig: {}, storeIndex, ignoreScripts: true } ) // Custom fetcher that maps custom git resolution to git-hosted tarball diff --git a/fetching/pick-fetcher/tsconfig.json b/fetching/pick-fetcher/tsconfig.json index d140e6474f..f6199da7ca 100644 --- a/fetching/pick-fetcher/tsconfig.json +++ b/fetching/pick-fetcher/tsconfig.json @@ -30,6 +30,9 @@ { "path": "../../store/create-cafs-store" }, + { + "path": "../../store/index" + }, { "path": "../fetcher-base" }, diff --git a/fetching/tarball-fetcher/package.json b/fetching/tarball-fetcher/package.json index 63ce722760..103e87a51b 100644 --- a/fetching/tarball-fetcher/package.json +++ b/fetching/tarball-fetcher/package.json @@ -40,13 +40,12 @@ "@pnpm/fs.packlist": "workspace:*", "@pnpm/graceful-fs": "workspace:*", "@pnpm/prepare-package": "workspace:*", + "@pnpm/store.index": "workspace:*", "@pnpm/types": "workspace:*", "@zkochan/retry": "catalog:", "lodash.throttle": "catalog:", "p-map-values": "catalog:", - "path-temp": "catalog:", - "ramda": "catalog:", - "rename-overwrite": "catalog:" + "ramda": "catalog:" }, "peerDependencies": { "@pnpm/logger": "catalog:", diff --git a/fetching/tarball-fetcher/src/gitHostedTarballFetcher.ts b/fetching/tarball-fetcher/src/gitHostedTarballFetcher.ts index de0adce4e1..8440e58e59 100644 --- a/fetching/tarball-fetcher/src/gitHostedTarballFetcher.ts +++ b/fetching/tarball-fetcher/src/gitHostedTarballFetcher.ts @@ -1,15 +1,13 @@ import assert from 'assert' -import fs from 'node:fs/promises' import util from 'util' import { type FetchFunction, type FetchOptions } from '@pnpm/fetcher-base' import { type Cafs, type FilesMap } from '@pnpm/cafs-types' import { packlist } from '@pnpm/fs.packlist' import { globalWarn } from '@pnpm/logger' import { preparePackage } from '@pnpm/prepare-package' +import { type StoreIndex } from '@pnpm/store.index' import { type BundledManifest } from '@pnpm/types' import { addFilesFromDir } from '@pnpm/worker' -import renameOverwrite from 'rename-overwrite' -import { fastPathTemp as pathTemp } from 'path-temp' interface Resolution { integrity?: string @@ -21,18 +19,22 @@ interface Resolution { export interface CreateGitHostedTarballFetcher { ignoreScripts?: boolean rawConfig: Record + storeIndex: StoreIndex unsafePerm?: boolean } export function createGitHostedTarballFetcher (fetchRemoteTarball: FetchFunction, fetcherOpts: CreateGitHostedTarballFetcher): FetchFunction { const fetch = async (cafs: Cafs, resolution: Resolution, opts: FetchOptions) => { - const tempIndexFile = pathTemp(opts.filesIndexFile) + const rawFilesIndexFile = `${opts.filesIndexFile}\traw` const { filesMap, manifest, requiresBuild } = await fetchRemoteTarball(cafs, resolution, { ...opts, - filesIndexFile: tempIndexFile, + filesIndexFile: rawFilesIndexFile, }) + // Flush any queued store index writes so that the raw files index entry + // written during tarball extraction is visible to subsequent reads. + fetcherOpts.storeIndex.flush() try { - const prepareResult = await prepareGitHostedPkg(filesMap, cafs, tempIndexFile, opts.filesIndexFile, fetcherOpts, opts, resolution) + const prepareResult = await prepareGitHostedPkg(filesMap, cafs, rawFilesIndexFile, opts.filesIndexFile, fetcherOpts, opts, resolution) if (prepareResult.ignoredBuild) { globalWarn(`The git-hosted package fetched from "${resolution.tarball}" has to be built but the build scripts were ignored.`) } @@ -60,7 +62,7 @@ interface PrepareGitHostedPkgResult { async function prepareGitHostedPkg ( filesMap: FilesMap, cafs: Cafs, - filesIndexFileNonBuilt: string, + rawFilesIndexFile: string, filesIndexFile: string, opts: CreateGitHostedTarballFetcher, fetcherOpts: FetchOptions, @@ -80,10 +82,13 @@ async function prepareGitHostedPkg ( allowBuild: fetcherOpts.allowBuild, }, tempLocation, resolution.path ?? '') const files = await packlist(pkgDir) + const { storeIndex } = opts if (!resolution.path && files.length === filesMap.size) { if (!shouldBeBuilt) { - if (filesIndexFileNonBuilt !== filesIndexFile) { - await renameOverwrite(filesIndexFileNonBuilt, filesIndexFile) + const data = storeIndex.get(rawFilesIndexFile) + if (data) { + storeIndex.set(filesIndexFile, data) + storeIndex.delete(rawFilesIndexFile) } return { filesMap, @@ -91,22 +96,21 @@ async function prepareGitHostedPkg ( } } if (opts.ignoreScripts) { + storeIndex.delete(rawFilesIndexFile) return { filesMap, ignoredBuild: true, } } } - try { - // The temporary index file may be deleted - await fs.unlink(filesIndexFileNonBuilt) - } catch {} + storeIndex.delete(rawFilesIndexFile) // Important! We cannot remove the temp location at this stage. // Even though we have the index of the package, // the linking of files to the store is in progress. return { ...await addFilesFromDir({ storeDir: cafs.storeDir, + storeIndex: opts.storeIndex, dir: pkgDir, files, filesIndexFile, diff --git a/fetching/tarball-fetcher/src/index.ts b/fetching/tarball-fetcher/src/index.ts index 518d7166d9..e9d8467e73 100644 --- a/fetching/tarball-fetcher/src/index.ts +++ b/fetching/tarball-fetcher/src/index.ts @@ -10,6 +10,7 @@ import { type GetAuthHeader, type RetryTimeoutOptions, } from '@pnpm/fetching-types' +import { type StoreIndex } from '@pnpm/store.index' import { TarballIntegrityError } from '@pnpm/worker' import { createDownloader, @@ -41,6 +42,7 @@ export function createTarballFetcher ( rawConfig: Record unsafePerm?: boolean ignoreScripts?: boolean + storeIndex: StoreIndex timeout?: number retry?: RetryTimeoutOptions offline?: boolean @@ -56,10 +58,11 @@ export function createTarballFetcher ( download, getAuthHeaderByURI: getAuthHeader, offline: opts.offline, + storeIndex: opts.storeIndex, }) as FetchFunction return { - localTarball: createLocalTarballFetcher(), + localTarball: createLocalTarballFetcher(opts.storeIndex), remoteTarball: remoteTarballFetcher, gitHostedTarball: createGitHostedTarballFetcher(remoteTarballFetcher, opts), } @@ -70,6 +73,7 @@ async function fetchFromTarball ( download: DownloadFunction getAuthHeaderByURI: (registry: string) => string | undefined offline?: boolean + storeIndex: StoreIndex }, cafs: Cafs, resolution: { @@ -86,6 +90,7 @@ async function fetchFromTarball ( return ctx.download(resolution.tarball, { getAuthHeaderByURI: ctx.getAuthHeaderByURI, cafs, + storeIndex: ctx.storeIndex, integrity: resolution.integrity, readManifest: opts.readManifest, onProgress: opts.onProgress, diff --git a/fetching/tarball-fetcher/src/localTarballFetcher.ts b/fetching/tarball-fetcher/src/localTarballFetcher.ts index 2ca5e828bf..b86b0aef24 100644 --- a/fetching/tarball-fetcher/src/localTarballFetcher.ts +++ b/fetching/tarball-fetcher/src/localTarballFetcher.ts @@ -2,6 +2,7 @@ import path from 'path' import { type FetchFunction, type FetchOptions } from '@pnpm/fetcher-base' import type { Cafs } from '@pnpm/cafs-types' import gfs from '@pnpm/graceful-fs' +import { type StoreIndex } from '@pnpm/store.index' import { addFilesFromTarball } from '@pnpm/worker' const isAbsolutePath = /^\/|^[A-Z]:/i @@ -12,12 +13,13 @@ interface Resolution { tarball: string } -export function createLocalTarballFetcher (): FetchFunction { +export function createLocalTarballFetcher (storeIndex: StoreIndex): FetchFunction { const fetch = (cafs: Cafs, resolution: Resolution, opts: FetchOptions) => { const tarball = resolvePath(opts.lockfileDir, resolution.tarball.slice(5)) const buffer = gfs.readFileSync(tarball) return addFilesFromTarball({ storeDir: cafs.storeDir, + storeIndex, buffer, filesIndexFile: opts.filesIndexFile, integrity: resolution.integrity, diff --git a/fetching/tarball-fetcher/src/remoteTarballFetcher.ts b/fetching/tarball-fetcher/src/remoteTarballFetcher.ts index 8630f5a979..63c87d4efd 100644 --- a/fetching/tarball-fetcher/src/remoteTarballFetcher.ts +++ b/fetching/tarball-fetcher/src/remoteTarballFetcher.ts @@ -7,6 +7,7 @@ import { type FetchResult, type FetchOptions } from '@pnpm/fetcher-base' import { type Cafs } from '@pnpm/cafs-types' import { type FetchFromRegistry } from '@pnpm/fetching-types' import { globalWarn } from '@pnpm/logger' +import { type StoreIndex } from '@pnpm/store.index' import { addFilesFromTarball } from '@pnpm/worker' import * as retry from '@zkochan/retry' import throttle from 'lodash.throttle' @@ -25,6 +26,7 @@ export type DownloadOptions = { onStart?: (totalSize: number | null, attempt: number) => void onProgress?: (downloaded: number) => void integrity?: string + storeIndex: StoreIndex } & Pick export type DownloadFunction = (url: string, opts: DownloadOptions) => Promise @@ -165,6 +167,7 @@ export function createDownloader ( return addFilesFromTarball({ buffer: data, storeDir: opts.cafs.storeDir, + storeIndex: opts.storeIndex, readManifest: opts.readManifest, integrity: opts.integrity, filesIndexFile: opts.filesIndexFile, diff --git a/fetching/tarball-fetcher/test/fetch.ts b/fetching/tarball-fetcher/test/fetch.ts index c6f2a6b29d..0c9befbffe 100644 --- a/fetching/tarball-fetcher/test/fetch.ts +++ b/fetching/tarball-fetcher/test/fetch.ts @@ -7,6 +7,7 @@ import { createFetchFromRegistry } from '@pnpm/fetch' import { createCafsStore } from '@pnpm/create-cafs-store' import { fixtures } from '@pnpm/test-fixtures' import { lexCompare } from '@pnpm/util.lex-comparator' +import { StoreIndex } from '@pnpm/store.index' import nock from 'nock' import ssri from 'ssri' import { temporaryDirectory } from 'tempy' @@ -34,6 +35,11 @@ beforeEach(() => { const storeDir = temporaryDirectory() const filesIndexFile = path.join(storeDir, 'index.json') const cafs = createCafsStore(storeDir) +const storeIndex = new StoreIndex(storeDir) + +afterAll(() => { + storeIndex.close() +}) const f = fixtures(import.meta.dirname) const tarballPath = f.find('babel-helper-hoist-variables-6.24.1.tgz') @@ -44,6 +50,7 @@ const fetchFromRegistry = createFetchFromRegistry({}) const getAuthHeader = () => undefined const fetch = createTarballFetcher(fetchFromRegistry, getAuthHeader, { rawConfig: {}, + storeIndex, retry: { maxTimeout: 100, minTimeout: 0, @@ -239,6 +246,7 @@ test("don't fail when fetching a local tarball in offline mode", async () => { const fetch = createTarballFetcher(fetchFromRegistry, getAuthHeader, { offline: true, rawConfig: {}, + storeIndex, retry: { maxTimeout: 100, minTimeout: 0, @@ -266,6 +274,7 @@ test('fail when trying to fetch a non-local tarball in offline mode', async () = const fetch = createTarballFetcher(fetchFromRegistry, getAuthHeader, { offline: true, rawConfig: {}, + storeIndex, retry: { maxTimeout: 100, minTimeout: 0, @@ -396,6 +405,7 @@ test('accessing private packages', async () => { const getAuthHeader = () => 'Bearer ofjergrg349gj3f2' const fetch = createTarballFetcher(fetchFromRegistry, getAuthHeader, { rawConfig: {}, + storeIndex, retry: { maxTimeout: 100, minTimeout: 0, @@ -504,6 +514,7 @@ test('do not build the package when scripts are ignored', async () => { const fetch = createTarballFetcher(fetchFromRegistry, getAuthHeader, { ignoreScripts: true, rawConfig: {}, + storeIndex, retry: { maxTimeout: 100, minTimeout: 0, @@ -549,6 +560,7 @@ test('use the subfolder when path is present', async () => { const fetch = createTarballFetcher(fetchFromRegistry, getAuthHeader, { ignoreScripts: true, rawConfig: {}, + storeIndex, retry: { maxTimeout: 100, minTimeout: 0, @@ -575,6 +587,7 @@ test('prevent directory traversal attack when path is present', async () => { const fetch = createTarballFetcher(fetchFromRegistry, getAuthHeader, { ignoreScripts: true, rawConfig: {}, + storeIndex, retry: { maxTimeout: 100, minTimeout: 0, @@ -599,6 +612,7 @@ test('fail when path is not exists', async () => { const fetch = createTarballFetcher(fetchFromRegistry, getAuthHeader, { ignoreScripts: true, rawConfig: {}, + storeIndex, retry: { maxTimeout: 100, minTimeout: 0, diff --git a/fetching/tarball-fetcher/tsconfig.json b/fetching/tarball-fetcher/tsconfig.json index 5e3830967c..f2eef358e8 100644 --- a/fetching/tarball-fetcher/tsconfig.json +++ b/fetching/tarball-fetcher/tsconfig.json @@ -45,6 +45,9 @@ { "path": "../../store/create-cafs-store" }, + { + "path": "../../store/index" + }, { "path": "../../worker" }, diff --git a/modules-mounter/daemon/package.json b/modules-mounter/daemon/package.json index 4c53bbcae5..c882f9c31c 100644 --- a/modules-mounter/daemon/package.json +++ b/modules-mounter/daemon/package.json @@ -38,12 +38,12 @@ "dependencies": { "@pnpm/config": "workspace:*", "@pnpm/dependency-path": "workspace:*", - "@pnpm/fs.msgpack-file": "workspace:*", "@pnpm/lockfile.fs": "workspace:*", "@pnpm/lockfile.utils": "workspace:*", "@pnpm/logger": "workspace:*", "@pnpm/store-path": "workspace:*", "@pnpm/store.cafs": "workspace:*", + "@pnpm/store.index": "workspace:*", "@pnpm/types": "workspace:*", "hyperdrive-schemas": "catalog:", "normalize-path": "catalog:" diff --git a/modules-mounter/daemon/src/createFuseHandlers.ts b/modules-mounter/daemon/src/createFuseHandlers.ts index 0084d432ef..d2a75795f8 100644 --- a/modules-mounter/daemon/src/createFuseHandlers.ts +++ b/modules-mounter/daemon/src/createFuseHandlers.ts @@ -1,7 +1,7 @@ // cspell:ignore ents import fs from 'fs' -import { readMsgpackFileSync } from '@pnpm/fs.msgpack-file' -import { getIndexFilePathInCafs, getFilePathByModeInCafs, type PackageFilesIndex } from '@pnpm/store.cafs' +import { StoreIndex, storeIndexKey } from '@pnpm/store.index' +import { getFilePathByModeInCafs, type PackageFilesIndex } from '@pnpm/store.cafs' import { type LockfileObject, readWantedLockfile, type PackageSnapshot, type TarballResolution } from '@pnpm/lockfile.fs' import { nameVerFromPkgSnapshot, @@ -39,6 +39,7 @@ export async function createFuseHandlers (lockfileDir: string, storeDir: string) } export function createFuseHandlersFromLockfile (lockfile: LockfileObject, storeDir: string): FuseHandlers { + const storeIndex = new StoreIndex(storeDir) const pkgSnapshotCache = new Map() const virtualNodeModules = makeVirtualNodeModules(lockfile) return { @@ -156,7 +157,7 @@ export function createFuseHandlersFromLockfile (lockfile: LockfileObject, storeD currentDirEntry = currentDirEntry.entries[parts.shift()!] } if (currentDirEntry?.entryType === 'index') { - const pkg = getPkgInfo(currentDirEntry.depPath, storeDir) + const pkg = getPkgInfo(currentDirEntry.depPath) if (pkg == null) { return null } @@ -168,13 +169,13 @@ export function createFuseHandlersFromLockfile (lockfile: LockfileObject, storeD } return currentDirEntry } - function getPkgInfo (depPath: string, storeDir: string) { + function getPkgInfo (depPath: string) { if (!pkgSnapshotCache.has(depPath)) { const pkgSnapshot = lockfile.packages?.[depPath as DepPath] if (pkgSnapshot == null) return undefined const nameVer = nameVerFromPkgSnapshot(depPath, pkgSnapshot) - const pkgIndexFilePath = getIndexFilePathInCafs(storeDir, (pkgSnapshot.resolution as TarballResolution).integrity!, `${nameVer.name}@${nameVer.version}`) - const pkgIndex = readMsgpackFileSync(pkgIndexFilePath) // TODO: maybe make it async? + const pkgIndexFilePath = storeIndexKey((pkgSnapshot.resolution as TarballResolution).integrity!, `${nameVer.name}@${nameVer.version}`) + const pkgIndex = storeIndex.get(pkgIndexFilePath) as PackageFilesIndex pkgSnapshotCache.set(depPath, { ...nameVer, pkgSnapshot, diff --git a/modules-mounter/daemon/tsconfig.json b/modules-mounter/daemon/tsconfig.json index 465d1e6988..7de082352f 100644 --- a/modules-mounter/daemon/tsconfig.json +++ b/modules-mounter/daemon/tsconfig.json @@ -12,9 +12,6 @@ { "path": "../../config/config" }, - { - "path": "../../fs/msgpack-file" - }, { "path": "../../lockfile/fs" }, @@ -36,6 +33,9 @@ { "path": "../../store/cafs" }, + { + "path": "../../store/index" + }, { "path": "../../store/store-path" } diff --git a/pkg-manager/client/package.json b/pkg-manager/client/package.json index 583513edc3..187c71d645 100644 --- a/pkg-manager/client/package.json +++ b/pkg-manager/client/package.json @@ -43,6 +43,7 @@ "@pnpm/network.auth-header": "workspace:*", "@pnpm/node.fetcher": "workspace:*", "@pnpm/resolver-base": "workspace:*", + "@pnpm/store.index": "workspace:*", "@pnpm/tarball-fetcher": "workspace:*", "@pnpm/types": "workspace:*", "ramda": "catalog:" diff --git a/pkg-manager/client/src/index.ts b/pkg-manager/client/src/index.ts index 13ef941f92..5b4caa8e1f 100644 --- a/pkg-manager/client/src/index.ts +++ b/pkg-manager/client/src/index.ts @@ -12,6 +12,7 @@ import { createDirectoryFetcher } from '@pnpm/directory-fetcher' import { createGitFetcher } from '@pnpm/git-fetcher' import { createTarballFetcher, type TarballFetchers } from '@pnpm/tarball-fetcher' import { createGetAuthHeaderByURI } from '@pnpm/network.auth-header' +import { type StoreIndex } from '@pnpm/store.index' import { createBinaryFetcher } from '@pnpm/fetching.binary-fetcher' export type { ResolveFunction } @@ -24,6 +25,7 @@ export type ClientOptions = { rawConfig: Record sslConfigs?: Record retry?: RetryTimeoutOptions + storeIndex: StoreIndex timeout?: number unsafePerm?: boolean userAgent?: string @@ -53,7 +55,7 @@ export function createClient (opts: ClientOptions): Client { } } -export function createResolver (opts: ClientOptions): { resolve: ResolveFunction, clearCache: () => void } { +export function createResolver (opts: Omit): { resolve: ResolveFunction, clearCache: () => void } { const fetchFromRegistry = createFetchFromRegistry(opts) const getAuthHeader = createGetAuthHeaderByURI({ allSettings: opts.authConfig, userSettings: opts.userConfig }) @@ -69,7 +71,7 @@ type Fetchers = { function createFetchers ( fetchFromRegistry: FetchFromRegistry, getAuthHeader: GetAuthHeader, - opts: Pick + opts: Pick ): Fetchers { const tarballFetchers = createTarballFetcher(fetchFromRegistry, getAuthHeader, opts) return { @@ -81,6 +83,7 @@ function createFetchers ( fetchFromRemoteTarball: tarballFetchers.remoteTarball, offline: opts.offline, rawConfig: opts.rawConfig, + storeIndex: opts.storeIndex, }), } } diff --git a/pkg-manager/client/test/index.ts b/pkg-manager/client/test/index.ts index 38104c5740..90a30f0c19 100644 --- a/pkg-manager/client/test/index.ts +++ b/pkg-manager/client/test/index.ts @@ -1,7 +1,15 @@ /// import { createClient, createResolver } from '@pnpm/client' +import { StoreIndex } from '@pnpm/store.index' + +const storeIndexes: StoreIndex[] = [] +afterAll(() => { + for (const si of storeIndexes) si.close() +}) test('createClient()', () => { + const storeIndex = new StoreIndex('.store') + storeIndexes.push(storeIndex) const client = createClient({ authConfig: { registry: 'https://registry.npmjs.org/' }, cacheDir: '', @@ -10,6 +18,7 @@ test('createClient()', () => { default: 'https://reigstry.npmjs.org/', }, storeDir: '.store', + storeIndex, }) expect(typeof client === 'object').toBeTruthy() }) diff --git a/pkg-manager/client/tsconfig.json b/pkg-manager/client/tsconfig.json index e9f1840254..735f562b8f 100644 --- a/pkg-manager/client/tsconfig.json +++ b/pkg-manager/client/tsconfig.json @@ -47,6 +47,9 @@ }, { "path": "../../resolving/resolver-base" + }, + { + "path": "../../store/index" } ] } diff --git a/pkg-manager/core/package.json b/pkg-manager/core/package.json index c703eb5952..49c68dc126 100644 --- a/pkg-manager/core/package.json +++ b/pkg-manager/core/package.json @@ -137,6 +137,7 @@ "@pnpm/registry-mock": "catalog:", "@pnpm/store-path": "workspace:*", "@pnpm/store.cafs": "workspace:*", + "@pnpm/store.index": "workspace:*", "@pnpm/test-fixtures": "workspace:*", "@pnpm/test-ipc-server": "workspace:*", "@pnpm/testing.temp-store": "workspace:*", diff --git a/pkg-manager/core/test/install/patch.ts b/pkg-manager/core/test/install/patch.ts index caa4e74da4..03620bf6f3 100644 --- a/pkg-manager/core/test/install/patch.ts +++ b/pkg-manager/core/test/install/patch.ts @@ -5,8 +5,9 @@ import { ENGINE_NAME } from '@pnpm/constants' import { install } from '@pnpm/core' import { type IgnoredScriptsLog } from '@pnpm/core-loggers' import { createHexHashFromFile } from '@pnpm/crypto.hash' -import { readMsgpackFileSync } from '@pnpm/fs.msgpack-file' import { prepareEmpty } from '@pnpm/prepare' +import { getIntegrity } from '@pnpm/registry-mock' +import { StoreIndex, storeIndexKey } from '@pnpm/store.index' import { fixtures } from '@pnpm/test-fixtures' import { jest } from '@jest/globals' import { sync as rimraf } from '@zkochan/rimraf' @@ -14,6 +15,11 @@ import { testDefaults } from '../utils/index.js' const f = fixtures(import.meta.dirname) +const storeIndexes: StoreIndex[] = [] +afterAll(() => { + for (const si of storeIndexes) si.close() +}) + test('patch package with exact version', async () => { const reporter = jest.fn() const project = prepareEmpty() @@ -55,8 +61,10 @@ test('patch package with exact version', async () => { }) expect(lockfile.snapshots[`is-positive@1.0.0(patch_hash=${patchFileHash})`]).toBeTruthy() - const filesIndexFile = path.join(opts.storeDir, 'index/c7/1ccf199e0fdae37aad13946b937d67bcd35fa111b84d21b3a19439cfdc2812-is-positive@1.0.0.mpk') - const filesIndex = readMsgpackFileSync(filesIndexFile) + const filesIndexKey = storeIndexKey(getIntegrity('is-positive', '1.0.0'), 'is-positive@1.0.0') + const storeIndex = new StoreIndex(opts.storeDir) + storeIndexes.push(storeIndex) + const filesIndex = storeIndex.get(filesIndexKey) as PackageFilesIndex expect(filesIndex.sideEffects).toBeTruthy() const sideEffectsKey = `${ENGINE_NAME};patch=${patchFileHash}` expect(filesIndex.sideEffects!.has(sideEffectsKey)).toBeTruthy() @@ -153,8 +161,10 @@ test('patch package with version range', async () => { }) expect(lockfile.snapshots[`is-positive@1.0.0(patch_hash=${patchFileHash})`]).toBeTruthy() - const filesIndexFile = path.join(opts.storeDir, 'index/c7/1ccf199e0fdae37aad13946b937d67bcd35fa111b84d21b3a19439cfdc2812-is-positive@1.0.0.mpk') - const filesIndex = readMsgpackFileSync(filesIndexFile) + const filesIndexKey = storeIndexKey(getIntegrity('is-positive', '1.0.0'), 'is-positive@1.0.0') + const storeIndex = new StoreIndex(opts.storeDir) + storeIndexes.push(storeIndex) + const filesIndex = storeIndex.get(filesIndexKey) as PackageFilesIndex expect(filesIndex.sideEffects).toBeTruthy() const sideEffectsKey = `${ENGINE_NAME};patch=${patchFileHash}` expect(filesIndex.sideEffects!.has(sideEffectsKey)).toBeTruthy() @@ -323,8 +333,10 @@ test('patch package when scripts are ignored', async () => { }) expect(lockfile.snapshots[`is-positive@1.0.0(patch_hash=${patchFileHash})`]).toBeTruthy() - const filesIndexFile = path.join(opts.storeDir, 'index/c7/1ccf199e0fdae37aad13946b937d67bcd35fa111b84d21b3a19439cfdc2812-is-positive@1.0.0.mpk') - const filesIndex = readMsgpackFileSync(filesIndexFile) + const filesIndexKey = storeIndexKey(getIntegrity('is-positive', '1.0.0'), 'is-positive@1.0.0') + const storeIndex = new StoreIndex(opts.storeDir) + storeIndexes.push(storeIndex) + const filesIndex = storeIndex.get(filesIndexKey) as PackageFilesIndex expect(filesIndex.sideEffects).toBeTruthy() const sideEffectsKey = `${ENGINE_NAME};patch=${patchFileHash}` expect(filesIndex.sideEffects!.has(sideEffectsKey)).toBeTruthy() @@ -414,8 +426,10 @@ test('patch package when the package is not in allowBuilds list', async () => { }) expect(lockfile.snapshots[`is-positive@1.0.0(patch_hash=${patchFileHash})`]).toBeTruthy() - const filesIndexFile = path.join(opts.storeDir, 'index/c7/1ccf199e0fdae37aad13946b937d67bcd35fa111b84d21b3a19439cfdc2812-is-positive@1.0.0.mpk') - const filesIndex = readMsgpackFileSync(filesIndexFile) + const filesIndexKey = storeIndexKey(getIntegrity('is-positive', '1.0.0'), 'is-positive@1.0.0') + const storeIndex = new StoreIndex(opts.storeDir) + storeIndexes.push(storeIndex) + const filesIndex = storeIndex.get(filesIndexKey) as PackageFilesIndex expect(filesIndex.sideEffects).toBeTruthy() const sideEffectsKey = `${ENGINE_NAME};patch=${patchFileHash}` expect(filesIndex.sideEffects!.has(sideEffectsKey)).toBeTruthy() diff --git a/pkg-manager/core/test/install/sideEffects.ts b/pkg-manager/core/test/install/sideEffects.ts index fa71f32d77..08d757df72 100644 --- a/pkg-manager/core/test/install/sideEffects.ts +++ b/pkg-manager/core/test/install/sideEffects.ts @@ -2,8 +2,8 @@ import fs from 'fs' import path from 'path' import { addDependenciesToPackage, install } from '@pnpm/core' import { hashObject } from '@pnpm/crypto.object-hasher' -import { readMsgpackFileSync, writeMsgpackFileSync } from '@pnpm/fs.msgpack-file' -import { getIndexFilePathInCafs, getFilePathByModeInCafs, type PackageFilesIndex } from '@pnpm/store.cafs' +import { getFilePathByModeInCafs, type PackageFilesIndex } from '@pnpm/store.cafs' +import { StoreIndex, storeIndexKey } from '@pnpm/store.index' import { getIntegrity, REGISTRY_MOCK_PORT } from '@pnpm/registry-mock' import { prepareEmpty } from '@pnpm/prepare' import { ENGINE_NAME } from '@pnpm/constants' @@ -12,6 +12,11 @@ import { testDefaults } from '../utils/index.js' const ENGINE_DIR = `${process.platform}-${process.arch}-node-${process.version.split('.')[0]}` +const storeIndexes: StoreIndex[] = [] +afterAll(() => { + for (const si of storeIndexes) si.close() +}) + test.skip('caching side effects of native package', async () => { prepareEmpty() @@ -82,8 +87,10 @@ test('using side effects cache', async () => { }, {}, {}, { packageImportMethod: 'copy' }) const { updatedManifest: manifest } = await addDependenciesToPackage({}, ['@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0'], opts) - const filesIndexFile = getIndexFilePathInCafs(opts.storeDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0') - const filesIndex = readMsgpackFileSync(filesIndexFile) + const filesIndexKey = storeIndexKey(getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0') + const storeIndex = new StoreIndex(opts.storeDir) + storeIndexes.push(storeIndex) + const filesIndex = storeIndex.get(filesIndexKey) as PackageFilesIndex expect(filesIndex.sideEffects).toBeTruthy() // files index has side effects const sideEffectsKey = `${ENGINE_NAME};deps=${hashObject({ id: `@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0:${getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0')}`, @@ -101,7 +108,7 @@ test('using side effects cache', async () => { expect(addedFiles.has('generated-by-preinstall.js')).toBeTruthy() expect(addedFiles.has('generated-by-postinstall.js')).toBeTruthy() addedFiles.delete('generated-by-postinstall.js') - writeMsgpackFileSync(filesIndexFile, filesIndex) + storeIndex.set(filesIndexKey, filesIndex) rimraf('node_modules') rimraf('pnpm-lock.yaml') // to avoid headless install @@ -169,9 +176,11 @@ test('uploading errors do not interrupt installation', async () => { expect(fs.existsSync('node_modules/@pnpm.e2e/pre-and-postinstall-scripts-example/generated-by-postinstall.js')).toBeTruthy() - const filesIndexFile = getIndexFilePathInCafs(opts.storeDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0') - const filesIndex = readMsgpackFileSync(filesIndexFile) - expect(filesIndex.sideEffects).toBeFalsy() + const filesIndexKey2 = storeIndexKey(getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0') + const storeIndex2 = new StoreIndex(opts.storeDir) + const filesIndex2 = storeIndex2.get(filesIndexKey2) as PackageFilesIndex + storeIndex2.close() + expect(filesIndex2.sideEffects).toBeFalsy() }) test('a postinstall script does not modify the original sources added to the store', async () => { @@ -187,8 +196,10 @@ test('a postinstall script does not modify the original sources added to the sto expect(fs.readFileSync('node_modules/@pnpm/postinstall-modifies-source/empty-file.txt', 'utf8')).toContain('hello') - const filesIndexFile = getIndexFilePathInCafs(opts.storeDir, getIntegrity('@pnpm/postinstall-modifies-source', '1.0.0'), '@pnpm/postinstall-modifies-source@1.0.0') - const filesIndex = readMsgpackFileSync(filesIndexFile) + const filesIndexKey3 = storeIndexKey(getIntegrity('@pnpm/postinstall-modifies-source', '1.0.0'), '@pnpm/postinstall-modifies-source@1.0.0') + const storeIndex3 = new StoreIndex(opts.storeDir) + const filesIndex = storeIndex3.get(filesIndexKey3) as PackageFilesIndex + storeIndex3.close() expect(filesIndex.sideEffects).toBeTruthy() expect(filesIndex.sideEffects?.has(`${ENGINE_NAME};deps=${hashObject({ id: `@pnpm/postinstall-modifies-source@1.0.0:${getIntegrity('@pnpm/postinstall-modifies-source', '1.0.0')}`, @@ -219,9 +230,11 @@ test('a corrupted side-effects cache is ignored', async () => { }) const { updatedManifest: manifest } = await addDependenciesToPackage({}, ['@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0'], opts) - const filesIndexFile = getIndexFilePathInCafs(opts.storeDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0') - const filesIndex = readMsgpackFileSync(filesIndexFile) - expect(filesIndex.sideEffects).toBeTruthy() // files index has side effects + const filesIndexKey4 = storeIndexKey(getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0') + const storeIndex4 = new StoreIndex(opts.storeDir) + const filesIndex4 = storeIndex4.get(filesIndexKey4) as PackageFilesIndex + storeIndex4.close() + expect(filesIndex4.sideEffects).toBeTruthy() // files index has side effects const sideEffectsKey = `${ENGINE_NAME};deps=${hashObject({ id: `@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0:${getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0')}`, deps: { @@ -232,11 +245,11 @@ test('a corrupted side-effects cache is ignored', async () => { }, })}` - expect(filesIndex.sideEffects).toBeTruthy() - expect(filesIndex.sideEffects!.has(sideEffectsKey)).toBeTruthy() - expect(filesIndex.sideEffects!.get(sideEffectsKey)!.added).toBeTruthy() - expect(filesIndex.sideEffects!.get(sideEffectsKey)!.added!.has('generated-by-preinstall.js')).toBeTruthy() - const sideEffectFileStat = filesIndex.sideEffects!.get(sideEffectsKey)!.added!.get('generated-by-preinstall.js')! + expect(filesIndex4.sideEffects).toBeTruthy() + expect(filesIndex4.sideEffects!.has(sideEffectsKey)).toBeTruthy() + expect(filesIndex4.sideEffects!.get(sideEffectsKey)!.added).toBeTruthy() + expect(filesIndex4.sideEffects!.get(sideEffectsKey)!.added!.has('generated-by-preinstall.js')).toBeTruthy() + const sideEffectFileStat = filesIndex4.sideEffects!.get(sideEffectsKey)!.added!.get('generated-by-preinstall.js')! const sideEffectFile = getFilePathByModeInCafs(opts.storeDir, sideEffectFileStat.digest, sideEffectFileStat.mode) expect(fs.existsSync(sideEffectFile)).toBeTruthy() rimraf(sideEffectFile) // we remove the side effect file to break the store diff --git a/pkg-manager/core/tsconfig.json b/pkg-manager/core/tsconfig.json index a8bbed1215..2e379d367b 100644 --- a/pkg-manager/core/tsconfig.json +++ b/pkg-manager/core/tsconfig.json @@ -153,6 +153,9 @@ { "path": "../../store/cafs" }, + { + "path": "../../store/index" + }, { "path": "../../store/store-controller-types" }, diff --git a/pkg-manager/headless/package.json b/pkg-manager/headless/package.json index 463240a681..ab906e4eb6 100644 --- a/pkg-manager/headless/package.json +++ b/pkg-manager/headless/package.json @@ -81,7 +81,6 @@ "@jest/globals": "catalog:", "@pnpm/assert-project": "workspace:*", "@pnpm/crypto.object-hasher": "workspace:*", - "@pnpm/fs.msgpack-file": "workspace:*", "@pnpm/headless": "workspace:*", "@pnpm/logger": "workspace:*", "@pnpm/prepare": "workspace:*", @@ -89,6 +88,7 @@ "@pnpm/registry-mock": "catalog:", "@pnpm/store-path": "workspace:*", "@pnpm/store.cafs": "workspace:*", + "@pnpm/store.index": "workspace:*", "@pnpm/test-fixtures": "workspace:*", "@pnpm/test-ipc-server": "workspace:*", "@pnpm/testing.temp-store": "workspace:*", diff --git a/pkg-manager/headless/src/index.ts b/pkg-manager/headless/src/index.ts index 01bce59a6f..7d185bea1c 100644 --- a/pkg-manager/headless/src/index.ts +++ b/pkg-manager/headless/src/index.ts @@ -664,8 +664,6 @@ export async function headlessInstall (opts: HeadlessOptions): Promise { + for (const si of storeIndexes) si.close() +}) + test('installing a simple project', async () => { const prefix = f.prepare('simple') const reporter = jest.fn() @@ -696,8 +701,10 @@ test.each([['isolated'], ['hoisted']])('using side effects cache with nodeLinker }, {}, {}, { packageImportMethod: 'copy' }) await headlessInstall(opts) - const cacheIntegrityPath = getIndexFilePathInCafs(opts.storeDir, getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0') - const cacheIntegrity = readMsgpackFileSync(cacheIntegrityPath) + const cacheIntegrityPath = storeIndexKey(getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0'), '@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0') + const storeIndex = new StoreIndex(opts.storeDir) + storeIndexes.push(storeIndex) + const cacheIntegrity = storeIndex.get(cacheIntegrityPath) as PackageFilesIndex expect(cacheIntegrity!.sideEffects).toBeTruthy() const sideEffectsKey = `${ENGINE_NAME};deps=${hashObject({ id: `@pnpm.e2e/pre-and-postinstall-scripts-example@1.0.0:${getIntegrity('@pnpm.e2e/pre-and-postinstall-scripts-example', '1.0.0')}`, @@ -712,7 +719,7 @@ test.each([['isolated'], ['hoisted']])('using side effects cache with nodeLinker cacheIntegrity!.sideEffects!.get(sideEffectsKey)!.added!.delete('generated-by-postinstall.js') expect(cacheIntegrity!.sideEffects!.get(sideEffectsKey)?.added?.has('generated-by-preinstall.js')).toBeTruthy() - writeMsgpackFileSync(cacheIntegrityPath, cacheIntegrity) + storeIndex.set(cacheIntegrityPath, cacheIntegrity) prefix = f.prepare('side-effects') const opts2 = await testDefaults({ diff --git a/pkg-manager/headless/tsconfig.json b/pkg-manager/headless/tsconfig.json index c1d4deb059..9af8800bc3 100644 --- a/pkg-manager/headless/tsconfig.json +++ b/pkg-manager/headless/tsconfig.json @@ -39,9 +39,6 @@ { "path": "../../exec/lifecycle" }, - { - "path": "../../fs/msgpack-file" - }, { "path": "../../fs/symlink-dependency" }, @@ -90,6 +87,9 @@ { "path": "../../store/cafs" }, + { + "path": "../../store/index" + }, { "path": "../../store/store-controller-types" }, diff --git a/pkg-manager/package-requester/package.json b/pkg-manager/package-requester/package.json index c20f60fe70..24b041f491 100644 --- a/pkg-manager/package-requester/package.json +++ b/pkg-manager/package-requester/package.json @@ -45,6 +45,7 @@ "@pnpm/resolver-base": "workspace:*", "@pnpm/store-controller-types": "workspace:*", "@pnpm/store.cafs": "workspace:*", + "@pnpm/store.index": "workspace:*", "@pnpm/types": "workspace:*", "detect-libc": "catalog:", "load-json-file": "catalog:", diff --git a/pkg-manager/package-requester/src/packageRequester.ts b/pkg-manager/package-requester/src/packageRequester.ts index 32bbe09650..fd03ae4f6f 100644 --- a/pkg-manager/package-requester/src/packageRequester.ts +++ b/pkg-manager/package-requester/src/packageRequester.ts @@ -1,9 +1,9 @@ import { createReadStream, promises as fs } from 'fs' import path from 'path' import { - getIndexFilePathInCafs as _getIndexFilePathInCafs, normalizeBundledManifest, } from '@pnpm/store.cafs' +import { gitHostedStoreIndexKey, storeIndexKey } from '@pnpm/store.index' import { fetchingProgressLogger, progressLogger } from '@pnpm/core-loggers' import { pickFetcher } from '@pnpm/pick-fetcher' import { PnpmError } from '@pnpm/error' @@ -96,7 +96,6 @@ export function createPackageRequester ( concurrency: networkConcurrency, }) - const getIndexFilePathInCafs = _getIndexFilePathInCafs.bind(null, opts.storeDir) const fetch = fetcher.bind(null, opts.fetchers, opts.cafs, opts.customFetchers) const readPkgFromCafs = _readPkgFromCafs.bind(null, { storeDir: opts.storeDir, @@ -107,7 +106,6 @@ export function createPackageRequester ( readPkgFromCafs, fetch, fetchingLocker: new Map(), - getIndexFilePathInCafs, requestsQueue: Object.assign(requestsQueue, { counter: 0, concurrency: networkConcurrency, @@ -130,7 +128,6 @@ export function createPackageRequester ( return Object.assign(requestPackage, { fetchPackageToStore, getFilesIndexFilePath: getFilesIndexFilePath.bind(null, { - getIndexFilePathInCafs, storeDir: opts.storeDir, virtualStoreDirMaxLength: opts.virtualStoreDirMaxLength, }), @@ -333,7 +330,6 @@ interface GetFilesIndexFilePathResult { function getFilesIndexFilePath ( ctx: { - getIndexFilePathInCafs: (integrity: string, pkgId: string) => string storeDir: string virtualStoreDirMaxLength: number }, @@ -344,7 +340,7 @@ function getFilesIndexFilePath ( if ((opts.pkg.resolution as TarballResolution).integrity) { return { target, - filesIndexFile: ctx.getIndexFilePathInCafs((opts.pkg.resolution as TarballResolution).integrity!, opts.pkg.id), + filesIndexFile: storeIndexKey((opts.pkg.resolution as TarballResolution).integrity!, opts.pkg.id), resolution: opts.pkg.resolution as AtomicResolution, } } @@ -354,14 +350,14 @@ function getFilesIndexFilePath ( if ((resolution as TarballResolution).integrity) { return { target, - filesIndexFile: ctx.getIndexFilePathInCafs((resolution as TarballResolution).integrity!, opts.pkg.id), + filesIndexFile: storeIndexKey((resolution as TarballResolution).integrity!, opts.pkg.id), resolution, } } } else { resolution = opts.pkg.resolution } - const filesIndexFile = path.join(target, opts.ignoreScripts ? 'integrity-not-built.mpk' : 'integrity.mpk') + const filesIndexFile = gitHostedStoreIndexKey(opts.pkg.id, { built: !opts.ignoreScripts }) return { filesIndexFile, target, resolution } } @@ -402,7 +398,6 @@ function fetchToStore ( opts: FetchOptions ) => Promise fetchingLocker: Map - getIndexFilePathInCafs: (integrity: string, pkgId: string) => string requestsQueue: { add: (fn: () => Promise, opts: { priority: number }) => Promise counter: number diff --git a/pkg-manager/package-requester/test/index.ts b/pkg-manager/package-requester/test/index.ts index 121faa1caf..aae8e91341 100644 --- a/pkg-manager/package-requester/test/index.ts +++ b/pkg-manager/package-requester/test/index.ts @@ -3,7 +3,7 @@ import fs from 'fs' import path from 'path' import { type PackageFilesIndex } from '@pnpm/store.cafs' import { createClient } from '@pnpm/client' -import { readMsgpackFileSync } from '@pnpm/fs.msgpack-file' +import { StoreIndex } from '@pnpm/store.index' import { streamParser } from '@pnpm/logger' import { createPackageRequester, type PackageResponse } from '@pnpm/package-requester' import { createCafsStore } from '@pnpm/create-cafs-store' @@ -26,14 +26,36 @@ const registries = { default: registry } const authConfig = { registry } +const storeIndexes: StoreIndex[] = [] +afterAll(() => { + for (const si of storeIndexes) si.close() +}) + +const topStoreIndex = new StoreIndex('.store') +storeIndexes.push(topStoreIndex) + const { resolve, fetchers } = createClient({ authConfig, cacheDir: '.store', storeDir: '.store', rawConfig: {}, registries, + storeIndex: topStoreIndex, }) +function createFetchersForStore (storeDir: string) { + const si = new StoreIndex(storeDir) + storeIndexes.push(si) + return createClient({ + authConfig, + rawConfig: {}, + cacheDir: storeDir, + storeDir, + registries, + storeIndex: si, + }).fetchers +} + afterEach(() => { nock.abortPendingRequests() nock.cleanAll() @@ -150,7 +172,6 @@ test('request package but skip fetching, when resolution is already available', update: false, }) as PackageResponse & { body: { - latest: string manifest: { name: string } } } @@ -160,7 +181,7 @@ test('request package but skip fetching, when resolution is already available', expect(pkgResponse.body.id).toBe('is-positive@1.0.0') expect(pkgResponse.body.isLocal).toBe(false) - expect(typeof pkgResponse.body.latest).toBe('string') + // latest may be undefined when the resolver's fast path resolves from the store cache expect(pkgResponse.body.manifest.name).toBe('is-positive') expect(!pkgResponse.body.normalizedBareSpecifier).toBeTruthy() expect(pkgResponse.body.resolution).toStrictEqual({ @@ -180,6 +201,7 @@ test('refetch local tarball if its integrity has changed', async () => { const wantedPackage = { bareSpecifier: tarball } const storeDir = temporaryDirectory() const cafs = createCafsStore(storeDir) + const localFetchers = createFetchersForStore(storeDir) const pkgId = `file:${normalize(tarballRelativePath)}` const requestPackageOpts = { downloadPriority: 0, @@ -193,7 +215,7 @@ test('refetch local tarball if its integrity has changed', async () => { { const requestPackage = createPackageRequester({ resolve, - fetchers, + fetchers: localFetchers, cafs, storeDir, verifyStoreIntegrity: true, @@ -225,7 +247,7 @@ test('refetch local tarball if its integrity has changed', async () => { { const requestPackage = createPackageRequester({ resolve, - fetchers, + fetchers: localFetchers, cafs, storeDir, verifyStoreIntegrity: true, @@ -252,7 +274,7 @@ test('refetch local tarball if its integrity has changed', async () => { { const requestPackage = createPackageRequester({ resolve, - fetchers, + fetchers: localFetchers, cafs, storeDir, verifyStoreIntegrity: true, @@ -287,6 +309,7 @@ test('refetch local tarball if its integrity has changed. The requester does not const wantedPackage = { bareSpecifier: tarball } const storeDir = path.join(projectDir, 'store') const cafs = createCafsStore(storeDir) + const localFetchers = createFetchersForStore(storeDir) const requestPackageOpts = { downloadPriority: 0, lockfileDir: projectDir, @@ -298,7 +321,7 @@ test('refetch local tarball if its integrity has changed. The requester does not { const requestPackage = createPackageRequester({ resolve, - fetchers, + fetchers: localFetchers, cafs, storeDir, verifyStoreIntegrity: true, @@ -321,7 +344,7 @@ test('refetch local tarball if its integrity has changed. The requester does not { const requestPackage = createPackageRequester({ resolve, - fetchers, + fetchers: localFetchers, cafs, storeDir, verifyStoreIntegrity: true, @@ -341,7 +364,7 @@ test('refetch local tarball if its integrity has changed. The requester does not { const requestPackage = createPackageRequester({ resolve, - fetchers, + fetchers: localFetchers, cafs, storeDir, verifyStoreIntegrity: true, @@ -421,9 +444,10 @@ test('force fetch when resolution integrity differs from current package integri test('fetchPackageToStore()', async () => { const storeDir = temporaryDirectory() const cafs = createCafsStore(storeDir) + const localFetchers = createFetchersForStore(storeDir) const packageRequester = createPackageRequester({ resolve, - fetchers, + fetchers: localFetchers, cafs, networkConcurrency: 1, storeDir, @@ -451,7 +475,9 @@ test('fetchPackageToStore()', async () => { expect(Array.from(files.filesMap.keys()).sort((a, b) => a.localeCompare(b))).toStrictEqual(['package.json', 'index.js', 'license', 'readme.md'].sort((a, b) => a.localeCompare(b))) expect(files.resolvedFrom).toBe('remote') - const indexFile = readMsgpackFileSync(fetchResult.filesIndexFile) + const storeIndex = new StoreIndex(storeDir) + storeIndexes.push(storeIndex) + const indexFile = storeIndex.get(fetchResult.filesIndexFile) as PackageFilesIndex expect(indexFile).toBeTruthy() expect(typeof indexFile.files.get('package.json')!.checkedAt).toBeTruthy() @@ -571,6 +597,7 @@ test('fetchPackageToStore() does not cache errors', async () => { cacheDir: '.pnpm', storeDir: '.store', registries, + storeIndex: topStoreIndex, }) const storeDir = temporaryDirectory() @@ -732,6 +759,7 @@ test('fetchPackageToStore() fetch raw manifest of cached package', async () => { test('refetch package to store if it has been modified', async () => { const storeDir = temporaryDirectory() const lockfileDir = temporaryDirectory() + const localFetchers = createFetchersForStore(storeDir) const pkgId = 'magic-hook@2.0.0' const resolution = { @@ -743,7 +771,7 @@ test('refetch package to store if it has been modified', async () => { const cafs = createCafsStore(storeDir) const packageRequester = createPackageRequester({ resolve, - fetchers, + fetchers: localFetchers, cafs, networkConcurrency: 1, storeDir, @@ -782,7 +810,7 @@ test('refetch package to store if it has been modified', async () => { const cafs = createCafsStore(storeDir) const packageRequester = createPackageRequester({ resolve, - fetchers, + fetchers: localFetchers, cafs, networkConcurrency: 1, storeDir, @@ -893,12 +921,13 @@ test('fetch a git package without a package.json', async () => { test('throw exception if the package data in the store differs from the expected data', async () => { const storeDir = temporaryDirectory() const cafs = createCafsStore(storeDir) + const localFetchers = createFetchersForStore(storeDir) let pkgResponse!: PackageResponse { const requestPackage = createPackageRequester({ resolve, - fetchers, + fetchers: localFetchers, cafs, networkConcurrency: 1, storeDir, @@ -920,7 +949,7 @@ test('throw exception if the package data in the store differs from the expected { const requestPackage = createPackageRequester({ resolve, - fetchers, + fetchers: localFetchers, cafs, networkConcurrency: 1, storeDir, @@ -944,7 +973,7 @@ test('throw exception if the package data in the store differs from the expected { const requestPackage = createPackageRequester({ resolve, - fetchers, + fetchers: localFetchers, cafs, networkConcurrency: 1, storeDir, @@ -968,7 +997,7 @@ test('throw exception if the package data in the store differs from the expected { const requestPackage = createPackageRequester({ resolve, - fetchers, + fetchers: localFetchers, cafs, networkConcurrency: 1, storeDir, diff --git a/pkg-manager/package-requester/tsconfig.json b/pkg-manager/package-requester/tsconfig.json index 5f65998184..77414b9eb7 100644 --- a/pkg-manager/package-requester/tsconfig.json +++ b/pkg-manager/package-requester/tsconfig.json @@ -57,6 +57,9 @@ { "path": "../../store/create-cafs-store" }, + { + "path": "../../store/index" + }, { "path": "../../store/store-controller-types" }, diff --git a/pkg-manager/plugin-commands-installation/package.json b/pkg-manager/plugin-commands-installation/package.json index 5b2d30acc3..74aabd2738 100644 --- a/pkg-manager/plugin-commands-installation/package.json +++ b/pkg-manager/plugin-commands-installation/package.json @@ -101,8 +101,10 @@ "@pnpm/plugin-commands-installation": "workspace:*", "@pnpm/prepare": "workspace:*", "@pnpm/registry-mock": "catalog:", + "@pnpm/store.index": "workspace:*", "@pnpm/test-fixtures": "workspace:*", "@pnpm/test-ipc-server": "workspace:*", + "@pnpm/worker": "workspace:*", "@pnpm/workspace.filter-packages-from-dir": "workspace:*", "@types/normalize-path": "catalog:", "@types/proxyquire": "catalog:", diff --git a/pkg-manager/plugin-commands-installation/test/fetch.ts b/pkg-manager/plugin-commands-installation/test/fetch.ts index 303ad5c545..a6b1124409 100644 --- a/pkg-manager/plugin-commands-installation/test/fetch.ts +++ b/pkg-manager/plugin-commands-installation/test/fetch.ts @@ -4,6 +4,8 @@ import { STORE_VERSION } from '@pnpm/constants' import { install, fetch } from '@pnpm/plugin-commands-installation' import { prepare } from '@pnpm/prepare' import { REGISTRY_MOCK_PORT } from '@pnpm/registry-mock' +import { closeAllStoreIndexes } from '@pnpm/store.index' +import { finishWorkers } from '@pnpm/worker' import { sync as rimraf } from '@zkochan/rimraf' const REGISTRY_URL = `http://localhost:${REGISTRY_MOCK_PORT}` @@ -204,6 +206,10 @@ test('fetch populates global virtual store links/', async () => { storeDir, }) + // Drain workers and close SQLite connections before removing the store (required on Windows) + await finishWorkers() + closeAllStoreIndexes() + // Remove the store — simulate a cold start with only the lockfile rimraf(storeDir) diff --git a/pkg-manager/plugin-commands-installation/tsconfig.json b/pkg-manager/plugin-commands-installation/tsconfig.json index ecc2806c52..2d46310d20 100644 --- a/pkg-manager/plugin-commands-installation/tsconfig.json +++ b/pkg-manager/plugin-commands-installation/tsconfig.json @@ -105,12 +105,18 @@ { "path": "../../reviewing/outdated" }, + { + "path": "../../store/index" + }, { "path": "../../store/package-store" }, { "path": "../../store/store-connection-manager" }, + { + "path": "../../worker" + }, { "path": "../../workspace/filter-packages-from-dir" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4fb7fc6b99..6b2964fd68 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -29,7 +29,7 @@ catalogs: version: 9.4.1 '@eslint/js': specifier: ^9.18.0 - version: 9.39.3 + version: 9.39.2 '@jest/globals': specifier: 30.0.5 version: 30.0.5 @@ -356,7 +356,7 @@ catalogs: version: 5.0.0 eslint: specifier: ^9.39.2 - version: 9.39.3 + version: 9.39.2 eslint-plugin-import-x: specifier: ^4.16.1 version: 4.16.1 @@ -704,7 +704,7 @@ catalogs: version: 5.9.2 typescript-eslint: specifier: ^8.53.0 - version: 8.56.1 + version: 8.56.0 unified: specifier: ^9.2.2 version: 9.2.2 @@ -857,10 +857,10 @@ importers: version: 9.2.0 eslint: specifier: 'catalog:' - version: 9.39.3(jiti@2.6.1) + version: 9.39.2(jiti@2.6.1) eslint-plugin-regexp: specifier: 'catalog:' - version: 2.10.0(eslint@9.39.3(jiti@2.6.1)) + version: 2.10.0(eslint@9.39.2(jiti@2.6.1)) husky: specifier: 'catalog:' version: 9.1.7 @@ -1004,6 +1004,9 @@ importers: '@pnpm/store.cafs': specifier: workspace:* version: link:../../store/cafs + '@pnpm/store.index': + specifier: workspace:* + version: link:../../store/index devDependencies: '@pnpm/assert-store': specifier: workspace:* @@ -1016,38 +1019,38 @@ importers: dependencies: '@eslint/js': specifier: 'catalog:' - version: 9.39.3 + version: 9.39.2 '@stylistic/eslint-plugin': specifier: 'catalog:' - version: 4.4.1(eslint@9.39.3(jiti@2.6.1))(typescript@5.9.2) + version: 4.4.1(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.2) eslint: specifier: 'catalog:' - version: 9.39.3(jiti@2.6.1) + version: 9.39.2(jiti@2.6.1) eslint-plugin-import-x: specifier: 'catalog:' - version: 4.16.1(@typescript-eslint/utils@8.56.1(eslint@9.39.3(jiti@2.6.1))(typescript@5.9.2))(eslint@9.39.3(jiti@2.6.1)) + version: 4.16.1(@typescript-eslint/utils@8.56.1(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.2))(eslint@9.39.2(jiti@2.6.1)) eslint-plugin-n: specifier: 'catalog:' - version: 17.24.0(eslint@9.39.3(jiti@2.6.1))(typescript@5.9.2) + version: 17.24.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.2) eslint-plugin-promise: specifier: 'catalog:' - version: 7.2.1(eslint@9.39.3(jiti@2.6.1)) + version: 7.2.1(eslint@9.39.2(jiti@2.6.1)) typescript: specifier: 'catalog:' version: 5.9.2 typescript-eslint: specifier: 'catalog:' - version: 8.56.1(eslint@9.39.3(jiti@2.6.1))(typescript@5.9.2) + version: 8.56.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.2) devDependencies: '@pnpm/eslint-config': specifier: workspace:* version: 'link:' '@typescript-eslint/utils': specifier: 'catalog:' - version: 8.56.1(eslint@9.39.3(jiti@2.6.1))(typescript@5.9.2) + version: 8.56.1(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.2) eslint-plugin-jest: specifier: 'catalog:' - version: 29.15.0(@typescript-eslint/eslint-plugin@8.56.1(@typescript-eslint/parser@8.56.1(eslint@9.39.3(jiti@2.6.1))(typescript@5.9.2))(eslint@9.39.3(jiti@2.6.1))(typescript@5.9.2))(eslint@9.39.3(jiti@2.6.1))(jest@30.2.0(@babel/types@7.29.0)(@types/node@22.19.13))(typescript@5.9.2) + version: 29.15.0(@typescript-eslint/eslint-plugin@8.56.1(@typescript-eslint/parser@8.56.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.2))(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.2))(eslint@9.39.2(jiti@2.6.1))(jest@30.2.0(@babel/types@7.29.0)(@types/node@22.19.13))(typescript@5.9.2) __utils__/get-release-text: dependencies: @@ -1243,6 +1246,9 @@ importers: '@pnpm/store.cafs': specifier: workspace:* version: link:../../store/cafs + '@pnpm/store.index': + specifier: workspace:* + version: link:../../store/index encode-registry: specifier: 'catalog:' version: 3.0.1 @@ -2270,6 +2276,9 @@ importers: '@pnpm/node.resolver': specifier: workspace:* version: link:../node.resolver + '@pnpm/store.index': + specifier: workspace:* + version: link:../../store/index '@pnpm/tarball-fetcher': specifier: workspace:* version: link:../../fetching/tarball-fetcher @@ -2654,9 +2663,6 @@ importers: '@pnpm/exec.pkg-requires-build': specifier: workspace:* version: link:../pkg-requires-build - '@pnpm/fs.msgpack-file': - specifier: workspace:* - version: link:../../fs/msgpack-file '@pnpm/get-context': specifier: workspace:* version: link:../../pkg-manager/get-context @@ -2699,6 +2705,9 @@ importers: '@pnpm/store.cafs': specifier: workspace:* version: link:../../store/cafs + '@pnpm/store.index': + specifier: workspace:* + version: link:../../store/index '@pnpm/types': specifier: workspace:* version: link:../../packages/types @@ -3013,6 +3022,9 @@ importers: '@pnpm/fetching-types': specifier: workspace:* version: link:../../network/fetching-types + '@pnpm/store.index': + specifier: workspace:* + version: link:../../store/index '@pnpm/worker': specifier: workspace:^ version: link:../../worker @@ -3121,6 +3133,9 @@ importers: '@pnpm/prepare-package': specifier: workspace:* version: link:../../exec/prepare-package + '@pnpm/store.index': + specifier: workspace:* + version: link:../../store/index '@pnpm/worker': specifier: workspace:^ version: link:../../worker @@ -3186,6 +3201,9 @@ importers: '@pnpm/pick-fetcher': specifier: workspace:* version: 'link:' + '@pnpm/store.index': + specifier: workspace:* + version: link:../../store/index '@pnpm/tarball-fetcher': specifier: workspace:* version: link:../tarball-fetcher @@ -3222,6 +3240,9 @@ importers: '@pnpm/prepare-package': specifier: workspace:* version: link:../../exec/prepare-package + '@pnpm/store.index': + specifier: workspace:* + version: link:../../store/index '@pnpm/types': specifier: workspace:* version: link:../../packages/types @@ -3237,15 +3258,9 @@ importers: p-map-values: specifier: 'catalog:' version: 1.0.0 - path-temp: - specifier: 'catalog:' - version: 2.1.1 ramda: specifier: 'catalog:' version: '@pnpm/ramda@0.28.1' - rename-overwrite: - specifier: 'catalog:' - version: 6.0.3 devDependencies: '@jest/globals': specifier: 'catalog:' @@ -4267,9 +4282,6 @@ importers: '@pnpm/dependency-path': specifier: workspace:* version: link:../../packages/dependency-path - '@pnpm/fs.msgpack-file': - specifier: workspace:* - version: link:../../fs/msgpack-file '@pnpm/lockfile.fs': specifier: workspace:* version: link:../../lockfile/fs @@ -4285,6 +4297,9 @@ importers: '@pnpm/store.cafs': specifier: workspace:* version: link:../../store/cafs + '@pnpm/store.index': + specifier: workspace:* + version: link:../../store/index '@pnpm/types': specifier: workspace:* version: link:../../packages/types @@ -4934,6 +4949,9 @@ importers: '@pnpm/resolver-base': specifier: workspace:* version: link:../../resolving/resolver-base + '@pnpm/store.index': + specifier: workspace:* + version: link:../../store/index '@pnpm/tarball-fetcher': specifier: workspace:* version: link:../../fetching/tarball-fetcher @@ -5185,6 +5203,9 @@ importers: '@pnpm/store.cafs': specifier: workspace:* version: link:../../store/cafs + '@pnpm/store.index': + specifier: workspace:* + version: link:../../store/index '@pnpm/test-fixtures': specifier: workspace:* version: link:../../__utils__/test-fixtures @@ -5437,9 +5458,6 @@ importers: '@pnpm/crypto.object-hasher': specifier: workspace:* version: link:../../crypto/object-hasher - '@pnpm/fs.msgpack-file': - specifier: workspace:* - version: link:../../fs/msgpack-file '@pnpm/headless': specifier: workspace:* version: 'link:' @@ -5461,6 +5479,9 @@ importers: '@pnpm/store.cafs': specifier: workspace:* version: link:../../store/cafs + '@pnpm/store.index': + specifier: workspace:* + version: link:../../store/index '@pnpm/test-fixtures': specifier: workspace:* version: link:../../__utils__/test-fixtures @@ -5757,6 +5778,9 @@ importers: '@pnpm/store.cafs': specifier: workspace:* version: link:../../store/cafs + '@pnpm/store.index': + specifier: workspace:* + version: link:../../store/index '@pnpm/types': specifier: workspace:* version: link:../../packages/types @@ -6038,12 +6062,18 @@ importers: '@pnpm/registry-mock': specifier: 'catalog:' version: 5.2.2(verdaccio@6.2.7(encoding@0.1.13)(typanion@3.14.0)) + '@pnpm/store.index': + specifier: workspace:* + version: link:../../store/index '@pnpm/test-fixtures': specifier: workspace:* version: link:../../__utils__/test-fixtures '@pnpm/test-ipc-server': specifier: workspace:* version: link:../../__utils__/test-ipc-server + '@pnpm/worker': + specifier: workspace:* + version: link:../../worker '@pnpm/workspace.filter-packages-from-dir': specifier: workspace:* version: link:../../workspace/filter-packages-from-dir @@ -6702,6 +6732,9 @@ importers: '@pnpm/store.cafs': specifier: workspace:* version: link:../store/cafs + '@pnpm/store.index': + specifier: workspace:* + version: link:../store/index '@pnpm/tabtab': specifier: 'catalog:' version: 0.5.4 @@ -7537,6 +7570,9 @@ importers: '@pnpm/store.cafs': specifier: workspace:* version: link:../../store/cafs + '@pnpm/store.index': + specifier: workspace:* + version: link:../../store/index '@pnpm/types': specifier: workspace:* version: link:../../packages/types @@ -7657,9 +7693,6 @@ importers: '@pnpm/dependency-path': specifier: workspace:* version: link:../../packages/dependency-path - '@pnpm/fs.msgpack-file': - specifier: workspace:* - version: link:../../fs/msgpack-file '@pnpm/lockfile.detect-dep-types': specifier: workspace:* version: link:../../lockfile/detect-dep-types @@ -7690,6 +7723,9 @@ importers: '@pnpm/store.cafs': specifier: workspace:* version: link:../../store/cafs + '@pnpm/store.index': + specifier: workspace:* + version: link:../../store/index '@pnpm/types': specifier: workspace:* version: link:../../packages/types @@ -7757,6 +7793,9 @@ importers: '@pnpm/read-package-json': specifier: workspace:* version: link:../../pkg-manifest/read-package-json + '@pnpm/store.index': + specifier: workspace:* + version: link:../../store/index '@pnpm/store.pkg-finder': specifier: workspace:* version: link:../../store/pkg-finder @@ -8229,6 +8268,9 @@ importers: '@pnpm/read-package-json': specifier: workspace:* version: link:../../pkg-manifest/read-package-json + '@pnpm/store.index': + specifier: workspace:* + version: link:../../store/index '@pnpm/store.pkg-finder': specifier: workspace:* version: link:../../store/pkg-finder @@ -8301,9 +8343,6 @@ importers: store/cafs: dependencies: - '@pnpm/crypto.integrity': - specifier: workspace:* - version: link:../../crypto/integrity '@pnpm/error': specifier: workspace:* version: link:../../packages/error @@ -8421,6 +8460,22 @@ importers: specifier: 'catalog:' version: 0.29.12 + store/index: + dependencies: + msgpackr: + specifier: 'catalog:' + version: 1.11.8 + devDependencies: + '@pnpm/store.index': + specifier: workspace:* + version: 'link:' + '@types/node': + specifier: 'catalog:' + version: 22.19.13 + tempy: + specifier: 'catalog:' + version: 3.0.0 + store/package-store: dependencies: '@pnpm/create-cafs-store': @@ -8435,9 +8490,6 @@ importers: '@pnpm/fetcher-base': specifier: workspace:* version: link:../../fetching/fetcher-base - '@pnpm/fs.msgpack-file': - specifier: workspace:* - version: link:../../fs/msgpack-file '@pnpm/hooks.types': specifier: workspace:* version: link:../../hooks/types @@ -8453,6 +8505,9 @@ importers: '@pnpm/store.cafs': specifier: workspace:* version: link:../cafs + '@pnpm/store.index': + specifier: workspace:* + version: link:../index '@pnpm/types': specifier: workspace:* version: link:../../packages/types @@ -8505,15 +8560,15 @@ importers: '@pnpm/directory-fetcher': specifier: workspace:* version: link:../../fetching/directory-fetcher - '@pnpm/fs.msgpack-file': - specifier: workspace:* - version: link:../../fs/msgpack-file '@pnpm/resolver-base': specifier: workspace:* version: link:../../resolving/resolver-base '@pnpm/store.cafs': specifier: workspace:* version: link:../cafs + '@pnpm/store.index': + specifier: workspace:* + version: link:../index devDependencies: '@pnpm/store.pkg-finder': specifier: workspace:* @@ -8536,9 +8591,6 @@ importers: '@pnpm/error': specifier: workspace:* version: link:../../packages/error - '@pnpm/fs.msgpack-file': - specifier: workspace:* - version: link:../../fs/msgpack-file '@pnpm/get-context': specifier: workspace:* version: link:../../pkg-manager/get-context @@ -8566,6 +8618,9 @@ importers: '@pnpm/store.cafs': specifier: workspace:* version: link:../cafs + '@pnpm/store.index': + specifier: workspace:* + version: link:../index '@pnpm/types': specifier: workspace:* version: link:../../packages/types @@ -8648,9 +8703,6 @@ importers: '@pnpm/error': specifier: workspace:* version: link:../../packages/error - '@pnpm/fs.msgpack-file': - specifier: workspace:* - version: link:../../fs/msgpack-file '@pnpm/graceful-fs': specifier: workspace:* version: link:../../fs/graceful-fs @@ -8669,6 +8721,9 @@ importers: '@pnpm/store.cafs': specifier: workspace:* version: link:../cafs + '@pnpm/store.index': + specifier: workspace:* + version: link:../index '@pnpm/types': specifier: workspace:* version: link:../../packages/types @@ -8715,6 +8770,9 @@ importers: '@pnpm/store-path': specifier: workspace:* version: link:../store-path + '@pnpm/store.index': + specifier: workspace:* + version: link:../index dir-is-case-sensitive: specifier: 'catalog:' version: 2.0.0 @@ -8808,6 +8866,9 @@ importers: '@pnpm/store-controller-types': specifier: workspace:* version: link:../../store/store-controller-types + '@pnpm/store.index': + specifier: workspace:* + version: link:../../store/index devDependencies: '@pnpm/testing.temp-store': specifier: workspace:* @@ -8928,15 +8989,15 @@ importers: '@pnpm/fs.hard-link-dir': specifier: workspace:* version: link:../fs/hard-link-dir - '@pnpm/fs.msgpack-file': - specifier: workspace:* - version: link:../fs/msgpack-file '@pnpm/graceful-fs': specifier: workspace:* version: link:../fs/graceful-fs '@pnpm/store.cafs': specifier: workspace:* version: link:../store/cafs + '@pnpm/store.index': + specifier: workspace:* + version: link:../store/index '@pnpm/symlink-dependency': specifier: workspace:* version: link:../fs/symlink-dependency @@ -9686,8 +9747,8 @@ packages: '@cspell/dict-csharp@4.0.8': resolution: {integrity: sha512-qmk45pKFHSxckl5mSlbHxmDitSsGMlk/XzFgt7emeTJWLNSTUK//MbYAkBNRtfzB4uD7pAFiKgpKgtJrTMRnrQ==} - '@cspell/dict-css@4.1.0': - resolution: {integrity: sha512-bfuvlTeGoK5QgXzzjn+PvqXU5J6mwraIdESNDSvPyplr/EbGFSuvgW3TOuoVNqW4WdDI7eM4tmoP5Dn1ZVgLag==} + '@cspell/dict-css@4.0.19': + resolution: {integrity: sha512-VYHtPnZt/Zd/ATbW3rtexWpBnHUohUrQOHff/2JBhsVgxOrksAxJnLAO43Q1ayLJBJUUwNVo+RU0sx0aaysZfg==} '@cspell/dict-dart@2.3.2': resolution: {integrity: sha512-sUiLW56t9gfZcu8iR/5EUg+KYyRD83Cjl3yjDEA2ApVuJvK1HhX+vn4e4k4YfjpUQMag8XO2AaRhARE09+/rqw==} @@ -9710,11 +9771,11 @@ packages: '@cspell/dict-en-common-misspellings@2.1.12': resolution: {integrity: sha512-14Eu6QGqyksqOd4fYPuRb58lK1Va7FQK9XxFsRKnZU8LhL3N+kj7YKDW+7aIaAN/0WGEqslGP6lGbQzNti8Akw==} - '@cspell/dict-en-gb-mit@3.1.19': - resolution: {integrity: sha512-Ni8eobxohZjVnzpUufMK2oLLRTUwFY4tF7pE2clAmM8ELgBaAheXlm5U52SpWmyhumz/c9fzburEYXKHG4spzQ==} + '@cspell/dict-en-gb-mit@3.1.18': + resolution: {integrity: sha512-AXaMzbaxhSc32MSzKX0cpwT+Thv1vPfxQz1nTly1VHw3wQcwPqVFSqrLOYwa8VNqAPR45583nnhD6iqJ9YESoQ==} - '@cspell/dict-en_us@4.4.30': - resolution: {integrity: sha512-+eVO/VNw8IzQpDIL/SCj+ytd5WbzbHZdU+GAM8eUY2ZU1KTxRw6BoDO+hEFB4cGkD9x+BXm0OKVGSWHNCSGdVw==} + '@cspell/dict-en_us@4.4.29': + resolution: {integrity: sha512-G3B27++9ziRdgbrY/G/QZdFAnMzzx17u8nCb2Xyd4q6luLpzViRM/CW3jA+Mb/cGT5zR/9N+Yz9SrGu1s0bq7g==} '@cspell/dict-filetypes@3.0.16': resolution: {integrity: sha512-SyrtuK2/sx+cr94jOp2/uOAb43ngZEVISUTRj4SR6SfoGULVV1iJS7Drqn7Ul9HJ731QDttwWlOUgcQ+yMRblg==} @@ -9776,10 +9837,10 @@ packages: '@cspell/dict-makefile@1.0.5': resolution: {integrity: sha512-4vrVt7bGiK8Rx98tfRbYo42Xo2IstJkAF4tLLDMNQLkQ86msDlYSKG1ZCk8Abg+EdNcFAjNhXIiNO+w4KflGAQ==} - '@cspell/dict-markdown@2.0.15': - resolution: {integrity: sha512-xz3LJfFCIJaxHu5Msu9UUSev1R1urVkERb5h1Yc5lJyNOkk/SQTSlNyME0Oma1sXlp6dLnIekL8GyeXJYszQ2w==} + '@cspell/dict-markdown@2.0.14': + resolution: {integrity: sha512-uLKPNJsUcumMQTsZZgAK9RgDLyQhUz/uvbQTEkvF/Q4XfC1i/BnA8XrOrd0+Vp6+tPOKyA+omI5LRWfMu5K/Lw==} peerDependencies: - '@cspell/dict-css': ^4.1.0 + '@cspell/dict-css': ^4.0.19 '@cspell/dict-html': ^4.0.14 '@cspell/dict-html-symbol-entities': ^4.0.5 '@cspell/dict-typescript': ^3.2.3 @@ -9790,8 +9851,8 @@ packages: '@cspell/dict-node@5.0.9': resolution: {integrity: sha512-hO+ga+uYZ/WA4OtiMEyKt5rDUlUyu3nXMf8KVEeqq2msYvAPdldKBGH7lGONg6R/rPhv53Rb+0Y1SLdoK1+7wQ==} - '@cspell/dict-npm@5.2.36': - resolution: {integrity: sha512-QeoanpVt8QrSxDQVseXn+qk4sy79TAkbgB0G7q3klsZAByr61gXd1yDQ/0wp8xXsyPx+M/BIj5Qd6B8GnozFLA==} + '@cspell/dict-npm@5.2.35': + resolution: {integrity: sha512-w0VIDUvzHSTt4S9pfvSatApxtCesLMFrDUYD0Wjtw91EBRkB2wVw/RV3q1Ni9Nzpx6pCFpcB7c1xBY8l22cyiQ==} '@cspell/dict-php@4.1.1': resolution: {integrity: sha512-EXelI+4AftmdIGtA8HL8kr4WlUE11OqCSVlnIgZekmTkEGSZdYnkFdiJ5IANSALtlQ1mghKjz+OFqVs6yowgWA==} @@ -9820,8 +9881,8 @@ packages: '@cspell/dict-shell@1.1.2': resolution: {integrity: sha512-WqOUvnwcHK1X61wAfwyXq04cn7KYyskg90j4lLg3sGGKMW9Sq13hs91pqrjC44Q+lQLgCobrTkMDw9Wyl9nRFA==} - '@cspell/dict-software-terms@5.1.24': - resolution: {integrity: sha512-Y+5b5mw8lnovcoyuiVJJX5PpNPMbdpNyILR4wJDsUMWPK2ZVcl0yyG2UYJmevY7jq/+LY48Ai9RSp0ARAlDzEQ==} + '@cspell/dict-software-terms@5.1.23': + resolution: {integrity: sha512-YzxBeqP1j8+hg/+pmw7XHvYrQLO5ttDpZ0rqZiS7y2vnku3Cv1OQZgt9y/3SsTgcUPSCWSRHGgWfrMGqEGNB6g==} '@cspell/dict-sql@2.2.1': resolution: {integrity: sha512-qDHF8MpAYCf4pWU8NKbnVGzkoxMNrFqBHyG/dgrlic5EQiKANCLELYtGlX5auIMDLmTf1inA0eNtv74tyRJ/vg==} @@ -10076,8 +10137,8 @@ packages: resolution: {integrity: sha512-4h4MVF8pmBsncB60r0wSJiIeUKTSD4m7FmTFThG8RHlsg9ajqckLm9OraguFGZE4vVdpiI1Q4+hFnisopmG6gQ==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} - '@eslint/js@9.39.3': - resolution: {integrity: sha512-1B1VkCq6FuUNlQvlBYb+1jDu/gV297TIs/OeiaSR9l1H27SVW55ONE1e1Vp16NqP683+xEGzxYtv4XCiDPaQiw==} + '@eslint/js@9.39.2': + resolution: {integrity: sha512-q1mjIoW1VX4IvSocvM/vbTiveKC4k9eLrajNEuSsmjymSDEbpGddtpfOoN7YGAqBK3NG+uqo8ia4PDTt8buCYA==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} '@eslint/object-schema@2.1.7': @@ -11268,6 +11329,14 @@ packages: '@types/yarnpkg__lockfile@1.1.9': resolution: {integrity: sha512-GD4Fk15UoP5NLCNor51YdfL9MSdldKCqOC9EssrRw3HVfar9wUZ5y8Lfnp+qVD6hIinLr8ygklDYnmlnlQo12Q==} + '@typescript-eslint/eslint-plugin@8.56.0': + resolution: {integrity: sha512-lRyPDLzNCuae71A3t9NEINBiTn7swyOhvUj3MyUOxb8x6g6vPEFoOU+ZRmGMusNC3X3YMhqMIX7i8ShqhT74Pw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + '@typescript-eslint/parser': ^8.56.0 + eslint: ^8.57.0 || ^9.0.0 || ^10.0.0 + typescript: '>=4.8.4 <6.0.0' + '@typescript-eslint/eslint-plugin@8.56.1': resolution: {integrity: sha512-Jz9ZztpB37dNC+HU2HI28Bs9QXpzCz+y/twHOwhyrIRdbuVDxSytJNDl6z/aAKlaRIwC7y8wJdkBv7FxYGgi0A==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} @@ -11276,29 +11345,52 @@ packages: eslint: ^8.57.0 || ^9.0.0 || ^10.0.0 typescript: '>=4.8.4 <6.0.0' - '@typescript-eslint/parser@8.56.1': - resolution: {integrity: sha512-klQbnPAAiGYFyI02+znpBRLyjL4/BrBd0nyWkdC0s/6xFLkXYQ8OoRrSkqacS1ddVxf/LDyODIKbQ5TgKAf/Fg==} + '@typescript-eslint/parser@8.56.0': + resolution: {integrity: sha512-IgSWvLobTDOjnaxAfDTIHaECbkNlAlKv2j5SjpB2v7QHKv1FIfjwMy8FsDbVfDX/KjmCmYICcw7uGaXLhtsLNg==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} peerDependencies: eslint: ^8.57.0 || ^9.0.0 || ^10.0.0 typescript: '>=4.8.4 <6.0.0' + '@typescript-eslint/project-service@8.56.0': + resolution: {integrity: sha512-M3rnyL1vIQOMeWxTWIW096/TtVP+8W3p/XnaFflhmcFp+U4zlxUxWj4XwNs6HbDeTtN4yun0GNTTDBw/SvufKg==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + typescript: '>=4.8.4 <6.0.0' + '@typescript-eslint/project-service@8.56.1': resolution: {integrity: sha512-TAdqQTzHNNvlVFfR+hu2PDJrURiwKsUvxFn1M0h95BB8ah5jejas08jUWG4dBA68jDMI988IvtfdAI53JzEHOQ==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} peerDependencies: typescript: '>=4.8.4 <6.0.0' + '@typescript-eslint/scope-manager@8.56.0': + resolution: {integrity: sha512-7UiO/XwMHquH+ZzfVCfUNkIXlp/yQjjnlYUyYz7pfvlK3/EyyN6BK+emDmGNyQLBtLGaYrTAI6KOw8tFucWL2w==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + '@typescript-eslint/scope-manager@8.56.1': resolution: {integrity: sha512-YAi4VDKcIZp0O4tz/haYKhmIDZFEUPOreKbfdAN3SzUDMcPhJ8QI99xQXqX+HoUVq8cs85eRKnD+rne2UAnj2w==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + '@typescript-eslint/tsconfig-utils@8.56.0': + resolution: {integrity: sha512-bSJoIIt4o3lKXD3xmDh9chZcjCz5Lk8xS7Rxn+6l5/pKrDpkCwtQNQQwZ2qRPk7TkUYhrq3WPIHXOXlbXP0itg==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + typescript: '>=4.8.4 <6.0.0' + '@typescript-eslint/tsconfig-utils@8.56.1': resolution: {integrity: sha512-qOtCYzKEeyr3aR9f28mPJqBty7+DBqsdd63eO0yyDwc6vgThj2UjWfJIcsFeSucYydqcuudMOprZ+x1SpF3ZuQ==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} peerDependencies: typescript: '>=4.8.4 <6.0.0' + '@typescript-eslint/type-utils@8.56.0': + resolution: {integrity: sha512-qX2L3HWOU2nuDs6GzglBeuFXviDODreS58tLY/BALPC7iu3Fa+J7EOTwnX9PdNBxUI7Uh0ntP0YWGnxCkXzmfA==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + eslint: ^8.57.0 || ^9.0.0 || ^10.0.0 + typescript: '>=4.8.4 <6.0.0' + '@typescript-eslint/type-utils@8.56.1': resolution: {integrity: sha512-yB/7dxi7MgTtGhZdaHCemf7PuwrHMenHjmzgUW1aJpO+bBU43OycnM3Wn+DdvDO/8zzA9HlhaJ0AUGuvri4oGg==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} @@ -11306,16 +11398,33 @@ packages: eslint: ^8.57.0 || ^9.0.0 || ^10.0.0 typescript: '>=4.8.4 <6.0.0' + '@typescript-eslint/types@8.56.0': + resolution: {integrity: sha512-DBsLPs3GsWhX5HylbP9HNG15U0bnwut55Lx12bHB9MpXxQ+R5GC8MwQe+N1UFXxAeQDvEsEDY6ZYwX03K7Z6HQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + '@typescript-eslint/types@8.56.1': resolution: {integrity: sha512-dbMkdIUkIkchgGDIv7KLUpa0Mda4IYjo4IAMJUZ+3xNoUXxMsk9YtKpTHSChRS85o+H9ftm51gsK1dZReY9CVw==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + '@typescript-eslint/typescript-estree@8.56.0': + resolution: {integrity: sha512-ex1nTUMWrseMltXUHmR2GAQ4d+WjkZCT4f+4bVsps8QEdh0vlBsaCokKTPlnqBFqqGaxilDNJG7b8dolW2m43Q==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + typescript: '>=4.8.4 <6.0.0' + '@typescript-eslint/typescript-estree@8.56.1': resolution: {integrity: sha512-qzUL1qgalIvKWAf9C1HpvBjif+Vm6rcT5wZd4VoMb9+Km3iS3Cv9DY6dMRMDtPnwRAFyAi7YXJpTIEXLvdfPxg==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} peerDependencies: typescript: '>=4.8.4 <6.0.0' + '@typescript-eslint/utils@8.56.0': + resolution: {integrity: sha512-RZ3Qsmi2nFGsS+n+kjLAYDPVlrzf7UhTffrDIKr+h2yzAlYP/y5ZulU0yeDEPItos2Ph46JAL5P/On3pe7kDIQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + eslint: ^8.57.0 || ^9.0.0 || ^10.0.0 + typescript: '>=4.8.4 <6.0.0' + '@typescript-eslint/utils@8.56.1': resolution: {integrity: sha512-HPAVNIME3tABJ61siYlHzSWCGtOoeP2RTIaHXFMPqjrQKCGB9OgUVdiNgH7TJS2JNIQ5qQ4RsAUDuGaGme/KOA==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} @@ -11323,6 +11432,10 @@ packages: eslint: ^8.57.0 || ^9.0.0 || ^10.0.0 typescript: '>=4.8.4 <6.0.0' + '@typescript-eslint/visitor-keys@8.56.0': + resolution: {integrity: sha512-q+SL+b+05Ud6LbEE35qe4A99P+htKTKVbyiNEe45eCbJFyh/HVK9QXwlrbz+Q4L8SOW4roxSVwXYj4DMBT7Ieg==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + '@typescript-eslint/visitor-keys@8.56.1': resolution: {integrity: sha512-KiROIzYdEV85YygXw6BI/Dx4fnBlFQu6Mq4QE4MOH9fFnhohw6wX/OAvDY2/C+ut0I3RSPKenvZJIVYqJNkhEw==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} @@ -11907,8 +12020,8 @@ packages: bare-buffer: optional: true - bare-os@3.7.0: - resolution: {integrity: sha512-64Rcwj8qlnTZU8Ps6JJEdSmxBEUGgI7g8l+lMtsJLl4IsfTcHMTfJ188u2iGV6P6YPRZrtv72B2kjn+hp+Yv3g==} + bare-os@3.7.1: + resolution: {integrity: sha512-ebvMaS5BgZKmJlvuWh14dg9rbUI84QeV3WlWn6Ph6lFI8jJoh7ADtVTyD2c93euwbe+zgi0DVrl4YmqXeM9aIA==} engines: {bare: '>=1.14.0'} bare-path@3.0.0: @@ -12283,8 +12396,8 @@ packages: resolution: {integrity: sha512-bmkUukX8wAOjHdN26xj5c4ctEV22TQ7dQYhSmuckKhToXrkUn0iIaolHdIxYYqD55nhpSPA9zPQ1yP57GdXP2A==} engines: {node: '>= 0.6.x'} - comment-json@4.6.1: - resolution: {integrity: sha512-kdBIsBGqD/sAeqvzeOhBvO/bhtpbfbIU/2lw7bp182FV1cVlY7gr1Jf3Q1I+NOsCk8e4gF5Sl9iYH5cNvVmx5w==} + comment-json@4.5.1: + resolution: {integrity: sha512-taEtr3ozUmOB7it68Jll7s0Pwm+aoiHyXKrEC8SEodL4rNpdfDLqa7PfBlrgFoCNNdR8ImL+muti5IGvktJAAg==} engines: {node: '>= 6'} comment-parser@1.4.5: @@ -12376,8 +12489,8 @@ packages: cosmiconfig: '>=9' typescript: '>=5' - cosmiconfig@9.0.1: - resolution: {integrity: sha512-hr4ihw+DBqcvrsEDioRO31Z17x71pUYoNe/4h6Z0wB72p7MU7/9gH8Q3s12NFhHPfYBBOV3qyfUxmr/Yn3shnQ==} + cosmiconfig@9.0.0: + resolution: {integrity: sha512-itvL5h8RETACmOTFc4UfIyB2RfEHi71Ax6E/PivVxq9NseKbOWpeyHEOIbmAw1rs8Ak0VursQNww7lf7YtUwzg==} engines: {node: '>=14'} peerDependencies: typescript: '>=4.9.5' @@ -12732,8 +12845,8 @@ packages: resolution: {integrity: sha512-w+5mJ3GuFL+NjVtJlvydShqE1eN3h3PbI7/5LAsYJP/2qtuMXjfL2LpHSRqo4b4eSF5K/DH1JXKUAHSB2UW50g==} engines: {node: '>= 0.4'} - es-toolkit@1.45.0: - resolution: {integrity: sha512-RArCX+Zea16+R1jg4mH223Z8p/ivbJjIkU3oC6ld2bdUfmDxiCkFYSi9zLOR2anucWJUeH4Djnzgd0im0nD3dw==} + es-toolkit@1.44.0: + resolution: {integrity: sha512-6penXeZalaV88MM3cGkFZZfOoLGWshWWfdy0tWw/RlVVyhvMaWSBTOvXNeiW3e5FwdS5ePW0LGEu17zT139ktg==} esbuild@0.25.12: resolution: {integrity: sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==} @@ -12843,8 +12956,8 @@ packages: resolution: {integrity: sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==} engines: {node: ^20.19.0 || ^22.13.0 || >=24} - eslint@9.39.3: - resolution: {integrity: sha512-VmQ+sifHUbI/IcSopBCF/HO3YiHQx/AVd3UVyYL6weuwW+HvON9VYn5l6Zl1WZzPWXPNZrSQpxwkkZ/VuvJZzg==} + eslint@9.39.2: + resolution: {integrity: sha512-LEyamqS7W5HB3ujJyvi0HQK/dtVINZvd5mAAp9eT5S/ujByGjiZLCzPcHVzuXbpJDJF/cxwHlfceVUDZ2lnSTw==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} hasBin: true peerDependencies: @@ -13063,8 +13176,8 @@ packages: resolution: {integrity: sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==} engines: {node: '>=16'} - flatted@3.3.4: - resolution: {integrity: sha512-3+mMldrTAPdta5kjX2G2J7iX4zxtnwpdA8Tr2ZSjkyPSanvbZAcy6flmtnXbEybHrDcU9641lxrMfFuUxVz9vA==} + flatted@3.3.3: + resolution: {integrity: sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg==} flush-write-stream@1.1.1: resolution: {integrity: sha512-3Z4XhFZ3992uIq0XOqb9AreonueSYphE6oYbpt5+3u06JWklbsPkNv3ZKkP9Bz/r+1MWCaMoSQ28P85+1Yc77w==} @@ -14504,6 +14617,10 @@ packages: resolution: {integrity: sha512-Brg/fp/iAVDOQoHxkuN5bEYhyQlZhxddI78yWsCbeEwTHXQjlNLtiJDUsp1GIptVqMI7/gkJMz4vVAc01mpoBw==} engines: {node: '>=10'} + minimatch@9.0.9: + resolution: {integrity: sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==} + engines: {node: '>=16 || 14 >=14.17'} + minimist-options@4.1.0: resolution: {integrity: sha512-Q4r8ghd80yhO/0j1O3B2BjweX3fiHg9cdOwjJd2J76Q135c+NDxGCqdYKQ1SKBuFfgWbAUzBfvYjPUEeNgqN1A==} engines: {node: '>= 6'} @@ -16222,8 +16339,8 @@ packages: types-ramda@0.29.10: resolution: {integrity: sha512-5PJiW/eiTPyXXBYGZOYGezMl6qj7keBiZheRwfjJZY26QPHsNrjfJnz0mru6oeqqoTHOni893Jfd6zyUXfQRWg==} - typescript-eslint@8.56.1: - resolution: {integrity: sha512-U4lM6pjmBX7J5wk4szltF7I1cGBHXZopnAXCMXb3+fZ3B/0Z3hq3wS/CCUB2NZBNAExK92mCU2tEohWuwVMsDQ==} + typescript-eslint@8.56.0: + resolution: {integrity: sha512-c7toRLrotJ9oixgdW7liukZpsnq5CZ7PuKztubGYlNppuTqhIoWfhgHo/7EU0v06gS2l/x0i2NEFK1qMIf0rIg==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} peerDependencies: eslint: ^8.57.0 || ^9.0.0 || ^10.0.0 @@ -17047,8 +17164,8 @@ snapshots: '@commitlint/execute-rule': 20.0.0 '@commitlint/resolve-extends': 20.4.0 '@commitlint/types': 20.4.0 - cosmiconfig: 9.0.1(typescript@5.9.2) - cosmiconfig-typescript-loader: 6.2.0(@types/node@22.19.13)(cosmiconfig@9.0.1(typescript@5.9.2))(typescript@5.9.2) + cosmiconfig: 9.0.0(typescript@5.9.2) + cosmiconfig-typescript-loader: 6.2.0(@types/node@22.19.13)(cosmiconfig@9.0.0(typescript@5.9.2))(typescript@5.9.2) is-plain-obj: 4.1.0 lodash.mergewith: 4.6.2 picocolors: 1.1.1 @@ -17129,7 +17246,7 @@ snapshots: '@cspell/dict-cpp': 6.0.15 '@cspell/dict-cryptocurrencies': 5.0.5 '@cspell/dict-csharp': 4.0.8 - '@cspell/dict-css': 4.1.0 + '@cspell/dict-css': 4.0.19 '@cspell/dict-dart': 2.3.2 '@cspell/dict-data-science': 2.0.13 '@cspell/dict-django': 4.1.6 @@ -17137,8 +17254,8 @@ snapshots: '@cspell/dict-dotnet': 5.0.12 '@cspell/dict-elixir': 4.0.8 '@cspell/dict-en-common-misspellings': 2.1.12 - '@cspell/dict-en-gb-mit': 3.1.19 - '@cspell/dict-en_us': 4.4.30 + '@cspell/dict-en-gb-mit': 3.1.18 + '@cspell/dict-en_us': 4.4.29 '@cspell/dict-filetypes': 3.0.16 '@cspell/dict-flutter': 1.1.1 '@cspell/dict-fonts': 4.0.5 @@ -17159,10 +17276,10 @@ snapshots: '@cspell/dict-lorem-ipsum': 4.0.5 '@cspell/dict-lua': 4.0.8 '@cspell/dict-makefile': 1.0.5 - '@cspell/dict-markdown': 2.0.15(@cspell/dict-css@4.1.0)(@cspell/dict-html-symbol-entities@4.0.5)(@cspell/dict-html@4.0.14)(@cspell/dict-typescript@3.2.3) + '@cspell/dict-markdown': 2.0.14(@cspell/dict-css@4.0.19)(@cspell/dict-html-symbol-entities@4.0.5)(@cspell/dict-html@4.0.14)(@cspell/dict-typescript@3.2.3) '@cspell/dict-monkeyc': 1.0.12 '@cspell/dict-node': 5.0.9 - '@cspell/dict-npm': 5.2.36 + '@cspell/dict-npm': 5.2.35 '@cspell/dict-php': 4.1.1 '@cspell/dict-powershell': 5.0.15 '@cspell/dict-public-licenses': 2.0.16 @@ -17172,7 +17289,7 @@ snapshots: '@cspell/dict-rust': 4.1.2 '@cspell/dict-scala': 5.0.9 '@cspell/dict-shell': 1.1.2 - '@cspell/dict-software-terms': 5.1.24 + '@cspell/dict-software-terms': 5.1.23 '@cspell/dict-sql': 2.2.1 '@cspell/dict-svelte': 1.0.7 '@cspell/dict-swift': 2.0.6 @@ -17212,7 +17329,7 @@ snapshots: '@cspell/dict-csharp@4.0.8': {} - '@cspell/dict-css@4.1.0': {} + '@cspell/dict-css@4.0.19': {} '@cspell/dict-dart@2.3.2': {} @@ -17228,9 +17345,9 @@ snapshots: '@cspell/dict-en-common-misspellings@2.1.12': {} - '@cspell/dict-en-gb-mit@3.1.19': {} + '@cspell/dict-en-gb-mit@3.1.18': {} - '@cspell/dict-en_us@4.4.30': {} + '@cspell/dict-en_us@4.4.29': {} '@cspell/dict-filetypes@3.0.16': {} @@ -17272,9 +17389,9 @@ snapshots: '@cspell/dict-makefile@1.0.5': {} - '@cspell/dict-markdown@2.0.15(@cspell/dict-css@4.1.0)(@cspell/dict-html-symbol-entities@4.0.5)(@cspell/dict-html@4.0.14)(@cspell/dict-typescript@3.2.3)': + '@cspell/dict-markdown@2.0.14(@cspell/dict-css@4.0.19)(@cspell/dict-html-symbol-entities@4.0.5)(@cspell/dict-html@4.0.14)(@cspell/dict-typescript@3.2.3)': dependencies: - '@cspell/dict-css': 4.1.0 + '@cspell/dict-css': 4.0.19 '@cspell/dict-html': 4.0.14 '@cspell/dict-html-symbol-entities': 4.0.5 '@cspell/dict-typescript': 3.2.3 @@ -17283,7 +17400,7 @@ snapshots: '@cspell/dict-node@5.0.9': {} - '@cspell/dict-npm@5.2.36': {} + '@cspell/dict-npm@5.2.35': {} '@cspell/dict-php@4.1.1': {} @@ -17305,7 +17422,7 @@ snapshots: '@cspell/dict-shell@1.1.2': {} - '@cspell/dict-software-terms@5.1.24': {} + '@cspell/dict-software-terms@5.1.23': {} '@cspell/dict-sql@2.2.1': {} @@ -17454,9 +17571,9 @@ snapshots: '@esbuild/win32-x64@0.25.12': optional: true - '@eslint-community/eslint-utils@4.9.1(eslint@9.39.3(jiti@2.6.1))': + '@eslint-community/eslint-utils@4.9.1(eslint@9.39.2(jiti@2.6.1))': dependencies: - eslint: 9.39.3(jiti@2.6.1) + eslint: 9.39.2(jiti@2.6.1) eslint-visitor-keys: 3.4.3 '@eslint-community/regexpp@4.12.2': {} @@ -17491,7 +17608,7 @@ snapshots: transitivePeerDependencies: - supports-color - '@eslint/js@9.39.3': {} + '@eslint/js@9.39.2': {} '@eslint/object-schema@2.1.7': {} @@ -17545,7 +17662,7 @@ snapshots: '@jest/console@30.2.0': dependencies: '@jest/types': 30.2.0 - '@types/node': 22.19.13 + '@types/node': 25.3.3 chalk: 4.1.2 jest-message-util: 30.2.0 jest-util: 30.2.0 @@ -17559,14 +17676,14 @@ snapshots: '@jest/test-result': 30.2.0 '@jest/transform': 30.2.0(@babel/types@7.29.0) '@jest/types': 30.2.0 - '@types/node': 22.19.13 + '@types/node': 25.3.3 ansi-escapes: 4.3.2 chalk: 4.1.2 ci-info: 4.4.0 exit-x: 0.2.2 graceful-fs: 4.2.11(patch_hash=68ebc232025360cb3dcd3081f4067f4e9fc022ab6b6f71a3230e86c7a5b337d1) jest-changed-files: 30.2.0 - jest-config: 30.2.0(@babel/types@7.29.0)(@types/node@22.19.13) + jest-config: 30.2.0(@babel/types@7.29.0)(@types/node@25.3.3) jest-haste-map: 30.2.0 jest-message-util: 30.2.0 jest-regex-util: 30.0.1 @@ -17601,7 +17718,7 @@ snapshots: dependencies: '@jest/fake-timers': 30.2.0 '@jest/types': 30.2.0 - '@types/node': 22.19.13 + '@types/node': 25.3.3 jest-mock: 30.2.0 '@jest/expect-utils@30.0.5': @@ -17639,7 +17756,7 @@ snapshots: dependencies: '@jest/types': 30.2.0 '@sinonjs/fake-timers': 13.0.5 - '@types/node': 22.19.13 + '@types/node': 25.3.3 jest-message-util: 30.2.0 jest-mock: 30.2.0 jest-util: 30.2.0 @@ -17668,7 +17785,7 @@ snapshots: '@jest/pattern@30.0.1': dependencies: - '@types/node': 22.19.13 + '@types/node': 25.3.3 jest-regex-util: 30.0.1 '@jest/reporters@30.2.0(@babel/types@7.29.0)': @@ -17679,7 +17796,7 @@ snapshots: '@jest/transform': 30.2.0(@babel/types@7.29.0) '@jest/types': 30.2.0 '@jridgewell/trace-mapping': 0.3.31 - '@types/node': 22.19.13 + '@types/node': 25.3.3 chalk: 4.1.2 collect-v8-coverage: 1.0.3 exit-x: 0.2.2 @@ -17809,7 +17926,7 @@ snapshots: '@jest/schemas': 30.0.5 '@types/istanbul-lib-coverage': 2.0.6 '@types/istanbul-reports': 3.0.4 - '@types/node': 22.19.13 + '@types/node': 25.3.3 '@types/yargs': 17.0.35 chalk: 4.1.2 @@ -19404,10 +19521,10 @@ snapshots: dependencies: '@sinonjs/commons': 3.0.1 - '@stylistic/eslint-plugin@4.4.1(eslint@9.39.3(jiti@2.6.1))(typescript@5.9.2)': + '@stylistic/eslint-plugin@4.4.1(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.2)': dependencies: - '@typescript-eslint/utils': 8.56.1(eslint@9.39.3(jiti@2.6.1))(typescript@5.9.2) - eslint: 9.39.3(jiti@2.6.1) + '@typescript-eslint/utils': 8.56.1(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.2) + eslint: 9.39.2(jiti@2.6.1) eslint-visitor-keys: 4.2.1 espree: 10.4.0 estraverse: 5.3.0 @@ -19506,7 +19623,7 @@ snapshots: '@types/isexe@2.0.2': dependencies: - '@types/node': 22.19.13 + '@types/node': 25.3.3 '@types/istanbul-lib-coverage@2.0.6': {} @@ -19608,7 +19725,7 @@ snapshots: '@types/responselike@1.0.0': dependencies: - '@types/node': 22.19.13 + '@types/node': 25.3.3 '@types/responselike@1.0.3': dependencies: @@ -19639,7 +19756,7 @@ snapshots: '@types/touch@3.1.5': dependencies: - '@types/node': 22.19.13 + '@types/node': 25.3.3 '@types/treeify@1.0.3': {} @@ -19661,15 +19778,15 @@ snapshots: '@types/yarnpkg__lockfile@1.1.9': {} - '@typescript-eslint/eslint-plugin@8.56.1(@typescript-eslint/parser@8.56.1(eslint@9.39.3(jiti@2.6.1))(typescript@5.9.2))(eslint@9.39.3(jiti@2.6.1))(typescript@5.9.2)': + '@typescript-eslint/eslint-plugin@8.56.0(@typescript-eslint/parser@8.56.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.2))(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.2)': dependencies: '@eslint-community/regexpp': 4.12.2 - '@typescript-eslint/parser': 8.56.1(eslint@9.39.3(jiti@2.6.1))(typescript@5.9.2) - '@typescript-eslint/scope-manager': 8.56.1 - '@typescript-eslint/type-utils': 8.56.1(eslint@9.39.3(jiti@2.6.1))(typescript@5.9.2) - '@typescript-eslint/utils': 8.56.1(eslint@9.39.3(jiti@2.6.1))(typescript@5.9.2) - '@typescript-eslint/visitor-keys': 8.56.1 - eslint: 9.39.3(jiti@2.6.1) + '@typescript-eslint/parser': 8.56.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.2) + '@typescript-eslint/scope-manager': 8.56.0 + '@typescript-eslint/type-utils': 8.56.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.2) + '@typescript-eslint/utils': 8.56.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.2) + '@typescript-eslint/visitor-keys': 8.56.0 + eslint: 9.39.2(jiti@2.6.1) ignore: 7.0.5 natural-compare: 1.4.0 ts-api-utils: 2.4.0(typescript@5.9.2) @@ -19677,14 +19794,40 @@ snapshots: transitivePeerDependencies: - supports-color - '@typescript-eslint/parser@8.56.1(eslint@9.39.3(jiti@2.6.1))(typescript@5.9.2)': + '@typescript-eslint/eslint-plugin@8.56.1(@typescript-eslint/parser@8.56.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.2))(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.2)': dependencies: + '@eslint-community/regexpp': 4.12.2 + '@typescript-eslint/parser': 8.56.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.2) '@typescript-eslint/scope-manager': 8.56.1 - '@typescript-eslint/types': 8.56.1 - '@typescript-eslint/typescript-estree': 8.56.1(typescript@5.9.2) + '@typescript-eslint/type-utils': 8.56.1(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.2) + '@typescript-eslint/utils': 8.56.1(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.2) '@typescript-eslint/visitor-keys': 8.56.1 + eslint: 9.39.2(jiti@2.6.1) + ignore: 7.0.5 + natural-compare: 1.4.0 + ts-api-utils: 2.4.0(typescript@5.9.2) + typescript: 5.9.2 + transitivePeerDependencies: + - supports-color + optional: true + + '@typescript-eslint/parser@8.56.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.2)': + dependencies: + '@typescript-eslint/scope-manager': 8.56.0 + '@typescript-eslint/types': 8.56.0 + '@typescript-eslint/typescript-estree': 8.56.0(typescript@5.9.2) + '@typescript-eslint/visitor-keys': 8.56.0 + debug: 4.4.3 + eslint: 9.39.2(jiti@2.6.1) + typescript: 5.9.2 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/project-service@8.56.0(typescript@5.9.2)': + dependencies: + '@typescript-eslint/tsconfig-utils': 8.56.1(typescript@5.9.2) + '@typescript-eslint/types': 8.56.1 debug: 4.4.3 - eslint: 9.39.3(jiti@2.6.1) typescript: 5.9.2 transitivePeerDependencies: - supports-color @@ -19698,29 +19841,68 @@ snapshots: transitivePeerDependencies: - supports-color + '@typescript-eslint/scope-manager@8.56.0': + dependencies: + '@typescript-eslint/types': 8.56.0 + '@typescript-eslint/visitor-keys': 8.56.0 + '@typescript-eslint/scope-manager@8.56.1': dependencies: '@typescript-eslint/types': 8.56.1 '@typescript-eslint/visitor-keys': 8.56.1 + '@typescript-eslint/tsconfig-utils@8.56.0(typescript@5.9.2)': + dependencies: + typescript: 5.9.2 + '@typescript-eslint/tsconfig-utils@8.56.1(typescript@5.9.2)': dependencies: typescript: 5.9.2 - '@typescript-eslint/type-utils@8.56.1(eslint@9.39.3(jiti@2.6.1))(typescript@5.9.2)': + '@typescript-eslint/type-utils@8.56.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.2)': dependencies: - '@typescript-eslint/types': 8.56.1 - '@typescript-eslint/typescript-estree': 8.56.1(typescript@5.9.2) - '@typescript-eslint/utils': 8.56.1(eslint@9.39.3(jiti@2.6.1))(typescript@5.9.2) + '@typescript-eslint/types': 8.56.0 + '@typescript-eslint/typescript-estree': 8.56.0(typescript@5.9.2) + '@typescript-eslint/utils': 8.56.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.2) debug: 4.4.3 - eslint: 9.39.3(jiti@2.6.1) + eslint: 9.39.2(jiti@2.6.1) ts-api-utils: 2.4.0(typescript@5.9.2) typescript: 5.9.2 transitivePeerDependencies: - supports-color + '@typescript-eslint/type-utils@8.56.1(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.2)': + dependencies: + '@typescript-eslint/types': 8.56.1 + '@typescript-eslint/typescript-estree': 8.56.1(typescript@5.9.2) + '@typescript-eslint/utils': 8.56.1(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.2) + debug: 4.4.3 + eslint: 9.39.2(jiti@2.6.1) + ts-api-utils: 2.4.0(typescript@5.9.2) + typescript: 5.9.2 + transitivePeerDependencies: + - supports-color + optional: true + + '@typescript-eslint/types@8.56.0': {} + '@typescript-eslint/types@8.56.1': {} + '@typescript-eslint/typescript-estree@8.56.0(typescript@5.9.2)': + dependencies: + '@typescript-eslint/project-service': 8.56.0(typescript@5.9.2) + '@typescript-eslint/tsconfig-utils': 8.56.0(typescript@5.9.2) + '@typescript-eslint/types': 8.56.0 + '@typescript-eslint/visitor-keys': 8.56.0 + debug: 4.4.3 + minimatch: 9.0.9 + semver: 7.7.4 + tinyglobby: 0.2.15 + ts-api-utils: 2.4.0(typescript@5.9.2) + typescript: 5.9.2 + transitivePeerDependencies: + - supports-color + '@typescript-eslint/typescript-estree@8.56.1(typescript@5.9.2)': dependencies: '@typescript-eslint/project-service': 8.56.1(typescript@5.9.2) @@ -19736,17 +19918,33 @@ snapshots: transitivePeerDependencies: - supports-color - '@typescript-eslint/utils@8.56.1(eslint@9.39.3(jiti@2.6.1))(typescript@5.9.2)': + '@typescript-eslint/utils@8.56.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.2)': dependencies: - '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.3(jiti@2.6.1)) - '@typescript-eslint/scope-manager': 8.56.1 - '@typescript-eslint/types': 8.56.1 - '@typescript-eslint/typescript-estree': 8.56.1(typescript@5.9.2) - eslint: 9.39.3(jiti@2.6.1) + '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.2(jiti@2.6.1)) + '@typescript-eslint/scope-manager': 8.56.0 + '@typescript-eslint/types': 8.56.0 + '@typescript-eslint/typescript-estree': 8.56.0(typescript@5.9.2) + eslint: 9.39.2(jiti@2.6.1) typescript: 5.9.2 transitivePeerDependencies: - supports-color + '@typescript-eslint/utils@8.56.1(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.2)': + dependencies: + '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.2(jiti@2.6.1)) + '@typescript-eslint/scope-manager': 8.56.1 + '@typescript-eslint/types': 8.56.1 + '@typescript-eslint/typescript-estree': 8.56.1(typescript@5.9.2) + eslint: 9.39.2(jiti@2.6.1) + typescript: 5.9.2 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/visitor-keys@8.56.0': + dependencies: + '@typescript-eslint/types': 8.56.0 + eslint-visitor-keys: 5.0.1 + '@typescript-eslint/visitor-keys@8.56.1': dependencies: '@typescript-eslint/types': 8.56.1 @@ -20049,7 +20247,7 @@ snapshots: cross-spawn: 7.0.6 diff: 8.0.3 dotenv: 16.6.1 - es-toolkit: 1.45.0 + es-toolkit: 1.44.0 fast-glob: 3.3.3 got: 11.8.6 hpagent: 1.2.0 @@ -20455,11 +20653,11 @@ snapshots: - bare-abort-controller - react-native-b4a - bare-os@3.7.0: {} + bare-os@3.7.1: {} bare-path@3.0.0: dependencies: - bare-os: 3.7.0 + bare-os: 3.7.1 bare-stream@2.8.0(bare-events@2.8.2): dependencies: @@ -20522,7 +20720,7 @@ snapshots: http-errors: 2.0.1 iconv-lite: 0.7.2 on-finished: 2.4.1 - qs: 6.14.2 + qs: 6.15.0 raw-body: 3.0.2 type-is: 2.0.1 transitivePeerDependencies: @@ -20856,7 +21054,7 @@ snapshots: dependencies: graceful-readlink: 1.0.1 - comment-json@4.6.1: + comment-json@4.5.1: dependencies: array-timsort: 1.0.3 core-util-is: 1.0.3 @@ -20949,14 +21147,14 @@ snapshots: object-assign: 4.1.1 vary: 1.1.2 - cosmiconfig-typescript-loader@6.2.0(@types/node@22.19.13)(cosmiconfig@9.0.1(typescript@5.9.2))(typescript@5.9.2): + cosmiconfig-typescript-loader@6.2.0(@types/node@22.19.13)(cosmiconfig@9.0.0(typescript@5.9.2))(typescript@5.9.2): dependencies: '@types/node': 22.19.13 - cosmiconfig: 9.0.1(typescript@5.9.2) + cosmiconfig: 9.0.0(typescript@5.9.2) jiti: 2.6.1 typescript: 5.9.2 - cosmiconfig@9.0.1(typescript@5.9.2): + cosmiconfig@9.0.0(typescript@5.9.2): dependencies: env-paths: 2.2.1 import-fresh: 3.3.1 @@ -20985,7 +21183,7 @@ snapshots: cspell-config-lib@9.2.0: dependencies: '@cspell/cspell-types': 9.2.0 - comment-json: 4.6.1 + comment-json: 4.5.1 smol-toml: 1.6.0 yaml: 2.8.2 @@ -21028,7 +21226,7 @@ snapshots: '@cspell/strong-weak-map': 9.2.0 '@cspell/url': 9.2.0 clear-module: 4.1.2 - comment-json: 4.6.1 + comment-json: 4.5.1 cspell-config-lib: 9.2.0 cspell-dictionary: 9.2.0 cspell-glob: 9.2.0 @@ -21067,7 +21265,7 @@ snapshots: cspell-io: 9.2.0 cspell-lib: 9.2.0 fast-json-stable-stringify: 2.1.0 - flatted: 3.3.4 + flatted: 3.3.3 semver: 7.7.4 tinyglobby: 0.2.15 @@ -21379,7 +21577,7 @@ snapshots: is-date-object: 1.1.0 is-symbol: 1.1.1 - es-toolkit@1.45.0: {} + es-toolkit@1.44.0: {} esbuild@0.25.12: optionalDependencies: @@ -21420,9 +21618,9 @@ snapshots: escape-string-regexp@5.0.0: {} - eslint-compat-utils@0.5.1(eslint@9.39.3(jiti@2.6.1)): + eslint-compat-utils@0.5.1(eslint@9.39.2(jiti@2.6.1)): dependencies: - eslint: 9.39.3(jiti@2.6.1) + eslint: 9.39.2(jiti@2.6.1) semver: 7.7.4 eslint-import-context@0.1.9(unrs-resolver@1.11.1): @@ -21432,19 +21630,19 @@ snapshots: optionalDependencies: unrs-resolver: 1.11.1 - eslint-plugin-es-x@7.8.0(eslint@9.39.3(jiti@2.6.1)): + eslint-plugin-es-x@7.8.0(eslint@9.39.2(jiti@2.6.1)): dependencies: - '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.3(jiti@2.6.1)) + '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.2(jiti@2.6.1)) '@eslint-community/regexpp': 4.12.2 - eslint: 9.39.3(jiti@2.6.1) - eslint-compat-utils: 0.5.1(eslint@9.39.3(jiti@2.6.1)) + eslint: 9.39.2(jiti@2.6.1) + eslint-compat-utils: 0.5.1(eslint@9.39.2(jiti@2.6.1)) - eslint-plugin-import-x@4.16.1(@typescript-eslint/utils@8.56.1(eslint@9.39.3(jiti@2.6.1))(typescript@5.9.2))(eslint@9.39.3(jiti@2.6.1)): + eslint-plugin-import-x@4.16.1(@typescript-eslint/utils@8.56.1(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.2))(eslint@9.39.2(jiti@2.6.1)): dependencies: '@typescript-eslint/types': 8.56.1 comment-parser: 1.4.5 debug: 4.4.3 - eslint: 9.39.3(jiti@2.6.1) + eslint: 9.39.2(jiti@2.6.1) eslint-import-context: 0.1.9(unrs-resolver@1.11.1) is-glob: 4.0.3 minimatch: 10.2.4 @@ -21452,27 +21650,27 @@ snapshots: stable-hash-x: 0.2.0 unrs-resolver: 1.11.1 optionalDependencies: - '@typescript-eslint/utils': 8.56.1(eslint@9.39.3(jiti@2.6.1))(typescript@5.9.2) + '@typescript-eslint/utils': 8.56.1(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.2) transitivePeerDependencies: - supports-color - eslint-plugin-jest@29.15.0(@typescript-eslint/eslint-plugin@8.56.1(@typescript-eslint/parser@8.56.1(eslint@9.39.3(jiti@2.6.1))(typescript@5.9.2))(eslint@9.39.3(jiti@2.6.1))(typescript@5.9.2))(eslint@9.39.3(jiti@2.6.1))(jest@30.2.0(@babel/types@7.29.0)(@types/node@22.19.13))(typescript@5.9.2): + eslint-plugin-jest@29.15.0(@typescript-eslint/eslint-plugin@8.56.1(@typescript-eslint/parser@8.56.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.2))(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.2))(eslint@9.39.2(jiti@2.6.1))(jest@30.2.0(@babel/types@7.29.0)(@types/node@22.19.13))(typescript@5.9.2): dependencies: - '@typescript-eslint/utils': 8.56.1(eslint@9.39.3(jiti@2.6.1))(typescript@5.9.2) - eslint: 9.39.3(jiti@2.6.1) + '@typescript-eslint/utils': 8.56.1(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.2) + eslint: 9.39.2(jiti@2.6.1) optionalDependencies: - '@typescript-eslint/eslint-plugin': 8.56.1(@typescript-eslint/parser@8.56.1(eslint@9.39.3(jiti@2.6.1))(typescript@5.9.2))(eslint@9.39.3(jiti@2.6.1))(typescript@5.9.2) + '@typescript-eslint/eslint-plugin': 8.56.1(@typescript-eslint/parser@8.56.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.2))(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.2) jest: 30.2.0(@babel/types@7.29.0)(@types/node@22.19.13) typescript: 5.9.2 transitivePeerDependencies: - supports-color - eslint-plugin-n@17.24.0(eslint@9.39.3(jiti@2.6.1))(typescript@5.9.2): + eslint-plugin-n@17.24.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.2): dependencies: - '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.3(jiti@2.6.1)) + '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.2(jiti@2.6.1)) enhanced-resolve: 5.20.0 - eslint: 9.39.3(jiti@2.6.1) - eslint-plugin-es-x: 7.8.0(eslint@9.39.3(jiti@2.6.1)) + eslint: 9.39.2(jiti@2.6.1) + eslint-plugin-es-x: 7.8.0(eslint@9.39.2(jiti@2.6.1)) get-tsconfig: 4.13.6 globals: 15.15.0 globrex: 0.1.2 @@ -21482,17 +21680,17 @@ snapshots: transitivePeerDependencies: - typescript - eslint-plugin-promise@7.2.1(eslint@9.39.3(jiti@2.6.1)): + eslint-plugin-promise@7.2.1(eslint@9.39.2(jiti@2.6.1)): dependencies: - '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.3(jiti@2.6.1)) - eslint: 9.39.3(jiti@2.6.1) + '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.2(jiti@2.6.1)) + eslint: 9.39.2(jiti@2.6.1) - eslint-plugin-regexp@2.10.0(eslint@9.39.3(jiti@2.6.1)): + eslint-plugin-regexp@2.10.0(eslint@9.39.2(jiti@2.6.1)): dependencies: - '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.3(jiti@2.6.1)) + '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.2(jiti@2.6.1)) '@eslint-community/regexpp': 4.12.2 comment-parser: 1.4.5 - eslint: 9.39.3(jiti@2.6.1) + eslint: 9.39.2(jiti@2.6.1) jsdoc-type-pratt-parser: 4.8.0 refa: 0.12.1 regexp-ast-analysis: 0.7.1 @@ -21509,15 +21707,15 @@ snapshots: eslint-visitor-keys@5.0.1: {} - eslint@9.39.3(jiti@2.6.1): + eslint@9.39.2(jiti@2.6.1): dependencies: - '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.3(jiti@2.6.1)) + '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.2(jiti@2.6.1)) '@eslint-community/regexpp': 4.12.2 '@eslint/config-array': 0.21.1 '@eslint/config-helpers': 0.4.2 '@eslint/core': 0.17.0 '@eslint/eslintrc': 3.3.4 - '@eslint/js': 9.39.3 + '@eslint/js': 9.39.2 '@eslint/plugin-kit': 0.4.1 '@humanfs/node': 0.16.7 '@humanwhocodes/module-importer': 1.0.1 @@ -21791,10 +21989,10 @@ snapshots: flat-cache@4.0.1: dependencies: - flatted: 3.3.4 + flatted: 3.3.3 keyv: 4.5.4 - flatted@3.3.4: {} + flatted@3.3.3: {} flush-write-stream@1.1.1: dependencies: @@ -22671,7 +22869,7 @@ snapshots: '@jest/expect': 30.2.0 '@jest/test-result': 30.2.0 '@jest/types': 30.2.0 - '@types/node': 22.19.13 + '@types/node': 25.3.3 chalk: 4.1.2 co: 4.6.0 dedent: 1.7.2 @@ -22745,6 +22943,39 @@ snapshots: - babel-plugin-macros - supports-color + jest-config@30.2.0(@babel/types@7.29.0)(@types/node@25.3.3): + dependencies: + '@babel/core': 7.29.0 + '@jest/get-type': 30.1.0 + '@jest/pattern': 30.0.1 + '@jest/test-sequencer': 30.2.0 + '@jest/types': 30.2.0 + babel-jest: 30.2.0(@babel/core@7.29.0)(@babel/types@7.29.0) + chalk: 4.1.2 + ci-info: 4.4.0 + deepmerge: 4.3.1 + glob: 11.1.0 + graceful-fs: 4.2.11(patch_hash=68ebc232025360cb3dcd3081f4067f4e9fc022ab6b6f71a3230e86c7a5b337d1) + jest-circus: 30.2.0(@babel/types@7.29.0) + jest-docblock: 30.2.0 + jest-environment-node: 30.2.0 + jest-regex-util: 30.0.1 + jest-resolve: 30.2.0 + jest-runner: 30.2.0(@babel/types@7.29.0) + jest-util: 30.2.0 + jest-validate: 30.2.0 + micromatch: 4.0.8 + parse-json: 5.2.0 + pretty-format: 30.2.0 + slash: 3.0.0 + strip-json-comments: 3.1.1 + optionalDependencies: + '@types/node': 25.3.3 + transitivePeerDependencies: + - '@babel/types' + - babel-plugin-macros + - supports-color + jest-diff@30.0.5: dependencies: '@jest/diff-sequences': 30.0.1 @@ -22776,7 +23007,7 @@ snapshots: '@jest/environment': 30.2.0 '@jest/fake-timers': 30.2.0 '@jest/types': 30.2.0 - '@types/node': 22.19.13 + '@types/node': 25.3.3 jest-mock: 30.2.0 jest-util: 30.2.0 jest-validate: 30.2.0 @@ -22817,7 +23048,7 @@ snapshots: jest-haste-map@30.2.0: dependencies: '@jest/types': 30.2.0 - '@types/node': 22.19.13 + '@types/node': 25.3.3 anymatch: 3.1.3 fb-watchman: 2.0.2 graceful-fs: 4.2.11(patch_hash=68ebc232025360cb3dcd3081f4067f4e9fc022ab6b6f71a3230e86c7a5b337d1) @@ -22881,7 +23112,7 @@ snapshots: jest-mock@30.2.0: dependencies: '@jest/types': 30.2.0 - '@types/node': 22.19.13 + '@types/node': 25.3.3 jest-util: 30.2.0 jest-pnp-resolver@1.2.3(jest-resolve@29.7.0): @@ -22933,7 +23164,7 @@ snapshots: '@jest/test-result': 30.2.0 '@jest/transform': 30.2.0(@babel/types@7.29.0) '@jest/types': 30.2.0 - '@types/node': 22.19.13 + '@types/node': 25.3.3 chalk: 4.1.2 emittery: 0.13.1 exit-x: 0.2.2 @@ -22963,7 +23194,7 @@ snapshots: '@jest/test-result': 30.2.0 '@jest/transform': 30.2.0(@babel/types@7.29.0) '@jest/types': 30.2.0 - '@types/node': 22.19.13 + '@types/node': 25.3.3 chalk: 4.1.2 cjs-module-lexer: 2.2.0 collect-v8-coverage: 1.0.3 @@ -23055,7 +23286,7 @@ snapshots: jest-util@30.2.0: dependencies: '@jest/types': 30.2.0 - '@types/node': 22.19.13 + '@types/node': 25.3.3 chalk: 4.1.2 ci-info: 4.4.0 graceful-fs: 4.2.11(patch_hash=68ebc232025360cb3dcd3081f4067f4e9fc022ab6b6f71a3230e86c7a5b337d1) @@ -23083,7 +23314,7 @@ snapshots: dependencies: '@jest/test-result': 30.2.0 '@jest/types': 30.2.0 - '@types/node': 22.19.13 + '@types/node': 25.3.3 ansi-escapes: 4.3.2 chalk: 4.1.2 emittery: 0.13.1 @@ -23107,7 +23338,7 @@ snapshots: jest-worker@30.2.0: dependencies: - '@types/node': 22.19.13 + '@types/node': 25.3.3 '@ungap/structured-clone': 1.3.0 jest-util: 30.2.0 merge-stream: 2.0.0 @@ -23567,6 +23798,10 @@ snapshots: dependencies: brace-expansion: 2.0.2 + minimatch@9.0.9: + dependencies: + brace-expansion: 2.0.2 + minimist-options@4.1.0: dependencies: arrify: 1.0.1 @@ -25409,13 +25644,13 @@ snapshots: dependencies: ts-toolbelt: 9.6.0 - typescript-eslint@8.56.1(eslint@9.39.3(jiti@2.6.1))(typescript@5.9.2): + typescript-eslint@8.56.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.2): dependencies: - '@typescript-eslint/eslint-plugin': 8.56.1(@typescript-eslint/parser@8.56.1(eslint@9.39.3(jiti@2.6.1))(typescript@5.9.2))(eslint@9.39.3(jiti@2.6.1))(typescript@5.9.2) - '@typescript-eslint/parser': 8.56.1(eslint@9.39.3(jiti@2.6.1))(typescript@5.9.2) - '@typescript-eslint/typescript-estree': 8.56.1(typescript@5.9.2) - '@typescript-eslint/utils': 8.56.1(eslint@9.39.3(jiti@2.6.1))(typescript@5.9.2) - eslint: 9.39.3(jiti@2.6.1) + '@typescript-eslint/eslint-plugin': 8.56.0(@typescript-eslint/parser@8.56.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.2))(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.2) + '@typescript-eslint/parser': 8.56.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.2) + '@typescript-eslint/typescript-estree': 8.56.0(typescript@5.9.2) + '@typescript-eslint/utils': 8.56.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.2) + eslint: 9.39.2(jiti@2.6.1) typescript: 5.9.2 transitivePeerDependencies: - supports-color @@ -25801,7 +26036,7 @@ snapshots: wide-align@1.1.5: dependencies: - string-width: 1.0.2 + string-width: 4.2.3 optional: true widest-line@3.1.0: diff --git a/pnpm/bundle.ts b/pnpm/bundle.ts index 41e7a9b734..0f39656c51 100644 --- a/pnpm/bundle.ts +++ b/pnpm/bundle.ts @@ -2,7 +2,12 @@ import { build } from 'esbuild' ;(async () => { try { - const banner = { js: `import { createRequire as _cr } from 'module';const require = _cr(import.meta.url); const __filename = import.meta.filename; const __dirname = import.meta.dirname` } + const banner = { js: [ + `import { createRequire as _cr } from 'module';const require = _cr(import.meta.url); const __filename = import.meta.filename; const __dirname = import.meta.dirname;`, + // Suppress "SQLite is an experimental feature" warnings. + // Must run before any module that loads node:sqlite. + `var _ew=process.emitWarning;process.emitWarning=function(w,...a){if(String(w).includes('SQLite')&&(a[0]==='ExperimentalWarning'||(a[0]&&a[0].type==='ExperimentalWarning')))return;return _ew.call(process,w,...a)};`, + ].join('') } await build({ entryPoints: ['lib/pnpm.js'], bundle: true, diff --git a/pnpm/package.json b/pnpm/package.json index be1adec73a..b9f8c93f9b 100644 --- a/pnpm/package.json +++ b/pnpm/package.json @@ -127,6 +127,7 @@ "@pnpm/read-project-manifest": "workspace:*", "@pnpm/registry-mock": "catalog:", "@pnpm/run-npm": "workspace:*", + "@pnpm/store.index": "workspace:*", "@pnpm/store.cafs": "workspace:*", "@pnpm/tabtab": "catalog:", "@pnpm/test-fixtures": "workspace:*", diff --git a/pnpm/test/install/misc.ts b/pnpm/test/install/misc.ts index 05fe4b57d0..7659bb9a64 100644 --- a/pnpm/test/install/misc.ts +++ b/pnpm/test/install/misc.ts @@ -1,13 +1,13 @@ import fs from 'fs' import path from 'path' import { STORE_VERSION, WANTED_LOCKFILE } from '@pnpm/constants' -import { readMsgpackFileSync, writeMsgpackFileSync } from '@pnpm/fs.msgpack-file' import { type LockfileObject } from '@pnpm/lockfile.types' import { prepare, prepareEmpty, preparePackages } from '@pnpm/prepare' import { readPackageJsonFromDir } from '@pnpm/read-package-json' import { readProjectManifest } from '@pnpm/read-project-manifest' import { getIntegrity } from '@pnpm/registry-mock' -import { getIndexFilePathInCafs, type PackageFilesIndex } from '@pnpm/store.cafs' +import { type PackageFilesIndex } from '@pnpm/store.cafs' +import { StoreIndex, storeIndexKey } from '@pnpm/store.index' import { lexCompare } from '@pnpm/util.lex-comparator' import { writeProjectManifest } from '@pnpm/write-project-manifest' import { fixtures } from '@pnpm/test-fixtures' @@ -25,6 +25,11 @@ import { const skipOnWindows = isWindows() ? test.skip : test const f = fixtures(import.meta.dirname) +const storeIndexes: StoreIndex[] = [] +afterAll(() => { + for (const si of storeIndexes) si.close() +}) + test('bin files are found by lifecycle scripts', () => { prepare({ dependencies: { @@ -159,12 +164,19 @@ test("don't fail on case insensitive filesystems when package has 2 files with s project.has('@pnpm.e2e/with-same-file-in-different-cases') - const { files: integrityFile } = readMsgpackFileSync(project.getPkgIndexFilePath('@pnpm.e2e/with-same-file-in-different-cases', '1.0.0')) - const packageFiles = Array.from(integrityFile.keys()).sort(lexCompare) + const storeDir = project.getStorePath() + const indexKey = storeIndexKey(getIntegrity('@pnpm.e2e/with-same-file-in-different-cases', '1.0.0'), '@pnpm.e2e/with-same-file-in-different-cases@1.0.0') + const si = new StoreIndex(storeDir) + let filesIndex: PackageFilesIndex + try { + filesIndex = si.get(indexKey) as PackageFilesIndex + } finally { + si.close() + } + const packageFiles = Array.from(filesIndex.files.keys()).sort(lexCompare) expect(packageFiles).toStrictEqual(['Foo.js', 'foo.js', 'package.json']) const files = fs.readdirSync('node_modules/@pnpm.e2e/with-same-file-in-different-cases') - const storeDir = project.getStorePath() if (await dirIsCaseSensitive.default(storeDir)) { expect([...files].sort(lexCompare)).toStrictEqual(['Foo.js', 'foo.js', 'package.json']) } else { @@ -505,9 +517,11 @@ test('installation fails when the stored package name and version do not match t await execPnpm(['add', '@pnpm.e2e/dep-of-pkg-with-1-dep@100.1.0', ...settings]) - const cacheIntegrityPath = getIndexFilePathInCafs(path.join(storeDir, STORE_VERSION), getIntegrity('@pnpm.e2e/dep-of-pkg-with-1-dep', '100.1.0'), '@pnpm.e2e/dep-of-pkg-with-1-dep@100.1.0') - const cacheIntegrity = readMsgpackFileSync(cacheIntegrityPath) - writeMsgpackFileSync(cacheIntegrityPath, { + const cacheIntegrityKey = storeIndexKey(getIntegrity('@pnpm.e2e/dep-of-pkg-with-1-dep', '100.1.0'), '@pnpm.e2e/dep-of-pkg-with-1-dep@100.1.0') + const storeIndex = new StoreIndex(path.join(storeDir, STORE_VERSION)) + storeIndexes.push(storeIndex) + const cacheIntegrity = storeIndex.get(cacheIntegrityKey) as PackageFilesIndex + storeIndex.set(cacheIntegrityKey, { ...cacheIntegrity, manifest: { ...cacheIntegrity.manifest, name: 'foo' }, }) diff --git a/pnpm/tsconfig.json b/pnpm/tsconfig.json index 5cb66ff712..a099643cf4 100644 --- a/pnpm/tsconfig.json +++ b/pnpm/tsconfig.json @@ -158,6 +158,9 @@ { "path": "../store/cafs" }, + { + "path": "../store/index" + }, { "path": "../store/plugin-commands-store" }, diff --git a/resolving/npm-resolver/package.json b/resolving/npm-resolver/package.json index 126f62989b..1f98aa92db 100644 --- a/resolving/npm-resolver/package.json +++ b/resolving/npm-resolver/package.json @@ -47,6 +47,7 @@ "@pnpm/resolver-base": "workspace:*", "@pnpm/resolving.jsr-specifier-parser": "workspace:*", "@pnpm/store.cafs": "workspace:*", + "@pnpm/store.index": "workspace:*", "@pnpm/types": "workspace:*", "@pnpm/workspace.spec-parser": "workspace:*", "@zkochan/retry": "catalog:", diff --git a/resolving/npm-resolver/src/index.ts b/resolving/npm-resolver/src/index.ts index 4c10449afd..1c681ee18c 100644 --- a/resolving/npm-resolver/src/index.ts +++ b/resolving/npm-resolver/src/index.ts @@ -19,7 +19,7 @@ import { type WorkspacePackages, type WorkspacePackagesByVersion, } from '@pnpm/resolver-base' -import { getIndexFilePathInCafs } from '@pnpm/store.cafs' +import { storeIndexKey } from '@pnpm/store.index' import { readPkgFromCafs, } from '@pnpm/worker' @@ -184,7 +184,7 @@ export function createNpmResolver ( let peekManifestFromStore: ResolveFromNpmContext['peekManifestFromStore'] | undefined if (storeDir) { peekManifestFromStore = async (peekOpts) => { - const filesIndexFile = getIndexFilePathInCafs(storeDir, peekOpts.integrity, peekOpts.id) + const filesIndexFile = storeIndexKey(peekOpts.integrity, peekOpts.id) const existingRequest = peekLockerForPeek.get(filesIndexFile) if (existingRequest != null) { return existingRequest diff --git a/resolving/npm-resolver/tsconfig.json b/resolving/npm-resolver/tsconfig.json index 8e776a271d..d6a6adf1b2 100644 --- a/resolving/npm-resolver/tsconfig.json +++ b/resolving/npm-resolver/tsconfig.json @@ -57,6 +57,9 @@ { "path": "../../store/cafs" }, + { + "path": "../../store/index" + }, { "path": "../../worker" }, diff --git a/reviewing/dependencies-hierarchy/package.json b/reviewing/dependencies-hierarchy/package.json index 30777f3c3e..4b3c9a0784 100644 --- a/reviewing/dependencies-hierarchy/package.json +++ b/reviewing/dependencies-hierarchy/package.json @@ -35,7 +35,6 @@ }, "dependencies": { "@pnpm/dependency-path": "workspace:*", - "@pnpm/fs.msgpack-file": "workspace:*", "@pnpm/lockfile.detect-dep-types": "workspace:*", "@pnpm/lockfile.fs": "workspace:*", "@pnpm/lockfile.utils": "workspace:*", @@ -46,6 +45,7 @@ "@pnpm/read-modules-dir": "workspace:*", "@pnpm/read-package-json": "workspace:*", "@pnpm/store.cafs": "workspace:*", + "@pnpm/store.index": "workspace:*", "@pnpm/types": "workspace:*", "@pnpm/util.lex-comparator": "catalog:", "load-json-file": "catalog:", diff --git a/reviewing/dependencies-hierarchy/src/buildDependenciesTree.ts b/reviewing/dependencies-hierarchy/src/buildDependenciesTree.ts index 15b88b23d3..36c073b4b2 100644 --- a/reviewing/dependencies-hierarchy/src/buildDependenciesTree.ts +++ b/reviewing/dependencies-hierarchy/src/buildDependenciesTree.ts @@ -9,6 +9,7 @@ import { } from '@pnpm/lockfile.fs' import { detectDepTypes } from '@pnpm/lockfile.detect-dep-types' import { readModulesManifest } from '@pnpm/modules-yaml' +import { StoreIndex } from '@pnpm/store.index' import { normalizeRegistries } from '@pnpm/normalize-registries' import { readModulesDir } from '@pnpm/read-modules-dir' import { safeReadPackageJsonFromDir } from '@pnpm/read-package-json' @@ -72,6 +73,8 @@ export async function buildDependenciesTree ( return result } + const storeDir = modules?.storeDir + const storeIndex = storeDir ? new StoreIndex(storeDir) : undefined const opts = { depth: maybeOpts.depth || 0, excludePeerDependencies: maybeOpts.excludePeerDependencies, @@ -87,7 +90,8 @@ export async function buildDependenciesTree ( search: maybeOpts.search, showDedupedSearchMatches: maybeOpts.showDedupedSearchMatches ?? (maybeOpts.search != null), skipped: new Set(modules?.skipped ?? []), - storeDir: modules?.storeDir, + storeDir, + storeIndex, modulesDir, virtualStoreDir: modules?.virtualStoreDir, virtualStoreDirMaxLength: modules?.virtualStoreDirMaxLength ?? maybeOpts.virtualStoreDirMaxLength, @@ -130,6 +134,7 @@ export async function buildDependenciesTree ( for (const [projectPath, dependenciesHierarchy] of pairs) { result[projectPath] = dependenciesHierarchy } + storeIndex?.close() return result } diff --git a/reviewing/dependencies-hierarchy/src/buildDependentsTree.ts b/reviewing/dependencies-hierarchy/src/buildDependentsTree.ts index f52ae3db66..a5a548d4ac 100644 --- a/reviewing/dependencies-hierarchy/src/buildDependentsTree.ts +++ b/reviewing/dependencies-hierarchy/src/buildDependentsTree.ts @@ -7,6 +7,7 @@ import { } from '@pnpm/lockfile.fs' import { nameVerFromPkgSnapshot } from '@pnpm/lockfile.utils' import { readModulesManifest } from '@pnpm/modules-yaml' +import { StoreIndex } from '@pnpm/store.index' import { normalizeRegistries } from '@pnpm/normalize-registries' import { type DependenciesField, type DependencyManifest, type Finder, type Registries } from '@pnpm/types' import { lexCompare } from '@pnpm/util.lex-comparator' @@ -90,6 +91,7 @@ export async function buildDependentsTree ( ...modules?.registries, }) const storeDir = modules?.storeDir + const storeIndex = storeDir ? new StoreIndex(storeDir) : undefined const virtualStoreDir = modules?.virtualStoreDir ?? path.join(modulesDir, '.pnpm') const virtualStoreDirMaxLength = modules?.virtualStoreDirMaxLength ?? 120 @@ -129,6 +131,7 @@ export async function buildDependentsTree ( registries, wantedPackages: currentPackages, storeDir, + storeIndex, }) // Scan all package nodes for matches. @@ -208,6 +211,7 @@ export async function buildDependentsTree ( if (versionCmp !== 0) return versionCmp return lexCompare(a.peersSuffixHash ?? '', b.peersSuffixHash ?? '') }) + storeIndex?.close() return trees } @@ -248,6 +252,7 @@ function resolvePackageNodes ( registries: Registries wantedPackages: PackageSnapshots storeDir?: string + storeIndex?: StoreIndex } ): Map DependencyManifest }> { const resolved = new Map DependencyManifest }>() diff --git a/reviewing/dependencies-hierarchy/src/getPkgInfo.ts b/reviewing/dependencies-hierarchy/src/getPkgInfo.ts index f2f999a310..63ed31633f 100644 --- a/reviewing/dependencies-hierarchy/src/getPkgInfo.ts +++ b/reviewing/dependencies-hierarchy/src/getPkgInfo.ts @@ -9,6 +9,7 @@ import { pkgSnapshotToResolution, } from '@pnpm/lockfile.utils' import { type DepTypes, DepType } from '@pnpm/lockfile.detect-dep-types' +import { type StoreIndex } from '@pnpm/store.index' import { type DependencyManifest, type Registries } from '@pnpm/types' import { refToRelative } from '@pnpm/dependency-path' import { readPackageJsonFromDirSync } from '@pnpm/read-package-json' @@ -24,6 +25,7 @@ export interface GetPkgInfoOpts { readonly registries: Registries readonly skipped: Set readonly storeDir?: string + readonly storeIndex?: StoreIndex readonly wantedPackages: PackageSnapshots readonly virtualStoreDir?: string readonly virtualStoreDirMaxLength: number @@ -141,8 +143,8 @@ export function getPkgInfo (opts: GetPkgInfoOpts): { pkgInfo: PackageInfo, readM return { pkgInfo: packageInfo, readManifest: () => { - if (integrity && opts.storeDir) { - const manifest = readManifestFromCafs(opts.storeDir, { integrity, name, version }) + if (integrity && opts.storeDir && opts.storeIndex) { + const manifest = readManifestFromCafs(opts.storeDir, opts.storeIndex, { integrity, name, version }) if (manifest) return manifest } return readPackageJsonFromDirSync(fullPackagePath) diff --git a/reviewing/dependencies-hierarchy/src/getTree.ts b/reviewing/dependencies-hierarchy/src/getTree.ts index 90ee06f61c..f05b51a8a3 100644 --- a/reviewing/dependencies-hierarchy/src/getTree.ts +++ b/reviewing/dependencies-hierarchy/src/getTree.ts @@ -1,6 +1,7 @@ import path from 'path' import { type PackageSnapshots, type ProjectSnapshot } from '@pnpm/lockfile.fs' import { type DepTypes } from '@pnpm/lockfile.detect-dep-types' +import { type StoreIndex } from '@pnpm/store.index' import { type Finder, type Registries } from '@pnpm/types' import { lexCompare } from '@pnpm/util.lex-comparator' import { type DependencyGraph } from './buildDependencyGraph.js' @@ -23,6 +24,7 @@ export interface BaseTreeOpts { registries: Registries depTypes: DepTypes storeDir?: string + storeIndex?: StoreIndex virtualStoreDir?: string virtualStoreDirMaxLength: number modulesDir?: string diff --git a/reviewing/dependencies-hierarchy/src/readManifestFromCafs.ts b/reviewing/dependencies-hierarchy/src/readManifestFromCafs.ts index fc27d3f1b5..1b21c248e0 100644 --- a/reviewing/dependencies-hierarchy/src/readManifestFromCafs.ts +++ b/reviewing/dependencies-hierarchy/src/readManifestFromCafs.ts @@ -1,21 +1,22 @@ -import { readMsgpackFileSync } from '@pnpm/fs.msgpack-file' +import { type StoreIndex, storeIndexKey } from '@pnpm/store.index' import { loadJsonFileSync } from 'load-json-file' -import { getIndexFilePathInCafs, getFilePathByModeInCafs, type PackageFilesIndex } from '@pnpm/store.cafs' +import { getFilePathByModeInCafs, type PackageFilesIndex } from '@pnpm/store.cafs' import { type DependencyManifest } from '@pnpm/types' /** * Attempts to read a package manifest from the content-addressable store (CAFS) * using its integrity hash. Returns `undefined` if the manifest cannot be read. */ -export function readManifestFromCafs (storeDir: string, pkg: { +export function readManifestFromCafs (storeDir: string, storeIndex: StoreIndex, pkg: { integrity: string name: string version: string }): DependencyManifest | undefined { try { const pkgId = `${pkg.name}@${pkg.version}` - const indexPath = getIndexFilePathInCafs(storeDir, pkg.integrity, pkgId) - const pkgIndex = readMsgpackFileSync(indexPath) + const indexPath = storeIndexKey(pkg.integrity, pkgId) + const pkgIndex = storeIndex.get(indexPath) as PackageFilesIndex | undefined + if (!pkgIndex) return undefined const pkgJsonEntry = pkgIndex.files.get('package.json') if (pkgJsonEntry) { const filePath = getFilePathByModeInCafs(storeDir, pkgJsonEntry.digest, pkgJsonEntry.mode) diff --git a/reviewing/dependencies-hierarchy/tsconfig.json b/reviewing/dependencies-hierarchy/tsconfig.json index 2f43034ac9..6b499948c5 100644 --- a/reviewing/dependencies-hierarchy/tsconfig.json +++ b/reviewing/dependencies-hierarchy/tsconfig.json @@ -18,9 +18,6 @@ { "path": "../../config/normalize-registries" }, - { - "path": "../../fs/msgpack-file" - }, { "path": "../../fs/read-modules-dir" }, @@ -50,6 +47,9 @@ }, { "path": "../../store/cafs" + }, + { + "path": "../../store/index" } ] } diff --git a/reviewing/license-scanner/package.json b/reviewing/license-scanner/package.json index 40f4652fda..811a1488a3 100644 --- a/reviewing/license-scanner/package.json +++ b/reviewing/license-scanner/package.json @@ -41,6 +41,7 @@ "@pnpm/lockfile.walker": "workspace:*", "@pnpm/package-is-installable": "workspace:*", "@pnpm/read-package-json": "workspace:*", + "@pnpm/store.index": "workspace:*", "@pnpm/store.pkg-finder": "workspace:*", "@pnpm/types": "workspace:*", "p-limit": "catalog:", diff --git a/reviewing/license-scanner/src/getPkgInfo.ts b/reviewing/license-scanner/src/getPkgInfo.ts index 866c7879e9..e6b5e6cda7 100644 --- a/reviewing/license-scanner/src/getPkgInfo.ts +++ b/reviewing/license-scanner/src/getPkgInfo.ts @@ -5,6 +5,7 @@ import { readPackageJson } from '@pnpm/read-package-json' import { depPathToFilename } from '@pnpm/dependency-path' import pLimit from 'p-limit' import { type PackageManifest, type Registries } from '@pnpm/types' +import { type StoreIndex } from '@pnpm/store.index' import { readPackageFileMap } from '@pnpm/store.pkg-finder' import { PnpmError } from '@pnpm/error' import type { LicensePackage } from './licenses.js' @@ -195,6 +196,7 @@ export interface PackageInfo { export interface GetPackageInfoOptions { storeDir: string + storeIndex: StoreIndex virtualStoreDir: string virtualStoreDirMaxLength: number dir: string @@ -229,6 +231,7 @@ export async function getPkgInfo ( pkg.id, { storeDir: opts.storeDir, + storeIndex: opts.storeIndex, lockfileDir: opts.dir, virtualStoreDirMaxLength: opts.virtualStoreDirMaxLength, } diff --git a/reviewing/license-scanner/src/lockfileToLicenseNodeTree.ts b/reviewing/license-scanner/src/lockfileToLicenseNodeTree.ts index 4c0bab67dc..8a6454e227 100644 --- a/reviewing/license-scanner/src/lockfileToLicenseNodeTree.ts +++ b/reviewing/license-scanner/src/lockfileToLicenseNodeTree.ts @@ -1,11 +1,12 @@ import { type LockfileObject, type TarballResolution } from '@pnpm/lockfile.types' -import { nameVerFromPkgSnapshot } from '@pnpm/lockfile.utils' +import { nameVerFromPkgSnapshot, packageIdFromSnapshot } from '@pnpm/lockfile.utils' import { packageIsInstallable } from '@pnpm/package-is-installable' import { lockfileWalkerGroupImporterSteps, type LockfileWalkerStep, } from '@pnpm/lockfile.walker' import { type DepTypes, DepType, detectDepTypes } from '@pnpm/lockfile.detect-dep-types' +import { StoreIndex } from '@pnpm/store.index' import { type SupportedArchitectures, type DependenciesField, type ProjectId, type Registries } from '@pnpm/types' import { map as mapValues } from 'ramda' import { getPkgInfo } from './getPkgInfo.js' @@ -33,6 +34,7 @@ export type LicenseNodeTree = Omit< export interface LicenseExtractOptions { storeDir: string + storeIndex: StoreIndex virtualStoreDir: string virtualStoreDirMaxLength: number modulesDir?: string @@ -71,7 +73,7 @@ export async function lockfileToLicenseNode ( const packageInfo = await getPkgInfo( { - id: pkgSnapshot.id ?? depPath, + id: packageIdFromSnapshot(depPath, pkgSnapshot), name, version, depPath, @@ -80,6 +82,7 @@ export async function lockfileToLicenseNode ( }, { storeDir: options.storeDir, + storeIndex: options.storeIndex, virtualStoreDir: options.virtualStoreDir, virtualStoreDirMaxLength: options.virtualStoreDirMaxLength, dir: options.dir, @@ -128,7 +131,7 @@ export async function lockfileToLicenseNodeTree ( opts: { include?: { [dependenciesField in DependenciesField]: boolean } includedImporterIds?: ProjectId[] - } & LicenseExtractOptions + } & Omit ): Promise { const importerWalkers = lockfileWalkerGroupImporterSteps( lockfile, @@ -136,11 +139,13 @@ export async function lockfileToLicenseNodeTree ( { include: opts?.include } ) const depTypes = detectDepTypes(lockfile) + const storeIndex = new StoreIndex(opts.storeDir) const dependencies = Object.fromEntries( await Promise.all( importerWalkers.map(async (importerWalker) => { const importerDeps = await lockfileToLicenseNode(importerWalker.step, { storeDir: opts.storeDir, + storeIndex, virtualStoreDir: opts.virtualStoreDir, virtualStoreDirMaxLength: opts.virtualStoreDirMaxLength, modulesDir: opts.modulesDir, @@ -158,6 +163,7 @@ export async function lockfileToLicenseNodeTree ( }) ) ) + storeIndex.close() const licenseNodeTree: LicenseNodeTree = { name: undefined, diff --git a/reviewing/license-scanner/test/getPkgInfo.spec.ts b/reviewing/license-scanner/test/getPkgInfo.spec.ts index 09328ce8b5..b6cbe040c7 100644 --- a/reviewing/license-scanner/test/getPkgInfo.spec.ts +++ b/reviewing/license-scanner/test/getPkgInfo.spec.ts @@ -1,3 +1,7 @@ +import fs from 'fs' +import os from 'os' +import path from 'path' +import { StoreIndex } from '@pnpm/store.index' import { getPkgInfo } from '../lib/getPkgInfo.js' export const DEFAULT_REGISTRIES = { @@ -6,6 +10,19 @@ export const DEFAULT_REGISTRIES = { } describe('licences', () => { + let storeDir: string + let storeIndex: StoreIndex + + beforeAll(() => { + storeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'pnpm-license-test-')) + storeIndex = new StoreIndex(storeDir) + }) + + afterAll(() => { + storeIndex.close() + fs.rmSync(storeDir, { recursive: true, force: true }) + }) + test('getPkgInfo() should throw error when package info can not be fetched', async () => { await expect( getPkgInfo( @@ -22,13 +39,14 @@ describe('licences', () => { registries: DEFAULT_REGISTRIES, }, { - storeDir: 'store-dir', + storeDir, + storeIndex, virtualStoreDir: 'virtual-store-dir', modulesDir: 'modules-dir', dir: 'workspace-dir', virtualStoreDirMaxLength: 120, } ) - ).rejects.toThrow(/Failed to find package index file for bogus-package@1\.0\.0 \(at .*16-bogus-package@1\.0\.0\.mpk\), please consider running 'pnpm install'/) + ).rejects.toThrow(/Failed to find package index file for bogus-package@1\.0\.0 \(at .*\), please consider running 'pnpm install'/) }) }) diff --git a/reviewing/license-scanner/test/licenses.spec.ts b/reviewing/license-scanner/test/licenses.spec.ts index 7899af9862..5e0d378f31 100644 --- a/reviewing/license-scanner/test/licenses.spec.ts +++ b/reviewing/license-scanner/test/licenses.spec.ts @@ -1,3 +1,6 @@ +import fs from 'fs' +import os from 'os' +import path from 'path' import { LOCKFILE_VERSION } from '@pnpm/constants' import type { DepPath, ProjectManifest, Registries, ProjectId } from '@pnpm/types' import type { LockfileObject } from '@pnpm/lockfile.fs' @@ -5,6 +8,11 @@ import { jest } from '@jest/globals' import type { LicensePackage } from '../lib/licenses.js' import type { GetPackageInfoOptions, PackageInfo } from '../lib/getPkgInfo.js' +const tmpStoreDir = fs.mkdtempSync(path.join(os.tmpdir(), 'pnpm-license-spec-')) +afterAll(() => { + fs.rmSync(tmpStoreDir, { recursive: true, force: true }) +}) + const actualModule = await import('../lib/getPkgInfo.js') jest.unstable_mockModule('../lib/getPkgInfo.js', () => { return { @@ -72,7 +80,7 @@ describe('licences', () => { virtualStoreDir: '/.pnpm', registries: {} as Registries, wantedLockfile: lockfile, - storeDir: '/opt/.pnpm', + storeDir: tmpStoreDir, virtualStoreDirMaxLength: 120, }) @@ -158,7 +166,7 @@ describe('licences', () => { virtualStoreDir: '/.pnpm', registries: {} as Registries, wantedLockfile: lockfile, - storeDir: '/opt/.pnpm', + storeDir: tmpStoreDir, includedImporterIds: ['packages/a'] as ProjectId[], virtualStoreDirMaxLength: 120, }) @@ -235,7 +243,7 @@ describe('licences', () => { virtualStoreDir: '/.pnpm', registries: {} as Registries, wantedLockfile: lockfile, - storeDir: '/opt/.pnpm', + storeDir: tmpStoreDir, virtualStoreDirMaxLength: 120, }) diff --git a/reviewing/license-scanner/tsconfig.json b/reviewing/license-scanner/tsconfig.json index 7ec15f5806..41330cc5b4 100644 --- a/reviewing/license-scanner/tsconfig.json +++ b/reviewing/license-scanner/tsconfig.json @@ -45,6 +45,9 @@ { "path": "../../pkg-manifest/read-package-json" }, + { + "path": "../../store/index" + }, { "path": "../../store/pkg-finder" } diff --git a/reviewing/outdated/src/createManifestGetter.ts b/reviewing/outdated/src/createManifestGetter.ts index c0a8dd263d..18bb763ab0 100644 --- a/reviewing/outdated/src/createManifestGetter.ts +++ b/reviewing/outdated/src/createManifestGetter.ts @@ -14,7 +14,7 @@ interface GetManifestOpts { minimumReleaseAgeExclude?: string[] } -export type ManifestGetterOptions = Omit +export type ManifestGetterOptions = Omit & GetManifestOpts & { fullMetadata: boolean, rawConfig: Record } diff --git a/reviewing/sbom/package.json b/reviewing/sbom/package.json index 507906101b..6659d552e1 100644 --- a/reviewing/sbom/package.json +++ b/reviewing/sbom/package.json @@ -40,6 +40,7 @@ "@pnpm/lockfile.utils": "workspace:*", "@pnpm/lockfile.walker": "workspace:*", "@pnpm/read-package-json": "workspace:*", + "@pnpm/store.index": "workspace:*", "@pnpm/store.pkg-finder": "workspace:*", "@pnpm/types": "workspace:*", "p-limit": "catalog:", diff --git a/reviewing/sbom/src/collectComponents.ts b/reviewing/sbom/src/collectComponents.ts index dcf15dad07..ab6652c208 100644 --- a/reviewing/sbom/src/collectComponents.ts +++ b/reviewing/sbom/src/collectComponents.ts @@ -6,6 +6,7 @@ import { } from '@pnpm/lockfile.walker' import { type DepTypes, DepType, detectDepTypes } from '@pnpm/lockfile.detect-dep-types' import { type DependenciesField, type ProjectId, type Registries } from '@pnpm/types' +import { StoreIndex } from '@pnpm/store.index' import { buildPurl, encodePurlName } from './purl.js' import { getPkgMetadata, type GetPkgMetadataOptions } from './getPkgMetadata.js' import { type SbomComponent, type SbomRelationship, type SbomResult, type SbomComponentType } from './types.js' @@ -42,9 +43,13 @@ export async function collectSbomComponents (opts: CollectSbomComponentsOptions) const relationships: SbomRelationship[] = [] const rootPurl = `pkg:npm/${encodePurlName(opts.rootName)}@${opts.rootVersion}` - const metadataOpts: GetPkgMetadataOptions | undefined = (!opts.lockfileOnly && opts.storeDir) + const storeIndex = (!opts.lockfileOnly && opts.storeDir) + ? new StoreIndex(opts.storeDir) + : undefined + const metadataOpts: GetPkgMetadataOptions | undefined = (storeIndex && opts.storeDir) ? { storeDir: opts.storeDir, + storeIndex, lockfileDir: opts.lockfileDir, virtualStoreDirMaxLength: opts.virtualStoreDirMaxLength ?? 120, } @@ -63,6 +68,7 @@ export async function collectSbomComponents (opts: CollectSbomComponentsOptions) ) }) ) + storeIndex?.close() return { rootComponent: { diff --git a/reviewing/sbom/src/getPkgMetadata.ts b/reviewing/sbom/src/getPkgMetadata.ts index f0d39fec07..16bcb6b58d 100644 --- a/reviewing/sbom/src/getPkgMetadata.ts +++ b/reviewing/sbom/src/getPkgMetadata.ts @@ -1,4 +1,5 @@ import { type PackageManifest, type Registries } from '@pnpm/types' +import { type StoreIndex } from '@pnpm/store.index' import { readPackageFileMap } from '@pnpm/store.pkg-finder' import { readPackageJson } from '@pnpm/read-package-json' import { type PackageSnapshot, pkgSnapshotToResolution } from '@pnpm/lockfile.utils' @@ -16,6 +17,7 @@ export interface PkgMetadata { export interface GetPkgMetadataOptions { storeDir: string + storeIndex: StoreIndex lockfileDir: string virtualStoreDirMaxLength: number } diff --git a/reviewing/sbom/tsconfig.json b/reviewing/sbom/tsconfig.json index e610fdba43..982c48cf7f 100644 --- a/reviewing/sbom/tsconfig.json +++ b/reviewing/sbom/tsconfig.json @@ -30,6 +30,9 @@ { "path": "../../pkg-manifest/read-package-json" }, + { + "path": "../../store/index" + }, { "path": "../../store/pkg-finder" } diff --git a/store/cafs-types/src/index.ts b/store/cafs-types/src/index.ts index 7a577773ce..a991951f28 100644 --- a/store/cafs-types/src/index.ts +++ b/store/cafs-types/src/index.ts @@ -71,7 +71,6 @@ export interface Cafs { addFilesFromDir: (dir: string) => AddToStoreResult addFilesFromTarball: (buffer: Buffer) => AddToStoreResult addFile: (buffer: Buffer, mode: number) => FileWriteResult - getIndexFilePathInCafs: (integrity: string, pkgId: string) => string getFilePathByModeInCafs: (digest: string, mode: number) => string importPackage: ImportPackageFunction tempDir: () => Promise diff --git a/store/cafs/package.json b/store/cafs/package.json index e2fbfc162d..eecc9c7b5e 100644 --- a/store/cafs/package.json +++ b/store/cafs/package.json @@ -31,7 +31,6 @@ "prepublishOnly": "pnpm run compile" }, "dependencies": { - "@pnpm/crypto.integrity": "workspace:*", "@pnpm/error": "workspace:*", "@pnpm/fetcher-base": "workspace:*", "@pnpm/graceful-fs": "workspace:*", diff --git a/store/cafs/src/getFilePathInCafs.ts b/store/cafs/src/getFilePathInCafs.ts index 90c245fe62..7a71856815 100644 --- a/store/cafs/src/getFilePathInCafs.ts +++ b/store/cafs/src/getFilePathInCafs.ts @@ -1,5 +1,4 @@ import path from 'path' -import { parseIntegrity } from '@pnpm/crypto.integrity' /** * Checks if a file mode has any executable permissions set. @@ -27,22 +26,6 @@ export function getFilePathByModeInCafs ( return path.join(storeDir, contentPathFromHex(fileType, hexDigest)) } -export function getIndexFilePathInCafs ( - storeDir: string, - integrity: string, - pkgId: string -): string { - const { hexDigest } = parseIntegrity(integrity) - const hex = hexDigest.substring(0, 64) - // Some registries allow identical content to be published under different package names or versions. - // To accommodate this, index files are stored using both the content hash and package identifier. - // This approach ensures that we can: - // 1. Validate that the integrity in the lockfile corresponds to the correct package, - // which might not be the case after a poorly resolved Git conflict. - // 2. Allow the same content to be referenced by different packages or different versions of the same package. - return path.join(storeDir, `index/${path.join(hex.slice(0, 2), hex.slice(2))}-${pkgId.replace(/[\\/:*?"<>|]/g, '+')}.mpk`) -} - export function contentPathFromHex (fileType: FileType, hex: string): string { const p = path.join('files', hex.slice(0, 2), hex.slice(2)) switch (fileType) { diff --git a/store/cafs/src/index.ts b/store/cafs/src/index.ts index 2f12c20182..9f394b297e 100644 --- a/store/cafs/src/index.ts +++ b/store/cafs/src/index.ts @@ -18,7 +18,6 @@ import { type VerifyResult, } from './checkPkgFilesIntegrity.js' import { - getIndexFilePathInCafs, contentPathFromHex, type FileType, getFilePathByModeInCafs, @@ -37,7 +36,6 @@ export { buildFileMapsFromIndex, type FileType, getFilePathByModeInCafs, - getIndexFilePathInCafs, type Integrity, type PackageFileInfo, type PackageFiles, @@ -60,7 +58,6 @@ export interface CafsFunctions { addFilesFromDir: (dirname: string, opts?: { files?: string[], readManifest?: boolean, includeNodeModules?: boolean }) => AddToStoreResult addFilesFromTarball: (tarballBuffer: Buffer, readManifest?: boolean) => AddToStoreResult addFile: (buffer: Buffer, mode: number) => FileWriteResult - getIndexFilePathInCafs: (integrity: string, pkgId: string) => string getFilePathByModeInCafs: (digest: string, mode: number) => string } @@ -71,7 +68,6 @@ export function createCafs (storeDir: string, { ignoreFile, cafsLocker }: Create addFilesFromDir: addFilesFromDir.bind(null, addBuffer), addFilesFromTarball: addFilesFromTarball.bind(null, addBuffer, ignoreFile ?? null), addFile: addBuffer, - getIndexFilePathInCafs: getIndexFilePathInCafs.bind(null, storeDir), getFilePathByModeInCafs: getFilePathByModeInCafs.bind(null, storeDir), } } diff --git a/store/cafs/tsconfig.json b/store/cafs/tsconfig.json index b2c9df6623..6c98976975 100644 --- a/store/cafs/tsconfig.json +++ b/store/cafs/tsconfig.json @@ -12,9 +12,6 @@ { "path": "../../__utils__/test-fixtures" }, - { - "path": "../../crypto/integrity" - }, { "path": "../../fetching/fetcher-base" }, diff --git a/store/index/README.md b/store/index/README.md new file mode 100644 index 0000000000..1fef7241f4 --- /dev/null +++ b/store/index/README.md @@ -0,0 +1,29 @@ +# @pnpm/store.index + +> SQLite-backed index for the pnpm content-addressable store + +## Why SQLite instead of individual index files? + +Previously, pnpm stored package metadata as individual JSON files under +`$STORE/index/`. Each resolved package had its own file, keyed by its integrity +hash. This worked but had several downsides at scale: + +- **Filesystem overhead.** Every lookup required `open` / `read` / `close` + syscalls, and every write needed an atomic `write` + `rename` per entry. + On repositories with thousands of dependencies the accumulated I/O was + significant. +- **Space inefficiency.** Small metadata entries still consumed a minimum + filesystem block each (typically 4 KiB), wasting space. +Storing all entries in a single SQLite database (`$STORE/index.db`) addresses +these issues: + +- **Fewer syscalls.** Reads and writes go through SQLite's page cache and + memory-mapped I/O instead of individual file operations. +- **Space efficiency.** Small entries share database pages instead of each + occupying a full filesystem block. +- **Batch writes.** Multiple entries can be inserted in a single transaction, + reducing disk flushes. + +## License + +MIT diff --git a/store/index/package.json b/store/index/package.json new file mode 100644 index 0000000000..92645d1002 --- /dev/null +++ b/store/index/package.json @@ -0,0 +1,48 @@ +{ + "name": "@pnpm/store.index", + "version": "1000.0.0-0", + "description": "SQLite-backed index for the pnpm content-addressable store", + "keywords": [ + "pnpm", + "pnpm11", + "store" + ], + "license": "MIT", + "funding": "https://opencollective.com/pnpm", + "repository": "https://github.com/pnpm/pnpm/tree/main/store/index", + "homepage": "https://github.com/pnpm/pnpm/tree/main/store/index#readme", + "bugs": { + "url": "https://github.com/pnpm/pnpm/issues" + }, + "type": "module", + "main": "lib/index.js", + "types": "lib/index.d.ts", + "exports": { + ".": "./lib/index.js" + }, + "files": [ + "lib", + "!*.map" + ], + "scripts": { + "lint": "eslint \"src/**/*.ts\" \"test/**/*.ts\"", + "_test": "cross-env NODE_OPTIONS=\"$NODE_OPTIONS --experimental-vm-modules\" jest", + "test": "pnpm run compile && pnpm run _test", + "compile": "tsgo --build && pnpm run lint --fix", + "prepublishOnly": "pnpm run compile" + }, + "dependencies": { + "msgpackr": "catalog:" + }, + "devDependencies": { + "@pnpm/store.index": "workspace:*", + "@types/node": "catalog:", + "tempy": "catalog:" + }, + "engines": { + "node": ">=22.13" + }, + "jest": { + "preset": "@pnpm/jest-config" + } +} diff --git a/store/index/src/index.ts b/store/index/src/index.ts new file mode 100644 index 0000000000..f786dcf8d4 --- /dev/null +++ b/store/index/src/index.ts @@ -0,0 +1,267 @@ +import { createRequire } from 'module' +import fs from 'fs' +import type { DatabaseSync as DatabaseSyncType, StatementSync } from 'node:sqlite' +import { Packr } from 'msgpackr' + +// Use createRequire to load node:sqlite because it is a prefix-only builtin +// that Jest's ESM module resolver cannot handle. +const req = createRequire(import.meta.url) +const { DatabaseSync } = req('node:sqlite') as { DatabaseSync: typeof DatabaseSyncType } + +const packr = new Packr({ + useRecords: true, + moreTypes: true, +}) + +const SQLITE_BUSY = 5 +const RETRY_DELAY_MS = 50 +const MAX_RETRIES = 100 // ~5 seconds total + +function sqliteRetry (fn: () => T): T { + for (let attempt = 0; ; attempt++) { + try { + return fn() + } catch (err: unknown) { + if (isSqliteBusy(err) && attempt < MAX_RETRIES) { + sleepSync(RETRY_DELAY_MS) + continue + } + throw err + } + } +} + +function isSqliteBusy (err: any): boolean { // eslint-disable-line @typescript-eslint/no-explicit-any + // errcode may be an extended error code (e.g. SQLITE_BUSY_RECOVERY = 261), + // so mask off the upper bits to get the primary error code. + return (err?.errcode & 0xFF) === SQLITE_BUSY +} + +const sleepBuffer = new Int32Array(new SharedArrayBuffer(4)) + +function sleepSync (ms: number): void { + Atomics.wait(sleepBuffer, 0, 0, ms) +} + +/** + * Pack data for storage using msgpackr. + * Use this when data will be packed in one thread and stored by another, + * to ensure the same Packr instance is used for pack and unpack within each thread. + */ +export function packForStorage (data: unknown): Uint8Array { + return packr.pack(data) +} + +/** + * Create a store index key from an integrity hash and package id. + * The key is `${integrity}\t${pkgId}` — tab-separated. + * Integrity strings never contain tabs, so this is unambiguous. + */ +export function storeIndexKey (integrity: string, pkgId: string): string { + return `${integrity}\t${pkgId}` +} + +export function gitHostedStoreIndexKey (pkgId: string, opts: { built: boolean }): string { + return storeIndexKey(pkgId, opts.built ? 'built' : 'not-built') +} + +const openInstances = new Set() + +/** + * Close all open StoreIndex instances. + * Useful in tests that need to remove the store directory. + */ +export function closeAllStoreIndexes (): void { + for (const si of openInstances) { + si.close() + } +} + +export class StoreIndex { + private db: DatabaseSyncType + private closed = false + private pendingWrites: Array<{ key: string, buffer: Uint8Array }> = [] + private flushScheduled = false + private stmtGet: StatementSync + private stmtSet: StatementSync + private stmtDel: StatementSync + private stmtHas: StatementSync + private stmtAll: StatementSync + private readonly exitHandler: () => void + + constructor (storeDir: string) { + const dbPath = `${storeDir}/index.db` + fs.mkdirSync(storeDir, { recursive: true }) + this.db = new DatabaseSync(dbPath) + // Set busy_timeout FIRST so SQLite's internal busy handler is active + // during all subsequent operations. On Windows, file locking is mandatory + // and concurrent processes (e.g. parallel dlx calls) will contend. + this.db.exec('PRAGMA busy_timeout=5000') + sqliteRetry(() => { + this.db.exec('PRAGMA journal_mode=WAL') + this.db.exec('PRAGMA synchronous=NORMAL') + // Increase memory map size to 512MB + this.db.exec('PRAGMA mmap_size=536870912') + // Increase page cache size to ~32MB + this.db.exec('PRAGMA cache_size=-32000') + this.db.exec('PRAGMA temp_store=MEMORY') + // Increase wal autocheckpoint interval to reduce I/O during heavy writes + this.db.exec('PRAGMA wal_autocheckpoint=10000') + this.db.exec(` + CREATE TABLE IF NOT EXISTS package_index ( + key TEXT PRIMARY KEY, + data BLOB NOT NULL + ) WITHOUT ROWID + `) + }) + this.stmtGet = this.db.prepare('SELECT data FROM package_index WHERE key = ?') + this.stmtSet = this.db.prepare('INSERT OR REPLACE INTO package_index (key, data) VALUES (?, ?)') + this.stmtDel = this.db.prepare('DELETE FROM package_index WHERE key = ?') + this.stmtHas = this.db.prepare('SELECT 1 FROM package_index WHERE key = ?') + this.stmtAll = this.db.prepare('SELECT key, data FROM package_index') + this.exitHandler = () => this.close() + process.on('exit', this.exitHandler) + openInstances.add(this) + } + + get (key: string): unknown | undefined { + const row = sqliteRetry(() => this.stmtGet.get(key)) as { data: Uint8Array } | undefined + if (row) { + return packr.unpack(row.data) + } + return undefined + } + + set (key: string, data: unknown): void { + const buffer = packr.pack(data) + sqliteRetry(() => { + this.stmtSet.run(key, buffer) + }) + } + + delete (key: string): boolean { + let result!: { changes: number | bigint } + sqliteRetry(() => { + result = this.stmtDel.run(key) + }) + return result.changes > 0 + } + + has (key: string): boolean { + return sqliteRetry(() => this.stmtHas.get(key)) != null + } + + /** + * Iterate over all index entries. + * Yields [key, data] pairs where key is `integrity\tpkgId`. + */ + * entries (): IterableIterator<[string, unknown]> { + for (const row of this.stmtAll.iterate() as IterableIterator<{ key: string, data: Uint8Array }>) { + yield [row.key, packr.unpack(row.data)] + } + } + + /** + * Queue pre-packed writes to be flushed on the next tick. + * Used by the fetch phase for throughput. + */ + queueWrites (writes: Array<{ key: string, buffer: Uint8Array }>): void { + for (const w of writes) { + this.pendingWrites.push(w) + } + if (!this.flushScheduled) { + this.flushScheduled = true + process.nextTick(() => this.flush()) + } + } + + /** + * Flush all pending queued writes immediately. + */ + flush (): void { + this.flushScheduled = false + if (this.pendingWrites.length === 0) return + this.setRawMany(this.pendingWrites) + this.pendingWrites = [] + } + + /** + * Write multiple pre-packed entries in a single transaction. + * The buffers must already be msgpack-encoded. + */ + setRawMany (entries: Array<{ key: string, buffer: Uint8Array }>): void { + if (this.closed || entries.length === 0) return + if (entries.length === 1) { + sqliteRetry(() => { + this.stmtSet.run(entries[0].key, entries[0].buffer) + }) + return + } + sqliteRetry(() => { + this.db.exec('BEGIN IMMEDIATE') + let committed = false + try { + for (const { key, buffer } of entries) { + this.stmtSet.run(key, buffer) + } + this.db.exec('COMMIT') + committed = true + } finally { + if (!committed) { + try { + this.db.exec('ROLLBACK') + } catch {} + } + } + }) + } + + /** + * Delete multiple index entries in a single transaction, + * then VACUUM to reclaim disk space. + */ + deleteMany (keys: string[]): void { + if (keys.length === 0) return + if (keys.length === 1) { + this.delete(keys[0]) + this.db.exec('VACUUM') + return + } + sqliteRetry(() => { + this.db.exec('BEGIN IMMEDIATE') + let committed = false + try { + for (const key of keys) { + this.stmtDel.run(key) + } + this.db.exec('COMMIT') + committed = true + } finally { + if (!committed) { + try { + this.db.exec('ROLLBACK') + } catch {} + } + } + }) + this.db.exec('VACUUM') + } + + close (): void { + if (this.closed) return + this.flush() + this.closed = true + openInstances.delete(this) + process.removeListener('exit', this.exitHandler) + try { + this.db.exec('PRAGMA optimize') + } catch { + // PRAGMA optimize is a performance hint; safe to ignore if the DB is locked. + } + try { + this.db.close() + } catch { + // The DB may be locked by another connection; the OS will reclaim it on process exit. + } + } +} diff --git a/store/index/test/index.ts b/store/index/test/index.ts new file mode 100644 index 0000000000..5a977e6b3f --- /dev/null +++ b/store/index/test/index.ts @@ -0,0 +1,46 @@ +import { StoreIndex, storeIndexKey } from '@pnpm/store.index' +import path from 'path' +import { temporaryDirectory } from 'tempy' + +test('StoreIndex round-trips data via SQLite key', () => { + const storeDir = path.join(temporaryDirectory(), 'store', 'v11') + const idx = new StoreIndex(storeDir) + try { + const key = storeIndexKey('sha512-abc123', 'lodash@4.17.21') + expect(idx.get(key)).toBeUndefined() + + const data = { algo: 'sha512', files: new Map([['index.js', { digest: 'abc', size: 100, mode: 0o644 }]]) } + idx.set(key, data) + + const result = idx.get(key) as typeof data + expect(result).toBeDefined() + expect(result.algo).toBe('sha512') + expect(result.files.get('index.js')?.digest).toBe('abc') + + expect(idx.has(key)).toBe(true) + expect(idx.delete(key)).toBe(true) + expect(idx.get(key)).toBeUndefined() + expect(idx.has(key)).toBe(false) + } finally { + idx.close() + } +}) + +test('StoreIndex entries() iterates all SQLite entries', () => { + const storeDir = path.join(temporaryDirectory(), 'store', 'v11') + const idx = new StoreIndex(storeDir) + try { + const key1 = storeIndexKey('sha512-aaa', 'pkg-a@1.0.0') + const key2 = storeIndexKey('sha512-bbb', 'pkg-b@2.0.0') + idx.set(key1, { a: 1 }) + idx.set(key2, { b: 2 }) + + const entries = [...idx.entries()] + expect(entries).toHaveLength(2) + const keys = entries.map(([k]) => k) + expect(keys).toContain(key1) + expect(keys).toContain(key2) + } finally { + idx.close() + } +}) diff --git a/store/index/test/tsconfig.json b/store/index/test/tsconfig.json new file mode 100644 index 0000000000..67ce5e1d0e --- /dev/null +++ b/store/index/test/tsconfig.json @@ -0,0 +1,18 @@ +{ + "extends": "../tsconfig.json", + "compilerOptions": { + "noEmit": false, + "outDir": "../node_modules/.test.lib", + "rootDir": "..", + "isolatedModules": true + }, + "include": [ + "**/*.ts", + "../../../__typings__/**/*.d.ts" + ], + "references": [ + { + "path": ".." + } + ] +} diff --git a/store/index/tsconfig.json b/store/index/tsconfig.json new file mode 100644 index 0000000000..c6f0399f60 --- /dev/null +++ b/store/index/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "@pnpm/tsconfig", + "compilerOptions": { + "outDir": "lib", + "rootDir": "src" + }, + "include": [ + "src/**/*.ts", + "../../__typings__/**/*.d.ts" + ], + "references": [] +} diff --git a/store/index/tsconfig.lint.json b/store/index/tsconfig.lint.json new file mode 100644 index 0000000000..1bbe711971 --- /dev/null +++ b/store/index/tsconfig.lint.json @@ -0,0 +1,8 @@ +{ + "extends": "./tsconfig.json", + "include": [ + "src/**/*.ts", + "test/**/*.ts", + "../../__typings__/**/*.d.ts" + ] +} diff --git a/store/package-store/package.json b/store/package-store/package.json index 09549ddf2f..a22246b80a 100644 --- a/store/package-store/package.json +++ b/store/package-store/package.json @@ -48,12 +48,12 @@ "@pnpm/crypto.hash": "workspace:*", "@pnpm/error": "workspace:*", "@pnpm/fetcher-base": "workspace:*", - "@pnpm/fs.msgpack-file": "workspace:*", "@pnpm/hooks.types": "workspace:*", "@pnpm/package-requester": "workspace:*", "@pnpm/resolver-base": "workspace:*", "@pnpm/store-controller-types": "workspace:*", "@pnpm/store.cafs": "workspace:*", + "@pnpm/store.index": "workspace:*", "@pnpm/types": "workspace:*", "@zkochan/rimraf": "catalog:", "is-subdir": "catalog:", diff --git a/store/package-store/src/storeController/index.ts b/store/package-store/src/storeController/index.ts index 9e67019820..ebe788e090 100644 --- a/store/package-store/src/storeController/index.ts +++ b/store/package-store/src/storeController/index.ts @@ -9,6 +9,7 @@ import { type StoreController, } from '@pnpm/store-controller-types' import { type CustomFetcher } from '@pnpm/hooks.types' +import { type StoreIndex } from '@pnpm/store.index' import { addFilesFromDir, importPackage, initStoreDir } from '@pnpm/worker' import { prune } from './prune.js' @@ -31,6 +32,7 @@ export interface CreatePackageStoreOptions { strictStorePkgContentCheck?: boolean clearResolutionCache: () => void customFetchers?: CustomFetcher[] + storeIndex: StoreIndex } export function createPackageStore ( @@ -64,7 +66,9 @@ export function createPackageStore ( }) return { - close: async () => {}, // eslint-disable-line:no-empty + close: async () => { + initOpts.storeIndex.flush() + }, fetchPackage: packageRequester.fetchPackageToStore, getFilesIndexFilePath: packageRequester.getFilesIndexFilePath, importPackage: initOpts.importPackage @@ -75,7 +79,7 @@ export function createPackageStore ( storeDir: initOpts.storeDir, targetDir, }), - prune: prune.bind(null, { storeDir, cacheDir: initOpts.cacheDir }), + prune: prune.bind(null, { storeDir, cacheDir: initOpts.cacheDir, storeIndex: initOpts.storeIndex }), requestPackage: packageRequester.requestPackage, upload, clearResolutionCache: initOpts.clearResolutionCache, @@ -84,6 +88,7 @@ export function createPackageStore ( async function upload (builtPkgLocation: string, opts: { filesIndexFile: string, sideEffectsCacheKey: string }): Promise { await addFilesFromDir({ storeDir: cafs.storeDir, + storeIndex: initOpts.storeIndex, dir: builtPkgLocation, sideEffectsCacheKey: opts.sideEffectsCacheKey, filesIndexFile: opts.filesIndexFile, diff --git a/store/package-store/src/storeController/prune.ts b/store/package-store/src/storeController/prune.ts index 87cef4df6a..5cf4cdc7db 100644 --- a/store/package-store/src/storeController/prune.ts +++ b/store/package-store/src/storeController/prune.ts @@ -1,7 +1,7 @@ import { type Dirent, promises as fs } from 'fs' import util from 'util' import path from 'path' -import { readMsgpackFile } from '@pnpm/fs.msgpack-file' +import { type StoreIndex } from '@pnpm/store.index' import { type PackageFilesIndex } from '@pnpm/store.cafs' import { globalInfo, globalWarn } from '@pnpm/logger' import rimraf from '@zkochan/rimraf' @@ -12,9 +12,10 @@ const BIG_ONE = BigInt(1) as unknown export interface PruneOptions { cacheDir: string storeDir: string + storeIndex: StoreIndex } -export async function prune ({ cacheDir, storeDir }: PruneOptions, removeAlienFiles?: boolean): Promise { +export async function prune ({ cacheDir, storeDir, storeIndex }: PruneOptions, removeAlienFiles?: boolean): Promise { // 1. First, prune the global virtual store // This must happen BEFORE pruning the CAS, because removing packages from // the virtual store will reduce hard link counts on files in the CAS @@ -37,17 +38,6 @@ export async function prune ({ cacheDir, storeDir }: PruneOptions, removeAlienFi // 3. Prune the content-addressable store (CAS) const cafsDir = path.join(storeDir, 'files') - const pkgIndexFiles = [] as string[] - const indexDir = path.join(storeDir, 'index') - await Promise.all((await getSubdirsSafely(indexDir)).map(async (dir) => { - const subdir = path.join(indexDir, dir) - await Promise.all((await fs.readdir(subdir)).map(async (fileName) => { - const filePath = path.join(subdir, fileName) - if (fileName.endsWith('.mpk')) { - pkgIndexFiles.push(filePath) - } - })) - })) const removedHashes = new Set() const dirs = await getSubdirsSafely(cafsDir) let fileCounter = 0 @@ -55,10 +45,6 @@ export async function prune ({ cacheDir, storeDir }: PruneOptions, removeAlienFi const subdir = path.join(cafsDir, dir) await Promise.all((await fs.readdir(subdir)).map(async (fileName) => { const filePath = path.join(subdir, fileName) - if (fileName.endsWith('.mpk')) { - pkgIndexFiles.push(filePath) - return - } const stat = await fs.stat(filePath) if (stat.isDirectory()) { if (removeAlienFiles) { @@ -82,17 +68,19 @@ export async function prune ({ cacheDir, storeDir }: PruneOptions, removeAlienFi })) globalInfo(`Removed ${fileCounter} file${fileCounter === 1 ? '' : 's'}`) - // 4. Clean up orphaned package index files + // 4. Clean up orphaned package index entries let pkgCounter = 0 - await Promise.all(pkgIndexFiles.map(async (pkgIndexFilePath) => { - const pkgFilesIndex = await readMsgpackFile(pkgIndexFilePath) + const toDelete: string[] = [] + for (const [filesIndexFile, data] of storeIndex.entries()) { + const pkgFilesIndex = data as PackageFilesIndex const pkgJson = pkgFilesIndex.files.get('package.json') // TODO: implement prune of Node.js packages, they don't have a package.json file if (pkgJson && removedHashes.has(pkgJson.digest)) { - await fs.unlink(pkgIndexFilePath) + toDelete.push(filesIndexFile) pkgCounter++ } - })) + } + storeIndex.deleteMany(toDelete) globalInfo(`Removed ${pkgCounter} package${pkgCounter === 1 ? '' : 's'}`) } diff --git a/store/package-store/test/index.ts b/store/package-store/test/index.ts index d7d2a18fec..cf87cf53fc 100644 --- a/store/package-store/test/index.ts +++ b/store/package-store/test/index.ts @@ -3,6 +3,7 @@ import path from 'path' import { createClient } from '@pnpm/client' import { createPackageStore } from '@pnpm/package-store' import { type FetchPackageToStoreFunction } from '@pnpm/store-controller-types' +import { StoreIndex } from '@pnpm/store.index' import { temporaryDirectory } from 'tempy' describe('store.importPackage()', () => { @@ -12,11 +13,13 @@ describe('store.importPackage()', () => { const cacheDir = path.join(tmp, 'cache') const registry = 'https://registry.npmjs.org/' const authConfig = { registry } + const storeIndex = new StoreIndex(storeDir) const { resolve, fetchers, clearResolutionCache } = createClient({ authConfig, cacheDir: path.join(tmp, 'cache'), storeDir: path.join(tmp, 'store'), rawConfig: {}, + storeIndex, registries: { default: registry, }, @@ -27,6 +30,7 @@ describe('store.importPackage()', () => { verifyStoreIntegrity: true, virtualStoreDirMaxLength: 120, clearResolutionCache, + storeIndex, }) const pkgId = 'registry.npmjs.org/is-positive/1.0.0' const fetchResponse = (storeController.fetchPackage as FetchPackageToStoreFunction)({ @@ -55,11 +59,13 @@ describe('store.importPackage()', () => { const cacheDir = path.join(tmp, 'cache') const registry = 'https://registry.npmjs.org/' const authConfig = { registry } + const storeIndex = new StoreIndex(storeDir) const { resolve, fetchers, clearResolutionCache } = createClient({ authConfig, cacheDir: path.join(tmp, 'cache'), storeDir: path.join(tmp, 'store'), rawConfig: {}, + storeIndex, registries: { default: registry, }, @@ -71,6 +77,7 @@ describe('store.importPackage()', () => { verifyStoreIntegrity: true, virtualStoreDirMaxLength: 120, clearResolutionCache, + storeIndex, }) const pkgId = 'registry.npmjs.org/is-positive/1.0.0' const fetchResponse = (storeController.fetchPackage as FetchPackageToStoreFunction)({ diff --git a/store/package-store/tsconfig.json b/store/package-store/tsconfig.json index a802c0cf53..c735b3ef86 100644 --- a/store/package-store/tsconfig.json +++ b/store/package-store/tsconfig.json @@ -18,9 +18,6 @@ { "path": "../../fetching/fetcher-base" }, - { - "path": "../../fs/msgpack-file" - }, { "path": "../../hooks/types" }, @@ -51,6 +48,9 @@ { "path": "../create-cafs-store" }, + { + "path": "../index" + }, { "path": "../store-controller-types" } diff --git a/store/pkg-finder/package.json b/store/pkg-finder/package.json index f8ccd3e977..dfcfe72dfb 100644 --- a/store/pkg-finder/package.json +++ b/store/pkg-finder/package.json @@ -32,9 +32,9 @@ "dependencies": { "@pnpm/dependency-path": "workspace:*", "@pnpm/directory-fetcher": "workspace:*", - "@pnpm/fs.msgpack-file": "workspace:*", "@pnpm/resolver-base": "workspace:*", - "@pnpm/store.cafs": "workspace:*" + "@pnpm/store.cafs": "workspace:*", + "@pnpm/store.index": "workspace:*" }, "devDependencies": { "@pnpm/store.pkg-finder": "workspace:*" diff --git a/store/pkg-finder/src/index.ts b/store/pkg-finder/src/index.ts index 6f8d98d9bb..56b6184159 100644 --- a/store/pkg-finder/src/index.ts +++ b/store/pkg-finder/src/index.ts @@ -1,12 +1,13 @@ import path from 'path' -import { depPathToFilename, parse } from '@pnpm/dependency-path' +import { parse } from '@pnpm/dependency-path' import { fetchFromDir } from '@pnpm/directory-fetcher' -import { readMsgpackFile } from '@pnpm/fs.msgpack-file' +import { type StoreIndex, storeIndexKey, gitHostedStoreIndexKey } from '@pnpm/store.index' import { type Resolution } from '@pnpm/resolver-base' -import { getFilePathByModeInCafs, getIndexFilePathInCafs, type PackageFilesIndex } from '@pnpm/store.cafs' +import { getFilePathByModeInCafs, type PackageFilesIndex } from '@pnpm/store.cafs' export interface ReadPackageFileMapOptions { storeDir: string + storeIndex: StoreIndex lockfileDir: string virtualStoreDirMaxLength: number } @@ -47,23 +48,26 @@ export async function readPackageFileMap ( let pkgIndexFilePath: string if (isPackageWithIntegrity) { const parsedId = parse(packageId) - pkgIndexFilePath = getIndexFilePathInCafs( - opts.storeDir, + pkgIndexFilePath = storeIndexKey( packageResolution.integrity as string, parsedId.nonSemverVersion ?? `${parsedId.name}@${parsedId.version}` ) } else if (!packageResolution.type && 'tarball' in packageResolution && packageResolution.tarball) { - const packageDirInStore = depPathToFilename(parse(packageId).nonSemverVersion ?? packageId, opts.virtualStoreDirMaxLength) - pkgIndexFilePath = path.join( - opts.storeDir, - packageDirInStore, - 'integrity.mpk' - ) + pkgIndexFilePath = gitHostedStoreIndexKey(packageId, { built: true }) } else { return undefined } - const { files: indexFiles } = await readMsgpackFile(pkgIndexFilePath) + const pkgFilesIndex = opts.storeIndex.get(pkgIndexFilePath) as PackageFilesIndex | undefined + if (!pkgFilesIndex) { + const err: NodeJS.ErrnoException = new Error( + `ENOENT: package index not found for '${pkgIndexFilePath}'` + ) + err.code = 'ENOENT' + err.path = pkgIndexFilePath + throw err + } + const { files: indexFiles } = pkgFilesIndex const files = new Map() for (const [name, info] of indexFiles) { files.set(name, getFilePathByModeInCafs(opts.storeDir, info.digest, info.mode)) diff --git a/store/pkg-finder/tsconfig.json b/store/pkg-finder/tsconfig.json index 2a4510ed93..10b78285b6 100644 --- a/store/pkg-finder/tsconfig.json +++ b/store/pkg-finder/tsconfig.json @@ -12,9 +12,6 @@ { "path": "../../fetching/directory-fetcher" }, - { - "path": "../../fs/msgpack-file" - }, { "path": "../../packages/dependency-path" }, @@ -23,6 +20,9 @@ }, { "path": "../cafs" + }, + { + "path": "../index" } ] } diff --git a/store/plugin-commands-store-inspecting/package.json b/store/plugin-commands-store-inspecting/package.json index 2f6eab4437..0a2782096e 100644 --- a/store/plugin-commands-store-inspecting/package.json +++ b/store/plugin-commands-store-inspecting/package.json @@ -34,13 +34,13 @@ "@pnpm/client": "workspace:*", "@pnpm/config": "workspace:*", "@pnpm/error": "workspace:*", - "@pnpm/fs.msgpack-file": "workspace:*", "@pnpm/graceful-fs": "workspace:*", "@pnpm/lockfile.types": "workspace:*", "@pnpm/object.key-sorting": "workspace:*", "@pnpm/parse-wanted-dependency": "workspace:*", "@pnpm/store-path": "workspace:*", "@pnpm/store.cafs": "workspace:*", + "@pnpm/store.index": "workspace:*", "@pnpm/types": "workspace:*", "@pnpm/util.lex-comparator": "catalog:", "chalk": "catalog:", diff --git a/store/plugin-commands-store-inspecting/src/catIndex.ts b/store/plugin-commands-store-inspecting/src/catIndex.ts index cc566bebdb..94efea858f 100644 --- a/store/plugin-commands-store-inspecting/src/catIndex.ts +++ b/store/plugin-commands-store-inspecting/src/catIndex.ts @@ -4,10 +4,10 @@ import { createResolver } from '@pnpm/client' import { type TarballResolution } from '@pnpm/lockfile.types' import { PnpmError } from '@pnpm/error' -import { readMsgpackFile } from '@pnpm/fs.msgpack-file' import { sortDeepKeys } from '@pnpm/object.key-sorting' +import { StoreIndex, storeIndexKey } from '@pnpm/store.index' import { getStorePath } from '@pnpm/store-path' -import { getIndexFilePathInCafs, type PackageFilesIndex } from '@pnpm/store.cafs' +import { type PackageFilesIndex } from '@pnpm/store.cafs' import { parseWantedDependency } from '@pnpm/parse-wanted-dependency' import { lexCompare } from '@pnpm/util.lex-comparator' @@ -82,19 +82,22 @@ export async function handler (opts: CatIndexCommandOptions, params: string[]): } ) - const filesIndexFile = getIndexFilePathInCafs( - storeDir, + const filesIndexFile = storeIndexKey( (pkgSnapshot.resolution as TarballResolution).integrity!.toString(), `${alias}@${bareSpecifier}` ) + const storeIndex = new StoreIndex(storeDir) try { - const pkgFilesIndex = await readMsgpackFile(filesIndexFile) + const pkgFilesIndex = storeIndex.get(filesIndexFile) as PackageFilesIndex | undefined + if (!pkgFilesIndex) { + throw new PnpmError( + 'INVALID_PACKAGE', + 'No corresponding index file found. You can use pnpm list to see if the package is installed.' + ) + } return JSON.stringify(sortDeepKeys(pkgFilesIndex), replacer, 2) - } catch { - throw new PnpmError( - 'INVALID_PACKAGE', - 'No corresponding index file found. You can use pnpm list to see if the package is installed.' - ) + } finally { + storeIndex.close() } } diff --git a/store/plugin-commands-store-inspecting/src/findHash.ts b/store/plugin-commands-store-inspecting/src/findHash.ts index b52eb48b7a..821b8b7974 100644 --- a/store/plugin-commands-store-inspecting/src/findHash.ts +++ b/store/plugin-commands-store-inspecting/src/findHash.ts @@ -1,10 +1,8 @@ -import path from 'path' -import fs from 'fs' import chalk from 'chalk' import { type Config } from '@pnpm/config' import { PnpmError } from '@pnpm/error' -import { readMsgpackFileSync } from '@pnpm/fs.msgpack-file' +import { StoreIndex } from '@pnpm/store.index' import { getStorePath } from '@pnpm/store-path' import { type PackageFilesIndex } from '@pnpm/store.cafs' @@ -36,7 +34,7 @@ export type FindHashCommandOptions = Pick export interface FindHashResult { name: string version: string - filesIndexFile: string + indexKey: string } export async function handler (opts: FindHashCommandOptions, params: string[]): Promise { @@ -61,52 +59,41 @@ export async function handler (opts: FindHashCommandOptions, params: string[]): storePath: opts.storeDir, pnpmHomeDir: opts.pnpmHomeDir, }) - const indexDir = path.join(storeDir, 'index') - const cafsChildrenDirs = fs.readdirSync(indexDir, { withFileTypes: true }).filter(file => file.isDirectory()) - const indexFiles: string[] = []; const result: FindHashResult[] = [] + const result: FindHashResult[] = [] + const storeIndex = new StoreIndex(storeDir) - for (const { name: dirName } of cafsChildrenDirs) { - const dirIndexFiles = fs - .readdirSync(`${indexDir}/${dirName}`) - .filter((fileName) => fileName.includes('.mpk')) - ?.map((fileName) => `${indexDir}/${dirName}/${fileName}`) + try { + for (const [indexKey, data] of storeIndex.entries()) { + const pkgFilesIndex = data as PackageFilesIndex + if (!pkgFilesIndex) continue - indexFiles.push(...dirIndexFiles) - } - - for (const filesIndexFile of indexFiles) { - let pkgFilesIndex: PackageFilesIndex | undefined - try { - pkgFilesIndex = readMsgpackFileSync(filesIndexFile) - } catch { - continue - } - if (!pkgFilesIndex) continue - - if (pkgFilesIndex.files) { - for (const file of pkgFilesIndex.files.values()) { - if (file?.digest === hash) { - result.push({ name: pkgFilesIndex.manifest?.name ?? 'unknown', version: pkgFilesIndex.manifest?.version ?? 'unknown', filesIndexFile: filesIndexFile.replace(indexDir, '') }) - - // a package is only found once. - continue - } - } - } - - if (pkgFilesIndex.sideEffects) { - for (const { added } of pkgFilesIndex.sideEffects.values()) { - if (!added) continue - for (const file of added.values()) { + if (pkgFilesIndex.files) { + for (const file of pkgFilesIndex.files.values()) { if (file?.digest === hash) { - result.push({ name: pkgFilesIndex.manifest?.name ?? 'unknown', version: pkgFilesIndex.manifest?.version ?? 'unknown', filesIndexFile: filesIndexFile.replace(indexDir, '') }) + result.push({ name: pkgFilesIndex.manifest?.name ?? 'unknown', version: pkgFilesIndex.manifest?.version ?? 'unknown', indexKey }) // a package is only found once. continue } } } + + if (pkgFilesIndex.sideEffects) { + for (const { added } of pkgFilesIndex.sideEffects.values()) { + if (!added) continue + for (const file of added.values()) { + if (file?.digest === hash) { + result.push({ name: pkgFilesIndex.manifest?.name ?? 'unknown', version: pkgFilesIndex.manifest?.version ?? 'unknown', indexKey }) + + // a package is only found once. + continue + } + } + } + } } + } finally { + storeIndex.close() } if (!result.length) { @@ -117,8 +104,8 @@ export async function handler (opts: FindHashCommandOptions, params: string[]): } let acc = '' - for (const { name, version, filesIndexFile } of result) { - acc += `${PACKAGE_INFO_CLR(name)}@${PACKAGE_INFO_CLR(version)} ${INDEX_PATH_CLR(filesIndexFile)}\n` + for (const { name, version, indexKey } of result) { + acc += `${PACKAGE_INFO_CLR(name)}@${PACKAGE_INFO_CLR(version)} ${INDEX_PATH_CLR(indexKey)}\n` } return acc } diff --git a/store/plugin-commands-store-inspecting/test/findHash.ts b/store/plugin-commands-store-inspecting/test/findHash.ts index 1c03c49ba9..8a6bbe6332 100644 --- a/store/plugin-commands-store-inspecting/test/findHash.ts +++ b/store/plugin-commands-store-inspecting/test/findHash.ts @@ -12,7 +12,7 @@ import { temporaryDirectory } from 'tempy' const pnpmBin = path.join(import.meta.dirname, '../../../pnpm/bin/pnpm.mjs') test('print index file path with hash', async () => { - const { PACKAGE_INFO_CLR, INDEX_PATH_CLR } = findHash + const { PACKAGE_INFO_CLR } = findHash prepare() const tmp = temporaryDirectory() const storeDir = path.join(tmp, 'store') @@ -26,9 +26,12 @@ test('print index file path with hash', async () => { storeDir, }, ['sha512-fXs1pWlUdqT2jkeoEJW/+odKZ2NwAyYkWea+plJKZI2xmhRKQi2e+nKGcClyDblgLwCLD912oMaua0+sTwwIrw==']) - expect(output).toBe(`${PACKAGE_INFO_CLR('lodash')}@${PACKAGE_INFO_CLR('4.17.19')} ${INDEX_PATH_CLR('/24/dbddf17111f46417d2fdaa260b1a37f9b3142340e4145efe3f0937d77eb56c-lodash@4.17.19.mpk')} -${PACKAGE_INFO_CLR('lodash')}@${PACKAGE_INFO_CLR('4.17.20')} ${INDEX_PATH_CLR('/3e/585d15c8a594e20d7de57b362ea81754c011acb2641a19f1b72c8531ea3982-lodash@4.17.20.mpk')} -`) + // The output contains colored package info and SQLite index keys (integrity\tpkgId) + expect(output).toContain(PACKAGE_INFO_CLR('lodash')) + expect(output).toContain(PACKAGE_INFO_CLR('4.17.19')) + expect(output).toContain(PACKAGE_INFO_CLR('4.17.20')) + expect(output).toContain('lodash@4.17.19') + expect(output).toContain('lodash@4.17.20') } }) diff --git a/store/plugin-commands-store-inspecting/tsconfig.json b/store/plugin-commands-store-inspecting/tsconfig.json index 0da273904c..889bb9fca3 100644 --- a/store/plugin-commands-store-inspecting/tsconfig.json +++ b/store/plugin-commands-store-inspecting/tsconfig.json @@ -18,9 +18,6 @@ { "path": "../../fs/graceful-fs" }, - { - "path": "../../fs/msgpack-file" - }, { "path": "../../lockfile/types" }, @@ -42,6 +39,9 @@ { "path": "../cafs" }, + { + "path": "../index" + }, { "path": "../store-path" } diff --git a/store/plugin-commands-store/package.json b/store/plugin-commands-store/package.json index 139dd3773a..6af3ac3056 100644 --- a/store/plugin-commands-store/package.json +++ b/store/plugin-commands-store/package.json @@ -37,7 +37,6 @@ "@pnpm/crypto.integrity": "workspace:*", "@pnpm/dependency-path": "workspace:*", "@pnpm/error": "workspace:*", - "@pnpm/fs.msgpack-file": "workspace:*", "@pnpm/get-context": "workspace:*", "@pnpm/global.packages": "workspace:*", "@pnpm/lockfile.utils": "workspace:*", @@ -47,6 +46,7 @@ "@pnpm/store-controller-types": "workspace:*", "@pnpm/store-path": "workspace:*", "@pnpm/store.cafs": "workspace:*", + "@pnpm/store.index": "workspace:*", "@pnpm/types": "workspace:*", "archy": "catalog:", "dint": "catalog:", diff --git a/store/plugin-commands-store/src/storeStatus/index.ts b/store/plugin-commands-store/src/storeStatus/index.ts index 4a086a5540..490017c392 100644 --- a/store/plugin-commands-store/src/storeStatus/index.ts +++ b/store/plugin-commands-store/src/storeStatus/index.ts @@ -1,6 +1,7 @@ import path from 'path' import { formatIntegrity } from '@pnpm/crypto.integrity' -import { getIndexFilePathInCafs, type PackageFilesIndex } from '@pnpm/store.cafs' +import { type PackageFilesIndex } from '@pnpm/store.cafs' +import { storeIndexKey, gitHostedStoreIndexKey } from '@pnpm/store.index' import { getContextForSingleImporter } from '@pnpm/get-context' import { nameVerFromPkgSnapshot, @@ -9,7 +10,7 @@ import { } from '@pnpm/lockfile.utils' import { streamParser } from '@pnpm/logger' import * as dp from '@pnpm/dependency-path' -import { readMsgpackFile } from '@pnpm/fs.msgpack-file' +import { StoreIndex } from '@pnpm/store.index' import { type DepPath } from '@pnpm/types' import dint from 'dint' import pFilter from 'p-filter' @@ -49,25 +50,34 @@ export async function storeStatus (maybeOpts: StoreStatusOptions): Promise { - const pkgIndexFilePath = integrity - ? getIndexFilePathInCafs(storeDir, integrity, id) - : path.join(storeDir, dp.depPathToFilename(id, maybeOpts.virtualStoreDirMaxLength), 'integrity.mpk') - const { algo, files } = await readMsgpackFile(pkgIndexFilePath) - // Transform files to dint format: { integrity: '-', size: number } - const dintFiles: Record = {} - for (const [filePath, { digest, size }] of files) { - dintFiles[filePath] = { - integrity: formatIntegrity(algo, digest), - size, + const storeIndex = new StoreIndex(storeDir) + try { + const modified = await pFilter(pkgs, async ({ id, integrity, depPath, name }) => { + const pkgIndexFilePath = integrity + ? storeIndexKey(integrity, id) + : gitHostedStoreIndexKey(id, { built: true }) + const pkgFilesIndex = storeIndex.get(pkgIndexFilePath) as PackageFilesIndex | undefined + if (!pkgFilesIndex) { + return false } + const { algo, files } = pkgFilesIndex + // Transform files to dint format: { integrity: '-', size: number } + const dintFiles: Record = {} + for (const [filePath, { digest, size }] of files) { + dintFiles[filePath] = { + integrity: formatIntegrity(algo, digest), + size, + } + } + return (await dint.check(path.join(virtualStoreDir, dp.depPathToFilename(depPath, maybeOpts.virtualStoreDirMaxLength), 'node_modules', name), dintFiles)) === false + }, { concurrency: 8 }) + + if ((reporter != null) && typeof reporter === 'function') { + streamParser.removeListener('data', reporter) } - return (await dint.check(path.join(virtualStoreDir, dp.depPathToFilename(depPath, maybeOpts.virtualStoreDirMaxLength), 'node_modules', name), dintFiles)) === false - }, { concurrency: 8 }) - if ((reporter != null) && typeof reporter === 'function') { - streamParser.removeListener('data', reporter) + return modified.map(({ pkgPath }) => pkgPath) + } finally { + storeIndex.close() } - - return modified.map(({ pkgPath }) => pkgPath) } diff --git a/store/plugin-commands-store/test/storeStatus.ts b/store/plugin-commands-store/test/storeStatus.ts index 0664ff0cb9..aed876fc74 100644 --- a/store/plugin-commands-store/test/storeStatus.ts +++ b/store/plugin-commands-store/test/storeStatus.ts @@ -11,6 +11,11 @@ import { temporaryDirectory } from 'tempy' const REGISTRY = `http://localhost:${REGISTRY_MOCK_PORT}/` const pnpmBin = path.join(import.meta.dirname, '../../../pnpm/bin/pnpm.mjs') +// Use an empty config dir to ensure the subprocess is not affected by +// the user's global pnpm config (e.g. enable-global-virtual-store). +const cleanConfigDir = temporaryDirectory() +const execaOpts = { env: { XDG_CONFIG_HOME: cleanConfigDir } } + test('CLI fails when store status finds modified packages', async () => { const project = prepare() const tmp = temporaryDirectory() @@ -24,7 +29,7 @@ test('CLI fails when store status finds modified packages', async () => { `--store-dir=${storeDir}`, `--registry=${REGISTRY}`, '--verify-store-integrity', - ]) + ], execaOpts) rimraf('node_modules/.pnpm/is-positive@3.1.0/node_modules/is-positive/index.js') @@ -72,7 +77,7 @@ test('CLI does not fail when store status does not find modified packages', asyn 'react@15.4.1', 'webpack@5.24.2', 'koorchik/node-mole-rpc', - ]) + ], execaOpts) // store status does not fail on not installed optional dependencies await execa('node', [ pnpmBin, @@ -82,7 +87,7 @@ test('CLI does not fail when store status does not find modified packages', asyn `--store-dir=${storeDir}`, `--registry=${REGISTRY}`, '--verify-store-integrity', - ]) + ], execaOpts) const modulesState = project.readModulesManifest() await store.handler({ @@ -125,7 +130,7 @@ storeDir: "${relativeStoreDir}" 'is-positive@3.1.0', `--registry=${REGISTRY}`, '--verify-store-integrity', - ]) + ], execaOpts) const modulesState = project.readModulesManifest() diff --git a/store/plugin-commands-store/tsconfig.json b/store/plugin-commands-store/tsconfig.json index fa7e450e30..f5e8fb6177 100644 --- a/store/plugin-commands-store/tsconfig.json +++ b/store/plugin-commands-store/tsconfig.json @@ -30,9 +30,6 @@ { "path": "../../exec/plugin-commands-script-runners" }, - { - "path": "../../fs/msgpack-file" - }, { "path": "../../global/packages" }, @@ -66,6 +63,9 @@ { "path": "../cafs" }, + { + "path": "../index" + }, { "path": "../package-store" }, diff --git a/store/store-connection-manager/package.json b/store/store-connection-manager/package.json index b55b4ec07f..ea73d98851 100644 --- a/store/store-connection-manager/package.json +++ b/store/store-connection-manager/package.json @@ -36,6 +36,7 @@ "@pnpm/config": "workspace:*", "@pnpm/package-store": "workspace:*", "@pnpm/store-path": "workspace:*", + "@pnpm/store.index": "workspace:*", "dir-is-case-sensitive": "catalog:" }, "peerDependencies": { diff --git a/store/store-connection-manager/src/createNewStoreController.ts b/store/store-connection-manager/src/createNewStoreController.ts index dd3f2d2ef5..cfa313fd85 100644 --- a/store/store-connection-manager/src/createNewStoreController.ts +++ b/store/store-connection-manager/src/createNewStoreController.ts @@ -3,6 +3,7 @@ import { createClient, type ClientOptions } from '@pnpm/client' import { type Config } from '@pnpm/config' import { createPackageStore, type CafsLocker, type StoreController } from '@pnpm/package-store' import { packageManager } from '@pnpm/cli-meta' +import { StoreIndex } from '@pnpm/store.index' type CreateResolverOptions = Pick filename !== 'package.json', storeDir, + storeIndex, verifyStoreIntegrity: true, virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, clearResolutionCache, diff --git a/testing/temp-store/tsconfig.json b/testing/temp-store/tsconfig.json index cfc86146de..5792c12ed8 100644 --- a/testing/temp-store/tsconfig.json +++ b/testing/temp-store/tsconfig.json @@ -12,6 +12,9 @@ { "path": "../../pkg-manager/client" }, + { + "path": "../../store/index" + }, { "path": "../../store/package-store" }, diff --git a/worker/package.json b/worker/package.json index 8299529099..84df3701df 100644 --- a/worker/package.json +++ b/worker/package.json @@ -39,9 +39,9 @@ "@pnpm/error": "workspace:*", "@pnpm/exec.pkg-requires-build": "workspace:*", "@pnpm/fs.hard-link-dir": "workspace:*", - "@pnpm/fs.msgpack-file": "workspace:*", "@pnpm/graceful-fs": "workspace:*", "@pnpm/store.cafs": "workspace:*", + "@pnpm/store.index": "workspace:*", "@pnpm/symlink-dependency": "workspace:*", "@rushstack/worker-pool": "catalog:", "is-windows": "catalog:", diff --git a/worker/src/index.ts b/worker/src/index.ts index 0456e993d6..9643c6aef4 100644 --- a/worker/src/index.ts +++ b/worker/src/index.ts @@ -9,6 +9,7 @@ import { type PackageFilesResponse, type FilesMap } from '@pnpm/cafs-types' import { type BundledManifest } from '@pnpm/types' import pLimit from 'p-limit' import { globalWarn } from '@pnpm/logger' +import { type StoreIndex } from '@pnpm/store.index' import { type TarballExtractMessage, type AddDirToStoreMessage, @@ -26,7 +27,11 @@ export async function restartWorkerPool (): Promise { export async function finishWorkers (): Promise { // @ts-expect-error - await global.finishWorkers?.() + const finish = global.finishWorkers + // @ts-expect-error + global.finishWorkers = undefined + await finish?.() + workerPool = undefined } function createTarballWorkerPool (): WorkerPool { @@ -74,7 +79,9 @@ interface AddFilesResult { integrity?: string } -type AddFilesFromDirOptions = Pick +type AddFilesFromDirOptions = Pick & { + storeIndex: StoreIndex +} export async function addFilesFromDir (opts: AddFilesFromDirOptions): Promise { if (!workerPool) { @@ -82,12 +89,17 @@ export async function addFilesFromDir (opts: AddFilesFromDirOptions): Promise((resolve, reject) => { - localWorker.once('message', ({ status, error, value }) => { + localWorker.once('message', ({ status, error, value, indexWrites }) => { workerPool!.checkinWorker(localWorker) if (status === 'error') { reject(new PnpmError(error.code ?? 'GIT_FETCH_FAILED', error.message as string)) return } + if (indexWrites) { + // Write immediately so that subsequent worker reads (e.g. side effects) + // see the committed data without waiting for nextTick. + opts.storeIndex.setRawMany(indexWrites) + } resolve(value) }) localWorker.postMessage({ @@ -140,6 +152,7 @@ If you think that this is the case, then run "pnpm store prune" and rerun the co } type AddFilesFromTarballOptions = Pick & { + storeIndex: StoreIndex url: string } @@ -149,7 +162,7 @@ export async function addFilesFromTarball (opts: AddFilesFromTarballOptions): Pr } const localWorker = await workerPool.checkoutWorkerAsync(true) return new Promise((resolve, reject) => { - localWorker.once('message', ({ status, error, value }) => { + localWorker.once('message', ({ status, error, value, indexWrites }) => { workerPool!.checkinWorker(localWorker) if (status === 'error') { if (error.type === 'integrity_validation_failed') { @@ -162,6 +175,9 @@ export async function addFilesFromTarball (opts: AddFilesFromTarballOptions): Pr reject(new PnpmError(error.code ?? 'TARBALL_EXTRACT', `Failed to add tarball from "${opts.url}" to store: ${error.message as string}`)) return } + if (indexWrites) { + opts.storeIndex.queueWrites(indexWrites) + } resolve(value) }) localWorker.postMessage({ diff --git a/worker/src/start.ts b/worker/src/start.ts index efe5b129b5..2d26e40457 100644 --- a/worker/src/start.ts +++ b/worker/src/start.ts @@ -2,11 +2,11 @@ import crypto from 'crypto' import path from 'path' import fs from 'fs' import { PnpmError } from '@pnpm/error' -import { type Cafs, type PackageFiles, type SideEffects, type SideEffectsDiff, type FilesMap } from '@pnpm/cafs-types' +import { type Cafs, type PackageFiles, type SideEffectsDiff, type FilesMap } from '@pnpm/cafs-types' import { createCafsStore } from '@pnpm/create-cafs-store' import { pkgRequiresBuild } from '@pnpm/exec.pkg-requires-build' import { hardLinkDir } from '@pnpm/fs.hard-link-dir' -import { readMsgpackFileSync, writeMsgpackFileSync } from '@pnpm/fs.msgpack-file' +import { StoreIndex, packForStorage } from '@pnpm/store.index' import { formatIntegrity, parseIntegrity } from '@pnpm/crypto.integrity' import { type CafsFunctions, @@ -17,7 +17,6 @@ import { normalizeBundledManifest, type PackageFilesIndex, type FilesIndex, - optimisticRenameOverwrite, type VerifyResult, } from '@pnpm/store.cafs' import { symlinkDependencySync } from '@pnpm/symlink-dependency' @@ -44,6 +43,14 @@ export function startWorker (): void { const cafsCache = new Map() const cafsStoreCache = new Map() const cafsLocker = new Map() +const storeIndexCache = new Map() + +function getStoreIndex (storeDir: string): StoreIndex { + if (!storeIndexCache.has(storeDir)) { + storeIndexCache.set(storeDir, new StoreIndex(storeDir)) + } + return storeIndexCache.get(storeDir)! +} async function handleMessage ( message: @@ -58,6 +65,13 @@ async function handleMessage ( ): Promise { if (message === false) { parentPort!.off('message', handleMessage) + // Explicitly close cached SQLite connections before exiting. + // process.exit() in a worker thread may not run C++ destructors, + // which would leave file descriptors and mmap regions open. + for (const idx of storeIndexCache.values()) { + idx.close() + } + storeIndexCache.clear() process.exit(0) } try { @@ -80,12 +94,7 @@ async function handleMessage ( } case 'readPkgFromCafs': { const { storeDir, filesIndexFile, verifyStoreIntegrity, expectedPkg, strictStorePkgContentCheck } = message - let pkgFilesIndex: PackageFilesIndex | undefined - try { - pkgFilesIndex = readMsgpackFileSync(filesIndexFile) - } catch { - // ignoring. It is fine if the integrity file is not present. Just refetch the package - } + const pkgFilesIndex = getStoreIndex(storeDir).get(filesIndexFile) as PackageFilesIndex | undefined if (!pkgFilesIndex) { parentPort!.postMessage({ status: 'success', @@ -195,7 +204,13 @@ function addTarballToStore ({ buffer, storeDir, integrity, filesIndexFile, appen } const { filesIntegrity, filesMap } = processFilesIndex(filesIndex) const bundledManifest = manifest != null ? normalizeBundledManifest(manifest) : undefined - const requiresBuild = writeFilesIndexFile(filesIndexFile, { algo: HASH_ALGORITHM, manifest: bundledManifest, files: filesIntegrity }) + const requiresBuild = pkgRequiresBuild(bundledManifest, filesIntegrity) + const pkgFilesIndex: PackageFilesIndex = { + requiresBuild, + manifest: bundledManifest, + algo: HASH_ALGORITHM, + files: filesIntegrity, + } return { status: 'success', value: { @@ -204,6 +219,7 @@ function addTarballToStore ({ buffer, storeDir, integrity, filesIndexFile, appen requiresBuild, integrity: integrity ?? calcIntegrity(buffer), }, + indexWrites: [{ key: filesIndexFile, buffer: packToShared(pkgFilesIndex) }], } } @@ -212,6 +228,19 @@ function calcIntegrity (buffer: Buffer): string { return formatIntegrity('sha512', calculatedHash) } +function packToShared (data: unknown): Uint8Array { + const packed = packForStorage(data) + const shared = new SharedArrayBuffer(packed.byteLength) + const view = new Uint8Array(shared) + view.set(packed) + return view +} + +interface IndexWrite { + key: string + buffer: Uint8Array +} + interface AddFilesFromDirResult { status: string value: { @@ -219,30 +248,32 @@ interface AddFilesFromDirResult { manifest?: BundledManifest requiresBuild: boolean } + indexWrites?: IndexWrite[] } function initStore ({ storeDir }: InitStoreMessage): { status: string } { fs.mkdirSync(storeDir, { recursive: true }) const hexChars = '0123456789abcdef'.split('') - for (const subDir of ['files', 'index']) { - const subDirPath = path.join(storeDir, subDir) - try { - fs.mkdirSync(subDirPath) - } catch { - // If a parallel process has already started creating the directories in the store, - // ignore if it already exists. - } - for (const hex1 of hexChars) { - for (const hex2 of hexChars) { - try { - fs.mkdirSync(path.join(subDirPath, `${hex1}${hex2}`)) - } catch { - // If a parallel process has already started creating the directories in the store, - // ignore if it already exists. - } + // Only create subdirectories for files/ — index/ is now managed by SQLite + const filesDirPath = path.join(storeDir, 'files') + try { + fs.mkdirSync(filesDirPath) + } catch { + // If a parallel process has already started creating the directories in the store, + // ignore if it already exists. + } + for (const hex1 of hexChars) { + for (const hex2 of hexChars) { + try { + fs.mkdirSync(path.join(filesDirPath, `${hex1}${hex2}`)) + } catch { + // If a parallel process has already started creating the directories in the store, + // ignore if it already exists. } } } + // Initialize the SQLite index database + getStoreIndex(storeDir) return { status: 'success' } } @@ -273,11 +304,10 @@ function addFilesFromDir ( const { filesIntegrity, filesMap } = processFilesIndex(filesIndex) const bundledManifest = manifest != null ? normalizeBundledManifest(manifest) : undefined let requiresBuild: boolean + let indexWrites: IndexWrite[] | undefined if (sideEffectsCacheKey) { - let existingFilesIndex!: PackageFilesIndex - try { - existingFilesIndex = readMsgpackFileSync(filesIndexFile) - } catch { + const existingFilesIndex = getStoreIndex(storeDir).get(filesIndexFile) as PackageFilesIndex | undefined + if (!existingFilesIndex) { // If there is no existing index file, then we cannot store the side effects. return { status: 'success', @@ -304,11 +334,18 @@ function addFilesFromDir ( } else { requiresBuild = existingFilesIndex.requiresBuild } - writeIndexFile(filesIndexFile, existingFilesIndex) + indexWrites = [{ key: filesIndexFile, buffer: packToShared(existingFilesIndex) }] } else { - requiresBuild = writeFilesIndexFile(filesIndexFile, { algo: HASH_ALGORITHM, manifest: bundledManifest, files: filesIntegrity }) + requiresBuild = pkgRequiresBuild(bundledManifest, filesIntegrity) + const pkgFilesIndex: PackageFilesIndex = { + requiresBuild, + manifest: bundledManifest, + algo: HASH_ALGORITHM, + files: filesIntegrity, + } + indexWrites = [{ key: filesIndexFile, buffer: packToShared(pkgFilesIndex) }] } - return { status: 'success', value: { filesMap, manifest: bundledManifest, requiresBuild } } + return { status: 'success', value: { filesMap, manifest: bundledManifest, requiresBuild }, indexWrites } } function addManifestToCafs (cafs: CafsFunctions, filesIndex: FilesIndex, manifest: DependencyManifest): void { @@ -412,36 +449,3 @@ function symlinkAllModules (opts: SymlinkAllModulesMessage): { status: 'success' return { status: 'success' } } -function writeFilesIndexFile ( - filesIndexFile: string, - { algo, manifest, files, sideEffects }: { - algo: string - manifest?: BundledManifest - files: PackageFiles - sideEffects?: SideEffects - } -): boolean { - const requiresBuild = pkgRequiresBuild(manifest, files) - const filesIndex: PackageFilesIndex = { - requiresBuild, - manifest, - algo, - files, - sideEffects, - } - writeIndexFile(filesIndexFile, filesIndex) - return requiresBuild -} - -function writeIndexFile (filePath: string, data: PackageFilesIndex): void { - const targetDir = path.dirname(filePath) - // TODO: use the API of @pnpm/cafs to write this file - // There is actually no need to create the directory in 99% of cases. - // So by using cafs API, we'll improve performance. - fs.mkdirSync(targetDir, { recursive: true }) - // Drop the last 10 characters and append the PID to create a shorter unique temp filename. - // This avoids ENAMETOOLONG errors on systems with path length limits. - const temp = `${filePath.slice(0, -10)}${process.pid}` - writeMsgpackFileSync(temp, data) - optimisticRenameOverwrite(temp, filePath) -} \ No newline at end of file diff --git a/worker/tsconfig.json b/worker/tsconfig.json index e0013f47c3..8bf981ee0c 100644 --- a/worker/tsconfig.json +++ b/worker/tsconfig.json @@ -21,9 +21,6 @@ { "path": "../fs/hard-link-dir" }, - { - "path": "../fs/msgpack-file" - }, { "path": "../fs/symlink-dependency" }, @@ -44,6 +41,9 @@ }, { "path": "../store/create-cafs-store" + }, + { + "path": "../store/index" } ] }