From 48dcd108c89ca585caf62a817d5caa115ae38245 Mon Sep 17 00:00:00 2001 From: Zoltan Kochan Date: Sat, 9 Sep 2023 18:56:43 +0300 Subject: [PATCH] perf: symlink dependencies in a worker thread (#7069) --- .changeset/five-islands-cough.md | 8 +++ .changeset/sharp-buckets-study.md | 5 ++ config/config/package.json | 2 +- env/plugin-commands-env/package.json | 2 +- fs/symlink-dependency/package.json | 2 +- fs/symlink-dependency/src/index.ts | 12 +++- pkg-manager/core/package.json | 2 +- pkg-manager/core/src/install/link.ts | 44 ++++++------ pkg-manager/headless/package.json | 3 +- pkg-manager/headless/src/index.ts | 33 +++------ pkg-manager/headless/tsconfig.json | 3 + pkg-manager/hoist/package.json | 2 +- pkg-manager/link-bins/package.json | 2 +- .../plugin-commands-installation/package.json | 2 +- pnpm-lock.yaml | 68 ++++++++++++++----- pnpm/package.json | 2 +- worker/package.json | 1 + worker/src/index.ts | 24 +++++++ worker/src/types.ts | 9 +++ worker/src/worker.ts | 21 +++++- worker/tsconfig.json | 3 + 21 files changed, 178 insertions(+), 72 deletions(-) create mode 100644 .changeset/five-islands-cough.md create mode 100644 .changeset/sharp-buckets-study.md diff --git a/.changeset/five-islands-cough.md b/.changeset/five-islands-cough.md new file mode 100644 index 0000000000..ae6e918fa6 --- /dev/null +++ b/.changeset/five-islands-cough.md @@ -0,0 +1,8 @@ +--- +"@pnpm/headless": minor +"@pnpm/core": minor +"@pnpm/worker": minor +"pnpm": patch +--- + +Improve performance of installation by using a worker for creating the symlinks inside `node_modules/.pnpm` [#7069](https://github.com/pnpm/pnpm/pull/7069). diff --git a/.changeset/sharp-buckets-study.md b/.changeset/sharp-buckets-study.md new file mode 100644 index 0000000000..56d563ddfd --- /dev/null +++ b/.changeset/sharp-buckets-study.md @@ -0,0 +1,5 @@ +--- +"@pnpm/symlink-dependency": minor +--- + +Add a sync symlinker. diff --git a/config/config/package.json b/config/config/package.json index af576ccf26..40f72cd959 100644 --- a/config/config/package.json +++ b/config/config/package.json @@ -62,7 +62,7 @@ "@types/is-windows": "^1.0.0", "@types/ramda": "0.28.20", "@types/which": "^2.0.2", - "symlink-dir": "^5.1.1" + "symlink-dir": "^5.2.0" }, "funding": "https://opencollective.com/pnpm", "exports": { diff --git a/env/plugin-commands-env/package.json b/env/plugin-commands-env/package.json index 398333003c..fc99b6a048 100644 --- a/env/plugin-commands-env/package.json +++ b/env/plugin-commands-env/package.json @@ -46,7 +46,7 @@ "load-json-file": "^6.2.0", "render-help": "^1.0.3", "semver": "^7.5.4", - "symlink-dir": "^5.1.1", + "symlink-dir": "^5.2.0", "write-json-file": "^4.3.0" }, "funding": "https://opencollective.com/pnpm", diff --git a/fs/symlink-dependency/package.json b/fs/symlink-dependency/package.json index a63fd8a37b..1c3c6db5f8 100644 --- a/fs/symlink-dependency/package.json +++ b/fs/symlink-dependency/package.json @@ -42,7 +42,7 @@ "dependencies": { "@pnpm/core-loggers": "workspace:*", "@pnpm/types": "workspace:*", - "symlink-dir": "^5.1.1" + "symlink-dir": "^5.2.0" }, "funding": "https://opencollective.com/pnpm", "exports": { diff --git a/fs/symlink-dependency/src/index.ts b/fs/symlink-dependency/src/index.ts index 9d97247a60..2a853f8c23 100644 --- a/fs/symlink-dependency/src/index.ts +++ b/fs/symlink-dependency/src/index.ts @@ -1,7 +1,7 @@ import path from 'path' import { linkLogger } from '@pnpm/core-loggers' - import symlinkDir from 'symlink-dir' + export { symlinkDirectRootDependency } from './symlinkDirectRootDependency' export async function symlinkDependency ( @@ -13,3 +13,13 @@ export async function symlinkDependency ( linkLogger.debug({ target: dependencyRealLocation, link }) return symlinkDir(dependencyRealLocation, link) } + +export function symlinkDependencySync ( + dependencyRealLocation: string, + destModulesDir: string, + importAs: string +) { + const link = path.join(destModulesDir, importAs) + linkLogger.debug({ target: dependencyRealLocation, link }) + return symlinkDir.sync(dependencyRealLocation, link) +} diff --git a/pkg-manager/core/package.json b/pkg-manager/core/package.json index 6c463000a3..d84d24eeb9 100644 --- a/pkg-manager/core/package.json +++ b/pkg-manager/core/package.json @@ -101,7 +101,7 @@ "read-yaml-file": "^2.1.0", "resolve-link-target": "^2.0.0", "sinon": "^15.2.0", - "symlink-dir": "^5.1.1", + "symlink-dir": "^5.2.0", "write-json-file": "^4.3.0", "write-yaml-file": "^5.0.0" }, diff --git a/pkg-manager/core/src/install/link.ts b/pkg-manager/core/src/install/link.ts index c883c93beb..3e93ccaede 100644 --- a/pkg-manager/core/src/install/link.ts +++ b/pkg-manager/core/src/install/link.ts @@ -27,6 +27,7 @@ import { type HoistedDependencies, type Registries, } from '@pnpm/types' +import { symlinkAllModules } from '@pnpm/worker' import pLimit from 'p-limit' import pathExists from 'path-exists' import equals from 'ramda/src/equals' @@ -472,25 +473,26 @@ async function linkAllModules ( optional: boolean } ) { - await Promise.all( - depNodes - .map(async ({ children, optionalDependencies, name, modules }) => { - const childrenToLink: Record = opts.optional - ? children - : pickBy((_, childAlias) => !optionalDependencies.has(childAlias), children) - - await Promise.all( - Object.entries(childrenToLink) - .map(async ([childAlias, childDepPath]) => { - if (childDepPath.startsWith('link:')) { - await limitLinking(() => symlinkDependency(path.resolve(opts.lockfileDir, childDepPath.slice(5)), modules, childAlias)) - return - } - const pkg = depGraph[childDepPath] - if (!pkg || !pkg.installable && pkg.optional || childAlias === name) return - await limitLinking(() => symlinkDependency(pkg.dir, modules, childAlias)) - }) - ) - }) - ) + await symlinkAllModules({ + deps: depNodes.map((depNode) => { + const children = opts.optional + ? depNode.children + : pickBy((_, childAlias) => !depNode.optionalDependencies.has(childAlias), depNode.children) + const childrenPaths: Record = {} + for (const [alias, childDepPath] of Object.entries(children ?? {})) { + if (childDepPath.startsWith('link:')) { + childrenPaths[alias] = path.resolve(opts.lockfileDir, childDepPath.slice(5)) + } else { + const pkg = depGraph[childDepPath] + if (!pkg || !pkg.installable && pkg.optional || alias === depNode.name) continue + childrenPaths[alias] = pkg.dir + } + } + return { + children: childrenPaths, + modules: depNode.modules, + name: depNode.name, + } + }), + }) } diff --git a/pkg-manager/headless/package.json b/pkg-manager/headless/package.json index 12d84bb313..404c51d3c4 100644 --- a/pkg-manager/headless/package.json +++ b/pkg-manager/headless/package.json @@ -12,7 +12,8 @@ "!*.map" ], "peerDependencies": { - "@pnpm/logger": "^5.0.0" + "@pnpm/logger": "^5.0.0", + "@pnpm/worker": "workspace:^" }, "devDependencies": { "@pnpm/assert-project": "workspace:*", diff --git a/pkg-manager/headless/src/index.ts b/pkg-manager/headless/src/index.ts index 5d6db16992..904c6322ee 100644 --- a/pkg-manager/headless/src/index.ts +++ b/pkg-manager/headless/src/index.ts @@ -56,6 +56,7 @@ import { import { symlinkDependency } from '@pnpm/symlink-dependency' import { type DependencyManifest, type HoistedDependencies, type ProjectManifest, type Registries, DEPENDENCIES_FIELDS } from '@pnpm/types' import * as dp from '@pnpm/dependency-path' +import { symlinkAllModules } from '@pnpm/worker' import pLimit from 'p-limit' import pathAbsolute from 'path-absolute' import equals from 'ramda/src/equals' @@ -378,7 +379,6 @@ export async function headlessInstall (opts: HeadlessOptions): Promise>, opts: { optional: boolean - lockfileDir: string } ) { - await Promise.all( - depNodes - .map(async (depNode) => { - const childrenToLink: Record = opts.optional - ? depNode.children - : pickBy((_, childAlias) => !depNode.optionalDependencies.has(childAlias), depNode.children) - - await Promise.all( - Object.entries(childrenToLink) - .map(async ([alias, pkgDir]) => { - // if (!pkg.installable && pkg.optional) return - if (alias === depNode.name) { - return - } - await limitLinking(() => symlinkDependency(pkgDir, depNode.modules, alias)) - }) - ) - }) - ) + await symlinkAllModules({ + deps: depNodes.map((depNode) => ({ + children: opts.optional + ? depNode.children + : pickBy((_, childAlias) => !depNode.optionalDependencies.has(childAlias), depNode.children), + modules: depNode.modules, + name: depNode.name, + })), + }) } diff --git a/pkg-manager/headless/tsconfig.json b/pkg-manager/headless/tsconfig.json index 0121615388..ae44c3b795 100644 --- a/pkg-manager/headless/tsconfig.json +++ b/pkg-manager/headless/tsconfig.json @@ -81,6 +81,9 @@ { "path": "../../store/store-path" }, + { + "path": "../../worker" + }, { "path": "../client" }, diff --git a/pkg-manager/hoist/package.json b/pkg-manager/hoist/package.json index 39f499d605..3580a87685 100644 --- a/pkg-manager/hoist/package.json +++ b/pkg-manager/hoist/package.json @@ -52,7 +52,7 @@ "is-subdir": "^1.2.0", "ramda": "npm:@pnpm/ramda@0.28.1", "resolve-link-target": "^2.0.0", - "symlink-dir": "^5.1.1" + "symlink-dir": "^5.2.0" }, "funding": "https://opencollective.com/pnpm", "exports": { diff --git a/pkg-manager/link-bins/package.json b/pkg-manager/link-bins/package.json index 91cadc14a3..4c2f84d202 100644 --- a/pkg-manager/link-bins/package.json +++ b/pkg-manager/link-bins/package.json @@ -46,7 +46,7 @@ "normalize-path": "^3.0.0", "p-settle": "^4.1.1", "ramda": "npm:@pnpm/ramda@0.28.1", - "symlink-dir": "^5.1.1" + "symlink-dir": "^5.2.0" }, "devDependencies": { "@pnpm/link-bins": "workspace:*", diff --git a/pkg-manager/plugin-commands-installation/package.json b/pkg-manager/plugin-commands-installation/package.json index 58d850b51e..991d21c61f 100644 --- a/pkg-manager/plugin-commands-installation/package.json +++ b/pkg-manager/plugin-commands-installation/package.json @@ -47,7 +47,7 @@ "proxyquire": "^2.1.3", "read-yaml-file": "^2.1.0", "sinon": "^15.2.0", - "symlink-dir": "^5.1.1", + "symlink-dir": "^5.2.0", "tempy": "^1.0.1", "write-json-file": "^4.3.0", "write-pkg": "4.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 896d1cbac3..b4f6917904 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -659,8 +659,8 @@ importers: specifier: ^2.0.2 version: 2.0.2 symlink-dir: - specifier: ^5.1.1 - version: 5.1.1 + specifier: ^5.2.0 + version: 5.2.0 config/matcher: dependencies: @@ -1009,8 +1009,8 @@ importers: specifier: ^7.5.4 version: 7.5.4 symlink-dir: - specifier: ^5.1.1 - version: 5.1.1 + specifier: ^5.2.0 + version: 5.2.0 write-json-file: specifier: ^4.3.0 version: 4.3.0 @@ -1763,8 +1763,8 @@ importers: specifier: workspace:* version: link:../../packages/types symlink-dir: - specifier: ^5.1.1 - version: 5.1.1 + specifier: ^5.2.0 + version: 5.2.0 devDependencies: '@pnpm/prepare': specifier: workspace:* @@ -3113,8 +3113,8 @@ importers: specifier: ^15.2.0 version: 15.2.0 symlink-dir: - specifier: ^5.1.1 - version: 5.1.1 + specifier: ^5.2.0 + version: 5.2.0 write-json-file: specifier: ^4.3.0 version: 4.3.0 @@ -3282,6 +3282,9 @@ importers: '@pnpm/types': specifier: workspace:* version: link:../../packages/types + '@pnpm/worker': + specifier: workspace:^ + version: link:../../worker '@zkochan/rimraf': specifier: ^2.1.3 version: 2.1.3 @@ -3407,8 +3410,8 @@ importers: specifier: ^2.0.0 version: 2.0.0 symlink-dir: - specifier: ^5.1.1 - version: 5.1.1 + specifier: ^5.2.0 + version: 5.2.0 devDependencies: '@pnpm/hoist': specifier: workspace:* @@ -3468,8 +3471,8 @@ importers: specifier: npm:@pnpm/ramda@0.28.1 version: /@pnpm/ramda@0.28.1 symlink-dir: - specifier: ^5.1.1 - version: 5.1.1 + specifier: ^5.2.0 + version: 5.2.0 devDependencies: '@pnpm/link-bins': specifier: workspace:* @@ -3906,8 +3909,8 @@ importers: specifier: ^15.2.0 version: 15.2.0 symlink-dir: - specifier: ^5.1.1 - version: 5.1.1 + specifier: ^5.2.0 + version: 5.2.0 tempy: specifier: ^1.0.1 version: 1.0.1 @@ -4537,8 +4540,8 @@ importers: specifier: ^6.0.1 version: 6.0.1 symlink-dir: - specifier: ^5.1.1 - version: 5.1.1 + specifier: ^5.2.0 + version: 5.2.0 tempy: specifier: ^1.0.1 version: 1.0.1 @@ -6017,6 +6020,9 @@ importers: '@pnpm/store.cafs': specifier: workspace:* version: link:../store/cafs + '@pnpm/symlink-dependency': + specifier: workspace:* + version: link:../fs/symlink-dependency '@rushstack/worker-pool': specifier: 0.3.34 version: 0.3.34(@types/node@16.18.48) @@ -8330,6 +8336,7 @@ packages: /@tootallnate/once@2.0.0: resolution: {integrity: sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==} engines: {node: '>= 10'} + requiresBuild: true dev: false /@tsconfig/node10@1.0.9: @@ -9177,6 +9184,7 @@ packages: /agent-base@6.0.2: resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==} engines: {node: '>= 6.0.0'} + requiresBuild: true dependencies: debug: 4.3.4 transitivePeerDependencies: @@ -9196,6 +9204,7 @@ packages: /agentkeepalive@4.5.0: resolution: {integrity: sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==} engines: {node: '>= 8.0.0'} + requiresBuild: true dependencies: humanize-ms: 1.2.1 dev: false @@ -9384,6 +9393,7 @@ packages: /are-we-there-yet@3.0.1: resolution: {integrity: sha512-QZW4EDmGwlYur0Yyf/b2uGucHQMa8aFUP7eu9ddR73vvhFyt4V0Vl3QHPcTNJ8l6qYOBdxgXdnBXQrHilfRQBg==} engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} + requiresBuild: true dependencies: delegates: 1.0.0 readable-stream: 3.6.2 @@ -9687,6 +9697,7 @@ packages: /brace-expansion@2.0.1: resolution: {integrity: sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==} + requiresBuild: true dependencies: balanced-match: 1.0.2 @@ -10198,6 +10209,7 @@ packages: /console-control-strings@1.1.0: resolution: {integrity: sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==} + requiresBuild: true /content-disposition@0.5.4: resolution: {integrity: sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==} @@ -10721,6 +10733,7 @@ packages: /env-paths@2.2.1: resolution: {integrity: sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==} engines: {node: '>=6'} + requiresBuild: true dev: false /envinfo@7.8.1: @@ -11641,6 +11654,7 @@ packages: /gauge@4.0.4: resolution: {integrity: sha512-f9m+BEN5jkg6a0fZjleidjN51VE1X+mPFQ2DJ0uv1V39oCLCbsGe6yjbBnp7eK7z/+GAon99a3nHuqbuuthyPg==} engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} + requiresBuild: true dependencies: aproba: 2.0.0 color-support: 1.1.3 @@ -12023,6 +12037,7 @@ packages: /http-proxy-agent@5.0.0: resolution: {integrity: sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==} engines: {node: '>= 6'} + requiresBuild: true dependencies: '@tootallnate/once': 2.0.0 agent-base: 6.0.2 @@ -12068,6 +12083,7 @@ packages: /humanize-ms@1.2.1: resolution: {integrity: sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==} + requiresBuild: true dependencies: ms: 2.1.3 dev: false @@ -12216,6 +12232,7 @@ packages: /ip@2.0.0: resolution: {integrity: sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ==} + requiresBuild: true dev: false /ipaddr.js@1.9.1: @@ -12361,6 +12378,7 @@ packages: /is-lambda@1.0.1: resolution: {integrity: sha512-z7CMFGNrENq5iFB9Bqo64Xk6Y9sg+epq1myIcdHaGnbMTYOxvzsEtdYqQUylB7LxfkvgrrjP32T6Ywciio9UIQ==} + requiresBuild: true dev: false /is-negated-glob@1.0.0: @@ -13373,11 +13391,13 @@ packages: /lru-cache@7.18.3: resolution: {integrity: sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==} engines: {node: '>=12'} + requiresBuild: true dev: false /lru-cache@9.1.2: resolution: {integrity: sha512-ERJq3FOzJTxBbFjZ7iDs+NiK4VI9Wz+RdrrAB8dio1oV+YvdPzUEE4QNiT2VD51DkIbCYRUUzCRkssXCHqSnKQ==} engines: {node: 14 || >=16.14} + requiresBuild: true dev: false /lunr-mutable-indexes@2.3.2: @@ -13728,6 +13748,7 @@ packages: /minipass-collect@1.0.2: resolution: {integrity: sha512-6T6lH0H8OG9kITm/Jm6tdooIbogG9e0tLgpY6mphXSm/A9u8Nq1ryBG+Qspiub9LjWlBPsPS3tWQ/Botq4FdxA==} engines: {node: '>= 8'} + requiresBuild: true dependencies: minipass: 3.3.6 dev: false @@ -13759,6 +13780,7 @@ packages: /minipass-flush@1.0.5: resolution: {integrity: sha512-JmQSYYpPUqX5Jyn1mXaRwOda1uQ8HP5KAT/oDSLCzt1BYRhQU0/hDtsB1ufZfEEzMZ9aAVmsBw8+FWsIXlClWw==} engines: {node: '>= 8'} + requiresBuild: true dependencies: minipass: 3.3.6 dev: false @@ -13766,6 +13788,7 @@ packages: /minipass-pipeline@1.2.4: resolution: {integrity: sha512-xuIq7cIOt09RPRJ19gdi4b+RiNvDFYe5JH+ggNvBqGqpQXcru3PcRmOZuHBKWK1Txf9+cQ+HMVN4d6z46LZP7A==} engines: {node: '>=8'} + requiresBuild: true dependencies: minipass: 3.3.6 dev: false @@ -13773,6 +13796,7 @@ packages: /minipass-sized@1.0.3: resolution: {integrity: sha512-MbkQQ2CTiBMlA2Dm/5cY+9SWFEN8pzzOXi6rlM5Xxq0Yqbda5ZQy9sU75a673FE9ZK0Zsbr6Y5iP6u9nktfg2g==} engines: {node: '>=8'} + requiresBuild: true dependencies: minipass: 3.3.6 dev: false @@ -14161,6 +14185,7 @@ packages: /npmlog@6.0.2: resolution: {integrity: sha512-/vBvz5Jfr9dT/aFWd0FIRf+T/Q2WBsLENygUaFUqstqsycmZAP/t5BvFJTK0viFmSUxiUKTUplWy5vt+rvKIxg==} engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} + requiresBuild: true dependencies: are-we-there-yet: 3.0.1 console-control-strings: 1.1.0 @@ -14748,6 +14773,7 @@ packages: /promise-retry@2.0.1: resolution: {integrity: sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==} engines: {node: '>=10'} + requiresBuild: true dependencies: err-code: 2.0.3 retry: 0.12.0 @@ -15253,6 +15279,7 @@ packages: /retry@0.12.0: resolution: {integrity: sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==} engines: {node: '>= 4'} + requiresBuild: true dev: false /retry@0.13.1: @@ -15450,6 +15477,7 @@ packages: /set-blocking@2.0.0: resolution: {integrity: sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==} + requiresBuild: true /setprototypeof@1.2.0: resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} @@ -15578,6 +15606,7 @@ packages: /smart-buffer@4.2.0: resolution: {integrity: sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==} engines: {node: '>= 6.0.0', npm: '>= 3.0.0'} + requiresBuild: true dev: false /smartwrap@2.0.2: @@ -15631,6 +15660,7 @@ packages: /socks@2.7.1: resolution: {integrity: sha512-7maUZy1N7uo6+WVEX6psASxtNlKaNVMlGQKkG/63nEDdLOWNbiUMoLK7X4uYoLhQstau72mLgfEWcXcwsaHbYQ==} engines: {node: '>= 10.13.0', npm: '>= 3.0.0'} + requiresBuild: true dependencies: ip: 2.0.0 smart-buffer: 4.2.0 @@ -15741,6 +15771,7 @@ packages: /ssri@10.0.4: resolution: {integrity: sha512-12+IR2CB2C28MMAw0Ncqwj5QbTcs0nGIhgJzYWzDkb21vWmfNI83KS4f3Ci6GI98WreIfG7o9UXp3C0qbpA8nQ==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} + requiresBuild: true dependencies: minipass: 5.0.0 @@ -15991,8 +16022,8 @@ packages: resolution: {integrity: sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==} engines: {node: '>= 0.4'} - /symlink-dir@5.1.1: - resolution: {integrity: sha512-kmVV2SfdoDksjJxStJ5N9u1ZZ5tQndCeUEG8St0tHI9BZe/ehZYbKB6eXPjo+AvFG1uRsDymUSGG0OLv2Ox8aQ==} + /symlink-dir@5.2.0: + resolution: {integrity: sha512-MPL1kiINPyLr9f4Q9qZVHmNqNNYF+x3Gq9fXFj9xXHmgYe57cIXSIgdNfhkqwp+LNgOaKcl8rI0hTinT5LbV8w==} engines: {node: '>=12.10'} hasBin: true dependencies: @@ -16949,6 +16980,7 @@ packages: /wrap-ansi@7.0.0: resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==} engines: {node: '>=10'} + requiresBuild: true dependencies: ansi-styles: 4.3.0 string-width: 4.2.3 diff --git a/pnpm/package.json b/pnpm/package.json index 31ad89d935..9bf5392e33 100644 --- a/pnpm/package.json +++ b/pnpm/package.json @@ -102,7 +102,7 @@ "semver": "^7.5.4", "split-cmd": "^1.0.1", "strip-ansi": "^6.0.1", - "symlink-dir": "^5.1.1", + "symlink-dir": "^5.2.0", "tempy": "^1.0.1", "tree-kill": "^1.2.2", "which": "^3.0.1", diff --git a/worker/package.json b/worker/package.json index 7d2ff74063..3029832dbb 100644 --- a/worker/package.json +++ b/worker/package.json @@ -36,6 +36,7 @@ "@pnpm/create-cafs-store": "workspace:*", "@pnpm/error": "workspace:*", "@pnpm/graceful-fs": "workspace:*", + "@pnpm/symlink-dependency": "workspace:*", "@pnpm/store.cafs": "workspace:*", "@rushstack/worker-pool": "0.3.34", "load-json-file": "^6.2.0" diff --git a/worker/src/index.ts b/worker/src/index.ts index 0a32fd4456..1a8afeee0e 100644 --- a/worker/src/index.ts +++ b/worker/src/index.ts @@ -8,6 +8,7 @@ import { type TarballExtractMessage, type AddDirToStoreMessage, type LinkPkgMessage, + type SymlinkAllModulesMessage, } from './types' let workerPool: WorkerPool | undefined @@ -196,3 +197,26 @@ export async function importPackage ( }) }) } + +export async function symlinkAllModules ( + opts: Omit +): Promise<{ isBuilt: boolean, importMethod: string | undefined }> { + if (!workerPool) { + workerPool = createTarballWorkerPool() + } + const localWorker = await workerPool.checkoutWorkerAsync(true) + return new Promise<{ isBuilt: boolean, importMethod: string | undefined }>((resolve, reject) => { + localWorker.once('message', ({ status, error, value }: any) => { // eslint-disable-line @typescript-eslint/no-explicit-any + workerPool!.checkinWorker(localWorker) + if (status === 'error') { + reject(new PnpmError('SYMLINK_FAILED', error as string)) + return + } + resolve(value) + }) + localWorker.postMessage({ + type: 'symlinkAllModules', + ...opts, + } as SymlinkAllModulesMessage) + }) +} diff --git a/worker/src/types.ts b/worker/src/types.ts index 4247175c44..7fcd5813cd 100644 --- a/worker/src/types.ts +++ b/worker/src/types.ts @@ -27,6 +27,15 @@ export interface LinkPkgMessage { keepModulesDir?: boolean } +export interface SymlinkAllModulesMessage { + type: 'symlinkAllModules' + deps: Array<{ + children: Record + modules: string + name: string + }> +} + export interface AddDirToStoreMessage { type: 'add-dir' cafsDir: string diff --git a/worker/src/worker.ts b/worker/src/worker.ts index c5f8614407..82a9b7c3cc 100644 --- a/worker/src/worker.ts +++ b/worker/src/worker.ts @@ -13,12 +13,14 @@ import { readManifestFromStore, type VerifyResult, } from '@pnpm/store.cafs' +import { symlinkDependencySync } from '@pnpm/symlink-dependency' import { sync as loadJsonFile } from 'load-json-file' import { parentPort } from 'worker_threads' import { type AddDirToStoreMessage, type ReadPkgFromCafsMessage, type LinkPkgMessage, + type SymlinkAllModulesMessage, type TarballExtractMessage, } from './types' @@ -30,7 +32,9 @@ const cafsCache = new Map>() const cafsStoreCache = new Map>() const cafsLocker = new Map() -async function handleMessage (message: TarballExtractMessage | LinkPkgMessage | AddDirToStoreMessage | ReadPkgFromCafsMessage | false): Promise { +async function handleMessage ( + message: TarballExtractMessage | LinkPkgMessage | AddDirToStoreMessage | ReadPkgFromCafsMessage | SymlinkAllModulesMessage | false +): Promise { if (message === false) { parentPort!.off('message', handleMessage) process.exit(0) @@ -86,6 +90,10 @@ async function handleMessage (message: TarballExtractMessage | LinkPkgMessage | }) break } + case 'symlinkAllModules': { + parentPort!.postMessage(symlinkAllModules(message)) + break + } } } catch (e: any) { // eslint-disable-line parentPort!.postMessage({ status: 'error', error: e.toString() }) @@ -184,6 +192,17 @@ function importPackage ({ return { status: 'success', value: { isBuilt, importMethod } } } +function symlinkAllModules (opts: SymlinkAllModulesMessage) { + for (const dep of opts.deps) { + for (const [alias, pkgDir] of Object.entries(dep.children)) { + if (alias !== dep.name) { + symlinkDependencySync(pkgDir, dep.modules, alias) + } + } + } + return { status: 'success' } +} + function writeFilesIndexFile ( filesIndexFile: string, { pkg, files }: { diff --git a/worker/tsconfig.json b/worker/tsconfig.json index 750472c240..652fe9a576 100644 --- a/worker/tsconfig.json +++ b/worker/tsconfig.json @@ -12,6 +12,9 @@ { "path": "../fs/graceful-fs" }, + { + "path": "../fs/symlink-dependency" + }, { "path": "../packages/error" },