From 04efe86469ac7a7952227df952b7dfde5ce13cad Mon Sep 17 00:00:00 2001
From: Zoltan Kochan <z@kochan.io>
Date: Fri, 27 Jan 2023 06:11:43 +0200
Subject: [PATCH] fix: out of memory exception with node-linker=hoisted (#5988)

ref #5909
---
 .changeset/cuddly-hounds-unite.md             |  6 +++
 .../headless/src/linkHoistedModules.ts        | 38 +++++++++++--------
 2 files changed, 29 insertions(+), 15 deletions(-)
 create mode 100644 .changeset/cuddly-hounds-unite.md

diff --git a/.changeset/cuddly-hounds-unite.md b/.changeset/cuddly-hounds-unite.md
new file mode 100644
index 0000000000..6cc402729f
--- /dev/null
+++ b/.changeset/cuddly-hounds-unite.md
@@ -0,0 +1,6 @@
+---
+"@pnpm/headless": patch
+"pnpm": patch
+---
+
+Fixed out of memory error that sometimes happens when `node-linker` is set to `hoisted`.
diff --git a/pkg-manager/headless/src/linkHoistedModules.ts b/pkg-manager/headless/src/linkHoistedModules.ts
index 4e539801d2..9fc34f38f7 100644
--- a/pkg-manager/headless/src/linkHoistedModules.ts
+++ b/pkg-manager/headless/src/linkHoistedModules.ts
@@ -11,6 +11,7 @@ import {
   PackageFilesResponse,
   StoreController,
 } from '@pnpm/store-controller-types'
+import pLimit from 'p-limit'
 import difference from 'ramda/src/difference'
 import isEmpty from 'ramda/src/isEmpty'
 import rimraf from '@zkochan/rimraf'
@@ -19,6 +20,8 @@ import {
   DependenciesGraph,
 } from './lockfileToDepGraph'
 
+const limitLinking = pLimit(16)
+
 export async function linkHoistedModules (
   storeController: StoreController,
   graph: DependenciesGraph,
@@ -111,22 +114,27 @@ async function linkAllPkgsInOrder (
             patchFileHash: depNode.patchFile?.hash,
           })
         }
-        const { importMethod, isBuilt } = await storeController.importPackage(depNode.dir, {
-          filesResponse,
-          force: opts.force || depNode.depPath !== prevGraph[dir]?.depPath,
-          keepModulesDir: true,
-          requiresBuild: depNode.requiresBuild || depNode.patchFile != null,
-          sideEffectsCacheKey,
-        })
-        if (importMethod) {
-          progressLogger.debug({
-            method: importMethod,
-            requester: opts.lockfileDir,
-            status: 'imported',
-            to: depNode.dir,
+        // Limiting the concurrency here fixes an out of memory error.
+        // It is not clear why it helps as importing is also limited inside fs.indexed-pkg-importer.
+        // The out of memory error was reproduced on the teambit/bit repository with the "rootComponents" feature turned on
+        await limitLinking(async () => {
+          const { importMethod, isBuilt } = await storeController.importPackage(depNode.dir, {
+            filesResponse,
+            force: opts.force || depNode.depPath !== prevGraph[dir]?.depPath,
+            keepModulesDir: true,
+            requiresBuild: depNode.requiresBuild || depNode.patchFile != null,
+            sideEffectsCacheKey,
           })
-        }
-        depNode.isBuilt = isBuilt
+          if (importMethod) {
+            progressLogger.debug({
+              method: importMethod,
+              requester: opts.lockfileDir,
+              status: 'imported',
+              to: depNode.dir,
+            })
+          }
+          depNode.isBuilt = isBuilt
+        })
       }
       return linkAllPkgsInOrder(storeController, graph, prevGraph, deps, dir, opts)
     })