refactor: linking direct dependencies (#5669)

.changeset/modern-otters-explode.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/pkg-manager.direct-dep-linker": minor
+---
+
+Initial release.

pkg-manager/core/package.json

@@ -41,6 +41,7 @@
     "@pnpm/npm-package-arg": "^1.0.0",
     "@pnpm/package-requester": "workspace:*",
     "@pnpm/parse-wanted-dependency": "workspace:*",
+    "@pnpm/pkg-manager.direct-dep-linker": "workspace:*",
     "@pnpm/prune-lockfile": "workspace:*",
     "@pnpm/read-modules-dir": "workspace:*",
     "@pnpm/read-package-json": "workspace:*",

pkg-manager/core/src/install/link.ts

@@ -3,13 +3,13 @@ import path from 'path'
 import { calcDepState, DepsStateCache } from '@pnpm/calc-dep-state'
 import {
   progressLogger,
-  rootLogger,
   stageLogger,
   statsLogger,
 } from '@pnpm/core-loggers'
 import {
   filterLockfileByImporters,
 } from '@pnpm/filter-lockfile'
+import { linkDirectDeps } from '@pnpm/pkg-manager.direct-dep-linker'
 import { hoist } from '@pnpm/hoist'
 import { Lockfile } from '@pnpm/lockfile-file'
 import { logger } from '@pnpm/logger'
@@ -21,7 +21,7 @@ import {
   LinkedDependency,
 } from '@pnpm/resolve-dependencies'
 import { StoreController } from '@pnpm/store-controller-types'
-import { symlinkDependency, symlinkDirectRootDependency } from '@pnpm/symlink-dependency'
+import { symlinkDependency } from '@pnpm/symlink-dependency'
 import {
   HoistedDependencies,
   Registries,
@@ -160,43 +160,49 @@ export async function linkPackages (
   })
 
   if (opts.symlink) {
-    await Promise.all(projects.map(async ({ id, manifest, modulesDir, rootDir }) => {
-      const deps = opts.dependenciesByProjectId[id]
-      const importerFromLockfile = newCurrentLockfile.importers[id]
-      await Promise.all([
-        ...Object.entries(deps)
-          .filter(([rootAlias]) => importerFromLockfile.specifiers[rootAlias])
-          .map(([rootAlias, depPath]) => ({ rootAlias, depGraphNode: depGraph[depPath] }))
-          .filter(({ depGraphNode }) => depGraphNode)
-          .map(async ({ rootAlias, depGraphNode }) => {
-            if (
-              (await symlinkDependency(depGraphNode.dir, modulesDir, rootAlias)).reused
-            ) return
-
-            const isDev = Boolean(manifest.devDependencies?.[depGraphNode.name])
-            const isOptional = Boolean(manifest.optionalDependencies?.[depGraphNode.name])
-            rootLogger.debug({
-              added: {
-                dependencyType: isDev && 'dev' || isOptional && 'optional' || 'prod',
-                id: depGraphNode.id,
-                latest: opts.outdatedDependencies[depGraphNode.id],
-                name: rootAlias,
-                realName: depGraphNode.name,
-                version: depGraphNode.version,
-              },
-              prefix: rootDir,
-            })
-          }),
-        ...opts.linkedDependenciesByProjectId[id].map(async (linkedDependency) => {
-          const depLocation = resolvePath(rootDir, linkedDependency.resolution.directory)
-          return symlinkDirectRootDependency(depLocation, modulesDir, linkedDependency.alias, {
-            fromDependenciesField: linkedDependency.dev && 'devDependencies' || linkedDependency.optional && 'optionalDependencies' || 'dependencies',
-            linkedPackage: linkedDependency,
-            prefix: rootDir,
-          })
-        }),
-      ])
-    }))
+    const projectsToLink = await Promise.all(
+      projects.map(async ({ id, manifest, modulesDir, rootDir }) => {
+        const deps = opts.dependenciesByProjectId[id]
+        const importerFromLockfile = newCurrentLockfile.importers[id]
+        return {
+          dir: rootDir,
+          modulesDir,
+          dependencies: await Promise.all([
+            ...Object.entries(deps)
+              .filter(([rootAlias]) => importerFromLockfile.specifiers[rootAlias])
+              .map(([rootAlias, depPath]) => ({ rootAlias, depGraphNode: depGraph[depPath] }))
+              .filter(({ depGraphNode }) => depGraphNode)
+              .map(async ({ rootAlias, depGraphNode }) => {
+                const isDev = Boolean(manifest.devDependencies?.[depGraphNode.name])
+                const isOptional = Boolean(manifest.optionalDependencies?.[depGraphNode.name])
+                return {
+                  alias: rootAlias,
+                  name: depGraphNode.name,
+                  version: depGraphNode.version,
+                  dir: depGraphNode.dir,
+                  id: depGraphNode.id,
+                  dependencyType: (isDev && 'dev' || isOptional && 'optional' || 'prod') as 'dev' | 'optional' | 'prod',
+                  latest: opts.outdatedDependencies[depGraphNode.id],
+                  isExternalLink: false,
+                }
+              }),
+            ...opts.linkedDependenciesByProjectId[id].map(async (linkedDependency) => {
+              const dir = resolvePath(rootDir, linkedDependency.resolution.directory)
+              return {
+                alias: linkedDependency.alias,
+                name: linkedDependency.name,
+                version: linkedDependency.version,
+                dir,
+                id: linkedDependency.resolution.directory,
+                dependencyType: (linkedDependency.dev && 'dev' || linkedDependency.optional && 'optional' || 'prod') as 'dev' | 'optional' | 'prod',
+                isExternalLink: true,
+              }
+            }),
+          ]),
+        }
+      })
+    )
+    await linkDirectDeps(projectsToLink)
   }
 
   let currentLockfile: Lockfile

pkg-manager/core/tsconfig.json

@@ -117,6 +117,9 @@
     {
       "path": "../client"
     },
+    {
+      "path": "../direct-dep-linker"
+    },
     {
       "path": "../get-context"
     },

pkg-manager/direct-dep-linker/README.md

@@ -0,0 +1,13 @@
+# @pnpm/pkg-manager.direct-dep-linker
+
+> Links direct dependencies to projects
+
+## Installation
+
+```
+pnpm install @pnpm/pkg-manager.direct-dep-linker
+```
+
+## License
+
+[MIT](LICENSE)

pkg-manager/direct-dep-linker/package.json

@@ -0,0 +1,44 @@
+{
+  "name": "@pnpm/pkg-manager.direct-dep-linker",
+  "description": "Fast installation using only pnpm-lock.yaml",
+  "version": "0.0.0",
+  "bugs": {
+    "url": "https://github.com/pnpm/pnpm/issues"
+  },
+  "main": "lib/index.js",
+  "types": "lib/index.d.ts",
+  "files": [
+    "lib",
+    "!*.map"
+  ],
+  "peerDependencies": {
+    "@pnpm/logger": "^5.0.0"
+  },
+  "devDependencies": {
+    "@pnpm/pkg-manager.direct-dep-linker": "workspace:*"
+  },
+  "homepage": "https://github.com/pnpm/pnpm/blob/main/pkg-manager/direct-dep-linker#readme",
+  "keywords": [
+    "pnpm7",
+    "pnpm"
+  ],
+  "license": "MIT",
+  "engines": {
+    "node": ">=14.6"
+  },
+  "repository": "https://github.com/pnpm/pnpm/blob/main/pkg-manager/direct-dep-linker",
+  "scripts": {
+    "lint": "eslint src/**/*.ts",
+    "test": "pnpm run compile",
+    "prepublishOnly": "pnpm run compile",
+    "compile": "tsc --build && pnpm run lint --fix"
+  },
+  "dependencies": {
+    "@pnpm/core-loggers": "workspace:*",
+    "@pnpm/symlink-dependency": "workspace:*"
+  },
+  "funding": "https://opencollective.com/pnpm",
+  "exports": {
+    ".": "./lib/index.js"
+  }
+}

pkg-manager/direct-dep-linker/src/index.ts

@@ -0,0 +1 @@
+export * from './linkDirectDeps'

pkg-manager/direct-dep-linker/src/linkDirectDeps.ts

@@ -0,0 +1,55 @@
+import { rootLogger } from '@pnpm/core-loggers'
+import { symlinkDependency, symlinkDirectRootDependency } from '@pnpm/symlink-dependency'
+
+export interface LinkedDirectDep {
+  alias: string
+  name: string
+  version: string
+  dir: string
+  id: string
+  dependencyType: 'prod' | 'dev' | 'optional'
+  isExternalLink: boolean
+  latest?: string
+}
+
+export interface ProjectToLink {
+  dir: string
+  modulesDir: string
+  dependencies: LinkedDirectDep[]
+}
+
+export async function linkDirectDeps (
+  projects: ProjectToLink[]
+) {
+  await Promise.all(projects.map(async (project) => {
+    await Promise.all(project.dependencies.map(async (dep) => {
+      if (dep.isExternalLink) {
+        await symlinkDirectRootDependency(dep.dir, project.modulesDir, dep.alias, {
+          fromDependenciesField: dep.dependencyType === 'dev' && 'devDependencies' ||
+            dep.dependencyType === 'optional' && 'optionalDependencies' ||
+            'dependencies',
+          linkedPackage: {
+            name: dep.name,
+            version: dep.version,
+          },
+          prefix: project.dir,
+        })
+        return
+      }
+      if ((await symlinkDependency(dep.dir, project.modulesDir, dep.alias)).reused) {
+        return
+      }
+      rootLogger.debug({
+        added: {
+          dependencyType: dep.dependencyType,
+          id: dep.id,
+          latest: dep.latest,
+          name: dep.alias,
+          realName: dep.name,
+          version: dep.version,
+        },
+        prefix: project.dir,
+      })
+    }))
+  }))
+}

pkg-manager/direct-dep-linker/tsconfig.json

@@ -0,0 +1,19 @@
+{
+  "extends": "@pnpm/tsconfig",
+  "compilerOptions": {
+    "outDir": "lib",
+    "rootDir": "src"
+  },
+  "include": [
+    "src/**/*.ts",
+    "../../__typings__/**/*.d.ts"
+  ],
+  "references": [
+    {
+      "path": "../../fs/symlink-dependency"
+    },
+    {
+      "path": "../../packages/core-loggers"
+    }
+  ]
+}

pkg-manager/direct-dep-linker/tsconfig.lint.json

@@ -0,0 +1,8 @@
+{
+  "extends": "./tsconfig.json",
+  "include": [
+    "src/**/*.ts",
+    "test/**/*.ts",
+    "../../__typings__/**/*.d.ts"
+  ]
+}

pkg-manager/headless/package.json

@@ -83,6 +83,7 @@
     "@pnpm/modules-yaml": "workspace:*",
     "@pnpm/package-is-installable": "workspace:*",
     "@pnpm/package-requester": "workspace:*",
+    "@pnpm/pkg-manager.direct-dep-linker": "workspace:*",
     "@pnpm/read-package-json": "workspace:*",
     "@pnpm/read-project-manifest": "workspace:*",
     "@pnpm/real-hoist": "workspace:*",

pkg-manager/headless/src/index.ts

@@ -9,7 +9,6 @@ import {
 import {
   packageManifestLogger,
   progressLogger,
-  rootLogger,
   stageLogger,
   statsLogger,
   summaryLogger,
@@ -55,7 +54,7 @@ import {
   PackageFilesResponse,
   StoreController,
 } from '@pnpm/store-controller-types'
-import { symlinkDependency, symlinkDirectRootDependency } from '@pnpm/symlink-dependency'
+import { symlinkDependency } from '@pnpm/symlink-dependency'
 import { DependencyManifest, HoistedDependencies, ProjectManifest, Registries } from '@pnpm/types'
 import * as dp from 'dependency-path'
 import pLimit from 'p-limit'
@@ -78,6 +77,7 @@ import {
   lockfileToDepGraph,
 } from './lockfileToDepGraph'
 import { lockfileToHoistedDepGraph } from './lockfileToHoistedDepGraph'
+import { linkDirectDeps, LinkedDirectDep } from '@pnpm/pkg-manager.direct-dep-linker'
 
 export { HoistingLimits }
 
@@ -576,26 +576,32 @@ async function symlinkDirectDependencies (
     symlink,
   }: SymlinkDirectDependenciesOpts
 ) {
-  await Promise.all(projects.map(async ({ rootDir, id, manifest, modulesDir }) => {
-    if (symlink !== false) {
-      await linkRootPackages(filteredLockfile, {
-        importerId: id,
-        importerModulesDir: modulesDir,
-        lockfileDir,
-        projectDir: rootDir,
-        projects,
-        registries,
-        rootDependencies: directDependenciesByImporterId[id],
-      })
-    }
-
+  projects.forEach(({ rootDir, manifest }) => {
     // Even though headless installation will never update the package.json
     // this needs to be logged because otherwise install summary won't be printed
     packageManifestLogger.debug({
       prefix: rootDir,
       updated: manifest,
     })
-  }))
+  })
+  if (symlink !== false) {
+    const projectsToLink = await Promise.all(
+      projects.map(async ({ rootDir, id, modulesDir }) => ({
+        dir: rootDir,
+        modulesDir,
+        dependencies: await getRootPackagesToLink(filteredLockfile, {
+          importerId: id,
+          importerModulesDir: modulesDir,
+          lockfileDir,
+          projectDir: rootDir,
+          projects,
+          registries,
+          rootDependencies: directDependenciesByImporterId[id],
+        }),
+      }))
+    )
+    await linkDirectDeps(projectsToLink)
+  }
 }
 
 async function linkBinsOfImporter (
@@ -617,7 +623,7 @@ async function linkBinsOfImporter (
   })
 }
 
-async function linkRootPackages (
+async function getRootPackagesToLink (
   lockfile: Lockfile,
   opts: {
     registries: Registries
@@ -628,7 +634,7 @@ async function linkRootPackages (
     lockfileDir: string
     rootDependencies: { [alias: string]: string }
   }
-) {
+): Promise<LinkedDirectDep[]> {
   const importerManifestsByImporterId = {} as { [id: string]: ProjectManifest }
   for (const { id, manifest } of opts.projects) {
     importerManifestsByImporterId[id] = manifest
@@ -639,7 +645,7 @@ async function linkRootPackages (
     ...projectSnapshot.dependencies,
     ...projectSnapshot.optionalDependencies,
   }
-  return Promise.all(
+  return (await Promise.all(
     Object.entries(allDeps)
       .map(async ([alias, ref]) => {
         if (ref.startsWith('link:')) {
@@ -659,23 +665,23 @@ async function linkRootPackages (
               return { name: alias, version: '0.0.0' }
             }
           })() as DependencyManifest
-          await symlinkDirectRootDependency(packageDir, opts.importerModulesDir, alias, {
-            fromDependenciesField: isDev && 'devDependencies' ||
-              isOptional && 'optionalDependencies' ||
-              'dependencies',
-            linkedPackage,
-            prefix: opts.projectDir,
-          })
-          return
+          return {
+            alias,
+            name: linkedPackage.name,
+            version: linkedPackage.version,
+            dir: packageDir,
+            id: ref,
+            isExternalLink: true,
+            dependencyType: isDev && 'dev' ||
+              isOptional && 'optional' ||
+              'prod',
+          }
         }
         const dir = opts.rootDependencies[alias]
         // Skipping linked packages
         if (!dir) {
           return
         }
-        if ((await symlinkDependency(dir, opts.importerModulesDir, alias)).reused) {
-          return
-        }
         const isDev = Boolean(projectSnapshot.devDependencies?.[alias])
         const isOptional = Boolean(projectSnapshot.optionalDependencies?.[alias])
 
@@ -685,19 +691,18 @@ async function linkRootPackages (
         if (pkgSnapshot == null) return // this won't ever happen. Just making typescript happy
         const pkgId = pkgSnapshot.id ?? dp.refToAbsolute(ref, alias, opts.registries) ?? undefined
         const pkgInfo = nameVerFromPkgSnapshot(depPath, pkgSnapshot)
-        rootLogger.debug({
-          added: {
-            dependencyType: isDev && 'dev' || isOptional && 'optional' || 'prod',
-            id: pkgId,
-            // latest: opts.outdatedPkgs[pkg.id],
-            name: alias,
-            realName: pkgInfo.name,
-            version: pkgInfo.version,
-          },
-          prefix: opts.projectDir,
-        })
+        return {
+          alias,
+          isExternalLink: false,
+          name: pkgInfo.name,
+          version: pkgInfo.version,
+          dependencyType: isDev && 'dev' || isOptional && 'optional' || 'prod',
+          dir,
+          id: pkgId,
+        }
       })
-  )
+  ))
+    .filter(Boolean) as LinkedDirectDep[]
 }
 
 const limitLinking = pLimit(16)

pkg-manager/headless/tsconfig.json

@@ -81,6 +81,9 @@
     {
       "path": "../client"
     },
+    {
+      "path": "../direct-dep-linker"
+    },
     {
       "path": "../hoist"
     },

pnpm-lock.yaml

@@ -2539,6 +2539,9 @@ importers:
       '@pnpm/parse-wanted-dependency':
         specifier: workspace:*
         version: link:../../packages/parse-wanted-dependency
+      '@pnpm/pkg-manager.direct-dep-linker':
+        specifier: workspace:*
+        version: link:../direct-dep-linker
       '@pnpm/prune-lockfile':
         specifier: workspace:*
         version: link:../../lockfile/prune-lockfile
@@ -2712,6 +2715,22 @@ importers:
         specifier: ^4.2.0
         version: 4.2.0
 
+  pkg-manager/direct-dep-linker:
+    dependencies:
+      '@pnpm/core-loggers':
+        specifier: workspace:*
+        version: link:../../packages/core-loggers
+      '@pnpm/logger':
+        specifier: ^5.0.0
+        version: 5.0.0
+      '@pnpm/symlink-dependency':
+        specifier: workspace:*
+        version: link:../../fs/symlink-dependency
+    devDependencies:
+      '@pnpm/pkg-manager.direct-dep-linker':
+        specifier: workspace:*
+        version: 'link:'
+
   pkg-manager/get-context:
     dependencies:
       '@pnpm/constants':
@@ -2814,6 +2833,9 @@ importers:
       '@pnpm/package-requester':
         specifier: workspace:*
         version: link:../package-requester
+      '@pnpm/pkg-manager.direct-dep-linker':
+        specifier: workspace:*
+        version: link:../direct-dep-linker
       '@pnpm/read-package-json':
         specifier: workspace:*
         version: link:../../pkg-manifest/read-package-json