From d5496cc3f8fb5696fb4236fb27a956ec93c50eda Mon Sep 17 00:00:00 2001 From: Zoltan Kochan Date: Wed, 23 Nov 2022 23:21:01 +0200 Subject: [PATCH] refactor: linking direct dependencies (#5669) --- .changeset/modern-otters-explode.md | 5 ++ pkg-manager/core/package.json | 1 + pkg-manager/core/src/install/link.ts | 84 +++++++++-------- pkg-manager/core/tsconfig.json | 3 + pkg-manager/direct-dep-linker/README.md | 13 +++ pkg-manager/direct-dep-linker/package.json | 44 +++++++++ pkg-manager/direct-dep-linker/src/index.ts | 1 + .../direct-dep-linker/src/linkDirectDeps.ts | 55 ++++++++++++ pkg-manager/direct-dep-linker/tsconfig.json | 19 ++++ .../direct-dep-linker/tsconfig.lint.json | 8 ++ pkg-manager/headless/package.json | 1 + pkg-manager/headless/src/index.ts | 89 ++++++++++--------- pkg-manager/headless/tsconfig.json | 3 + pnpm-lock.yaml | 22 +++++ 14 files changed, 267 insertions(+), 81 deletions(-) create mode 100644 .changeset/modern-otters-explode.md create mode 100644 pkg-manager/direct-dep-linker/README.md create mode 100644 pkg-manager/direct-dep-linker/package.json create mode 100644 pkg-manager/direct-dep-linker/src/index.ts create mode 100644 pkg-manager/direct-dep-linker/src/linkDirectDeps.ts create mode 100644 pkg-manager/direct-dep-linker/tsconfig.json create mode 100644 pkg-manager/direct-dep-linker/tsconfig.lint.json diff --git a/.changeset/modern-otters-explode.md b/.changeset/modern-otters-explode.md new file mode 100644 index 0000000000..229598cc55 --- /dev/null +++ b/.changeset/modern-otters-explode.md @@ -0,0 +1,5 @@ +--- +"@pnpm/pkg-manager.direct-dep-linker": minor +--- + +Initial release. diff --git a/pkg-manager/core/package.json b/pkg-manager/core/package.json index a4a0332f0e..526e5803f0 100644 --- a/pkg-manager/core/package.json +++ b/pkg-manager/core/package.json @@ -41,6 +41,7 @@ "@pnpm/npm-package-arg": "^1.0.0", "@pnpm/package-requester": "workspace:*", "@pnpm/parse-wanted-dependency": "workspace:*", + "@pnpm/pkg-manager.direct-dep-linker": "workspace:*", "@pnpm/prune-lockfile": "workspace:*", "@pnpm/read-modules-dir": "workspace:*", "@pnpm/read-package-json": "workspace:*", diff --git a/pkg-manager/core/src/install/link.ts b/pkg-manager/core/src/install/link.ts index 8c5182df54..1819972c40 100644 --- a/pkg-manager/core/src/install/link.ts +++ b/pkg-manager/core/src/install/link.ts @@ -3,13 +3,13 @@ import path from 'path' import { calcDepState, DepsStateCache } from '@pnpm/calc-dep-state' import { progressLogger, - rootLogger, stageLogger, statsLogger, } from '@pnpm/core-loggers' import { filterLockfileByImporters, } from '@pnpm/filter-lockfile' +import { linkDirectDeps } from '@pnpm/pkg-manager.direct-dep-linker' import { hoist } from '@pnpm/hoist' import { Lockfile } from '@pnpm/lockfile-file' import { logger } from '@pnpm/logger' @@ -21,7 +21,7 @@ import { LinkedDependency, } from '@pnpm/resolve-dependencies' import { StoreController } from '@pnpm/store-controller-types' -import { symlinkDependency, symlinkDirectRootDependency } from '@pnpm/symlink-dependency' +import { symlinkDependency } from '@pnpm/symlink-dependency' import { HoistedDependencies, Registries, @@ -160,43 +160,49 @@ export async function linkPackages ( }) if (opts.symlink) { - await Promise.all(projects.map(async ({ id, manifest, modulesDir, rootDir }) => { - const deps = opts.dependenciesByProjectId[id] - const importerFromLockfile = newCurrentLockfile.importers[id] - await Promise.all([ - ...Object.entries(deps) - .filter(([rootAlias]) => importerFromLockfile.specifiers[rootAlias]) - .map(([rootAlias, depPath]) => ({ rootAlias, depGraphNode: depGraph[depPath] })) - .filter(({ depGraphNode }) => depGraphNode) - .map(async ({ rootAlias, depGraphNode }) => { - if ( - (await symlinkDependency(depGraphNode.dir, modulesDir, rootAlias)).reused - ) return - - const isDev = Boolean(manifest.devDependencies?.[depGraphNode.name]) - const isOptional = Boolean(manifest.optionalDependencies?.[depGraphNode.name]) - rootLogger.debug({ - added: { - dependencyType: isDev && 'dev' || isOptional && 'optional' || 'prod', - id: depGraphNode.id, - latest: opts.outdatedDependencies[depGraphNode.id], - name: rootAlias, - realName: depGraphNode.name, - version: depGraphNode.version, - }, - prefix: rootDir, - }) - }), - ...opts.linkedDependenciesByProjectId[id].map(async (linkedDependency) => { - const depLocation = resolvePath(rootDir, linkedDependency.resolution.directory) - return symlinkDirectRootDependency(depLocation, modulesDir, linkedDependency.alias, { - fromDependenciesField: linkedDependency.dev && 'devDependencies' || linkedDependency.optional && 'optionalDependencies' || 'dependencies', - linkedPackage: linkedDependency, - prefix: rootDir, - }) - }), - ]) - })) + const projectsToLink = await Promise.all( + projects.map(async ({ id, manifest, modulesDir, rootDir }) => { + const deps = opts.dependenciesByProjectId[id] + const importerFromLockfile = newCurrentLockfile.importers[id] + return { + dir: rootDir, + modulesDir, + dependencies: await Promise.all([ + ...Object.entries(deps) + .filter(([rootAlias]) => importerFromLockfile.specifiers[rootAlias]) + .map(([rootAlias, depPath]) => ({ rootAlias, depGraphNode: depGraph[depPath] })) + .filter(({ depGraphNode }) => depGraphNode) + .map(async ({ rootAlias, depGraphNode }) => { + const isDev = Boolean(manifest.devDependencies?.[depGraphNode.name]) + const isOptional = Boolean(manifest.optionalDependencies?.[depGraphNode.name]) + return { + alias: rootAlias, + name: depGraphNode.name, + version: depGraphNode.version, + dir: depGraphNode.dir, + id: depGraphNode.id, + dependencyType: (isDev && 'dev' || isOptional && 'optional' || 'prod') as 'dev' | 'optional' | 'prod', + latest: opts.outdatedDependencies[depGraphNode.id], + isExternalLink: false, + } + }), + ...opts.linkedDependenciesByProjectId[id].map(async (linkedDependency) => { + const dir = resolvePath(rootDir, linkedDependency.resolution.directory) + return { + alias: linkedDependency.alias, + name: linkedDependency.name, + version: linkedDependency.version, + dir, + id: linkedDependency.resolution.directory, + dependencyType: (linkedDependency.dev && 'dev' || linkedDependency.optional && 'optional' || 'prod') as 'dev' | 'optional' | 'prod', + isExternalLink: true, + } + }), + ]), + } + }) + ) + await linkDirectDeps(projectsToLink) } let currentLockfile: Lockfile diff --git a/pkg-manager/core/tsconfig.json b/pkg-manager/core/tsconfig.json index 728eb65aff..9120825cf2 100644 --- a/pkg-manager/core/tsconfig.json +++ b/pkg-manager/core/tsconfig.json @@ -117,6 +117,9 @@ { "path": "../client" }, + { + "path": "../direct-dep-linker" + }, { "path": "../get-context" }, diff --git a/pkg-manager/direct-dep-linker/README.md b/pkg-manager/direct-dep-linker/README.md new file mode 100644 index 0000000000..95e550026c --- /dev/null +++ b/pkg-manager/direct-dep-linker/README.md @@ -0,0 +1,13 @@ +# @pnpm/pkg-manager.direct-dep-linker + +> Links direct dependencies to projects + +## Installation + +``` +pnpm install @pnpm/pkg-manager.direct-dep-linker +``` + +## License + +[MIT](LICENSE) diff --git a/pkg-manager/direct-dep-linker/package.json b/pkg-manager/direct-dep-linker/package.json new file mode 100644 index 0000000000..4a9f14b69f --- /dev/null +++ b/pkg-manager/direct-dep-linker/package.json @@ -0,0 +1,44 @@ +{ + "name": "@pnpm/pkg-manager.direct-dep-linker", + "description": "Fast installation using only pnpm-lock.yaml", + "version": "0.0.0", + "bugs": { + "url": "https://github.com/pnpm/pnpm/issues" + }, + "main": "lib/index.js", + "types": "lib/index.d.ts", + "files": [ + "lib", + "!*.map" + ], + "peerDependencies": { + "@pnpm/logger": "^5.0.0" + }, + "devDependencies": { + "@pnpm/pkg-manager.direct-dep-linker": "workspace:*" + }, + "homepage": "https://github.com/pnpm/pnpm/blob/main/pkg-manager/direct-dep-linker#readme", + "keywords": [ + "pnpm7", + "pnpm" + ], + "license": "MIT", + "engines": { + "node": ">=14.6" + }, + "repository": "https://github.com/pnpm/pnpm/blob/main/pkg-manager/direct-dep-linker", + "scripts": { + "lint": "eslint src/**/*.ts", + "test": "pnpm run compile", + "prepublishOnly": "pnpm run compile", + "compile": "tsc --build && pnpm run lint --fix" + }, + "dependencies": { + "@pnpm/core-loggers": "workspace:*", + "@pnpm/symlink-dependency": "workspace:*" + }, + "funding": "https://opencollective.com/pnpm", + "exports": { + ".": "./lib/index.js" + } +} diff --git a/pkg-manager/direct-dep-linker/src/index.ts b/pkg-manager/direct-dep-linker/src/index.ts new file mode 100644 index 0000000000..15bf5a8d95 --- /dev/null +++ b/pkg-manager/direct-dep-linker/src/index.ts @@ -0,0 +1 @@ +export * from './linkDirectDeps' diff --git a/pkg-manager/direct-dep-linker/src/linkDirectDeps.ts b/pkg-manager/direct-dep-linker/src/linkDirectDeps.ts new file mode 100644 index 0000000000..81fed66ce0 --- /dev/null +++ b/pkg-manager/direct-dep-linker/src/linkDirectDeps.ts @@ -0,0 +1,55 @@ +import { rootLogger } from '@pnpm/core-loggers' +import { symlinkDependency, symlinkDirectRootDependency } from '@pnpm/symlink-dependency' + +export interface LinkedDirectDep { + alias: string + name: string + version: string + dir: string + id: string + dependencyType: 'prod' | 'dev' | 'optional' + isExternalLink: boolean + latest?: string +} + +export interface ProjectToLink { + dir: string + modulesDir: string + dependencies: LinkedDirectDep[] +} + +export async function linkDirectDeps ( + projects: ProjectToLink[] +) { + await Promise.all(projects.map(async (project) => { + await Promise.all(project.dependencies.map(async (dep) => { + if (dep.isExternalLink) { + await symlinkDirectRootDependency(dep.dir, project.modulesDir, dep.alias, { + fromDependenciesField: dep.dependencyType === 'dev' && 'devDependencies' || + dep.dependencyType === 'optional' && 'optionalDependencies' || + 'dependencies', + linkedPackage: { + name: dep.name, + version: dep.version, + }, + prefix: project.dir, + }) + return + } + if ((await symlinkDependency(dep.dir, project.modulesDir, dep.alias)).reused) { + return + } + rootLogger.debug({ + added: { + dependencyType: dep.dependencyType, + id: dep.id, + latest: dep.latest, + name: dep.alias, + realName: dep.name, + version: dep.version, + }, + prefix: project.dir, + }) + })) + })) +} diff --git a/pkg-manager/direct-dep-linker/tsconfig.json b/pkg-manager/direct-dep-linker/tsconfig.json new file mode 100644 index 0000000000..2b2e3ca812 --- /dev/null +++ b/pkg-manager/direct-dep-linker/tsconfig.json @@ -0,0 +1,19 @@ +{ + "extends": "@pnpm/tsconfig", + "compilerOptions": { + "outDir": "lib", + "rootDir": "src" + }, + "include": [ + "src/**/*.ts", + "../../__typings__/**/*.d.ts" + ], + "references": [ + { + "path": "../../fs/symlink-dependency" + }, + { + "path": "../../packages/core-loggers" + } + ] +} diff --git a/pkg-manager/direct-dep-linker/tsconfig.lint.json b/pkg-manager/direct-dep-linker/tsconfig.lint.json new file mode 100644 index 0000000000..1bbe711971 --- /dev/null +++ b/pkg-manager/direct-dep-linker/tsconfig.lint.json @@ -0,0 +1,8 @@ +{ + "extends": "./tsconfig.json", + "include": [ + "src/**/*.ts", + "test/**/*.ts", + "../../__typings__/**/*.d.ts" + ] +} diff --git a/pkg-manager/headless/package.json b/pkg-manager/headless/package.json index 8e9b62363c..fb3df4fa87 100644 --- a/pkg-manager/headless/package.json +++ b/pkg-manager/headless/package.json @@ -83,6 +83,7 @@ "@pnpm/modules-yaml": "workspace:*", "@pnpm/package-is-installable": "workspace:*", "@pnpm/package-requester": "workspace:*", + "@pnpm/pkg-manager.direct-dep-linker": "workspace:*", "@pnpm/read-package-json": "workspace:*", "@pnpm/read-project-manifest": "workspace:*", "@pnpm/real-hoist": "workspace:*", diff --git a/pkg-manager/headless/src/index.ts b/pkg-manager/headless/src/index.ts index 9273c1e2e7..9bc9f028a1 100644 --- a/pkg-manager/headless/src/index.ts +++ b/pkg-manager/headless/src/index.ts @@ -9,7 +9,6 @@ import { import { packageManifestLogger, progressLogger, - rootLogger, stageLogger, statsLogger, summaryLogger, @@ -55,7 +54,7 @@ import { PackageFilesResponse, StoreController, } from '@pnpm/store-controller-types' -import { symlinkDependency, symlinkDirectRootDependency } from '@pnpm/symlink-dependency' +import { symlinkDependency } from '@pnpm/symlink-dependency' import { DependencyManifest, HoistedDependencies, ProjectManifest, Registries } from '@pnpm/types' import * as dp from 'dependency-path' import pLimit from 'p-limit' @@ -78,6 +77,7 @@ import { lockfileToDepGraph, } from './lockfileToDepGraph' import { lockfileToHoistedDepGraph } from './lockfileToHoistedDepGraph' +import { linkDirectDeps, LinkedDirectDep } from '@pnpm/pkg-manager.direct-dep-linker' export { HoistingLimits } @@ -576,26 +576,32 @@ async function symlinkDirectDependencies ( symlink, }: SymlinkDirectDependenciesOpts ) { - await Promise.all(projects.map(async ({ rootDir, id, manifest, modulesDir }) => { - if (symlink !== false) { - await linkRootPackages(filteredLockfile, { - importerId: id, - importerModulesDir: modulesDir, - lockfileDir, - projectDir: rootDir, - projects, - registries, - rootDependencies: directDependenciesByImporterId[id], - }) - } - + projects.forEach(({ rootDir, manifest }) => { // Even though headless installation will never update the package.json // this needs to be logged because otherwise install summary won't be printed packageManifestLogger.debug({ prefix: rootDir, updated: manifest, }) - })) + }) + if (symlink !== false) { + const projectsToLink = await Promise.all( + projects.map(async ({ rootDir, id, modulesDir }) => ({ + dir: rootDir, + modulesDir, + dependencies: await getRootPackagesToLink(filteredLockfile, { + importerId: id, + importerModulesDir: modulesDir, + lockfileDir, + projectDir: rootDir, + projects, + registries, + rootDependencies: directDependenciesByImporterId[id], + }), + })) + ) + await linkDirectDeps(projectsToLink) + } } async function linkBinsOfImporter ( @@ -617,7 +623,7 @@ async function linkBinsOfImporter ( }) } -async function linkRootPackages ( +async function getRootPackagesToLink ( lockfile: Lockfile, opts: { registries: Registries @@ -628,7 +634,7 @@ async function linkRootPackages ( lockfileDir: string rootDependencies: { [alias: string]: string } } -) { +): Promise { const importerManifestsByImporterId = {} as { [id: string]: ProjectManifest } for (const { id, manifest } of opts.projects) { importerManifestsByImporterId[id] = manifest @@ -639,7 +645,7 @@ async function linkRootPackages ( ...projectSnapshot.dependencies, ...projectSnapshot.optionalDependencies, } - return Promise.all( + return (await Promise.all( Object.entries(allDeps) .map(async ([alias, ref]) => { if (ref.startsWith('link:')) { @@ -659,23 +665,23 @@ async function linkRootPackages ( return { name: alias, version: '0.0.0' } } })() as DependencyManifest - await symlinkDirectRootDependency(packageDir, opts.importerModulesDir, alias, { - fromDependenciesField: isDev && 'devDependencies' || - isOptional && 'optionalDependencies' || - 'dependencies', - linkedPackage, - prefix: opts.projectDir, - }) - return + return { + alias, + name: linkedPackage.name, + version: linkedPackage.version, + dir: packageDir, + id: ref, + isExternalLink: true, + dependencyType: isDev && 'dev' || + isOptional && 'optional' || + 'prod', + } } const dir = opts.rootDependencies[alias] // Skipping linked packages if (!dir) { return } - if ((await symlinkDependency(dir, opts.importerModulesDir, alias)).reused) { - return - } const isDev = Boolean(projectSnapshot.devDependencies?.[alias]) const isOptional = Boolean(projectSnapshot.optionalDependencies?.[alias]) @@ -685,19 +691,18 @@ async function linkRootPackages ( if (pkgSnapshot == null) return // this won't ever happen. Just making typescript happy const pkgId = pkgSnapshot.id ?? dp.refToAbsolute(ref, alias, opts.registries) ?? undefined const pkgInfo = nameVerFromPkgSnapshot(depPath, pkgSnapshot) - rootLogger.debug({ - added: { - dependencyType: isDev && 'dev' || isOptional && 'optional' || 'prod', - id: pkgId, - // latest: opts.outdatedPkgs[pkg.id], - name: alias, - realName: pkgInfo.name, - version: pkgInfo.version, - }, - prefix: opts.projectDir, - }) + return { + alias, + isExternalLink: false, + name: pkgInfo.name, + version: pkgInfo.version, + dependencyType: isDev && 'dev' || isOptional && 'optional' || 'prod', + dir, + id: pkgId, + } }) - ) + )) + .filter(Boolean) as LinkedDirectDep[] } const limitLinking = pLimit(16) diff --git a/pkg-manager/headless/tsconfig.json b/pkg-manager/headless/tsconfig.json index ae88546019..cd1d0fa50e 100644 --- a/pkg-manager/headless/tsconfig.json +++ b/pkg-manager/headless/tsconfig.json @@ -81,6 +81,9 @@ { "path": "../client" }, + { + "path": "../direct-dep-linker" + }, { "path": "../hoist" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d38bfc9234..c4ac509d33 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2539,6 +2539,9 @@ importers: '@pnpm/parse-wanted-dependency': specifier: workspace:* version: link:../../packages/parse-wanted-dependency + '@pnpm/pkg-manager.direct-dep-linker': + specifier: workspace:* + version: link:../direct-dep-linker '@pnpm/prune-lockfile': specifier: workspace:* version: link:../../lockfile/prune-lockfile @@ -2712,6 +2715,22 @@ importers: specifier: ^4.2.0 version: 4.2.0 + pkg-manager/direct-dep-linker: + dependencies: + '@pnpm/core-loggers': + specifier: workspace:* + version: link:../../packages/core-loggers + '@pnpm/logger': + specifier: ^5.0.0 + version: 5.0.0 + '@pnpm/symlink-dependency': + specifier: workspace:* + version: link:../../fs/symlink-dependency + devDependencies: + '@pnpm/pkg-manager.direct-dep-linker': + specifier: workspace:* + version: 'link:' + pkg-manager/get-context: dependencies: '@pnpm/constants': @@ -2814,6 +2833,9 @@ importers: '@pnpm/package-requester': specifier: workspace:* version: link:../package-requester + '@pnpm/pkg-manager.direct-dep-linker': + specifier: workspace:* + version: link:../direct-dep-linker '@pnpm/read-package-json': specifier: workspace:* version: link:../../pkg-manifest/read-package-json