From 2ca77fc21ef46c609acc4cee0ddb69fc19a835f3 Mon Sep 17 00:00:00 2001 From: Zoltan Kochan Date: Sun, 7 Apr 2019 02:14:32 +0300 Subject: [PATCH] fix: bins should be linked after running lifecycle scripts PR #1764 * refactor: create @pnpm/build-modules * fix: bins should be linked after running lifecycle scripts close #1608 * fix: rebuild should link bins * fix: rebuild should link all bins * fix(deps): update @pnpm/link-bins to v4.0.2 * test: lifecycle scripts run before linking bins * test: bins are linked even if lifecycle scripts are ignored * fix: building modules in headless mode --- packages/build-modules/LICENSE | 22 ++ packages/build-modules/README.md | 17 ++ packages/build-modules/package.json | 65 ++++++ packages/build-modules/src/index.ts | 208 ++++++++++++++++++ packages/build-modules/tsconfig.json | 24 ++ packages/build-modules/tslint.json | 3 + packages/build-modules/typings/index.d.ts | 4 + packages/headless/package.json | 9 +- packages/headless/src/index.ts | 52 +++-- .../headless/src/runDependenciesScripts.ts | 123 ----------- packages/outdated/package.json | 2 +- packages/pnpm/package.json | 2 +- packages/supi/package.json | 5 +- packages/supi/src/install/index.ts | 144 ++++-------- packages/supi/src/install/link.ts | 70 +----- packages/supi/src/install/resolvePeers.ts | 4 +- packages/supi/src/install/updateLockfile.ts | 4 +- packages/supi/src/rebuild/index.ts | 50 ++++- .../supi/test/install/lifecycleScripts.ts | 67 ++++++ packages/supi/test/rebuild.ts | 20 ++ pnpm-lock.yaml | 87 ++++++-- 21 files changed, 638 insertions(+), 344 deletions(-) create mode 100644 packages/build-modules/LICENSE create mode 100644 packages/build-modules/README.md create mode 100644 packages/build-modules/package.json create mode 100644 packages/build-modules/src/index.ts create mode 100644 packages/build-modules/tsconfig.json create mode 100644 packages/build-modules/tslint.json create mode 100644 packages/build-modules/typings/index.d.ts delete mode 100644 packages/headless/src/runDependenciesScripts.ts diff --git a/packages/build-modules/LICENSE b/packages/build-modules/LICENSE new file mode 100644 index 0000000000..757f1faa24 --- /dev/null +++ b/packages/build-modules/LICENSE @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2015-2016 Rico Sta. Cruz +Copyright (c) 2016-2019 Zoltan Kochan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/build-modules/README.md b/packages/build-modules/README.md new file mode 100644 index 0000000000..04fbc3e430 --- /dev/null +++ b/packages/build-modules/README.md @@ -0,0 +1,17 @@ +# @pnpm/build-modules + +> Build packages in node_modules + + +[![npm version](https://img.shields.io/npm/v/@pnpm/build-modules.svg)](https://www.npmjs.com/package/@pnpm/build-modules) + + +## Installation + +```sh +npm i -S @pnpm/build-modules +``` + +## License + +[MIT](./LICENSE) © [Zoltan Kochan](https://www.kochan.io/) diff --git a/packages/build-modules/package.json b/packages/build-modules/package.json new file mode 100644 index 0000000000..9891034e3d --- /dev/null +++ b/packages/build-modules/package.json @@ -0,0 +1,65 @@ +{ + "name": "@pnpm/build-modules", + "version": "0.0.0", + "description": "Build packages in node_modules", + "main": "lib/index.js", + "typings": "lib/index.d.ts", + "files": [ + "lib" + ], + "engines": { + "node": ">=8" + }, + "scripts": { + "lint": "tslint -c tslint.json src/**/*.ts test/**/*.ts", + "tsc": "rimraf lib && tsc", + "test": "npm run tsc && npm run lint && mos t", + "md": "mos", + "prepublishOnly": "npm run tsc" + }, + "repository": "https://github.com/pnpm/pnpm/blob/master/packages/build-modules", + "keywords": [ + "pnpm", + "resolver", + "npm" + ], + "author": "Zoltan Kochan (https://www.kochan.io/)", + "license": "MIT", + "bugs": { + "url": "https://github.com/pnpm/pnpm/issues" + }, + "homepage": "https://pnpm.js.org", + "dependencies": { + "@pnpm/constants": "1.0.0", + "@pnpm/core-loggers": "3.0.0", + "@pnpm/lifecycle": "5.0.0", + "@pnpm/link-bins": "4.0.2", + "@pnpm/read-package-json": "2.0.1", + "@pnpm/store-controller-types": "3.0.0", + "@pnpm/types": "3.0.0", + "@types/node": "*", + "@types/ramda": "0.26.6", + "graph-sequencer": "2.0.0", + "ramda": "0.26.1", + "run-groups": "2.0.0" + }, + "devDependencies": { + "@pnpm/build-modules": "link:", + "@pnpm/logger": "2.1.0", + "@pnpm/tslint-config": "0.0.0", + "mos": "2.0.0-alpha.3", + "mos-plugin-readme": "1.0.4", + "rimraf": "2.6.3", + "ts-node": "8.0.3", + "tslint": "5.15.0", + "typescript": "3.4.1" + }, + "mos": { + "plugins": [ + "readme" + ], + "installation": { + "useShortAlias": true + } + } +} diff --git a/packages/build-modules/src/index.ts b/packages/build-modules/src/index.ts new file mode 100644 index 0000000000..1af6432cd8 --- /dev/null +++ b/packages/build-modules/src/index.ts @@ -0,0 +1,208 @@ +import { ENGINE_NAME } from '@pnpm/constants' +import { skippedOptionalDependencyLogger } from '@pnpm/core-loggers' +import { runPostinstallHooks } from '@pnpm/lifecycle' +import linkBins, { linkBinsOfPackages } from '@pnpm/link-bins' +import logger from '@pnpm/logger' +import { fromDir as readPackageFromDir } from '@pnpm/read-package-json' +import { StoreController } from '@pnpm/store-controller-types' +import { PackageJson } from '@pnpm/types' +import graphSequencer = require('graph-sequencer') +import path = require('path') +import R = require('ramda') +import runGroups from 'run-groups' + +export default async ( + depGraph: DependenciesGraph, + rootDepPaths: string[], + opts: { + childConcurrency?: number, + depsToBuild?: Set, + optional: boolean, + prefix: string, + rawNpmConfig: object, + unsafePerm: boolean, + userAgent: string, + sideEffectsCacheWrite: boolean, + storeController: StoreController, + rootNodeModulesDir: string, + }, +) => { + const warn = (message: string) => logger.warn({ message, prefix: opts.prefix }) + // postinstall hooks + const nodesToBuild = new Set() + getSubgraphToBuild(depGraph, rootDepPaths, nodesToBuild, new Set()) + const onlyFromBuildGraph = R.filter((depPath: string) => nodesToBuild.has(depPath)) + + const nodesToBuildArray = Array.from(nodesToBuild) + const graph = new Map( + nodesToBuildArray + .map((depPath) => [depPath, onlyFromBuildGraph(R.values(depGraph[depPath].children))]) as Array<[string, string[]]>, + ) + const graphSequencerResult = graphSequencer({ + graph, + groups: [nodesToBuildArray], + }) + const chunks = graphSequencerResult.chunks as string[][] + const buildDepOpts = { ...opts, warn } + const groups = chunks.map((chunk) => { + chunk = chunk.filter((depPath) => depGraph[depPath].requiresBuild && !depGraph[depPath].isBuilt) + if (opts.depsToBuild) { + chunk = chunk.filter((depPath) => opts.depsToBuild!.has(depPath)) + } + + return chunk.map((depPath: string) => + async () => buildDependency(depPath, depGraph, buildDepOpts) + ) + }) + await runGroups(opts.childConcurrency || 4, groups) +} + +async function buildDependency ( + depPath: string, + depGraph: DependenciesGraph, + opts: { + optional: boolean, + prefix: string, + rawNpmConfig: object, + rootNodeModulesDir: string, + sideEffectsCacheWrite: boolean, + storeController: StoreController, + unsafePerm: boolean, + warn: (message: string) => void, + } +) { + const depNode = depGraph[depPath] + try { + await linkBinsOfDependencies(depNode, depGraph, opts) + const hasSideEffects = await runPostinstallHooks({ + depPath, + optional: depNode.optional, + pkgRoot: depNode.peripheralLocation, + prepare: depNode.prepare, + rawNpmConfig: opts.rawNpmConfig, + rootNodeModulesDir: opts.rootNodeModulesDir, + unsafePerm: opts.unsafePerm || false, + }) + if (hasSideEffects && opts.sideEffectsCacheWrite) { + try { + await opts.storeController.upload(depNode.peripheralLocation, { + engine: ENGINE_NAME, + packageId: depNode.packageId, + }) + } catch (err) { + if (err && err.statusCode === 403) { + logger.warn({ + message: `The store server disabled upload requests, could not upload ${depNode.packageId}`, + prefix: opts.prefix, + }) + } else { + logger.warn({ + error: err, + message: `An error occurred while uploading ${depNode.packageId}`, + prefix: opts.prefix, + }) + } + } + } + } catch (err) { + if (depNode.optional) { + // TODO: add parents field to the log + const pkg = await readPackageFromDir(path.join(depNode.peripheralLocation)) + skippedOptionalDependencyLogger.debug({ + details: err.toString(), + package: { + id: depNode.packageId, + name: pkg.name, + version: pkg.version, + }, + prefix: opts.prefix, + reason: 'build_failure', + }) + return + } + throw err + } +} + +function getSubgraphToBuild ( + graph: DependenciesGraph, + entryNodes: string[], + nodesToBuild: Set, + walked: Set, +) { + let currentShouldBeBuilt = false + for (const depPath of entryNodes) { + if (!graph[depPath]) return // packages that are already in node_modules are skipped + if (nodesToBuild.has(depPath)) { + currentShouldBeBuilt = true + } + if (walked.has(depPath)) continue + walked.add(depPath) + const childShouldBeBuilt = getSubgraphToBuild(graph, R.values(graph[depPath].children), nodesToBuild, walked) + || graph[depPath].requiresBuild + if (childShouldBeBuilt) { + nodesToBuild.add(depPath) + currentShouldBeBuilt = true + } + } + return currentShouldBeBuilt +} + +export interface DependenciesGraphNode { + fetchingRawManifest?: Promise, + hasBundledDependencies: boolean, + peripheralLocation: string, + children: {[alias: string]: string}, + optional: boolean, + optionalDependencies: Set, + packageId: string, // TODO: this option is currently only needed when running postinstall scripts but even there it should be not used + installable?: boolean, + isBuilt?: boolean, + requiresBuild?: boolean, + prepare: boolean, + hasBin: boolean, +} + +export interface DependenciesGraph { + [depPath: string]: DependenciesGraphNode +} + +export async function linkBinsOfDependencies ( + depNode: DependenciesGraphNode, + depGraph: DependenciesGraph, + opts: { + optional: boolean, + warn: (message: string) => void, + }, +) { + const childrenToLink = opts.optional + ? depNode.children + : R.keys(depNode.children) + .reduce((nonOptionalChildren, childAlias: string) => { + if (!depNode.optionalDependencies.has(childAlias)) { + nonOptionalChildren[childAlias] = depNode.children[childAlias] + } + return nonOptionalChildren + }, {}) + + const pkgs = await Promise.all( + R.keys(childrenToLink) + .filter((alias) => depGraph[childrenToLink[alias]].hasBin && depGraph[childrenToLink[alias]].installable !== false) + .map(async (alias) => { + const dep = depGraph[childrenToLink[alias]] + return { + location: dep.peripheralLocation, + manifest: dep.fetchingRawManifest && (await dep.fetchingRawManifest) || await readPackageFromDir(dep.peripheralLocation), + } + }), + ) + + const binPath = path.join(depNode.peripheralLocation, 'node_modules', '.bin') + await linkBinsOfPackages(pkgs, binPath, { warn: opts.warn }) + + // link also the bundled dependencies` bins + if (depNode.hasBundledDependencies) { + const bundledModules = path.join(depNode.peripheralLocation, 'node_modules') + await linkBins(bundledModules, binPath, { warn: opts.warn }) + } +} diff --git a/packages/build-modules/tsconfig.json b/packages/build-modules/tsconfig.json new file mode 100644 index 0000000000..88257b3393 --- /dev/null +++ b/packages/build-modules/tsconfig.json @@ -0,0 +1,24 @@ +{ + "compilerOptions": { + "removeComments": false, + "preserveConstEnums": true, + "sourceMap": true, + "declaration": true, + "noImplicitAny": true, + "noImplicitReturns": true, + "suppressImplicitAnyIndexErrors": true, + "allowSyntheticDefaultImports": true, + "strictNullChecks": true, + "target": "es2017", + "outDir": "lib", + "module": "commonjs", + "moduleResolution": "node" + }, + "include": [ + "src/**/*.ts", + "typings/**/*.d.ts" + ], + "atom": { + "rewriteTsconfig": true + } +} diff --git a/packages/build-modules/tslint.json b/packages/build-modules/tslint.json new file mode 100644 index 0000000000..568acfabec --- /dev/null +++ b/packages/build-modules/tslint.json @@ -0,0 +1,3 @@ +{ + "extends": "@pnpm/tslint-config" +} diff --git a/packages/build-modules/typings/index.d.ts b/packages/build-modules/typings/index.d.ts new file mode 100644 index 0000000000..ca53f914ef --- /dev/null +++ b/packages/build-modules/typings/index.d.ts @@ -0,0 +1,4 @@ +declare module 'graph-sequencer' { + const anything: any; + export = anything; +} diff --git a/packages/headless/package.json b/packages/headless/package.json index 318e299e38..9f55364578 100644 --- a/packages/headless/package.json +++ b/packages/headless/package.json @@ -41,7 +41,7 @@ "isexe": "2.0.0", "mz": "2.7.0", "npm-run-all": "4.1.5", - "pnpm-registry-mock": "2.9.0", + "pnpm-registry-mock": "2.11.0", "rimraf": "2.6.3", "rimraf-then": "1.0.1", "sinon": "7.3.1", @@ -83,11 +83,12 @@ "prepareFixtures": "npm-run-all -p -r pnpm-registry-mock runPrepareFixtures" }, "dependencies": { + "@pnpm/build-modules": "0.0.0", "@pnpm/constants": "1.0.0", "@pnpm/core-loggers": "3.0.0", "@pnpm/filter-lockfile": "1.0.1", "@pnpm/lifecycle": "5.0.0", - "@pnpm/link-bins": "4.0.1", + "@pnpm/link-bins": "4.0.2", "@pnpm/lockfile-file": "1.0.1", "@pnpm/lockfile-utils": "1.0.1", "@pnpm/modules-cleaner": "3.0.3", @@ -102,10 +103,8 @@ "@pnpm/utils": "0.10.1", "@types/ramda": "0.25.34", "dependency-path": "3.0.1", - "graph-sequencer": "2.0.0", "p-limit": "2.2.0", "path-exists": "4.0.0", - "ramda": "0.26.1", - "run-groups": "2.0.0" + "ramda": "0.26.1" } } diff --git a/packages/headless/src/index.ts b/packages/headless/src/index.ts index eb653b73df..81b4deb9d0 100644 --- a/packages/headless/src/index.ts +++ b/packages/headless/src/index.ts @@ -1,3 +1,4 @@ +import buildModules from '@pnpm/build-modules' import { ENGINE_NAME, LAYOUT_VERSION, @@ -54,7 +55,6 @@ import fs = require('mz/fs') import pLimit from 'p-limit' import path = require('path') import R = require('ramda') -import runDependenciesScripts from './runDependenciesScripts' const brokenNodeModulesLogger = logger('_broken_node_modules') @@ -221,8 +221,6 @@ export default async (opts: HeadlessOptions) => { }) } - await linkAllBins(depGraph, { optional: opts.include.optionalDependencies, warn }) - await Promise.all(opts.importers.map(async (importer) => { if (importer.shamefullyFlatten) { importer.hoistedAliases = await shamefullyFlattenByLockfile(filteredLockfile, importer.id, { @@ -253,8 +251,6 @@ export default async (opts: HeadlessOptions) => { registries: opts.registries, rootDependencies: res.directDependenciesByImporterId[importer.id], }) - const bin = path.join(importer.modulesDir, '.bin') - await linkBins(importer.modulesDir, bin, { warn }) // Even though headless installation will never update the package.json // this needs to be logged because otherwise install summary won't be printed @@ -307,20 +303,31 @@ export default async (opts: HeadlessOptions) => { }) if (!opts.ignoreScripts) { + const directNodes = new Set() for (const importer of opts.importers) { - await runDependenciesScripts(depGraph, R.values(res.directDependenciesByImporterId[importer.id]).filter((loc) => depGraph[loc]), { - childConcurrency: opts.childConcurrency, - prefix: importer.prefix, - rawNpmConfig: opts.rawNpmConfig, - rootNodeModulesDir: importer.modulesDir, - sideEffectsCacheWrite: opts.sideEffectsCacheWrite, - storeController: opts.storeController, - unsafePerm: opts.unsafePerm, - userAgent: opts.userAgent, - }) + R + .values(res.directDependenciesByImporterId[importer.id]) + .filter((loc) => depGraph[loc]) + .forEach((loc) => { + directNodes.add(loc) + }) } + await buildModules(depGraph, Array.from(directNodes), { + childConcurrency: opts.childConcurrency, + optional: opts.include.optionalDependencies, + prefix: opts.lockfileDirectory, + rawNpmConfig: opts.rawNpmConfig, + rootNodeModulesDir: virtualStoreDir, + sideEffectsCacheWrite: opts.sideEffectsCacheWrite, + storeController: opts.storeController, + unsafePerm: opts.unsafePerm, + userAgent: opts.userAgent, + }) } + await linkAllBins(depGraph, { optional: opts.include.optionalDependencies, warn }) + await Promise.all(opts.importers.map(linkBinsOfImporter)) + // waiting till package requests are finished await Promise.all(R.values(depGraph).map((depNode) => depNode.finishing)) @@ -342,6 +349,17 @@ export default async (opts: HeadlessOptions) => { } } +function linkBinsOfImporter ( + { modulesDir, bin, prefix }: { + bin: string, + modulesDir: string, + prefix: string, + }, +) { + const warn = (message: string) => logger.warn({ message, prefix }) + return linkBins(modulesDir, bin, { warn }) +} + async function linkRootPackages ( lockfile: Lockfile, opts: { @@ -496,8 +514,8 @@ async function lockfileToDepGraph ( name: pkgName, optional: !!pkgSnapshot.optional, optionalDependencies: new Set(R.keys(pkgSnapshot.optionalDependencies)), + packageId, peripheralLocation, - pkgId: packageId, prepare: pkgSnapshot.prepare === true, relDepPath, requiresBuild: pkgSnapshot.requiresBuild === true, @@ -597,7 +615,7 @@ export interface DependenciesGraphNode { optionalDependencies: Set, optional: boolean, relDepPath: string, // this option is only needed for saving pendingBuild when running with --ignore-scripts flag - pkgId: string, // TODO: this option is currently only needed when running postinstall scripts but even there it should be not used + packageId: string, // TODO: this option is currently only needed when running postinstall scripts but even there it should be not used isBuilt: boolean, requiresBuild: boolean, prepare: boolean, diff --git a/packages/headless/src/runDependenciesScripts.ts b/packages/headless/src/runDependenciesScripts.ts deleted file mode 100644 index 8803ee060f..0000000000 --- a/packages/headless/src/runDependenciesScripts.ts +++ /dev/null @@ -1,123 +0,0 @@ -// TODO: move to separate package. It is used in supi/lib/install.ts as well - -import { ENGINE_NAME } from '@pnpm/constants' -import { skippedOptionalDependencyLogger } from '@pnpm/core-loggers' -import { runPostinstallHooks } from '@pnpm/lifecycle' -import logger from '@pnpm/logger' -import { fromDir as readPackageFromDir } from '@pnpm/read-package-json' -import { StoreController } from '@pnpm/store-controller-types' -import graphSequencer = require('graph-sequencer') -import path = require('path') -import R = require('ramda') -import runGroups from 'run-groups' -import { DependenciesGraph } from '.' - -export default async ( - depGraph: DependenciesGraph, - rootDepPaths: string[], - opts: { - childConcurrency?: number, - prefix: string, - rawNpmConfig: object, - unsafePerm: boolean, - userAgent: string, - sideEffectsCacheWrite: boolean, - storeController: StoreController, - rootNodeModulesDir: string, - }, -) => { - // postinstall hooks - const nodesToBuild = new Set() - getSubgraphToBuild(depGraph, rootDepPaths, nodesToBuild, new Set()) - const onlyFromBuildGraph = R.filter((depPath: string) => nodesToBuild.has(depPath)) - - const nodesToBuildArray = Array.from(nodesToBuild) - const graph = new Map( - nodesToBuildArray - .map((depPath) => [depPath, onlyFromBuildGraph(R.values(depGraph[depPath].children))]) as Array<[string, string[]]>, - ) - const graphSequencerResult = graphSequencer({ - graph, - groups: [nodesToBuildArray], - }) - const chunks = graphSequencerResult.chunks as string[][] - const groups = chunks.map((chunk) => chunk.filter((depPath) => depGraph[depPath].requiresBuild && !depGraph[depPath].isBuilt).map((depPath: string) => - async () => { - const depNode = depGraph[depPath] - try { - const hasSideEffects = await runPostinstallHooks({ - depPath, - optional: depNode.optional, - pkgRoot: depNode.peripheralLocation, - prepare: depNode.prepare, - rawNpmConfig: opts.rawNpmConfig, - rootNodeModulesDir: opts.rootNodeModulesDir, - unsafePerm: opts.unsafePerm || false, - }) - if (hasSideEffects && opts.sideEffectsCacheWrite) { - try { - await opts.storeController.upload(depNode.peripheralLocation, { - engine: ENGINE_NAME, - packageId: depNode.pkgId, - }) - } catch (err) { - if (err && err.statusCode === 403) { - logger.warn({ - message: `The store server disabled upload requests, could not upload ${depNode.pkgId}`, - prefix: opts.prefix, - }) - } else { - logger.warn({ - error: err, - message: `An error occurred while uploading ${depNode.pkgId}`, - prefix: opts.prefix, - }) - } - } - } - } catch (err) { - if (depNode.optional) { - // TODO: add parents field to the log - const pkg = await readPackageFromDir(path.join(depNode.peripheralLocation)) - skippedOptionalDependencyLogger.debug({ - details: err.toString(), - package: { - id: depNode.pkgId, - name: pkg.name, - version: pkg.version, - }, - prefix: opts.prefix, - reason: 'build_failure', - }) - return - } - throw err - } - } - )) - await runGroups(opts.childConcurrency || 4, groups) -} - -function getSubgraphToBuild ( - graph: DependenciesGraph, - entryNodes: string[], - nodesToBuild: Set, - walked: Set, -) { - let currentShouldBeBuilt = false - for (const depPath of entryNodes) { - if (!graph[depPath]) return // packages that are already in node_modules are skipped - if (nodesToBuild.has(depPath)) { - currentShouldBeBuilt = true - } - if (walked.has(depPath)) continue - walked.add(depPath) - const childShouldBeBuilt = getSubgraphToBuild(graph, R.values(graph[depPath].children), nodesToBuild, walked) - || graph[depPath].requiresBuild - if (childShouldBeBuilt) { - nodesToBuild.add(depPath) - currentShouldBeBuilt = true - } - } - return currentShouldBeBuilt -} diff --git a/packages/outdated/package.json b/packages/outdated/package.json index c40983b858..46302b3668 100644 --- a/packages/outdated/package.json +++ b/packages/outdated/package.json @@ -58,7 +58,7 @@ "mos": "2.0.0-alpha.3", "mos-plugin-readme": "1.0.4", "npm-run-all": "4.1.5", - "pnpm-registry-mock": "2.9.0", + "pnpm-registry-mock": "2.11.0", "tape": "4.10.1", "ts-node": "8.0.3", "tslint": "5.15.0", diff --git a/packages/pnpm/package.json b/packages/pnpm/package.json index 6e4079779a..60539c3996 100644 --- a/packages/pnpm/package.json +++ b/packages/pnpm/package.json @@ -117,7 +117,7 @@ "p-any": "1.1.0", "path-exists": "4.0.0", "pnpm": "link:", - "pnpm-registry-mock": "2.9.0", + "pnpm-registry-mock": "2.11.0", "retry": "0.12.0", "rimraf": "2.6.3", "rimraf-then": "1.0.1", diff --git a/packages/supi/package.json b/packages/supi/package.json index 9f81989327..7daadc9134 100644 --- a/packages/supi/package.json +++ b/packages/supi/package.json @@ -19,6 +19,7 @@ "@pnpm/logger": "^2.1.0" }, "dependencies": { + "@pnpm/build-modules": "0.0.0", "@pnpm/check-package": "3.0.0", "@pnpm/constants": "1.0.0", "@pnpm/core-loggers": "3.0.0", @@ -26,7 +27,7 @@ "@pnpm/fs-locker": "1.0.3", "@pnpm/headless": "6.0.4", "@pnpm/lifecycle": "5.0.0", - "@pnpm/link-bins": "4.0.1", + "@pnpm/link-bins": "4.0.2", "@pnpm/lockfile-file": "1.0.1", "@pnpm/lockfile-utils": "1.0.1", "@pnpm/modules-cleaner": "3.0.3", @@ -114,7 +115,7 @@ "npm-scripts-info": "0.3.9", "path-exists": "4.0.0", "path-name": "1.0.0", - "pnpm-registry-mock": "2.9.0", + "pnpm-registry-mock": "2.11.0", "read-pkg": "4.0.1", "read-yaml-file": "1.1.0", "rimraf": "2.6.3", diff --git a/packages/supi/src/install/index.ts b/packages/supi/src/install/index.ts index aa9d68790e..3f1e1f1420 100644 --- a/packages/supi/src/install/index.ts +++ b/packages/supi/src/install/index.ts @@ -1,20 +1,19 @@ +import buildModules, { linkBinsOfDependencies } from '@pnpm/build-modules' import { - ENGINE_NAME, LAYOUT_VERSION, LOCKFILE_VERSION, WANTED_LOCKFILE, } from '@pnpm/constants' import { packageJsonLogger, - skippedOptionalDependencyLogger, stageLogger, summaryLogger, } from '@pnpm/core-loggers' import headless from '@pnpm/headless' import { runLifecycleHooksConcurrently, - runPostinstallHooks, } from '@pnpm/lifecycle' +import linkBins from '@pnpm/link-bins' import { Lockfile, LockfileImporter, @@ -46,15 +45,14 @@ import { WantedDependency, } from '@pnpm/utils' import * as dp from 'dependency-path' -import graphSequencer = require('graph-sequencer') import isInnerLink = require('is-inner-link') import isSubdir = require('is-subdir') import pEvery from 'p-every' import pFilter = require('p-filter') +import pLimit = require('p-limit') import path = require('path') import R = require('ramda') import rimraf = require('rimraf-then') -import runGroups from 'run-groups' import semver = require('semver') import { PnpmError } from '../errorTypes' import getContext, { ImportersOptions, PnpmContext } from '../getContext' @@ -72,6 +70,7 @@ import extendOptions, { } from './extendInstallOptions' import linkPackages, { DependenciesGraph, + DependenciesGraphNode, Importer as ImporterToLink, } from './link' import { absolutePathToRef } from './lockfile' @@ -847,78 +846,34 @@ async function installInContext ( ]) // postinstall hooks - if (!(opts.ignoreScripts || !result.newDepPaths || !result.newDepPaths.length)) { + if (!opts.ignoreScripts && result.newDepPaths && result.newDepPaths.length) { const depPaths = Object.keys(result.depGraph) const rootNodes = depPaths.filter((depPath) => result.depGraph[depPath].depth === 0) - const nodesToBuild = new Set() - getSubgraphToBuild(result.depGraph, rootNodes, nodesToBuild, new Set()) - const onlyFromBuildGraph = R.filter((depPath: string) => nodesToBuild.has(depPath)) - const nodesToBuildArray = Array.from(nodesToBuild) - const graph = new Map( - nodesToBuildArray - .map((depPath) => [depPath, onlyFromBuildGraph(R.values(result.depGraph[depPath].children))]) as Array<[string, string[]]>, - ) - const graphSequencerResult = graphSequencer({ - graph, - groups: [nodesToBuildArray], + await buildModules(result.depGraph, rootNodes, { + childConcurrency: opts.childConcurrency, + depsToBuild: new Set(result.newDepPaths), + optional: opts.include.optionalDependencies, + prefix: ctx.lockfileDirectory, + rawNpmConfig: opts.rawNpmConfig, + rootNodeModulesDir: ctx.virtualStoreDir, + sideEffectsCacheWrite: opts.sideEffectsCacheWrite, + storeController: opts.storeController, + unsafePerm: opts.unsafePerm, + userAgent: opts.userAgent, }) - const chunks = graphSequencerResult.chunks as string[][] - const groups = chunks.map((chunk) => chunk - .filter((depPath) => result.depGraph[depPath].requiresBuild && !result.depGraph[depPath].isBuilt && result.newDepPaths.indexOf(depPath) !== -1) - .map((depPath) => result.depGraph[depPath]) - .map((pkg) => async () => { - try { - const hasSideEffects = await runPostinstallHooks({ - depPath: pkg.absolutePath, - optional: pkg.optional, - pkgRoot: pkg.peripheralLocation, - prepare: pkg.prepare, - rawNpmConfig: opts.rawNpmConfig, - rootNodeModulesDir: ctx.virtualStoreDir, - unsafePerm: opts.unsafePerm || false, - }) - if (hasSideEffects && opts.sideEffectsCacheWrite) { - try { - await opts.storeController.upload(pkg.peripheralLocation, { - engine: ENGINE_NAME, - packageId: pkg.id, - }) - } catch (err) { - if (err && err.statusCode === 403) { - logger.warn({ - message: `The store server disabled upload requests, could not upload ${pkg.id}`, - prefix: ctx.lockfileDirectory, - }) - } else { - logger.warn({ - error: err, - message: `An error occurred while uploading ${pkg.id}`, - prefix: ctx.lockfileDirectory, - }) - } - } - } - } catch (err) { - if (resolvedPackagesByPackageId[pkg.id].optional) { - // TODO: add parents field to the log - skippedOptionalDependencyLogger.debug({ - details: err.toString(), - package: { - id: pkg.id, - name: pkg.name, - version: pkg.version, - }, - prefix: opts.lockfileDirectory, - reason: 'build_failure', - }) - return - } - throw err - } - }), - ) - await runGroups(opts.childConcurrency, groups) + } + + if (result.newDepPaths && result.newDepPaths.length) { + const newPkgs = R.props(result.newDepPaths, result.depGraph) + await linkAllBins(newPkgs, result.depGraph, { + optional: opts.include.optionalDependencies, + warn: (message: string) => logger.warn({ message, prefix: opts.lockfileDirectory }), + }) + } + + if (!opts.lockfileOnly) { + await Promise.all(importersToLink.map(linkBinsOfImporter)) } } @@ -939,6 +894,26 @@ async function installInContext ( await opts.storeController.close() } +const limitLinking = pLimit(16) + +function linkBinsOfImporter ({ modulesDir, bin, prefix }: ImporterToLink) { + const warn = (message: string) => logger.warn({ message, prefix }) + return linkBins(modulesDir, bin, { warn }) +} + +async function linkAllBins ( + depNodes: DependenciesGraphNode[], + depGraph: DependenciesGraph, + opts: { + optional: boolean, + warn: (message: string) => void, + }, +) { + return Promise.all( + depNodes.map((depNode => limitLinking(async () => linkBinsOfDependencies(depNode, depGraph, opts)))), + ) +} + function modulesIsUpToDate ( opts: { defaultRegistry: string, @@ -955,29 +930,6 @@ function modulesIsUpToDate ( return R.equals(R.keys(opts.wantedLockfile.packages), currentWithSkipped) } -function getSubgraphToBuild ( - graph: DependenciesGraph, - entryNodes: string[], - nodesToBuild: Set, - walked: Set, -) { - let currentShouldBeBuilt = false - for (const depPath of entryNodes) { - if (nodesToBuild.has(depPath)) { - currentShouldBeBuilt = true - } - if (walked.has(depPath)) continue - walked.add(depPath) - const childShouldBeBuilt = getSubgraphToBuild(graph, R.values(graph[depPath].children), nodesToBuild, walked) - || graph[depPath].requiresBuild - if (childShouldBeBuilt) { - nodesToBuild.add(depPath) - currentShouldBeBuilt = true - } - } - return currentShouldBeBuilt -} - function addDirectDependenciesToLockfile ( newPkg: PackageJson, lockfileImporter: LockfileImporter, diff --git a/packages/supi/src/install/link.ts b/packages/supi/src/install/link.ts index 7133a2d33a..d212e76678 100644 --- a/packages/supi/src/install/link.ts +++ b/packages/supi/src/install/link.ts @@ -10,12 +10,10 @@ import { import filterLockfile, { filterLockfileByImporters, } from '@pnpm/filter-lockfile' -import linkBins, { linkBinsOfPackages } from '@pnpm/link-bins' import { Lockfile } from '@pnpm/lockfile-file' import logger from '@pnpm/logger' import { prune } from '@pnpm/modules-cleaner' import { IncludedDependencies } from '@pnpm/modules-yaml' -import { fromDir as readPackageFromDir } from '@pnpm/read-package-json' import { DependenciesTree, LinkedDependency } from '@pnpm/resolve-dependencies' import shamefullyFlattenGraph from '@pnpm/shamefully-flatten' import { StoreController } from '@pnpm/store-controller-types' @@ -23,7 +21,6 @@ import symlinkDependency, { symlinkDirectRootDependency } from '@pnpm/symlink-de import { PackageJson, Registries } from '@pnpm/types' import * as dp from 'dependency-path' import fs = require('mz/fs') -import pFilter = require('p-filter') import pLimit = require('p-limit') import path = require('path') import R = require('ramda') @@ -36,7 +33,10 @@ import updateLockfile from './updateLockfile' const brokenNodeModulesLogger = logger('_broken_node_modules') -export { DependenciesGraph } +export { + DependenciesGraph, + DependenciesGraphNode, +} export interface Importer { bin: string, @@ -134,7 +134,7 @@ export default async function linkPackages ( opts.skipped.delete(relDepPath) return true } - if (opts.wantedToBeSkippedPackageIds.has(depNode.id)) { + if (opts.wantedToBeSkippedPackageIds.has(depNode.packageId)) { opts.skipped.add(relDepPath) return false } @@ -225,8 +225,8 @@ export default async function linkPackages ( rootLogger.debug({ added: { dependencyType: isDev && 'dev' || isOptional && 'optional' || 'prod', - id: depGraphNode.id, - latest: opts.outdatedDependencies[depGraphNode.id], + id: depGraphNode.packageId, + latest: opts.outdatedDependencies[depGraphNode.packageId], name: rootAlias, realName: depGraphNode.name, version: depGraphNode.version, @@ -341,8 +341,6 @@ export default async function linkPackages ( })), ), ) - - await Promise.all(importers.map(linkBinsOfImporter)) } return { @@ -354,11 +352,6 @@ export default async function linkPackages ( } } -function linkBinsOfImporter ({ modulesDir, bin, prefix }: Importer) { - const warn = (message: string) => logger.warn({ message, prefix }) - return linkBins(modulesDir, bin, { warn }) -} - const isAbsolutePath = /^[/]|^[A-Za-z]:/ // This function is copied from @pnpm/local-resolver @@ -440,11 +433,6 @@ async function linkNewPackages ( linkAllPkgs(opts.storeController, newPkgs, opts), ]) - await linkAllBins(newPkgs, depGraph, { - optional: opts.optional, - warn: (message: string) => logger.warn({ message, prefix: opts.lockfileDirectory }), - }) - return newDepPaths } @@ -501,50 +489,6 @@ async function linkAllPkgs ( ) } -async function linkAllBins ( - depNodes: DependenciesGraphNode[], - depGraph: DependenciesGraph, - opts: { - optional: boolean, - warn: (message: string) => void, - }, -) { - return Promise.all( - depNodes.map((depNode) => limitLinking(async () => { - const childrenToLink = opts.optional - ? depNode.children - : R.keys(depNode.children) - .reduce((nonOptionalChildren, childAlias) => { - if (!depNode.optionalDependencies.has(childAlias)) { - nonOptionalChildren[childAlias] = depNode.children[childAlias] - } - return nonOptionalChildren - }, {}) - - const pkgs = await Promise.all( - R.keys(childrenToLink) - .filter((alias) => depGraph[childrenToLink[alias]].hasBin && depGraph[childrenToLink[alias]].installable) - .map(async (alias) => { - const dep = depGraph[childrenToLink[alias]] - return { - location: dep.peripheralLocation, - manifest: (await dep.fetchingRawManifest) || await readPackageFromDir(dep.peripheralLocation), - } - }), - ) - - const binPath = path.join(depNode.peripheralLocation, 'node_modules', '.bin') - await linkBinsOfPackages(pkgs, binPath, { warn: opts.warn }) - - // link also the bundled dependencies` bins - if (depNode.hasBundledDependencies) { - const bundledModules = path.join(depNode.peripheralLocation, 'node_modules') - await linkBins(bundledModules, binPath, { warn: opts.warn }) - } - })), - ) -} - async function linkAllModules ( depNodes: DependenciesGraphNode[], depGraph: DependenciesGraph, diff --git a/packages/supi/src/install/resolvePeers.ts b/packages/supi/src/install/resolvePeers.ts index dd69fa46cb..133c5ea399 100644 --- a/packages/supi/src/install/resolvePeers.ts +++ b/packages/supi/src/install/resolvePeers.ts @@ -42,7 +42,7 @@ export interface DependenciesGraphNode { prod: boolean, dev: boolean, optional: boolean, - id: string, + packageId: string, installable: boolean, additionalInfo: { deprecated?: string, @@ -243,7 +243,6 @@ function resolvePeersOfNode ( fetchingRawManifest: node.resolvedPackage.fetchingRawManifest, hasBin: node.resolvedPackage.hasBin, hasBundledDependencies: node.resolvedPackage.hasBundledDependencies, - id: node.resolvedPackage.id, independent, installable: node.installable, isBuilt: !!node.resolvedPackage.engineCache, @@ -252,6 +251,7 @@ function resolvePeersOfNode ( name: node.resolvedPackage.name, optional: node.resolvedPackage.optional, optionalDependencies: node.resolvedPackage.optionalDependencies, + packageId: node.resolvedPackage.id, peripheralLocation, prepare: node.resolvedPackage.prepare, prod: node.resolvedPackage.prod, diff --git a/packages/supi/src/install/updateLockfile.ts b/packages/supi/src/install/updateLockfile.ts index 131d063218..4efd7053f7 100644 --- a/packages/supi/src/install/updateLockfile.ts +++ b/packages/supi/src/install/updateLockfile.ts @@ -126,8 +126,8 @@ function toLockfileDependency ( if (depNode.optional) { result['optional'] = true } - if (opts.relDepPath[0] !== '/' && opts.depPath !== depNode.id) { - result['id'] = depNode.id + if (opts.relDepPath[0] !== '/' && opts.depPath !== depNode.packageId) { + result['id'] = depNode.packageId } if (pkg.peerDependencies) { result['peerDependencies'] = pkg.peerDependencies diff --git a/packages/supi/src/rebuild/index.ts b/packages/supi/src/rebuild/index.ts index ae7c123f29..673d1d9599 100644 --- a/packages/supi/src/rebuild/index.ts +++ b/packages/supi/src/rebuild/index.ts @@ -8,6 +8,7 @@ import { runLifecycleHooksConcurrently, runPostinstallHooks, } from '@pnpm/lifecycle' +import linkBins from '@pnpm/link-bins' import { Lockfile, nameVerFromPkgSnapshot, @@ -20,6 +21,7 @@ import pkgIdToFilename from '@pnpm/pkgid-to-filename' import npa = require('@zkochan/npm-package-arg') import * as dp from 'dependency-path' import graphSequencer = require('graph-sequencer') +import pLimit = require('p-limit') import path = require('path') import R = require('ramda') import runGroups from 'run-groups' @@ -113,7 +115,7 @@ export async function rebuildPkgs ( new Set(pkgs), ctx.virtualStoreDir, ctx.currentLockfile, - ctx.importers.map((importer) => importer.id), + ctx.importers, opts, ) } @@ -135,16 +137,13 @@ export async function rebuild ( idsToRebuild = ctx.pendingBuilds } else if (ctx.currentLockfile && ctx.currentLockfile.packages) { idsToRebuild = R.keys(ctx.currentLockfile.packages) - } else { - return } - if (idsToRebuild.length === 0) return const pkgsThatWereRebuilt = await _rebuild( new Set(idsToRebuild), ctx.virtualStoreDir, ctx.currentLockfile, - ctx.importers.map((importer) => importer.id), + ctx.importers, opts, ) @@ -232,11 +231,13 @@ function getSubgraphToBuild ( return currentShouldBeBuilt } +const limitLinking = pLimit(16) + async function _rebuild ( pkgsToRebuild: Set, - modules: string, + rootNodeModulesDir: string, lockfile: Lockfile, - importerIds: string[], + importers: Array<{ id: string, prefix: string }>, opts: StrictRebuildOptions, ) { const pkgsThatWereRebuilt = new Set() @@ -245,8 +246,8 @@ async function _rebuild ( const entryNodes = [] as string[] - importerIds.forEach((importerId) => { - const lockfileImporter = lockfile.importers[importerId] + importers.forEach((importer) => { + const lockfileImporter = lockfile.importers[importer.id] R.toPairs({ ...(opts.development && lockfileImporter.devDependencies || {}), ...(opts.production && lockfileImporter.dependencies || {}), @@ -274,7 +275,7 @@ async function _rebuild ( groups: [nodesToBuildAndTransitiveArray], }) const chunks = graphSequencerResult.chunks as string[][] - + const warn = (message: string) => logger.warn({ message, prefix: opts.prefix }) const groups = chunks.map((chunk) => chunk.filter((relDepPath) => pkgsToRebuild.has(relDepPath)).map((relDepPath) => async () => { const pkgSnapshot = pkgSnapshots[relDepPath] @@ -282,7 +283,7 @@ async function _rebuild ( const pkgInfo = nameVerFromPkgSnapshot(relDepPath, pkgSnapshot) const independent = opts.independentLeaves && packageIsIndependent(pkgSnapshot) const pkgRoot = !independent - ? path.join(modules, `.${pkgIdToFilename(depPath, opts.lockfileDirectory)}`, 'node_modules', pkgInfo.name) + ? path.join(rootNodeModulesDir, `.${pkgIdToFilename(depPath, opts.lockfileDirectory)}`, 'node_modules', pkgInfo.name) : await ( async () => { const { directory } = await opts.storeController.getPackageLocation(pkgSnapshot.id || depPath, pkgInfo.name, { @@ -293,13 +294,18 @@ async function _rebuild ( } )() try { + if (!independent) { + const modules = path.join(rootNodeModulesDir, `.${pkgIdToFilename(depPath, opts.lockfileDirectory)}`, 'node_modules') + const binPath = path.join(pkgRoot, 'node_modules', '.bin') + await linkBins(modules, binPath, { warn }) + } await runPostinstallHooks({ depPath, optional: pkgSnapshot.optional === true, pkgRoot, prepare: pkgSnapshot.prepare, rawNpmConfig: opts.rawNpmConfig, - rootNodeModulesDir: modules, + rootNodeModulesDir, unsafePerm: opts.unsafePerm || false, }) pkgsThatWereRebuilt.add(relDepPath) @@ -325,5 +331,25 @@ async function _rebuild ( await runGroups(opts.childConcurrency || 5, groups) + // It may be optimized because some bins were already linked before running lifecycle scripts + await Promise.all( + R + .keys(pkgSnapshots) + .filter((relDepPath) => !packageIsIndependent(pkgSnapshots[relDepPath])) + .map((relDepPath) => limitLinking(() => { + const depPath = dp.resolve(opts.registries, relDepPath) + const pkgSnapshot = pkgSnapshots[relDepPath] + const pkgInfo = nameVerFromPkgSnapshot(relDepPath, pkgSnapshot) + const modules = path.join(rootNodeModulesDir, `.${pkgIdToFilename(depPath, opts.lockfileDirectory)}`, 'node_modules') + const binPath = path.join(modules, pkgInfo.name, 'node_modules', '.bin') + return linkBins(modules, binPath, { warn }) + })), + ) + await Promise.all(importers.map((importer) => limitLinking(() => { + const modules = path.join(importer.prefix, 'node_modules') + const binPath = path.join(modules, '.bin') + return linkBins(modules, binPath, { warn }) + }))) + return pkgsThatWereRebuilt } diff --git a/packages/supi/test/install/lifecycleScripts.ts b/packages/supi/test/install/lifecycleScripts.ts index 60a634e65b..afa4e9bbaf 100644 --- a/packages/supi/test/install/lifecycleScripts.ts +++ b/packages/supi/test/install/lifecycleScripts.ts @@ -10,6 +10,7 @@ import sinon = require('sinon') import { addDependenciesToPackage, install, + mutateModules, } from 'supi' import tape = require('tape') import promisifyTape from 'tape-promise' @@ -18,6 +19,7 @@ import { testDefaults } from '../utils' const pkgRoot = path.join(__dirname, '..', '..') const test = promisifyTape(tape) +const testOnly = promisifyTape(tape.only) test('run pre/postinstall scripts', async (t: tape.Test) => { const project = prepare(t) @@ -276,3 +278,68 @@ test('run prepare script for git-hosted dependencies', async (t: tape.Test) => { const lockfile = await project.loadLockfile() t.ok(lockfile.packages['github.com/zkochan/install-scripts-example/2de638b8b572cd1e87b74f4540754145fb2c0ebb'].prepare === true, `prepare field added to ${WANTED_LOCKFILE}`) }) + +test('lifecycle scripts run before linking bins', async (t: tape.Test) => { + const project = prepare(t) + + await addDependenciesToPackage(['generated-bins'], await testDefaults()) + + await project.isExecutable('.bin/cmd1') + await project.isExecutable('.bin/cmd2') + + await rimraf('node_modules') + + await mutateModules( + [ + { + buildIndex: 0, + mutation: 'install', + prefix: process.cwd(), + } + ], + await testDefaults({ frozenLockfile: true }), + ) + + await project.isExecutable('.bin/cmd1') + await project.isExecutable('.bin/cmd2') +}) + +test('bins are linked even if lifecycle scripts are ignored', async (t: tape.Test) => { + const project = prepare(t) + + await addDependenciesToPackage( + [ + 'pkg-with-peer-having-bin', + 'peer-with-bin', + 'pre-and-postinstall-scripts-example', + ], + await testDefaults({ ignoreScripts: true }), + ) + + await project.isExecutable('.bin/peer-with-bin') + await project.isExecutable('pkg-with-peer-having-bin/node_modules/.bin/hello-world-js-bin') + + // Verifying that the scripts were ignored + t.ok(await exists('node_modules/pre-and-postinstall-scripts-example/package.json')) + t.notOk(await exists('node_modules/pre-and-postinstall-scripts-example/generated-by-preinstall.js'), 'scripts were ignored indeed') + + await rimraf('node_modules') + + await mutateModules( + [ + { + buildIndex: 0, + mutation: 'install', + prefix: process.cwd(), + } + ], + await testDefaults({ frozenLockfile: true, ignoreScripts: true }), + ) + + await project.isExecutable('.bin/peer-with-bin') + await project.isExecutable('pkg-with-peer-having-bin/node_modules/.bin/hello-world-js-bin') + + // Verifying that the scripts were ignored + t.ok(await exists('node_modules/pre-and-postinstall-scripts-example/package.json')) + t.notOk(await exists('node_modules/pre-and-postinstall-scripts-example/generated-by-preinstall.js'), 'scripts were ignored indeed') +}) diff --git a/packages/supi/test/rebuild.ts b/packages/supi/test/rebuild.ts index a19cbd04d9..a6099d6698 100644 --- a/packages/supi/test/rebuild.ts +++ b/packages/supi/test/rebuild.ts @@ -247,3 +247,23 @@ test('rebuild multiple packages in correct order', async (t: tape.Test) => { t.deepEqual(outputs1, ['project-1', 'project-2']) t.deepEqual(outputs2, ['project-1', 'project-3']) }) + +test('rebuild links bins', async (t: tape.Test) => { + const project = prepare(t) + + await addDependenciesToPackage(['has-generated-bins-as-dep', 'generated-bins'], await testDefaults({ ignoreScripts: true })) + + t.notOk(await exists(path.resolve('node_modules/.bin/cmd1'))) + t.notOk(await exists(path.resolve('node_modules/.bin/cmd2'))) + + t.ok(await exists(path.resolve('node_modules/has-generated-bins-as-dep/package.json'))) + t.notOk(await exists(path.resolve('node_modules/has-generated-bins-as-dep/node_modules/.bin/cmd1'))) + t.notOk(await exists(path.resolve('node_modules/has-generated-bins-as-dep/node_modules/.bin/cmd2'))) + + await rebuild([{ buildIndex: 0, prefix: process.cwd() }], await testDefaults({ rawNpmConfig: { pending: true } })) + + await project.isExecutable('.bin/cmd1') + await project.isExecutable('.bin/cmd2') + await project.isExecutable('has-generated-bins-as-dep/node_modules/.bin/cmd1') + await project.isExecutable('has-generated-bins-as-dep/node_modules/.bin/cmd2') +}) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9cb4aea05e..c42c7a817f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1,4 +1,50 @@ importers: + packages/build-modules: + dependencies: + '@pnpm/constants': 'link:../constants' + '@pnpm/core-loggers': 'link:../core-loggers' + '@pnpm/lifecycle': 'link:../lifecycle' + '@pnpm/link-bins': 4.0.2 + '@pnpm/read-package-json': 2.0.1 + '@pnpm/store-controller-types': 'link:../store-controller-types' + '@pnpm/types': 'link:../types' + '@types/node': 11.13.0 + '@types/ramda': 0.26.6 + graph-sequencer: 2.0.0 + ramda: 0.26.1 + run-groups: 2.0.0 + devDependencies: + '@pnpm/build-modules': 'link:' + '@pnpm/logger': 2.1.0 + '@pnpm/tslint-config': 'link:../../utils/tslint-config' + mos: 2.0.0-alpha.3 + mos-plugin-readme: 1.0.4 + rimraf: 2.6.3 + ts-node: 8.0.3_typescript@3.4.1 + tslint: 5.15.0_typescript@3.4.1 + typescript: 3.4.1 + specifiers: + '@pnpm/build-modules': 'link:' + '@pnpm/constants': 1.0.0 + '@pnpm/core-loggers': 3.0.0 + '@pnpm/lifecycle': 5.0.0 + '@pnpm/link-bins': 4.0.2 + '@pnpm/logger': 2.1.0 + '@pnpm/read-package-json': 2.0.1 + '@pnpm/store-controller-types': 3.0.0 + '@pnpm/tslint-config': 0.0.0 + '@pnpm/types': 3.0.0 + '@types/node': '*' + '@types/ramda': 0.26.6 + graph-sequencer: 2.0.0 + mos: 2.0.0-alpha.3 + mos-plugin-readme: 1.0.4 + ramda: 0.26.1 + rimraf: 2.6.3 + run-groups: 2.0.0 + ts-node: 8.0.3 + tslint: 5.15.0 + typescript: 3.4.1 packages/config: dependencies: '@pnpm/types': 'link:../types' @@ -495,11 +541,12 @@ importers: typescript: 3.4.1 packages/headless: dependencies: + '@pnpm/build-modules': 'link:../build-modules' '@pnpm/constants': 'link:../constants' '@pnpm/core-loggers': 'link:../core-loggers' '@pnpm/filter-lockfile': 'link:../filter-lockfile' '@pnpm/lifecycle': 'link:../lifecycle' - '@pnpm/link-bins': 4.0.1 + '@pnpm/link-bins': 4.0.2 '@pnpm/lockfile-file': 'link:../lockfile-file' '@pnpm/lockfile-utils': 'link:../lockfile-utils' '@pnpm/modules-cleaner': 'link:../modules-cleaner' @@ -514,11 +561,9 @@ importers: '@pnpm/utils': 'link:../utils' '@types/ramda': 0.25.34 dependency-path: 'link:../dependency-path' - graph-sequencer: 2.0.0 p-limit: 2.2.0 path-exists: 4.0.0 ramda: 0.26.1 - run-groups: 2.0.0 devDependencies: '@pnpm/assert-project': 'link:../../privatePackages/assert-project' '@pnpm/default-fetcher': 'link:../default-fetcher' @@ -542,7 +587,7 @@ importers: isexe: 2.0.0 mz: 2.7.0 npm-run-all: 4.1.5 - pnpm-registry-mock: 2.9.0 + pnpm-registry-mock: 2.11.0 rimraf: 2.6.3 rimraf-then: 1.0.1 sinon: 7.3.1 @@ -554,6 +599,7 @@ importers: typescript: 3.4.1 specifiers: '@pnpm/assert-project': 'link:../../privatePackages/assert-project' + '@pnpm/build-modules': 0.0.0 '@pnpm/constants': 1.0.0 '@pnpm/core-loggers': 3.0.0 '@pnpm/default-fetcher': 'file:../default-fetcher' @@ -561,7 +607,7 @@ importers: '@pnpm/filter-lockfile': 1.0.1 '@pnpm/headless': 'link:' '@pnpm/lifecycle': 5.0.0 - '@pnpm/link-bins': 4.0.1 + '@pnpm/link-bins': 4.0.2 '@pnpm/lockfile-file': 1.0.1 '@pnpm/lockfile-utils': 1.0.1 '@pnpm/logger': 2.1.0 @@ -590,18 +636,16 @@ importers: '@types/tempy': 0.2.0 dependency-path: 3.0.1 fs-extra: 7.0.1 - graph-sequencer: 2.0.0 is-windows: 1.0.2 isexe: 2.0.0 mz: 2.7.0 npm-run-all: 4.1.5 p-limit: 2.2.0 path-exists: 4.0.0 - pnpm-registry-mock: 2.9.0 + pnpm-registry-mock: 2.11.0 ramda: 0.26.1 rimraf: 2.6.3 rimraf-then: 1.0.1 - run-groups: 2.0.0 sinon: 7.3.1 tape: 4.10.1 tape-promise: 4.0.0 @@ -1063,7 +1107,7 @@ importers: mos: 2.0.0-alpha.3 mos-plugin-readme: 1.0.4 npm-run-all: 4.1.5 - pnpm-registry-mock: 2.9.0 + pnpm-registry-mock: 2.11.0 tape: 4.10.1 ts-node: 8.0.3_typescript@3.4.1 tslint: 5.15.0_typescript@3.4.1 @@ -1085,7 +1129,7 @@ importers: mos: 2.0.0-alpha.3 mos-plugin-readme: 1.0.4 npm-run-all: 4.1.5 - pnpm-registry-mock: 2.9.0 + pnpm-registry-mock: 2.11.0 tape: 4.10.1 ts-node: 8.0.3 tslint: 5.15.0 @@ -1391,7 +1435,7 @@ importers: p-any: 1.1.0 path-exists: 4.0.0 pnpm: 'link:' - pnpm-registry-mock: 2.9.0 + pnpm-registry-mock: 2.11.0 retry: 0.12.0 rimraf: 2.6.3 rimraf-then: 1.0.1 @@ -1491,7 +1535,7 @@ importers: pkgs-graph: 3.0.0 pnpm: 'link:' pnpm-file-reporter: 0.1.0 - pnpm-registry-mock: 2.9.0 + pnpm-registry-mock: 2.11.0 process-exists: 3.1.0 ramda: 0.26.1 read-ini-file: 2.0.0 @@ -1776,6 +1820,7 @@ importers: typescript: 3.4.1 packages/supi: dependencies: + '@pnpm/build-modules': 'link:../build-modules' '@pnpm/check-package': 3.0.0 '@pnpm/constants': 'link:../constants' '@pnpm/core-loggers': 'link:../core-loggers' @@ -1783,7 +1828,7 @@ importers: '@pnpm/fs-locker': 1.0.3 '@pnpm/headless': 'link:../headless' '@pnpm/lifecycle': 'link:../lifecycle' - '@pnpm/link-bins': 4.0.1 + '@pnpm/link-bins': 4.0.2 '@pnpm/lockfile-file': 'link:../lockfile-file' '@pnpm/lockfile-utils': 'link:../lockfile-utils' '@pnpm/modules-cleaner': 'link:../modules-cleaner' @@ -1870,7 +1915,7 @@ importers: npm-scripts-info: 0.3.9 path-exists: 4.0.0 path-name: 1.0.0 - pnpm-registry-mock: 2.9.0 + pnpm-registry-mock: 2.11.0 read-pkg: 4.0.1 read-yaml-file: 1.1.0 rimraf: 2.6.3 @@ -1887,6 +1932,7 @@ importers: specifiers: '@pnpm/assert-project': 'link:../../privatePackages/assert-project' '@pnpm/assert-store': 'link:../../privatePackages/assert-store' + '@pnpm/build-modules': 0.0.0 '@pnpm/check-package': 3.0.0 '@pnpm/constants': 1.0.0 '@pnpm/core-loggers': 3.0.0 @@ -1896,7 +1942,7 @@ importers: '@pnpm/fs-locker': 1.0.3 '@pnpm/headless': 6.0.4 '@pnpm/lifecycle': 5.0.0 - '@pnpm/link-bins': 4.0.1 + '@pnpm/link-bins': 4.0.2 '@pnpm/lockfile-file': 1.0.1 '@pnpm/lockfile-utils': 1.0.1 '@pnpm/logger': 2.1.0 @@ -1967,7 +2013,7 @@ importers: path-absolute: 1.0.1 path-exists: 4.0.0 path-name: 1.0.0 - pnpm-registry-mock: 2.9.0 + pnpm-registry-mock: 2.11.0 ramda: 0.26.1 read-pkg: 4.0.1 read-yaml-file: 1.1.0 @@ -2503,7 +2549,7 @@ packages: node: '>=4' resolution: integrity: sha512-ECFtJCNpqeyjSGwhv+IgnqDRLQclI/d6GRkq+UxyPW3jSvB7AK8qjxuMNFoNG78pwwi1zuJpMLgMvaYPHI/IMA== - /@pnpm/link-bins/4.0.1: + /@pnpm/link-bins/4.0.2: dependencies: '@pnpm/package-bins': 3.0.1 '@pnpm/read-package-json': 2.0.1 @@ -2513,6 +2559,7 @@ packages: '@types/ramda': 0.26.6 '@zkochan/cmd-shim': 3.1.0 arr-flatten: 1.1.0 + is-subdir: 1.0.3 is-windows: 1.0.2 mkdirp-promise: 5.0.1 mz: 2.7.0 @@ -2523,7 +2570,7 @@ packages: engines: node: '>=8' resolution: - integrity: sha512-7rtKxnqYi5TtO4bDdq3IjTNQPLfVzKN5JsWVcCfoUk8T2e7HFu5ULSspMsWcqLFLcMZM3cN3YYmnaB0gf/weHg== + integrity: sha512-69ZxoeiSRxtOOmeBlYXEOiNZPY0fTinePQhXmLcNE5g8+ZUGaSD6KKDWJ6fW2rSrc21m4hcuadpdoqVlyfA6/w== /@pnpm/logger/2.1.0: dependencies: '@types/node': 10.14.4 @@ -8031,7 +8078,7 @@ packages: node: '>=4' resolution: integrity: sha512-ZvCSe4hTJHWfAXZUgydkbRre3h1svqtlb3Wq70fkbdZF4Wbvr0LEQRG5kCsiEnkt27PpdzlZ2b3XmV2dvUpHwg== - /pnpm-registry-mock/2.9.0: + /pnpm-registry-mock/2.11.0: dependencies: anonymous-npm-registry-client: 0.1.2 cpr: 3.0.1 @@ -8042,7 +8089,7 @@ packages: node: '>=6' hasBin: true resolution: - integrity: sha512-n/1xE+5290VcmBh44/zpg2JoZU0e4BkoApwJAUb8a22FuFYg3EjyhfeuQJtyfZCmdPgmhLekxNlRA4rgNA0Qng== + integrity: sha512-c1cOVtuDutugSgWp5Et02x+JBJXAKGmPF59ADuYocLbZETUJ/s9ZtHarfAN+eEq5BZEXgA0o8u1hPYxVTk286w== /posix-character-classes/0.1.1: dev: false engines: