feat: incremental rebuild (#30)

* fix: run node-gyp rebuild when install is not specified

* feat: after pnpm install --ignore-script, pnpm rebuild is incremental

* refactor: add --pending option to rebuild, only store pending ids

* fix: javascript magic to avoid if statement

* fix: update pendingBuilds also when removing packages

* fix: remove old code, use testDefaults correctly

* test: add test for rebuild --pending

* chore: make my IDE and TSLint happy

* chore: @types/es6-promise makes IntelliJ happy and Travis sad

* fix: use Set, only append to pendingBuilds if ignoreScripts is true

* test: pendingBuilds handled correctly

* test: install and uninstall behavior with pendingBuilds

* fix: saveModules only when needed

* fix: Set has size, not length, add comment about the use of .concat

src/api/extendOptions.ts

@@ -95,5 +95,6 @@ export default async (
     extendedOpts.prefix = path.join(extendedOpts.prefix, subfolder)
   }
   extendedOpts.rawNpmConfig['registry'] = extendedOpts.registry
+  extendedOpts.pending = extendedOpts.rawNpmConfig['pending']
   return extendedOpts
 }

src/api/getContext.ts

@@ -35,6 +35,7 @@ export type PnpmContext = {
   currentShrinkwrap: Shrinkwrap,
   wantedShrinkwrap: Shrinkwrap,
   skipped: Set<string>,
+  pendingBuilds: string[],
 }
 
 export default async function getContext (opts: StrictSupiOptions, installType?: 'named' | 'general'): Promise<PnpmContext> {
@@ -97,6 +98,7 @@ export default async function getContext (opts: StrictSupiOptions, installType?:
     existsCurrentShrinkwrap: !!files[2],
     storeController: files[3],
     skipped: new Set(modules && modules.skipped || []),
+    pendingBuilds: modules && modules.pendingBuilds || [],
   }
   packageJsonLogger.debug({ initial: ctx.pkg })
 

src/api/install.ts

@@ -4,6 +4,7 @@ import {
   PnpmOptions,
   StrictPnpmOptions,
 } from '@pnpm/types'
+import * as dp from 'dependency-path'
 import path = require('path')
 import logger, {
   streamParser,
@@ -521,9 +522,18 @@ async function installInContext (
     outdatedPkgs: installCtx.outdatedPkgs,
   })
 
+  ctx.pendingBuilds = ctx.pendingBuilds
+    .filter(pkgId => !result.removedPkgIds.has(dp.resolve(ctx.wantedShrinkwrap.registry, pkgId)))
+
+  if (opts.ignoreScripts) {
+    // we can use concat here because we always only append new packages, which are guaranteed to not be there by definition
+    ctx.pendingBuilds = ctx.pendingBuilds
+      .concat(result.newPkgResolvedIds.map(absolutePath => dp.relative(ctx.wantedShrinkwrap.registry, absolutePath)))
+  }
+
   await Promise.all([
     saveShrinkwrap(ctx.root, result.wantedShrinkwrap, result.currentShrinkwrap),
-    result.currentShrinkwrap.packages === undefined
+    result.currentShrinkwrap.packages === undefined && result.removedPkgIds.size === 0
       ? Promise.resolve()
       : saveModules(path.join(ctx.root, 'node_modules'), {
         packageManager: `${opts.packageManager.name}@${opts.packageManager.version}`,
@@ -531,13 +541,13 @@ async function installInContext (
         skipped: Array.from(installCtx.skipped),
         layoutVersion: LAYOUT_VERSION,
         independentLeaves: opts.independentLeaves,
+        pendingBuilds: ctx.pendingBuilds,
       }),
   ])
 
   // postinstall hooks
   if (!(opts.ignoreScripts || !result.newPkgResolvedIds || !result.newPkgResolvedIds.length)) {
     const limitChild = pLimit(opts.childConcurrency)
-    const linkedPkgsMapValues = R.values(result.linkedPkgsMap)
     await Promise.all(
       R.props<string, DependencyTreeNode>(result.newPkgResolvedIds, result.linkedPkgsMap)
         .map(pkg => limitChild(async () => {

src/api/rebuild.ts

@@ -14,6 +14,7 @@ import {
 import npa = require('@zkochan/npm-package-arg')
 import semver = require('semver')
 import getPkgInfoFromShr from '../getPkgInfoFromShr'
+import {save as saveModules, LAYOUT_VERSION} from '../fs/modulesController';
 
 type PackageToRebuild = {
   relativeDepPath: string,
@@ -22,8 +23,8 @@ type PackageToRebuild = {
   pkgShr: DependencyShrinkwrap
 }
 
-function getPackagesInfo (packages: ResolvedPackages): PackageToRebuild[] {
-  return R.keys(packages)
+function getPackagesInfo (packages: ResolvedPackages, idsToRebuild: string[]): PackageToRebuild[] {
+  return idsToRebuild
     .map(relativeDepPath => {
       const pkgShr = packages[relativeDepPath]
       const pkgInfo = getPkgInfoFromShr(relativeDepPath, pkgShr)
@@ -76,7 +77,7 @@ export async function rebuildPkgs (pkgSpecs: string[], maybeOpts: PnpmOptions) {
     }
   })
 
-  const pkgs = getPackagesInfo(packages)
+  const pkgs = getPackagesInfo(packages, R.keys(packages))
     .filter(pkg => matches(searched, pkg))
 
   await _rebuild(pkgs, modules, ctx.currentShrinkwrap.registry, opts)
@@ -106,12 +107,28 @@ export async function rebuild (maybeOpts: PnpmOptions) {
   await ctx.storeController.close() // TODO: storeController should not be created at all in this case
   const modules = path.join(opts.prefix, 'node_modules')
 
-  if (!ctx.currentShrinkwrap || !ctx.currentShrinkwrap.packages) return
-  const packages = ctx.currentShrinkwrap.packages
+  let idsToRebuild: string[] = []
 
-  const pkgs = getPackagesInfo(packages)
+  if (opts.pending) {
+    idsToRebuild = ctx.pendingBuilds
+  } else if (ctx.currentShrinkwrap && ctx.currentShrinkwrap.packages) {
+    idsToRebuild = R.keys(ctx.currentShrinkwrap.packages)
+  } else {
+    return
+  }
+
+  const pkgs = getPackagesInfo(ctx.currentShrinkwrap.packages || {}, idsToRebuild)
 
   await _rebuild(pkgs, modules, ctx.currentShrinkwrap.registry, opts)
+
+  await saveModules(path.join(ctx.root, 'node_modules'), {
+    packageManager: `${opts.packageManager.name}@${opts.packageManager.version}`,
+    store: ctx.storePath,
+    skipped: Array.from(ctx.skipped),
+    layoutVersion: LAYOUT_VERSION,
+    independentLeaves: opts.independentLeaves,
+    pendingBuilds: [],
+  })
 }
 
 async function _rebuild (

src/api/removeOrphanPkgs.ts

@@ -18,7 +18,7 @@ export default async function removeOrphanPkgs (
     storeController: StoreController,
     pruneStore?: boolean,
   }
-): Promise<string[]> {
+): Promise<Set<string>> {
   const oldPkgs = R.toPairs(R.mergeAll(R.map(depType => opts.oldShrinkwrap[depType], dependenciesTypes)))
   const newPkgs = R.toPairs(R.mergeAll(R.map(depType => opts.newShrinkwrap[depType], dependenciesTypes)))
 
@@ -58,7 +58,7 @@ export default async function removeOrphanPkgs (
 
   await opts.storeController.saveState()
 
-  return notDependents
+  return new Set(notDependents)
 }
 
 function getPackageIds (

src/api/uninstall.ts

@@ -1,5 +1,6 @@
 import rimraf = require('rimraf-then')
 import path = require('path')
+import * as dp from 'dependency-path'
 import getContext, {PnpmContext} from './getContext'
 import getSaveType from '../getSaveType'
 import removeDeps from '../removeDeps'
@@ -65,6 +66,7 @@ export async function uninstallInContext (pkgsToUninstall: string[], ctx: PnpmCo
     storeController: ctx.storeController,
     bin: opts.bin,
   })
+  ctx.pendingBuilds = ctx.pendingBuilds.filter(pkgId => !removedPkgIds.has(dp.resolve(newShr.registry, pkgId)))
   await ctx.storeController.close()
   const currentShrinkwrap = makePartialCurrentShrinkwrap
     ? pruneShrinkwrap(ctx.currentShrinkwrap, pkg)
@@ -73,9 +75,10 @@ export async function uninstallInContext (pkgsToUninstall: string[], ctx: PnpmCo
   await saveModules(path.join(ctx.root, 'node_modules'), {
     packageManager: `${opts.packageManager.name}@${opts.packageManager.version}`,
     store: ctx.storePath,
-    skipped: Array.from(ctx.skipped).filter(pkgId => removedPkgIds.indexOf(pkgId) === -1),
+    skipped: Array.from(ctx.skipped).filter(pkgId => !removedPkgIds.has(pkgId)),
     layoutVersion: LAYOUT_VERSION,
     independentLeaves: opts.independentLeaves,
+    pendingBuilds: ctx.pendingBuilds,
   })
   await removeOuterLinks(pkgsToUninstall, path.join(ctx.root, 'node_modules'), {
     storePath: ctx.storePath,