feat: incremental rebuild (#30)

* fix: run node-gyp rebuild when install is not specified

* feat: after pnpm install --ignore-script, pnpm rebuild is incremental

* refactor: add --pending option to rebuild, only store pending ids

* fix: javascript magic to avoid if statement

* fix: update pendingBuilds also when removing packages

* fix: remove old code, use testDefaults correctly

* test: add test for rebuild --pending

* chore: make my IDE and TSLint happy

* chore: @types/es6-promise makes IntelliJ happy and Travis sad

* fix: use Set, only append to pendingBuilds if ignoreScripts is true

* test: pendingBuilds handled correctly

* test: install and uninstall behavior with pendingBuilds

* fix: saveModules only when needed

* fix: Set has size, not length, add comment about the use of .concat

.gitignore

@@ -17,3 +17,6 @@ lib
 
 # Visual Studio Code configs
 .vscode/
+
+# JetBrains IDEs
+.idea/

src/api/extendOptions.ts

@@ -95,5 +95,6 @@ export default async (
     extendedOpts.prefix = path.join(extendedOpts.prefix, subfolder)
   }
   extendedOpts.rawNpmConfig['registry'] = extendedOpts.registry
+  extendedOpts.pending = extendedOpts.rawNpmConfig['pending']
   return extendedOpts
 }

src/api/getContext.ts

@@ -35,6 +35,7 @@ export type PnpmContext = {
   currentShrinkwrap: Shrinkwrap,
   wantedShrinkwrap: Shrinkwrap,
   skipped: Set<string>,
+  pendingBuilds: string[],
 }
 
 export default async function getContext (opts: StrictSupiOptions, installType?: 'named' | 'general'): Promise<PnpmContext> {
@@ -97,6 +98,7 @@ export default async function getContext (opts: StrictSupiOptions, installType?:
     existsCurrentShrinkwrap: !!files[2],
     storeController: files[3],
     skipped: new Set(modules && modules.skipped || []),
+    pendingBuilds: modules && modules.pendingBuilds || [],
   }
   packageJsonLogger.debug({ initial: ctx.pkg })
 

src/api/install.ts

@@ -4,6 +4,7 @@ import {
   PnpmOptions,
   StrictPnpmOptions,
 } from '@pnpm/types'
+import * as dp from 'dependency-path'
 import path = require('path')
 import logger, {
   streamParser,
@@ -521,9 +522,18 @@ async function installInContext (
     outdatedPkgs: installCtx.outdatedPkgs,
   })
 
+  ctx.pendingBuilds = ctx.pendingBuilds
+    .filter(pkgId => !result.removedPkgIds.has(dp.resolve(ctx.wantedShrinkwrap.registry, pkgId)))
+
+  if (opts.ignoreScripts) {
+    // we can use concat here because we always only append new packages, which are guaranteed to not be there by definition
+    ctx.pendingBuilds = ctx.pendingBuilds
+      .concat(result.newPkgResolvedIds.map(absolutePath => dp.relative(ctx.wantedShrinkwrap.registry, absolutePath)))
+  }
+
   await Promise.all([
     saveShrinkwrap(ctx.root, result.wantedShrinkwrap, result.currentShrinkwrap),
-    result.currentShrinkwrap.packages === undefined
+    result.currentShrinkwrap.packages === undefined && result.removedPkgIds.size === 0
       ? Promise.resolve()
       : saveModules(path.join(ctx.root, 'node_modules'), {
         packageManager: `${opts.packageManager.name}@${opts.packageManager.version}`,
@@ -531,13 +541,13 @@ async function installInContext (
         skipped: Array.from(installCtx.skipped),
         layoutVersion: LAYOUT_VERSION,
         independentLeaves: opts.independentLeaves,
+        pendingBuilds: ctx.pendingBuilds,
       }),
   ])
 
   // postinstall hooks
   if (!(opts.ignoreScripts || !result.newPkgResolvedIds || !result.newPkgResolvedIds.length)) {
     const limitChild = pLimit(opts.childConcurrency)
-    const linkedPkgsMapValues = R.values(result.linkedPkgsMap)
     await Promise.all(
       R.props<string, DependencyTreeNode>(result.newPkgResolvedIds, result.linkedPkgsMap)
         .map(pkg => limitChild(async () => {

src/api/rebuild.ts

@@ -14,6 +14,7 @@ import {
 import npa = require('@zkochan/npm-package-arg')
 import semver = require('semver')
 import getPkgInfoFromShr from '../getPkgInfoFromShr'
+import {save as saveModules, LAYOUT_VERSION} from '../fs/modulesController';
 
 type PackageToRebuild = {
   relativeDepPath: string,
@@ -22,8 +23,8 @@ type PackageToRebuild = {
   pkgShr: DependencyShrinkwrap
 }
 
-function getPackagesInfo (packages: ResolvedPackages): PackageToRebuild[] {
-  return R.keys(packages)
+function getPackagesInfo (packages: ResolvedPackages, idsToRebuild: string[]): PackageToRebuild[] {
+  return idsToRebuild
     .map(relativeDepPath => {
       const pkgShr = packages[relativeDepPath]
       const pkgInfo = getPkgInfoFromShr(relativeDepPath, pkgShr)
@@ -76,7 +77,7 @@ export async function rebuildPkgs (pkgSpecs: string[], maybeOpts: PnpmOptions) {
     }
   })
 
-  const pkgs = getPackagesInfo(packages)
+  const pkgs = getPackagesInfo(packages, R.keys(packages))
     .filter(pkg => matches(searched, pkg))
 
   await _rebuild(pkgs, modules, ctx.currentShrinkwrap.registry, opts)
@@ -106,12 +107,28 @@ export async function rebuild (maybeOpts: PnpmOptions) {
   await ctx.storeController.close() // TODO: storeController should not be created at all in this case
   const modules = path.join(opts.prefix, 'node_modules')
 
-  if (!ctx.currentShrinkwrap || !ctx.currentShrinkwrap.packages) return
-  const packages = ctx.currentShrinkwrap.packages
+  let idsToRebuild: string[] = []
 
-  const pkgs = getPackagesInfo(packages)
+  if (opts.pending) {
+    idsToRebuild = ctx.pendingBuilds
+  } else if (ctx.currentShrinkwrap && ctx.currentShrinkwrap.packages) {
+    idsToRebuild = R.keys(ctx.currentShrinkwrap.packages)
+  } else {
+    return
+  }
+
+  const pkgs = getPackagesInfo(ctx.currentShrinkwrap.packages || {}, idsToRebuild)
 
   await _rebuild(pkgs, modules, ctx.currentShrinkwrap.registry, opts)
+
+  await saveModules(path.join(ctx.root, 'node_modules'), {
+    packageManager: `${opts.packageManager.name}@${opts.packageManager.version}`,
+    store: ctx.storePath,
+    skipped: Array.from(ctx.skipped),
+    layoutVersion: LAYOUT_VERSION,
+    independentLeaves: opts.independentLeaves,
+    pendingBuilds: [],
+  })
 }
 
 async function _rebuild (

src/api/removeOrphanPkgs.ts

@@ -18,7 +18,7 @@ export default async function removeOrphanPkgs (
     storeController: StoreController,
     pruneStore?: boolean,
   }
-): Promise<string[]> {
+): Promise<Set<string>> {
   const oldPkgs = R.toPairs(R.mergeAll(R.map(depType => opts.oldShrinkwrap[depType], dependenciesTypes)))
   const newPkgs = R.toPairs(R.mergeAll(R.map(depType => opts.newShrinkwrap[depType], dependenciesTypes)))
 
@@ -58,7 +58,7 @@ export default async function removeOrphanPkgs (
 
   await opts.storeController.saveState()
 
-  return notDependents
+  return new Set(notDependents)
 }
 
 function getPackageIds (

src/api/uninstall.ts

@@ -1,5 +1,6 @@
 import rimraf = require('rimraf-then')
 import path = require('path')
+import * as dp from 'dependency-path'
 import getContext, {PnpmContext} from './getContext'
 import getSaveType from '../getSaveType'
 import removeDeps from '../removeDeps'
@@ -65,6 +66,7 @@ export async function uninstallInContext (pkgsToUninstall: string[], ctx: PnpmCo
     storeController: ctx.storeController,
     bin: opts.bin,
   })
+  ctx.pendingBuilds = ctx.pendingBuilds.filter(pkgId => !removedPkgIds.has(dp.resolve(newShr.registry, pkgId)))
   await ctx.storeController.close()
   const currentShrinkwrap = makePartialCurrentShrinkwrap
     ? pruneShrinkwrap(ctx.currentShrinkwrap, pkg)
@@ -73,9 +75,10 @@ export async function uninstallInContext (pkgsToUninstall: string[], ctx: PnpmCo
   await saveModules(path.join(ctx.root, 'node_modules'), {
     packageManager: `${opts.packageManager.name}@${opts.packageManager.version}`,
     store: ctx.storePath,
-    skipped: Array.from(ctx.skipped).filter(pkgId => removedPkgIds.indexOf(pkgId) === -1),
+    skipped: Array.from(ctx.skipped).filter(pkgId => !removedPkgIds.has(pkgId)),
     layoutVersion: LAYOUT_VERSION,
     independentLeaves: opts.independentLeaves,
+    pendingBuilds: ctx.pendingBuilds,
   })
   await removeOuterLinks(pkgsToUninstall, path.join(ctx.root, 'node_modules'), {
     storePath: ctx.storePath,

src/fs/modulesController.ts

@@ -14,6 +14,7 @@ export type Modules = {
   skipped: string[],
   layoutVersion: number,
   independentLeaves: boolean,
+  pendingBuilds: string[],
 }
 
 export async function read (modulesPath: string): Promise<Modules | null> {

src/link/index.ts

@@ -53,6 +53,7 @@ export default async function (
   wantedShrinkwrap: Shrinkwrap,
   currentShrinkwrap: Shrinkwrap,
   newPkgResolvedIds: string[],
+  removedPkgIds: Set<string>,
 }> {
   // TODO: decide what kind of logging should be here.
   // The `Creating dependency tree` is not good to report in all cases as
@@ -62,7 +63,7 @@ export default async function (
   const pkgsToLink = resolvePeersResult.resolvedTree
   const newShr = updateShrinkwrap(pkgsToLink, opts.wantedShrinkwrap, opts.pkg)
 
-  await removeOrphanPkgs({
+  const removedPkgIds = await removeOrphanPkgs({
     oldShrinkwrap: opts.currentShrinkwrap,
     newShrinkwrap: newShr,
     prefix: opts.root,
@@ -156,6 +157,7 @@ export default async function (
     wantedShrinkwrap: newShr,
     currentShrinkwrap,
     newPkgResolvedIds,
+    removedPkgIds,
   }
 }
 

src/types.ts

@@ -22,4 +22,5 @@ export type SupiOptions = PnpmOptions & {
 
 export type StrictSupiOptions = StrictPnpmOptions & {
   storeController?: StoreController
+  pending?: boolean
 }

test/install/lifecycleScripts.ts

@@ -110,7 +110,7 @@ test('do not run install scripts if unsafePerm is false', async (t: tape.Test) =
       postinstall: `node -e "process.stdout.write('postinstall')" | json-append output.json`,
     }
   })
-  const opts = Object.assign(testDefaults(), { unsafePerm: false })
+  const opts = testDefaults({ unsafePerm: false })
   await installPkgs(['json-append@1.1.1'], opts)
   await install(opts)
 

test/rebuild.ts

@@ -52,3 +52,39 @@ test('rebuilds specific dependencies', async function (t: tape.Test) {
   const generatedByPostinstall = project.requireModule('install-scripts-example-for-pnpm/generated-by-postinstall')
   t.ok(typeof generatedByPostinstall === 'function', 'generatedByPostinstall() is available')
 })
+
+test('rebuild with pending option', async function (t: tape.Test) {
+  const project = prepare(t)
+  await installPkgs(['pre-and-postinstall-scripts-example'], testDefaults({ignoreScripts: true}))
+  await installPkgs(['zkochan/install-scripts-example'], testDefaults({ignoreScripts: true}))
+
+  let modules = await project.loadModules()
+  t.doesNotEqual(modules['pendingBuilds'].length, 0)
+
+  await project.hasNot('pre-and-postinstall-scripts-example/generated-by-preinstall')
+  await project.hasNot('pre-and-postinstall-scripts-example/generated-by-postinstall')
+
+  await project.hasNot('install-scripts-example-for-pnpm/generated-by-preinstall')
+  await project.hasNot('install-scripts-example-for-pnpm/generated-by-postinstall')
+
+  await rebuild(testDefaults({rawNpmConfig: {'pending': true}}))
+
+  modules = await project.loadModules()
+  t.equal(modules['pendingBuilds'].length, 0)
+
+  {
+    const generatedByPreinstall = project.requireModule('pre-and-postinstall-scripts-example/generated-by-preinstall')
+    t.ok(typeof generatedByPreinstall === 'function', 'generatedByPreinstall() is available')
+
+    const generatedByPostinstall = project.requireModule('pre-and-postinstall-scripts-example/generated-by-postinstall')
+    t.ok(typeof generatedByPostinstall === 'function', 'generatedByPostinstall() is available')
+  }
+
+  {
+    const generatedByPreinstall = project.requireModule('install-scripts-example-for-pnpm/generated-by-preinstall')
+    t.ok(typeof generatedByPreinstall === 'function', 'generatedByPreinstall() is available')
+
+    const generatedByPostinstall = project.requireModule('install-scripts-example-for-pnpm/generated-by-postinstall')
+    t.ok(typeof generatedByPostinstall === 'function', 'generatedByPostinstall() is available')
+  }
+})

test/shrinkwrap.ts

@@ -23,6 +23,9 @@ test('shrinkwrap file has correct format', async (t: tape.Test) => {
 
   await installPkgs(['pkg-with-1-dep', '@rstacruz/tap-spec@4.1.1', 'kevva/is-negative#1d7e288222b53a0cab90a331f1865220ec29560c'], testDefaults({save: true}))
 
+  const modules = await project.loadModules()
+  t.equal(modules['pendingBuilds'].length, 0)
+
   const shr = await project.loadShrinkwrap()
   const id = '/pkg-with-1-dep/100.0.0'
 
@@ -670,3 +673,24 @@ test('updating shrinkwrap version 3 to 3.1', async (t: tape.Test) => {
   t.equal(shr.shrinkwrapMinorVersion, 4)
   t.ok(shr.packages['/abc/1.0.0/peer-a@1.0.0+peer-b@1.0.0+peer-c@1.0.0'].peerDependencies)
 })
+
+test('pendingBuilds gets updated if install removes packages', async (t: tape.Test) => {
+  const project = prepare(t, {
+    dependencies: {
+      'is-negative': '2.1.0',
+      'sh-hello-world': '1.0.1',
+    },
+  })
+
+  await install(testDefaults({ ignoreScripts: true }))
+  const modules1 = await project.loadModules()
+
+  await project.rewriteDependencies({
+    'is-negative': '2.1.0',
+  })
+
+  await install(testDefaults({ ignoreScripts: true }))
+  const modules2 = await project.loadModules()
+
+  t.ok(modules1['pendingBuilds'].length > modules2['pendingBuilds'].length, 'pendingBuilds gets updated when install removes packages')
+})

test/uninstall.ts

@@ -140,3 +140,18 @@ test('relative link is uninstalled', async (t: tape.Test) => {
 
   await project.hasNot(linkedPkgName)
 })
+
+test('pendingBuilds gets updated after uninstall', async (t: tape.Test) => {
+  const project = prepare(t)
+
+  await installPkgs(['is-negative@2.1.0', 'sh-hello-world@1.0.1'], testDefaults({save: true, ignoreScripts: true}))
+
+  const modules1 = await project.loadModules()
+  t.doesNotEqual(modules1['pendingBuilds'].length, 0, 'installPkgs should update pendingBuilds')
+
+  await uninstall(['sh-hello-world'], testDefaults({save: true}))
+
+  const modules2 = await project.loadModules()
+  t.doesNotEqual(modules2['pendingBuilds'].length, 0, 'uninstall should not remove all the pendingBuilds')
+  t.ok(modules1['pendingBuilds'].length > modules2['pendingBuilds'].length, 'uninstall should update pendingBuilds')
+})

test/utils/prepare.ts

@@ -24,7 +24,8 @@ export default function prepare (t: Test, pkg?: Object) {
   const dirname = dirNumber.toString()
   const pkgTmpPath = path.join(tmpPath, dirname, 'project')
   mkdirp.sync(pkgTmpPath)
-  writePkg.sync(pkgTmpPath, Object.assign({name: 'project', version: '0.0.0'}, pkg))
+  let pkgJson = Object.assign({name: 'project', version: '0.0.0'}, pkg)
+  writePkg.sync(pkgTmpPath, pkgJson)
   process.chdir(pkgTmpPath)
   t.pass(`create testing package ${dirname}`)
 
@@ -90,6 +91,18 @@ export default function prepare (t: Test, pkg?: Object) {
         throw err
       }
     },
+    loadModules: async () => {
+      try {
+        return await loadYamlFile<any>('node_modules/.modules.yaml') // tslint:disable-line
+      } catch (err) {
+        if (err.code === 'ENOENT') return null
+        throw err
+      }
+    },
+    rewriteDependencies: async (deps) => {
+      pkgJson = Object.assign(pkgJson, { dependencies: deps })
+      writePkg.sync(pkgTmpPath, pkgJson)
+    },
   }
   return project
 }