feat: add independent-leaves config

BREAKING CHANGE:

independent dependencies are not symlinked from the global store
by default

Close #821

README.md

@@ -116,6 +116,15 @@ Can be passed in via a CLI option. `--no-lock` to set it to false. E.g.: `pnpm i
 > If you experience issues similar to the ones described in [#594](https://github.com/pnpm/pnpm/issues/594), use this option to disable locking.
 > In the meanwhile, we'll try to find a solution that will make locking work for everyone.
 
+#### independent-leaves
+
+* Default: **false**
+* Type: **Boolean**
+
+If true, symlinks leaf dependencies directly from the global store. Leaf dependencies are
+packages that have no dependencies of their own. Setting this config to `true` might break some packages
+that rely on location but gives an average of **8% installation speed improvement**.
+
 ## Benchmark
 
 pnpm is faster than npm and yarn. See [this](https://github.com/zkochan/node-package-manager-benchmark)

src/api/extendOptions.ts

@@ -38,6 +38,7 @@ const defaults = (opts: PnpmOptions) => {
     update: false,
     repeatInstallDepth: -1,
     optional: true,
+    independentLeaves: false,
   }
 }
 
@@ -66,7 +67,8 @@ export default (opts?: PnpmOptions): StrictPnpmOptions => {
   }
   extendedOpts.registry = normalizeRegistryUrl(extendedOpts.registry)
   if (extendedOpts.global) {
-    extendedOpts.prefix = path.join(extendedOpts.prefix, LAYOUT_VERSION.toString())
+    const subfolder = LAYOUT_VERSION.toString() + (extendedOpts.independentLeaves ? '_independent_leaves' : '')
+    extendedOpts.prefix = path.join(extendedOpts.prefix, subfolder)
   }
   return extendedOpts
 }

src/api/getContext.ts

@@ -41,6 +41,14 @@ export default async function getContext (opts: StrictPnpmOptions, installType?:
 
   if (modules) {
     try {
+      if (Boolean(modules.independentLeaves) !== opts.independentLeaves) {
+        if (modules.independentLeaves) {
+          throw new Error(`This node_modules was installed with --independent-leaves option.
+            Use this option or run same command with --force to recreated node_modules`)
+        }
+        throw new Error(`This node_modules was not installed with the --independent-leaves option.
+          Don't use --independent-leaves run same command with --force to recreated node_modules`)
+      }
       checkCompatibility(modules, {storePath, modulesPath})
     } catch (err) {
       if (!opts.force) throw err

src/api/install.ts

@@ -377,6 +377,7 @@ async function installInContext (
     storePath: ctx.storePath,
     skipped: ctx.skipped,
     pkg: newPkg || ctx.pkg,
+    independentLeaves: opts.independentLeaves
   })
 
   await saveShrinkwrap(ctx.root, result.shrinkwrap)
@@ -385,6 +386,7 @@ async function installInContext (
     storePath: ctx.storePath,
     skipped: Array.from(installCtx.skipped),
     layoutVersion: LAYOUT_VERSION,
+    independentLeaves: opts.independentLeaves,
   })
 
   // postinstall hooks

src/api/uninstall.ts

@@ -60,6 +60,7 @@ export async function uninstallInContext (pkgsToUninstall: string[], ctx: PnpmCo
     storePath: ctx.storePath,
     skipped: Array.from(ctx.skipped).filter(pkgId => removedPkgIds.indexOf(pkgId) === -1),
     layoutVersion: LAYOUT_VERSION,
+    independentLeaves: opts.independentLeaves,
   })
   await removeOuterLinks(pkgsToUninstall, path.join(ctx.root, 'node_modules'), {storePath: ctx.storePath})
 }

src/bin/pnpm.ts

@@ -63,6 +63,7 @@ async function run (argv: string[]) {
     'child-concurrency': Number,
     'offline': Boolean,
     'reporter': String,
+    'independent-leaves': Boolean,
   }
   const types = R.merge(npmDefaults.types, pnpmTypes)
   const cliConf = nopt(

src/fs/modulesController.ts

@@ -13,6 +13,7 @@ export type Modules = {
   storePath: string,
   skipped: string[],
   layoutVersion: number,
+  independentLeaves: boolean,
 }
 
 export async function read (modulesPath: string): Promise<Modules | null> {

src/link/index.ts

@@ -40,6 +40,7 @@ export default async function (
     storePath: string,
     skipped: Set<string>,
     pkg: Package,
+    independentLeaves: boolean,
   }
 ): Promise<{
   linkedPkgsMap: DependencyTreeNodeMap,
@@ -47,7 +48,7 @@ export default async function (
   newPkgResolvedIds: string[],
 }> {
   const topPkgIds = topPkgs.map(pkg => pkg.id)
-  const pkgsToLink = await resolvePeers(tree, rootNodeIds, topPkgIds, opts.topParents)
+  const pkgsToLink = await resolvePeers(tree, rootNodeIds, topPkgIds, opts.topParents, opts.independentLeaves)
   const newShr = updateShrinkwrap(pkgsToLink, opts.shrinkwrap, opts.pkg)
 
   await removeOrphanPkgs(opts.privateShrinkwrap, newShr, opts.root, opts.storePath)

src/link/resolvePeers.ts

@@ -41,7 +41,8 @@ export default function (
   topPkgIds: string[],
   // only the top dependencies that were already installed
   // to avoid warnings about unresolved peer dependencies
-  topParents: {name: string, version: string}[]
+  topParents: {name: string, version: string}[],
+  independentLeaves: boolean
 ): DependencyTreeNodeMap {
   const pkgsByName = R.fromPairs(
     topParents.map((parent: {name: string, version: string}): R.KeyValuePair<string, ParentRef> => [
@@ -55,7 +56,7 @@ export default function (
 
   const nodeIdToResolvedId = {}
   const resolvedTree: DependencyTreeNodeMap = {}
-  resolvePeersOfChildren(rootNodeIds, pkgsByName, tree, nodeIdToResolvedId, resolvedTree)
+  resolvePeersOfChildren(rootNodeIds, pkgsByName, tree, nodeIdToResolvedId, resolvedTree, independentLeaves)
 
   R.values(resolvedTree).forEach(node => {
     node.children = node.children.map(child => nodeIdToResolvedId[child])
@@ -68,11 +69,12 @@ function resolvePeersOfNode (
   parentPkgs: ParentRefs,
   tree: TreeNodeMap,
   nodeIdToResolvedId: {[nodeId: string]: string},
-  resolvedTree: DependencyTreeNodeMap
+  resolvedTree: DependencyTreeNodeMap,
+  independentLeaves: boolean
 ): string[] {
   const node = tree[nodeId]
 
-  const unknownResolvedPeersOfChildren = resolvePeersOfChildren(node.children, parentPkgs, tree, nodeIdToResolvedId, resolvedTree)
+  const unknownResolvedPeersOfChildren = resolvePeersOfChildren(node.children, parentPkgs, tree, nodeIdToResolvedId, resolvedTree, independentLeaves)
 
   const resolvedPeers = R.isEmpty(node.pkg.peerDependencies)
     ? []
@@ -97,7 +99,7 @@ function resolvePeersOfNode (
 
   nodeIdToResolvedId[nodeId] = resolvedId
   if (!resolvedTree[resolvedId] || resolvedTree[resolvedId].depth > node.depth) {
-    const independent = !node.children.length && R.isEmpty(node.pkg.peerDependencies)
+    const independent = independentLeaves && !node.children.length && R.isEmpty(node.pkg.peerDependencies)
     const pathToUnpacked = path.join(node.pkg.path, 'node_modules', node.pkg.name)
     const hardlinkedLocation = !independent
       ? path.join(modules, node.pkg.name)
@@ -129,7 +131,8 @@ function resolvePeersOfChildren (
   parentParentPkgs: ParentRefs,
   tree: {[nodeId: string]: TreeNode},
   nodeIdToResolvedId: {[nodeId: string]: string},
-  resolvedTree: DependencyTreeNodeMap
+  resolvedTree: DependencyTreeNodeMap,
+  independentLeaves: boolean
 ): string[] {
   const trees: DependencyTreeNodeMap[] = []
   const unknownResolvedPeersOfChildren: string[] = []
@@ -138,7 +141,7 @@ function resolvePeersOfChildren (
   )
 
   for (const child of children) {
-    const allResolvedPeers = resolvePeersOfNode(child, parentPkgs, tree, nodeIdToResolvedId, resolvedTree)
+    const allResolvedPeers = resolvePeersOfNode(child, parentPkgs, tree, nodeIdToResolvedId, resolvedTree, independentLeaves)
 
     const unknownResolvedPeersOfChild = allResolvedPeers
       .filter((resolvedPeerNodeId: string) => children.indexOf(resolvedPeerNodeId) === -1)

src/types.ts

@@ -50,6 +50,7 @@ export type PnpmOptions = {
   lock?: boolean,
   childConcurrency?: number,
   repeatInstallDepth?: number,
+  independentLeaves?: boolean,
 
   // cannot be specified via configs
   update?: boolean,
@@ -103,6 +104,7 @@ export type StrictPnpmOptions = {
   lock: boolean,
   childConcurrency: number,
   repeatInstallDepth: number,
+  independentLeaves: boolean,
 
   // cannot be specified via configs
   update: boolean,

test/breakingChanges.ts

@@ -77,7 +77,7 @@ test('fail on non-compatible store when forced during named installation', async
 
 async function saveModulesYaml (pnpmVersion: string, storePath: string) {
   await mkdirp('node_modules')
-  await fs.writeFile('node_modules/.modules.yaml', `packageManager: pnpm@${pnpmVersion}\nstorePath: ${storePath}`)
+  await fs.writeFile('node_modules/.modules.yaml', `packageManager: pnpm@${pnpmVersion}\nstorePath: ${storePath}\nindependentLeaves: false`)
 }
 
 test('fail on non-compatible shrinkwrap.yaml', async t => {

test/install/independentLeaves.ts

@@ -0,0 +1,60 @@
+import tape = require('tape')
+import path = require('path')
+import promisifyTape from 'tape-promise'
+import {
+  prepare,
+  testDefaults,
+} from '../utils'
+import {installPkgs} from '../../src'
+
+const test = promisifyTape(tape)
+
+test('install with --independent-leaves', async function (t: tape.Test) {
+  const project = prepare(t)
+  await installPkgs(['rimraf@2.5.1'], testDefaults({independentLeaves: true}))
+
+  const m = project.requireModule('rimraf')
+  t.ok(typeof m === 'function', 'rimraf() is available')
+  await project.isExecutable('.bin/rimraf')
+})
+
+test('--independent-leaves throws exception when executed on node_modules installed w/o the option', async function (t: tape.Test) {
+  const project = prepare(t)
+  await installPkgs(['is-positive'], testDefaults({independentLeaves: false}))
+
+  try {
+    await installPkgs(['is-negative'], testDefaults({independentLeaves: true}))
+    t.fail('installation should have failed')
+  } catch (err) {
+    t.ok(err.message.indexOf('This node_modules was not installed with the --independent-leaves option.') === 0)
+  }
+})
+
+test('--no-independent-leaves throws exception when executed on node_modules installed with --independent-leaves', async function (t: tape.Test) {
+  const project = prepare(t)
+  await installPkgs(['is-positive'], testDefaults({independentLeaves: true}))
+
+  try {
+    await installPkgs(['is-negative'], testDefaults({independentLeaves: false}))
+    t.fail('installation should have failed')
+  } catch (err) {
+    t.ok(err.message.indexOf('This node_modules was installed with --independent-leaves option.') === 0)
+  }
+})
+
+test('global installation with --independent-leaves', async function (t: tape.Test) {
+  prepare(t)
+  const globalPrefix = path.resolve('..', 'global')
+  const opts = testDefaults({global: true, prefix: globalPrefix, independentLeaves: true})
+  await installPkgs(['is-positive'], opts)
+
+  // there was an issue when subsequent installations were removing everything installed prior
+  // https://github.com/pnpm/pnpm/issues/808
+  await installPkgs(['is-negative'], opts)
+
+  const isPositive = require(path.join(globalPrefix, '1_independent_leaves', 'node_modules', 'is-positive'))
+  t.ok(typeof isPositive === 'function', 'isPositive() is available')
+
+  const isNegative = require(path.join(globalPrefix, '1_independent_leaves', 'node_modules', 'is-negative'))
+  t.ok(typeof isNegative === 'function', 'isNegative() is available')
+})

test/install/index.ts

@@ -9,3 +9,4 @@ import './auth'
 import './local'
 import './updatingPkgJson'
 import './global'
+import './independentLeaves'