perf: headless install supports shamefully-flatten config

packages/headless/package.json

@@ -92,6 +92,7 @@
     "@pnpm/package-requester": "4.1.8",
     "@pnpm/pkgid-to-filename": "2.0.0",
     "@pnpm/read-package-json": "1.0.1",
+    "@pnpm/shamefully-flatten": "0.0.0",
     "@pnpm/store-controller-types": "0.0.0",
     "@pnpm/symlink-dependency": "0.0.0",
     "@pnpm/types": "2.0.0",

packages/headless/src/index.ts

@@ -24,6 +24,7 @@ import {
 } from '@pnpm/package-requester'
 import pkgIdToFilename from '@pnpm/pkgid-to-filename'
 import { fromDir as readPackageFromDir } from '@pnpm/read-package-json'
+import { shamefullyFlattenByShrinkwrap } from '@pnpm/shamefully-flatten'
 import symlinkDependency from '@pnpm/symlink-dependency'
 import {
   PackageFilesResponse,
@@ -64,6 +65,7 @@ export interface HeadlessOptions {
   include: IncludedDependencies,
   independentLeaves: boolean,
   importerId?: string,
+  shamefullyFlatten: boolean,
   shrinkwrapDirectory?: string,
   storeController: StoreController,
   verifyStoreIntegrity: boolean,
@@ -103,6 +105,7 @@ export default async (opts: HeadlessOptions) => {
     throw new Error('Headless installation requires a shrinkwrap.yaml file')
   }
 
+  const shamefullyFlatten = opts.shamefullyFlatten === true
   const currentShrinkwrap = opts.currentShrinkwrap || await readCurrent(shrinkwrapDirectory, { ignoreIncompatible: false })
   const importerId = getImporterId(shrinkwrapDirectory, opts.prefix)
   const virtualStoreDir = await realNodeModulesDir(shrinkwrapDirectory)
@@ -113,7 +116,7 @@ export default async (opts: HeadlessOptions) => {
       importers: {
         [importerId]: {
           hoistedAliases: {},
-          shamefullyFlatten: false, // shamefully flatten is not supported yet by headless install
+          shamefullyFlatten,
         },
       },
       pendingBuilds: [] as string[],
@@ -166,7 +169,7 @@ export default async (opts: HeadlessOptions) => {
           id: importerId,
           modulesDir,
           prefix: opts.prefix,
-          shamefullyFlatten: false,
+          shamefullyFlatten,
         },
       ],
       newShrinkwrap: filterShrinkwrap(wantedShrinkwrap, filterOpts),
@@ -221,6 +224,15 @@ export default async (opts: HeadlessOptions) => {
 
   await linkAllBins(depGraph, { optional: opts.include.optionalDependencies, warn })
 
+  if (shamefullyFlatten) {
+    modules.importers[importerId].hoistedAliases = await shamefullyFlattenByShrinkwrap(filteredShrinkwrap, importerId, {
+      defaultRegistry: registries.default,
+      modulesDir,
+      prefix: opts.prefix,
+      virtualStoreDir,
+    })
+  }
+
   await linkRootPackages(filteredShrinkwrap, {
     defaultRegistry: registries.default,
     importerId,

packages/headless/test/fixtures/simple-shamefully-flatten/package.json

@@ -0,0 +1,14 @@
+{
+  "name": "simple-shamefully-flatten",
+  "version": "1.0.0",
+  "dependencies": {
+    "is-positive": "^1.0.0",
+    "rimraf": "^2.6.2"
+  },
+  "devDependencies": {
+    "is-negative": "^2.1.0"
+  },
+  "optionalDependencies": {
+    "colors": "1.2.0"
+  }
+}

packages/headless/test/fixtures/simple-shamefully-flatten/shrinkwrap.yaml

@@ -0,0 +1,105 @@
+dependencies:
+  is-positive: 1.0.0
+  rimraf: 2.6.2
+devDependencies:
+  is-negative: 2.1.0
+optionalDependencies:
+  colors: 1.2.0
+packages:
+  /balanced-match/1.0.0:
+    dev: false
+    resolution:
+      integrity: sha1-ibTRmasr7kneFk6gK4nORi1xt2c=
+  /brace-expansion/1.1.11:
+    dependencies:
+      balanced-match: 1.0.0
+      concat-map: 0.0.1
+    dev: false
+    resolution:
+      integrity: sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==
+  /colors/1.2.0:
+    dev: false
+    engines:
+      node: '>=0.1.90'
+    optional: true
+    resolution:
+      integrity: sha512-lweugcX5nailCqZBttArTojZZpHGWhmFJX78KJHlxwhM8tLAy5QCgRgRxrubrksdvA+2Y3inWG5TToyyjL82BQ==
+  /concat-map/0.0.1:
+    dev: false
+    resolution:
+      integrity: sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=
+  /fs.realpath/1.0.0:
+    dev: false
+    resolution:
+      integrity: sha1-FQStJSMVjKpA20onh8sBQRmU6k8=
+  /glob/7.1.2:
+    dependencies:
+      fs.realpath: 1.0.0
+      inflight: 1.0.6
+      inherits: 2.0.3
+      minimatch: 3.0.4
+      once: 1.4.0
+      path-is-absolute: 1.0.1
+    dev: false
+    resolution:
+      integrity: sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==
+  /inflight/1.0.6:
+    dependencies:
+      once: 1.4.0
+      wrappy: 1.0.2
+    dev: false
+    resolution:
+      integrity: sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=
+  /inherits/2.0.3:
+    dev: false
+    resolution:
+      integrity: sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=
+  /is-negative/2.1.0:
+    dev: true
+    engines:
+      node: '>=0.10.0'
+    resolution:
+      integrity: sha1-8Nhjd6oVpkw0lh84rCqb4rQKEYc=
+  /is-positive/1.0.0:
+    dev: false
+    engines:
+      node: '>=0.10.0'
+    resolution:
+      integrity: sha1-iACYVrZKLx632LsBeUGEJK4EUss=
+  /minimatch/3.0.4:
+    dependencies:
+      brace-expansion: 1.1.11
+    dev: false
+    resolution:
+      integrity: sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==
+  /once/1.4.0:
+    dependencies:
+      wrappy: 1.0.2
+    dev: false
+    resolution:
+      integrity: sha1-WDsap3WWHUsROsF9nFC6753Xa9E=
+  /path-is-absolute/1.0.1:
+    dev: false
+    engines:
+      node: '>=0.10.0'
+    resolution:
+      integrity: sha1-F0uSaHNVNP+8es5r9TpanhtcX18=
+  /rimraf/2.6.2:
+    dependencies:
+      glob: 7.1.2
+    dev: false
+    hasBin: true
+    resolution:
+      integrity: sha512-lreewLK/BlghmxtfH36YYVg1i8IAce4TI7oao75I1g245+6BctqTVQiBP3YUJ9C6DQOXJmkYR9X9fCLtCOJc5w==
+  /wrappy/1.0.2:
+    dev: false
+    resolution:
+      integrity: sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=
+registry: 'http://localhost:4873/'
+shrinkwrapMinorVersion: 8
+shrinkwrapVersion: 3
+specifiers:
+  colors: 1.2.0
+  is-negative: ^2.1.0
+  is-positive: ^1.0.0
+  rimraf: ^2.6.2

packages/headless/test/index.ts

@@ -473,3 +473,59 @@ test('independent-leaves: installing a simple project', async (t) => {
 
   t.end()
 })
+
+test('installing with shamefullyFlatten = true', async (t) => {
+  const prefix = path.join(fixtures, 'simple-shamefully-flatten')
+  const reporter = sinon.spy()
+
+  await headless(await testDefaults({ prefix, reporter, shamefullyFlatten: true }))
+
+  const project = assertProject(t, prefix)
+  t.ok(project.requireModule('is-positive'), 'prod dep installed')
+  t.ok(project.requireModule('rimraf'), 'prod dep installed')
+  t.ok(project.requireModule('glob'), 'prod subdep hoisted')
+  t.ok(project.requireModule('is-negative'), 'dev dep installed')
+  t.ok(project.requireModule('colors'), 'optional dep installed')
+
+  // test that independent leaves is false by default
+  t.ok(project.has('.localhost+4873/colors'), 'colors is not symlinked from the store')
+
+  await project.isExecutable('.bin/rimraf')
+
+  t.ok(await project.loadCurrentShrinkwrap())
+  t.ok(await project.loadModules())
+
+  t.ok(reporter.calledWithMatch({
+    initial: require(path.join(prefix, 'package.json')),
+    level: 'debug',
+    name: 'pnpm:package-json',
+  } as PackageJsonLog), 'initial package.json logged')
+  t.ok(reporter.calledWithMatch({
+    added: 15,
+    level: 'debug',
+    name: 'pnpm:stats',
+    prefix,
+  } as StatsLog), 'added stat')
+  t.ok(reporter.calledWithMatch({
+    level: 'debug',
+    name: 'pnpm:stats',
+    prefix,
+    removed: 0,
+  } as StatsLog), 'removed stat')
+  t.ok(reporter.calledWithMatch({
+    level: 'debug',
+    message: 'importing_done',
+    name: 'pnpm:stage',
+  } as StageLog), 'importing stage done logged')
+  t.ok(reporter.calledWithMatch({
+    level: 'debug',
+    pkgId: 'localhost+4873/is-negative/2.1.0',
+    status: 'resolving_content',
+  }), 'logs that package is being resolved')
+
+  const modules = await project.loadModules()
+
+  t.deepEqual(modules!.importers['.'].hoistedAliases['localhost+4873/balanced-match/1.0.0'], ['balanced-match'], 'hoisted field populated in .modules.yaml')
+
+  t.end()
+})

packages/supi/src/install/index.ts

@@ -118,14 +118,7 @@ export async function install (maybeOpts: InstallOptions & {
         satisfiesPackageJson(ctx.wantedShrinkwrap, importer.pkg, importer.id) &&
         await linkedPackagesAreUpToDate(importer.pkg, ctx.wantedShrinkwrap.importers[importer.id], importer.prefix, opts.localPackages))
       ) {
-        if (importer.shamefullyFlatten) {
-          if (opts.frozenShrinkwrap) {
-            logger.warn({
-              message: 'Headless installation does not support flat node_modules layout yet',
-              prefix: importer.prefix,
-            })
-          }
-        } else if (!ctx.existsWantedShrinkwrap) {
+        if (!ctx.existsWantedShrinkwrap) {
           if (R.keys(importer.pkg.dependencies).length || R.keys(importer.pkg.devDependencies).length || R.keys(importer.pkg.optionalDependencies).length) {
             throw new Error('Headless installation requires a shrinkwrap.yaml file')
           }
@@ -137,6 +130,7 @@ export async function install (maybeOpts: InstallOptions & {
             importerId: importer.id,
             packageJson: importer.pkg,
             prefix: importer.prefix,
+            shamefullyFlatten: opts.shamefullyFlatten,
             shrinkwrapDirectory: ctx.shrinkwrapDirectory,
             wantedShrinkwrap: ctx.wantedShrinkwrap,
           } as HeadlessOptions)

packages/supi/test/install/frozenShrinkwrap.ts

@@ -190,3 +190,31 @@ test('frozen-shrinkwrap: should not fail if no shrinkwrap.yaml is present and pr
 
   await install(await testDefaults({ frozenShrinkwrap: true }))
 })
+
+test('prefer-frozen-shrinkwrap+shamefully-flatten: should prefer headless installation when shrinkwrap.yaml satisfies package.json', async (t) => {
+  const project = prepare(t, {
+    dependencies: {
+      'pkg-with-1-dep': '100.0.0',
+    },
+  })
+
+  await install(await testDefaults({ shrinkwrapOnly: true }))
+
+  await project.hasNot('pkg-with-1-dep')
+
+  const reporter = sinon.spy()
+  await install(await testDefaults({
+    preferFrozenShrinkwrap: true,
+    reporter,
+    shamefullyFlatten: true,
+  }))
+
+  t.ok(reporter.calledWithMatch({
+    level: 'info',
+    message: 'Performing headless installation',
+    name: 'pnpm',
+  }), 'start of headless installation logged')
+
+  await project.has('pkg-with-1-dep')
+  await project.has('dep-of-pkg-with-1-dep')
+})

shrinkwrap.yaml

@@ -269,6 +269,7 @@ importers:
       '@pnpm/package-requester': 'link:../package-requester'
       '@pnpm/pkgid-to-filename': 2.0.0
       '@pnpm/read-package-json': 1.0.1
+      '@pnpm/shamefully-flatten': 'link:../shamefully-flatten'
       '@pnpm/store-controller-types': 'link:../store-controller-types'
       '@pnpm/symlink-dependency': 'link:../symlink-dependency'
       '@pnpm/types': 'link:../types'
@@ -328,6 +329,7 @@ importers:
       '@pnpm/package-store': 0.0.0
       '@pnpm/pkgid-to-filename': 2.0.0
       '@pnpm/read-package-json': 1.0.1
+      '@pnpm/shamefully-flatten': 0.0.0
       '@pnpm/store-controller-types': 0.0.0
       '@pnpm/store-path': 1.0.3
       '@pnpm/symlink-dependency': 0.0.0