feat: "recursive rebuild" supports "shared-workspace-shrinkwrap"

PR #1597

* feat: "recursive rebuild" supports "shared-workspace-shrinkwarp"

close #1596

* fix: recursive rebuild + independen-leaves and shared shrinkwrap

* fix: build workspace package in correct order during r-ve install

* test: always create the store in the current test's temp folder

* style: sort imports in tests

* test: correct order of workspace package builds

* feat: build workspace packages concurrently during headless install

* refactor: use run-groups

* test: stages should run in correct order
This commit is contained in:
Zoltan Kochan
2019-01-07 22:16:25 +02:00
committed by GitHub
parent 184f9ba0c7
commit 33a5830637
30 changed files with 780 additions and 442 deletions

View File

@@ -99,11 +99,13 @@
"@pnpm/symlink-dependency": "1.1.3",
"@pnpm/types": "2.0.0",
"@pnpm/utils": "0.9.1",
"@types/p-limit": "2.0.0",
"@types/ramda": "0.25.34",
"dependency-path": "2.0.1",
"graph-sequencer": "2.0.0",
"p-limit": "2.1.0",
"path-exists": "3.0.0",
"ramda": "0.26.1"
"ramda": "0.26.1",
"run-groups": "1.0.0"
}
}

View File

@@ -9,7 +9,7 @@ import {
import filterShrinkwrap, {
filterByImporters as filterShrinkwrapByImporters,
} from '@pnpm/filter-shrinkwrap'
import runLifecycleHooks from '@pnpm/lifecycle'
import { runLifecycleHooksConcurrently } from '@pnpm/lifecycle'
import linkBins, { linkBinsOfPackages } from '@pnpm/link-bins'
import logger, {
LogBase,
@@ -32,6 +32,7 @@ import {
} from '@pnpm/shrinkwrap-file'
import {
nameVerFromPkgSnapshot,
packageIsIndependent,
pkgSnapshotToResolution,
satisfiesPackageJson,
} from '@pnpm/shrinkwrap-utils'
@@ -62,6 +63,7 @@ export interface HeadlessOptions {
independentLeaves: boolean,
importers: Array<{
bin: string,
buildIndex: number,
hoistedAliases: {[depPath: string]: string[]}
modulesDir: string,
id: string,
@@ -117,24 +119,20 @@ export default async (opts: HeadlessOptions) => {
}
}
const scriptsOpts = {
optional: false,
rawNpmConfig: opts.rawNpmConfig,
stdio: opts.ownLifecycleHooksStdio || 'inherit',
unsafePerm: opts.unsafePerm || false,
}
if (!opts.ignoreScripts) {
for (const importer of opts.importers) {
const scripts = !opts.ignoreScripts && importer.pkg.scripts || {}
const scriptsOpts = {
depPath: importer.prefix,
optional: false,
pkgRoot: importer.prefix,
rawNpmConfig: opts.rawNpmConfig,
rootNodeModulesDir: importer.modulesDir,
stdio: opts.ownLifecycleHooksStdio || 'inherit',
unsafePerm: opts.unsafePerm || false,
}
if (scripts.preinstall) {
await runLifecycleHooks('preinstall', importer.pkg, scriptsOpts)
}
}
await runLifecycleHooksConcurrently(
['preinstall'],
opts.importers,
opts.childConcurrency || 5,
scriptsOpts,
)
}
const filterOpts = {
@@ -313,32 +311,12 @@ export default async (opts: HeadlessOptions) => {
await opts.storeController.close()
if (!opts.ignoreScripts) {
for (const importer of opts.importers) {
if (!importer.pkg.scripts) continue
const scriptsOpts = {
depPath: importer.prefix,
optional: false,
pkgRoot: importer.prefix,
rawNpmConfig: opts.rawNpmConfig,
rootNodeModulesDir: importer.modulesDir,
stdio: opts.ownLifecycleHooksStdio || 'inherit',
unsafePerm: opts.unsafePerm || false,
}
if (importer.pkg.scripts.install) {
await runLifecycleHooks('install', importer.pkg, scriptsOpts)
}
if (importer.pkg.scripts.postinstall) {
await runLifecycleHooks('postinstall', importer.pkg, scriptsOpts)
}
if (importer.pkg.scripts.prepublish) {
await runLifecycleHooks('prepublish', importer.pkg, scriptsOpts)
}
if (importer.pkg.scripts.prepare) {
await runLifecycleHooks('prepare', importer.pkg, scriptsOpts)
}
}
await runLifecycleHooksConcurrently(
['install', 'postinstall', 'prepublish', 'prepare'],
opts.importers,
opts.childConcurrency || 5,
scriptsOpts,
)
}
if (reporter) {
@@ -443,7 +421,6 @@ async function shrinkwrapToDepGraph (
}
const depPath = dp.resolve(opts.defaultRegistry, relDepPath)
const pkgSnapshot = shr.packages[relDepPath]
const independent = opts.independentLeaves && pkgIsIndependent(pkgSnapshot)
const resolution = pkgSnapshotToResolution(relDepPath, pkgSnapshot, opts.defaultRegistry)
// TODO: optimize. This info can be already returned by pkgSnapshotToResolution()
const pkgName = nameVerFromPkgSnapshot(relDepPath, pkgSnapshot).name
@@ -475,10 +452,8 @@ async function shrinkwrapToDepGraph (
targetEngine: opts.sideEffectsCacheRead && !opts.force && ENGINE_NAME || undefined,
})
// NOTE: This code will not convert the depPath with peer deps correctly
// Unfortunately, there is currently no way to tell if the last dir in the path is originally there or added to separate
// the diferent peer dependency sets
const modules = path.join(opts.virtualStoreDir, `.${pkgIdToFilename(depPath, opts.prefix)}`, 'node_modules')
const independent = opts.independentLeaves && packageIsIndependent(pkgSnapshot)
const peripheralLocation = !independent
? path.join(modules, pkgName)
: pkgLocation.directory
@@ -555,7 +530,7 @@ async function getChildrenPaths (
const childPkgSnapshot = ctx.pkgSnapshotsByRelDepPaths[childRelDepPath]
if (ctx.graph[childDepPath]) {
children[alias] = ctx.graph[childDepPath].peripheralLocation
} else if (ctx.independentLeaves && pkgIsIndependent(childPkgSnapshot)) {
} else if (ctx.independentLeaves && packageIsIndependent(childPkgSnapshot)) {
const pkgId = childPkgSnapshot.id || childDepPath
const pkgName = nameVerFromPkgSnapshot(childRelDepPath, childPkgSnapshot).name
const pkgLocation = await ctx.storeController.getPackageLocation(pkgId, pkgName, {
@@ -576,10 +551,6 @@ async function getChildrenPaths (
return children
}
function pkgIsIndependent (pkgSnapshot: PackageSnapshot) {
return pkgSnapshot.dependencies === undefined && pkgSnapshot.optionalDependencies === undefined
}
export interface DependenciesGraphNode {
hasBundledDependencies: boolean,
centralLocation: string,

View File

@@ -6,9 +6,9 @@ import logger from '@pnpm/logger'
import { fromDir as readPackageFromDir } from '@pnpm/read-package-json'
import { StoreController } from '@pnpm/store-controller-types'
import graphSequencer = require('graph-sequencer')
import pLimit = require('p-limit')
import path = require('path')
import R = require('ramda')
import runGroups from 'run-groups'
import { DependenciesGraph } from '.'
import { ENGINE_NAME } from './constants'
@@ -27,9 +27,6 @@ export default async (
},
) => {
// postinstall hooks
const limitChild = pLimit(opts.childConcurrency || 4)
const depPaths = Object.keys(depGraph)
const nodesToBuild = new Set<string>()
getSubgraphToBuild(depGraph, rootDepPaths, nodesToBuild, new Set<string>())
const onlyFromBuildGraph = R.filter((depPath: string) => nodesToBuild.has(depPath))
@@ -44,63 +41,61 @@ export default async (
groups: [nodesToBuildArray],
})
const chunks = graphSequencerResult.chunks as string[][]
for (const chunk of chunks) {
await Promise.all(chunk
.filter((depPath) => depGraph[depPath].requiresBuild && !depGraph[depPath].isBuilt)
.map((depPath: string) => limitChild(async () => {
const depNode = depGraph[depPath]
try {
const hasSideEffects = await runPostinstallHooks({
depPath,
optional: depNode.optional,
pkgRoot: depNode.peripheralLocation,
prepare: depNode.prepare,
rawNpmConfig: opts.rawNpmConfig,
rootNodeModulesDir: opts.rootNodeModulesDir,
unsafePerm: opts.unsafePerm || false,
})
if (hasSideEffects && opts.sideEffectsCacheWrite) {
try {
await opts.storeController.upload(depNode.peripheralLocation, {
engine: ENGINE_NAME,
pkgId: depNode.pkgId,
const groups = chunks.map((chunk) => chunk.filter((depPath) => depGraph[depPath].requiresBuild && !depGraph[depPath].isBuilt).map((depPath: string) =>
async () => {
const depNode = depGraph[depPath]
try {
const hasSideEffects = await runPostinstallHooks({
depPath,
optional: depNode.optional,
pkgRoot: depNode.peripheralLocation,
prepare: depNode.prepare,
rawNpmConfig: opts.rawNpmConfig,
rootNodeModulesDir: opts.rootNodeModulesDir,
unsafePerm: opts.unsafePerm || false,
})
if (hasSideEffects && opts.sideEffectsCacheWrite) {
try {
await opts.storeController.upload(depNode.peripheralLocation, {
engine: ENGINE_NAME,
pkgId: depNode.pkgId,
})
} catch (err) {
if (err && err.statusCode === 403) {
logger.warn({
message: `The store server disabled upload requests, could not upload ${depNode.pkgId}`,
prefix: opts.prefix,
})
} else {
logger.warn({
error: err,
message: `An error occurred while uploading ${depNode.pkgId}`,
prefix: opts.prefix,
})
} catch (err) {
if (err && err.statusCode === 403) {
logger.warn({
message: `The store server disabled upload requests, could not upload ${depNode.pkgId}`,
prefix: opts.prefix,
})
} else {
logger.warn({
error: err,
message: `An error occurred while uploading ${depNode.pkgId}`,
prefix: opts.prefix,
})
}
}
}
} catch (err) {
if (depNode.optional) {
// TODO: add parents field to the log
const pkg = await readPackageFromDir(path.join(depNode.peripheralLocation))
skippedOptionalDependencyLogger.debug({
details: err.toString(),
package: {
id: depNode.pkgId,
name: pkg.name,
version: pkg.version,
},
prefix: opts.prefix,
reason: 'build_failure',
})
return
}
throw err
}
})))
}
} catch (err) {
if (depNode.optional) {
// TODO: add parents field to the log
const pkg = await readPackageFromDir(path.join(depNode.peripheralLocation))
skippedOptionalDependencyLogger.debug({
details: err.toString(),
package: {
id: depNode.pkgId,
name: pkg.name,
version: pkg.version,
},
prefix: opts.prefix,
reason: 'build_failure',
})
return
}
throw err
}
}
))
await runGroups(opts.childConcurrency || 4, groups)
}
function getSubgraphToBuild (

View File

@@ -1,8 +1,3 @@
declare module 'p-limit' {
const anything: any;
export = anything;
}
declare module 'is-exe' {
const anything: any;
export = anything;

View File

@@ -38,7 +38,8 @@
"@pnpm/read-package-json": "1.1.1",
"@pnpm/types": "2.0.0",
"@zkochan/npm-lifecycle": "2.2.0",
"path-exists": "3.0.0"
"path-exists": "3.0.0",
"run-groups": "1.0.0"
},
"devDependencies": {
"@pnpm/lifecycle": "link:",

View File

@@ -1,11 +1,11 @@
import { lifecycleLogger } from '@pnpm/core-loggers'
import { fromDir as readPackageJsonFromDir } from '@pnpm/read-package-json'
import { PackageJson } from '@pnpm/types'
import lifecycle = require('@zkochan/npm-lifecycle')
import path = require('path')
import exists = require('path-exists')
import runLifecycleHook from './runLifecycleHook'
import runLifecycleHooksConcurrently from './runLifecycleHooksConcurrently'
function noop () {} // tslint:disable-line:no-empty
export default runLifecycleHook
export { runLifecycleHooksConcurrently }
export async function runPostinstallHooks (
opts: {
@@ -42,79 +42,6 @@ export async function runPostinstallHooks (
return !!scripts.preinstall || !!scripts.install || !!scripts.postinstall
}
export default async function runLifecycleHook (
stage: string,
pkg: PackageJson,
opts: {
depPath: string,
optional?: boolean,
pkgRoot: string,
rawNpmConfig: object,
rootNodeModulesDir: string,
stdio?: string,
unsafePerm: boolean,
},
) {
const optional = opts.optional === true
if (opts.stdio !== 'inherit') {
lifecycleLogger.debug({
depPath: opts.depPath,
optional,
script: pkg.scripts![stage],
stage,
wd: opts.pkgRoot,
})
}
return lifecycle(pkg, stage, opts.pkgRoot, {
config: opts.rawNpmConfig,
dir: opts.rootNodeModulesDir,
log: {
clearProgress: noop,
info: noop,
level: opts.stdio === 'inherit' ? undefined : 'silent',
pause: noop,
resume: noop,
showProgress: noop,
silly: npmLog,
verbose: npmLog,
warn: noop,
},
runConcurrently: true,
stdio: opts.stdio || 'pipe',
unsafePerm: opts.unsafePerm,
})
function npmLog (prefix: string, logid: string, stdtype: string, line: string) {
switch (stdtype) {
case 'stdout':
case 'stderr':
lifecycleLogger.debug({
depPath: opts.depPath,
line: line.toString(),
stage,
stdio: stdtype,
wd: opts.pkgRoot,
})
return
case 'Returned: code:':
if (opts.stdio === 'inherit') {
// Preventing the pnpm reporter from overriding the project's script output
return
}
const code = arguments[3]
lifecycleLogger.debug({
depPath: opts.depPath,
exitCode: code,
optional,
stage,
wd: opts.pkgRoot,
})
return
}
}
}
/**
* Run node-gyp when binding.gyp is available. Only do this when there's no
* `install` script (see `npm help scripts`).

View File

@@ -0,0 +1,78 @@
import { lifecycleLogger } from '@pnpm/core-loggers'
import { PackageJson } from '@pnpm/types'
import lifecycle = require('@zkochan/npm-lifecycle')
function noop () {} // tslint:disable-line:no-empty
export default async function runLifecycleHook (
stage: string,
pkg: PackageJson,
opts: {
depPath: string,
optional?: boolean,
pkgRoot: string,
rawNpmConfig: object,
rootNodeModulesDir: string,
stdio?: string,
unsafePerm: boolean,
},
) {
const optional = opts.optional === true
if (opts.stdio !== 'inherit') {
lifecycleLogger.debug({
depPath: opts.depPath,
optional,
script: pkg.scripts![stage],
stage,
wd: opts.pkgRoot,
})
}
return lifecycle(pkg, stage, opts.pkgRoot, {
config: opts.rawNpmConfig,
dir: opts.rootNodeModulesDir,
log: {
clearProgress: noop,
info: noop,
level: opts.stdio === 'inherit' ? undefined : 'silent',
pause: noop,
resume: noop,
showProgress: noop,
silly: npmLog,
verbose: npmLog,
warn: noop,
},
runConcurrently: true,
stdio: opts.stdio || 'pipe',
unsafePerm: opts.unsafePerm,
})
function npmLog (prefix: string, logid: string, stdtype: string, line: string) {
switch (stdtype) {
case 'stdout':
case 'stderr':
lifecycleLogger.debug({
depPath: opts.depPath,
line: line.toString(),
stage,
stdio: stdtype,
wd: opts.pkgRoot,
})
return
case 'Returned: code:':
if (opts.stdio === 'inherit') {
// Preventing the pnpm reporter from overriding the project's script output
return
}
const code = arguments[3]
lifecycleLogger.debug({
depPath: opts.depPath,
exitCode: code,
optional,
stage,
wd: opts.pkgRoot,
})
return
}
}
}

View File

@@ -0,0 +1,44 @@
import { PackageJson } from '@pnpm/types'
import runGroups from 'run-groups'
import runLifecycleHook from './runLifecycleHook'
export default async function runLifecycleHooksConcurrently (
stages: string[],
importers: Array<{ buildIndex: number, pkg: PackageJson, prefix: string, modulesDir: string }>,
childConcurrency: number,
opts: {
rawNpmConfig: object,
stdio?: string,
unsafePerm: boolean,
},
) {
const importersByBuildIndex = new Map<number, Array<{ prefix: string, pkg: PackageJson, modulesDir: string }>>()
for (const importer of importers) {
if (!importersByBuildIndex.has(importer.buildIndex)) {
importersByBuildIndex.set(importer.buildIndex, [importer])
} else {
importersByBuildIndex.get(importer.buildIndex)!.push(importer)
}
}
const sortedBuildIndexes = Array.from(importersByBuildIndex.keys()).sort()
const groups = sortedBuildIndexes.map((buildIndex) => {
const importers = importersByBuildIndex.get(buildIndex) as Array<{ prefix: string, pkg: PackageJson, modulesDir: string }>
return importers.map((importer) =>
async () => {
const runLifecycleHookOpts = {
depPath: importer.prefix,
pkgRoot: importer.prefix,
rawNpmConfig: opts.rawNpmConfig,
rootNodeModulesDir: importer.modulesDir,
stdio: opts.stdio,
unsafePerm: opts.unsafePerm,
}
for (const stage of stages) {
if (!importer.pkg.scripts || !importer.pkg.scripts[stage]) continue
await runLifecycleHook(stage, importer.pkg, runLifecycleHookOpts)
}
}
)
})
await runGroups(childConcurrency, groups)
}

View File

@@ -84,6 +84,6 @@ export default async function installCmd (
if (opts.ignoreScripts) return
await rebuild({ ...opts, pending: true } as any) // tslint:disable-line:no-any
await rebuild([{ buildIndex: 0, prefix: opts.prefix }], { ...opts, pending: true } as any) // tslint:disable-line:no-any
}
}

View File

@@ -6,6 +6,7 @@ export default async (input: string[], opts: PnpmOptions) => {
const store = await createStoreController(opts)
return mutateModules([
{
buildIndex: 0,
mutation: 'install',
prefix: process.cwd(),
pruneDirectDependencies: true,

View File

@@ -1,9 +1,8 @@
import storePath from '@pnpm/store-path'
import path = require('path')
import {
rebuild,
rebuildPkgs,
} from 'supi'
import createStoreController from '../createStoreController'
import { PnpmOptions } from '../types'
export default async function (
@@ -11,12 +10,14 @@ export default async function (
opts: PnpmOptions,
command: string,
) {
const store = await createStoreController(opts)
const rebuildOpts = Object.assign(opts, {
store: await storePath(opts.prefix, opts.store),
store: store.path,
storeController: store.ctrl,
})
if (args.length === 0) {
await rebuild(rebuildOpts)
await rebuild([{ buildIndex: 0, prefix: rebuildOpts.prefix }], rebuildOpts)
}
await rebuildPkgs(args, rebuildOpts)
await rebuildPkgs([{ prefix: rebuildOpts.prefix }], args, rebuildOpts)
}

View File

@@ -206,20 +206,28 @@ export async function recursive (
const memReadLocalConfigs = mem(readLocalConfigs)
if (cmdFullName !== 'rebuild') {
let pkgPaths = chunks.length === 0
? chunks[0]
: Object.keys(pkgGraphResult.graph).sort()
if (opts.shrinkwrapDirectory && ['install', 'uninstall', 'update'].indexOf(cmdFullName) !== -1) {
function getImporters () {
const importers = [] as Array<{ buildIndex: number, prefix: string }>
chunks.forEach((prefixes: string[], buildIndex) => {
if (opts.ignoredPackages) {
pkgPaths = pkgPaths.filter((prefix) => !opts.ignoredPackages!.has(prefix))
prefixes = prefixes.filter((prefix) => !opts.ignoredPackages!.has(prefix))
}
prefixes.forEach((prefix) => {
importers.push({ buildIndex, prefix })
})
})
return importers
}
if (cmdFullName !== 'rebuild') {
if (opts.shrinkwrapDirectory && ['install', 'uninstall', 'update'].indexOf(cmdFullName) !== -1) {
let importers = getImporters()
const isFromWorkspace = isSubdir.bind(null, opts.shrinkwrapDirectory)
pkgPaths = await pFilter(pkgPaths, async (pkgPath: string) => isFromWorkspace(await fs.realpath(pkgPath)))
if (pkgPaths.length === 0) return true
importers = await pFilter(importers, async ({ prefix }: { prefix: string }) => isFromWorkspace(await fs.realpath(prefix)))
if (importers.length === 0) return true
const hooks = opts.ignorePnpmfile ? {} : requireHooks(opts.shrinkwrapDirectory, opts)
const mutation = cmdFullName === 'uninstall' ? 'uninstallSome' : (input.length === 0 ? 'install' : 'installSome')
const importers = await Promise.all<MutatedImporter>(pkgPaths.map(async (prefix) => {
const mutatedImporters = await Promise.all<MutatedImporter>(importers.map(async ({ buildIndex, prefix }) => {
const localConfigs = await memReadLocalConfigs(prefix)
const shamefullyFlatten = typeof localConfigs.shamefullyFlatten === 'boolean'
? localConfigs.shamefullyFlatten
@@ -248,13 +256,14 @@ export async function recursive (
} as MutatedImporter
case 'install':
return {
buildIndex,
mutation,
prefix,
shamefullyFlatten,
} as MutatedImporter
}
}))
await mutateModules(importers, {
await mutateModules(mutatedImporters, {
...installOpts,
hooks,
storeController: store.ctrl,
@@ -262,6 +271,10 @@ export async function recursive (
return true
}
let pkgPaths = chunks.length === 0
? chunks[0]
: Object.keys(pkgGraphResult.graph).sort()
let action!: any // tslint:disable-line:no-any
switch (cmdFullName) {
case 'unlink':
@@ -322,8 +335,23 @@ export async function recursive (
cmdFullName === 'rebuild' ||
!opts.shrinkwrapOnly && !opts.ignoreScripts && (cmdFullName === 'install' || cmdFullName === 'update' || cmdFullName === 'unlink')
) {
const action = (
cmdFullName !== 'rebuild' || input.length === 0
? rebuild
: (importers: any, opts: any) => rebuildPkgs(importers, input, opts) // tslint:disable-line
)
if (opts.shrinkwrapDirectory) {
const importers = getImporters()
await action(
importers,
{
...installOpts,
pending: cmdFullName !== 'rebuild' || opts.pending === true,
},
)
return true
}
const limitRebuild = pLimit(opts.workspaceConcurrency)
const action = (cmdFullName !== 'rebuild' || input.length === 0 ? rebuild : rebuildPkgs.bind(null, input))
for (const chunk of chunks) {
await Promise.all(chunk.map((prefix: string) =>
limitRebuild(async () => {
@@ -332,17 +360,20 @@ export async function recursive (
return
}
const localConfigs = await memReadLocalConfigs(prefix)
await action({
...installOpts,
...localConfigs,
bin: path.join(prefix, 'node_modules', '.bin'),
pending: cmdFullName !== 'rebuild' || opts.pending === true,
prefix,
rawNpmConfig: {
...installOpts.rawNpmConfig,
...localConfigs.rawNpmConfig,
await action(
[{ buildIndex: 0, prefix }],
{
...installOpts,
...localConfigs,
bin: path.join(prefix, 'node_modules', '.bin'),
pending: cmdFullName !== 'rebuild' || opts.pending === true,
prefix,
rawNpmConfig: {
...installOpts.rawNpmConfig,
...localConfigs.rawNpmConfig,
},
},
})
)
result.passes++
} catch (err) {
logger.info(err)

View File

@@ -463,6 +463,131 @@ test('recursive install with link-workspace-packages and shared-workspace-shrink
}
})
test('recursive install with shared-workspace-shrinkwrap builds workspace packages in correct order', async (t: tape.Test) => {
const jsonAppend = (append: string, target: string) => `node -e "process.stdout.write('${append}')" | json-append ${target}`
const projects = preparePackages(t, [
{
name: 'project-999',
version: '1.0.0',
dependencies: {
'json-append': '1',
},
scripts: {
install: `${jsonAppend('project-999-install', '../output1.json')} && ${jsonAppend('project-999-install', '../output2.json')}`,
postinstall: `${jsonAppend('project-999-postinstall', '../output1.json')} && ${jsonAppend('project-999-postinstall', '../output2.json')}`,
prepare: `${jsonAppend('project-999-prepare', '../output1.json')} && ${jsonAppend('project-999-prepare', '../output2.json')}`,
prepublish: `${jsonAppend('project-999-prepublish', '../output1.json')} && ${jsonAppend('project-999-prepublish', '../output2.json')}`,
},
},
{
name: 'project-1',
version: '1.0.0',
devDependencies: {
'json-append': '1',
'project-999': '1.0.0',
},
scripts: {
install: jsonAppend('project-1-install', '../output1.json'),
postinstall: jsonAppend('project-1-postinstall', '../output1.json'),
prepare: jsonAppend('project-1-prepare', '../output1.json'),
prepublish: jsonAppend('project-1-prepublish', '../output1.json'),
},
},
{
name: 'project-2',
version: '1.0.0',
devDependencies: {
'json-append': '1',
'project-999': '1.0.0',
},
scripts: {
install: jsonAppend('project-2-install', '../output2.json'),
postinstall: jsonAppend('project-2-postinstall', '../output2.json'),
prepare: jsonAppend('project-2-prepare', '../output2.json'),
prepublish: jsonAppend('project-2-prepublish', '../output2.json'),
},
},
])
await writeYamlFile('pnpm-workspace.yaml', { packages: ['**', '!store/**'] })
await execPnpm('recursive', 'install', '--link-workspace-packages', '--shared-workspace-shrinkwrap=true', '--store', 'store')
{
const outputs1 = await import(path.resolve('output1.json')) as string[]
t.deepEqual(
outputs1,
[
'project-999-install',
'project-999-postinstall',
'project-999-prepublish',
'project-999-prepare',
'project-1-install',
'project-1-postinstall',
'project-1-prepublish',
'project-1-prepare',
],
)
const outputs2 = await import(path.resolve('output2.json')) as string[]
t.deepEqual(
outputs2,
[
'project-999-install',
'project-999-postinstall',
'project-999-prepublish',
'project-999-prepare',
'project-2-install',
'project-2-postinstall',
'project-2-prepublish',
'project-2-prepare',
],
)
}
await rimraf('node_modules')
await rimraf('output1.json')
await rimraf('output2.json')
// TODO: duplicate this test in @pnpm/headless
await execPnpm('recursive', 'install', '--frozen-shrinkwrap', '--link-workspace-packages', '--shared-workspace-shrinkwrap=true')
{
const outputs1 = await import(path.resolve('output1.json')) as string[]
t.deepEqual(
outputs1,
[
'project-999-install',
'project-999-postinstall',
'project-999-prepublish',
'project-999-prepare',
'project-1-install',
'project-1-postinstall',
'project-1-prepublish',
'project-1-prepare',
],
)
const outputs2 = await import(path.resolve('output2.json')) as string[]
t.deepEqual(
outputs2,
[
'project-999-install',
'project-999-postinstall',
'project-999-prepublish',
'project-999-prepare',
'project-2-install',
'project-2-postinstall',
'project-2-prepublish',
'project-2-prepare',
],
)
}
})
test('recursive installation with shared-workspace-shrinkwrap and a readPackage hook', async (t) => {
const projects = preparePackages(t, [
{

View File

@@ -2,7 +2,7 @@ import logger from '@pnpm/logger'
import pkgIdToFilename from '@pnpm/pkgid-to-filename'
import {
nameVerFromPkgSnapshot,
PackageSnapshot,
packageIsIndependent,
PackageSnapshots,
Shrinkwrap,
} from '@pnpm/shrinkwrap-utils'
@@ -80,7 +80,7 @@ async function getDependencies (
const absolutePath = dp.resolve(opts.registry, depRelPath)
const pkgName = nameVerFromPkgSnapshot(depRelPath, pkgSnapshot).name
const modules = path.join(opts.virtualStoreDir, `.${pkgIdToFilename(absolutePath, opts.prefix)}`, 'node_modules')
const independent = opts.getIndependentPackageLocation && pkgIsIndependent(pkgSnapshot)
const independent = opts.getIndependentPackageLocation && packageIsIndependent(pkgSnapshot)
const allDeps = {
...pkgSnapshot.dependencies,
...pkgSnapshot.optionalDependencies,
@@ -115,10 +115,6 @@ async function getDependencies (
]
}
function pkgIsIndependent (pkgSnapshot: PackageSnapshot) {
return pkgSnapshot.dependencies === undefined && pkgSnapshot.optionalDependencies === undefined
}
export interface Dependency {
name: string,
location: string,

View File

@@ -1,11 +1,13 @@
export * from '@pnpm/shrinkwrap-types'
import nameVerFromPkgSnapshot from './nameVerFromPkgSnapshot'
import packageIsIndependent from './packageIsIndependent'
import pkgSnapshotToResolution from './pkgSnapshotToResolution'
import satisfiesPackageJson from './satisfiesPackageJson'
export {
nameVerFromPkgSnapshot,
packageIsIndependent,
pkgSnapshotToResolution,
satisfiesPackageJson,
}

View File

@@ -0,0 +1,5 @@
import { PackageSnapshot } from '@pnpm/shrinkwrap-types'
export default ({ dependencies, optionalDependencies }: PackageSnapshot) => {
return dependencies === undefined && optionalDependencies === undefined
}

View File

@@ -80,6 +80,7 @@
"replace-string": "2.0.0",
"resolve-link-target": "1.0.1",
"rimraf-then": "1.0.1",
"run-groups": "1.0.0",
"semver": "5.6.0",
"symlink-dir": "2.0.2",
"util.promisify": "1.0.0",

View File

@@ -1,4 +1,3 @@
import logger from '@pnpm/logger'
import { IncludedDependencies } from '@pnpm/modules-yaml'
import { LocalPackages } from '@pnpm/resolver-base'
import { Shrinkwrap } from '@pnpm/shrinkwrap-file'

View File

@@ -5,7 +5,10 @@ import {
summaryLogger,
} from '@pnpm/core-loggers'
import headless from '@pnpm/headless'
import runLifecycleHooks, { runPostinstallHooks } from '@pnpm/lifecycle'
import {
runLifecycleHooksConcurrently,
runPostinstallHooks,
} from '@pnpm/lifecycle'
import logger, {
streamParser,
} from '@pnpm/logger'
@@ -41,10 +44,10 @@ import isInnerLink = require('is-inner-link')
import isSubdir = require('is-subdir')
import pEvery = require('p-every')
import pFilter = require('p-filter')
import pLimit = require('p-limit')
import path = require('path')
import R = require('ramda')
import rimraf = require('rimraf-then')
import runGroups from 'run-groups'
import semver = require('semver')
import {
ENGINE_NAME,
@@ -73,6 +76,7 @@ import linkPackages, {
import { absolutePathToRef } from './shrinkwrap'
export type DependenciesMutation = {
buildIndex: number,
mutation: 'install',
pruneDirectDependencies?: boolean,
} | {
@@ -103,7 +107,16 @@ export function install (
},
},
) {
return mutateModules([{ prefix: opts.prefix || process.cwd(), mutation: 'install' }], opts)
return mutateModules(
[
{
buildIndex: 0,
mutation: 'install',
prefix: opts.prefix || process.cwd(),
},
],
opts,
)
}
export type MutatedImporter = ImportersOptions & DependenciesMutation
@@ -185,7 +198,17 @@ export async function mutateModules (
currentShrinkwrap: ctx.currentShrinkwrap,
force: opts.force,
ignoreScripts: opts.ignoreScripts,
importers: ctx.importers,
importers: ctx.importers as Array<{
bin: string,
buildIndex: number,
hoistedAliases: {[depPath: string]: string[]}
id: string,
modulesDir: string,
pkg: PackageJson,
prefix: string,
pruneDirectDependencies?: boolean,
shamefullyFlatten: boolean,
}>,
include: opts.include,
independentLeaves: opts.independentLeaves,
ownLifecycleHooksStdio: opts.ownLifecycleHooksStdio,
@@ -209,6 +232,22 @@ export async function mutateModules (
}
const importersToInstall = [] as ImporterToUpdate[]
const importersToBeInstalled = ctx.importers.filter((importer) => importer.mutation === 'install') as Array<{ buildIndex: number, prefix: string, pkg: PackageJson, modulesDir: string }>
const scriptsOpts = {
rawNpmConfig: opts.rawNpmConfig,
stdio: opts.ownLifecycleHooksStdio,
unsafePerm: opts.unsafePerm || false,
}
if (!opts.ignoreScripts) {
await runLifecycleHooksConcurrently(
['preinstall'],
importersToBeInstalled,
opts.childConcurrency,
scriptsOpts,
)
}
// TODO: make it concurrent
for (const importer of ctx.importers) {
switch (importer.mutation) {
@@ -328,20 +367,6 @@ export async function mutateModules (
})
}
const scriptsOpts = {
depPath: importer.prefix,
optional: false,
pkgRoot: importer.prefix,
rawNpmConfig: opts.rawNpmConfig,
rootNodeModulesDir: importer.modulesDir,
stdio: opts.ownLifecycleHooksStdio,
unsafePerm: opts.unsafePerm || false,
}
if (scripts.preinstall) {
await runLifecycleHooks('preinstall', importer.pkg, scriptsOpts)
}
importersToInstall.push({
pruneDirectDependencies: false,
...importer,
@@ -375,33 +400,12 @@ export async function mutateModules (
updateShrinkwrapMinorVersion: true,
})
for (const importer of ctx.importers) {
if (importer.mutation !== 'install') continue
const scripts = !opts.ignoreScripts && importer.pkg && importer.pkg.scripts || {}
const scriptsOpts = {
depPath: importer.prefix,
optional: false,
pkgRoot: importer.prefix,
rawNpmConfig: opts.rawNpmConfig,
rootNodeModulesDir: importer.modulesDir,
stdio: opts.ownLifecycleHooksStdio,
unsafePerm: opts.unsafePerm || false,
}
if (scripts.install) {
await runLifecycleHooks('install', importer.pkg, scriptsOpts)
}
if (scripts.postinstall) {
await runLifecycleHooks('postinstall', importer.pkg, scriptsOpts)
}
if (scripts.prepublish) {
await runLifecycleHooks('prepublish', importer.pkg, scriptsOpts)
}
if (scripts.prepare) {
await runLifecycleHooks('prepare', importer.pkg, scriptsOpts)
}
if (!opts.ignoreScripts) {
await runLifecycleHooksConcurrently(['install', 'postinstall', 'prepublish', 'prepare'],
importersToBeInstalled,
opts.childConcurrency,
scriptsOpts,
)
}
}
}
@@ -838,8 +842,6 @@ async function installInContext (
// postinstall hooks
if (!(opts.ignoreScripts || !result.newDepPaths || !result.newDepPaths.length)) {
const limitChild = pLimit(opts.childConcurrency)
const depPaths = Object.keys(result.depGraph)
const rootNodes = depPaths.filter((depPath) => result.depGraph[depPath].depth === 0)
const nodesToBuild = new Set<string>()
@@ -856,63 +858,61 @@ async function installInContext (
groups: [nodesToBuildArray],
})
const chunks = graphSequencerResult.chunks as string[][]
for (const chunk of chunks) {
await Promise.all(chunk
.filter((depPath) => result.depGraph[depPath].requiresBuild && !result.depGraph[depPath].isBuilt && result.newDepPaths.indexOf(depPath) !== -1)
.map((depPath) => result.depGraph[depPath])
.map((pkg) => limitChild(async () => {
try {
const hasSideEffects = await runPostinstallHooks({
depPath: pkg.absolutePath,
optional: pkg.optional,
pkgRoot: pkg.peripheralLocation,
prepare: pkg.prepare,
rawNpmConfig: opts.rawNpmConfig,
rootNodeModulesDir: ctx.virtualStoreDir,
unsafePerm: opts.unsafePerm || false,
})
if (hasSideEffects && opts.sideEffectsCacheWrite) {
try {
await opts.storeController.upload(pkg.peripheralLocation, {
engine: ENGINE_NAME,
pkgId: pkg.id,
const groups = chunks.map((chunk) => chunk
.filter((depPath) => result.depGraph[depPath].requiresBuild && !result.depGraph[depPath].isBuilt && result.newDepPaths.indexOf(depPath) !== -1)
.map((depPath) => result.depGraph[depPath])
.map((pkg) => async () => {
try {
const hasSideEffects = await runPostinstallHooks({
depPath: pkg.absolutePath,
optional: pkg.optional,
pkgRoot: pkg.peripheralLocation,
prepare: pkg.prepare,
rawNpmConfig: opts.rawNpmConfig,
rootNodeModulesDir: ctx.virtualStoreDir,
unsafePerm: opts.unsafePerm || false,
})
if (hasSideEffects && opts.sideEffectsCacheWrite) {
try {
await opts.storeController.upload(pkg.peripheralLocation, {
engine: ENGINE_NAME,
pkgId: pkg.id,
})
} catch (err) {
if (err && err.statusCode === 403) {
logger.warn({
message: `The store server disabled upload requests, could not upload ${pkg.id}`,
prefix: ctx.shrinkwrapDirectory,
})
} else {
logger.warn({
error: err,
message: `An error occurred while uploading ${pkg.id}`,
prefix: ctx.shrinkwrapDirectory,
})
} catch (err) {
if (err && err.statusCode === 403) {
logger.warn({
message: `The store server disabled upload requests, could not upload ${pkg.id}`,
prefix: ctx.shrinkwrapDirectory,
})
} else {
logger.warn({
error: err,
message: `An error occurred while uploading ${pkg.id}`,
prefix: ctx.shrinkwrapDirectory,
})
}
}
}
} catch (err) {
if (resolvedPackagesByPackageId[pkg.id].optional) {
// TODO: add parents field to the log
skippedOptionalDependencyLogger.debug({
details: err.toString(),
package: {
id: pkg.id,
name: pkg.name,
version: pkg.version,
},
prefix: opts.shrinkwrapDirectory,
reason: 'build_failure',
})
return
}
throw err
}
},
)))
}
} catch (err) {
if (resolvedPackagesByPackageId[pkg.id].optional) {
// TODO: add parents field to the log
skippedOptionalDependencyLogger.debug({
details: err.toString(),
package: {
id: pkg.id,
name: pkg.name,
version: pkg.version,
},
prefix: opts.shrinkwrapDirectory,
reason: 'build_failure',
})
return
}
throw err
}
}),
)
await runGroups(opts.childConcurrency, groups)
}
}

View File

@@ -1,3 +1,4 @@
import { StoreController } from '@pnpm/store-controller-types'
import { Registries } from '@pnpm/types'
import { DEFAULT_REGISTRIES, normalizeRegistries } from '@pnpm/utils'
import path = require('path')
@@ -8,7 +9,9 @@ export interface RebuildOptions {
childConcurrency?: number,
prefix?: string,
shrinkwrapDirectory?: string,
sideEffectsCacheRead?: boolean,
store: string, // TODO: remove this property
storeController: StoreController,
independentLeaves?: boolean,
force?: boolean,
forceSharedShrinkwrap?: boolean,
@@ -36,6 +39,7 @@ export type StrictRebuildOptions = RebuildOptions & {
prefix: string,
store: string,
shrinkwrapDirectory: string,
sideEffectsCacheRead: boolean,
independentLeaves: boolean,
force: boolean,
forceSharedShrinkwrap: boolean,
@@ -77,6 +81,7 @@ const defaults = async (opts: RebuildOptions) => {
shamefullyFlatten: false,
shrinkwrap: true,
shrinkwrapDirectory,
sideEffectsCacheRead: false,
store: opts.store,
unsafePerm: process.platform === 'win32' ||
process.platform === 'cygwin' ||

View File

@@ -1,22 +1,28 @@
import { skippedOptionalDependencyLogger } from '@pnpm/core-loggers'
import runLifecycleHooks, { runPostinstallHooks } from '@pnpm/lifecycle'
import {
runLifecycleHooksConcurrently,
runPostinstallHooks,
} from '@pnpm/lifecycle'
import logger, { streamParser } from '@pnpm/logger'
import { write as writeModulesYaml } from '@pnpm/modules-yaml'
import {
nameVerFromPkgSnapshot,
packageIsIndependent,
PackageSnapshots,
Shrinkwrap,
} from '@pnpm/shrinkwrap-utils'
import { PackageJson } from '@pnpm/types'
import npa = require('@zkochan/npm-package-arg')
import * as dp from 'dependency-path'
import graphSequencer = require('graph-sequencer')
import pLimit = require('p-limit')
import path = require('path')
import R = require('ramda')
import runGroups from 'run-groups'
import semver = require('semver')
import { LAYOUT_VERSION } from '../constants'
import { getContextForSingleImporter } from '../getContext'
import {
ENGINE_NAME,
LAYOUT_VERSION,
} from '../constants'
import getContext from '../getContext'
import extendOptions, {
RebuildOptions,
StrictRebuildOptions,
@@ -65,6 +71,7 @@ type PackageSelector = string | {
}
export async function rebuildPkgs (
importers: Array<{ prefix: string }>,
pkgSpecs: string[],
maybeOpts: RebuildOptions,
) {
@@ -73,7 +80,7 @@ export async function rebuildPkgs (
streamParser.on('data', reporter)
}
const opts = await extendOptions(maybeOpts)
const ctx = await getContextForSingleImporter(opts)
const ctx = await getContext(importers, opts)
if (!ctx.currentShrinkwrap || !ctx.currentShrinkwrap.packages) return
const packages = ctx.currentShrinkwrap.packages
@@ -92,18 +99,33 @@ export async function rebuildPkgs (
}
})
const pkgs = findPackages(packages, searched, { prefix: ctx.prefix })
let pkgs = [] as string[]
for (const importer of importers) {
pkgs = [
...pkgs,
...findPackages(packages, searched, { prefix: importer.prefix }),
]
}
await _rebuild(new Set(pkgs), ctx.virtualStoreDir, ctx.currentShrinkwrap, ctx.importerId, opts)
await _rebuild(
new Set(pkgs),
ctx.virtualStoreDir,
ctx.currentShrinkwrap,
ctx.importers.map((importer) => importer.id),
opts,
)
}
export async function rebuild (maybeOpts: RebuildOptions) {
export async function rebuild (
importers: Array<{ buildIndex: number, prefix: string }>,
maybeOpts: RebuildOptions,
) {
const reporter = maybeOpts && maybeOpts.reporter
if (reporter) {
streamParser.on('data', reporter)
}
const opts = await extendOptions(maybeOpts)
const ctx = await getContextForSingleImporter(opts)
const ctx = await getContext(importers, opts)
let idsToRebuild: string[] = []
@@ -116,28 +138,43 @@ export async function rebuild (maybeOpts: RebuildOptions) {
}
if (idsToRebuild.length === 0) return
const pkgsThatWereRebuilt = await _rebuild(new Set(idsToRebuild), ctx.virtualStoreDir, ctx.currentShrinkwrap, ctx.importerId, opts)
const pkgsThatWereRebuilt = await _rebuild(
new Set(idsToRebuild),
ctx.virtualStoreDir,
ctx.currentShrinkwrap,
ctx.importers.map((importer) => importer.id),
opts,
)
ctx.pendingBuilds = ctx.pendingBuilds.filter((relDepPath) => !pkgsThatWereRebuilt.has(relDepPath))
if (ctx.pkg && ctx.pkg.scripts && (!opts.pending || ctx.pendingBuilds.indexOf(ctx.importerId) !== -1)) {
await runLifecycleHooksInDir(opts.prefix, ctx.pkg, {
rawNpmConfig: opts.rawNpmConfig,
rootNodeModulesDir: ctx.modulesDir,
unsafePerm: opts.unsafePerm,
})
ctx.pendingBuilds.splice(ctx.pendingBuilds.indexOf(ctx.importerId), 1)
const scriptsOpts = {
rawNpmConfig: opts.rawNpmConfig,
unsafePerm: opts.unsafePerm || false,
}
await runLifecycleHooksConcurrently(
['preinstall', 'install', 'postinstall', 'prepublish', 'prepare'],
ctx.importers,
opts.childConcurrency || 5,
scriptsOpts,
)
for (const importer of ctx.importers) {
if (importer.pkg && importer.pkg.scripts && (!opts.pending || ctx.pendingBuilds.indexOf(importer.id) !== -1)) {
ctx.pendingBuilds.splice(ctx.pendingBuilds.indexOf(importer.id), 1)
}
}
await writeModulesYaml(ctx.virtualStoreDir, {
...ctx.modulesFile,
importers: {
...ctx.modulesFile && ctx.modulesFile.importers,
[ctx.importerId]: {
hoistedAliases: ctx.hoistedAliases,
shamefullyFlatten: opts.shamefullyFlatten,
},
...ctx.importers.reduce((acc, importer) => {
acc[importer.id] = {
hoistedAliases: importer.hoistedAliases,
shamefullyFlatten: importer.shamefullyFlatten,
}
return acc
}, {}),
},
included: ctx.include,
independentLeaves: opts.independentLeaves,
@@ -150,40 +187,6 @@ export async function rebuild (maybeOpts: RebuildOptions) {
})
}
async function runLifecycleHooksInDir (
prefix: string,
pkg: PackageJson,
opts: {
rawNpmConfig: object,
rootNodeModulesDir: string,
unsafePerm: boolean,
},
) {
const scriptsOpts = {
depPath: prefix,
optional: false,
pkgRoot: prefix,
rawNpmConfig: opts.rawNpmConfig,
rootNodeModulesDir: opts.rootNodeModulesDir,
unsafePerm: opts.unsafePerm || false,
}
if (pkg.scripts!.preinstall) {
await runLifecycleHooks('preinstall', pkg, scriptsOpts)
}
if (pkg.scripts!.install) {
await runLifecycleHooks('install', pkg, scriptsOpts)
}
if (pkg.scripts!.postinstall) {
await runLifecycleHooks('postinstall', pkg, scriptsOpts)
}
if (pkg.scripts!.prepublish) {
await runLifecycleHooks('prepublish', pkg, scriptsOpts)
}
if (pkg.scripts!.prepare) {
await runLifecycleHooks('prepare', pkg, scriptsOpts)
}
}
function getSubgraphToBuild (
pkgSnapshots: PackageSnapshots,
entryNodes: string[],
@@ -231,22 +234,28 @@ async function _rebuild (
pkgsToRebuild: Set<string>,
modules: string,
shr: Shrinkwrap,
importerId: string,
importerIds: string[],
opts: StrictRebuildOptions,
) {
const pkgsThatWereRebuilt = new Set()
const limitChild = pLimit(opts.childConcurrency)
const graph = new Map()
const pkgSnapshots: PackageSnapshots = shr.packages || {}
const shrImporter = shr.importers[importerId]
const entryNodes = R.toPairs({
...(opts.development && shrImporter.devDependencies || {}),
...(opts.production && shrImporter.dependencies || {}),
...(opts.optional && shrImporter.optionalDependencies || {}),
const entryNodes = [] as string[]
importerIds.forEach((importerId) => {
const shrImporter = shr.importers[importerId]
R.toPairs({
...(opts.development && shrImporter.devDependencies || {}),
...(opts.production && shrImporter.dependencies || {}),
...(opts.optional && shrImporter.optionalDependencies || {}),
})
.map((pair) => dp.refToRelative(pair[1], pair[0]))
.filter((nodeId) => nodeId !== null)
.forEach((relDepPath) => {
entryNodes.push(relDepPath as string)
})
})
.map((pair) => dp.refToRelative(pair[1], pair[0]))
.filter((nodeId) => nodeId !== null) as string[]
const nodesToBuildAndTransitive = new Set()
getSubgraphToBuild(pkgSnapshots, entryNodes, nodesToBuildAndTransitive, new Set(), { optional: opts.optional === true, pkgsToRebuild })
@@ -264,46 +273,55 @@ async function _rebuild (
})
const chunks = graphSequencerResult.chunks as string[][]
for (const chunk of chunks) {
await Promise.all(chunk
.filter((relDepPath) => pkgsToRebuild.has(relDepPath))
.map((relDepPath) => {
const pkgSnapshot = pkgSnapshots[relDepPath]
return limitChild(async () => {
const depAbsolutePath = dp.resolve(opts.registries.default, relDepPath)
const pkgInfo = nameVerFromPkgSnapshot(relDepPath, pkgSnapshot)
try {
await runPostinstallHooks({
depPath: depAbsolutePath,
optional: pkgSnapshot.optional === true,
pkgRoot: path.join(modules, `.${depAbsolutePath}`, 'node_modules', pkgInfo.name),
prepare: pkgSnapshot.prepare,
rawNpmConfig: opts.rawNpmConfig,
rootNodeModulesDir: modules,
unsafePerm: opts.unsafePerm || false,
const groups = chunks.map((chunk) => chunk.filter((relDepPath) => pkgsToRebuild.has(relDepPath)).map((relDepPath) =>
async () => {
const pkgSnapshot = pkgSnapshots[relDepPath]
const depPath = dp.resolve(opts.registries.default, relDepPath)
const pkgInfo = nameVerFromPkgSnapshot(relDepPath, pkgSnapshot)
const independent = opts.independentLeaves && packageIsIndependent(pkgSnapshot)
const pkgRoot = !independent
? path.join(modules, `.${depPath}`, 'node_modules', pkgInfo.name)
: await (
async () => {
const { directory } = await opts.storeController.getPackageLocation(pkgSnapshot.id || depPath, pkgInfo.name, {
importerPrefix: opts.prefix,
targetEngine: opts.sideEffectsCacheRead && !opts.force && ENGINE_NAME || undefined,
})
pkgsThatWereRebuilt.add(relDepPath)
} catch (err) {
if (pkgSnapshot.optional) {
// TODO: add parents field to the log
skippedOptionalDependencyLogger.debug({
details: err.toString(),
package: {
id: pkgSnapshot.id || depAbsolutePath,
name: pkgInfo.name,
version: pkgInfo.version,
},
prefix: opts.prefix,
reason: 'build_failure',
})
return
}
throw err
return directory
}
)()
try {
await runPostinstallHooks({
depPath,
optional: pkgSnapshot.optional === true,
pkgRoot,
prepare: pkgSnapshot.prepare,
rawNpmConfig: opts.rawNpmConfig,
rootNodeModulesDir: modules,
unsafePerm: opts.unsafePerm || false,
})
}),
)
}
pkgsThatWereRebuilt.add(relDepPath)
} catch (err) {
if (pkgSnapshot.optional) {
// TODO: add parents field to the log
skippedOptionalDependencyLogger.debug({
details: err.toString(),
package: {
id: pkgSnapshot.id || depPath,
name: pkgInfo.name,
version: pkgInfo.version,
},
prefix: opts.prefix,
reason: 'build_failure',
})
return
}
throw err
}
}
))
await runGroups(opts.childConcurrency || 5, groups)
return pkgsThatWereRebuilt
}

View File

@@ -80,10 +80,12 @@ test('frozen-shrinkwrap: fail on a shared shrinkwrap.yaml that does not satisfy
const importers: MutatedImporter[] = [
{
buildIndex: 0,
mutation: 'install',
prefix: path.resolve('p1'),
},
{
buildIndex: 0,
mutation: 'install',
prefix: path.resolve('p2'),
},
@@ -247,10 +249,12 @@ test('prefer-frozen-shrinkwrap: should prefer frozen-shrinkwrap when package has
const importers: MutatedImporter[] = [
{
buildIndex: 0,
mutation: 'install',
prefix: path.resolve('p1'),
},
{
buildIndex: 0,
mutation: 'install',
prefix: path.resolve('p2'),
},

View File

@@ -38,10 +38,12 @@ test('install only the dependencies of the specified importer', async (t) => {
const importers: MutatedImporter[] = [
{
buildIndex: 0,
mutation: 'install',
prefix: path.resolve('project-1'),
},
{
buildIndex: 0,
mutation: 'install',
prefix: path.resolve('project-2'),
},
@@ -80,10 +82,12 @@ test('dependencies of other importers are not pruned when installing for a subse
await mutateModules([
{
buildIndex: 0,
mutation: 'install',
prefix: path.resolve('project-1'),
},
{
buildIndex: 0,
mutation: 'install',
prefix: path.resolve('project-2'),
},
@@ -131,10 +135,12 @@ test('dependencies of other importers are not pruned when (headless) installing
const importers: MutatedImporter[] = [
{
buildIndex: 0,
mutation: 'install',
prefix: path.resolve('project-1'),
},
{
buildIndex: 0,
mutation: 'install',
prefix: path.resolve('project-2'),
},
@@ -171,6 +177,7 @@ test('adding a new dev dependency to project that uses a shared shrinkwrap', asy
await mutateModules([
{
buildIndex: 0,
mutation: 'install',
prefix: path.resolve('project-1'),
},
@@ -206,10 +213,12 @@ test('headless install is used when package link to another package in the works
const importers: MutatedImporter[] = [
{
buildIndex: 0,
mutation: 'install',
prefix: path.resolve('project-1'),
},
{
buildIndex: 0,
mutation: 'install',
prefix: path.resolve('project-2'),
},
@@ -252,6 +261,7 @@ test('current shrinkwrap contains only installed dependencies when adding a new
await mutateModules([
{
buildIndex: 0,
mutation: 'install',
prefix: path.resolve('project-1'),
},
@@ -259,6 +269,7 @@ test('current shrinkwrap contains only installed dependencies when adding a new
await mutateModules([
{
buildIndex: 0,
mutation: 'install',
prefix: path.resolve('project-2'),
},

View File

@@ -292,7 +292,10 @@ test('rebuild should not fail on incomplete shrinkwrap.yaml', async (t: tape.Tes
const reporter = sinon.spy()
await rebuild(await testDefaults({ pending: true, reporter }))
await rebuild([{
buildIndex: 0,
prefix: process.cwd(),
}], await testDefaults({ pending: true, reporter }))
t.ok(reporter.calledWithMatch({
level: 'debug',

View File

@@ -41,6 +41,7 @@ test('prune removes extraneous packages', async (t: tape.Test) => {
await mutateModules(
[
{
buildIndex: 0,
mutation: 'install',
prefix: process.cwd(),
pruneDirectDependencies: true,

View File

@@ -1,9 +1,10 @@
import prepare from '@pnpm/prepare'
import prepare, { preparePackages } from '@pnpm/prepare'
import ncpCB = require('ncp')
import path = require('path')
import exists = require('path-exists')
import {
addDependenciesToPackage,
mutateModules,
rebuild,
rebuildPkgs,
} from 'supi'
@@ -13,10 +14,11 @@ import promisify = require('util.promisify')
import {
pathToLocalPkg,
testDefaults,
} from './utils'
} from './utils'
const ncp = promisify(ncpCB.ncp)
const test = promisifyTape(tape)
const testOnly = promisifyTape(tape.only)
test('rebuilds dependencies', async (t: tape.Test) => {
const project = prepare(t)
@@ -30,7 +32,7 @@ test('rebuilds dependencies', async (t: tape.Test) => {
'github.com/zkochan/install-scripts-example/2de638b8b572cd1e87b74f4540754145fb2c0ebb',
])
await rebuild(await testDefaults())
await rebuild([{ buildIndex: 0, prefix: process.cwd() }], await testDefaults())
modules = await project.loadModules()
t.ok(modules)
@@ -62,7 +64,7 @@ test('rebuild does not fail when a linked package is present', async (t: tape.Te
await addDependenciesToPackage(['link:../local-pkg', 'is-positive'], await testDefaults())
await rebuild(await testDefaults())
await rebuild([{ buildIndex: 0, prefix: process.cwd() }], await testDefaults())
// see related issue https://github.com/pnpm/pnpm/issues/1155
t.pass('rebuild did not fail')
@@ -75,7 +77,7 @@ test('rebuilds specific dependencies', async (t: tape.Test) => {
'zkochan/install-scripts-example'
], await testDefaults({ targetDependenciesField: 'devDependencies', ignoreScripts: true }))
await rebuildPkgs(['install-scripts-example-for-pnpm'], await testDefaults())
await rebuildPkgs([{ prefix: process.cwd() }], ['install-scripts-example-for-pnpm'], await testDefaults())
await project.hasNot('pre-and-postinstall-scripts-example/generated-by-preinstall')
await project.hasNot('pre-and-postinstall-scripts-example/generated-by-postinstall')
@@ -104,7 +106,7 @@ test('rebuild with pending option', async (t: tape.Test) => {
await project.hasNot('install-scripts-example-for-pnpm/generated-by-preinstall')
await project.hasNot('install-scripts-example-for-pnpm/generated-by-postinstall')
await rebuild(await testDefaults({ rawNpmConfig: { pending: true } }))
await rebuild([{ buildIndex: 0, prefix: process.cwd() }], await testDefaults({ rawNpmConfig: { pending: true } }))
modules = await project.loadModules()
t.ok(modules)
@@ -139,7 +141,7 @@ test('rebuild dependencies in correct order', async (t: tape.Test) => {
await project.hasNot('.localhost+4873/with-postinstall-b/1.0.0/node_modules/with-postinstall-b/output.json')
await project.hasNot('with-postinstall-a/output.json')
await rebuild(await testDefaults({ rawNpmConfig: { pending: true } }))
await rebuild([{ buildIndex: 0, prefix: process.cwd() }], await testDefaults({ rawNpmConfig: { pending: true } }))
modules = await project.loadModules()
t.ok(modules)
@@ -147,3 +149,101 @@ test('rebuild dependencies in correct order', async (t: tape.Test) => {
t.ok(+project.requireModule('.localhost+4873/with-postinstall-b/1.0.0/node_modules/with-postinstall-b/output.json')[0] < +project.requireModule('with-postinstall-a/output.json')[0])
})
test('rebuild dependencies in correct order when node_modules uses independent-leaves', async (t: tape.Test) => {
const project = prepare(t)
await addDependenciesToPackage(['with-postinstall-a'], await testDefaults({ ignoreScripts: true, independentLeaves: true }))
let modules = await project.loadModules()
t.ok(modules)
t.doesNotEqual(modules!.pendingBuilds.length, 0)
await project.hasNot('.localhost+4873/with-postinstall-b/1.0.0/node_modules/with-postinstall-b/output.json')
await project.hasNot('with-postinstall-a/output.json')
await rebuild([{ buildIndex: 0, prefix: process.cwd() }], await testDefaults({ rawNpmConfig: { pending: true }, independentLeaves: true }))
modules = await project.loadModules()
t.ok(modules)
t.equal(modules!.pendingBuilds.length, 0)
t.ok(+project.requireModule('.localhost+4873/with-postinstall-b/1.0.0/node_modules/with-postinstall-b/output.json')[0] < +project.requireModule('with-postinstall-a/output.json')[0])
})
test('rebuild multiple packages in correct order', async (t: tape.Test) => {
const projects = preparePackages(t, [
{
name: 'project-1',
version: '1.0.0',
dependencies: {
'json-append': '1',
},
scripts: {
postinstall: `node -e "process.stdout.write('project-1')" | json-append ../output1.json && node -e "process.stdout.write('project-1')" | json-append ../output2.json`,
},
},
{
name: 'project-2',
version: '1.0.0',
dependencies: {
'json-append': '1',
'project-1': '1'
},
scripts: {
postinstall: `node -e "process.stdout.write('project-2')" | json-append ../output1.json`,
},
},
{
name: 'project-3',
version: '1.0.0',
dependencies: {
'json-append': '1',
'project-1': '1'
},
scripts: {
postinstall: `node -e "process.stdout.write('project-3')" | json-append ../output2.json`,
},
},
{
name: 'project-0',
version: '1.0.0',
dependencies: {},
},
])
const importers = [
{
buildIndex: 1,
prefix: path.resolve('project-3'),
},
{
buildIndex: 1,
prefix: path.resolve('project-2'),
},
{
buildIndex: 0,
prefix: path.resolve('project-1'),
},
{
buildIndex: 0,
prefix: path.resolve('project-0'),
},
]
await mutateModules(
importers.map((importer) => ({ ...importer, mutation: 'install' as 'install' })),
await testDefaults({ ignoreScripts: true }),
)
await rebuild(importers, await testDefaults())
const outputs1 = await import(path.resolve('output1.json')) as string[]
const outputs2 = await import(path.resolve('output2.json')) as string[]
t.deepEqual(outputs1, ['project-1', 'project-2'])
t.deepEqual(outputs2, ['project-1', 'project-3'])
})

View File

@@ -945,8 +945,10 @@ test('packages installed via tarball URL from the default registry are normalize
test('shrinkwrap file has correct format when shrinkwrap directory does not equal the prefix directory', async (t: tape.Test) => {
const project = prepare(t)
const store = path.resolve('..', '.store')
await addDependenciesToPackage(['pkg-with-1-dep', '@rstacruz/tap-spec@4.1.1', 'kevva/is-negative#1d7e288222b53a0cab90a331f1865220ec29560c'],
await testDefaults({ save: true, shrinkwrapDirectory: path.resolve('..') }))
await testDefaults({ save: true, shrinkwrapDirectory: path.resolve('..'), store }))
t.ok(!await exists('node_modules/.modules.yaml'), ".modules.yaml in importer's node_modules not created")
@@ -989,7 +991,7 @@ test('shrinkwrap file has correct format when shrinkwrap directory does not equa
process.chdir('project-2')
await addDependenciesToPackage(['is-positive'], await testDefaults({ save: true, shrinkwrapDirectory: path.resolve('..') }))
await addDependenciesToPackage(['is-positive'], await testDefaults({ save: true, shrinkwrapDirectory: path.resolve('..'), store }))
{
const shr = await readYamlFile<Shrinkwrap>(path.join('..', 'shrinkwrap.yaml'))
@@ -1091,10 +1093,12 @@ test('doing named installation when shared shrinkwrap.yaml exists already', asyn
await mutateModules(
[
{
buildIndex: 0,
mutation: 'install',
prefix: path.resolve('pkg1'),
},
{
buildIndex: 0,
mutation: 'install',
prefix: path.resolve('pkg2'),
},

View File

@@ -228,10 +228,12 @@ test('uninstalling a dependency from package that uses shared shrinkwrap', async
await mutateModules(
[
{
buildIndex: 0,
mutation: 'install',
prefix: path.resolve('project-1'),
},
{
buildIndex: 0,
mutation: 'install',
prefix: path.resolve('project-2'),
},

View File

@@ -20,7 +20,7 @@ export default async function testDefaults (
fetchOpts?: any, // tslint:disable-line
storeOpts?: any, // tslint:disable-line
): Promise<InstallOptions & {globalPrefix: string, globalBin: string}> {
let store = opts && opts.store || path.resolve('..', '.store')
let store = opts && opts.store || path.resolve('.store')
store = await storePath(opts && opts.prefix || process.cwd(), store)
const rawNpmConfig = { registry }
const storeController = await createStore(

View File

@@ -339,12 +339,14 @@ importers:
'@pnpm/symlink-dependency': 'link:../symlink-dependency'
'@pnpm/types': 'link:../types'
'@pnpm/utils': 'link:../utils'
'@types/p-limit': 2.0.0
'@types/ramda': 0.25.34
dependency-path: 'link:../dependency-path'
graph-sequencer: 2.0.0
p-limit: 2.1.0
path-exists: 3.0.0
ramda: 0.26.1
run-groups: 1.0.0
devDependencies:
'@pnpm/assert-project': 'link:../../privatePackages/assert-project'
'@pnpm/default-fetcher': 'link:../default-fetcher'
@@ -407,6 +409,7 @@ importers:
'@types/fs-extra': 5.0.4
'@types/mz': 0.0.32
'@types/node': 10.12.18
'@types/p-limit': 2.0.0
'@types/path-exists': 3.0.0
'@types/ramda': 0.25.34
'@types/rimraf': 2.0.2
@@ -426,6 +429,7 @@ importers:
ramda: 0.26.1
rimraf: 2.6.3
rimraf-then: 1.0.1
run-groups: 1.0.0
sinon: 7.2.2
tape: 4.9.2
tape-promise: 4.0.0
@@ -440,6 +444,7 @@ importers:
'@pnpm/types': 'link:../types'
'@zkochan/npm-lifecycle': 2.2.0
path-exists: 3.0.0
run-groups: 1.0.0
devDependencies:
'@pnpm/lifecycle': 'link:'
'@pnpm/logger': 2.1.0
@@ -475,6 +480,7 @@ importers:
mos-plugin-readme: 1.0.4
path-exists: 3.0.0
rimraf: 2.6.3
run-groups: 1.0.0
tape: 4.9.2
ts-node: 7.0.1
tslint: 5.12.0
@@ -1455,6 +1461,7 @@ importers:
replace-string: 2.0.0
resolve-link-target: 1.0.1
rimraf-then: 1.0.1
run-groups: 1.0.0
semver: 5.6.0
symlink-dir: 2.0.2
util.promisify: 1.0.0
@@ -1595,6 +1602,7 @@ importers:
resolve-link-target: 1.0.1
rimraf: 2.6.3
rimraf-then: 1.0.1
run-groups: 1.0.0
semver: 5.6.0
sepia: 2.0.2
sinon: 7.2.2
@@ -8720,6 +8728,14 @@ packages:
dev: true
resolution:
integrity: sha1-yK1KXhEGYeQCp9IbUw4AnyX444k=
/run-groups/1.0.0:
dependencies:
p-limit: 2.1.0
dev: false
engines:
node: '>=6'
resolution:
integrity: sha512-zB8L/p3NZd7st8jjMHyyiJQrQKM8hufFDyZRy5b2OZDmJWwPB+Sq9nR3ORFE1Vw2k4NLcCNlUuD1RsP2oQGw9w==
/run-node/1.0.0:
dev: true
engines: