feat: save optional deps separately in the lockfile

For correct prunning of the lockfile, it has to be known
which deps are optional. Also it has to be known which
deps are dev deps in the root.

BREAKING CHANGE:

lockfile format changed
This commit is contained in:
zkochan
2017-06-05 21:54:43 +03:00
parent 5acc4671f0
commit 2c04aa69da
17 changed files with 80 additions and 266 deletions

View File

@@ -68,6 +68,7 @@
"pnpm-default-reporter": "^0.6.0",
"pnpm-file-reporter": "^0.0.1",
"pnpm-install-checks": "^1.1.0",
"pnpm-lockfile": "^0.1.0",
"pnpm-logger": "^0.3.0",
"proper-lockfile": "^2.0.0",
"ramda": "^0.24.1",
@@ -159,7 +160,8 @@
"pnpm-install-checks",
"pnpm-logger",
"pnpm-registry-mock",
"remove-all-except-outer-links"
"remove-all-except-outer-links",
"pnpm-lockfile"
]
}
}

View File

@@ -67,6 +67,7 @@ dependencies:
pnpm-default-reporter@^0.6.0: 0.6.0
pnpm-file-reporter@^0.0.1: 0.0.1
pnpm-install-checks@^1.1.0: 1.1.0
pnpm-lockfile@^0.1.0: 0.1.0
pnpm-logger@^0.3.0: 0.3.0
pnpm-registry-mock@^0.10.0: 0.10.0
proper-lockfile@^2.0.0: 2.0.1
@@ -1561,6 +1562,16 @@ packages:
dependencies:
semver: 5.3.0
resolution: 741d9979762fdfad93f3e469deb4a814d3430008
/pnpm-lockfile/0.1.0:
dependencies:
'@types/ramda': 0.0.11
is-ci: 1.0.10
load-yaml-file: 0.1.0
pnpm-logger: 0.3.0
ramda: 0.24.1
rimraf-then: 1.0.1
write-yaml-file: 1.0.0
resolution: ef53cd6e78da43b5956afa721867c8c71c2b61d7
/pnpm-logger/0.0.0:
dependencies:
bole: 3.0.2

View File

@@ -7,7 +7,7 @@ import {
read as readShrinkwrap,
readPrivate as readPrivateShrinkwrap,
Shrinkwrap,
} from '../fs/shrinkwrap'
} from 'pnpm-lockfile'
import {
read as readModules,
} from '../fs/modulesController'

View File

@@ -23,8 +23,8 @@ import {
save as saveShrinkwrap,
Shrinkwrap,
ResolvedDependencies,
pkgIdToRef,
} from '../fs/shrinkwrap'
} from 'pnpm-lockfile'
import {pkgIdToRef} from '../fs/shrinkwrap'
import {save as saveModules} from '../fs/modulesController'
import mkdirp = require('mkdirp-promise')
import createMemoize, {MemoizedFunc} from '../memoize'
@@ -297,7 +297,16 @@ async function installInContext (
const getSpecFromPkg = (depName: string) => deps[depName] || devDeps[depName] || optionalDeps[depName]
pkgs.forEach(dep => {
ctx.shrinkwrap.dependencies[dep.name] = pkgIdToRef(dep.id, dep.name, dep.resolution, ctx.shrinkwrap.registry)
const ref = pkgIdToRef(dep.id, dep.name, dep.resolution, ctx.shrinkwrap.registry)
if (dep.dev) {
ctx.shrinkwrap.devDependencies = ctx.shrinkwrap.devDependencies || {}
ctx.shrinkwrap.devDependencies[dep.name] = ref
} else if (dep.optional) {
ctx.shrinkwrap.optionalDependencies = ctx.shrinkwrap.optionalDependencies || {}
ctx.shrinkwrap.optionalDependencies[dep.name] = ref
} else {
ctx.shrinkwrap.dependencies[dep.name] = ref
}
ctx.shrinkwrap.specifiers[dep.name] = getSpecFromPkg(dep.name)
})
}

View File

@@ -11,7 +11,7 @@ import {PackageSpec} from '../resolve'
import {
ResolvedDependencies,
prune as pruneShrinkwrap,
} from '../fs/shrinkwrap'
} from 'pnpm-lockfile'
export async function prune(maybeOpts?: PnpmOptions): Promise<void> {
const opts = extendOptions(maybeOpts)

View File

@@ -1,9 +1,9 @@
import rimraf = require('rimraf-then')
import path = require('path')
import {
Shrinkwrap,
shortIdToFullId,
} from '../fs/shrinkwrap'
import {Shrinkwrap} from 'pnpm-lockfile'
import {read as readStore, save as saveStore} from '../fs/storeController'
import R = require('ramda')
import {PackageSpec} from '../resolve'
@@ -15,8 +15,8 @@ export default async function removeOrphanPkgs (
root: string,
storePath: string
): Promise<string[]> {
const oldPkgNames = Object.keys(oldShr.dependencies)
const newPkgNames = Object.keys(newShr.dependencies)
const oldPkgNames = Object.keys(oldShr.specifiers)
const newPkgNames = Object.keys(newShr.specifiers)
const removedTopDeps = R.difference(oldPkgNames, newPkgNames)

View File

@@ -10,7 +10,7 @@ import {
Shrinkwrap,
save as saveShrinkwrap,
prune as pruneShrinkwrap,
} from '../fs/shrinkwrap'
} from 'pnpm-lockfile'
import {
save as saveModules
} from '../fs/modulesController'

View File

@@ -1,238 +1,5 @@
import path = require('path')
import {Resolution, PackageSpec} from '../resolve'
import {PnpmError} from '../errorTypes'
import logger from 'pnpm-logger'
import loadYamlFile = require('load-yaml-file')
import writeYamlFile = require('write-yaml-file')
import R = require('ramda')
import rimraf = require('rimraf-then')
import isCI = require('is-ci')
import {Resolution} from '../resolve'
import getRegistryName from '../resolve/npm/getRegistryName'
import npa = require('npm-package-arg')
import pnpmPkgJson from '../pnpmPkgJson'
import {Package} from '../types'
const shrinkwrapLogger = logger('shrinkwrap')
export const SHRINKWRAP_FILENAME = 'shrinkwrap.yaml'
export const PRIVATE_SHRINKWRAP_FILENAME = path.join('node_modules', '.shrinkwrap.yaml')
const SHRINKWRAP_VERSION = 3
const CREATED_WITH = `${pnpmPkgJson.name}@${pnpmPkgJson.version}`
class ShrinkwrapBreakingChangeError extends PnpmError {
constructor (filename: string) {
super('SHRINKWRAP_BREAKING_CHANGE', `Shrinkwrap file ${filename} not compatible with current pnpm`)
this.filename = filename
}
filename: string
}
function getDefaultShrinkwrap (registry: string) {
return {
version: SHRINKWRAP_VERSION,
createdWith: CREATED_WITH,
specifiers: {},
dependencies: {},
packages: {},
registry,
}
}
export type Shrinkwrap = {
version: number,
createdWith: string,
specifiers: ResolvedDependencies,
dependencies: ResolvedDependencies,
packages: ResolvedPackages,
registry: string,
}
export type ResolvedPackages = {
[pkgId: string]: DependencyShrinkwrap,
}
export type ShrinkwrapResolution = Resolution | {
integrity: string,
}
export type DependencyShrinkwrap = {
id?: string,
dev?: true,
optional?: true,
resolution: ShrinkwrapResolution,
dependencies?: ResolvedDependencies,
}
/*** @example
* {
* "foo": "registry.npmjs.org/foo/1.0.1"
* }
*/
export type ResolvedDependencies = {
[pkgName: string]: string,
}
export async function readPrivate (
pkgPath: string,
opts: {
force: boolean,
registry: string,
}
): Promise<Shrinkwrap> {
const shrinkwrapPath = path.join(pkgPath, PRIVATE_SHRINKWRAP_FILENAME)
let shrinkwrap
try {
shrinkwrap = await loadYamlFile<Shrinkwrap>(shrinkwrapPath)
} catch (err) {
if ((<NodeJS.ErrnoException>err).code !== 'ENOENT') {
throw err
}
return getDefaultShrinkwrap(opts.registry)
}
if (shrinkwrap && shrinkwrap.version === SHRINKWRAP_VERSION) {
return shrinkwrap
}
if (opts.force || isCI) {
shrinkwrapLogger.warn(`Ignoring not compatible shrinkwrap file at ${shrinkwrapPath}`)
return getDefaultShrinkwrap(opts.registry)
}
throw new ShrinkwrapBreakingChangeError(shrinkwrapPath)
}
export async function read (
pkgPath: string,
opts: {
force: boolean,
registry: string,
}): Promise<Shrinkwrap> {
const shrinkwrapPath = path.join(pkgPath, SHRINKWRAP_FILENAME)
let shrinkwrap
try {
shrinkwrap = await loadYamlFile<Shrinkwrap>(shrinkwrapPath)
} catch (err) {
if ((<NodeJS.ErrnoException>err).code !== 'ENOENT') {
throw err
}
return getDefaultShrinkwrap(opts.registry)
}
if (shrinkwrap && shrinkwrap.version === SHRINKWRAP_VERSION) {
return shrinkwrap
}
if (opts.force || isCI) {
shrinkwrapLogger.warn(`Ignoring not compatible shrinkwrap file at ${shrinkwrapPath}`)
return getDefaultShrinkwrap(opts.registry)
}
throw new ShrinkwrapBreakingChangeError(shrinkwrapPath)
}
export function save (pkgPath: string, shrinkwrap: Shrinkwrap) {
const shrinkwrapPath = path.join(pkgPath, SHRINKWRAP_FILENAME)
const privateShrinkwrapPath = path.join(pkgPath, PRIVATE_SHRINKWRAP_FILENAME)
// empty shrinkwrap is not saved
if (Object.keys(shrinkwrap.dependencies).length === 0) {
return Promise.all([
rimraf(shrinkwrapPath),
rimraf(privateShrinkwrapPath),
])
}
const formatOpts = {
sortKeys: true,
lineWidth: 1000,
noCompatMode: true,
}
return Promise.all([
writeYamlFile(shrinkwrapPath, shrinkwrap, formatOpts),
writeYamlFile(privateShrinkwrapPath, shrinkwrap, formatOpts),
])
}
export function prune (shr: Shrinkwrap, pkg: Package): Shrinkwrap {
const packages: ResolvedPackages = {}
const optionalDependencies = R.keys(pkg.optionalDependencies)
const dependencies = R.difference(R.keys(pkg.dependencies), optionalDependencies)
const devDependencies = R.difference(R.difference(R.keys(pkg.devDependencies), optionalDependencies), dependencies)
copyDependencyTree(packages, shr, {
registry: shr.registry,
dependencies: devDependencies,
dev: true,
})
copyDependencyTree(packages, shr, {
registry: shr.registry,
dependencies: optionalDependencies,
optional: true,
})
copyDependencyTree(packages, shr, {
registry: shr.registry,
dependencies,
})
const allDeps = R.reduce(R.union, [], [optionalDependencies, devDependencies, dependencies])
const specifiers: ResolvedDependencies = {}
const shrDependencies: ResolvedDependencies = {}
R.keys(shr.specifiers).forEach(depName => {
if (allDeps.indexOf(depName) === -1) return
specifiers[depName] = shr.specifiers[depName]
shrDependencies[depName] = shr.dependencies[depName]
})
return {
version: SHRINKWRAP_VERSION,
createdWith: shr.createdWith || CREATED_WITH,
specifiers,
registry: shr.registry,
dependencies: shrDependencies,
packages,
}
}
function copyDependencyTree (
resolvedPackages: ResolvedPackages,
shr: Shrinkwrap,
opts: {
registry: string,
dependencies: string[],
dev?: boolean,
optional?: boolean,
}
): ResolvedPackages {
let pkgIds: string[] = opts.dependencies
.map((pkgName: string) => getPkgShortId(shr.dependencies[pkgName], pkgName))
const checked = new Set<string>()
while (pkgIds.length) {
let nextPkgIds: string[] = []
for (let pkgId of pkgIds) {
if (checked.has(pkgId)) continue
checked.add(pkgId)
if (!shr.packages[pkgId]) {
logger.warn(`Cannot find resolution of ${pkgId} in shrinkwrap file`)
continue
}
const depShr = shr.packages[pkgId]
resolvedPackages[pkgId] = depShr
if (opts.optional) {
depShr.optional = true
} else {
delete depShr.optional
}
if (opts.dev) {
depShr.dev = true
} else {
delete depShr.dev
}
const newDependencies = R.keys(depShr.dependencies)
.map((pkgName: string) => getPkgShortId(<string>(depShr.dependencies && depShr.dependencies[pkgName]), pkgName))
.filter((newPkgId: string) => !checked.has(newPkgId))
nextPkgIds = R.union(nextPkgIds, newDependencies)
}
pkgIds = nextPkgIds
}
return resolvedPackages
}
export function shortIdToFullId (
shortId: string,

View File

@@ -12,11 +12,13 @@ import logStatus from '../logging/logInstallStatus'
import fs = require('mz/fs')
import {Got} from '../network/got'
import {
DependencyShrinkwrap,
ResolvedDependencies,
getPkgId,
getPkgShortId,
} from '../fs/shrinkwrap'
import {
DependencyShrinkwrap,
ResolvedDependencies,
} from 'pnpm-lockfile'
import {Resolution, PackageSpec, PackageMeta} from '../resolve'
import depsToSpecs from '../depsToSpecs'
import getIsInstallable from './getIsInstallable'
@@ -39,6 +41,7 @@ export type InstalledPackage = {
name: string,
version: string,
peerDependencies: Dependencies,
optionalDependencies: Set<string>,
hasBundledDependencies: boolean,
localLocation: string,
}
@@ -223,6 +226,7 @@ async function install (
path: fetchedPkg.path,
specRaw: spec.raw,
peerDependencies: pkg.peerDependencies || {},
optionalDependencies: new Set(R.keys(pkg.optionalDependencies)),
hasBundledDependencies: !!(pkg.bundledDependencies || pkg.bundleDependencies),
localLocation: path.join(options.nodeModules, `.${pkgIdToFilename(fetchedPkg.id)}`),
}

View File

@@ -14,7 +14,8 @@ import {Resolution} from '../resolve'
import resolvePeers, {DependencyTreeNode, DependencyTreeNodeMap} from './resolvePeers'
import logStatus from '../logging/logInstallStatus'
import updateShrinkwrap from './updateShrinkwrap'
import {Shrinkwrap, shortIdToFullId, DependencyShrinkwrap} from '../fs/shrinkwrap'
import {shortIdToFullId} from '../fs/shrinkwrap'
import {Shrinkwrap, DependencyShrinkwrap} from 'pnpm-lockfile'
import removeOrphanPkgs from '../api/removeOrphanPkgs'
import ncpCB = require('ncp')
import thenify = require('thenify')
@@ -105,7 +106,6 @@ function filterShrinkwrap (
}
return {
version: shr.version,
createdWith: shr.createdWith,
registry: shr.registry,
specifiers: shr.specifiers,
packages: R.fromPairs(pairs),
@@ -120,6 +120,7 @@ async function linkNewPackages (
force: boolean,
global: boolean,
baseNodeModules: string,
optional: boolean,
}
): Promise<string[]> {
const nextPkgResolvedIds = R.keys(shrinkwrap.packages)
@@ -149,14 +150,14 @@ async function linkNewPackages (
// TODO: no need to relink everything. Can be relinked only what was changed
for (const shortId of nextPkgResolvedIds) {
if (privateShrinkwrap.packages[shortId] &&
!R.equals(privateShrinkwrap.packages[shortId].dependencies, shrinkwrap.packages[shortId].dependencies) ) {
!R.equals(privateShrinkwrap.packages[shortId].dependencies, shrinkwrap.packages[shortId].dependencies)) {
const resolvedId = shortIdToFullId(shortId, shrinkwrap.registry)
newPkgs.push(pkgsToLink[resolvedId])
}
}
}
await linkAllModules(newPkgs, pkgsToLink)
await linkAllModules(newPkgs, pkgsToLink, {optional: opts.optional})
return newPkgResolvedIds
}
@@ -182,10 +183,13 @@ async function linkAllPkgs (
async function linkAllModules (
pkgs: DependencyTreeNode[],
pkgMap: DependencyTreeNodeMap
pkgMap: DependencyTreeNodeMap,
opts: {
optional: boolean,
}
) {
return Promise.all(
pkgs.map(pkg => limitLinking(() => linkModules(pkg, pkgMap)))
pkgs.map(pkg => limitLinking(() => linkModules(pkg, pkgMap, opts)))
)
}
@@ -233,10 +237,17 @@ async function isSameFile (file1: string, file2: string) {
async function linkModules (
dependency: DependencyTreeNode,
pkgMap: DependencyTreeNodeMap
pkgMap: DependencyTreeNodeMap,
opts: {
optional: boolean,
}
) {
const childrenToLink = opts.optional
? dependency.children
: dependency.children.filter(child => !dependency.optionalDependencies.has(pkgMap[child].name))
await Promise.all(
R.props<DependencyTreeNode>(dependency.children, pkgMap)
R.props<DependencyTreeNode>(childrenToLink, pkgMap)
.filter(child => child.installable)
.map(child => symlinkDependencyTo(child, dependency.modules))
)

View File

@@ -16,6 +16,7 @@ export type DependencyTreeNode = {
resolution: Resolution,
hardlinkedLocation: string,
children: string[],
optionalDependencies: Set<string>,
depth: number,
resolvedId: string,
dev: boolean,
@@ -102,6 +103,7 @@ function resolvePeersOfNode (
path: node.pkg.path,
modules,
hardlinkedLocation,
optionalDependencies: node.pkg.optionalDependencies,
children: R.union(node.children, resolvedPeers),
depth: node.depth,
resolvedId,

View File

@@ -1,12 +1,14 @@
import {
pkgShortId,
pkgIdToRef,
} from '../fs/shrinkwrap'
import {
Shrinkwrap,
DependencyShrinkwrap,
ShrinkwrapResolution,
pkgShortId,
pkgIdToRef,
ResolvedDependencies,
prune as pruneShrinkwrap,
} from '../fs/shrinkwrap'
} from 'pnpm-lockfile'
import {DependencyTreeNodeMap, DependencyTreeNode} from './resolvePeers'
import {Resolution} from '../resolve'
import R = require('ramda')
@@ -19,15 +21,18 @@ export default function (
): Shrinkwrap {
for (const resolvedId of R.keys(pkgsToLink)) {
const shortId = pkgShortId(resolvedId, shrinkwrap.registry)
const result = R.partition((childResolvedId: string) => pkgsToLink[resolvedId].optionalDependencies.has(pkgsToLink[childResolvedId].name), pkgsToLink[resolvedId].children)
shrinkwrap.packages[shortId] = toShrDependency({
resolvedId,
id: pkgsToLink[resolvedId].id,
shortId,
resolution: pkgsToLink[resolvedId].resolution,
updatedDeps: pkgsToLink[resolvedId].children,
updatedOptionalDeps: result[0],
updatedDeps: result[1],
registry: shrinkwrap.registry,
pkgsToLink,
prevResolvedDeps: shrinkwrap.packages[shortId] && shrinkwrap.packages[shortId].dependencies || {},
prevResolvedOptionalDeps: shrinkwrap.packages[shortId] && shrinkwrap.packages[shortId].optionalDependencies || {},
dev: pkgsToLink[resolvedId].dev,
optional: pkgsToLink[resolvedId].optional,
})
@@ -43,20 +48,26 @@ function toShrDependency (
resolution: Resolution,
registry: string,
updatedDeps: string[],
updatedOptionalDeps: string[],
pkgsToLink: DependencyTreeNodeMap,
prevResolvedDeps: ResolvedDependencies,
prevResolvedOptionalDeps: ResolvedDependencies,
dev: boolean,
optional: boolean,
}
): DependencyShrinkwrap {
const shrResolution = toShrResolution(opts.shortId, opts.resolution)
const newResolvedDeps = updateResolvedDeps(opts.prevResolvedDeps, opts.updatedDeps, opts.registry, opts.pkgsToLink)
const newResolvedOptionalDeps = updateResolvedDeps(opts.prevResolvedOptionalDeps, opts.updatedOptionalDeps, opts.registry, opts.pkgsToLink)
const result = {
resolution: shrResolution
}
if (!R.isEmpty(newResolvedDeps)) {
result['dependencies'] = newResolvedDeps
}
if (!R.isEmpty(newResolvedOptionalDeps)) {
result['optionalDependencies'] = newResolvedOptionalDeps
}
if (opts.dev) {
result['dev'] = true
}

View File

@@ -12,7 +12,6 @@ import {
pathToLocalPkg,
local,
} from '../utils'
import pnpmPkgJson from '../../src/pnpmPkgJson'
const ncp = thenify(ncpCB.ncp)
const test = promisifyTape(tape)
@@ -55,7 +54,6 @@ test('local file', async function (t: tape.Test) {
},
registry: 'http://localhost:4873/',
version: 3,
createdWith: `${pnpmPkgJson.name}@${pnpmPkgJson.version}`
})
})

View File

@@ -120,7 +120,7 @@ test('not installing optional dependencies when optional is false', async (t: ta
project.has('pkg-with-good-optional')
t.ok(deepRequireCwd(['pkg-with-good-optional', 'dep-of-pkg-with-1-dep', './package.json']))
t.notOk(deepRequireCwd.silent(['pkg-with-good-optional', 'is-positive', './package.json']))
t.notOk(deepRequireCwd.silent(['pkg-with-good-optional', 'is-positive', './package.json']), 'optional subdep not installed')
})
test('optional dependency has bigger priority than regular dependency', async (t: tape.Test) => {

View File

@@ -49,9 +49,9 @@ test('shrinkwrap file has dev deps even when installing for prod only', async (t
const shr = await project.loadShrinkwrap()
const id = '/is-negative/2.1.0'
t.ok(shr.dependencies, 'has dependencies field')
t.ok(shr.devDependencies, 'has devDependencies field')
t.equal(shr.dependencies['is-negative'], '2.1.0', 'has dependency resolved')
t.equal(shr.devDependencies['is-negative'], '2.1.0', 'has dev dependency resolved')
t.ok(shr.packages[id], `has resolution for ${id}`)
})
@@ -195,7 +195,7 @@ test('subdeps are updated on repeat install if outer shrinkwrap.yaml does not ma
shr.packages['/dep-of-pkg-with-1-dep/100.1.0'] = {
resolution: {
integrity: 'sha1-ChbBDewTLAqLCzb793Fo5VDvg/g=',
integrity: 'sha1-sdzLq5q5h7h61HeCB+HLf+lI+zw=',
},
}

View File

@@ -23,7 +23,7 @@ export default function prepare (t: Test, pkg?: Object) {
const dirname = dirNumber.toString()
const pkgTmpPath = path.join(tmpPath, dirname, 'project')
mkdirp.sync(pkgTmpPath)
const json = JSON.stringify(Object.assign({name: 'foo', version: '0.0.0'}, pkg))
const json = JSON.stringify(Object.assign({name: 'foo', version: '0.0.0'}, pkg), null, 2)
fs.writeFileSync(path.join(pkgTmpPath, 'package.json'), json, 'utf-8')
process.chdir(pkgTmpPath)
t.pass(`create testing package ${dirname}`)

View File

@@ -55,7 +55,6 @@
"src/fs/pkgIdToFilename.ts",
"src/fs/readPkg.ts",
"src/fs/safeReadPkg.ts",
"src/fs/shrinkwrap.ts",
"src/fs/storeController.ts",
"src/getSaveType.ts",
"src/index.ts",