feat: custom modules directory

close #2471
PR #2474
This commit is contained in:
Zoltan Kochan authored and GitHub committed 2020-04-12 23:40:43 +03:00
1 parent 6afba68819
commit f70e817fef
17 files changed
+210 -45

No files matched your search

+1
View File
@@ -110,6 +110,7 @@ export interface Config {
sort: boolean,
strictPeerDependencies: boolean,
lockfileDir?: string,
modulesDir?: string,
sharedWorkspaceLockfile?: boolean,
useLockfile: boolean,
resolutionStrategy: 'fast' | 'fewer-dependencies',
+1
View File
@@ -53,6 +53,7 @@ export const types = Object.assign({
'lockfile-directory': String, // TODO: deprecate
'lockfile-only': Boolean,
'loglevel': ['silent', 'error', 'warn', 'info', 'debug'],
'modules-dir': String,
'network-concurrency': Number,
'npm-path': String,
'offline': Boolean,
+33 -20
View File
@@ -50,6 +50,7 @@ export interface PnpmContext<T> {
export interface ProjectOptions {
binsDir?: string,
manifest: ProjectManifest,
modulesDir?: string,
rootDir: string,
}
@@ -61,6 +62,7 @@ export default async function getContext<T> (
forceSharedLockfile: boolean,
extraBinPaths: string[],
lockfileDir: string,
modulesDir?: string,
hooks?: {
readPackage?: ReadPackageHook,
},
@@ -80,8 +82,9 @@ export default async function getContext<T> (
forceShamefullyHoist?: boolean,
},
): Promise<PnpmContext<T>> {
let importersContext = await readProjectsContext(projects, opts.lockfileDir)
const virtualStoreDir = pathAbsolute(opts.virtualStoreDir ?? 'node_modules/.pnpm', opts.lockfileDir)
const modulesDir = opts.modulesDir ?? 'node_modules'
let importersContext = await readProjectsContext(projects, { lockfileDir: opts.lockfileDir, modulesDir })
const virtualStoreDir = pathAbsolute(opts.virtualStoreDir ?? path.join(modulesDir, '.pnpm'), opts.lockfileDir)
if (importersContext.modules) {
const { purged } = await validateNodeModules(importersContext.modules, importersContext.projects, {
@@ -89,6 +92,7 @@ export default async function getContext<T> (
forceNewNodeModules: opts.forceNewNodeModules === true,
include: opts.include,
lockfileDir: opts.lockfileDir,
modulesDir,
registries: opts.registries,
storeDir: opts.storeDir,
virtualStoreDir,
@@ -103,7 +107,10 @@ export default async function getContext<T> (
shamefullyHoist: opts.shamefullyHoist,
})
if (purged) {
importersContext = await readProjectsContext(projects, opts.lockfileDir)
importersContext = await readProjectsContext(projects, {
lockfileDir: opts.lockfileDir,
modulesDir,
})
}
}
@@ -177,6 +184,7 @@ async function validateNodeModules (
forceNewNodeModules: boolean,
include?: IncludedDependencies,
lockfileDir: string,
modulesDir: string,
registries: Registries,
storeDir: string,
virtualStoreDir: string,
@@ -200,14 +208,14 @@ async function validateNodeModules (
if (modules.shamefullyHoist) {
throw new PnpmError(
'SHAMEFULLY_HOIST_WANTED',
'This "node_modules" folder was created using the --shamefully-hoist option.'
+ ' You must add that option, or else run "pnpm install" to recreate the "node_modules" folder.',
'This modules directory was created using the --shamefully-hoist option.'
+ ' You must add that option, or else run "pnpm install" to recreate the modules directory.',
)
}
throw new PnpmError(
'SHAMEFULLY_HOIST_NOT_WANTED',
'This "node_modules" folder was created without the --shamefully-hoist option.'
+ ' You must remove that option, or else "pnpm install" to recreate the "node_modules" folder.',
'This modules directory was created without the --shamefully-hoist option.'
+ ' You must remove that option, or else "pnpm install" to recreate the modules directory.',
)
}
if (opts.forceIndependentLeaves && Boolean(modules.independentLeaves) !== opts.independentLeaves) {
@@ -215,7 +223,7 @@ async function validateNodeModules (
await Promise.all(projects.map(purgeModulesDirsOfImporter))
if (!rootProject) {
await purgeModulesDirsOfImporter({
modulesDir: path.join(opts.lockfileDir, 'node_modules'),
modulesDir: path.join(opts.lockfileDir, opts.modulesDir),
rootDir: opts.lockfileDir,
})
}
@@ -224,14 +232,14 @@ async function validateNodeModules (
if (modules.independentLeaves) {
throw new PnpmError(
'INDEPENDENT_LEAVES_WANTED',
'This "node_modules" folder was created using the --independent-leaves option.'
+ ' You must add that option, or else run "pnpm install" to recreate the "node_modules" folder.',
'This modules directory was created using the --independent-leaves option.'
+ ' You must add that option, or else run "pnpm install" to recreate the modules directory.',
)
}
throw new PnpmError(
'INDEPENDENT_LEAVES_NOT_WANTED',
'This "node_modules" folder was created without the --independent-leaves option.'
+ ' You must remove that option, or else "pnpm install" to recreate the "node_modules" folder.',
'This modules directory was created without the --independent-leaves option.'
+ ' You must remove that option, or else "pnpm install" to recreate the modules directory.',
)
}
let purged = false
@@ -241,14 +249,14 @@ async function validateNodeModules (
if (opts.currentHoistPattern) {
throw new PnpmError(
'HOISTING_WANTED',
'This "node_modules" folder was created using the --hoist-pattern option.'
+ ' You must add this option, or else add the --force option to recreate the "node_modules" folder.',
'This modules directory was created using the --hoist-pattern option.'
+ ' You must add this option, or else add the --force option to recreate the modules directory.',
)
}
throw new PnpmError(
'HOISTING_NOT_WANTED',
'This "node_modules" folder was created without the --hoist-pattern option.'
+ ' You must remove that option, or else run "pnpm install" to recreate the "node_modules" folder.',
'This modules directory was created without the --hoist-pattern option.'
+ ' You must remove that option, or else run "pnpm install" to recreate the modules directory.',
)
}
} catch (err) {
@@ -268,7 +276,7 @@ async function validateNodeModules (
for (const depsField of DEPENDENCIES_FIELDS) {
if (opts.include[depsField] !== modules.included[depsField]) {
throw new PnpmError('INCLUDED_DEPS_CONFLICT',
`node_modules (at "${opts.lockfileDir}") was installed with ${stringifyIncludedDeps(modules.included)}. ` +
`modules directory (at "${opts.lockfileDir}") was installed with ${stringifyIncludedDeps(modules.included)}. ` +
`Current install wants ${stringifyIncludedDeps(opts.include)}.`,
)
}
@@ -285,11 +293,11 @@ async function validateNodeModules (
await Promise.all(projects.map(purgeModulesDirsOfImporter))
return { purged: true }
}
throw new PnpmError('REGISTRIES_MISMATCH', `This "node_modules" directory was created using the following registries configuration: ${JSON.stringify(modules.registries)}. The current configuration is ${JSON.stringify(opts.registries)}. To recreate "node_modules" using the new settings, run "pnpm install".`)
throw new PnpmError('REGISTRIES_MISMATCH', `This modules directory was created using the following registries configuration: ${JSON.stringify(modules.registries)}. The current configuration is ${JSON.stringify(opts.registries)}. To recreate recreate the modules directory using the new settings, run "pnpm install".`)
}
if (purged && !rootProject) {
await purgeModulesDirsOfImporter({
modulesDir: path.join(opts.lockfileDir, 'node_modules'),
modulesDir: path.join(opts.lockfileDir, opts.modulesDir),
rootDir: opts.lockfileDir,
})
}
@@ -351,6 +359,7 @@ export async function getContextForSingleImporter (
forceSharedLockfile: boolean,
extraBinPaths: string[],
lockfileDir: string,
modulesDir?: string,
hooks?: {
readPackage?: ReadPackageHook,
},
@@ -390,7 +399,10 @@ export async function getContextForSingleImporter (
rootDir: opts.dir,
},
],
opts.lockfileDir,
{
lockfileDir: opts.lockfileDir,
modulesDir: opts.modulesDir,
},
)
const storeDir = opts.storeDir
@@ -406,6 +418,7 @@ export async function getContextForSingleImporter (
forceNewNodeModules: opts.forceNewNodeModules === true,
include: opts.include,
lockfileDir: opts.lockfileDir,
modulesDir: opts.modulesDir ?? 'node_modules',
registries: opts.registries,
storeDir: opts.storeDir,
virtualStoreDir,
+4 -2
View File
@@ -89,6 +89,7 @@ export interface HeadlessOptions {
hoistedAliases: {[depPath: string]: string[]}
hoistPattern?: string[],
lockfileDir: string,
modulesDir?: string,
virtualStoreDir?: string,
shamefullyHoist: boolean,
storeController: StoreController,
@@ -125,8 +126,9 @@ export default async (opts: HeadlessOptions) => {
throw new Error(`Headless installation requires a ${WANTED_LOCKFILE} file`)
}
const rootModulesDir = await realpathMissing(path.join(lockfileDir, 'node_modules'))
const virtualStoreDir = pathAbsolute(opts.virtualStoreDir ?? 'node_modules/.pnpm', lockfileDir)
const relativeModulesDir = opts.modulesDir ?? 'node_modules'
const rootModulesDir = await realpathMissing(path.join(lockfileDir, relativeModulesDir))
const virtualStoreDir = pathAbsolute(opts.virtualStoreDir ?? path.join(relativeModulesDir, '.pnpm'), lockfileDir)
const currentLockfile = opts.currentLockfile || await readCurrentLockfile(virtualStoreDir, { ignoreIncompatible: false })
const hoistedModulesDir = opts.shamefullyHoist
? rootModulesDir : path.join(virtualStoreDir, 'node_modules')
+4 -4
View File
@@ -254,7 +254,7 @@ test('installing with independent-leaves and hoistPattern=*', async (t) => {
rootDir: lockfileDir,
},
],
lockfileDir,
{ lockfileDir },
)
await headless(await testDefaults({
@@ -743,7 +743,7 @@ test('using side effects cache and hoistPattern=*', async (t) => {
rootDir: lockfileDir,
},
],
lockfileDir,
{ lockfileDir },
)
// Right now, hardlink does not work with side effects, so we specify copy as the packageImportMethod
@@ -788,7 +788,7 @@ test('installing in a workspace', async (t) => {
rootDir: path.join(workspaceFixture, 'bar'),
},
],
workspaceFixture,
{ lockfileDir: workspaceFixture },
)
projects = await Promise.all(
@@ -831,7 +831,7 @@ test('independent-leaves: installing in a workspace', async (t) => {
rootDir: path.join(workspaceFixture, 'bar'),
},
],
workspaceFixture,
{ lockfileDir: workspaceFixture },
)
await headless(await testDefaults({
+1 -1
View File
@@ -32,7 +32,7 @@ export default async function testDefaults (
rootDir: lockfileDir,
},
],
lockfileDir,
{ lockfileDir },
)
storeDir = await storePath(lockfileDir, storeDir)
const rawConfig = { registry }
@@ -121,11 +121,15 @@ export function help () {
name: '--prefer-frozen-lockfile',
},
{
description: `The directory in which the ${WANTED_LOCKFILE} of the package will be created. Several projects may share a single lockfile`,
description: `The directory in which the ${WANTED_LOCKFILE} of the package will be created. Several projects may share a single lockfile.`,
name: '--lockfile-dir <dir>',
},
{
description: 'Dependencies inside node_modules have access only to their listed dependencies',
description: 'The directory in which dependencies will be installed (instead of node_modules)',
name: '--modules-dir <dir>',
},
{
description: 'Dependencies inside the modules directory will have access only to their listed dependencies',
name: '--no-hoist',
},
{
@@ -47,6 +47,7 @@ type RecursiveOptions = CreateStoreControllerOptions & Pick<Config,
'linkWorkspacePackages' |
'lockfileDir' |
'lockfileOnly' |
'modulesDir' |
'pnpmfile' |
'rawLocalConfig' |
'registries' |
@@ -179,6 +180,7 @@ export default async function recursive (
const mutatedImporters = [] as MutatedProject[]
await Promise.all(importers.map(async ({ buildIndex, rootDir }) => {
const localConfig = await memReadLocalConfig(rootDir)
const modulesDir = localConfig.modulesDir ?? opts.modulesDir
const { manifest, writeProjectManifest } = manifestsByPath[rootDir]
let currentInput = [...params]
if (updateMatch) {
@@ -212,6 +214,7 @@ export default async function recursive (
mutatedImporters.push({
dependencyNames: currentInput,
manifest,
modulesDir,
mutation,
rootDir,
targetDependenciesField,
@@ -222,6 +225,7 @@ export default async function recursive (
allowNew: cmdFullName === 'install' || cmdFullName === 'add',
dependencySelectors: currentInput,
manifest,
modulesDir,
mutation,
peer: opts.savePeer,
pinnedVersion: getPinnedVersion({
@@ -236,6 +240,7 @@ export default async function recursive (
mutatedImporters.push({
buildIndex,
manifest,
modulesDir,
mutation,
rootDir,
} as MutatedProject)
@@ -584,3 +584,41 @@ test('recursive install on workspace with custom lockfile-dir', async (t) => {
t.end()
})
test('recursive install in a monorepo with different modules directories', async (t) => {
const projects = preparePackages(t, [
{
name: 'project-1',
version: '1.0.0',
dependencies: {
'is-positive': '1.0.0',
},
},
{
name: 'project-2',
version: '1.0.0',
dependencies: {
'is-positive': '1.0.0',
},
},
])
await fs.writeFile('project-1/.npmrc', 'modules-dir=modules_1', 'utf8')
await fs.writeFile('project-2/.npmrc', 'modules-dir=modules_2', 'utf8')
const { allProjects, selectedProjectsGraph } = await readProjects(process.cwd(), [])
await install.handler({
...DEFAULT_OPTS,
allProjects,
dir: process.cwd(),
recursive: true,
selectedProjectsGraph,
workspaceDir: process.cwd(),
})
await projects['project-1'].has('is-positive', 'modules_1')
await projects['project-2'].has('is-positive', 'modules_2')
t.end()
})
+10 -6
View File
@@ -7,19 +7,22 @@ import realpathMissing = require('realpath-missing')
export interface ProjectOptions {
binsDir?: string,
modulesDir?: string,
rootDir: string,
}
export default async function <T>(
projects: (ProjectOptions & T)[],
lockfileDir: string,
opts: {
lockfileDir: string,
modulesDir?: string,
},
): Promise<{
currentHoistPattern?: string[],
hoist?: boolean,
hoistedAliases: { [depPath: string]: string[] },
projects: Array<{
id: string,
modulesDir: string,
} & T & Required<ProjectOptions>>,
include: Record<DependenciesField, boolean>,
independentLeaves: boolean | undefined,
@@ -30,7 +33,8 @@ export default async function <T>(
shamefullyHoist?: boolean,
skipped: Set<string>,
}> {
const rootModulesDir = await realpathMissing(path.join(lockfileDir, 'node_modules'))
const relativeModulesDir = opts.modulesDir ?? 'node_modules'
const rootModulesDir = await realpathMissing(path.join(opts.lockfileDir, relativeModulesDir))
const modules = await readModulesYaml(rootModulesDir)
return {
currentHoistPattern: modules?.hoistPattern || undefined,
@@ -42,12 +46,12 @@ export default async function <T>(
pendingBuilds: modules?.pendingBuilds || [],
projects: await Promise.all(
projects.map(async (project) => {
const modulesDir = await realpathMissing(path.join(project.rootDir, 'node_modules'))
const importerId = getLockfileImporterId(lockfileDir, project.rootDir)
const modulesDir = await realpathMissing(path.join(project.rootDir, project.modulesDir ?? relativeModulesDir))
const importerId = getLockfileImporterId(opts.lockfileDir, project.rootDir)
return {
...project,
binsDir: project.binsDir || path.join(project.rootDir, 'node_modules', '.bin'),
binsDir: project.binsDir ?? path.join(project.rootDir, relativeModulesDir, '.bin'),
id: importerId,
modulesDir,
}
@@ -32,6 +32,7 @@ export interface StrictInstallOptions {
depth: number,
resolutionStrategy: 'fast' | 'fewer-dependencies',
lockfileDir: string,
modulesDir: string,
rawConfig: object,
verifyStoreIntegrity: boolean,
engineStrict: boolean,
+1
View File
@@ -219,6 +219,7 @@ export async function mutateModules (
include: opts.include,
independentLeaves: opts.independentLeaves,
lockfileDir: ctx.lockfileDir,
modulesDir: opts.modulesDir,
ownLifecycleHooksStdio: opts.ownLifecycleHooksStdio,
packageManager: opts.packageManager,
pendingBuilds: ctx.pendingBuilds,
+2 -2
View File
@@ -169,7 +169,7 @@ test('hoistPattern=* throws exception when executed on node_modules installed w/
t.fail('installation should have failed')
} catch (err) {
t.equal(err['code'], 'ERR_PNPM_HOISTING_NOT_WANTED') // tslint:disable-line:no-string-literal
t.ok(err.message.indexOf('This "node_modules" folder was created without the --hoist-pattern option.') === 0)
t.ok(err.message.indexOf('This modules directory was created without the --hoist-pattern option.') === 0)
}
})
@@ -187,7 +187,7 @@ test('hoistPattern=undefined throws exception when executed on node_modules inst
t.fail('installation should have failed')
} catch (err) {
t.equal(err['code'], 'ERR_PNPM_HOISTING_WANTED') // tslint:disable-line:no-string-literal
t.ok(err.message.indexOf('This "node_modules" folder was created using the --hoist-pattern option.') === 0)
t.ok(err.message.indexOf('This modules directory was created using the --hoist-pattern option.') === 0)
}
// Instatll doesn't fail if the value of hoistPattern isn't forced
@@ -35,7 +35,7 @@ test('--independent-leaves throws exception when executed on node_modules instal
t.fail('installation should have failed')
} catch (err) {
t.equal(err['code'], 'ERR_PNPM_INDEPENDENT_LEAVES_NOT_WANTED') // tslint:disable-line:no-string-literal
t.ok(err.message.indexOf('This "node_modules" folder was created without the --independent-leaves option.') === 0)
t.ok(err.message.indexOf('This modules directory was created without the --independent-leaves option.') === 0)
}
// Install doesn't fail if independentLeaves is not forced
@@ -61,7 +61,7 @@ test('--no-independent-leaves throws exception when executed on node_modules ins
t.fail('installation should have failed')
} catch (err) {
t.equal(err['code'], 'ERR_PNPM_INDEPENDENT_LEAVES_WANTED') // tslint:disable-line:no-string-literal
t.ok(err.message.indexOf('This "node_modules" folder was created using the --independent-leaves option.') === 0)
t.ok(err.message.indexOf('This modules directory was created using the --independent-leaves option.') === 0)
}
})
+1
View File
@@ -13,6 +13,7 @@ import './local'
import './lockfileDir'
import './lockfileOnly'
import './misc'
import './modulesDir'
import './multipleImporters'
import './only'
import './optionalDependencies'
+92
View File
@@ -0,0 +1,92 @@
import { prepareEmpty, preparePackages } from '@pnpm/prepare'
import rimraf = require('@zkochan/rimraf')
import path = require('path')
import {
install,
MutatedProject,
mutateModules,
} from 'supi'
import tape = require('tape')
import promisifyTape from 'tape-promise'
import { testDefaults } from '../utils'
const test = promisifyTape(tape)
test('installing to a custom modules directory', async (t: tape.Test) => {
const project = prepareEmpty(t)
await install({
dependencies: {
'is-positive': '1.0.0',
},
}, await testDefaults({ modulesDir: 'pnpm_modules' }))
await project.has('is-positive', 'pnpm_modules')
await rimraf('pnpm_modules')
await project.hasNot('is-positive', 'pnpm_modules')
await install({
dependencies: {
'is-positive': '1.0.0',
},
}, await testDefaults({ frozenLockfile: true, modulesDir: 'pnpm_modules' }))
await project.has('is-positive', 'pnpm_modules')
})
test('using different custom modules directory for every project', async (t) => {
const projects = preparePackages(t, [
{
location: 'project-1',
package: {
name: 'project-1',
dependencies: { 'is-positive': '1.0.0' },
},
},
{
location: 'project-2',
package: {
name: 'project-2',
dependencies: { 'is-positive': '1.0.0' },
},
},
])
const importers: MutatedProject[] = [
{
buildIndex: 0,
manifest: {
name: 'project-1',
version: '1.0.0',
dependencies: {
'is-positive': '1.0.0',
},
},
modulesDir: 'modules_1',
mutation: 'install',
rootDir: path.resolve('project-1'),
},
{
buildIndex: 0,
manifest: {
name: 'project-2',
version: '1.0.0',
dependencies: {
'is-positive': '1.0.0',
},
},
modulesDir: 'modules_2',
mutation: 'install',
rootDir: path.resolve('project-2'),
},
]
await mutateModules(importers, await testDefaults())
await projects['project-1'].has('is-positive', 'modules_1')
await projects['project-2'].has('is-positive', 'modules_2')
})
+8 -6
View File
@@ -16,8 +16,8 @@ export type RawLockfile = Lockfile & Partial<ProjectSnapshot>
export interface Project {
requireModule: NodeRequireFunction
has (pkgName: string): Promise<void>
hasNot (pkgName: string): Promise<void>
has (pkgName: string, modulesDir?: string): Promise<void>
hasNot (pkgName: string, modulesDir?: string): Promise<void>
getStorePath (): Promise<string>
resolve (pkgName: string, version?: string, relativePath?: string): Promise<string>
storeHas (pkgName: string, version?: string): Promise<string>
@@ -75,11 +75,13 @@ export default (t: Test, projectPath: string, encodedRegistryName?: string): Pro
requireModule (pkgName: string) {
return require(path.join(modules, pkgName))
},
async has (pkgName: string) {
t.ok(await exists(path.join(modules, pkgName)), `${pkgName} is in node_modules`)
async has (pkgName: string, _modulesDir?: string) {
const md = _modulesDir ? path.join(projectPath, _modulesDir) : modules
t.ok(await exists(path.join(md, pkgName)), `${pkgName} is in ${md}`)
},
async hasNot (pkgName: string) {
t.notOk(await exists(path.join(modules, pkgName)), `${pkgName} is not in node_modules`)
async hasNot (pkgName: string, _modulesDir?: string) {
const md = _modulesDir ? path.join(projectPath, _modulesDir) : modules
t.notOk(await exists(path.join(md, pkgName)), `${pkgName} is not in ${md}`)
},
async getStorePath () {
const store = await getStoreInstance()