feat(core): add a disableRelinkFromStore option (#7075)

.changeset/beige-otters-wonder.md

@@ -0,0 +1,12 @@
+---
+"@pnpm/store-connection-manager": minor
+"@pnpm/store-controller-types": minor
+"@pnpm/fs.indexed-pkg-importer": minor
+"@pnpm/create-cafs-store": minor
+"@pnpm/headless": minor
+"@pnpm/core": minor
+"@pnpm/cafs-types": minor
+"@pnpm/worker": patch
+---
+
+New option added: disableRelinkFromStore.

config/config/src/Config.ts

@@ -119,7 +119,6 @@ export interface Config {
   fetchingConcurrency?: number
   lockfileOnly?: boolean // like npm's --package-lock-only
   childConcurrency?: number
-  repeatInstallDepth?: number
   ignorePnpmfile?: boolean
   pnpmfile: string
   hooks?: Hooks

fs/indexed-pkg-importer/src/index.ts

@@ -125,7 +125,7 @@ function hardlinkPkg (
   if (
     !opts.fromStore ||
     opts.force ||
-    !pkgLinkedToStore(opts.filesMap, to)
+    !opts.disableRelinkFromStore && !pkgLinkedToStore(opts.filesMap, to)
   ) {
     importIndexedDir(importFile, to, opts.filesMap, opts)
     return 'hardlink'

pkg-manager/core/src/install/extendInstallOptions.ts

@@ -124,6 +124,17 @@ export interface StrictInstallOptions {
   extendNodePath: boolean
   excludeLinksFromLockfile: boolean
   confirmModulesPurge: boolean
+  /**
+   * Don't relink packages if they are not hard linked from the store.
+   * This also applies to injected dependencies, which are linked from the local package's location.
+   *
+   * This option was added to fix an issue with Bit CLI.
+   * Bit compile adds dist directories to the injected dependencies, so if pnpm were to relink them,
+   * the dist directories would be deleted.
+   *
+   * The option might be used in the future to improve performance.
+   */
+  disableRelinkFromStore: boolean
 }
 
 export type InstallOptions =

pkg-manager/core/src/install/index.ts

@@ -1094,6 +1094,7 @@ const _installInContext: InstallFunction = async (projects, ctx, opts) => {
         dedupeDirectDeps: opts.dedupeDirectDeps,
         dependenciesByProjectId,
         depsStateCache,
+        disableRelinkFromStore: opts.disableRelinkFromStore,
         extraNodePaths: ctx.extraNodePaths,
         force: opts.force,
         hoistedDependencies: ctx.hoistedDependencies,

pkg-manager/core/src/install/link.ts

@@ -49,6 +49,7 @@ export async function linkPackages (
     currentLockfile: Lockfile
     dedupeDirectDeps: boolean
     dependenciesByProjectId: Record<string, Record<string, string>>
+    disableRelinkFromStore?: boolean
     force: boolean
     depsStateCache: DepsStateCache
     extraNodePaths: string[]
@@ -143,6 +144,7 @@ export async function linkPackages (
     newCurrentLockfile,
     depGraph,
     {
+      disableRelinkFromStore: opts.disableRelinkFromStore,
       force: opts.force,
       depsStateCache: opts.depsStateCache,
       ignoreScripts: opts.ignoreScripts,
@@ -301,6 +303,7 @@ async function linkNewPackages (
   depGraph: DependenciesGraph,
   opts: {
     depsStateCache: DepsStateCache
+    disableRelinkFromStore?: boolean
     force: boolean
     optional: boolean
     ignoreScripts: boolean
@@ -366,6 +369,7 @@ async function linkNewPackages (
     linkAllPkgs(opts.storeController, newPkgs, {
       depGraph,
       depsStateCache: opts.depsStateCache,
+      disableRelinkFromStore: opts.disableRelinkFromStore,
       force: opts.force,
       ignoreScripts: opts.ignoreScripts,
       lockfileDir: opts.lockfileDir,
@@ -418,6 +422,7 @@ async function linkAllPkgs (
   opts: {
     depGraph: DependenciesGraph
     depsStateCache: DepsStateCache
+    disableRelinkFromStore?: boolean
     force: boolean
     ignoreScripts: boolean
     lockfileDir: string
@@ -439,6 +444,7 @@ async function linkAllPkgs (
         })
       }
       const { importMethod, isBuilt } = await storeController.importPackage(depNode.dir, {
+        disableRelinkFromStore: opts.disableRelinkFromStore,
         filesResponse: files,
         force: opts.force,
         sideEffectsCacheKey,

pkg-manager/core/test/install/injectLocalPackages.ts

@@ -1606,3 +1606,179 @@ test('injected package is kept up-to-date when it is hoisted to multiple places'
   expect(modulesState?.injectedDeps?.['project-3'][0]).toEqual(path.join('project-1', 'node_modules', 'is-positive'))
   expect(modulesState?.injectedDeps?.['project-3'][1]).toEqual(path.join('project-2', 'node_modules', 'is-positive'))
 })
+
+test('relink injected dependency on install by default', async () => {
+  const depManifest = {
+    name: 'dep',
+    version: '1.0.0',
+  }
+  const mainManifest = {
+    name: 'main',
+    version: '1.0.0',
+    dependencies: {
+      dep: 'workspace:1.0.0',
+    },
+    dependenciesMeta: {
+      dep: {
+        injected: true,
+      },
+    },
+  }
+  preparePackages([
+    {
+      location: 'dep',
+      package: depManifest,
+    },
+    {
+      location: 'main',
+      package: mainManifest,
+    },
+  ])
+  fs.writeFileSync('dep/index.js', 'console.log("dep")')
+  const importers: MutatedProject[] = [
+    {
+      mutation: 'install',
+      rootDir: path.resolve('dep'),
+    },
+    {
+      mutation: 'install',
+      rootDir: path.resolve('main'),
+    },
+  ]
+  const allProjects: ProjectOptions[] = [
+    {
+      buildIndex: 0,
+      manifest: depManifest,
+      rootDir: path.resolve('dep'),
+    },
+    {
+      buildIndex: 0,
+      manifest: mainManifest,
+      rootDir: path.resolve('main'),
+    },
+  ]
+  const workspacePackages = {
+    dep: {
+      '1.0.0': {
+        dir: path.resolve('dep'),
+        manifest: depManifest,
+      },
+    },
+  }
+  await mutateModules(importers, await testDefaults({
+    allProjects,
+    workspacePackages,
+    packageImportMethod: 'hardlink',
+    fastUnpack: false,
+  }))
+
+  const indexJsPath = path.resolve('main/node_modules/dep/index.js')
+  const getInode = () => fs.statSync(indexJsPath).ino
+  const storeInode = getInode()
+
+  // rewriting index.js, to destroy the link
+  fs.unlinkSync(indexJsPath)
+  fs.writeFileSync(indexJsPath, 'console.log("dep updated")')
+
+  expect(storeInode).not.toEqual(getInode())
+
+  await mutateModules(importers, await testDefaults({
+    allProjects,
+    workspacePackages,
+    packageImportMethod: 'hardlink',
+    fastUnpack: false,
+  }))
+
+  expect(storeInode).toEqual(getInode())
+})
+
+test('do not relink injected dependency on install when disableRelinkFromStore is set to true', async () => {
+  const depManifest = {
+    name: 'dep',
+    version: '1.0.0',
+  }
+  const mainManifest = {
+    name: 'main',
+    version: '1.0.0',
+    dependencies: {
+      dep: 'workspace:1.0.0',
+    },
+    dependenciesMeta: {
+      dep: {
+        injected: true,
+      },
+    },
+  }
+  preparePackages([
+    {
+      location: 'dep',
+      package: depManifest,
+    },
+    {
+      location: 'main',
+      package: mainManifest,
+    },
+  ])
+  fs.writeFileSync('dep/index.js', 'console.log("dep")')
+  const importers: MutatedProject[] = [
+    {
+      mutation: 'install',
+      rootDir: path.resolve('dep'),
+    },
+    {
+      mutation: 'install',
+      rootDir: path.resolve('main'),
+    },
+  ]
+  const allProjects: ProjectOptions[] = [
+    {
+      buildIndex: 0,
+      manifest: depManifest,
+      rootDir: path.resolve('dep'),
+    },
+    {
+      buildIndex: 0,
+      manifest: mainManifest,
+      rootDir: path.resolve('main'),
+    },
+  ]
+  const workspacePackages = {
+    dep: {
+      '1.0.0': {
+        dir: path.resolve('dep'),
+        manifest: depManifest,
+      },
+    },
+  }
+  await mutateModules(importers, await testDefaults({
+    allProjects,
+    workspacePackages,
+    packageImportMethod: 'hardlink',
+    fastUnpack: false,
+  }))
+
+  const pkgJsonPath = path.resolve('main/node_modules/dep/package.json')
+  const getInode = () => fs.statSync(pkgJsonPath).ino
+  const storeInode = getInode()
+
+  // rewriting index.js, to destroy the link
+  const pkgJsonContent = fs.readFileSync(pkgJsonPath, 'utf8')
+  fs.unlinkSync(pkgJsonPath)
+  fs.writeFileSync(pkgJsonPath, pkgJsonContent)
+
+  const newInode = getInode()
+
+  expect(storeInode).not.toEqual(newInode)
+
+  await mutateModules(importers, await testDefaults({
+    allProjects,
+    workspacePackages,
+    packageImportMethod: 'hardlink',
+    fastUnpack: false,
+    disableRelinkFromStore: true,
+  }, {}, {}, {
+    relinkLocalDirDeps: false,
+  }))
+
+  expect(newInode).toEqual(getInode())
+})

pkg-manager/headless/src/index.ts

@@ -132,6 +132,7 @@ export interface HeadlessOptions {
   sideEffectsCacheRead: boolean
   sideEffectsCacheWrite: boolean
   symlink?: boolean
+  disableRelinkFromStore?: boolean
   force: boolean
   storeDir: string
   rawConfig: object
@@ -353,6 +354,7 @@ export async function headlessInstall (opts: HeadlessOptions): Promise<Installat
   if (opts.nodeLinker === 'hoisted' && hierarchy && prevGraph) {
     await linkHoistedModules(opts.storeController, graph, prevGraph, hierarchy, {
       depsStateCache,
+      disableRelinkFromStore: opts.disableRelinkFromStore,
       force: opts.force,
       ignoreScripts: opts.ignoreScripts,
       lockfileDir: opts.lockfileDir,
@@ -383,6 +385,7 @@ export async function headlessInstall (opts: HeadlessOptions): Promise<Installat
         }),
       linkAllPkgs(opts.storeController, depNodes, {
         force: opts.force,
+        disableRelinkFromStore: opts.disableRelinkFromStore,
         depGraph: graph,
         depsStateCache,
         ignoreScripts: opts.ignoreScripts,
@@ -770,6 +773,7 @@ async function linkAllPkgs (
   opts: {
     depGraph: DependenciesGraph
     depsStateCache: DepsStateCache
+    disableRelinkFromStore?: boolean
     force: boolean
     ignoreScripts: boolean
     lockfileDir: string
@@ -796,6 +800,7 @@ async function linkAllPkgs (
       const { importMethod, isBuilt } = await storeController.importPackage(depNode.dir, {
         filesResponse,
         force: opts.force,
+        disableRelinkFromStore: opts.disableRelinkFromStore,
         requiresBuild: depNode.requiresBuild || depNode.patchFile != null,
         sideEffectsCacheKey,
       })

pkg-manager/headless/src/linkHoistedModules.ts

@@ -29,6 +29,7 @@ export async function linkHoistedModules (
   hierarchy: DepHierarchy,
   opts: {
     depsStateCache: DepsStateCache
+    disableRelinkFromStore?: boolean
     force: boolean
     ignoreScripts: boolean
     lockfileDir: string
@@ -86,6 +87,7 @@ async function linkAllPkgsInOrder (
   parentDir: string,
   opts: {
     depsStateCache: DepsStateCache
+    disableRelinkFromStore?: boolean
     force: boolean
     ignoreScripts: boolean
     lockfileDir: string
@@ -121,6 +123,7 @@ async function linkAllPkgsInOrder (
           const { importMethod, isBuilt } = await storeController.importPackage(depNode.dir, {
             filesResponse,
             force: opts.force || depNode.depPath !== prevGraph[dir]?.depPath,
+            disableRelinkFromStore: opts.disableRelinkFromStore,
             keepModulesDir: true,
             requiresBuild: depNode.requiresBuild || depNode.patchFile != null,
             sideEffectsCacheKey,

store/cafs-types/src/index.ts

@@ -22,6 +22,7 @@ export type PackageFilesResponse = {
 })
 
 export interface ImportPackageOpts {
+  disableRelinkFromStore?: boolean
   requiresBuild?: boolean
   sideEffectsCacheKey?: string
   filesResponse: PackageFilesResponse

store/create-cafs-store/src/index.ts

@@ -39,6 +39,7 @@ export function createPackageImporterAsync (
       : (opts.filesResponse.packageImportMethod ?? packageImportMethod)
     const impPkg = cachedImporterCreator(pkgImportMethod)
     const importMethod = await impPkg(to, {
+      disableRelinkFromStore: opts.disableRelinkFromStore,
       filesMap,
       fromStore: opts.filesResponse.fromStore,
       force: opts.force,
@@ -67,6 +68,7 @@ function createPackageImporter (
       : (opts.filesResponse.packageImportMethod ?? packageImportMethod)
     const impPkg = cachedImporterCreator(pkgImportMethod)
     const importMethod = impPkg(to, {
+      disableRelinkFromStore: opts.disableRelinkFromStore,
       filesMap,
       fromStore: opts.filesResponse.fromStore,
       force: opts.force,

store/store-connection-manager/src/createNewStoreController.ts

@@ -44,6 +44,14 @@ export type CreateNewStoreControllerOptions = CreateResolverOptions & Pick<Confi
 > & {
   cafsLocker?: CafsLocker
   ignoreFile?: (filename: string) => boolean
+  /**
+   * By default, injected (also known as local) dependencies are relinked on every install
+   * to ensure that the links are always up-to-date.
+   * If another tool persists modifications to the injected dependencies,
+   * this option can be set to `false`.
+   * For example, 'bit compile' writes the compiled code directly to the node_modules directory,
+   * eliminating the need to relink the package.
+   */
   relinkLocalDirDeps?: boolean
 } & Partial<Pick<Config, 'userConfig' | 'deployAllFiles'>> & Pick<ClientOptions, 'resolveSymlinksInInjectedDirs'>
 

store/store-controller-types/src/index.ts

@@ -161,6 +161,7 @@ export interface PackageResponse {
 export type FilesMap = Record<string, string>
 
 export interface ImportOptions {
+  disableRelinkFromStore?: boolean
   filesMap: FilesMap
   force: boolean
   fromStore: boolean

worker/src/types.ts

@@ -25,6 +25,7 @@ export interface LinkPkgMessage {
   requiresBuild?: boolean
   force: boolean
   keepModulesDir?: boolean
+  disableRelinkFromStore?: boolean
 }
 
 export interface SymlinkAllModulesMessage {

worker/src/worker.ts

@@ -176,6 +176,7 @@ function importPackage ({
   requiresBuild,
   force,
   keepModulesDir,
+  disableRelinkFromStore,
 }: LinkPkgMessage) {
   const cacheKey = JSON.stringify({ storeDir, packageImportMethod })
   if (!cafsStoreCache.has(cacheKey)) {
@@ -185,6 +186,7 @@ function importPackage ({
   const { importMethod, isBuilt } = cafsStore.importPackage(targetDir, {
     filesResponse,
     force,
+    disableRelinkFromStore,
     requiresBuild,
     sideEffectsCacheKey,
     keepModulesDir,