fix: should always treat local file dependency as new dependency (#6623)

close #5381
close #6502

.changeset/grumpy-falcons-flow.md

@@ -0,0 +1,8 @@
+---
+"@pnpm/core": patch
+"@pnpm/lockfile-utils": patch
+"@pnpm/headless": patch
+"pnpm": patch
+---
+
+Should always treat local file dependency as new dependency [#5381](https://github.com/pnpm/pnpm/issues/5381)

lockfile/lockfile-utils/src/index.ts

@@ -6,6 +6,7 @@ export { packageIdFromSnapshot } from './packageIdFromSnapshot'
 export { packageIsIndependent } from './packageIsIndependent'
 export { pkgSnapshotToResolution } from './pkgSnapshotToResolution'
 export { satisfiesPackageManifest } from './satisfiesPackageManifest'
+export { refIsLocalTarball, refIsLocalDirectory } from './refIsLocalTarball'
 export * from '@pnpm/lockfile-types'
 
 // for backward compatibility

lockfile/lockfile-utils/src/refIsLocalTarball.ts

@@ -0,0 +1,7 @@
+export function refIsLocalTarball (ref: string) {
+  return ref.startsWith('file:') && (ref.endsWith('.tgz') || ref.endsWith('.tar.gz') || ref.endsWith('.tar'))
+}
+
+export function refIsLocalDirectory (ref: string) {
+  return ref.startsWith('file:') && !refIsLocalTarball(ref)
+}

pkg-manager/core/src/install/allProjectsAreUpToDate.ts

@@ -1,14 +1,17 @@
 import path from 'path'
 import { type ProjectOptions } from '@pnpm/get-context'
 import {
+  type PackageSnapshot,
   type Lockfile,
   type ProjectSnapshot,
+  type PackageSnapshots,
 } from '@pnpm/lockfile-file'
-import { satisfiesPackageManifest } from '@pnpm/lockfile-utils'
+import { refIsLocalDirectory, refIsLocalTarball, satisfiesPackageManifest } from '@pnpm/lockfile-utils'
 import { safeReadPackageJsonFromDir } from '@pnpm/read-package-json'
-import { type WorkspacePackages } from '@pnpm/resolver-base'
+import { type DirectoryResolution, type WorkspacePackages } from '@pnpm/resolver-base'
 import {
   DEPENDENCIES_FIELDS,
+  DEPENDENCIES_OR_PEER_FIELDS,
   type DependencyManifest,
   type ProjectManifest,
 } from '@pnpm/types'
@@ -24,6 +27,7 @@ export async function allProjectsAreUpToDate (
     linkWorkspacePackages: boolean
     wantedLockfile: Lockfile
     workspacePackages: WorkspacePackages
+    lockfileDir: string
   }
 ) {
   const manifestsByDir = opts.workspacePackages ? getWorkspacePackagesByDirectory(opts.workspacePackages) : {}
@@ -35,6 +39,8 @@ export async function allProjectsAreUpToDate (
     linkWorkspacePackages: opts.linkWorkspacePackages,
     manifestsByDir,
     workspacePackages: opts.workspacePackages,
+    lockfilePackages: opts.wantedLockfile.packages,
+    lockfileDir: opts.lockfileDir,
   })
   return pEvery(projects, (project) => {
     const importer = opts.wantedLockfile.importers[project.id]
@@ -63,10 +69,14 @@ async function linkedPackagesAreUpToDate (
     linkWorkspacePackages,
     manifestsByDir,
     workspacePackages,
+    lockfilePackages,
+    lockfileDir,
   }: {
     linkWorkspacePackages: boolean
     manifestsByDir: Record<string, DependencyManifest>
     workspacePackages: WorkspacePackages
+    lockfilePackages?: PackageSnapshots
+    lockfileDir: string
   },
   project: {
     dir: string
@@ -87,6 +97,9 @@ async function linkedPackagesAreUpToDate (
           const currentSpec = manifestDeps[depName]
           if (!currentSpec) return true
           const lockfileRef = lockfileDeps[depName]
+          if (refIsLocalDirectory(project.snapshot.specifiers[depName])) {
+            return isLocalFileDepUpdated(lockfileDir, lockfilePackages?.[lockfileRef])
+          }
           const isLinked = lockfileRef.startsWith('link:')
           if (
             isLinked &&
@@ -120,6 +133,40 @@ async function linkedPackagesAreUpToDate (
   )
 }
 
+async function isLocalFileDepUpdated (lockfileDir: string, pkgSnapshot: PackageSnapshot | undefined) {
+  if (!pkgSnapshot) return false
+  const localDepDir = path.join(lockfileDir, (pkgSnapshot.resolution as DirectoryResolution).directory)
+  const manifest = await safeReadPackageJsonFromDir(localDepDir)
+  if (!manifest) return false
+  for (const depField of DEPENDENCIES_OR_PEER_FIELDS) {
+    if (depField === 'devDependencies') continue
+    const manifestDeps = manifest[depField] ?? {}
+    const lockfileDeps = pkgSnapshot[depField] ?? {}
+
+    // Lock file has more dependencies than the current manifest, e.g. some dependencies are removed.
+    if (Object.keys(lockfileDeps).some(depName => !manifestDeps[depName])) {
+      return false
+    }
+
+    for (const depName of Object.keys(manifestDeps)) {
+      // If a dependency does not exist in the lock file, e.g. a new dependency is added to the current manifest.
+      // We need to do full resolution again.
+      if (!lockfileDeps[depName]) {
+        return false
+      }
+      const currentSpec = manifestDeps[depName]
+      // We do not care about the link dependencies of local dependency.
+      if (currentSpec.startsWith('file:') || currentSpec.startsWith('link:') || currentSpec.startsWith('workspace:')) continue
+      if (semver.satisfies(lockfileDeps[depName], getVersionRange(currentSpec), { loose: true })) {
+        continue
+      } else {
+        return false
+      }
+    }
+  }
+  return true
+}
+
 function getVersionRange (spec: string) {
   if (spec.startsWith('workspace:')) return spec.slice(10)
   if (spec.startsWith('npm:')) {
@@ -136,7 +183,3 @@ function hasLocalTarballDepsInRoot (importer: ProjectSnapshot) {
     any(refIsLocalTarball, Object.values(importer.devDependencies ?? {})) ||
     any(refIsLocalTarball, Object.values(importer.optionalDependencies ?? {}))
 }
-
-function refIsLocalTarball (ref: string) {
-  return ref.startsWith('file:') && (ref.endsWith('.tgz') || ref.endsWith('.tar.gz') || ref.endsWith('.tar'))
-}

pkg-manager/core/src/install/index.ts

@@ -386,6 +386,7 @@ export async function mutateModules (
           linkWorkspacePackages: opts.linkWorkspacePackagesDepth >= 0,
           wantedLockfile: ctx.wantedLockfile,
           workspacePackages: opts.workspacePackages,
+          lockfileDir: opts.lockfileDir,
         })
       )
     ) {

pkg-manager/core/src/install/link.ts

@@ -37,6 +37,7 @@ import pick from 'ramda/src/pick'
 import pickBy from 'ramda/src/pickBy'
 import props from 'ramda/src/props'
 import { type ImporterToUpdate } from './index'
+import { refIsLocalDirectory } from '@pnpm/lockfile-utils'
 
 const brokenModulesLogger = logger('_broken_node_modules')
 
@@ -389,6 +390,9 @@ async function selectNewFromWantedDeps (
         const prevDep = prevDeps[depPath]
         if (
           prevDep &&
+          // Local file should always be treated as a new dependency
+          // https://github.com/pnpm/pnpm/issues/5381
+          !refIsLocalDirectory(depNode.depPath) &&
           (depNode.resolution as TarballResolution).integrity === (prevDep.resolution as TarballResolution).integrity
         ) {
           if (await pathExists(depNode.dir)) {

pkg-manager/core/test/allProjectsAreUpToDate.test.ts

@@ -1,4 +1,7 @@
+import { prepareEmpty } from '@pnpm/prepare'
 import { allProjectsAreUpToDate } from '../lib/install/allProjectsAreUpToDate'
+import { writeFile, mkdir } from 'fs/promises'
+import { type Lockfile } from '@pnpm/lockfile-file'
 
 const fooManifest = {
   name: 'foo',
@@ -52,6 +55,7 @@ test('allProjectsAreUpToDate(): works with packages linked through the workspace
       lockfileVersion: 5,
     },
     workspacePackages,
+    lockfileDir: '',
   })).toBeTruthy()
 })
 
@@ -94,6 +98,7 @@ test('allProjectsAreUpToDate(): works with aliased local dependencies', async ()
       lockfileVersion: 5,
     },
     workspacePackages,
+    lockfileDir: '',
   })).toBeTruthy()
 })
 
@@ -136,6 +141,7 @@ test('allProjectsAreUpToDate(): works with aliased local dependencies that speci
       lockfileVersion: 5,
     },
     workspacePackages,
+    lockfileDir: '',
   })).toBeTruthy()
 })
 
@@ -178,6 +184,7 @@ test('allProjectsAreUpToDate(): returns false if the aliased dependency version
       lockfileVersion: 5,
     },
     workspacePackages,
+    lockfileDir: '',
   })).toBeFalsy()
 })
 
@@ -271,6 +278,7 @@ test('allProjectsAreUpToDate(): use link and registry version if linkWorkspacePa
           lockfileVersion: 5,
         },
         workspacePackages,
+        lockfileDir: '',
       }
     )
   ).toBeTruthy()
@@ -320,6 +328,7 @@ test('allProjectsAreUpToDate(): returns false if dependenciesMeta differs', asyn
       lockfileVersion: 5,
     },
     workspacePackages,
+    lockfileDir: '',
   })).toBeFalsy()
 })
 
@@ -372,5 +381,119 @@ test('allProjectsAreUpToDate(): returns true if dependenciesMeta matches', async
       lockfileVersion: 5,
     },
     workspacePackages,
+    lockfileDir: '',
   })).toBeTruthy()
 })
+
+describe('local file dependency', () => {
+  beforeEach(async () => {
+    prepareEmpty()
+    await mkdir('local-dir')
+    await writeFile('./local-dir/package.json', JSON.stringify({
+      name: 'local-dir',
+      version: '1.0.0',
+      dependencies: {
+        'is-positive': '2.0.0',
+      },
+    }))
+  })
+  const projects = [
+    {
+      buildIndex: 0,
+      id: 'bar',
+      manifest: {
+        dependencies: {
+          local: 'file:./local-dir',
+        },
+      },
+      rootDir: 'bar',
+    },
+    {
+      buildIndex: 0,
+      id: 'foo',
+      manifest: fooManifest,
+      rootDir: 'foo',
+    },
+  ]
+  const options = {
+    autoInstallPeers: false,
+    excludeLinksFromLockfile: false,
+    linkWorkspacePackages: true,
+    wantedLockfile: {
+      importers: {
+        bar: {
+          dependencies: {
+            local: 'file:./local-dir',
+          },
+          specifiers: {
+            local: 'file:./local-dir',
+          },
+        },
+        foo: {
+          specifiers: {},
+        },
+      },
+      packages: {
+        'file:./local-dir': {
+          resolution: { directory: './local-dir', type: 'directory' },
+          name: 'local-dir',
+          version: '1.0.0',
+          dependencies: {
+            'is-positive': '2.0.0',
+          },
+          dev: false,
+        },
+      },
+      lockfileVersion: 5,
+    } as Lockfile,
+    workspacePackages,
+    lockfileDir: process.cwd(),
+  }
+  test('allProjectsAreUpToDate(): returns true if local file not changed', async () => {
+    expect(await allProjectsAreUpToDate(projects, {
+      ...options,
+      lockfileDir: process.cwd(),
+    })).toBeTruthy()
+  })
+
+  test('allProjectsAreUpToDate(): returns false if add new dependency to local file', async () => {
+    await writeFile('./local-dir/package.json', JSON.stringify({
+      name: 'local-dir',
+      version: '1.0.0',
+      dependencies: {
+        'is-positive': '2.0.0',
+        'is-odd': '1.0.0',
+      },
+    }))
+    expect(await allProjectsAreUpToDate(projects, {
+      ...options,
+      lockfileDir: process.cwd(),
+    })).toBeFalsy()
+  })
+
+  test('allProjectsAreUpToDate(): returns false if update dependency in local file', async () => {
+    await writeFile('./local-dir/package.json', JSON.stringify({
+      name: 'local-dir',
+      version: '1.0.0',
+      dependencies: {
+        'is-positive': '3.0.0',
+      },
+    }))
+    expect(await allProjectsAreUpToDate(projects, {
+      ...options,
+      lockfileDir: process.cwd(),
+    })).toBeFalsy()
+  })
+
+  test('allProjectsAreUpToDate(): returns false if remove dependency in local file', async () => {
+    await writeFile('./local-dir/package.json', JSON.stringify({
+      name: 'local-dir',
+      version: '1.0.0',
+      dependencies: {},
+    }))
+    expect(await allProjectsAreUpToDate(projects, {
+      ...options,
+      lockfileDir: process.cwd(),
+    })).toBeFalsy()
+  })
+})

pkg-manager/core/test/install/local.ts

@@ -371,3 +371,104 @@ test('resolution should not fail when a peer is resolved from a local package an
   await mutateModules(importers, await testDefaults({ allProjects, lockfileOnly: true, strictPeerDependencies: false }))
   // All we need to know in this test is that installation doesn't fail
 })
+
+test('re-install should update local file dependency', async () => {
+  const project = prepareEmpty()
+  f.copy('local-pkg', path.resolve('..', 'local-pkg'))
+
+  const manifest = await addDependenciesToPackage({}, ['file:../local-pkg'], await testDefaults())
+
+  const expectedSpecs = { 'local-pkg': `file:..${path.sep}local-pkg` }
+  expect(manifest.dependencies).toStrictEqual(expectedSpecs)
+
+  const m = project.requireModule('local-pkg')
+
+  expect(m).toBeTruthy()
+  await expect(fs.access('./node_modules/local-pkg/add.js')).rejects.toThrow()
+
+  let lockfile = await project.readLockfile()
+
+  expect(lockfile).toStrictEqual({
+    settings: {
+      autoInstallPeers: true,
+      excludeLinksFromLockfile: false,
+    },
+    dependencies: {
+      'local-pkg': {
+        specifier: expectedSpecs['local-pkg'],
+        version: 'file:../local-pkg',
+      },
+    },
+    packages: {
+      'file:../local-pkg': {
+        resolution: { directory: '../local-pkg', type: 'directory' },
+        name: 'local-pkg',
+        version: '1.0.0',
+        dev: false,
+      },
+    },
+    lockfileVersion: LOCKFILE_VERSION,
+  })
+
+  // add file
+  await fs.writeFile('../local-pkg/add.js', 'added', 'utf8')
+  await install(manifest, await testDefaults())
+  await expect(fs.access('./node_modules/local-pkg/add.js')).resolves.toBeUndefined()
+
+  // remove file
+  await fs.rm('../local-pkg/add.js')
+  await install(manifest, await testDefaults())
+  await expect(fs.access('./node_modules/local-pkg/add.js')).rejects.toThrow()
+
+  // add dependency
+  await expect(fs.access('./node_modules/.pnpm/is-positive@1.0.0')).rejects.toThrow()
+  await fs.writeFile('../local-pkg/package.json', JSON.stringify({
+    name: 'local-pkg',
+    version: '1.0.0',
+    dependencies: {
+      'is-positive': '1.0.0',
+    },
+  }), 'utf8')
+  await install(manifest, await testDefaults())
+  await expect(fs.access('./node_modules/.pnpm/is-positive@1.0.0')).resolves.toBeUndefined()
+  lockfile = await project.readLockfile()
+  expect(lockfile).toMatchObject({
+    packages: {
+      'file:../local-pkg': {
+        resolution: { directory: '../local-pkg', type: 'directory' },
+        name: 'local-pkg',
+        version: '1.0.0',
+        dev: false,
+        dependencies: {
+          'is-positive': '1.0.0',
+        },
+      },
+    },
+  })
+
+  // update dependency
+  await fs.writeFile('../local-pkg/package.json', JSON.stringify({
+    name: 'local-pkg',
+    version: '1.0.0',
+    dependencies: {
+      'is-positive': '2.0.0',
+    },
+  }), 'utf8')
+  await install(manifest, await testDefaults())
+  await expect(fs.access('./node_modules/.pnpm/is-positive@2.0.0')).resolves.toBeUndefined()
+  lockfile = await project.readLockfile()
+  expect(lockfile).toMatchObject({
+    packages: {
+      'file:../local-pkg': {
+        resolution: { directory: '../local-pkg', type: 'directory' },
+        name: 'local-pkg',
+        version: '1.0.0',
+        dev: false,
+        dependencies: {
+          'is-positive': '2.0.0',
+        },
+      },
+    },
+    lockfileVersion: LOCKFILE_VERSION,
+  })
+})

pkg-manager/headless/src/lockfileToDepGraph.ts

@@ -11,6 +11,7 @@ import {
   nameVerFromPkgSnapshot,
   packageIdFromSnapshot,
   pkgSnapshotToResolution,
+  refIsLocalDirectory,
 } from '@pnpm/lockfile-utils'
 import { logger } from '@pnpm/logger'
 import { type IncludedDependencies } from '@pnpm/modules-yaml'
@@ -127,6 +128,7 @@ export async function lockfileToDepGraph (
         }
         const dir = path.join(modules, pkgName)
         if (
+          !refIsLocalDirectory(depPath) &&
           currentPackages[depPath] && equals(currentPackages[depPath].dependencies, lockfile.packages![depPath].dependencies) &&
           equals(currentPackages[depPath].optionalDependencies, lockfile.packages![depPath].optionalDependencies)
         ) {