feat(install): add --fix-lockfile for install to support autofix broken lockfile (#3729)

ref #3659

Co-authored-by: wumingliang.0113 <wumingliang.0113@bytedance.com>
Co-authored-by: Zoltan Kochan <z@kochan.io>

.changeset/fuzzy-adults-suffer.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/package-requester": patch
+---
+
+Always fetch the bundled manifest.

.changeset/tender-grapes-leave.md

@@ -0,0 +1,6 @@
+---
+"@pnpm/plugin-commands-installation": minor
+"supi": patch
+---
+
+Adding --fix-lockfile for the install command to support autofix broken lockfile

.changeset/three-knives-flow.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/resolve-dependencies": patch
+---
+
+`requiresBuild` fields should be updated when a full resolution is forced.

packages/package-requester/src/packageRequester.ts

@@ -234,7 +234,7 @@ async function resolveAndFetch (
   }
 
   const fetchResult = ctx.fetchPackageToStore({
-    fetchRawManifest: updated || (manifest == null),
+    fetchRawManifest: true,
     force: forceFetch,
     lockfileDir: options.lockfileDir,
     pkg: {

packages/plugin-commands-installation/src/install.ts

@@ -67,6 +67,7 @@ export function rcOptionsTypes () {
 export const cliOptionsTypes = () => ({
   ...rcOptionsTypes(),
   ...pick(['force'], allTypes),
+  'fix-lockfile': Boolean,
   recursive: Boolean,
 })
 
@@ -131,6 +132,10 @@ For options that may be used with `-r`, see "pnpm help recursive"',
             description: `The directory in which the ${WANTED_LOCKFILE} of the package will be created. Several projects may share a single lockfile.`,
             name: '--lockfile-dir <dir>',
           },
+          {
+            description: 'Fix broken lockfile entries automatically',
+            name: '--fix-lockfile',
+          },
           {
             description: 'The directory in which dependencies will be installed (instead of node_modules)',
             name: '--modules-dir <dir>',
@@ -286,6 +291,7 @@ export type InstallCommandOptions = Pick<Config,
   argv: {
     original: string[]
   }
+  fixLockfile?: boolean
   useBetaCli?: boolean
   recursive?: boolean
   workspace?: boolean

packages/resolve-dependencies/src/index.ts

@@ -173,8 +173,17 @@ export default async function (
       }
     }
   }
+
   const { newLockfile, pendingRequiresBuilds } = updateLockfile(dependenciesGraph, opts.wantedLockfile, opts.virtualStoreDir, opts.registries) // eslint-disable-line:prefer-const
 
+  if (opts.forceFullResolution && opts.wantedLockfile != null) {
+    for (const [depPath, pkg] of Object.entries(dependenciesGraph)) {
+      // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing
+      if (opts.neverBuiltDependencies?.has(pkg.name) || opts.wantedLockfile.packages?.[depPath] == null || pkg.requiresBuild) continue
+      pendingRequiresBuilds.push(depPath)
+    }
+  }
+
   // waiting till package requests are finished
   const waitTillAllFetchingsFinish = async () => Promise.all(Object.values(resolvedPackagesByDepPath).map(async ({ finishing }) => finishing?.()))
 

packages/resolve-dependencies/src/resolveDependencies.ts

@@ -526,7 +526,9 @@ function getInfoFromLockfile (
       dependencyLockfile,
       depPath,
       pkgId: packageIdFromSnapshot(depPath, dependencyLockfile, registries),
-      resolution: pkgSnapshotToResolution(depPath, dependencyLockfile, registries),
+      // resolution may not exist if lockfile is broken, and an unexpected error will be thrown
+      // if resolution does not exist, return undefined so it can be autofixed later
+      resolution: dependencyLockfile.resolution && pkgSnapshotToResolution(depPath, dependencyLockfile, registries),
     }
   } else {
     return {

packages/supi/src/install/extendInstallOptions.ts

@@ -21,6 +21,7 @@ export interface StrictInstallOptions {
   useLockfile: boolean
   linkWorkspacePackagesDepth: number
   lockfileOnly: boolean
+  fixLockfile: boolean
   ignorePackageManifest: boolean
   preferFrozenLockfile: boolean
   saveWorkspaceProtocol: boolean

packages/supi/src/install/index.ts

@@ -194,7 +194,8 @@ export async function mutateModules (
     let needsFullResolution = !maybeOpts.ignorePackageManifest && (
       !equals(ctx.wantedLockfile.overrides ?? {}, overrides ?? {}) ||
       !equals((ctx.wantedLockfile.neverBuiltDependencies ?? []).sort(), (neverBuiltDependencies ?? []).sort()) ||
-      ctx.wantedLockfile.packageExtensionsChecksum !== packageExtensionsChecksum)
+      ctx.wantedLockfile.packageExtensionsChecksum !== packageExtensionsChecksum) ||
+      opts.fixLockfile
     if (needsFullResolution) {
       ctx.wantedLockfile.overrides = overrides
       ctx.wantedLockfile.neverBuiltDependencies = neverBuiltDependencies
@@ -206,6 +207,7 @@ export async function mutateModules (
       !ctx.lockfileHadConflicts &&
       !opts.lockfileOnly &&
       !opts.update &&
+      !opts.fixLockfile &&
       installsOnly &&
       (
         frozenLockfile ||
@@ -725,6 +727,23 @@ const _installInContext: InstallFunction = async (projects, ctx, opts) => {
     workspacePackages: opts.workspacePackages,
   })
   const projectsToResolve = await Promise.all(projects.map(async (project) => _toResolveImporter(project)))
+
+  // Ignore some field when fixing lockfile, so this filed can be regenereated
+  // and make sure it's up-to-date
+  if (
+    opts.fixLockfile &&
+    (ctx.wantedLockfile.packages != null) &&
+    !isEmpty(ctx.wantedLockfile.packages)
+  ) {
+    ctx.wantedLockfile.packages = Object.entries(ctx.wantedLockfile.packages).reduce((pre, [depPath, snapshot]) => ({
+      ...pre,
+      [depPath]: {
+        name: snapshot.name,
+        version: snapshot.version,
+      },
+    }), {})
+  }
+
   let {
     dependenciesGraph,
     dependenciesByProjectId,

packages/supi/test/install/fixLockfile.ts

@@ -0,0 +1,61 @@
+import { LOCKFILE_VERSION, WANTED_LOCKFILE } from '@pnpm/constants'
+import { prepareEmpty } from '@pnpm/prepare'
+import { install } from 'supi'
+import writeYamlFile from 'write-yaml-file'
+import readYamlFile from 'read-yaml-file'
+import { Lockfile, PackageSnapshots } from '@pnpm/lockfile-file'
+import { testDefaults } from '../utils'
+
+test('fix broken lockfile with --fix-lockfile', async () => {
+  prepareEmpty()
+
+  await writeYamlFile(WANTED_LOCKFILE, {
+    dependencies: {
+      '@types/semver': '5.3.31',
+    },
+    devDependencies: {
+      fsevents: '2.3.2',
+    },
+    lockfileVersion: LOCKFILE_VERSION,
+    packages: {
+      '/@types/semver/5.3.31': {
+        // resolution: {
+        //   integrity: 'sha1-uZnX2TX0P1IHsBsA094ghS9Mp18=',
+        // },
+      },
+      '/core-js-pure/3.16.2': {
+        resolution: {
+          integrity: 'sha512-oxKe64UH049mJqrKkynWp6Vu0Rlm/BTXO/bJZuN2mmR3RtOFNepLlSWDd1eo16PzHpQAoNG97rLU1V/YxesJjw==',
+        },
+        // requiresBuild: true,
+        // dev: true
+      },
+    },
+    specifiers: {
+      '@types/semver': '^5.3.31',
+      fsevents: '^2.3.2',
+    },
+  }, { lineWidth: 1000 })
+
+  await install({
+    dependencies: {
+      '@types/semver': '^5.3.31',
+    },
+    devDependencies: {
+      'core-js-pure': '^3.16.2',
+    },
+  }, await testDefaults({ fixLockfile: true }))
+
+  const lockfile: Lockfile = await readYamlFile(WANTED_LOCKFILE)
+  expect(Object.keys(lockfile.packages as PackageSnapshots).length).toBe(2)
+  expect(lockfile.packages?.['/@types/semver/5.3.31']).toBeTruthy()
+  expect(lockfile.packages?.['/@types/semver/5.3.31']?.resolution).toEqual({
+    integrity: 'sha1-uZnX2TX0P1IHsBsA094ghS9Mp18=',
+  })
+  expect(lockfile.packages?.['/core-js-pure/3.16.2']).toBeTruthy()
+  expect(lockfile.packages?.['/core-js-pure/3.16.2']?.resolution).toEqual({
+    integrity: 'sha512-oxKe64UH049mJqrKkynWp6Vu0Rlm/BTXO/bJZuN2mmR3RtOFNepLlSWDd1eo16PzHpQAoNG97rLU1V/YxesJjw==',
+  })
+  expect(lockfile.packages?.['/core-js-pure/3.16.2']?.requiresBuild).toBeTruthy()
+  expect(lockfile.packages?.['/core-js-pure/3.16.2']?.dev).toBeTruthy()
+})