feat: allow to use a custom package importer (#5141)

.changeset/thick-pens-glow.md

@@ -0,0 +1,13 @@
+---
+"@pnpm/config": minor
+"@pnpm/core": minor
+"@pnpm/create-cafs-store": minor
+"@pnpm/fs.indexed-pkg-importer": minor
+"@pnpm/package-store": minor
+"pnpm": minor
+"@pnpm/pnpmfile": minor
+"@pnpm/store-connection-manager": minor
+"@pnpm/store-controller-types": minor
+---
+
+Support a new hook for passing a custom package importer to the store controller.

packages/create-cafs-store/src/index.ts

@@ -6,6 +6,7 @@ import createCafs, {
 import { PackageFilesResponse } from '@pnpm/fetcher-base'
 import { createIndexedPkgImporter } from '@pnpm/fs.indexed-pkg-importer'
 import {
+  ImportIndexedPackage,
   ImportPackageFunction,
   PackageFileInfo,
 } from '@pnpm/store-controller-types'
@@ -14,11 +15,14 @@ import pathTemp from 'path-temp'
 
 function createPackageImporter (
   opts: {
+    importIndexedPackage?: ImportIndexedPackage
     packageImportMethod?: 'auto' | 'hardlink' | 'copy' | 'clone' | 'clone-or-copy'
     cafsDir: string
   }
 ): ImportPackageFunction {
-  const cachedImporterCreator = memoize(createIndexedPkgImporter)
+  const cachedImporterCreator = opts.importIndexedPackage
+    ? () => opts.importIndexedPackage!
+    : memoize(createIndexedPkgImporter)
   const packageImportMethod = opts.packageImportMethod
   const gfm = getFlatMap.bind(null, opts.cafsDir)
   return async (to, opts) => {
@@ -63,12 +67,14 @@ export default function createCafsStore (
   storeDir: string,
   opts?: {
     ignoreFile?: (filename: string) => boolean
+    importPackage?: ImportIndexedPackage
     packageImportMethod?: 'auto' | 'hardlink' | 'copy' | 'clone' | 'clone-or-copy'
   }
 ) {
   const cafsDir = path.join(storeDir, 'files')
   const baseTempDir = path.join(storeDir, 'tmp')
   const importPackage = createPackageImporter({
+    importIndexedPackage: opts?.importPackage,
     packageImportMethod: opts?.packageImportMethod,
     cafsDir,
   })

packages/fs.indexed-pkg-importer/package.json

@@ -16,6 +16,7 @@
   },
   "dependencies": {
     "@pnpm/core-loggers": "workspace:*",
+    "@pnpm/store-controller-types": "workspace:*",
     "@zkochan/rimraf": "^2.1.2",
     "make-empty-dir": "^2.0.0",
     "p-limit": "^3.1.0",

packages/fs.indexed-pkg-importer/src/index.ts

@@ -2,25 +2,16 @@ import { constants, promises as fs, Stats } from 'fs'
 import path from 'path'
 import { globalInfo, globalWarn } from '@pnpm/logger'
 import { packageImportMethodLogger } from '@pnpm/core-loggers'
+import { FilesMap, ImportOptions, ImportIndexedPackage } from '@pnpm/store-controller-types'
 import pLimit from 'p-limit'
 import exists from 'path-exists'
 import importIndexedDir, { ImportFile } from './importIndexedDir'
 
 const limitLinking = pLimit(16)
 
-type FilesMap = Record<string, string>
-
-interface ImportOptions {
-  filesMap: FilesMap
-  force: boolean
-  fromStore: boolean
-}
-
-type ImportFunction = (to: string, opts: ImportOptions) => Promise<string | undefined>
-
 export function createIndexedPkgImporter (
   packageImportMethod?: 'auto' | 'hardlink' | 'copy' | 'clone' | 'clone-or-copy'
-): ImportFunction {
+): ImportIndexedPackage {
   const importPackage = createImportPackage(packageImportMethod)
   return async (to, opts) => limitLinking(async () => importPackage(to, opts))
 }
@@ -51,7 +42,7 @@ function createImportPackage (packageImportMethod?: 'auto' | 'hardlink' | 'copy'
   }
 }
 
-function createAutoImporter (): ImportFunction {
+function createAutoImporter (): ImportIndexedPackage {
   let auto = initialAuto
 
   return async (to, opts) => auto(to, opts)
@@ -89,7 +80,7 @@ function createAutoImporter (): ImportFunction {
   }
 }
 
-function createCloneOrCopyImporter (): ImportFunction {
+function createCloneOrCopyImporter (): ImportIndexedPackage {
   let auto = initialAuto
 
   return async (to, opts) => auto(to, opts)

packages/fs.indexed-pkg-importer/tsconfig.json

@@ -14,6 +14,9 @@
     },
     {
       "path": "../core-loggers"
+    },
+    {
+      "path": "../store-controller-types"
     }
   ]
 }

packages/package-store/src/storeController/index.ts

@@ -6,6 +6,7 @@ import { FetchFunction } from '@pnpm/fetcher-base'
 import createPackageRequester from '@pnpm/package-requester'
 import { ResolveFunction } from '@pnpm/resolver-base'
 import {
+  ImportIndexedPackage,
   PackageFileInfo,
   StoreController,
 } from '@pnpm/store-controller-types'
@@ -20,6 +21,7 @@ export default async function (
     engineStrict?: boolean
     force?: boolean
     nodeVersion?: string
+    importPackage?: ImportIndexedPackage
     pnpmVersion?: string
     ignoreFile?: (filename: string) => boolean
     storeDir: string

packages/pnpm/test/hooks.ts

@@ -99,3 +99,26 @@ test('filterLog hook filters peer dependency warning', async () => {
     expect.not.stringContaining('requires a peer of rollup')
   )
 })
+
+test('importPackage hooks', async () => {
+  prepare()
+  const pnpmfile = `
+const fs = require('fs')
+
+module.exports = { hooks: { importPackage } }
+
+function importPackage (to, opts) {
+  fs.writeFileSync('args.json', JSON.stringify([to, opts]), 'utf8')
+  return {}
+}`
+  await fs.writeFile('.pnpmfile.cjs', pnpmfile, 'utf8')
+  await execPnpm(['add', 'is-positive@1.0.0'])
+  const [to, opts] = await loadJsonFile<any>('args.json') // eslint-disable-line
+  expect(typeof to).toBe('string')
+  expect(Object.keys(opts.filesMap).sort()).toStrictEqual([
+    'index.js',
+    'license',
+    'package.json',
+    'readme.md',
+  ])
+})

packages/pnpmfile/package.json

@@ -36,6 +36,7 @@
     "@pnpm/core-loggers": "workspace:*",
     "@pnpm/error": "workspace:*",
     "@pnpm/lockfile-types": "workspace:*",
+    "@pnpm/store-controller-types": "workspace:*",
     "@pnpm/types": "workspace:*",
     "chalk": "^4.1.2",
     "path-absolute": "^1.0.1"

packages/pnpmfile/src/requireHooks.ts

@@ -3,6 +3,7 @@ import { hookLogger } from '@pnpm/core-loggers'
 import pathAbsolute from 'path-absolute'
 import type { Lockfile } from '@pnpm/lockfile-types'
 import type { Log } from '@pnpm/core-loggers'
+import { ImportIndexedPackage } from '@pnpm/store-controller-types'
 import requirePnpmfile from './requirePnpmfile'
 
 interface HookContext {
@@ -14,6 +15,7 @@ interface Hooks {
   readPackage?: (pkg: any, context: HookContext) => any
   afterAllResolved?: (lockfile: Lockfile, context: HookContext) => Lockfile | Promise<Lockfile>
   filterLog?: (log: Log) => boolean
+  importPackage?: ImportIndexedPackage
 }
 
 // eslint-disable-next-line
@@ -27,6 +29,7 @@ export interface CookedHooks {
   readPackage?: Cook<Required<Hooks>['readPackage']>
   afterAllResolved?: Cook<Required<Hooks>['afterAllResolved']>
   filterLog?: Cook<Required<Hooks>['filterLog']>
+  importPackage?: ImportIndexedPackage
 }
 
 export default function requireHooks (
@@ -75,6 +78,7 @@ export default function requireHooks (
   } else {
     cookedHooks.filterLog = globalFilterLog ?? filterLog
   }
+  cookedHooks.importPackage = hooks.importPackage ?? globalHooks.importPackage
   return cookedHooks
 }
 

packages/pnpmfile/tsconfig.json

@@ -18,6 +18,9 @@
     {
       "path": "../lockfile-types"
     },
+    {
+      "path": "../store-controller-types"
+    },
     {
       "path": "../types"
     }

packages/store-connection-manager/src/createNewStoreController.ts

@@ -22,6 +22,7 @@ export type CreateNewStoreControllerOptions = CreateResolverOptions & Pick<Confi
 | 'nodeVersion'
 | 'fetchTimeout'
 | 'gitShallowHosts'
+| 'hooks'
 | 'httpProxy'
 | 'httpsProxy'
 | 'key'
@@ -81,6 +82,7 @@ export default async (
       nodeVersion: opts.nodeVersion,
       pnpmVersion: pnpm.version,
       ignoreFile: opts.ignoreFile,
+      importPackage: opts.hooks?.importPackage,
       networkConcurrency: opts.networkConcurrency,
       packageImportMethod: opts.packageImportMethod,
       storeDir: opts.storeDir,

packages/store-controller-types/src/index.ts

@@ -136,3 +136,13 @@ export interface PackageResponse {
     }
   )
 }
+
+export type FilesMap = Record<string, string>
+
+export interface ImportOptions {
+  filesMap: FilesMap
+  force: boolean
+  fromStore: boolean
+}
+
+export type ImportIndexedPackage = (to: string, opts: ImportOptions) => Promise<string | undefined>

pnpm-lock.yaml

@@ -1353,6 +1353,9 @@ importers:
       '@pnpm/core-loggers':
         specifier: workspace:*
         version: link:../core-loggers
+      '@pnpm/store-controller-types':
+        specifier: workspace:*
+        version: link:../store-controller-types
       '@zkochan/rimraf':
         specifier: ^2.1.2
         version: 2.1.2
@@ -4383,6 +4386,9 @@ importers:
       '@pnpm/lockfile-types':
         specifier: workspace:*
         version: link:../lockfile-types
+      '@pnpm/store-controller-types':
+        specifier: workspace:*
+        version: link:../store-controller-types
       '@pnpm/types':
         specifier: workspace:*
         version: link:../types