feat: add hook for custom fetchers (#5185)

* feat: add hook for custom fetchers

* fixup: simplify fetcher overwrites

* fixup: refactor some types and implement fetcher hooks

* fixup: add changeset

.changeset/shy-sloths-beam.md

@@ -0,0 +1,15 @@
+---
+"@pnpm/client": minor
+"@pnpm/directory-fetcher": minor
+"@pnpm/fetcher-base": minor
+"@pnpm/git-fetcher": minor
+"@pnpm/package-requester": minor
+"@pnpm/package-store": minor
+"@pnpm/pick-fetcher": minor
+"pnpm": minor
+"@pnpm/pnpmfile": minor
+"@pnpm/resolver-base": minor
+"@pnpm/store-connection-manager": minor
+---
+
+Add hook for adding custom fetchers.

packages/client/package.json

@@ -43,6 +43,7 @@
   },
   "devDependencies": {
     "@pnpm/client": "workspace:*",
+    "@pnpm/fetcher-base": "workspace:*",
     "@pnpm/logger": "^4.0.0"
   },
   "funding": "https://opencollective.com/pnpm",

packages/client/src/index.ts

@@ -4,6 +4,7 @@ import createResolve, {
 } from '@pnpm/default-resolver'
 import { AgentOptions, createFetchFromRegistry } from '@pnpm/fetch'
 import { FetchFromRegistry, GetCredentials, RetryTimeoutOptions } from '@pnpm/fetching-types'
+import type { CustomFetchers } from '@pnpm/fetcher-base'
 import createDirectoryFetcher from '@pnpm/directory-fetcher'
 import fetchFromGit from '@pnpm/git-fetcher'
 import createTarballFetcher from '@pnpm/tarball-fetcher'
@@ -14,6 +15,7 @@ export { ResolveFunction }
 
 export type ClientOptions = {
   authConfig: Record<string, string>
+  customFetchers?: CustomFetchers
   retry?: RetryTimeoutOptions
   timeout?: number
   userAgent?: string
@@ -25,7 +27,7 @@ export default function (opts: ClientOptions) {
   const fetchFromRegistry = createFetchFromRegistry(opts)
   const getCredentials = mem((registry: string) => getCredentialsByURI(opts.authConfig, registry, opts.userConfig))
   return {
-    fetchers: createFetchers(fetchFromRegistry, getCredentials, opts),
+    fetchers: createFetchers(fetchFromRegistry, getCredentials, opts, opts.customFetchers),
     resolve: createResolve(fetchFromRegistry, getCredentials, opts),
   }
 }
@@ -39,11 +41,23 @@ export function createResolver (opts: ClientOptions) {
 function createFetchers (
   fetchFromRegistry: FetchFromRegistry,
   getCredentials: GetCredentials,
-  opts: Pick<ClientOptions, 'retry' | 'gitShallowHosts'>
+  opts: Pick<ClientOptions, 'retry' | 'gitShallowHosts'>,
+  customFetchers?: CustomFetchers
 ) {
-  return {
+  const defaultFetchers = {
     ...createTarballFetcher(fetchFromRegistry, getCredentials, opts),
     ...fetchFromGit(opts),
     ...createDirectoryFetcher(),
   }
+
+  const overwrites = Object.entries(customFetchers ?? {})
+    .reduce((acc, [fetcherName, factory]) => {
+      acc[fetcherName] = factory({ defaultFetchers })
+      return acc
+    }, {})
+
+  return {
+    ...defaultFetchers,
+    ...overwrites,
+  }
 }

packages/client/tsconfig.json

@@ -18,6 +18,9 @@
     {
       "path": "../fetch"
     },
+    {
+      "path": "../fetcher-base"
+    },
     {
       "path": "../fetching-types"
     },

packages/directory-fetcher/src/index.ts

@@ -1,16 +1,10 @@
 import { promises as fs } from 'fs'
 import path from 'path'
-import { Cafs, DeferredManifestPromise } from '@pnpm/fetcher-base'
+import type { DirectoryFetcher, DirectoryFetcherOptions } from '@pnpm/fetcher-base'
 import { safeReadProjectManifestOnly } from '@pnpm/read-project-manifest'
-import { DirectoryResolution } from '@pnpm/resolver-base'
 import fromPairs from 'ramda/src/fromPairs'
 import packlist from 'npm-packlist'
 
-export interface DirectoryFetcherOptions {
-  lockfileDir: string
-  manifest?: DeferredManifestPromise
-}
-
 export interface CreateDirectoryFetcherOptions {
   includeOnlyPackageFiles?: boolean
 }
@@ -19,15 +13,14 @@ export default (
   opts?: CreateDirectoryFetcherOptions
 ) => {
   const fetchFromDir = opts?.includeOnlyPackageFiles ? fetchPackageFilesFromDir : fetchAllFilesFromDir
+
+  const directoryFetcher: DirectoryFetcher = (cafs, resolution, opts) => {
+    const dir = path.join(opts.lockfileDir, resolution.directory)
+    return fetchFromDir(dir, opts)
+  }
+
   return {
-    directory: (
-      cafs: Cafs,
-      resolution: DirectoryResolution,
-      opts: DirectoryFetcherOptions
-    ) => {
-      const dir = path.join(opts.lockfileDir, resolution.directory)
-      return fetchFromDir(dir, opts)
-    },
+    directory: directoryFetcher,
   }
 }
 

packages/fetcher-base/src/index.ts

@@ -1,4 +1,4 @@
-import { Resolution } from '@pnpm/resolver-base'
+import { Resolution, GitResolution, DirectoryResolution } from '@pnpm/resolver-base'
 import { DependencyManifest } from '@pnpm/types'
 import { IntegrityLike } from 'ssri'
 
@@ -52,11 +52,11 @@ export interface DeferredManifestPromise {
   reject: (err: Error) => void
 }
 
-export type FetchFunction = (
+export type FetchFunction<FetcherResolution = Resolution, Options = FetchOptions, Result = FetchResult> = (
   cafs: Cafs,
-  resolution: Resolution,
-  opts: FetchOptions
-) => Promise<FetchResult>
+  resolution: FetcherResolution,
+  opts: Options
+) => Promise<Result>
 
 export type FetchResult = {
   local?: false
@@ -78,3 +78,44 @@ export interface FilesIndex {
     writeResult: Promise<FileWriteResult>
   }
 }
+
+export interface GitFetcherOptions {
+  manifest?: DeferredManifestPromise
+}
+
+export type GitFetcher = FetchFunction<GitResolution, GitFetcherOptions, { filesIndex: FilesIndex }>
+
+export interface DirectoryFetcherOptions {
+  lockfileDir: string
+  manifest?: DeferredManifestPromise
+}
+
+export interface DirectoryFetcherResult {
+  local: true
+  filesIndex: Record<string, string>
+  packageImportMethod: 'hardlink'
+}
+
+export type DirectoryFetcher = FetchFunction<DirectoryResolution, DirectoryFetcherOptions, DirectoryFetcherResult>
+
+export interface Fetchers {
+  localTarball: FetchFunction
+  remoteTarball: FetchFunction
+  gitHostedTarball: FetchFunction
+  directory: DirectoryFetcher
+  git: GitFetcher
+}
+
+interface CustomFetcherFactoryOptions {
+  defaultFetchers: Fetchers
+}
+
+export type CustomFetcherFactory<Fetcher> = (opts: CustomFetcherFactoryOptions) => Fetcher
+
+export interface CustomFetchers {
+  localTarball?: CustomFetcherFactory<FetchFunction>
+  remoteTarball?: CustomFetcherFactory<FetchFunction>
+  gitHostedTarball?: CustomFetcherFactory<FetchFunction>
+  directory?: CustomFetcherFactory<DirectoryFetcher>
+  git?: CustomFetcherFactory<GitFetcher>
+}

packages/git-fetcher/src/index.ts

@@ -1,5 +1,5 @@
 import path from 'path'
-import { Cafs, DeferredManifestPromise } from '@pnpm/fetcher-base'
+import type { GitFetcher } from '@pnpm/fetcher-base'
 import preparePackage from '@pnpm/prepare-package'
 import rimraf from '@zkochan/rimraf'
 import execa from 'execa'
@@ -7,36 +7,29 @@ import { URL } from 'url'
 
 export default (createOpts?: { gitShallowHosts?: string[] }) => {
   const allowedHosts = new Set(createOpts?.gitShallowHosts ?? [])
+
+  const gitFetcher: GitFetcher = async (cafs, resolution, opts) => {
+    const tempLocation = await cafs.tempDir()
+    if (allowedHosts.size > 0 && shouldUseShallow(resolution.repo, allowedHosts)) {
+      await execGit(['init'], { cwd: tempLocation })
+      await execGit(['remote', 'add', 'origin', resolution.repo], { cwd: tempLocation })
+      await execGit(['fetch', '--depth', '1', 'origin', resolution.commit], { cwd: tempLocation })
+    } else {
+      await execGit(['clone', resolution.repo, tempLocation])
+    }
+    await execGit(['checkout', resolution.commit], { cwd: tempLocation })
+    await preparePackage(tempLocation)
+    // removing /.git to make directory integrity calculation faster
+    await rimraf(path.join(tempLocation, '.git'))
+    const filesIndex = await cafs.addFilesFromDir(tempLocation, opts.manifest)
+    // Important! We cannot remove the temp location at this stage.
+    // Even though we have the index of the package,
+    // the linking of files to the store is in progress.
+    return { filesIndex }
+  }
+
   return {
-    git: async function fetchFromGit (
-      cafs: Cafs,
-      resolution: {
-        commit: string
-        repo: string
-        type: 'git'
-      },
-      opts: {
-        manifest?: DeferredManifestPromise
-      }
-    ) {
-      const tempLocation = await cafs.tempDir()
-      if (allowedHosts.size > 0 && shouldUseShallow(resolution.repo, allowedHosts)) {
-        await execGit(['init'], { cwd: tempLocation })
-        await execGit(['remote', 'add', 'origin', resolution.repo], { cwd: tempLocation })
-        await execGit(['fetch', '--depth', '1', 'origin', resolution.commit], { cwd: tempLocation })
-      } else {
-        await execGit(['clone', resolution.repo, tempLocation])
-      }
-      await execGit(['checkout', resolution.commit], { cwd: tempLocation })
-      await preparePackage(tempLocation)
-      // removing /.git to make directory integrity calculation faster
-      await rimraf(path.join(tempLocation, '.git'))
-      const filesIndex = await cafs.addFilesFromDir(tempLocation, opts.manifest)
-      // Important! We cannot remove the temp location at this stage.
-      // Even though we have the index of the package,
-      // the linking of files to the store is in progress.
-      return { filesIndex }
-    },
+    git: gitFetcher,
   }
 }
 

packages/package-requester/src/packageRequester.ts

@@ -15,7 +15,7 @@ import PnpmError from '@pnpm/error'
 import {
   Cafs,
   DeferredManifestPromise,
-  FetchFunction,
+  Fetchers,
   FetchOptions,
   FetchResult,
   PackageFilesResponse,
@@ -88,7 +88,7 @@ export default function (
     nodeVersion?: string
     pnpmVersion?: string
     resolve: ResolveFunction
-    fetchers: {[type: string]: FetchFunction}
+    fetchers: Fetchers
     cafs: Cafs
     ignoreFile?: (filename: string) => boolean
     networkConcurrency?: number
@@ -657,7 +657,7 @@ async function tarballIsUpToDate (
 }
 
 async function fetcher (
-  fetcherByHostingType: {[hostingType: string]: FetchFunction},
+  fetcherByHostingType: Fetchers,
   cafs: Cafs,
   packageId: string,
   resolution: Resolution,

packages/package-store/src/storeController/index.ts

@@ -2,7 +2,7 @@ import {
   PackageFilesIndex,
 } from '@pnpm/cafs'
 import createCafsStore from '@pnpm/create-cafs-store'
-import { FetchFunction } from '@pnpm/fetcher-base'
+import { Fetchers } from '@pnpm/fetcher-base'
 import createPackageRequester from '@pnpm/package-requester'
 import { ResolveFunction } from '@pnpm/resolver-base'
 import {
@@ -16,7 +16,7 @@ import prune from './prune'
 
 export default async function (
   resolve: ResolveFunction,
-  fetchers: {[type: string]: FetchFunction},
+  fetchers: Fetchers,
   initOpts: {
     engineStrict?: boolean
     force?: boolean

packages/pick-fetcher/src/index.ts

@@ -1,7 +1,7 @@
 import type { Resolution } from '@pnpm/resolver-base'
-import type { FetchFunction } from '@pnpm/fetcher-base'
+import type { Fetchers } from '@pnpm/fetcher-base'
 
-export function pickFetcher (fetcherByHostingType: {[hostingType: string]: FetchFunction}, resolution: Resolution) {
+export function pickFetcher (fetcherByHostingType: Partial<Fetchers>, resolution: Resolution) {
   let fetcherType = resolution.type
 
   if (resolution.type == null) {

packages/pnpm/test/hooks.ts

@@ -132,3 +132,70 @@ test('importPackage hooks', async () => {
     'readme.md',
   ])
 })
+
+test('should use default fetchers if no custom fetchers are defined', async () => {
+  const project = prepare()
+
+  const pnpmfile = `
+    const fs = require('fs')
+
+    module.exports = {
+      hooks: {
+        fetchers: {}
+      }
+    }
+  `
+
+  const npmrc = `
+    global-pnpmfile=.pnpmfile.cjs
+  `
+
+  await fs.writeFile('.npmrc', npmrc, 'utf8')
+  await fs.writeFile('.pnpmfile.cjs', pnpmfile, 'utf8')
+
+  await execPnpm(['add', 'is-positive@1.0.0'])
+
+  await project.cafsHas('is-positive', '1.0.0')
+})
+
+test('custom fetcher can call default fetcher', async () => {
+  const project = prepare()
+
+  const pnpmfile = `
+    const fs = require('fs')
+
+    module.exports = {
+      hooks: {
+        fetchers: {
+          remoteTarball: ({ defaultFetchers }) => {
+            return (cafs, resolution, opts) => {
+              fs.writeFileSync('args.json', JSON.stringify({ resolution, opts }), 'utf8')
+              return defaultFetchers.remoteTarball(cafs, resolution, opts)
+            }
+          }
+        }
+      }
+    }
+  `
+
+  const npmrc = `
+    global-pnpmfile=.pnpmfile.cjs
+  `
+
+  await fs.writeFile('.npmrc', npmrc, 'utf8')
+  await fs.writeFile('.pnpmfile.cjs', pnpmfile, 'utf8')
+
+  await execPnpm(['add', 'is-positive@1.0.0'])
+
+  await project.cafsHas('is-positive', '1.0.0')
+
+  const args = await loadJsonFile<any>('args.json') // eslint-disable-line
+
+  expect(args.resolution).toEqual({
+    integrity: 'sha512-xxzPGZ4P2uN6rROUa5N9Z7zTX6ERuE0hs6GUOc/cKBLF2NqKc16UwqHMt3tFg4CO6EBTE5UecUasg+3jZx3Ckg==',
+    registry: 'http://localhost:7782/',
+    tarball: 'http://localhost:7782/is-positive/-/is-positive-1.0.0.tgz',
+  })
+
+  expect(args.opts).toBeDefined()
+})

packages/pnpmfile/package.json

@@ -29,6 +29,7 @@
   },
   "homepage": "https://github.com/pnpm/pnpm/blob/main/packages/pnpmfile#readme",
   "devDependencies": {
+    "@pnpm/fetcher-base": "workspace:*",
     "@pnpm/logger": "^4.0.0",
     "@pnpm/pnpmfile": "workspace:*"
   },

packages/pnpmfile/src/requireHooks.ts

@@ -4,6 +4,7 @@ import { hookLogger } from '@pnpm/core-loggers'
 import pathAbsolute from 'path-absolute'
 import type { Lockfile } from '@pnpm/lockfile-types'
 import type { Log } from '@pnpm/core-loggers'
+import type { CustomFetchers } from '@pnpm/fetcher-base'
 import { ImportIndexedPackage } from '@pnpm/store-controller-types'
 import requirePnpmfile from './requirePnpmfile'
 
@@ -18,6 +19,7 @@ interface Hooks {
   afterAllResolved?: (lockfile: Lockfile, context: HookContext) => Lockfile | Promise<Lockfile>
   filterLog?: (log: Log) => boolean
   importPackage?: ImportIndexedPackage
+  fetchers?: CustomFetchers
 }
 
 // eslint-disable-next-line
@@ -33,6 +35,7 @@ export interface CookedHooks {
   afterAllResolved?: Cook<Required<Hooks>['afterAllResolved']>
   filterLog?: Cook<Required<Hooks>['filterLog']>
   importPackage?: ImportIndexedPackage
+  fetchers?: CustomFetchers
 }
 
 export default function requireHooks (
@@ -82,7 +85,7 @@ export default function requireHooks (
     cookedHooks.filterLog = globalFilterLog ?? filterLog
   }
 
-  // `importPackage` and `preResolution` can only be defined via a global pnpmfile
+  // `importPackage`, `preResolution` and `fetchers` can only be defined via a global pnpmfile
 
   cookedHooks.importPackage = globalHooks.importPackage
 
@@ -92,6 +95,8 @@ export default function requireHooks (
     ? (ctx: PreResolutionHookContext) => preResolutionHook(ctx, createPreResolutionHookLogger(prefix))
     : undefined
 
+  cookedHooks.fetchers = globalHooks.fetchers
+
   return cookedHooks
 }
 

packages/pnpmfile/tsconfig.json

@@ -18,6 +18,9 @@
     {
       "path": "../error"
     },
+    {
+      "path": "../fetcher-base"
+    },
     {
       "path": "../lockfile-types"
     },

packages/resolver-base/src/index.ts

@@ -21,9 +21,16 @@ export interface DirectoryResolution {
   directory: string
 }
 
+export interface GitResolution {
+  commit: string
+  repo: string
+  type: 'git'
+}
+
 export type Resolution =
   TarballResolution |
   DirectoryResolution |
+  GitResolution |
   ({ type: string } & object)
 
 export interface ResolveResult {

packages/store-connection-manager/src/createNewStoreController.ts

@@ -45,6 +45,7 @@ export default async (
   opts: CreateNewStoreControllerOptions
 ) => {
   const { resolve, fetchers } = createClient({
+    customFetchers: opts.hooks?.fetchers,
     userConfig: opts.userConfig,
     authConfig: opts.rawConfig,
     ca: opts.ca,

pnpm-lock.yaml

@@ -429,6 +429,9 @@ importers:
       '@pnpm/client':
         specifier: workspace:*
         version: 'link:'
+      '@pnpm/fetcher-base':
+        specifier: workspace:*
+        version: link:../fetcher-base
       '@pnpm/logger':
         specifier: ^4.0.0
         version: 4.0.0
@@ -4393,6 +4396,9 @@ importers:
         specifier: ^1.0.1
         version: 1.0.1
     devDependencies:
+      '@pnpm/fetcher-base':
+        specifier: workspace:*
+        version: link:../fetcher-base
       '@pnpm/logger':
         specifier: ^4.0.0
         version: 4.0.0
@@ -8794,8 +8800,8 @@ packages:
     engines: {node: '>=10'}
     hasBin: true
     dependencies:
-      JSONStream: 1.3.5
       is-text-path: 1.0.1
+      JSONStream: 1.3.5
       lodash: 4.17.21
       meow: 8.1.2
       split2: 3.2.2
@@ -15470,7 +15476,6 @@ packages:
       '@verdaccio/readme': 10.4.1
       '@verdaccio/streams': 10.2.0
       '@verdaccio/ui-theme': 6.0.0-6-next.25
-      JSONStream: 1.3.5
       async: 3.2.4
       body-parser: 1.20.0
       clipanion: 3.1.0
@@ -15487,6 +15492,7 @@ packages:
       handlebars: 4.7.7
       http-errors: 2.0.0
       js-yaml: /@zkochan/js-yaml/0.0.6
+      JSONStream: 1.3.5
       jsonwebtoken: 8.5.1
       kleur: 4.1.5
       lodash: 4.17.21