feat(git-fetcher): shallow clone when fetching git resource (#4548)

Co-authored-by: Zoltan Kochan <z@kochan.io>

.changeset/curly-spiders-search.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/git-fetcher": minor
+---
+
+feat(git-fetcher): shallow clone when fetching git resource

.changeset/stale-apples-shop.md

@@ -0,0 +1,8 @@
+---
+"@pnpm/config": minor
+"@pnpm/client": minor
+"@pnpm/store-connection-manager": minor
+"pnpm": minor
+---
+
+New setting added: `git-shallow-hosts`. When cloning repositories from "shallow-hosts", pnpm will use shallow cloning to fetch only the needed commit, not all the history [#4548](https://github.com/pnpm/pnpm/pull/4548).

packages/client/src/index.ts

@@ -18,6 +18,7 @@ export type ClientOptions = {
   timeout?: number
   userAgent?: string
   userConfig?: Record<string, string>
+  gitShallowHosts?: string[]
 } & ResolverFactoryOptions & AgentOptions
 
 export default function (opts: ClientOptions) {
@@ -38,13 +39,11 @@ export function createResolver (opts: ClientOptions) {
 function createFetchers (
   fetchFromRegistry: FetchFromRegistry,
   getCredentials: GetCredentials,
-  opts: {
-    retry?: RetryTimeoutOptions
-  }
+  opts: Pick<ClientOptions, 'retry' | 'gitShallowHosts'>
 ) {
   return {
     ...createTarballFetcher(fetchFromRegistry, getCredentials, opts),
-    ...fetchFromGit(),
+    ...fetchFromGit(opts),
     ...createDirectoryFetcher(),
   }
 }

packages/config/src/Config.ts

@@ -140,6 +140,7 @@ export interface Config {
   enableModulesDir: boolean
   modulesCacheMaxAge: number
   embedReadme?: boolean
+  gitShallowHosts?: string[]
 
   registries: Registries
   ignoreWorkspaceRootCheck: boolean

packages/config/src/index.ts

@@ -48,6 +48,7 @@ export const types = Object.assign({
   'filter-prod': [String, Array],
   'frozen-lockfile': Boolean,
   'git-checks': Boolean,
+  'git-shallow-hosts': Array,
   'global-bin-dir': String,
   'global-dir': String,
   'global-path': String,
@@ -174,6 +175,14 @@ export default async (
     'fetch-retry-maxtimeout': 60000,
     'fetch-retry-mintimeout': 10000,
     'fetch-timeout': 60000,
+    'git-shallow-hosts': [
+      // Follow https://github.com/npm/git/blob/1e1dbd26bd5b87ca055defecc3679777cb480e2a/lib/clone.js#L13-L19
+      'github.com',
+      'gist.github.com',
+      'gitlab.com',
+      'bitbucket.com',
+      'bitbucket.org',
+    ],
     globalconfig: npmDefaults.globalconfig,
     hoist: true,
     'hoist-pattern': ['*'],

packages/git-fetcher/src/index.ts

@@ -3,8 +3,10 @@ import { Cafs, DeferredManifestPromise } from '@pnpm/fetcher-base'
 import preparePackage from '@pnpm/prepare-package'
 import rimraf from '@zkochan/rimraf'
 import execa from 'execa'
+import { URL } from 'url'
 
-export default () => {
+export default (createOpts?: { gitShallowHosts?: string[] }) => {
+  const allowedHosts = new Set(createOpts?.gitShallowHosts ?? [])
   return {
     git: async function fetchFromGit (
       cafs: Cafs,
@@ -18,7 +20,13 @@ export default () => {
       }
     ) {
       const tempLocation = await cafs.tempDir()
-      await execGit(['clone', resolution.repo, tempLocation])
+      if (allowedHosts.size > 0 && shouldUseShallow(resolution.repo, allowedHosts)) {
+        await execGit(['init'], { cwd: tempLocation })
+        await execGit(['remote', 'add', 'origin', resolution.repo], { cwd: tempLocation })
+        await execGit(['fetch', '--depth', '1', 'origin', resolution.commit], { cwd: tempLocation })
+      } else {
+        await execGit(['clone', resolution.repo, tempLocation])
+      }
       await execGit(['checkout', resolution.commit], { cwd: tempLocation })
       await preparePackage(tempLocation)
       // removing /.git to make directory integrity calculation faster
@@ -32,6 +40,18 @@ export default () => {
   }
 }
 
+function shouldUseShallow (repoUrl: string, allowedHosts: Set<string>): boolean {
+  try {
+    const { host } = new URL(repoUrl)
+    if (allowedHosts.has(host)) {
+      return true
+    }
+  } catch (e) {
+    // URL might be malformed
+  }
+  return false
+}
+
 function prefixGitArgs (): string[] {
   return process.platform === 'win32' ? ['-c', 'core.longpaths=true'] : []
 }

packages/git-fetcher/test/index.ts

@@ -5,6 +5,20 @@ import createFetcher from '@pnpm/git-fetcher'
 import { DependencyManifest } from '@pnpm/types'
 import pDefer from 'p-defer'
 import tempy from 'tempy'
+import execa from 'execa'
+
+jest.mock('execa', () => {
+  const originalModule = jest.requireActual('execa')
+  return {
+    __esModule: true,
+    ...originalModule,
+    default: jest.fn(originalModule.default),
+  }
+})
+
+beforeEach(() => {
+  (execa as jest.Mock).mockClear()
+})
 
 test('fetch', async () => {
   const cafsDir = tempy.directory()
@@ -78,3 +92,38 @@ test('fetch a big repository', async () => {
     }, { manifest })
   await Promise.all(Object.values(filesIndex).map(({ writeResult }) => writeResult))
 })
+
+test('still able to shallow fetch for allowed hosts', async () => {
+  const cafsDir = tempy.directory()
+  const fetch = createFetcher({ gitShallowHosts: ['github.com'] }).git
+  const manifest = pDefer<DependencyManifest>()
+  const resolution = {
+    commit: 'c9b30e71d704cd30fa71f2edd1ecc7dcc4985493',
+    repo: 'https://github.com/kevva/is-positive.git',
+    type: 'git' as const,
+  }
+  const { filesIndex } = await fetch(createCafsStore(cafsDir), resolution, {
+    manifest,
+  })
+  const calls = (execa as jest.Mock).mock.calls
+  const expectedCalls = [
+    ['git', [...prefixGitArgs(), 'init']],
+    ['git', [...prefixGitArgs(), 'remote', 'add', 'origin', resolution.repo]],
+    [
+      'git',
+      [...prefixGitArgs(), 'fetch', '--depth', '1', 'origin', resolution.commit],
+    ],
+  ]
+  for (let i = 1; i < expectedCalls.length; i++) {
+    // Discard final argument as it passes temporary directory
+    expect(calls[i].slice(0, -1)).toEqual(expectedCalls[i])
+  }
+  expect(filesIndex['package.json']).toBeTruthy()
+  expect(filesIndex['package.json'].writeResult).toBeTruthy()
+  const name = (await manifest.promise).name
+  expect(name).toEqual('is-positive')
+})
+
+function prefixGitArgs (): string[] {
+  return process.platform === 'win32' ? ['-c', 'core.longpaths=true'] : []
+}

packages/store-connection-manager/src/createNewStoreController.ts

@@ -22,6 +22,7 @@ export type CreateNewStoreControllerOptions = CreateResolverOptions & Pick<Confi
 | 'force'
 | 'nodeVersion'
 | 'fetchTimeout'
+| 'gitShallowHosts'
 | 'httpProxy'
 | 'httpsProxy'
 | 'key'
@@ -71,6 +72,7 @@ export default async (
         ? (opts.networkConcurrency * 3)
         : undefined
     ),
+    gitShallowHosts: opts.gitShallowHosts,
   })
   await fs.mkdir(opts.storeDir, { recursive: true })
   return {