fix(git): don't use GitHub API to avoid rate limit errors

Additionally, use ls-remote to resolve refs to commits, before clonings

package.json

@@ -28,6 +28,7 @@
     "@types/semver": "5.3.30",
     "@types/update-notifier": "0.0.28",
     "@zkochan/cmd-shim": "2.2.1",
+    "@zkochan/hosted-git-info": "2.1.5",
     "arr-flatten": "1.0.1",
     "bole": "3.0.2",
     "byline": "5.0.0",
@@ -38,6 +39,7 @@
     "commondir": "1.0.1",
     "cross-spawn": "^5.0.0",
     "delocalize-dependencies": "0.1.0",
+    "execa": "0.6.0",
     "exists-file": "3.0.0",
     "find-up": "2.1.0",
     "is-ci": "1.0.10",

src/install/fetchResolution.ts

@@ -1,7 +1,7 @@
 import {fetchFromRemoteTarball, FetchOptions} from '../resolve/fetch'
 import {ResolveResult} from '../resolve'
 import logger from 'pnpm-logger'
-import spawn = require('cross-spawn')
+import execa = require('execa')
 
 const gitLogger = logger('git')
 
@@ -12,8 +12,8 @@ export default async function fetchRes (res: ResolveResult, target: string, opts
         tarball: res.tarball
       }, opts)
   }
-  if (res.repo && res.ref) {
-      return clone(res.repo, res.ref, target)
+  if (res.repo && res.commitId) {
+      return clone(res.repo, res.commitId, target)
   }
   if (res.fetch) {
       return res.fetch(target)
@@ -23,29 +23,19 @@ export default async function fetchRes (res: ResolveResult, target: string, opts
 /**
  * clone a git repository.
  */
-export async function clone (repo: string, ref: string, dest: string) {
-  await new Promise((resolve, reject) => {
-    const args = ['clone', '-b', ref, repo, dest, '--single-branch']
-    gitLogger.debug(`cloning git repository from ${repo}`)
-    const git = spawnGit(args)
-    let errMsg = ''
-    git.stderr.on('data', (data: string) => errMsg += data)
-    git.on('close', (code: number) => (code ? errorHandler() : resolve()))
-
-    function errorHandler () {
-      gitLogger.debug(`failed to clone repository from ${repo}`)
-      reject(new Error(`failed to clone repository from ${repo}
-        ${errMsg}`))
-    }
-  })
+async function clone (repo: string, commitId: string, dest: string) {
+  const args = ['clone', '-b', commitId, repo, dest, '--single-branch']
+  gitLogger.debug(`cloning git repository from ${repo}`)
+  await execGit(['clone', repo, dest])
+  await execGit(['checkout', commitId], {cwd: dest})
 }
 
 function prefixGitArgs (): string[] {
   return process.platform === 'win32' ? ['-c', 'core.longpaths=true'] : []
 }
 
-function spawnGit (args: string[]) {
+function execGit (args: string[], opts?: Object) {
   gitLogger.debug(`executing git with args ${args}`)
   const fullArgs = prefixGitArgs().concat(args || [])
-  return spawn('git', fullArgs)
+  return execa('git', fullArgs, opts)
 }

src/resolve/git.ts

@@ -1,18 +1,37 @@
-import spawn = require('cross-spawn')
+import execa = require('execa')
 import {PackageSpec, ResolveOptions, ResolveResult} from '.'
 import {delimiter} from './createPkgId'
+import hostedGitInfo = require('@zkochan/hosted-git-info')
 
 // TODO: resolve ref to commit
-export default async function resolveGithub (parsedSpec: PackageSpec, opts: ResolveOptions): Promise<ResolveResult> {
-  const parts = parsedSpec.spec.split('#')
+export default async function resolveGit (parsedSpec: PackageSpec, opts: ResolveOptions): Promise<ResolveResult> {
+  const parts = normalizeRepoUrl(parsedSpec.spec).split('#')
   const repo = parts[0]
   const ref = parts[1] || 'master'
+  const commitId = await resolveRef(repo, ref)
   return {
-    id: parsedSpec.spec
+    id: repo
       .replace(/^.*:\/\/(git@)?/, '')
       .replace(/:/g, delimiter)
-      .replace(/\.git$/, ''),
+      .replace(/\.git$/, '') + commitId,
     repo,
-    ref,
+    commitId,
   }
 }
+
+async function resolveRef(repo: string, ref: string) {
+  const result = await execa('git', ['ls-remote', '--refs', repo, ref])
+  // should output something like:
+  //   572bc3d4e16220c2e986091249e62a5913294b25    	refs/heads/master
+
+  // if no ref was found, assume that ref is the commit ID
+  if (!result.stdout) return ref
+
+  return result.stdout.match(/^[a-z0-9]+/)
+}
+
+function normalizeRepoUrl (repoUrl: string) {
+  const hosted = hostedGitInfo.fromUrl(repoUrl)
+  if (!hosted) return repoUrl
+  return hosted.getDefaultRepresentation() == 'shortcut' ? hosted.git() : hosted.toString()
+}

src/resolve/github.ts

@@ -1,51 +0,0 @@
-import path = require('path')
-import {HostedPackageSpec, ResolveOptions, ResolveResult} from '.'
-
-/**
- * Resolves a 'hosted' package hosted on 'github'.
- */
-export default async function resolveGithub (spec: HostedPackageSpec, opts: ResolveOptions): Promise<ResolveResult> {
-  const ghSpec = parseGithubSpec(spec)
-  // the ref should be a commit sha. Otherwise it would't be unique
-  // and couldn't be saved in a machine store
-  ghSpec.ref = await resolveRef(ghSpec)
-  return {
-    id: path.join('github.com', ghSpec.owner, ghSpec.repo, ghSpec.ref),
-    tarball: `https://codeload.github.com/${ghSpec.owner}/${ghSpec.repo}/tar.gz/${ghSpec.ref}`
-  }
-
-  async function resolveRef (spec: GitHubSpec) {
-    const url = [
-      'https://api.github.com/repos',
-      spec.owner,
-      spec.repo,
-      'commits',
-      spec.ref
-    ].join('/')
-    const body = await opts.got.getJSON<GitHubRepoResponse>(url)
-    return body.sha
-  }
-}
-
-const PARSE_GITHUB_RE = /^github:([^\/]+)\/([^#]+)(#(.+))?$/
-
-function parseGithubSpec (spec: HostedPackageSpec): GitHubSpec {
-  const m = PARSE_GITHUB_RE.exec(spec.hosted.shortcut)
-  if (!m) {
-    throw new Error('cannot parse: ' + spec.hosted.shortcut)
-  }
-  const owner = m[1]
-  const repo = m[2]
-  const ref = m[4] || 'HEAD'
-  return {owner, repo, ref}
-}
-
-type GitHubSpec = {
-  owner: string,
-  repo: string,
-  ref: string
-}
-
-type GitHubRepoResponse = {
-  sha: string
-}

src/resolve/index.ts

@@ -1,6 +1,5 @@
 import resolveFromNpm from './npm'
 import resolveFromTarball from './tarball'
-import resolveFromGithub from './github'
 import resolveFromLocal from './local'
 import resolveFromGit from './git'
 import {Got} from '../network/got'
@@ -14,7 +13,7 @@ export type ResolveResult = {
   shasum?: string
   root?: string,
   repo?: string,
-  ref?: string,
+  commitId?: string,
   fetch?: (target: string) => Promise<void>,
 }
 
@@ -68,11 +67,6 @@ export default function (spec: PackageSpec, opts: ResolveOptions): Promise<Resol
     case 'local':
       return resolveFromLocal(spec, opts)
     case 'hosted':
-      const hspec = <HostedPackageSpec>spec
-      if (hspec.hosted.type === 'github' && !isSsh(hspec.spec)) {
-        return resolveFromGithub(hspec, opts)
-      }
-      return resolveFromGit(spec, opts)
     case 'git':
       return resolveFromGit(spec, opts)
     default:

tsconfig.json

@@ -80,7 +80,6 @@
     "src/resolve/fetch.ts",
     "src/resolve/getTarballName.ts",
     "src/resolve/git.ts",
-    "src/resolve/github.ts",
     "src/resolve/index.ts",
     "src/resolve/local.ts",
     "src/resolve/npm.ts",

typings/local.d.ts

@@ -212,3 +212,13 @@ declare module 'ndjson' {
   const anything: any;
   export = anything;
 }
+
+declare module 'execa' {
+  const anything: any;
+  export = anything;
+}
+
+declare module '@zkochan/hosted-git-info' {
+  const anything: any;
+  export = anything;
+}