From 01fdfbad2eef78e1a4c09f282473a9750147f277 Mon Sep 17 00:00:00 2001 From: zkochan Date: Tue, 17 Jan 2017 23:39:25 +0200 Subject: [PATCH] fix(git): don't use GitHub API to avoid rate limit errors Additionally, use ls-remote to resolve refs to commits, before clonings --- package.json | 2 ++ src/install/fetchResolution.ts | 30 +++++++------------- src/resolve/git.ts | 31 +++++++++++++++++---- src/resolve/github.ts | 51 ---------------------------------- src/resolve/index.ts | 8 +----- tsconfig.json | 1 - typings/local.d.ts | 10 +++++++ 7 files changed, 48 insertions(+), 85 deletions(-) delete mode 100644 src/resolve/github.ts diff --git a/package.json b/package.json index 389ec0531d..89106c8ac6 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "@types/semver": "5.3.30", "@types/update-notifier": "0.0.28", "@zkochan/cmd-shim": "2.2.1", + "@zkochan/hosted-git-info": "2.1.5", "arr-flatten": "1.0.1", "bole": "3.0.2", "byline": "5.0.0", @@ -38,6 +39,7 @@ "commondir": "1.0.1", "cross-spawn": "^5.0.0", "delocalize-dependencies": "0.1.0", + "execa": "0.6.0", "exists-file": "3.0.0", "find-up": "2.1.0", "is-ci": "1.0.10", diff --git a/src/install/fetchResolution.ts b/src/install/fetchResolution.ts index a39c34b68f..84c8a9ff59 100644 --- a/src/install/fetchResolution.ts +++ b/src/install/fetchResolution.ts @@ -1,7 +1,7 @@ import {fetchFromRemoteTarball, FetchOptions} from '../resolve/fetch' import {ResolveResult} from '../resolve' import logger from 'pnpm-logger' -import spawn = require('cross-spawn') +import execa = require('execa') const gitLogger = logger('git') @@ -12,8 +12,8 @@ export default async function fetchRes (res: ResolveResult, target: string, opts tarball: res.tarball }, opts) } - if (res.repo && res.ref) { - return clone(res.repo, res.ref, target) + if (res.repo && res.commitId) { + return clone(res.repo, res.commitId, target) } if (res.fetch) { return res.fetch(target) @@ -23,29 +23,19 @@ export default async function fetchRes (res: ResolveResult, target: string, opts /** * clone a git repository. */ -export async function clone (repo: string, ref: string, dest: string) { - await new Promise((resolve, reject) => { - const args = ['clone', '-b', ref, repo, dest, '--single-branch'] - gitLogger.debug(`cloning git repository from ${repo}`) - const git = spawnGit(args) - let errMsg = '' - git.stderr.on('data', (data: string) => errMsg += data) - git.on('close', (code: number) => (code ? errorHandler() : resolve())) - - function errorHandler () { - gitLogger.debug(`failed to clone repository from ${repo}`) - reject(new Error(`failed to clone repository from ${repo} - ${errMsg}`)) - } - }) +async function clone (repo: string, commitId: string, dest: string) { + const args = ['clone', '-b', commitId, repo, dest, '--single-branch'] + gitLogger.debug(`cloning git repository from ${repo}`) + await execGit(['clone', repo, dest]) + await execGit(['checkout', commitId], {cwd: dest}) } function prefixGitArgs (): string[] { return process.platform === 'win32' ? ['-c', 'core.longpaths=true'] : [] } -function spawnGit (args: string[]) { +function execGit (args: string[], opts?: Object) { gitLogger.debug(`executing git with args ${args}`) const fullArgs = prefixGitArgs().concat(args || []) - return spawn('git', fullArgs) + return execa('git', fullArgs, opts) } diff --git a/src/resolve/git.ts b/src/resolve/git.ts index 1858ea5e9b..990b3bb4cd 100644 --- a/src/resolve/git.ts +++ b/src/resolve/git.ts @@ -1,18 +1,37 @@ -import spawn = require('cross-spawn') +import execa = require('execa') import {PackageSpec, ResolveOptions, ResolveResult} from '.' import {delimiter} from './createPkgId' +import hostedGitInfo = require('@zkochan/hosted-git-info') // TODO: resolve ref to commit -export default async function resolveGithub (parsedSpec: PackageSpec, opts: ResolveOptions): Promise { - const parts = parsedSpec.spec.split('#') +export default async function resolveGit (parsedSpec: PackageSpec, opts: ResolveOptions): Promise { + const parts = normalizeRepoUrl(parsedSpec.spec).split('#') const repo = parts[0] const ref = parts[1] || 'master' + const commitId = await resolveRef(repo, ref) return { - id: parsedSpec.spec + id: repo .replace(/^.*:\/\/(git@)?/, '') .replace(/:/g, delimiter) - .replace(/\.git$/, ''), + .replace(/\.git$/, '') + commitId, repo, - ref, + commitId, } } + +async function resolveRef(repo: string, ref: string) { + const result = await execa('git', ['ls-remote', '--refs', repo, ref]) + // should output something like: + // 572bc3d4e16220c2e986091249e62a5913294b25 refs/heads/master + + // if no ref was found, assume that ref is the commit ID + if (!result.stdout) return ref + + return result.stdout.match(/^[a-z0-9]+/) +} + +function normalizeRepoUrl (repoUrl: string) { + const hosted = hostedGitInfo.fromUrl(repoUrl) + if (!hosted) return repoUrl + return hosted.getDefaultRepresentation() == 'shortcut' ? hosted.git() : hosted.toString() +} diff --git a/src/resolve/github.ts b/src/resolve/github.ts deleted file mode 100644 index 9136d72f1f..0000000000 --- a/src/resolve/github.ts +++ /dev/null @@ -1,51 +0,0 @@ -import path = require('path') -import {HostedPackageSpec, ResolveOptions, ResolveResult} from '.' - -/** - * Resolves a 'hosted' package hosted on 'github'. - */ -export default async function resolveGithub (spec: HostedPackageSpec, opts: ResolveOptions): Promise { - const ghSpec = parseGithubSpec(spec) - // the ref should be a commit sha. Otherwise it would't be unique - // and couldn't be saved in a machine store - ghSpec.ref = await resolveRef(ghSpec) - return { - id: path.join('github.com', ghSpec.owner, ghSpec.repo, ghSpec.ref), - tarball: `https://codeload.github.com/${ghSpec.owner}/${ghSpec.repo}/tar.gz/${ghSpec.ref}` - } - - async function resolveRef (spec: GitHubSpec) { - const url = [ - 'https://api.github.com/repos', - spec.owner, - spec.repo, - 'commits', - spec.ref - ].join('/') - const body = await opts.got.getJSON(url) - return body.sha - } -} - -const PARSE_GITHUB_RE = /^github:([^\/]+)\/([^#]+)(#(.+))?$/ - -function parseGithubSpec (spec: HostedPackageSpec): GitHubSpec { - const m = PARSE_GITHUB_RE.exec(spec.hosted.shortcut) - if (!m) { - throw new Error('cannot parse: ' + spec.hosted.shortcut) - } - const owner = m[1] - const repo = m[2] - const ref = m[4] || 'HEAD' - return {owner, repo, ref} -} - -type GitHubSpec = { - owner: string, - repo: string, - ref: string -} - -type GitHubRepoResponse = { - sha: string -} diff --git a/src/resolve/index.ts b/src/resolve/index.ts index e3f988e79a..3858cf5322 100644 --- a/src/resolve/index.ts +++ b/src/resolve/index.ts @@ -1,6 +1,5 @@ import resolveFromNpm from './npm' import resolveFromTarball from './tarball' -import resolveFromGithub from './github' import resolveFromLocal from './local' import resolveFromGit from './git' import {Got} from '../network/got' @@ -14,7 +13,7 @@ export type ResolveResult = { shasum?: string root?: string, repo?: string, - ref?: string, + commitId?: string, fetch?: (target: string) => Promise, } @@ -68,11 +67,6 @@ export default function (spec: PackageSpec, opts: ResolveOptions): Promisespec - if (hspec.hosted.type === 'github' && !isSsh(hspec.spec)) { - return resolveFromGithub(hspec, opts) - } - return resolveFromGit(spec, opts) case 'git': return resolveFromGit(spec, opts) default: diff --git a/tsconfig.json b/tsconfig.json index 2d6b3d01af..1f7060dc66 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -80,7 +80,6 @@ "src/resolve/fetch.ts", "src/resolve/getTarballName.ts", "src/resolve/git.ts", - "src/resolve/github.ts", "src/resolve/index.ts", "src/resolve/local.ts", "src/resolve/npm.ts", diff --git a/typings/local.d.ts b/typings/local.d.ts index 103b6eba7c..ba47db870a 100644 --- a/typings/local.d.ts +++ b/typings/local.d.ts @@ -212,3 +212,13 @@ declare module 'ndjson' { const anything: any; export = anything; } + +declare module 'execa' { + const anything: any; + export = anything; +} + +declare module '@zkochan/hosted-git-info' { + const anything: any; + export = anything; +}