feat: init

This commit is contained in:
Zoltan Kochan
2017-12-04 23:20:38 +02:00
commit 307bd951f0
21 changed files with 4772 additions and 0 deletions

11
.editorconfig Normal file
View File

@@ -0,0 +1,11 @@
root = true
[*]
insert_final_newline = true
charset = utf-8
trim_trailing_whitespace = true
end_of_line = lf
[*.{ts,js,json}]
indent_style = space
indent_size = 2

3
.gitattributes vendored Normal file
View File

@@ -0,0 +1,3 @@
* text eol=lf
*.tgz binary

21
.gitignore vendored Normal file
View File

@@ -0,0 +1,21 @@
# Logs
logs
*.log
npm-debug.log*
# Dependency directory
node_modules
# Coverage directory used by tools like istanbul
coverage
fixtures
.tmp
_docpress
lib
# Visual Studio Code configs
.vscode/
.store

2
.npmrc Normal file
View File

@@ -0,0 +1,2 @@
tag-version-prefix =
message = chore(release): %s

15
.travis.yml Normal file
View File

@@ -0,0 +1,15 @@
language: node_js
node_js:
- 4
- 6
- 8
- 9
sudo: false
before_install:
- curl -L https://unpkg.com/@pnpm/self-installer | node
install:
- pnpm install
script:
- npm test
notifications:
email: false

21
LICENSE Normal file
View File

@@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) 2017 Zoltan Kochan <z@kochan.io>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

17
README.md Normal file
View File

@@ -0,0 +1,17 @@
# @pnpm/package-requester
> Concurrent downloader of npm-compatible packages
<!--@shields('npm', 'travis')-->
[![npm version](https://img.shields.io/npm/v/@pnpm/package-requester.svg)](https://www.npmjs.com/package/@pnpm/package-requester) [![Build Status](https://img.shields.io/travis/pnpm/package-requester/master.svg)](https://travis-ci.org/pnpm/package-requester)
<!--/@-->
## Installation
```sh
npm i -S @pnpm/logger @pnpm/package-requester
```
## License
[MIT](./LICENSE) © [Zoltan Kochan](https://www.kochan.io/)

81
package.json Normal file
View File

@@ -0,0 +1,81 @@
{
"name": "@pnpm/package-requester",
"version": "0.0.0",
"description": "Concurrent downloader of npm-compatible packages",
"main": "lib/index.js",
"typings": "lib/index.d.ts",
"files": [
"lib"
],
"engines": {
"node": ">=4"
},
"scripts": {
"lint": "tslint -c tslint.json --project .",
"tsc": "rimraf lib && tsc",
"test": "npm run lint && preview && ts-node test && mos t",
"md": "mos",
"prepublishOnly": "npm run tsc"
},
"repository": {
"type": "git",
"url": "git+https://github.com/pnpm/package-requester.git"
},
"keywords": [
"pnpm",
"resolver",
"npm"
],
"author": "Zoltan Kochan <z@kochan.io> (https://www.kochan.io/)",
"license": "MIT",
"bugs": {
"url": "https://github.com/pnpm/package-requester/issues"
},
"homepage": "https://github.com/pnpm/package-requester#readme",
"peerDependencies": {
"@pnpm/logger": "^1.0.0"
},
"dependencies": {
"@pnpm/types": "^1.2.1",
"@types/load-json-file": "^2.0.7",
"@types/mz": "^0.0.32",
"@types/p-queue": "^1.1.0",
"@types/write-json-file": "^2.2.1",
"load-json-file": "^4.0.0",
"mkdirp-promise": "^5.0.1",
"mz": "^2.7.0",
"p-limit": "^1.1.0",
"p-queue": "^2.3.0",
"package-store": "^0.9.0",
"path-exists": "^3.0.0",
"read-package-json": "^2.0.12",
"rename-overwrite": "^1.0.0",
"rimraf-then": "^1.0.1",
"symlink-dir": "^1.1.0",
"unpack-stream": "^2.2.0",
"util.promisify": "^1.0.0",
"write-json-file": "^2.3.0"
},
"devDependencies": {
"@pnpm/logger": "^1.0.0",
"@pnpm/npm-resolver": "^0.3.0",
"@pnpm/tarball-fetcher": "^0.2.0",
"@types/tape": "^4.2.31",
"mos": "^2.0.0-alpha.3",
"mos-plugin-readme": "^1.0.4",
"package-preview": "^1.0.1",
"rimraf": "^2.6.2",
"tape": "^4.8.0",
"ts-node": "^3.3.0",
"tslint": "^5.8.0",
"typescript": "^2.6.1"
},
"mos": {
"plugins": [
"readme"
],
"installation": {
"useShortAlias": true
}
}
}

3944
shrinkwrap.yaml Normal file
View File

File diff suppressed because it is too large Load Diff

16
src/fetchTypes.ts Normal file
View File

@@ -0,0 +1,16 @@
import * as unpackStream from 'unpack-stream'
import {Resolution} from './resolveTypes'
export interface FetchOptions {
cachedTarballLocation: string,
pkgId: string,
prefix: string,
onStart?: (totalSize: number | null, attempt: number) => void,
onProgress?: (downloaded: number) => void,
}
export type FetchFunction = (
resolution: Resolution,
target: string,
opts: FetchOptions,
) => Promise<unpackStream.Index>

14
src/fs/readPkg.ts Normal file
View File

@@ -0,0 +1,14 @@
import {PackageJson} from '@pnpm/types'
import path = require('path')
import readPackageJsonCB = require('read-package-json')
import promisify = require('util.promisify')
const readPackageJson = promisify(readPackageJsonCB)
export default function readPkg (pkgPath: string): Promise<PackageJson> {
return readPackageJson(pkgPath)
}
export function fromDir (pkgPath: string): Promise<PackageJson> {
return readPkg(path.join(pkgPath, 'package.json'))
}

16
src/fs/safeReadPkg.ts Normal file
View File

@@ -0,0 +1,16 @@
import {PackageJson} from '@pnpm/types'
import path = require('path')
import readPkg from './readPkg'
export default async function safeReadPkg (pkgPath: string): Promise<PackageJson | null> {
try {
return await readPkg(pkgPath)
} catch (err) {
if ((err as NodeJS.ErrnoException).code !== 'ENOENT') throw err
return null
}
}
export function fromDir (pkgPath: string): Promise<PackageJson | null> {
return safeReadPkg(path.join(pkgPath, 'package.json'))
}

8
src/index.ts Normal file
View File

@@ -0,0 +1,8 @@
import packageRequester from './packageRequester'
export default packageRequester
export {
ProgressLog,
Log,
} from './loggers'

43
src/loggers.ts Normal file
View File

@@ -0,0 +1,43 @@
import baseLogger, {
LogBase,
Logger,
} from '@pnpm/logger'
export const progressLogger = baseLogger('progress') as Logger<ProgressMessage>
export interface LoggedPkg {
rawSpec: string,
name?: string,
dependentId?: string,
}
// Not all of this message types are used in this project
// some of them can be removed
export type ProgressMessage = {
pkgId: string,
status: 'fetched' | 'installed' | 'dependencies_installed' | 'found_in_store' | 'resolving_content',
} | {
pkgId: string,
pkg: LoggedPkg,
status: 'resolved',
} | {
pkg: LoggedPkg,
status: 'resolving' | 'error' | 'installing',
} | {
pkgId: string,
status: 'fetching_started',
size: number | null,
attempt: number,
} | {
pkgId: string,
status: 'fetching_progress',
downloaded: number,
} | {
status: 'downloaded_manifest',
pkgId: string,
pkgVersion: string,
}
export type ProgressLog = {name: 'pnpm:progress'} & LogBase & ProgressMessage
export type Log = ProgressLog

21
src/memoize.ts Normal file
View File

@@ -0,0 +1,21 @@
import pLimit = require('p-limit')
interface CachedPromises<T> {
[name: string]: Promise<T>
}
export type MemoizedFunc<T> = (key: string, fn: () => Promise<T>) => Promise<T>
/**
* Save promises for later
*/
export default function memoize <T> (concurrency?: number): MemoizedFunc<T> {
const locks: CachedPromises<T> = {}
const limit = concurrency && pLimit(concurrency)
return (key: string, fn: () => Promise<T>): Promise<T> => {
if (locks[key]) return locks[key]
locks[key] = limit && limit(fn) || fn()
return locks[key]
}
}

372
src/packageRequester.ts Normal file
View File

@@ -0,0 +1,372 @@
import logger from '@pnpm/logger'
import {PackageJson} from '@pnpm/types'
import {Stats} from 'fs'
import loadJsonFile = require('load-json-file')
import mkdirp = require('mkdirp-promise')
import fs = require('mz/fs')
import PQueue = require('p-queue')
import {
pkgIdToFilename,
pkgIsUntouched,
Store,
} from 'package-store'
import path = require('path')
import exists = require('path-exists')
import renameOverwrite = require('rename-overwrite')
import rimraf = require('rimraf-then')
import symlinkDir = require('symlink-dir')
import * as unpackStream from 'unpack-stream'
import writeJsonFile = require('write-json-file')
import {
FetchFunction,
FetchOptions,
} from './fetchTypes'
import {fromDir as readPkgFromDir} from './fs/readPkg'
import {fromDir as safeReadPkgFromDir} from './fs/safeReadPkg'
import {LoggedPkg, progressLogger} from './loggers'
import memoize, {MemoizedFunc} from './memoize'
import {
DirectoryResolution,
Resolution,
ResolveFunction,
ResolveOptions,
ResolveResult,
WantedDependency,
} from './resolveTypes'
export interface PackageContentInfo {
isNew: boolean,
index: {},
}
export type FetchedPackage = {
isLocal: true,
resolution: DirectoryResolution,
pkg: PackageJson,
id: string,
normalizedPref?: string,
} | {
isLocal: false,
fetchingPkg: Promise<PackageJson>,
fetchingFiles: Promise<PackageContentInfo>,
calculatingIntegrity: Promise<void>,
path: string,
id: string,
resolution: Resolution,
// This is useful for recommending updates.
// If latest does not equal the version of the
// resolved package, it is out-of-date.
latest?: string,
normalizedPref?: string,
}
export default function (
resolve: ResolveFunction,
fetchers: {[type: string]: FetchFunction},
opts: {
networkConcurrency: number,
},
) {
opts = opts || {}
const networkConcurrency = opts.networkConcurrency || 16
const requestsQueue = new PQueue({
concurrency: networkConcurrency,
})
requestsQueue['counter'] = 0 // tslint:disable-line
requestsQueue['concurrency'] = networkConcurrency // tslint:disable-line
const fetch = fetcher.bind(null, fetchers)
return resolveAndFetch.bind(null,
requestsQueue,
resolve,
fetch,
)
}
async function resolveAndFetch (
requestsQueue: {add: <T>(fn: () => Promise<T>, opts: {priority: number}) => Promise<T>},
resolve: ResolveFunction,
fetch: FetchFunction,
wantedDependency: {
alias?: string,
pref: string,
},
options: {
downloadPriority: number,
fetchingLocker: {
[pkgId: string]: {
calculatingIntegrity: Promise<void>,
fetchingFiles: Promise<PackageContentInfo>,
fetchingPkg: Promise<PackageJson>,
},
},
loggedPkg: LoggedPkg,
offline: boolean,
pkgId?: string,
prefix: string,
registry: string,
shrinkwrapResolution?: Resolution,
storeIndex: Store,
storePath: string,
update?: boolean,
verifyStoreIntegrity: boolean,
},
): Promise<FetchedPackage> {
try {
let latest: string | undefined
let pkg: PackageJson | undefined
let normalizedPref: string | undefined
let resolution = options.shrinkwrapResolution
let pkgId = options.pkgId
if (!resolution || options.update) {
const resolveResult = await requestsQueue.add<ResolveResult>(() => resolve(wantedDependency, {
prefix: options.prefix,
registry: options.registry,
}), {priority: options.downloadPriority})
// keep the shrinkwrap resolution when possible
// to keep the original shasum
if (pkgId !== resolveResult.id || !resolution) {
resolution = resolveResult.resolution
}
pkgId = resolveResult.id
pkg = resolveResult.package
latest = resolveResult.latest
normalizedPref = resolveResult.normalizedPref
}
const id = pkgId as string
progressLogger.debug({status: 'resolved', pkgId: id, pkg: options.loggedPkg})
if (resolution.type === 'directory') {
if (!pkg) {
throw new Error(`Couldn't read package.json of local dependency ${wantedDependency.alias ? wantedDependency.alias + '@' : ''}${wantedDependency.pref}`)
}
return {
id,
isLocal: true,
normalizedPref,
pkg,
resolution: resolution as DirectoryResolution,
}
}
const targetRelative = pkgIdToFilename(id)
const target = path.join(options.storePath, targetRelative)
if (!options.fetchingLocker[id]) {
options.fetchingLocker[id] = fetchToStore({
fetch,
pkg,
pkgId: id,
prefix: options.prefix,
requestsQueue,
resolution: resolution as Resolution,
storeIndex: options.storeIndex,
storePath: options.storePath,
target,
targetRelative,
verifyStoreIntegrity: options.verifyStoreIntegrity,
})
}
return {
calculatingIntegrity: options.fetchingLocker[id].calculatingIntegrity,
fetchingFiles: options.fetchingLocker[id].fetchingFiles,
fetchingPkg: options.fetchingLocker[id].fetchingPkg,
id,
isLocal: false,
latest,
normalizedPref,
path: target,
resolution,
}
} catch (err) {
progressLogger.debug({status: 'error', pkg: options.loggedPkg})
throw err
}
}
function fetchToStore (opts: {
fetch: FetchFunction,
requestsQueue: {add: <T>(fn: () => Promise<T>, opts: {priority: number}) => Promise<T>},
pkg?: PackageJson,
pkgId: string,
prefix: string,
resolution: Resolution,
target: string,
targetRelative: string,
storePath: string,
storeIndex: Store,
verifyStoreIntegrity: boolean,
}): {
fetchingFiles: Promise<PackageContentInfo>,
fetchingPkg: Promise<PackageJson>,
calculatingIntegrity: Promise<void>,
} {
const fetchingPkg = differed<PackageJson>()
const fetchingFiles = differed<PackageContentInfo>()
const calculatingIntegrity = differed<void>()
doFetchToStore()
return {
calculatingIntegrity: calculatingIntegrity.promise,
fetchingFiles: fetchingFiles.promise,
fetchingPkg: opts.pkg && Promise.resolve(opts.pkg) || fetchingPkg.promise,
}
async function doFetchToStore () {
try {
progressLogger.debug({
pkgId: opts.pkgId,
status: 'resolving_content',
})
const target = opts.target
const linkToUnpacked = path.join(target, 'package')
// We can safely assume that if there is no data about the package in `store.json` then
// it is not in the store yet.
// In case there is record about the package in `store.json`, we check it in the file system just in case
const targetExists = opts.storeIndex[opts.targetRelative] && await exists(path.join(linkToUnpacked, 'package.json'))
if (targetExists) {
// if target exists and it wasn't modified, then no need to refetch it
const satisfiedIntegrity = opts.verifyStoreIntegrity
? await pkgIsUntouched(linkToUnpacked)
: await loadJsonFile(path.join(path.dirname(linkToUnpacked), 'integrity.json'))
if (satisfiedIntegrity) {
progressLogger.debug({
pkgId: opts.pkgId,
status: 'found_in_store',
})
fetchingFiles.resolve({
index: satisfiedIntegrity,
isNew: false,
})
if (!opts.pkg) {
readPkgFromDir(linkToUnpacked)
.then(fetchingPkg.resolve)
.catch(fetchingPkg.reject)
}
calculatingIntegrity.resolve(undefined)
return
}
logger.warn(`Refetching ${target} to store, as it was modified`)
}
// We fetch into targetStage directory first and then fs.rename() it to the
// target directory.
const targetStage = `${target}_stage`
await rimraf(targetStage)
let packageIndex: {} = {}
await Promise.all([
(async () => {
// Tarballs are requested first because they are bigger than metadata files.
// However, when one line is left available, allow it to be picked up by a metadata request.
// This is done in order to avoid situations when tarballs are downloaded in chunks
// As much tarballs should be downloaded simultaneously as possible.
const priority = (++opts.requestsQueue['counter'] % opts.requestsQueue['concurrency'] === 0 ? -1 : 1) * 1000 // tslint:disable-line
packageIndex = await opts.requestsQueue.add(() => opts.fetch(opts.resolution, targetStage, {
cachedTarballLocation: path.join(opts.storePath, opts.pkgId, 'packed.tgz'),
onProgress: (downloaded) => {
progressLogger.debug({status: 'fetching_progress', pkgId: opts.pkgId, downloaded})
},
onStart: (size, attempt) => {
progressLogger.debug({status: 'fetching_started', pkgId: opts.pkgId, size, attempt})
},
pkgId: opts.pkgId,
prefix: opts.prefix,
}), {priority})
})(),
// removing only the folder with the unpacked files
// not touching tarball and integrity.json
targetExists && await rimraf(path.join(target, 'node_modules')),
])
progressLogger.debug({
pkgId: opts.pkgId,
status: 'fetched',
})
// fetchingFilse shouldn't care about when this is saved at all
if (!targetExists) {
(async () => {
const integrity = opts.verifyStoreIntegrity
? await (packageIndex as unpackStream.Index).integrityPromise
: await (packageIndex as unpackStream.Index).headers
writeJsonFile(path.join(target, 'integrity.json'), integrity, {indent: null})
calculatingIntegrity.resolve(undefined)
})()
} else {
calculatingIntegrity.resolve(undefined)
}
let pkg: PackageJson
if (opts.pkg) {
pkg = opts.pkg
} else {
pkg = await readPkgFromDir(targetStage)
fetchingPkg.resolve(pkg)
}
const unpacked = path.join(target, 'node_modules', pkg.name)
await mkdirp(path.dirname(unpacked))
// rename(oldPath, newPath) is an atomic operation, so we do it at the
// end
await renameOverwrite(targetStage, unpacked)
await symlinkDir(unpacked, linkToUnpacked)
fetchingFiles.resolve({
index: (packageIndex as unpackStream.Index).headers,
isNew: true,
})
} catch (err) {
fetchingFiles.reject(err)
if (!opts.pkg) {
fetchingPkg.reject(err)
}
}
}
}
// tslint:disable-next-line
function noop () {}
function differed<T> (): {
promise: Promise<T>,
resolve: (v: T) => void,
reject: (err: Error) => void,
} {
let pResolve: (v: T) => void = noop
let pReject: (err: Error) => void = noop
const promise = new Promise<T>((resolve, reject) => {
pResolve = resolve
pReject = reject
})
return {
promise,
reject: pReject,
resolve: pResolve,
}
}
async function fetcher (
fetcherByHostingType: {[hostingType: string]: FetchFunction},
resolution: Resolution,
target: string,
opts: FetchOptions,
): Promise<unpackStream.Index> {
const fetch = fetcherByHostingType[resolution.type || 'tarball']
if (!fetch) {
throw new Error(`Fetching for dependency type "${resolution.type}" is not supported`)
}
return await fetch(resolution, target, opts)
}

47
src/resolveTypes.ts Normal file
View File

@@ -0,0 +1,47 @@
import {PackageJson} from '@pnpm/types'
/**
* tarball hosted remotely
*/
export interface TarballResolution {
type?: undefined,
tarball: string,
integrity?: string,
// needed in some cases to get the auth token
// sometimes the tarball URL is under a different path
// and the auth token is specified for the registry only
registry?: string,
}
/**
* directory on a file system
*/
export interface DirectoryResolution {
type: 'directory',
directory: string,
}
export type Resolution =
TarballResolution |
DirectoryResolution |
({ type: string } & object)
export interface ResolveResult {
id: string,
resolution: Resolution,
package?: PackageJson,
latest?: string,
normalizedPref?: string, // is null for npm-hosted dependencies
}
export interface ResolveOptions {
registry: string,
prefix: string,
}
export interface WantedDependency {
alias?: string,
pref: string,
}
export type ResolveFunction = (wantedDependency: WantedDependency, opts: ResolveOptions) => Promise<ResolveResult>

18
test/index.ts Normal file
View File

@@ -0,0 +1,18 @@
import test = require('tape')
import createPackageRequester from '@pnpm/package-requester'
import createResolver from '@pnpm/npm-resolver'
import createFetcher from '@pnpm/tarball-fetcher'
const resolve = createResolver({rawNpmConfig: {}})
const fetch = createFetcher({
alwaysAuth: false,
registry: 'https://registry.npmjs.org/',
strictSsl: false,
rawNpmConfig: {},
})
test('createPackageRequester', t => {
const requestPackage = createPackageRequester(resolve, fetch, {networkConcurrency: 1})
t.equal(typeof requestPackage, 'function')
t.end()
})

24
tsconfig.json Normal file
View File

@@ -0,0 +1,24 @@
{
"compilerOptions": {
"removeComments": false,
"preserveConstEnums": true,
"sourceMap": true,
"declaration": true,
"noImplicitAny": true,
"noImplicitReturns": true,
"suppressImplicitAnyIndexErrors": true,
"allowSyntheticDefaultImports": true,
"strictNullChecks": true,
"target": "es6",
"outDir": "lib",
"module": "commonjs",
"moduleResolution": "node"
},
"include": [
"src/**/*.ts",
"typings/**/*.d.ts"
],
"atom": {
"rewriteTsconfig": true
}
}

44
tslint.json Normal file
View File

@@ -0,0 +1,44 @@
{
"extends": "tslint:recommended",
"rules": {
"curly": false,
"eofline": false,
"align": [true, "parameters"],
"class-name": true,
"indent": [true, "spaces"],
"max-line-length": false,
"no-any": true,
"no-consecutive-blank-lines": true,
"no-trailing-whitespace": true,
"no-duplicate-variable": true,
"no-var-keyword": true,
"no-unused-expression": true,
"no-use-before-declare": true,
"no-var-requires": true,
"no-require-imports": false,
"space-before-function-paren": [true, "always"],
"interface-name": [true, "never-prefix"],
"no-console": false,
"one-line": [true,
"check-else",
"check-whitespace",
"check-open-brace"],
"quotemark": [true,
"single",
"avoid-escape"],
"semicolon": false,
"typedef-whitespace": [true, {
"call-signature": "nospace",
"index-signature": "nospace",
"parameter": "nospace",
"property-declaration": "nospace",
"variable-declaration": "nospace"
}],
"whitespace": [true,
"check-branch",
"check-decl",
"check-operator",
"check-separator",
"check-type"]
}
}

34
typings/index.d.ts vendored Normal file
View File

@@ -0,0 +1,34 @@
declare module 'p-limit' {
const anything: any;
export = anything;
}
declare module 'util.promisify' {
const anything: any;
export = anything;
}
declare module 'read-package-json' {
const anything: any;
export = anything;
}
declare module 'mkdirp-promise' {
const anything: any;
export = anything;
}
declare module 'rimraf-then' {
const anything: any;
export = anything;
}
declare module 'path-exists' {
const anything: any;
export = anything;
}
declare module 'rename-overwrite' {
const anything: any;
export = anything;
}