diff --git a/.changeset/chatty-penguins-explain.md b/.changeset/chatty-penguins-explain.md new file mode 100644 index 0000000000..3de336f3fe --- /dev/null +++ b/.changeset/chatty-penguins-explain.md @@ -0,0 +1,5 @@ +--- +"@pnpm/config": major +--- + +Remove httpsProxy from the object returned by @pnpm/config. httpsProxy is assigned to proxy. diff --git a/.changeset/cyan-weeks-collect.md b/.changeset/cyan-weeks-collect.md new file mode 100644 index 0000000000..be6f0d52b6 --- /dev/null +++ b/.changeset/cyan-weeks-collect.md @@ -0,0 +1,5 @@ +--- +"@pnpm/fetch": minor +--- + +fetchFromRegistry() added. diff --git a/.changeset/fair-kangaroos-hug.md b/.changeset/fair-kangaroos-hug.md new file mode 100644 index 0000000000..ce63ad8496 --- /dev/null +++ b/.changeset/fair-kangaroos-hug.md @@ -0,0 +1,6 @@ +--- +"@pnpm/client": major +"@pnpm/fetching-types": major +--- + +Initial version. diff --git a/.changeset/perfect-jobs-march.md b/.changeset/perfect-jobs-march.md new file mode 100644 index 0000000000..d37399349b --- /dev/null +++ b/.changeset/perfect-jobs-march.md @@ -0,0 +1,7 @@ +--- +"@pnpm/default-resolver": major +"@pnpm/npm-resolver": major +"@pnpm/tarball-fetcher": major +--- + +Breaking changes to the API. fetchFromRegistry and getCredentials are passed in through arguments. diff --git a/packages/audit/src/index.ts b/packages/audit/src/index.ts index 9bc4a7d6c5..3595a14884 100644 --- a/packages/audit/src/index.ts +++ b/packages/audit/src/index.ts @@ -1,5 +1,5 @@ import PnpmError from '@pnpm/error' -import fetch, { RetryOpts } from '@pnpm/fetch' +import fetch, { RetryTimeoutOptions } from '@pnpm/fetch' import { Lockfile } from '@pnpm/lockfile-types' import { DependenciesField } from '@pnpm/types' import lockfileToAuditTree from './lockfileToAuditTree' @@ -12,7 +12,7 @@ export default async function audit ( opts: { include?: { [dependenciesField in DependenciesField]: boolean }, registry: string, - retry?: RetryOpts, + retry?: RetryTimeoutOptions, } ) { const auditTree = lockfileToAuditTree(lockfile, { include: opts.include }) diff --git a/packages/client/README.md b/packages/client/README.md new file mode 100644 index 0000000000..e7ac50e7dc --- /dev/null +++ b/packages/client/README.md @@ -0,0 +1,17 @@ +# @pnpm/client + +> Creates the package resolve and fetch functions + + +[![npm version](https://img.shields.io/npm/v/@pnpm/client.svg)](https://www.npmjs.com/package/@pnpm/client) + + +## Installation + +```sh + add @pnpm/client +``` + +## License + +MIT © [Zoltan Kochan](https://www.kochan.io/) diff --git a/packages/default-fetcher/package.json b/packages/client/package.json similarity index 67% rename from packages/default-fetcher/package.json rename to packages/client/package.json index 22a76345da..16f1be7692 100644 --- a/packages/default-fetcher/package.json +++ b/packages/client/package.json @@ -1,7 +1,7 @@ { - "name": "@pnpm/default-fetcher", - "version": "6.0.9", - "description": "pnpm's default package fetcher", + "name": "@pnpm/client", + "version": "0.0.0", + "description": "Creates the package resolve and fetch functions", "main": "lib/index.js", "typings": "lib/index.d.ts", "files": [ @@ -13,12 +13,12 @@ }, "scripts": { "lint": "tslint -c ../../tslint.json src/**/*.ts test/**/*.ts", - "_test": "cd ../.. && c8 --reporter lcov --reports-dir packages/default-fetcher/coverage ts-node packages/default-fetcher/test --type-check", + "_test": "cd ../.. && c8 --reporter lcov --reports-dir packages/client/coverage ts-node packages/client/test --type-check", "test": "pnpm run compile && pnpm run _test", "prepublishOnly": "pnpm run compile", "compile": "rimraf lib tsconfig.tsbuildinfo && tsc --build" }, - "repository": "https://github.com/pnpm/pnpm/blob/master/packages/default-fetcher", + "repository": "https://github.com/pnpm/pnpm/blob/master/packages/client", "keywords": [ "pnpm", "resolver", @@ -29,14 +29,19 @@ "bugs": { "url": "https://github.com/pnpm/pnpm/issues" }, - "homepage": "https://github.com/pnpm/pnpm/blob/master/packages/default-fetcher#readme", + "homepage": "https://github.com/pnpm/pnpm/blob/master/packages/client#readme", "dependencies": { + "@pnpm/default-resolver": "workspace:^9.0.3", + "@pnpm/fetch": "workspace:^2.0.2", + "@pnpm/fetching-types": "workspace:^0.0.0", "@pnpm/git-fetcher": "workspace:3.0.4", "@pnpm/resolver-base": "workspace:7.0.3", - "@pnpm/tarball-fetcher": "workspace:7.1.4" + "@pnpm/tarball-fetcher": "workspace:7.1.4", + "credentials-by-uri": "^2.0.0", + "mem": "^6.1.0" }, "devDependencies": { - "@pnpm/default-fetcher": "link:", + "@pnpm/client": "link:", "@pnpm/logger": "3.2.2" }, "funding": "https://opencollective.com/pnpm" diff --git a/packages/client/src/index.ts b/packages/client/src/index.ts new file mode 100644 index 0000000000..f31abde17b --- /dev/null +++ b/packages/client/src/index.ts @@ -0,0 +1,39 @@ +import createResolve, { ResolverFactoryOptions } from '@pnpm/default-resolver' +import { createFetchFromRegistry } from '@pnpm/fetch' +import { FetchFromRegistry, GetCredentials, RetryTimeoutOptions } from '@pnpm/fetching-types' +import fetchFromGit from '@pnpm/git-fetcher' +import createTarballFetcher from '@pnpm/tarball-fetcher' +import getCredentialsByURI = require('credentials-by-uri') +import mem = require('mem') + +export default function (opts: { + ca?: string, + cert?: string, + key?: string, + localAddress?: string, + proxy?: string, + authConfig: Record, + retry?: RetryTimeoutOptions, + strictSSL?: boolean, + userAgent?: string, +} & ResolverFactoryOptions) { + const fetchFromRegistry = createFetchFromRegistry(opts) + const getCredentials = mem((registry: string) => getCredentialsByURI(opts.authConfig, registry)) + return { + fetchers: createFetchers(fetchFromRegistry, getCredentials, opts), + resolve: createResolve(fetchFromRegistry, getCredentials, opts), + } +} + +function createFetchers ( + fetchFromRegistry: FetchFromRegistry, + getCredentials: GetCredentials, + opts: { + retry?: RetryTimeoutOptions, + } +) { + return { + ...createTarballFetcher(fetchFromRegistry, getCredentials, opts), + ...fetchFromGit(), + } +} diff --git a/packages/client/test/index.ts b/packages/client/test/index.ts new file mode 100644 index 0000000000..0517ae805f --- /dev/null +++ b/packages/client/test/index.ts @@ -0,0 +1,13 @@ +/// +import createClient from '@pnpm/client' +import test = require('tape') + +test('createClient()', t => { + const client = createClient({ + authConfig: { registry: 'https://registry.npmjs.org/' }, + metaCache: new Map(), + storeDir: '', + }) + t.equal(typeof client, 'object') + t.end() +}) diff --git a/packages/default-fetcher/tsconfig.json b/packages/client/tsconfig.json similarity index 71% rename from packages/default-fetcher/tsconfig.json rename to packages/client/tsconfig.json index 235cb18b1c..30338a6bae 100644 --- a/packages/default-fetcher/tsconfig.json +++ b/packages/client/tsconfig.json @@ -9,6 +9,15 @@ "../../typings/**/*.d.ts" ], "references": [ + { + "path": "../default-resolver" + }, + { + "path": "../fetch" + }, + { + "path": "../fetching-types" + }, { "path": "../git-fetcher" }, diff --git a/packages/config/src/Config.ts b/packages/config/src/Config.ts index 75d20e2c39..d1ba87f4eb 100644 --- a/packages/config/src/Config.ts +++ b/packages/config/src/Config.ts @@ -55,6 +55,7 @@ export interface Config { name: string, version: string, }, + preferOffline?: boolean, sideEffectsCache?: boolean, sideEffectsCacheReadonly?: boolean, shamefullyHoist?: boolean, @@ -64,7 +65,6 @@ export interface Config { // proxy proxy?: string, - httpsProxy?: string, localAddress?: string, // ssl @@ -116,9 +116,10 @@ export interface Config { export interface ConfigWithDeprecatedSettings extends Config { frozenShrinkwrap?: boolean, globalPrefix?: string, + httpsProxy?: string, lockfileDirectory?: string, - shrinkwrapDirectory?: string, - shrinkwrapOnly?: boolean, preferFrozenShrinkwrap?: boolean, sharedWorkspaceShrinkwrap?: boolean, + shrinkwrapDirectory?: string, + shrinkwrapOnly?: boolean, } diff --git a/packages/config/src/index.ts b/packages/config/src/index.ts index 05e08dc94d..ffa708069c 100644 --- a/packages/config/src/index.ts +++ b/packages/config/src/index.ts @@ -334,5 +334,8 @@ export default async ( break } } + if (pnpmConfig.httpsProxy) { + pnpmConfig.proxy = pnpmConfig.httpsProxy + } return { config: pnpmConfig, warnings } } diff --git a/packages/default-fetcher/CHANGELOG.md b/packages/default-fetcher/CHANGELOG.md deleted file mode 100644 index 034a916cf0..0000000000 --- a/packages/default-fetcher/CHANGELOG.md +++ /dev/null @@ -1,142 +0,0 @@ -# @pnpm/default-fetcher - -## 6.0.9 - -### Patch Changes - -- Updated dependencies [b7b026822] - - @pnpm/tarball-fetcher@7.1.4 - -## 6.0.8 - -### Patch Changes - -- @pnpm/git-fetcher@3.0.4 -- @pnpm/resolver-base@7.0.3 -- @pnpm/tarball-fetcher@7.1.3 - -## 6.0.7 - -### Patch Changes - -- Updated dependencies [1520e3d6f] - - @pnpm/tarball-fetcher@7.1.2 - -## 6.0.6 - -### Patch Changes - -- @pnpm/git-fetcher@3.0.3 -- @pnpm/resolver-base@7.0.2 -- @pnpm/tarball-fetcher@7.1.1 - -## 6.0.5 - -### Patch Changes - -- @pnpm/git-fetcher@3.0.2 -- @pnpm/tarball-fetcher@7.1.0 - -## 6.0.4 - -### Patch Changes - -- Updated dependencies [2ebb7af33] - - @pnpm/tarball-fetcher@7.1.0 - - @pnpm/git-fetcher@3.0.2 - -## 6.0.3 - -### Patch Changes - -- @pnpm/tarball-fetcher@7.0.1 - -## 6.0.2 - -### Patch Changes - -- Updated dependencies [bcd4aa1aa] - - @pnpm/tarball-fetcher@7.0.0 - - @pnpm/git-fetcher@3.0.2 - -## 6.0.1 - -### Patch Changes - -- Updated dependencies [187615f87] - - @pnpm/git-fetcher@3.0.1 - -## 6.0.0 - -### Major Changes - -- b6a82072e: Using a content-addressable filesystem for storing packages. - -### Patch Changes - -- Updated dependencies [c47babd52] -- Updated dependencies [7db36dcb3] -- Updated dependencies [f516d266c] -- Updated dependencies [f93583d52] -- Updated dependencies [b6a82072e] -- Updated dependencies [1ae66a0dc] -- Updated dependencies [42e6490d1] - - @pnpm/tarball-fetcher@6.0.0 - - @pnpm/git-fetcher@3.0.0 - - @pnpm/resolver-base@7.0.1 - -## 6.0.0-alpha.5 - -## 6.0.0-alpha.4 - -### Patch Changes - -- Updated dependencies [c47babd5] - - @pnpm/tarball-fetcher@6.0.0-alpha.4 - - @pnpm/git-fetcher@2.0.11-alpha.4 - - @pnpm/resolver-base@7.0.1-alpha.0 - -## 6.0.0-alpha.3 - -### Patch Changes - -- Updated dependencies [1ae66a0dc] - - @pnpm/tarball-fetcher@6.0.0-alpha.3 - - @pnpm/git-fetcher@2.0.10-alpha.3 - -## 5.1.19-alpha.2 - -### Patch Changes - -- Updated dependencies [7db36dcb3] -- Updated dependencies [42e6490d1] - - @pnpm/tarball-fetcher@6.0.0-alpha.2 - - @pnpm/git-fetcher@3.0.0-alpha.2 - -## 6.0.0-alpha.1 - -### Patch Changes - -- Updated dependencies [4f62d0383] -- Updated dependencies [f93583d52] - - @pnpm/tarball-fetcher@6.0.0-alpha.1 - - @pnpm/git-fetcher@2.0.10-alpha.1 - -## 6.0.0-alpha.0 - -### Major Changes - -- 91c4b5954: Using a content-addressable filesystem for storing packages. - -### Patch Changes - -- Updated dependencies [91c4b5954] - - @pnpm/git-fetcher@3.0.0-alpha.0 - - @pnpm/tarball-fetcher@6.0.0-alpha.0 - -## 5.1.18 - -### Patch Changes - -- Updated dependencies [907c63a48] - - @pnpm/tarball-fetcher@5.1.15 diff --git a/packages/default-fetcher/README.md b/packages/default-fetcher/README.md deleted file mode 100644 index e096185cf5..0000000000 --- a/packages/default-fetcher/README.md +++ /dev/null @@ -1,17 +0,0 @@ -# @pnpm/default-fetcher - -> pnpm's default package fetcher - - -[![npm version](https://img.shields.io/npm/v/@pnpm/default-fetcher.svg)](https://www.npmjs.com/package/@pnpm/default-fetcher) - - -## Installation - -```sh - add @pnpm/default-fetcher -``` - -## License - -MIT © [Zoltan Kochan](https://www.kochan.io/) diff --git a/packages/default-fetcher/src/index.ts b/packages/default-fetcher/src/index.ts deleted file mode 100644 index 6927196b2c..0000000000 --- a/packages/default-fetcher/src/index.ts +++ /dev/null @@ -1,28 +0,0 @@ -import fetchFromGit from '@pnpm/git-fetcher' -import createTarballFetcher from '@pnpm/tarball-fetcher' - -export default function ( - opts: { - alwaysAuth?: boolean, - registry: string, - rawConfig: object, - strictSsl?: boolean, - proxy?: string, - httpsProxy?: string, - localAddress?: string, - cert?: string, - key?: string, - ca?: string, - fetchRetries?: number, - fetchRetryFactor?: number, - fetchRetryMintimeout?: number, - fetchRetryMaxtimeout?: number, - userAgent?: string, - offline?: boolean, - } -) { - return { - ...createTarballFetcher(opts), - ...fetchFromGit(), - } -} diff --git a/packages/default-fetcher/test/index.ts b/packages/default-fetcher/test/index.ts deleted file mode 100644 index 704e3227ce..0000000000 --- a/packages/default-fetcher/test/index.ts +++ /dev/null @@ -1,14 +0,0 @@ -/// -import createFetcher from '@pnpm/default-fetcher' -import test = require('tape') - -test('createFetcher()', t => { - const fetcher = createFetcher({ - alwaysAuth: false, - rawConfig: {}, - registry: 'https://registry.npmjs.org/', - strictSsl: false, - }) - t.equal(typeof fetcher, 'object') - t.end() -}) diff --git a/packages/default-resolver/package.json b/packages/default-resolver/package.json index d8344fb31c..133aaa65c6 100644 --- a/packages/default-resolver/package.json +++ b/packages/default-resolver/package.json @@ -32,6 +32,7 @@ "homepage": "https://github.com/pnpm/pnpm/blob/master/packages/default-resolver#readme", "dependencies": { "@pnpm/error": "workspace:1.2.0", + "@pnpm/fetching-types": "workspace:^0.0.0", "@pnpm/git-resolver": "workspace:4.0.15", "@pnpm/local-resolver": "workspace:5.0.10", "@pnpm/npm-resolver": "workspace:8.1.2", @@ -40,6 +41,7 @@ }, "devDependencies": { "@pnpm/default-resolver": "link:", + "@pnpm/fetch": "workspace:^2.0.2", "@pnpm/logger": "3.2.2" }, "funding": "https://opencollective.com/pnpm" diff --git a/packages/default-resolver/src/index.ts b/packages/default-resolver/src/index.ts index 00beb0d1dc..28485ce8a1 100644 --- a/packages/default-resolver/src/index.ts +++ b/packages/default-resolver/src/index.ts @@ -1,4 +1,5 @@ import PnpmError from '@pnpm/error' +import { FetchFromRegistry, GetCredentials } from '@pnpm/fetching-types' import createResolveFromGit from '@pnpm/git-resolver' import resolveFromLocal from '@pnpm/local-resolver' import createResolveFromNpm, { @@ -18,9 +19,11 @@ export { } export default function createResolver ( + fetchFromRegistry: FetchFromRegistry, + getCredentials: GetCredentials, pnpmOpts: ResolverFactoryOptions ): ResolveFunction { - const resolveFromNpm = createResolveFromNpm(pnpmOpts) + const resolveFromNpm = createResolveFromNpm(fetchFromRegistry, getCredentials, pnpmOpts) const resolveFromGit = createResolveFromGit(pnpmOpts) return async (wantedDependency, opts) => { const resolution = await resolveFromNpm(wantedDependency, opts as ResolveFromNpmOptions) diff --git a/packages/default-resolver/test/index.ts b/packages/default-resolver/test/index.ts index 13c8b7902a..6db2ec4b51 100644 --- a/packages/default-resolver/test/index.ts +++ b/packages/default-resolver/test/index.ts @@ -1,13 +1,12 @@ /// import createResolver from '@pnpm/default-resolver' +import { createFetchFromRegistry } from '@pnpm/fetch' import test = require('tape') test('createResolver()', t => { - const resolve = createResolver({ + const getCredentials = () => ({ authHeaderValue: '', alwaysAuth: false }) + const resolve = createResolver(createFetchFromRegistry({}), getCredentials, { metaCache: new Map(), - rawConfig: { - registry: 'https://registry.npmjs.org/', - }, storeDir: '.store', }) t.equal(typeof resolve, 'function') diff --git a/packages/default-resolver/tsconfig.json b/packages/default-resolver/tsconfig.json index 18e2904041..764be7c9a6 100644 --- a/packages/default-resolver/tsconfig.json +++ b/packages/default-resolver/tsconfig.json @@ -12,6 +12,9 @@ { "path": "../error" }, + { + "path": "../fetching-types" + }, { "path": "../git-resolver" }, @@ -26,6 +29,9 @@ }, { "path": "../tarball-resolver" + }, + { + "path": "../fetch" } ] } diff --git a/packages/fetch-from-npm-registry/CHANGELOG.md b/packages/fetch-from-npm-registry/CHANGELOG.md deleted file mode 100644 index fa86b10249..0000000000 --- a/packages/fetch-from-npm-registry/CHANGELOG.md +++ /dev/null @@ -1,37 +0,0 @@ -# fetch-from-npm-registry - -## 4.1.2 - -### Patch Changes - -- @pnpm/fetch@2.0.2 - -## 4.1.1 - -### Patch Changes - -- @pnpm/fetch@2.0.1 - -## 4.1.0 - -### Minor Changes - -- 2ebb7af33: Print a warning when request fails and a retry will happen. Breaking changes in the programmatic API of `@pnpm/fetch`. - -### Patch Changes - -- Updated dependencies [2ebb7af33] - - @pnpm/fetch@2.0.0 - -## 4.0.3 - -### Patch Changes - -- 872f81ca1: Don't remove authorization headers when redirecting requests to the same host. - -## 4.0.3 - -### Patch Changes - -- @pnpm/fetch@1.0.4 -- @pnpm/npm-registry-agent@2.0.4 diff --git a/packages/fetch-from-npm-registry/README.md b/packages/fetch-from-npm-registry/README.md deleted file mode 100644 index 28cb1528c9..0000000000 --- a/packages/fetch-from-npm-registry/README.md +++ /dev/null @@ -1,57 +0,0 @@ -# fetch-from-npm-registry - -> A fetch lib specifically for using with the npm registry - - -[![npm version](https://img.shields.io/npm/v/fetch-from-npm-registry.svg)](https://www.npmjs.com/package/fetch-from-npm-registry) - - -## Installation - -```sh - add fetch-from-npm-registry -``` - -## Usage - - -```js -'use strict' -const createFetcher = require('fetch-from-npm-registry').default - -const fetchFromNpmRegistry = createFetcher({userAgent: 'fetch-from-npm-registry'}) - -fetchFromNpmRegistry('https://registry.npmjs.org/is-positive') - .then(res => res.json()) - .then(metadata => console.log(JSON.stringify(metadata.versions['1.0.0'], null, 2))) - //> { - // "name": "is-positive", - // "version": "1.0.0", - // "devDependencies": { - // "ava": "^0.0.4" - // }, - // "_hasShrinkwrap": false, - // "directories": {}, - // "dist": { - // "shasum": "88009856b64a2f1eb7d8bb0179418424ae0452cb", - // "tarball": "https://registry.npmjs.org/is-positive/-/is-positive-1.0.0.tgz" - // }, - // "engines": { - // "node": ">=0.10.0" - // } - // } -``` - - -## API - -### `fetchFromNpmRegistry(url, opts)` - -#### Arguments - -- **url** - _String_ - url to request -- **opts.fullMetadata** - _Boolean_ - If true, don't attempt to fetch filtered ("corgi") registry metadata. (default: false) - -## License - -MIT © [Zoltan Kochan](https://www.kochan.io/) diff --git a/packages/fetch-from-npm-registry/example.js b/packages/fetch-from-npm-registry/example.js deleted file mode 100644 index 42425e932d..0000000000 --- a/packages/fetch-from-npm-registry/example.js +++ /dev/null @@ -1,8 +0,0 @@ -'use strict' -const createFetcher = require('fetch-from-npm-registry').default - -const fetchFromNpmRegistry = createFetcher({userAgent: 'fetch-from-npm-registry'}) - -fetchFromNpmRegistry('https://registry.npmjs.org/is-positive') - .then(res => res.json()) - .then(metadata => console.log(JSON.stringify(metadata.versions['1.0.0'], null, 2))) diff --git a/packages/fetch-from-npm-registry/package.json b/packages/fetch-from-npm-registry/package.json deleted file mode 100644 index 93d44ce8ca..0000000000 --- a/packages/fetch-from-npm-registry/package.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "name": "fetch-from-npm-registry", - "version": "4.1.2", - "description": "A fetch lib specifically for using with the npm registry", - "main": "lib/index.js", - "typings": "lib/index.d.ts", - "files": [ - "lib", - "!*.map" - ], - "engines": { - "node": ">=10.13" - }, - "scripts": { - "lint": "tslint -c ../../tslint.json src/**/*.ts test/**/*.ts", - "_test": "cd ../.. && c8 --reporter lcov --reports-dir packages/fetch-from-npm-registry/coverage ts-node packages/fetch-from-npm-registry/test --type-check", - "test": "pnpm run compile && pnpm run _test", - "prepublishOnly": "pnpm run compile", - "compile": "rimraf lib tsconfig.tsbuildinfo && tsc --build" - }, - "repository": "https://github.com/pnpm/pnpm/blob/master/packages/fetch-from-npm-registry", - "keywords": [ - "fetch", - "npm" - ], - "author": "Zoltan Kochan (https://www.kochan.io/)", - "license": "MIT", - "bugs": { - "url": "https://github.com/pnpm/pnpm/issues" - }, - "homepage": "https://github.com/pnpm/pnpm/blob/master/packages/fetch-from-npm-registry#readme", - "peerDependencies": { - "@pnpm/logger": "^3.1.0" - }, - "dependencies": { - "@pnpm/fetch": "workspace:2.0.2", - "@pnpm/npm-registry-agent": "workspace:2.0.3" - }, - "devDependencies": { - "fetch-from-npm-registry": "link:", - "nock": "12.0.3" - }, - "funding": "https://opencollective.com/pnpm" -} diff --git a/packages/fetch/package.json b/packages/fetch/package.json index 25b5c82850..3171189a7c 100644 --- a/packages/fetch/package.json +++ b/packages/fetch/package.json @@ -13,7 +13,8 @@ }, "scripts": { "lint": "tslint -c ../../tslint.json src/**/*.ts test/**/*.ts", - "test": "pnpm run compile", + "_test": "cd ../.. && c8 --reporter lcov --reports-dir packages/fetch/coverage ts-node packages/fetch/test --type-check", + "test": "pnpm run compile && pnpm run _test", "prepublishOnly": "pnpm run compile", "compile": "rimraf lib tsconfig.tsbuildinfo && tsc --build" }, @@ -33,14 +34,18 @@ "homepage": "https://github.com/pnpm/pnpm/blob/master/packages/fetch#readme", "dependencies": { "@pnpm/core-loggers": "workspace:^4.1.2", + "@pnpm/fetching-types": "workspace:^0.0.0", + "@pnpm/npm-registry-agent": "workspace:^2.0.3", "@zkochan/retry": "^0.2.0", "node-fetch": "2.6.0", "node-fetch-unix": "2.3.0" }, "devDependencies": { + "@pnpm/fetch": "link:", "@pnpm/logger": "^3.2.2", "@types/node-fetch": "^2.5.7", - "cpy-cli": "^3.1.1" + "cpy-cli": "^3.1.1", + "nock": "^13.0.0" }, "funding": "https://opencollective.com/pnpm" } diff --git a/packages/fetch/src/fetch.ts b/packages/fetch/src/fetch.ts new file mode 100644 index 0000000000..c1dea93823 --- /dev/null +++ b/packages/fetch/src/fetch.ts @@ -0,0 +1,95 @@ +import { requestRetryLogger } from '@pnpm/core-loggers' +import { operation, RetryTimeoutOptions } from '@zkochan/retry' +import { Request, RequestInit as NodeRequestInit, Response } from 'node-fetch' +import fetch = require('node-fetch-unix') + +// retry settings +const MIN_TIMEOUT = 10 +const MAX_RETRIES = 5 +const MAX_RETRY_AFTER = 20 +const FACTOR = 6 + +export { Response, RetryTimeoutOptions } + +interface URLLike { + href: string +} + +export type RequestInfo = string | URLLike | Request + +export interface RequestInit extends NodeRequestInit { + retry?: RetryTimeoutOptions +} + +export const isRedirect = fetch.isRedirect + +export default async function fetchRetry (url: RequestInfo, opts: RequestInit = {}): Promise { + const retryOpts = Object.assign({ + factor: FACTOR, + // timeouts will be [10, 60, 360, 2160, 12960] + // (before randomization is added) + maxRetryAfter: MAX_RETRY_AFTER, + minTimeout: MIN_TIMEOUT, + retries: MAX_RETRIES, + }, opts.retry) + + const op = operation(retryOpts) + + try { + return await new Promise((resolve, reject) => op.attempt(async (attempt) => { + try { + // this will be retried + const res = await fetch(url, opts) + if ((res.status >= 500 && res.status < 600) || res.status === 429) { + throw new ResponseError(res) + } else { + resolve(res) + return + } + } catch (error) { + const timeout = op.retry(error) + if (timeout === false) { + reject(op.mainError()) + return + } + requestRetryLogger.debug({ + attempt, + error, + maxRetries: retryOpts.retries, + method: opts.method ?? 'GET', + timeout, + url: url.toString(), + }) + } + })) + } catch (err) { + if (err instanceof ResponseError) { + return err.res + } + throw err + } +} + +class ResponseError extends Error { + public res: Response + public code: number + public status: number + public statusCode: number + public url: string + constructor (res: Response) { + super(res.statusText) + + if (Error.captureStackTrace) { + Error.captureStackTrace(this, ResponseError) + } + + this.name = this.constructor.name + this.res = res + + // backward compat + this.code = this.status = this.statusCode = res.status + this.url = res.url + } +} + +exports.ResponseError = ResponseError diff --git a/packages/fetch-from-npm-registry/src/index.ts b/packages/fetch/src/fetchFromRegistry.ts similarity index 86% rename from packages/fetch-from-npm-registry/src/index.ts rename to packages/fetch/src/fetchFromRegistry.ts index 1fe9acfccf..eaffa772bd 100644 --- a/packages/fetch-from-npm-registry/src/index.ts +++ b/packages/fetch/src/fetchFromRegistry.ts @@ -1,6 +1,7 @@ -import fetch, { isRedirect, Response } from '@pnpm/fetch' +import { FetchFromRegistry } from '@pnpm/fetching-types' import npmRegistryAgent from '@pnpm/npm-registry-agent' import { URL } from 'url' +import fetch, { isRedirect, Response } from './fetch' const USER_AGENT = 'pnpm' // or maybe make it `${pkg.name}/${pkg.version} (+https://npm.im/${pkg.name})` @@ -8,8 +9,6 @@ const CORGI_DOC = 'application/vnd.npm.install-v1+json; q=1.0, application/json; const JSON_DOC = 'application/json' const MAX_FOLLOWED_REDIRECTS = 20 -export type FetchFromRegistry = (url: string, opts?: { authHeaderValue?: string }) => Promise - export default function ( defaultOpts: { fullMetadata?: boolean, @@ -21,14 +20,6 @@ export default function ( cert?: string, key?: string, strictSSL?: boolean, - // retry - retry?: { - retries?: number, - factor?: number, - minTimeout?: number, - maxTimeout?: number, - randomize?: boolean, - }, userAgent?: string, } ): FetchFromRegistry { @@ -49,6 +40,7 @@ export default function ( const agent = npmRegistryAgent(urlObject.href, { ...defaultOpts, ...opts, + strictSSL: defaultOpts.strictSSL ?? true, } as any) // tslint:disable-line headers['connection'] = agent ? 'keep-alive' : 'close' @@ -60,7 +52,7 @@ export default function ( compress: false, headers, redirect: 'manual', - retry: defaultOpts.retry, + retry: opts?.retry, }) if (!isRedirect(response.status) || redirects >= MAX_FOLLOWED_REDIRECTS) { return response diff --git a/packages/fetch/src/index.ts b/packages/fetch/src/index.ts index 9f065b614a..f71e891d48 100644 --- a/packages/fetch/src/index.ts +++ b/packages/fetch/src/index.ts @@ -1,102 +1,6 @@ -import { requestRetryLogger } from '@pnpm/core-loggers' -import * as retry from '@zkochan/retry' -import { Request, RequestInit as NodeRequestInit, Response } from 'node-fetch' -import fetch = require('node-fetch-unix') +import { FetchFromRegistry } from '@pnpm/fetching-types' +import fetch, { RetryTimeoutOptions } from './fetch' +import createFetchFromRegistry from './fetchFromRegistry' -// retry settings -const MIN_TIMEOUT = 10 -const MAX_RETRIES = 5 -const MAX_RETRY_AFTER = 20 -const FACTOR = 6 - -export { Response } - -export interface RetryOpts { - factor?: number - maxTimeout?: number - minTimeout?: number - retries?: number -} - -interface URLLike { - href: string -} - -export type RequestInfo = string | URLLike | Request - -export interface RequestInit extends NodeRequestInit { - retry?: RetryOpts -} - -export const isRedirect = fetch.isRedirect - -export default async function fetchRetry (url: RequestInfo, opts: RequestInit = {}): Promise { - const retryOpts = Object.assign({ - factor: FACTOR, - // timeouts will be [10, 60, 360, 2160, 12960] - // (before randomization is added) - maxRetryAfter: MAX_RETRY_AFTER, - minTimeout: MIN_TIMEOUT, - retries: MAX_RETRIES, - }, opts.retry) - - const op = retry.operation(retryOpts) - - try { - return await new Promise((resolve, reject) => op.attempt(async (attempt) => { - try { - // this will be retried - const res = await fetch(url, opts) - if ((res.status >= 500 && res.status < 600) || res.status === 429) { - throw new ResponseError(res) - } else { - resolve(res) - return - } - } catch (error) { - const timeout = op.retry(error) - if (timeout === false) { - reject(op.mainError()) - return - } - requestRetryLogger.debug({ - attempt, - error, - maxRetries: retryOpts.retries, - method: opts.method ?? 'GET', - timeout, - url: url.toString(), - }) - } - })) - } catch (err) { - if (err instanceof ResponseError) { - return err.res - } - throw err - } -} - -class ResponseError extends Error { - public res: Response - public code: number - public status: number - public statusCode: number - public url: string - constructor (res: Response) { - super(res.statusText) - - if (Error.captureStackTrace) { - Error.captureStackTrace(this, ResponseError) - } - - this.name = this.constructor.name - this.res = res - - // backward compat - this.code = this.status = this.statusCode = res.status - this.url = res.url - } -} - -exports.ResponseError = ResponseError +export default fetch +export { createFetchFromRegistry, FetchFromRegistry, RetryTimeoutOptions } diff --git a/packages/fetch-from-npm-registry/test/index.ts b/packages/fetch/test/fetchFromRegistry.ts similarity index 69% rename from packages/fetch-from-npm-registry/test/index.ts rename to packages/fetch/test/fetchFromRegistry.ts index 48dac8cf18..c35b6ba9d7 100644 --- a/packages/fetch-from-npm-registry/test/index.ts +++ b/packages/fetch/test/fetchFromRegistry.ts @@ -1,20 +1,20 @@ /// -import createRegClient from 'fetch-from-npm-registry' +import { createFetchFromRegistry } from '@pnpm/fetch' import nock = require('nock') import test = require('tape') -test('fetchFromNpmRegistry', async t => { - const fetchFromNpmRegistry = createRegClient({}) - const res = await fetchFromNpmRegistry('https://registry.npmjs.org/is-positive') +test('fetchFromRegistry', async t => { + const fetchFromRegistry = createFetchFromRegistry({}) + const res = await fetchFromRegistry('https://registry.npmjs.org/is-positive') const metadata = await res.json() t.equal(metadata.name, 'is-positive') t.notOk(metadata.versions['1.0.0'].scripts) t.end() }) -test('fetchFromNpmRegistry fullMetadata', async t => { - const fetchFromNpmRegistry = createRegClient({ fullMetadata: true }) - const res = await fetchFromNpmRegistry('https://registry.npmjs.org/is-positive') +test('fetchFromRegistry fullMetadata', async t => { + const fetchFromRegistry = createFetchFromRegistry({ fullMetadata: true }) + const res = await fetchFromRegistry('https://registry.npmjs.org/is-positive') const metadata = await res.json() t.equal(metadata.name, 'is-positive') t.ok(metadata.versions['1.0.0'].scripts) @@ -31,8 +31,8 @@ test('authorization headers are removed before redirection if the target is on a .get('/is-positive') .reply(200, { ok: true }) - const fetchFromNpmRegistry = createRegClient({ fullMetadata: true }) - const res = await fetchFromNpmRegistry( + const fetchFromRegistry = createFetchFromRegistry({ fullMetadata: true }) + const res = await fetchFromRegistry( 'http://registry.pnpm.js.org/is-positive', { authHeaderValue: 'Bearer 123' } ) @@ -54,8 +54,8 @@ test('authorization headers are not removed before redirection if the target is .get('/is-positive-new') .reply(200, { ok: true }) - const fetchFromNpmRegistry = createRegClient({ fullMetadata: true }) - const res = await fetchFromNpmRegistry( + const fetchFromRegistry = createFetchFromRegistry({ fullMetadata: true }) + const res = await fetchFromRegistry( 'http://registry.pnpm.js.org/is-positive', { authHeaderValue: 'Bearer 123' } ) @@ -66,10 +66,10 @@ test('authorization headers are not removed before redirection if the target is }) test('switch to the correct agent for requests on redirect from http: to https:', async (t) => { - const fetchFromNpmRegistry = createRegClient({ fullMetadata: true }) + const fetchFromRegistry = createFetchFromRegistry({ fullMetadata: true }) // We can test this on any endpoint that redirects from http: to https: - const { status } = await fetchFromNpmRegistry('http://pnpm.js.org/css/main.css') + const { status } = await fetchFromRegistry('http://pnpm.js.org/css/main.css') t.equal(status, 200) t.end() diff --git a/packages/fetch/test/index.ts b/packages/fetch/test/index.ts new file mode 100644 index 0000000000..b02626c1eb --- /dev/null +++ b/packages/fetch/test/index.ts @@ -0,0 +1 @@ +import './fetchFromRegistry' diff --git a/packages/fetch/tsconfig.json b/packages/fetch/tsconfig.json index 79d8a13f9a..c68ae3d1c0 100644 --- a/packages/fetch/tsconfig.json +++ b/packages/fetch/tsconfig.json @@ -11,6 +11,12 @@ "references": [ { "path": "../core-loggers" + }, + { + "path": "../fetching-types" + }, + { + "path": "../npm-registry-agent" } ] } diff --git a/packages/fetching-types/README.md b/packages/fetching-types/README.md new file mode 100644 index 0000000000..55b080426e --- /dev/null +++ b/packages/fetching-types/README.md @@ -0,0 +1,7 @@ +# @pnpm/fetching-types + +> Types for the fetching + +## License + +MIT diff --git a/packages/fetching-types/package.json b/packages/fetching-types/package.json new file mode 100644 index 0000000000..973dd47c8b --- /dev/null +++ b/packages/fetching-types/package.json @@ -0,0 +1,35 @@ +{ + "name": "@pnpm/fetching-types", + "version": "0.0.0", + "description": "Types for fetching", + "main": "lib/index.js", + "typings": "lib/index.d.ts", + "engines": { + "node": ">=10.13" + }, + "files": [ + "lib", + "!*.map" + ], + "repository": "https://github.com/pnpm/pnpm/blob/master/packages/fetching-types", + "keywords": [ + "pnpm", + "types" + ], + "author": "Zoltan Kochan (https://www.kochan.io/)", + "license": "MIT", + "bugs": { + "url": "https://github.com/pnpm/pnpm/issues" + }, + "homepage": "https://github.com/pnpm/pnpm/blob/master/packages/fetching-types#readme", + "scripts": { + "compile": "rimraf lib tsconfig.tsbuildinfo && tsc --build", + "prepublishOnly": "pnpm run compile", + "test": "pnpm run compile" + }, + "funding": "https://opencollective.com/pnpm", + "dependencies": { + "@types/node-fetch": "^2.5.7", + "@zkochan/retry": "^0.2.0" + } +} diff --git a/packages/fetching-types/src/index.ts b/packages/fetching-types/src/index.ts new file mode 100644 index 0000000000..99ac2a7be0 --- /dev/null +++ b/packages/fetching-types/src/index.ts @@ -0,0 +1,17 @@ +import { RetryTimeoutOptions } from '@zkochan/retry' +import { Response } from 'node-fetch' + +export { RetryTimeoutOptions } + +export type FetchFromRegistry = ( + url: string, + opts?: { + authHeaderValue?: string, + retry?: RetryTimeoutOptions, + } +) => Promise + +export type GetCredentials = (registry: string) => { + authHeaderValue: string | undefined, + alwaysAuth: boolean | undefined, +} diff --git a/packages/fetch-from-npm-registry/tsconfig.json b/packages/fetching-types/tsconfig.json similarity index 61% rename from packages/fetch-from-npm-registry/tsconfig.json rename to packages/fetching-types/tsconfig.json index e88fc8d21a..41aa731a88 100644 --- a/packages/fetch-from-npm-registry/tsconfig.json +++ b/packages/fetching-types/tsconfig.json @@ -8,12 +8,5 @@ "src/**/*.ts", "../../typings/**/*.d.ts" ], - "references": [ - { - "path": "../fetch" - }, - { - "path": "../npm-registry-agent" - } - ] + "references": [] } diff --git a/packages/headless/package.json b/packages/headless/package.json index de645e1b02..58c664c06c 100644 --- a/packages/headless/package.json +++ b/packages/headless/package.json @@ -17,8 +17,7 @@ }, "devDependencies": { "@pnpm/assert-project": "workspace:*", - "@pnpm/default-fetcher": "workspace:*", - "@pnpm/default-resolver": "workspace:*", + "@pnpm/client": "workspace:^0.0.0", "@pnpm/headless": "link:", "@pnpm/logger": "3.2.2", "@pnpm/package-store": "workspace:*", @@ -36,7 +35,6 @@ "load-json-file": "6.2.0", "mz": "2.7.0", "npm-run-all": "4.1.5", - "read-yaml-file": "2.0.0", "sinon": "9.0.2", "tape-promise": "4.0.0", "tempy": "0.5.0", diff --git a/packages/headless/test/index.ts b/packages/headless/test/index.ts index aba5d30f63..407f51dbec 100644 --- a/packages/headless/test/index.ts +++ b/packages/headless/test/index.ts @@ -18,7 +18,6 @@ import loadJsonFile = require('load-json-file') import fs = require('mz/fs') import path = require('path') import exists = require('path-exists') -import readYamlFile from 'read-yaml-file' import sinon = require('sinon') import test = require('tape') import tempy = require('tempy') diff --git a/packages/headless/test/utils/testDefaults.ts b/packages/headless/test/utils/testDefaults.ts index d160fae5fc..8c564e93ef 100644 --- a/packages/headless/test/utils/testDefaults.ts +++ b/packages/headless/test/utils/testDefaults.ts @@ -1,5 +1,4 @@ -import createFetcher from '@pnpm/default-fetcher' -import createResolver from '@pnpm/default-resolver' +import createClient from '@pnpm/client' import { HeadlessOptions } from '@pnpm/headless' import createStore from '@pnpm/package-store' import { fromDir as readPackageJsonFromDir } from '@pnpm/read-package-json' @@ -12,10 +11,10 @@ import tempy = require('tempy') const registry = `http://localhost:${REGISTRY_MOCK_PORT}/` const retryOpts = { - fetchRetries: 2, - fetchRetryFactor: 10, - fetchRetryMaxtimeout: 60_000, - fetchRetryMintimeout: 10_000, + factor: 10, + retries: 2, + retryMaxtimeout: 60_000, + retryMintimeout: 10_000, } export default async function testDefaults ( @@ -35,23 +34,18 @@ export default async function testDefaults ( { lockfileDir } ) storeDir = await storePath(lockfileDir, storeDir) - const rawConfig = { registry } + const authConfig = { registry } + const { resolve, fetchers } = createClient({ + authConfig, + metaCache: new Map(), + retry: retryOpts, + storeDir, + ...resolveOpts, + ...fetchOpts, + }) const storeController = await createStore( - createResolver({ - metaCache: new Map(), - rawConfig, - storeDir, - strictSsl: true, - ...retryOpts, - ...resolveOpts, - }), - createFetcher({ - alwaysAuth: true, - rawConfig, - registry, - ...retryOpts, - ...fetchOpts, - }), + resolve, + fetchers, { storeDir, ...storeOpts, diff --git a/packages/headless/tsconfig.json b/packages/headless/tsconfig.json index 8b6fb49e7b..d2c18847d3 100644 --- a/packages/headless/tsconfig.json +++ b/packages/headless/tsconfig.json @@ -70,10 +70,7 @@ "path": "../../privatePackages/assert-project" }, { - "path": "../default-fetcher" - }, - { - "path": "../default-resolver" + "path": "../client" }, { "path": "../package-store" diff --git a/packages/npm-resolver/package.json b/packages/npm-resolver/package.json index 1d41eef67c..9d0b43813c 100644 --- a/packages/npm-resolver/package.json +++ b/packages/npm-resolver/package.json @@ -35,14 +35,12 @@ }, "dependencies": { "@pnpm/error": "workspace:1.2.0", + "@pnpm/fetching-types": "workspace:^0.0.0", "@pnpm/resolve-workspace-range": "workspace:1.0.1", "@pnpm/resolver-base": "workspace:7.0.3", "@pnpm/types": "workspace:6.2.0", - "credentials-by-uri": "2.0.0", "encode-registry": "2.0.2", - "fetch-from-npm-registry": "workspace:4.1.2", "load-json-file": "6.2.0", - "mem": "^6.1.0", "mz": "^2.7.0", "normalize-path": "3.0.0", "p-limit": "^3.0.1", @@ -55,6 +53,7 @@ "version-selector-type": "^3.0.0" }, "devDependencies": { + "@pnpm/fetch": "workspace:^2.0.2", "@pnpm/logger": "3.2.2", "@pnpm/npm-resolver": "link:", "@types/mz": "^2.7.1", diff --git a/packages/npm-resolver/src/fetch.ts b/packages/npm-resolver/src/fetch.ts index da95752501..75eda2575b 100644 --- a/packages/npm-resolver/src/fetch.ts +++ b/packages/npm-resolver/src/fetch.ts @@ -1,5 +1,5 @@ import PnpmError from '@pnpm/error' -import { FetchFromRegistry } from 'fetch-from-npm-registry' +import { FetchFromRegistry, RetryTimeoutOptions } from '@pnpm/fetching-types' import url = require('url') import { PackageMeta } from './pickPackage' @@ -38,12 +38,13 @@ class RegistryResponseError extends PnpmError { export default async function fromRegistry ( fetch: FetchFromRegistry, + retry: RetryTimeoutOptions, pkgName: string, registry: string, authHeaderValue?: string ) { const uri = toUri(pkgName, registry) - const response = await fetch(uri, { authHeaderValue }) as RegistryResponse + const response = await fetch(uri, { authHeaderValue, retry }) as RegistryResponse if (response.status > 400) { throw new RegistryResponseError({ package: pkgName, diff --git a/packages/npm-resolver/src/index.ts b/packages/npm-resolver/src/index.ts index 5e6e449a54..2195692e0a 100644 --- a/packages/npm-resolver/src/index.ts +++ b/packages/npm-resolver/src/index.ts @@ -1,4 +1,9 @@ import PnpmError from '@pnpm/error' +import { + FetchFromRegistry, + GetCredentials, + RetryTimeoutOptions, +} from '@pnpm/fetching-types' import resolveWorkspaceRange from '@pnpm/resolve-workspace-range' import { PreferredVersions, @@ -7,9 +12,6 @@ import { WorkspacePackages, } from '@pnpm/resolver-base' import { DependencyManifest } from '@pnpm/types' -import getCredentialsByURI = require('credentials-by-uri') -import createRegFetcher from 'fetch-from-npm-registry' -import mem = require('mem') import normalize = require('normalize-path') import pMemoize = require('p-memoize') import path = require('path') @@ -48,64 +50,30 @@ const META_DIR = 'metadata' const FULL_META_DIR = 'metadata-full' export interface ResolverFactoryOptions { - rawConfig: object, metaCache: PackageMetaCache, storeDir: string, - cert?: string, fullMetadata?: boolean, - key?: string, - ca?: string, - strictSsl?: boolean, - proxy?: string, - httpsProxy?: string, - localAddress?: string, - userAgent?: string, offline?: boolean, preferOffline?: boolean, - fetchRetries?: number, - fetchRetryFactor?: number, - fetchRetryMintimeout?: number, - fetchRetryMaxtimeout?: number, + retry?: RetryTimeoutOptions, } export default function createResolver ( + fetchFromRegistry: FetchFromRegistry, + getCredentials: GetCredentials, opts: ResolverFactoryOptions ) { - if (typeof opts.rawConfig !== 'object') { // tslint:disable-line - throw new TypeError('`opts.rawConfig` is required and needs to be an object') - } - if (typeof opts.rawConfig['registry'] !== 'string') { // tslint:disable-line - throw new TypeError('`opts.rawConfig.registry` is required and needs to be a string') - } if (typeof opts.metaCache !== 'object') { // tslint:disable-line throw new TypeError('`opts.metaCache` is required and needs to be an object') } if (typeof opts.storeDir !== 'string') { // tslint:disable-line throw new TypeError('`opts.storeDir` is required and needs to be a string') } - const fetch = pMemoize(fromRegistry.bind(null, createRegFetcher({ - ca: opts.ca, - cert: opts.cert, - fullMetadata: opts.fullMetadata, - key: opts.key, - localAddress: opts.localAddress, - proxy: opts.httpsProxy || opts.proxy, - retry: { - factor: opts.fetchRetryFactor, - maxTimeout: opts.fetchRetryMaxtimeout, - minTimeout: opts.fetchRetryMintimeout, - retries: opts.fetchRetries, - }, - strictSSL: opts.strictSsl, - userAgent: opts.userAgent, - })), { + const fetch = pMemoize(fromRegistry.bind(null, fetchFromRegistry, opts.retry ?? {}), { cacheKey: (...args) => JSON.stringify(args), maxAge: 1000 * 20, // 20 seconds }) - const getCreds = getCredentialsByURI.bind(null, opts.rawConfig) - const getAuthHeaderValueByURI = mem( - (registry: string) => getCreds(registry).authHeaderValue - ) + const getAuthHeaderValueByURI = (registry: string) => getCredentials(registry).authHeaderValue return resolveNpm.bind(null, { getAuthHeaderValueByURI, pickPackage: pickPackage.bind(null, { diff --git a/packages/npm-resolver/test/index.ts b/packages/npm-resolver/test/index.ts index 9167b4327f..a0bc96e970 100644 --- a/packages/npm-resolver/test/index.ts +++ b/packages/npm-resolver/test/index.ts @@ -1,5 +1,6 @@ /// -import createResolveFromNpm from '@pnpm/npm-resolver' +import { createFetchFromRegistry } from '@pnpm/fetch' +import _createResolveFromNpm from '@pnpm/npm-resolver' import loadJsonFile = require('load-json-file') import nock = require('nock') import path = require('path') @@ -18,6 +19,10 @@ const registry = 'https://registry.npmjs.org/' const delay = (time) => new Promise((resolve) => setTimeout(() => resolve(), time)) +const fetch = createFetchFromRegistry({}) +const getCredentials = () => ({ authHeaderValue: undefined, alwaysAuth: undefined }) +const createResolveFromNpm = _createResolveFromNpm.bind(null, fetch, getCredentials) + async function retryLoadJsonFile (filePath: string) { let retry = 0 while (true) { diff --git a/packages/npm-resolver/tsconfig.json b/packages/npm-resolver/tsconfig.json index 7f967367c5..b0cd070e82 100644 --- a/packages/npm-resolver/tsconfig.json +++ b/packages/npm-resolver/tsconfig.json @@ -13,6 +13,9 @@ { "path": "../error" }, + { + "path": "../fetching-types" + }, { "path": "../resolve-workspace-range" }, @@ -23,7 +26,7 @@ "path": "../types" }, { - "path": "../fetch-from-npm-registry" + "path": "../fetch" } ] } diff --git a/packages/outdated/package.json b/packages/outdated/package.json index 9fb00e0566..339d17e3b1 100644 --- a/packages/outdated/package.json +++ b/packages/outdated/package.json @@ -38,6 +38,7 @@ "@pnpm/constants": "workspace:4.0.0", "@pnpm/default-resolver": "workspace:9.0.3", "@pnpm/error": "workspace:1.2.0", + "@pnpm/fetch": "workspace:^2.0.2", "@pnpm/lockfile-file": "workspace:3.0.11", "@pnpm/lockfile-utils": "workspace:2.0.15", "@pnpm/manifest-utils": "workspace:1.0.3", @@ -46,8 +47,10 @@ "@pnpm/pick-registry-for-package": "workspace:1.0.3", "@pnpm/store-path": "^4.0.0", "@pnpm/types": "workspace:6.2.0", + "credentials-by-uri": "^2.0.0", "dependency-path": "workspace:5.0.2", "lru-cache": "^5.1.1", + "mem": "^6.1.0", "ramda": "^0.27.0", "semver": "^7.3.2" }, diff --git a/packages/outdated/src/createManifestGetter.ts b/packages/outdated/src/createManifestGetter.ts index f8ec7338d1..564d15dbf7 100644 --- a/packages/outdated/src/createManifestGetter.ts +++ b/packages/outdated/src/createManifestGetter.ts @@ -1,11 +1,15 @@ import createResolver, { ResolveFunction, ResolverFactoryOptions } from '@pnpm/default-resolver' +import { createFetchFromRegistry } from '@pnpm/fetch' import pickRegistryForPackage from '@pnpm/pick-registry-for-package' import { DependencyManifest, Registries } from '@pnpm/types' +import getCredentialsByURI = require('credentials-by-uri') import LRU = require('lru-cache') +import mem = require('mem') type GetManifestOpts = { dir: string, lockfileDir: string, + rawConfig: object, registries: Registries, } @@ -14,8 +18,9 @@ export type ManifestGetterOptions = Omit & export function createManifestGetter ( opts: ManifestGetterOptions ): (packageName: string, pref: string) => Promise { - const resolve = createResolver(Object.assign(opts, { - fullMetadata: false, + const fetch = createFetchFromRegistry(opts) + const getCredentials = mem((registry: string) => getCredentialsByURI(opts.rawConfig, registry)) + const resolve = createResolver(fetch, getCredentials, Object.assign(opts, { metaCache: new LRU({ max: 10000, maxAge: 120 * 1000, // 2 minutes diff --git a/packages/outdated/test/getManifest.spec.ts b/packages/outdated/test/getManifest.spec.ts index be9970f5a3..fcc5922341 100644 --- a/packages/outdated/test/getManifest.spec.ts +++ b/packages/outdated/test/getManifest.spec.ts @@ -8,6 +8,7 @@ test('getManifest()', async (t) => { const opts = { dir: '', lockfileDir: '', + rawConfig: {}, registries: { '@scope': 'https://pnpm.js.org/', 'default': 'https://registry.npmjs.org/', diff --git a/packages/outdated/tsconfig.json b/packages/outdated/tsconfig.json index fe2c41439e..65d99fbffa 100644 --- a/packages/outdated/tsconfig.json +++ b/packages/outdated/tsconfig.json @@ -18,6 +18,9 @@ { "path": "../error" }, + { + "path": "../fetch" + }, { "path": "../lockfile-file" }, diff --git a/packages/package-requester/package.json b/packages/package-requester/package.json index 12648be4c2..8012278e53 100644 --- a/packages/package-requester/package.json +++ b/packages/package-requester/package.json @@ -55,11 +55,9 @@ "ssri": "6.0.1" }, "devDependencies": { - "@pnpm/local-resolver": "workspace:*", + "@pnpm/client": "workspace:*", "@pnpm/logger": "3.2.2", - "@pnpm/npm-resolver": "workspace:*", "@pnpm/package-requester": "link:", - "@pnpm/tarball-fetcher": "workspace:*", "@types/mz": "^2.7.1", "@types/ncp": "^2.0.4", "@types/normalize-path": "^3.0.0", diff --git a/packages/package-requester/test/index.ts b/packages/package-requester/test/index.ts index bd61aced23..628c6b3097 100644 --- a/packages/package-requester/test/index.ts +++ b/packages/package-requester/test/index.ts @@ -1,12 +1,9 @@ /// import { getFilePathInCafs } from '@pnpm/cafs' -import localResolver from '@pnpm/local-resolver' +import createClient from '@pnpm/client' import { streamParser } from '@pnpm/logger' -import createResolver from '@pnpm/npm-resolver' import createPackageRequester, { PackageFilesResponse, PackageResponse } from '@pnpm/package-requester' import pkgIdToFilename from '@pnpm/pkgid-to-filename' -import { ResolveFunction } from '@pnpm/resolver-base' -import createFetcher from '@pnpm/tarball-fetcher' import { DependencyManifest } from '@pnpm/types' import delay from 'delay' import fs = require('mz/fs') @@ -23,24 +20,18 @@ const registry = 'https://registry.npmjs.org/' const IS_POSTIVE_TARBALL = path.join(__dirname, 'is-positive-1.0.0.tgz') const ncp = promisify(ncpCB as any) // tslint:disable-line:no-any -const rawConfig = { registry } +const authConfig = { registry } -const resolve = createResolver({ +const { resolve, fetchers } = createClient({ + authConfig, metaCache: new Map(), - rawConfig, storeDir: '.store', -}) as ResolveFunction -const fetch = createFetcher({ - alwaysAuth: false, - rawConfig, - registry: 'https://registry.npmjs.org/', - strictSsl: false, }) test('request package', async t => { const storeDir = tempy.directory() t.comment(storeDir) - const requestPackage = createPackageRequester(resolve, fetch, { + const requestPackage = createPackageRequester(resolve, fetchers, { networkConcurrency: 1, storeDir, verifyStoreIntegrity: true, @@ -82,7 +73,7 @@ test('request package', async t => { }) test('request package but skip fetching', async t => { - const requestPackage = createPackageRequester(resolve, fetch, { + const requestPackage = createPackageRequester(resolve, fetchers, { networkConcurrency: 1, storeDir: '.store', verifyStoreIntegrity: true, @@ -120,7 +111,7 @@ test('request package but skip fetching', async t => { }) test('request package but skip fetching, when resolution is already available', async t => { - const requestPackage = createPackageRequester(resolve, fetch, { + const requestPackage = createPackageRequester(resolve, fetchers, { networkConcurrency: 1, storeDir: '.store', verifyStoreIntegrity: true, @@ -192,7 +183,7 @@ test('refetch local tarball if its integrity has changed', async t => { } { - const requestPackage = createPackageRequester(localResolver as ResolveFunction, fetch, { + const requestPackage = createPackageRequester(resolve, fetchers, { storeDir, verifyStoreIntegrity: true, }) @@ -219,7 +210,7 @@ test('refetch local tarball if its integrity has changed', async t => { await delay(50) { - const requestPackage = createPackageRequester(localResolver as ResolveFunction, fetch, { + const requestPackage = createPackageRequester(resolve, fetchers, { storeDir, verifyStoreIntegrity: true, }) @@ -240,7 +231,7 @@ test('refetch local tarball if its integrity has changed', async t => { } { - const requestPackage = createPackageRequester(localResolver as ResolveFunction, fetch, { + const requestPackage = createPackageRequester(resolve, fetchers, { storeDir, verifyStoreIntegrity: true, }) @@ -283,7 +274,7 @@ test('refetch local tarball if its integrity has changed. The requester does not } { - const requestPackage = createPackageRequester(localResolver as ResolveFunction, fetch, { + const requestPackage = createPackageRequester(resolve, fetchers, { storeDir, verifyStoreIntegrity: true, }) @@ -304,7 +295,7 @@ test('refetch local tarball if its integrity has changed. The requester does not await delay(50) { - const requestPackage = createPackageRequester(localResolver as ResolveFunction, fetch, { + const requestPackage = createPackageRequester(resolve, fetchers, { storeDir, verifyStoreIntegrity: true, }) @@ -322,7 +313,7 @@ test('refetch local tarball if its integrity has changed. The requester does not } { - const requestPackage = createPackageRequester(localResolver as ResolveFunction, fetch, { + const requestPackage = createPackageRequester(resolve, fetchers, { storeDir, verifyStoreIntegrity: true, }) @@ -342,7 +333,7 @@ test('refetch local tarball if its integrity has changed. The requester does not }) test('fetchPackageToStore()', async (t) => { - const packageRequester = createPackageRequester(resolve, fetch, { + const packageRequester = createPackageRequester(resolve, fetchers, { networkConcurrency: 1, storeDir: tempy.directory(), verifyStoreIntegrity: true, @@ -401,7 +392,7 @@ test('fetchPackageToStore()', async (t) => { test('fetchPackageToStore() concurrency check', async (t) => { const storeDir = tempy.directory() const cafsDir = path.join(storeDir, 'files') - const packageRequester = createPackageRequester(resolve, fetch, { + const packageRequester = createPackageRequester(resolve, fetchers, { networkConcurrency: 1, storeDir, verifyStoreIntegrity: true, @@ -480,15 +471,14 @@ test('fetchPackageToStore() does not cache errors', async (t) => { .get('/is-positive/-/is-positive-1.0.0.tgz') .replyWithFile(200, IS_POSTIVE_TARBALL) - const noRetryFetch = createFetcher({ - alwaysAuth: false, - fetchRetries: 0, - rawConfig, - registry: 'https://registry.npmjs.org/', - strictSsl: false, + const noRetry = createClient({ + authConfig, + metaCache: new Map(), + retry: { retries: 0 }, + storeDir: '.pnpm', }) - const packageRequester = createPackageRequester(resolve, noRetryFetch, { + const packageRequester = createPackageRequester(noRetry.resolve, noRetry.fetchers, { networkConcurrency: 1, storeDir: tempy.directory(), verifyStoreIntegrity: true, @@ -539,7 +529,7 @@ test('fetchPackageToStore() does not cache errors', async (t) => { // This test was added to cover the issue described here: https://github.com/pnpm/supi/issues/65 test('always return a package manifest in the response', async t => { nock.cleanAll() - const requestPackage = createPackageRequester(resolve, fetch, { + const requestPackage = createPackageRequester(resolve, fetchers, { networkConcurrency: 1, storeDir: tempy.directory(), verifyStoreIntegrity: true, @@ -597,7 +587,7 @@ test('fetchPackageToStore() fetch raw manifest of cached package', async (t) => .get('/is-positive/-/is-positive-1.0.0.tgz') .replyWithFile(200, IS_POSTIVE_TARBALL) - const packageRequester = createPackageRequester(resolve, fetch, { + const packageRequester = createPackageRequester(resolve, fetchers, { networkConcurrency: 1, storeDir: tempy.directory(), verifyStoreIntegrity: true, @@ -644,7 +634,7 @@ test('refetch package to store if it has been modified', async (t) => { let indexJsFile!: string { - const packageRequester = createPackageRequester(resolve, fetch, { + const packageRequester = createPackageRequester(resolve, fetchers, { networkConcurrency: 1, storeDir, verifyStoreIntegrity: true, @@ -669,7 +659,7 @@ test('refetch package to store if it has been modified', async (t) => { streamParser.on('data', reporter) { - const packageRequester = createPackageRequester(resolve, fetch, { + const packageRequester = createPackageRequester(resolve, fetchers, { networkConcurrency: 1, storeDir, verifyStoreIntegrity: true, diff --git a/packages/package-requester/tsconfig.json b/packages/package-requester/tsconfig.json index 421d230940..cb475dec10 100644 --- a/packages/package-requester/tsconfig.json +++ b/packages/package-requester/tsconfig.json @@ -31,13 +31,7 @@ "path": "../types" }, { - "path": "../local-resolver" - }, - { - "path": "../npm-resolver" - }, - { - "path": "../tarball-fetcher" + "path": "../client" } ] } diff --git a/packages/package-store/package.json b/packages/package-store/package.json index 4cbdd83a6f..11b4828edd 100644 --- a/packages/package-store/package.json +++ b/packages/package-store/package.json @@ -37,10 +37,9 @@ "write-json-file": "4.0.0" }, "devDependencies": { + "@pnpm/client": "workspace:^0.0.0", "@pnpm/logger": "3.2.2", - "@pnpm/npm-resolver": "workspace:*", "@pnpm/package-store": "link:", - "@pnpm/tarball-fetcher": "workspace:*", "@types/mz": "^2.7.1", "@types/proxyquire": "^1.3.28", "@types/ramda": "^0.27.6", diff --git a/packages/package-store/test/index.ts b/packages/package-store/test/index.ts index 4758e999d8..edd160ab6e 100644 --- a/packages/package-store/test/index.ts +++ b/packages/package-store/test/index.ts @@ -1,8 +1,6 @@ /// -import createResolver from '@pnpm/npm-resolver' +import createClient from '@pnpm/client' import createStore from '@pnpm/package-store' -import { ResolveFunction } from '@pnpm/resolver-base' -import createFetcher from '@pnpm/tarball-fetcher' import path = require('path') import test = require('tape') import tempy = require('tempy') @@ -11,17 +9,13 @@ import './createImportPackage.spec' test('store.importPackage()', async (t) => { const storeDir = tempy.directory() const registry = 'https://registry.npmjs.org/' - const rawConfig = { registry } - const resolver = createResolver({ + const authConfig = { registry } + const { resolve, fetchers } = createClient({ + authConfig, metaCache: new Map(), - rawConfig, storeDir, - }) as ResolveFunction - const fetcher = createFetcher({ - rawConfig, - registry, }) - const storeController = await createStore(resolver, fetcher, { + const storeController = await createStore(resolve, fetchers, { storeDir, verifyStoreIntegrity: true, }) @@ -48,17 +42,13 @@ test('store.importPackage()', async (t) => { test('store.importPackage() by copying', async (t) => { const storeDir = tempy.directory() const registry = 'https://registry.npmjs.org/' - const rawConfig = { registry } - const resolver = createResolver({ + const authConfig = { registry } + const { resolve, fetchers } = createClient({ + authConfig, metaCache: new Map(), - rawConfig, storeDir, - }) as ResolveFunction - const fetcher = createFetcher({ - rawConfig, - registry, }) - const storeController = await createStore(resolver, fetcher, { + const storeController = await createStore(resolve, fetchers, { packageImportMethod: 'copy', storeDir, verifyStoreIntegrity: true, diff --git a/packages/package-store/tsconfig.json b/packages/package-store/tsconfig.json index 691709e96f..f3ea52dd0a 100644 --- a/packages/package-store/tsconfig.json +++ b/packages/package-store/tsconfig.json @@ -31,10 +31,7 @@ "path": "../types" }, { - "path": "../npm-resolver" - }, - { - "path": "../tarball-fetcher" + "path": "../client" } ] } diff --git a/packages/plugin-commands-outdated/src/outdated.ts b/packages/plugin-commands-outdated/src/outdated.ts index 4bfc476e3f..35d3e9ef5b 100644 --- a/packages/plugin-commands-outdated/src/outdated.ts +++ b/packages/plugin-commands-outdated/src/outdated.ts @@ -133,7 +133,6 @@ export type OutdatedCommandOptions = { | 'fetchRetryMaxtimeout' | 'fetchRetryMintimeout' | 'global' - | 'httpsProxy' | 'key' | 'localAddress' | 'lockfileDir' diff --git a/packages/plugin-commands-publishing/package.json b/packages/plugin-commands-publishing/package.json index 5ae7f2dd76..c0774ba9f6 100644 --- a/packages/plugin-commands-publishing/package.json +++ b/packages/plugin-commands-publishing/package.json @@ -55,6 +55,7 @@ "@pnpm/cli-utils": "workspace:0.4.11", "@pnpm/config": "workspace:10.0.1", "@pnpm/error": "workspace:1.2.0", + "@pnpm/fetch": "workspace:^2.0.2", "@pnpm/lifecycle": "workspace:9.1.3", "@pnpm/npm-resolver": "workspace:8.1.2", "@pnpm/pick-registry-for-package": "workspace:1.0.3", @@ -66,9 +67,11 @@ "@pnpm/types": "workspace:6.2.0", "@zkochan/rimraf": "1.0.0", "cp-file": "^9.0.0", + "credentials-by-uri": "^2.0.0", "enquirer": "^2.3.5", "fast-glob": "^3.2.4", "lru-cache": "^5.1.1", + "mem": "^6.1.0", "mz": "^2.7.0", "p-filter": "^2.1.0", "ramda": "^0.27.0", diff --git a/packages/plugin-commands-publishing/src/recursivePublish.ts b/packages/plugin-commands-publishing/src/recursivePublish.ts index 8d384da9be..95d80ac3f7 100644 --- a/packages/plugin-commands-publishing/src/recursivePublish.ts +++ b/packages/plugin-commands-publishing/src/recursivePublish.ts @@ -1,4 +1,5 @@ import { Config } from '@pnpm/config' +import { createFetchFromRegistry } from '@pnpm/fetch' import createResolver from '@pnpm/npm-resolver' import pickRegistryForPackage from '@pnpm/pick-registry-for-package' import { ResolveFunction } from '@pnpm/resolver-base' @@ -6,7 +7,9 @@ import runNpm from '@pnpm/run-npm' import sortPackages from '@pnpm/sort-packages' import storePath from '@pnpm/store-path' import { Registries } from '@pnpm/types' +import getCredentialsByURI = require('credentials-by-uri') import LRU = require('lru-cache') +import mem = require('mem') import pFilter = require('p-filter') import { handler as publish } from './publish' @@ -26,7 +29,6 @@ Partial wsPkg.package) const storeDir = await storePath(opts.workspaceDir, opts.storeDir) - const resolve = createResolver(Object.assign(opts, { - fullMetadata: false, + const fetch = createFetchFromRegistry(opts) + const getCredentials = mem((registry: string) => getCredentialsByURI(opts.rawConfig, registry)) + const resolve = createResolver(fetch, getCredentials, Object.assign(opts, { metaCache: new LRU({ max: 10000, maxAge: 120 * 1000, // 2 minutes diff --git a/packages/plugin-commands-publishing/tsconfig.json b/packages/plugin-commands-publishing/tsconfig.json index 4f9f45fb9d..cdfaa7c3a5 100644 --- a/packages/plugin-commands-publishing/tsconfig.json +++ b/packages/plugin-commands-publishing/tsconfig.json @@ -18,6 +18,9 @@ { "path": "../error" }, + { + "path": "../fetch" + }, { "path": "../lifecycle" }, diff --git a/packages/server/package.json b/packages/server/package.json index 3205126813..46900604d6 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -33,12 +33,11 @@ "@pnpm/logger": "^3.1.0" }, "devDependencies": { + "@pnpm/client": "workspace:^0.0.0", "@pnpm/logger": "3.2.2", - "@pnpm/npm-resolver": "workspace:*", "@pnpm/package-requester": "workspace:*", "@pnpm/package-store": "workspace:*", "@pnpm/server": "link:", - "@pnpm/tarball-fetcher": "workspace:*", "@types/mz": "^2.7.1", "@types/node-fetch": "^2.5.7", "@types/uuid": "^8.0.0", diff --git a/packages/server/test/index.ts b/packages/server/test/index.ts index 961b1b015b..a98805f987 100644 --- a/packages/server/test/index.ts +++ b/packages/server/test/index.ts @@ -1,9 +1,8 @@ /// -import createResolver, { PackageMetaCache } from '@pnpm/npm-resolver' +import createClient from '@pnpm/client' import createStore from '@pnpm/package-store' import { connectStoreController, createServer } from '@pnpm/server' -import { PackageFilesResponse, ResolveFunction } from '@pnpm/store-controller-types' -import createFetcher from '@pnpm/tarball-fetcher' +import { PackageFilesResponse } from '@pnpm/store-controller-types' import rimraf = require('@zkochan/rimraf') import isPortReachable = require('is-port-reachable') import loadJsonFile = require('load-json-file') @@ -19,17 +18,11 @@ async function createStoreController (storeDir?: string) { if (!storeDir) { storeDir = tempy.directory() } - const rawConfig = { registry } - const resolve = createResolver({ - metaCache: new Map() as PackageMetaCache, - rawConfig, + const authConfig = { registry } + const { resolve, fetchers } = createClient({ + authConfig, + metaCache: new Map(), storeDir, - }) as ResolveFunction - const fetchers = createFetcher({ - alwaysAuth: true, - rawConfig, - registry, - strictSsl: true, }) return createStore(resolve, fetchers, { networkConcurrency: 1, diff --git a/packages/server/tsconfig.json b/packages/server/tsconfig.json index 1dc0ca6d72..fc28c72809 100644 --- a/packages/server/tsconfig.json +++ b/packages/server/tsconfig.json @@ -19,16 +19,13 @@ "path": "../types" }, { - "path": "../npm-resolver" + "path": "../client" }, { "path": "../package-requester" }, { "path": "../package-store" - }, - { - "path": "../tarball-fetcher" } ] } diff --git a/packages/store-connection-manager/package.json b/packages/store-connection-manager/package.json index 0f6ed6cd86..246b84d9a2 100644 --- a/packages/store-connection-manager/package.json +++ b/packages/store-connection-manager/package.json @@ -36,9 +36,8 @@ }, "dependencies": { "@pnpm/cli-meta": "workspace:1.0.0", + "@pnpm/client": "workspace:^0.0.0", "@pnpm/config": "workspace:10.0.1", - "@pnpm/default-fetcher": "workspace:6.0.9", - "@pnpm/default-resolver": "workspace:9.0.3", "@pnpm/error": "workspace:1.2.0", "@pnpm/package-store": "workspace:9.0.11", "@pnpm/server": "workspace:8.0.4", diff --git a/packages/store-connection-manager/src/createNewStoreController.ts b/packages/store-connection-manager/src/createNewStoreController.ts index bb04e67080..c5d1dda33b 100644 --- a/packages/store-connection-manager/src/createNewStoreController.ts +++ b/packages/store-connection-manager/src/createNewStoreController.ts @@ -1,15 +1,34 @@ +import createClient from '@pnpm/client' import { Config } from '@pnpm/config' -import createFetcher from '@pnpm/default-fetcher' import createStore from '@pnpm/package-store' +import LRU = require('lru-cache') import fs = require('mz/fs') import path = require('path') -import createResolver, { CreateResolverOptions } from './createResolver' + +type CreateResolverOptions = Pick & Required> export type CreateNewStoreControllerOptions = CreateResolverOptions & Pick & { ignoreFile?: (filename: string) => boolean, @@ -18,21 +37,40 @@ export type CreateNewStoreControllerOptions = CreateResolverOptions & Pick { - const sopts = Object.assign(opts, { - registry: opts.registry || 'https://registry.npmjs.org/', + const { resolve, fetchers } = createClient({ + authConfig: opts.rawConfig, + ca: opts.ca, + cert: opts.cert, + fullMetadata: false, + key: opts.key, + localAddress: opts.localAddress, + metaCache: new LRU({ + max: 10000, + maxAge: 120 * 1000, // 2 minutes + }) as any, // tslint:disable-line:no-any + offline: opts.offline, + preferOffline: opts.preferOffline, + proxy: opts.proxy, + retry: { + factor: opts.fetchRetryFactor, + maxTimeout: opts.fetchRetryMaxtimeout, + minTimeout: opts.fetchRetryMintimeout, + retries: opts.fetchRetries, + }, + storeDir: opts.storeDir, + strictSSL: opts.strictSsl ?? true, + userAgent: opts.userAgent, }) - const resolve = createResolver(sopts) - await fs.mkdir(sopts.storeDir, { recursive: true }) - const fetchers = createFetcher(sopts) + await fs.mkdir(opts.storeDir, { recursive: true }) return { ctrl: await createStore(resolve, fetchers, { - ignoreFile: sopts.ignoreFile, - networkConcurrency: sopts.networkConcurrency, - packageImportMethod: sopts.packageImportMethod, - storeDir: sopts.storeDir, - verifyStoreIntegrity: typeof sopts.verifyStoreIntegrity === 'boolean' ? - sopts.verifyStoreIntegrity : true, + ignoreFile: opts.ignoreFile, + networkConcurrency: opts.networkConcurrency, + packageImportMethod: opts.packageImportMethod, + storeDir: opts.storeDir, + verifyStoreIntegrity: typeof opts.verifyStoreIntegrity === 'boolean' ? + opts.verifyStoreIntegrity : true, }), - dir: sopts.storeDir, + dir: opts.storeDir, } } diff --git a/packages/store-connection-manager/src/createResolver.ts b/packages/store-connection-manager/src/createResolver.ts deleted file mode 100644 index d4229688f7..0000000000 --- a/packages/store-connection-manager/src/createResolver.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { Config } from '@pnpm/config' -import createResolver from '@pnpm/default-resolver' -import LRU = require('lru-cache') - -export type CreateResolverOptions = Pick & Required> - -export default function ( - opts: CreateResolverOptions -) { - return createResolver(Object.assign(opts, { - fullMetadata: false, - metaCache: new LRU({ - max: 10000, - maxAge: 120 * 1000, // 2 minutes - }) as any, // tslint:disable-line:no-any - })) -} diff --git a/packages/store-connection-manager/tsconfig.json b/packages/store-connection-manager/tsconfig.json index 4786cbc6fc..4a17922c86 100644 --- a/packages/store-connection-manager/tsconfig.json +++ b/packages/store-connection-manager/tsconfig.json @@ -12,15 +12,12 @@ { "path": "../cli-meta" }, + { + "path": "../client" + }, { "path": "../config" }, - { - "path": "../default-fetcher" - }, - { - "path": "../default-resolver" - }, { "path": "../error" }, diff --git a/packages/supi/package.json b/packages/supi/package.json index 381352a918..f64d2eed64 100644 --- a/packages/supi/package.json +++ b/packages/supi/package.json @@ -72,8 +72,7 @@ "devDependencies": { "@pnpm/assert-project": "workspace:*", "@pnpm/assert-store": "workspace:*", - "@pnpm/default-fetcher": "workspace:*", - "@pnpm/default-resolver": "workspace:*", + "@pnpm/client": "workspace:^0.0.0", "@pnpm/logger": "3.2.2", "@pnpm/package-store": "workspace:*", "@pnpm/prepare": "workspace:0.0.7", diff --git a/packages/supi/test/install/auth.ts b/packages/supi/test/install/auth.ts index f99591e333..e90d63cf0a 100644 --- a/packages/supi/test/install/auth.ts +++ b/packages/supi/test/install/auth.ts @@ -25,14 +25,14 @@ test('a package that need authentication', async (t: tape.Test) => { }, (err: Error, d: { token: string }) => err ? reject(err) : resolve(d)) }) as {token: string} - let rawConfig = { + let authConfig = { [`//localhost:${REGISTRY_MOCK_PORT}/:_authToken`]: data.token, 'registry': `http://localhost:${REGISTRY_MOCK_PORT}/`, } const manifest = await addDependenciesToPackage({}, ['needs-auth'], await testDefaults({}, { - rawConfig, + authConfig, }, { - rawConfig, + authConfig, })) await project.has('needs-auth') @@ -42,15 +42,15 @@ test('a package that need authentication', async (t: tape.Test) => { await rimraf('node_modules') await rimraf(path.join('..', '.store')) - rawConfig = { + authConfig = { [`//localhost:${REGISTRY_MOCK_PORT}/:_authToken`]: data.token, 'registry': 'https://registry.npmjs.org/', } await addDependenciesToPackage(manifest, ['needs-auth'], await testDefaults({}, { - rawConfig, + authConfig, registry: 'https://registry.npmjs.org/', }, { - rawConfig, + authConfig, })) await project.has('needs-auth') @@ -72,15 +72,15 @@ test('installing a package that need authentication, using password', async (t: }) as {token: string} const encodedPassword = Buffer.from('bar').toString('base64') - let rawConfig = { + let authConfig = { [`//localhost:${REGISTRY_MOCK_PORT}/:_password`]: encodedPassword, [`//localhost:${REGISTRY_MOCK_PORT}/:username`]: 'foo', 'registry': `http://localhost:${REGISTRY_MOCK_PORT}/`, } await addDependenciesToPackage({}, ['needs-auth'], await testDefaults({}, { - rawConfig, + authConfig, }, { - rawConfig, + authConfig, })) await project.has('needs-auth') @@ -101,15 +101,15 @@ test('a package that need authentication, legacy way', async (t: tape.Test) => { }, (err: Error, d: object) => err ? reject(err) : resolve(d)) }) - const rawConfig = { + const authConfig = { '_auth': 'Zm9vOmJhcg==', // base64 encoded foo:bar 'always-auth': true, 'registry': `http://localhost:${REGISTRY_MOCK_PORT}`, } await addDependenciesToPackage({}, ['needs-auth'], await testDefaults({}, { - rawConfig, + authConfig, }, { - rawConfig, + authConfig, })) await project.has('needs-auth') @@ -130,16 +130,16 @@ test('a scoped package that need authentication specific to scope', async (t: ta }, (err: Error, d: { token: string }) => err ? reject(err) : resolve(d)) }) as {token: string} - const rawConfig = { + const authConfig = { [`//localhost:${REGISTRY_MOCK_PORT}/:_authToken`]: data.token, '@private:registry': `http://localhost:${REGISTRY_MOCK_PORT}/`, 'registry': 'https://registry.npmjs.org/', } let opts = await testDefaults({}, { - rawConfig, + authConfig, registry: 'https://registry.npmjs.org/', }, { - rawConfig, + authConfig, }) const manifest = await addDependenciesToPackage({}, ['@private/foo'], opts) @@ -151,10 +151,10 @@ test('a scoped package that need authentication specific to scope', async (t: ta // Recreating options to have a new storeController with clean cache opts = await testDefaults({}, { - rawConfig, + authConfig, registry: 'https://registry.npmjs.org/', }, { - rawConfig, + authConfig, }) await addDependenciesToPackage(manifest, ['@private/foo'], opts) @@ -176,17 +176,17 @@ test('a scoped package that need legacy authentication specific to scope', async }, (err: Error, d: { token: string }) => err ? reject(err) : resolve(d)) }) - const rawConfig = { + const authConfig = { [`//localhost:${REGISTRY_MOCK_PORT}/:_auth`]: 'Zm9vOmJhcg==', // base64 encoded foo:bar [`//localhost:${REGISTRY_MOCK_PORT}/:always-auth`]: true, '@private:registry': `http://localhost:${REGISTRY_MOCK_PORT}/`, 'registry': 'https://registry.npmjs.org/', } let opts = await testDefaults({}, { - rawConfig, + authConfig, registry: 'https://registry.npmjs.org/', }, { - rawConfig, + authConfig, }) const manifest = await addDependenciesToPackage({}, ['@private/foo'], opts) @@ -198,10 +198,10 @@ test('a scoped package that need legacy authentication specific to scope', async // Recreating options to have a new storeController with clean cache opts = await testDefaults({}, { - rawConfig, + authConfig, registry: 'https://registry.npmjs.org/', }, { - rawConfig, + authConfig, }) await addDependenciesToPackage(manifest, ['@private/foo'], opts) @@ -223,7 +223,7 @@ test('a package that need authentication reuses authorization tokens for tarball }, (err: Error, d: { token: string }) => err ? reject(err) : resolve(d)) }) as {token: string} - const rawConfig = { + const authConfig = { [`//127.0.0.1:${REGISTRY_MOCK_PORT}/:_authToken`]: data.token, [`//127.0.0.1:${REGISTRY_MOCK_PORT}/:always-auth`]: true, 'registry': `http://127.0.0.1:${REGISTRY_MOCK_PORT}`, @@ -233,10 +233,10 @@ test('a package that need authentication reuses authorization tokens for tarball default: `http://127.0.0.1:${REGISTRY_MOCK_PORT}`, }, }, { - rawConfig, + authConfig, registry: `http://127.0.0.1:${REGISTRY_MOCK_PORT}`, }, { - rawConfig, + authConfig, })) await project.has('needs-auth') @@ -257,7 +257,7 @@ test('a package that need authentication reuses authorization tokens for tarball }, (err: Error, d: { token: string }) => err ? reject(err) : resolve(d)) }) as {token: string} - const rawConfig = { + const authConfig = { [`//127.0.0.1:${REGISTRY_MOCK_PORT}/:_authToken`]: data.token, [`//127.0.0.1:${REGISTRY_MOCK_PORT}/:always-auth`]: true, 'registry': `http://127.0.0.1:${REGISTRY_MOCK_PORT}`, @@ -267,10 +267,10 @@ test('a package that need authentication reuses authorization tokens for tarball default: `http://127.0.0.1:${REGISTRY_MOCK_PORT}`, }, }, { - rawConfig, + authConfig, registry: `http://127.0.0.1:${REGISTRY_MOCK_PORT}`, }, { - rawConfig, + authConfig, }) const manifest = await addDependenciesToPackage({}, ['needs-auth'], opts) @@ -285,10 +285,10 @@ test('a package that need authentication reuses authorization tokens for tarball default: `http://127.0.0.1:${REGISTRY_MOCK_PORT}`, }, }, { - rawConfig, + authConfig, registry: `http://127.0.0.1:${REGISTRY_MOCK_PORT}`, }, { - rawConfig, + authConfig, }) await install(manifest, opts) diff --git a/packages/supi/test/utils/testDefaults.ts b/packages/supi/test/utils/testDefaults.ts index 3dc8b90701..bab67cfb4e 100644 --- a/packages/supi/test/utils/testDefaults.ts +++ b/packages/supi/test/utils/testDefaults.ts @@ -1,5 +1,4 @@ -import createFetcher from '@pnpm/default-fetcher' -import createResolver from '@pnpm/default-resolver' +import createClient from '@pnpm/client' import createStore from '@pnpm/package-store' import { REGISTRY_MOCK_PORT } from '@pnpm/registry-mock' import { StoreController } from '@pnpm/store-controller-types' @@ -11,10 +10,10 @@ import { InstallOptions } from 'supi' const registry = `http://localhost:${REGISTRY_MOCK_PORT}/` const retryOpts = { - fetchRetries: 4, - fetchRetryFactor: 10, - fetchRetryMaxtimeout: 60_000, - fetchRetryMintimeout: 10_000, + retries: 4, + retryFactor: 10, + retryMaxtimeout: 60_000, + retryMintimeout: 10_000, } export default async function testDefaults ( @@ -37,24 +36,18 @@ export default async function testDefaults ( > { let storeDir = opts && opts.storeDir || path.resolve('.store') storeDir = await storePath(opts && opts.prefix || process.cwd(), storeDir) - const rawConfig = { registry } + const authConfig = { registry } + const { resolve, fetchers } = createClient({ + authConfig, + metaCache: new Map(), + retry: retryOpts, + storeDir, + ...resolveOpts, + ...fetchOpts, + }) const storeController = await createStore( - createResolver({ - fullMetadata: false, - metaCache: new Map(), - rawConfig, - storeDir, - strictSsl: true, - ...retryOpts, - ...resolveOpts, - }), - createFetcher({ - alwaysAuth: true, - rawConfig, - registry, - ...retryOpts, - ...fetchOpts, - }), + resolve, + fetchers, { ignoreFile: opts?.fastUnpack === false ? undefined : (filename) => filename !== 'package.json', storeDir, diff --git a/packages/supi/tsconfig.json b/packages/supi/tsconfig.json index 9e234b2a95..5491ce788b 100644 --- a/packages/supi/tsconfig.json +++ b/packages/supi/tsconfig.json @@ -106,10 +106,7 @@ "path": "../../privatePackages/assert-store" }, { - "path": "../default-fetcher" - }, - { - "path": "../default-resolver" + "path": "../client" }, { "path": "../package-store" diff --git a/packages/tarball-fetcher/package.json b/packages/tarball-fetcher/package.json index 2412ae4415..4b2795cdd1 100644 --- a/packages/tarball-fetcher/package.json +++ b/packages/tarball-fetcher/package.json @@ -37,20 +37,16 @@ "@pnpm/core-loggers": "workspace:^4.1.2", "@pnpm/error": "workspace:1.2.0", "@pnpm/fetcher-base": "workspace:8.0.2", + "@pnpm/fetching-types": "workspace:^0.0.0", "@zkochan/retry": "^0.2.0", - "credentials-by-uri": "^2.0.0", - "fetch-from-npm-registry": "workspace:4.1.2", "graceful-fs": "^4.2.4", - "mem": "^6.1.0", "mz": "^2.7.0", - "path-temp": "^2.0.0", - "rimraf": "^3.0.2", "ssri": "6.0.1" }, "devDependencies": { "@pnpm/cafs": "workspace:1.0.5", + "@pnpm/fetch": "workspace:^2.0.2", "@pnpm/logger": "3.2.2", - "@pnpm/read-package-json": "workspace:3.1.3", "@pnpm/tarball-fetcher": "link:", "@types/graceful-fs": "^4.1.3", "@types/mz": "^2.7.1", diff --git a/packages/tarball-fetcher/src/createDownloader.ts b/packages/tarball-fetcher/src/createDownloader.ts index e0c706b884..b2753904f9 100644 --- a/packages/tarball-fetcher/src/createDownloader.ts +++ b/packages/tarball-fetcher/src/createDownloader.ts @@ -6,13 +6,9 @@ import { FetchResult, FilesIndex, } from '@pnpm/fetcher-base' +import { FetchFromRegistry } from '@pnpm/fetching-types' import * as retry from '@zkochan/retry' -import createFetcher from 'fetch-from-npm-registry' import { IncomingMessage } from 'http' -import fs = require('mz/fs') -import path = require('path') -import pathTemp = require('path-temp') -import rimraf = require('rimraf') import ssri = require('ssri') import urlLib = require('url') import { BadTarballError } from './errorTypes' @@ -76,17 +72,8 @@ export interface NpmRegistryClient { } export default ( + fetchFromRegistry: FetchFromRegistry, gotOpts: { - alwaysAuth: boolean, - registry: string, - // proxy - proxy?: string, - localAddress?: string, - // ssl - ca?: string, - cert?: string, - key?: string, - strictSSL?: boolean, // retry retry?: { retries?: number, @@ -95,26 +82,8 @@ export default ( maxTimeout?: number, randomize?: boolean, }, - userAgent?: string, } ): DownloadFunction => { - const fetchFromNpmRegistry = createFetcher({ - ca: gotOpts.ca, - cert: gotOpts.cert, - key: gotOpts.key, - localAddress: gotOpts.localAddress, - proxy: gotOpts.proxy, - strictSSL: gotOpts.strictSSL, - userAgent: gotOpts.userAgent, - - // The fetch library can retry requests on bad HTTP responses. - // However, it is not enough to retry on bad HTTP responses only. - // Requests should also be retried when the tarball's integrity check fails. - // Hence, we tell fetch to not retry, - // and we perform the retries from this function instead. - retry: { retries: 0 }, - }) - const retryOpts = { factor: 10, maxTimeout: 6e4, // 1 minute @@ -172,8 +141,14 @@ export default ( async function fetch (currentAttempt: number): Promise { try { - const res = await fetchFromNpmRegistry(url, { + const res = await fetchFromRegistry(url, { authHeaderValue: shouldAuth ? opts.auth?.authHeaderValue : undefined, + // The fetch library can retry requests on bad HTTP responses. + // However, it is not enough to retry on bad HTTP responses only. + // Requests should also be retried when the tarball's integrity check fails. + // Hence, we tell fetch to not retry, + // and we perform the retries from this function instead. + retry: { retries: 0 }, }) if (res.status !== 200) { diff --git a/packages/tarball-fetcher/src/index.ts b/packages/tarball-fetcher/src/index.ts index c473ea820b..7a387d8bba 100644 --- a/packages/tarball-fetcher/src/index.ts +++ b/packages/tarball-fetcher/src/index.ts @@ -6,59 +6,31 @@ import { FetchOptions, FetchResult, } from '@pnpm/fetcher-base' -import getCredentialsByURI = require('credentials-by-uri') -import mem = require('mem') +import { + FetchFromRegistry, + GetCredentials, + RetryTimeoutOptions, +} from '@pnpm/fetching-types' import fs = require('mz/fs') import path = require('path') import ssri = require('ssri') import createDownloader, { DownloadFunction } from './createDownloader' export default function ( + fetchFromRegistry: FetchFromRegistry, + getCredentials: GetCredentials, opts: { - registry: string, - rawConfig: object, - alwaysAuth?: boolean, - proxy?: string, - httpsProxy?: string, - localAddress?: string, - cert?: string, - key?: string, - ca?: string, - strictSsl?: boolean, - fetchRetries?: number, - fetchRetryFactor?: number, - fetchRetryMintimeout?: number, - fetchRetryMaxtimeout?: number, - userAgent?: string, + retry?: RetryTimeoutOptions, offline?: boolean, } ): { tarball: FetchFunction } { - const download = createDownloader({ - alwaysAuth: opts.alwaysAuth || false, - ca: opts.ca, - cert: opts.cert, - key: opts.key, - localAddress: opts.localAddress, - proxy: opts.httpsProxy || opts.proxy, - registry: opts.registry, - retry: { - factor: opts.fetchRetryFactor, - maxTimeout: opts.fetchRetryMaxtimeout, - minTimeout: opts.fetchRetryMintimeout, - retries: opts.fetchRetries, - }, - // TODO: cover with tests this option - // https://github.com/pnpm/pnpm/issues/1062 - strictSSL: typeof opts.strictSsl === 'boolean' - ? opts.strictSsl - : true, - userAgent: opts.userAgent, + const download = createDownloader(fetchFromRegistry, { + retry: opts.retry, }) - const getCreds = getCredentialsByURI.bind(null, opts.rawConfig) return { tarball: fetchFromTarball.bind(null, { download, - getCredentialsByURI: mem((registry: string) => getCreds(registry)), + getCredentialsByURI: getCredentials, offline: opts.offline, }) as FetchFunction, } diff --git a/packages/tarball-fetcher/test/download.ts b/packages/tarball-fetcher/test/download.ts index 1a45523f48..4a61cf1e8d 100644 --- a/packages/tarball-fetcher/test/download.ts +++ b/packages/tarball-fetcher/test/download.ts @@ -1,10 +1,8 @@ /// -import createCafs, { getFilePathByModeInCafs as _getFilePathByModeInCafs } from '@pnpm/cafs' -import { LogBase, streamParser } from '@pnpm/logger' -import readPackage from '@pnpm/read-package-json' +import createCafs from '@pnpm/cafs' +import { createFetchFromRegistry } from '@pnpm/fetch' import createFetcher from '@pnpm/tarball-fetcher' import cpFile = require('cp-file') -import { existsSync } from 'fs' import fs = require('mz/fs') import nock = require('nock') import path = require('path') @@ -15,20 +13,19 @@ import tempy = require('tempy') const cafsDir = tempy.directory() console.log(cafsDir) const cafs = createCafs(cafsDir) -const getFilePathByModeInCafs = _getFilePathByModeInCafs.bind(_getFilePathByModeInCafs, cafsDir) const tarballPath = path.join(__dirname, 'tars', 'babel-helper-hoist-variables-6.24.1.tgz') const tarballSize = 1279 const tarballIntegrity = 'sha1-HssnaJydJVE+rbyZFKc/VAi+enY=' const registry = 'http://example.com/' -const fetch = createFetcher({ - fetchRetries: 1, - fetchRetryMaxtimeout: 100, - fetchRetryMintimeout: 0, - rawConfig: { - registry, +const fetchFromRegistry = createFetchFromRegistry({}) +const getCredentials = () => ({ authHeaderValue: undefined, alwaysAuth: undefined }) +const fetch = createFetcher(fetchFromRegistry, getCredentials, { + retry: { + maxTimeout: 100, + minTimeout: 0, + retries: 1, }, - registry, }) test('fail when tarball size does not match content-length', async t => { @@ -227,15 +224,13 @@ test("don't fail when fetching a local tarball in offline mode", async (t) => { tarball: `file:${tarballAbsoluteLocation}`, } - const fetch = createFetcher({ - fetchRetries: 1, - fetchRetryMaxtimeout: 100, - fetchRetryMintimeout: 0, + const fetch = createFetcher(fetchFromRegistry, getCredentials, { offline: true, - rawConfig: { - registry, + retry: { + maxTimeout: 100, + minTimeout: 0, + retries: 1, }, - registry, }) const { filesIndex } = await fetch.tarball(cafs, resolution, { lockfileDir: process.cwd(), @@ -258,15 +253,13 @@ test('fail when trying to fetch a non-local tarball in offline mode', async (t) let err!: Error try { - const fetch = createFetcher({ - fetchRetries: 1, - fetchRetryMaxtimeout: 100, - fetchRetryMintimeout: 0, + const fetch = createFetcher(fetchFromRegistry, getCredentials, { offline: true, - rawConfig: { - registry, + retry: { + maxTimeout: 100, + minTimeout: 0, + retries: 1, }, - registry, }) await fetch.tarball(cafs, resolution, { lockfileDir: process.cwd(), @@ -359,16 +352,16 @@ test('accessing private packages', async t => { process.chdir(tempy.directory()) t.comment(`testing in ${process.cwd()}`) - const fetch = createFetcher({ - alwaysAuth: true, - fetchRetries: 1, - fetchRetryMaxtimeout: 100, - fetchRetryMintimeout: 0, - rawConfig: { - '//example.com/:_authToken': 'ofjergrg349gj3f2', - registry, + const getCredentials = () => ({ + alwaysAuth: undefined, + authHeaderValue: 'Bearer ofjergrg349gj3f2', + }) + const fetch = createFetcher(fetchFromRegistry, getCredentials, { + retry: { + maxTimeout: 100, + minTimeout: 0, + retries: 1, }, - registry, }) const resolution = { diff --git a/packages/tarball-fetcher/tsconfig.json b/packages/tarball-fetcher/tsconfig.json index 6c1761a08e..fc952b382d 100644 --- a/packages/tarball-fetcher/tsconfig.json +++ b/packages/tarball-fetcher/tsconfig.json @@ -19,13 +19,13 @@ "path": "../fetcher-base" }, { - "path": "../fetch-from-npm-registry" + "path": "../fetching-types" }, { "path": "../cafs" }, { - "path": "../read-package-json" + "path": "../fetch" } ] } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b1a1d3109f..d8fab54217 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -176,6 +176,30 @@ importers: '@types/ramda': ^0.27.6 chalk: ^4.1.0 load-json-file: ^6.2.0 + packages/client: + dependencies: + '@pnpm/default-resolver': 'link:../default-resolver' + '@pnpm/fetch': 'link:../fetch' + '@pnpm/fetching-types': 'link:../fetching-types' + '@pnpm/git-fetcher': 'link:../git-fetcher' + '@pnpm/resolver-base': 'link:../resolver-base' + '@pnpm/tarball-fetcher': 'link:../tarball-fetcher' + credentials-by-uri: 2.0.0 + mem: 6.1.0 + devDependencies: + '@pnpm/client': 'link:' + '@pnpm/logger': 3.2.2 + specifiers: + '@pnpm/client': 'link:' + '@pnpm/default-resolver': 'workspace:^9.0.3' + '@pnpm/fetch': 'workspace:^2.0.2' + '@pnpm/fetching-types': 'workspace:^0.0.0' + '@pnpm/git-fetcher': 'workspace:3.0.4' + '@pnpm/logger': 3.2.2 + '@pnpm/resolver-base': 'workspace:7.0.3' + '@pnpm/tarball-fetcher': 'workspace:7.1.4' + credentials-by-uri: ^2.0.0 + mem: ^6.1.0 packages/command: specifiers: {} packages/common-cli-options-help: @@ -224,20 +248,6 @@ importers: specifiers: '@pnpm/logger': 3.2.2 '@pnpm/types': 'workspace:^6.2.0' - packages/default-fetcher: - dependencies: - '@pnpm/git-fetcher': 'link:../git-fetcher' - '@pnpm/resolver-base': 'link:../resolver-base' - '@pnpm/tarball-fetcher': 'link:../tarball-fetcher' - devDependencies: - '@pnpm/default-fetcher': 'link:' - '@pnpm/logger': 3.2.2 - specifiers: - '@pnpm/default-fetcher': 'link:' - '@pnpm/git-fetcher': 'workspace:3.0.4' - '@pnpm/logger': 3.2.2 - '@pnpm/resolver-base': 'workspace:7.0.3' - '@pnpm/tarball-fetcher': 'workspace:7.1.4' packages/default-reporter: dependencies: '@pnpm/config': 'link:../config' @@ -301,6 +311,7 @@ importers: packages/default-resolver: dependencies: '@pnpm/error': 'link:../error' + '@pnpm/fetching-types': 'link:../fetching-types' '@pnpm/git-resolver': 'link:../git-resolver' '@pnpm/local-resolver': 'link:../local-resolver' '@pnpm/npm-resolver': 'link:../npm-resolver' @@ -308,10 +319,13 @@ importers: '@pnpm/tarball-resolver': 'link:../tarball-resolver' devDependencies: '@pnpm/default-resolver': 'link:' + '@pnpm/fetch': 'link:../fetch' '@pnpm/logger': 3.2.2 specifiers: '@pnpm/default-resolver': 'link:' '@pnpm/error': 'workspace:1.2.0' + '@pnpm/fetch': 'workspace:^2.0.2' + '@pnpm/fetching-types': 'workspace:^0.0.0' '@pnpm/git-resolver': 'workspace:4.0.15' '@pnpm/local-resolver': 'workspace:5.0.10' '@pnpm/logger': 3.2.2 @@ -373,33 +387,29 @@ importers: packages/fetch: dependencies: '@pnpm/core-loggers': 'link:../core-loggers' + '@pnpm/fetching-types': 'link:../fetching-types' + '@pnpm/npm-registry-agent': 'link:../npm-registry-agent' '@zkochan/retry': 0.2.0 node-fetch: 2.6.0 node-fetch-unix: 2.3.0 devDependencies: + '@pnpm/fetch': 'link:' '@pnpm/logger': 3.2.2 '@types/node-fetch': 2.5.7 cpy-cli: 3.1.1 + nock: 13.0.0 specifiers: '@pnpm/core-loggers': 'workspace:^4.1.2' + '@pnpm/fetch': 'link:' + '@pnpm/fetching-types': 'workspace:^0.0.0' '@pnpm/logger': ^3.2.2 + '@pnpm/npm-registry-agent': 'workspace:^2.0.3' '@types/node-fetch': ^2.5.7 '@zkochan/retry': ^0.2.0 cpy-cli: ^3.1.1 + nock: ^13.0.0 node-fetch: 2.6.0 node-fetch-unix: 2.3.0 - packages/fetch-from-npm-registry: - dependencies: - '@pnpm/fetch': 'link:../fetch' - '@pnpm/npm-registry-agent': 'link:../npm-registry-agent' - devDependencies: - fetch-from-npm-registry: 'link:' - nock: 12.0.3 - specifiers: - '@pnpm/fetch': 'workspace:2.0.2' - '@pnpm/npm-registry-agent': 'workspace:2.0.3' - fetch-from-npm-registry: 'link:' - nock: 12.0.3 packages/fetcher-base: dependencies: '@pnpm/resolver-base': 'link:../resolver-base' @@ -409,6 +419,13 @@ importers: '@pnpm/resolver-base': 'workspace:7.0.3' '@pnpm/types': 'workspace:6.2.0' '@types/ssri': ^6.0.3 + packages/fetching-types: + dependencies: + '@types/node-fetch': 2.5.7 + '@zkochan/retry': 0.2.0 + specifiers: + '@types/node-fetch': ^2.5.7 + '@zkochan/retry': ^0.2.0 packages/filter-lockfile: dependencies: '@pnpm/constants': 'link:../constants' @@ -643,8 +660,7 @@ importers: realpath-missing: 1.0.0 devDependencies: '@pnpm/assert-project': 'link:../../privatePackages/assert-project' - '@pnpm/default-fetcher': 'link:../default-fetcher' - '@pnpm/default-resolver': 'link:../default-resolver' + '@pnpm/client': 'link:../client' '@pnpm/headless': 'link:' '@pnpm/logger': 3.2.2 '@pnpm/package-store': 'link:../package-store' @@ -662,7 +678,6 @@ importers: load-json-file: 6.2.0 mz: 2.7.0 npm-run-all: 4.1.5 - read-yaml-file: 2.0.0 sinon: 9.0.2 tape-promise: 4.0.0 tempy: 0.5.0 @@ -670,10 +685,9 @@ importers: specifiers: '@pnpm/assert-project': 'workspace:*' '@pnpm/build-modules': 'workspace:5.0.6' + '@pnpm/client': 'workspace:^0.0.0' '@pnpm/constants': 'workspace:4.0.0' '@pnpm/core-loggers': 'workspace:4.1.2' - '@pnpm/default-fetcher': 'workspace:*' - '@pnpm/default-resolver': 'workspace:*' '@pnpm/error': 'workspace:1.2.0' '@pnpm/filter-lockfile': 'workspace:4.0.4' '@pnpm/headless': 'link:' @@ -712,7 +726,6 @@ importers: path-absolute: 1.0.1 path-exists: 4.0.0 ramda: 0.27.0 - read-yaml-file: 2.0.0 realpath-missing: 1.0.0 sinon: 9.0.2 tape-promise: 4.0.0 @@ -1091,14 +1104,12 @@ importers: packages/npm-resolver: dependencies: '@pnpm/error': 'link:../error' + '@pnpm/fetching-types': 'link:../fetching-types' '@pnpm/resolve-workspace-range': 'link:../resolve-workspace-range' '@pnpm/resolver-base': 'link:../resolver-base' '@pnpm/types': 'link:../types' - credentials-by-uri: 2.0.0 encode-registry: 2.0.2 - fetch-from-npm-registry: 'link:../fetch-from-npm-registry' load-json-file: 6.2.0 - mem: 6.1.0 mz: 2.7.0 normalize-path: 3.0.0 p-limit: 3.0.1 @@ -1110,6 +1121,7 @@ importers: ssri: 6.0.1 version-selector-type: 3.0.0 devDependencies: + '@pnpm/fetch': 'link:../fetch' '@pnpm/logger': 3.2.2 '@pnpm/npm-resolver': 'link:' '@types/mz': 2.7.1 @@ -1121,6 +1133,8 @@ importers: tempy: 0.5.0 specifiers: '@pnpm/error': 'workspace:1.2.0' + '@pnpm/fetch': 'workspace:^2.0.2' + '@pnpm/fetching-types': 'workspace:^0.0.0' '@pnpm/logger': 3.2.2 '@pnpm/npm-resolver': 'link:' '@pnpm/resolve-workspace-range': 'workspace:1.0.1' @@ -1130,11 +1144,8 @@ importers: '@types/normalize-path': ^3.0.0 '@types/semver': ^7.2.0 '@types/ssri': ^6.0.3 - credentials-by-uri: 2.0.0 encode-registry: 2.0.2 - fetch-from-npm-registry: 'workspace:4.1.2' load-json-file: 6.2.0 - mem: ^6.1.0 mz: ^2.7.0 nock: 12.0.3 normalize-path: 3.0.0 @@ -1153,6 +1164,7 @@ importers: '@pnpm/constants': 'link:../constants' '@pnpm/default-resolver': 'link:../default-resolver' '@pnpm/error': 'link:../error' + '@pnpm/fetch': 'link:../fetch' '@pnpm/lockfile-file': 'link:../lockfile-file' '@pnpm/lockfile-utils': 'link:../lockfile-utils' '@pnpm/manifest-utils': 'link:../manifest-utils' @@ -1161,8 +1173,10 @@ importers: '@pnpm/pick-registry-for-package': 'link:../pick-registry-for-package' '@pnpm/store-path': 4.0.0 '@pnpm/types': 'link:../types' + credentials-by-uri: 2.0.0 dependency-path: 'link:../dependency-path' lru-cache: 5.1.1 + mem: 6.1.0 ramda: 0.27.0 semver: 7.3.2 devDependencies: @@ -1176,6 +1190,7 @@ importers: '@pnpm/constants': 'workspace:4.0.0' '@pnpm/default-resolver': 'workspace:9.0.3' '@pnpm/error': 'workspace:1.2.0' + '@pnpm/fetch': 'workspace:^2.0.2' '@pnpm/lockfile-file': 'workspace:3.0.11' '@pnpm/lockfile-utils': 'workspace:2.0.15' '@pnpm/logger': 3.2.2 @@ -1189,8 +1204,10 @@ importers: '@types/lru-cache': ^5.1.0 '@types/ramda': ^0.27.6 '@types/semver': ^7.2.0 + credentials-by-uri: ^2.0.0 dependency-path: 'workspace:5.0.2' lru-cache: ^5.1.1 + mem: ^6.1.0 npm-run-all: 4.1.5 ramda: ^0.27.0 semver: ^7.3.2 @@ -1253,11 +1270,9 @@ importers: rename-overwrite: 3.0.0 ssri: 6.0.1 devDependencies: - '@pnpm/local-resolver': 'link:../local-resolver' + '@pnpm/client': 'link:../client' '@pnpm/logger': 3.2.2 - '@pnpm/npm-resolver': 'link:../npm-resolver' '@pnpm/package-requester': 'link:' - '@pnpm/tarball-fetcher': 'link:../tarball-fetcher' '@types/mz': 2.7.1 '@types/ncp': 2.0.4 '@types/normalize-path': 3.0.0 @@ -1272,17 +1287,15 @@ importers: tempy: 0.5.0 specifiers: '@pnpm/cafs': 'workspace:1.0.5' + '@pnpm/client': 'workspace:*' '@pnpm/core-loggers': 'workspace:4.1.2' '@pnpm/fetcher-base': 'workspace:8.0.2' - '@pnpm/local-resolver': 'workspace:*' '@pnpm/logger': 3.2.2 - '@pnpm/npm-resolver': 'workspace:*' '@pnpm/package-requester': 'link:' '@pnpm/pkgid-to-filename': 3.0.0 '@pnpm/read-package-json': 'workspace:3.1.3' '@pnpm/resolver-base': 'workspace:7.0.3' '@pnpm/store-controller-types': 'workspace:8.0.2' - '@pnpm/tarball-fetcher': 'workspace:*' '@pnpm/types': 'workspace:6.2.0' '@types/mz': ^2.7.1 '@types/ncp': ^2.0.4 @@ -1328,10 +1341,9 @@ importers: ssri: 8.0.0 write-json-file: 4.0.0 devDependencies: + '@pnpm/client': 'link:../client' '@pnpm/logger': 3.2.2 - '@pnpm/npm-resolver': 'link:../npm-resolver' '@pnpm/package-store': 'link:' - '@pnpm/tarball-fetcher': 'link:../tarball-fetcher' '@types/mz': 2.7.1 '@types/proxyquire': 1.3.28 '@types/ramda': 0.27.6 @@ -1342,15 +1354,14 @@ importers: tempy: 0.5.0 specifiers: '@pnpm/cafs': 'workspace:1.0.5' + '@pnpm/client': 'workspace:^0.0.0' '@pnpm/core-loggers': 'workspace:4.1.2' '@pnpm/fetcher-base': 'workspace:8.0.2' '@pnpm/logger': 3.2.2 - '@pnpm/npm-resolver': 'workspace:*' '@pnpm/package-requester': 'workspace:12.0.6' '@pnpm/package-store': 'link:' '@pnpm/resolver-base': 'workspace:7.0.3' '@pnpm/store-controller-types': 'workspace:8.0.2' - '@pnpm/tarball-fetcher': 'workspace:*' '@pnpm/types': 'workspace:6.2.0' '@types/mz': ^2.7.1 '@types/proxyquire': ^1.3.28 @@ -1725,6 +1736,7 @@ importers: '@pnpm/cli-utils': 'link:../cli-utils' '@pnpm/config': 'link:../config' '@pnpm/error': 'link:../error' + '@pnpm/fetch': 'link:../fetch' '@pnpm/lifecycle': 'link:../lifecycle' '@pnpm/npm-resolver': 'link:../npm-resolver' '@pnpm/pick-registry-for-package': 'link:../pick-registry-for-package' @@ -1736,9 +1748,11 @@ importers: '@pnpm/types': 'link:../types' '@zkochan/rimraf': 1.0.0 cp-file: 9.0.0 + credentials-by-uri: 2.0.0 enquirer: 2.3.5 fast-glob: 3.2.4 lru-cache: 5.1.1 + mem: 6.1.0 mz: 2.7.0 p-filter: 2.1.0 ramda: 0.27.0 @@ -1766,6 +1780,7 @@ importers: '@pnpm/cli-utils': 'workspace:0.4.11' '@pnpm/config': 'workspace:10.0.1' '@pnpm/error': 'workspace:1.2.0' + '@pnpm/fetch': 'workspace:^2.0.2' '@pnpm/filter-workspace-packages': 'workspace:2.1.4' '@pnpm/lifecycle': 'workspace:9.1.3' '@pnpm/npm-resolver': 'workspace:8.1.2' @@ -1786,11 +1801,13 @@ importers: '@types/sinon': ^9.0.4 '@zkochan/rimraf': 1.0.0 cp-file: ^9.0.0 + credentials-by-uri: ^2.0.0 cross-spawn: ^7.0.3 enquirer: ^2.3.5 execa: ^4.0.2 fast-glob: ^3.2.4 lru-cache: ^5.1.1 + mem: ^6.1.0 mz: ^2.7.0 p-filter: ^2.1.0 path-exists: 4.0.0 @@ -2464,12 +2481,11 @@ importers: promise-share: 1.0.0 uuid: 3.4.0 devDependencies: + '@pnpm/client': 'link:../client' '@pnpm/logger': 3.2.2 - '@pnpm/npm-resolver': 'link:../npm-resolver' '@pnpm/package-requester': 'link:../package-requester' '@pnpm/package-store': 'link:../package-store' '@pnpm/server': 'link:' - '@pnpm/tarball-fetcher': 'link:../tarball-fetcher' '@types/mz': 2.7.1 '@types/node-fetch': 2.5.7 '@types/uuid': 8.0.0 @@ -2480,14 +2496,13 @@ importers: node-fetch: 2.6.0 tempy: 0.5.0 specifiers: + '@pnpm/client': 'workspace:^0.0.0' '@pnpm/fetch': 'workspace:2.0.2' '@pnpm/logger': 3.2.2 - '@pnpm/npm-resolver': 'workspace:*' '@pnpm/package-requester': 'workspace:*' '@pnpm/package-store': 'workspace:*' '@pnpm/server': 'link:' '@pnpm/store-controller-types': 'workspace:8.0.2' - '@pnpm/tarball-fetcher': 'workspace:*' '@pnpm/types': 'workspace:6.2.0' '@types/mz': ^2.7.1 '@types/node-fetch': ^2.5.7 @@ -2514,9 +2529,8 @@ importers: packages/store-connection-manager: dependencies: '@pnpm/cli-meta': 'link:../cli-meta' + '@pnpm/client': 'link:../client' '@pnpm/config': 'link:../config' - '@pnpm/default-fetcher': 'link:../default-fetcher' - '@pnpm/default-resolver': 'link:../default-resolver' '@pnpm/error': 'link:../error' '@pnpm/package-store': 'link:../package-store' '@pnpm/server': 'link:../server' @@ -2533,9 +2547,8 @@ importers: '@types/mz': 2.7.1 specifiers: '@pnpm/cli-meta': 'workspace:1.0.0' + '@pnpm/client': 'workspace:^0.0.0' '@pnpm/config': 'workspace:10.0.1' - '@pnpm/default-fetcher': 'workspace:6.0.9' - '@pnpm/default-resolver': 'workspace:9.0.3' '@pnpm/error': 'workspace:1.2.0' '@pnpm/logger': 3.2.2 '@pnpm/package-store': 'workspace:9.0.11' @@ -2613,8 +2626,7 @@ importers: devDependencies: '@pnpm/assert-project': 'link:../../privatePackages/assert-project' '@pnpm/assert-store': 'link:../../privatePackages/assert-store' - '@pnpm/default-fetcher': 'link:../default-fetcher' - '@pnpm/default-resolver': 'link:../default-resolver' + '@pnpm/client': 'link:../client' '@pnpm/logger': 3.2.2 '@pnpm/package-store': 'link:../package-store' '@pnpm/prepare': 'link:../../privatePackages/prepare' @@ -2656,10 +2668,9 @@ importers: '@pnpm/assert-project': 'workspace:*' '@pnpm/assert-store': 'workspace:*' '@pnpm/build-modules': 'workspace:5.0.6' + '@pnpm/client': 'workspace:^0.0.0' '@pnpm/constants': 'workspace:4.0.0' '@pnpm/core-loggers': 'workspace:4.1.2' - '@pnpm/default-fetcher': 'workspace:*' - '@pnpm/default-resolver': 'workspace:*' '@pnpm/error': 'workspace:1.2.0' '@pnpm/filter-lockfile': 'workspace:4.0.4' '@pnpm/get-context': 'workspace:3.0.1' @@ -2767,19 +2778,15 @@ importers: '@pnpm/core-loggers': 'link:../core-loggers' '@pnpm/error': 'link:../error' '@pnpm/fetcher-base': 'link:../fetcher-base' + '@pnpm/fetching-types': 'link:../fetching-types' '@zkochan/retry': 0.2.0 - credentials-by-uri: 2.0.0 - fetch-from-npm-registry: 'link:../fetch-from-npm-registry' graceful-fs: 4.2.4 - mem: 6.1.0 mz: 2.7.0 - path-temp: 2.0.0 - rimraf: 3.0.2 ssri: 6.0.1 devDependencies: '@pnpm/cafs': 'link:../cafs' + '@pnpm/fetch': 'link:../fetch' '@pnpm/logger': 3.2.2 - '@pnpm/read-package-json': 'link:../read-package-json' '@pnpm/tarball-fetcher': 'link:' '@types/graceful-fs': 4.1.3 '@types/mz': 2.7.1 @@ -2793,9 +2800,10 @@ importers: '@pnpm/cafs': 'workspace:1.0.5' '@pnpm/core-loggers': 'workspace:^4.1.2' '@pnpm/error': 'workspace:1.2.0' + '@pnpm/fetch': 'workspace:^2.0.2' '@pnpm/fetcher-base': 'workspace:8.0.2' + '@pnpm/fetching-types': 'workspace:^0.0.0' '@pnpm/logger': 3.2.2 - '@pnpm/read-package-json': 'workspace:3.1.3' '@pnpm/tarball-fetcher': 'link:' '@types/graceful-fs': ^4.1.3 '@types/mz': ^2.7.1 @@ -2804,14 +2812,9 @@ importers: '@types/ssri': ^6.0.3 '@zkochan/retry': ^0.2.0 cp-file: 9.0.0 - credentials-by-uri: ^2.0.0 - fetch-from-npm-registry: 'workspace:4.1.2' graceful-fs: ^4.2.4 - mem: ^6.1.0 mz: ^2.7.0 nock: 12.0.3 - path-temp: ^2.0.0 - rimraf: ^3.0.2 ssri: 6.0.1 tempy: 0.5.0 packages/tarball-resolver: @@ -3638,9 +3641,8 @@ packages: integrity: sha512-erpimpT1pH8QfeNg77ypnjwz6CGMqrnL4DewVbqFzD9FXzSULjmG3KzjZnLNe7bzTSZm2W9DpkHyqop1g1KmgQ== /@types/node-fetch/2.5.7: dependencies: - '@types/node': 14.0.13 + '@types/node': 14.0.14 form-data: 3.0.0 - dev: true resolution: integrity: sha512-o2WVNf5UhWRkxlf6eq+jMZDu7kjgpgJfl4xVNlvryc95O/6F2ld8ztKX+qu+Rjyet93WAWm5LjeX9H5FGkODvw== /@types/node/12.12.47: @@ -3654,6 +3656,9 @@ packages: /@types/node/14.0.13: resolution: integrity: sha512-rouEWBImiRaSJsVA+ITTFM6ZxibuAlTuNOCyxVbwreu6k6+ujs7DfnU9o+PShFhET78pMBl3eH+AGSI5eOTkPA== + /@types/node/14.0.14: + resolution: + integrity: sha512-syUgf67ZQpaJj01/tRTknkMNoBBLWJOBODF0Zm4NrXmiSuxjymFrxnTu1QVYRubhVkRcZLYZG8STTwJRdVm/WQ== /@types/nopt/3.0.29: resolution: integrity: sha1-8Z3z20yX7hRZonQAKDIKcdcJZM4= @@ -6641,7 +6646,6 @@ packages: asynckit: 0.4.0 combined-stream: 1.0.8 mime-types: 2.1.27 - dev: true engines: node: '>= 6' resolution: @@ -8397,6 +8401,10 @@ packages: dev: true resolution: integrity: sha1-DdOXEhPHxW34gJd9UEyI+0cal6w= + /lodash.set/4.3.2: + dev: true + resolution: + integrity: sha1-2HV7HagH3eJIFrDWqEvqGnYjCyM= /lodash.sortby/4.7.0: dev: true resolution: @@ -9048,6 +9056,17 @@ packages: node: '>= 10.13' resolution: integrity: sha512-QNb/j8kbFnKCiyqi9C5DD0jH/FubFGj5rt9NQFONXwQm3IPB0CULECg/eS3AU1KgZb/6SwUa4/DTRKhVxkGABw== + /nock/13.0.0: + dependencies: + debug: 4.1.1 + json-stringify-safe: 5.0.1 + lodash.set: 4.3.2 + propagate: 2.0.1 + dev: true + engines: + node: '>= 10.13' + resolution: + integrity: sha512-FiW8t91Je5yG5MVT1r+go1Z9bX3rCYIEjenUYeZrEl2v8aTWdIX336itrmQaKUO8Ske5Z7RHR7OIzr/9p0Ujjg== /node-fetch-unix/2.3.0: dev: false engines: