diff --git a/.changeset/eleven-swans-switch.md b/.changeset/eleven-swans-switch.md new file mode 100644 index 0000000000..0541adf15d --- /dev/null +++ b/.changeset/eleven-swans-switch.md @@ -0,0 +1,5 @@ +--- +"@pnpm/node.fetcher": major +--- + +Initial release. diff --git a/packages/node.fetcher/README.md b/packages/node.fetcher/README.md new file mode 100644 index 0000000000..a20e12d2a1 --- /dev/null +++ b/packages/node.fetcher/README.md @@ -0,0 +1,17 @@ +# @pnpm/node.fetcher + +> Node.js artifacts fetcher + +Downloads and extracts the right Node.js artifact for the active platform and architecture. + +[![npm version](https://img.shields.io/npm/v/@pnpm/node.fetcher.svg)](https://www.npmjs.com/package/@pnpm/node.fetcher) + +## Installation + +```sh +pnpm add @pnpm/node.fetcher +``` + +## License + +MIT diff --git a/packages/node.fetcher/jest.config.js b/packages/node.fetcher/jest.config.js new file mode 100644 index 0000000000..95f86a2128 --- /dev/null +++ b/packages/node.fetcher/jest.config.js @@ -0,0 +1,3 @@ +const config = require('../../jest.config.js') + +module.exports = config diff --git a/packages/node.fetcher/package.json b/packages/node.fetcher/package.json new file mode 100644 index 0000000000..bb0c0fcfa8 --- /dev/null +++ b/packages/node.fetcher/package.json @@ -0,0 +1,52 @@ +{ + "name": "@pnpm/node.fetcher", + "version": "0.0.0", + "description": "Node.js artifacts fetcher", + "funding": "https://opencollective.com/pnpm", + "main": "lib/index.js", + "types": "lib/index.d.ts", + "files": [ + "lib", + "!*.map" + ], + "exports": { + ".": "./lib/index.js" + }, + "engines": { + "node": ">=14.6" + }, + "scripts": { + "lint": "eslint src/**/*.ts test/**/*.ts", + "_test": "jest", + "test": "pnpm run compile && pnpm run _test", + "prepublishOnly": "pnpm run compile", + "compile": "tsc --build && pnpm run lint --fix" + }, + "repository": "https://github.com/pnpm/pnpm/blob/main/packages/node.fetcher", + "keywords": [ + "pnpm7", + "pnpm", + "env", + "node.js" + ], + "license": "MIT", + "bugs": { + "url": "https://github.com/pnpm/pnpm/issues" + }, + "homepage": "https://github.com/pnpm/pnpm/blob/main/packages/node.fetcher#readme", + "dependencies": { + "@pnpm/create-cafs-store": "workspace:1.1.0", + "@pnpm/fetching-types": "workspace:3.0.0", + "@pnpm/fetcher-base": "workspace:12.1.0", + "@pnpm/tarball-fetcher": "workspace:10.0.5", + "adm-zip": "^0.5.5", + "rename-overwrite": "^4.0.2", + "tempy": "^1.0.0" + }, + "devDependencies": { + "@pnpm/node.fetcher": "workspace:0.0.0", + "@pnpm/prepare": "workspace:*", + "@types/adm-zip": "^0.4.34", + "node-fetch": "3.0.0-beta.9" + } +} diff --git a/packages/node.fetcher/src/getNodeTarball.ts b/packages/node.fetcher/src/getNodeTarball.ts new file mode 100644 index 0000000000..9ec31e6de3 --- /dev/null +++ b/packages/node.fetcher/src/getNodeTarball.ts @@ -0,0 +1,17 @@ +import normalizeArch from './normalizeArch' + +export function getNodeTarball ( + nodeVersion: string, + nodeMirror: string, + processPlatform: string, + processArch: string +) { + const platform = processPlatform === 'win32' ? 'win' : processPlatform + const arch = normalizeArch(processPlatform, processArch) + const extension = platform === 'win' ? 'zip' : 'tar.gz' + const pkgName = `node-v${nodeVersion}-${platform}-${arch}` + return { + pkgName, + tarball: `${nodeMirror}v${nodeVersion}/${pkgName}.${extension}`, + } +} diff --git a/packages/node.fetcher/src/index.ts b/packages/node.fetcher/src/index.ts new file mode 100644 index 0000000000..abf69b5f74 --- /dev/null +++ b/packages/node.fetcher/src/index.ts @@ -0,0 +1,63 @@ +import fs from 'fs' +import path from 'path' +import { + FetchFromRegistry, + RetryTimeoutOptions, +} from '@pnpm/fetching-types' +import { FilesIndex } from '@pnpm/fetcher-base' +import createCafsStore from '@pnpm/create-cafs-store' +import createFetcher, { waitForFilesIndex } from '@pnpm/tarball-fetcher' +import AdmZip from 'adm-zip' +import renameOverwrite from 'rename-overwrite' +import tempy from 'tempy' +import { getNodeTarball } from './getNodeTarball' + +export interface FetchNodeOptions { + cafsDir: string + fetchTimeout?: number + nodeMirrorBaseUrl: string + retry?: RetryTimeoutOptions +} + +export async function fetchNode (fetch: FetchFromRegistry, version: string, targetDir: string, opts: FetchNodeOptions) { + const { tarball, pkgName } = getNodeTarball(version, opts.nodeMirrorBaseUrl, process.platform, process.arch) + if (tarball.endsWith('.zip')) { + await downloadAndUnpackZip(fetch, tarball, targetDir, pkgName) + return + } + const getCredentials = () => ({ authHeaderValue: undefined, alwaysAuth: undefined }) + const { tarball: fetchTarball } = createFetcher(fetch, getCredentials, { + retry: opts.retry, + timeout: opts.fetchTimeout, + }) + const cafs = createCafsStore(opts.cafsDir) + const { filesIndex } = await fetchTarball(cafs, { tarball }, { + lockfileDir: process.cwd(), + }) + await cafs.importPackage(targetDir, { + filesResponse: { + filesIndex: await waitForFilesIndex(filesIndex as FilesIndex), + fromStore: false, + }, + force: true, + }) +} + +async function downloadAndUnpackZip ( + fetchFromRegistry: FetchFromRegistry, + zipUrl: string, + targetDir: string, + pkgName: string +) { + const response = await fetchFromRegistry(zipUrl) + const tmp = path.join(tempy.directory(), 'pnpm.zip') + const dest = fs.createWriteStream(tmp) + await new Promise((resolve, reject) => { + response.body!.pipe(dest).on('error', reject).on('close', resolve) + }) + const zip = new AdmZip(tmp) + const nodeDir = path.dirname(targetDir) + zip.extractAllTo(nodeDir, true) + await renameOverwrite(path.join(nodeDir, pkgName), targetDir) + await fs.promises.unlink(tmp) +} diff --git a/packages/plugin-commands-env/src/normalizeArch.ts b/packages/node.fetcher/src/normalizeArch.ts similarity index 100% rename from packages/plugin-commands-env/src/normalizeArch.ts rename to packages/node.fetcher/src/normalizeArch.ts diff --git a/packages/node.fetcher/test/getNodeTarball.test.ts b/packages/node.fetcher/test/getNodeTarball.test.ts new file mode 100644 index 0000000000..d8aeb67819 --- /dev/null +++ b/packages/node.fetcher/test/getNodeTarball.test.ts @@ -0,0 +1,36 @@ +import { getNodeTarball } from '../lib/getNodeTarball' + +test.each([ + [ + '16.0.0', + 'https://nodejs.org/download/release/', + 'win32', + 'ia32', + { + pkgName: 'node-v16.0.0-win-x86', + tarball: 'https://nodejs.org/download/release/v16.0.0/node-v16.0.0-win-x86.zip', + }, + ], + [ + '16.0.0', + 'https://nodejs.org/download/release/', + 'linux', + 'arm', + { + pkgName: 'node-v16.0.0-linux-armv7l', + tarball: 'https://nodejs.org/download/release/v16.0.0/node-v16.0.0-linux-armv7l.tar.gz', + }, + ], + [ + '16.0.0', + 'https://nodejs.org/download/release/', + 'linux', + 'x64', + { + pkgName: 'node-v16.0.0-linux-x64', + tarball: 'https://nodejs.org/download/release/v16.0.0/node-v16.0.0-linux-x64.tar.gz', + }, + ], +])('getNodeTarball', (version, nodeMirrorBaseUrl, platform, arch, tarball) => { + expect(getNodeTarball(version, nodeMirrorBaseUrl, platform, arch)).toStrictEqual(tarball) +}) diff --git a/packages/node.fetcher/test/node.test.ts b/packages/node.fetcher/test/node.test.ts new file mode 100644 index 0000000000..d8b739e6a8 --- /dev/null +++ b/packages/node.fetcher/test/node.test.ts @@ -0,0 +1,40 @@ +import AdmZip from 'adm-zip' +import { Response } from 'node-fetch' +import path from 'path' +import { Readable } from 'stream' +import { fetchNode, FetchNodeOptions } from '@pnpm/node.fetcher' +import { tempDir } from '@pnpm/prepare' + +const fetchMock = jest.fn(async (url: string) => { + if (url.endsWith('.zip')) { + // The Windows code path for pnpm's node bootstrapping expects a subdir + // within the .zip file. + const pkgName = path.basename(url, '.zip') + const zip = new AdmZip() + zip.addFile(`${pkgName}/dummy-file`, Buffer.from('test')) + + return new Response(Readable.from(zip.toBuffer())) + } + + return new Response(Readable.from(Buffer.alloc(0))) +}) + +beforeEach(() => { + fetchMock.mockClear() +}) + +test('install Node uses node-mirror:release option', async () => { + tempDir() + + const nodeMirrorBaseUrl = 'https://pnpm-node-mirror-test.localhost/download/release/' + const opts: FetchNodeOptions = { + nodeMirrorBaseUrl, + cafsDir: path.resolve('files'), + } + + await fetchNode(fetchMock, '16.4.0', path.resolve('node'), opts) + + for (const call of fetchMock.mock.calls) { + expect(call[0]).toMatch(nodeMirrorBaseUrl) + } +}) diff --git a/packages/node.fetcher/test/normalizeArch.test.ts b/packages/node.fetcher/test/normalizeArch.test.ts new file mode 100644 index 0000000000..71222c234b --- /dev/null +++ b/packages/node.fetcher/test/normalizeArch.test.ts @@ -0,0 +1,9 @@ +import normalizeArch from '../lib/normalizeArch' + +test.each([ + ['win32', 'ia32', 'x86'], + ['linux', 'arm', 'armv7l'], // Raspberry Pi 4 + ['linux', 'x64', 'x64'], +])('normalizedArch(%s, %s)', (platform, arch, normalizedArch) => { + expect(normalizeArch(platform, arch)).toBe(normalizedArch) +}) diff --git a/packages/node.fetcher/tsconfig.json b/packages/node.fetcher/tsconfig.json new file mode 100644 index 0000000000..a706e0b485 --- /dev/null +++ b/packages/node.fetcher/tsconfig.json @@ -0,0 +1,28 @@ +{ + "extends": "@pnpm/tsconfig", + "compilerOptions": { + "outDir": "lib", + "rootDir": "src" + }, + "include": [ + "src/**/*.ts", + "../../typings/**/*.d.ts" + ], + "references": [ + { + "path": "../../privatePackages/prepare" + }, + { + "path": "../create-cafs-store" + }, + { + "path": "../fetcher-base" + }, + { + "path": "../fetching-types" + }, + { + "path": "../tarball-fetcher" + } + ] +} diff --git a/packages/node.fetcher/tsconfig.lint.json b/packages/node.fetcher/tsconfig.lint.json new file mode 100644 index 0000000000..0dc5add6b7 --- /dev/null +++ b/packages/node.fetcher/tsconfig.lint.json @@ -0,0 +1,8 @@ +{ + "extends": "./tsconfig.json", + "include": [ + "src/**/*.ts", + "test/**/*.ts", + "../../typings/**/*.d.ts" + ] +} diff --git a/packages/plugin-commands-env/package.json b/packages/plugin-commands-env/package.json index 911781f369..74a58e9ffe 100644 --- a/packages/plugin-commands-env/package.json +++ b/packages/plugin-commands-env/package.json @@ -32,19 +32,15 @@ "dependencies": { "@pnpm/cli-utils": "workspace:0.7.13", "@pnpm/config": "workspace:15.3.0", - "@pnpm/create-cafs-store": "workspace:1.1.0", "@pnpm/error": "workspace:3.0.1", "@pnpm/fetch": "workspace:5.0.3", "@pnpm/fetcher-base": "workspace:12.1.0", + "@pnpm/node.fetcher": "workspace:0.0.0", "@pnpm/store-path": "workspace:6.0.0", - "@pnpm/tarball-fetcher": "workspace:10.0.5", "@zkochan/cmd-shim": "^5.2.2", - "adm-zip": "^0.5.5", "load-json-file": "^6.2.0", - "rename-overwrite": "^4.0.2", "render-help": "^1.0.1", "semver": "^7.3.4", - "tempy": "^1.0.0", "version-selector-type": "^3.0.0", "write-json-file": "^4.3.0" }, @@ -54,6 +50,7 @@ "@pnpm/prepare": "workspace:*", "@types/adm-zip": "^0.4.34", "@types/semver": "^7.3.4", + "adm-zip": "^0.5.5", "execa": "npm:safe-execa@^0.1.1", "nock": "12.0.3", "node-fetch": "3.0.0-beta.9", diff --git a/packages/plugin-commands-env/src/node.ts b/packages/plugin-commands-env/src/node.ts index 3e937e210f..3d6bb2ab59 100644 --- a/packages/plugin-commands-env/src/node.ts +++ b/packages/plugin-commands-env/src/node.ts @@ -2,16 +2,10 @@ import fs from 'fs' import path from 'path' import { Config } from '@pnpm/config' import { createFetchFromRegistry, FetchFromRegistry } from '@pnpm/fetch' -import { FilesIndex } from '@pnpm/fetcher-base' -import createCafsStore from '@pnpm/create-cafs-store' +import { fetchNode } from '@pnpm/node.fetcher' import storePath from '@pnpm/store-path' -import createFetcher, { waitForFilesIndex } from '@pnpm/tarball-fetcher' -import AdmZip from 'adm-zip' -import renameOverwrite from 'rename-overwrite' -import tempy from 'tempy' import loadJsonFile from 'load-json-file' import writeJsonFile from 'write-json-file' -import normalizeArch from './normalizeArch' import getNodeMirror from './getNodeMirror' import { parseNodeEditionSpecifier } from './parseNodeEditionSpecifier' @@ -71,76 +65,26 @@ export async function getNodeDir (fetch: FetchFromRegistry, opts: NvmNodeCommand await fs.promises.mkdir(nodesDir, { recursive: true }) const versionDir = path.join(nodesDir, opts.useNodeVersion) if (!fs.existsSync(versionDir)) { - await installNode(fetch, opts.useNodeVersion, versionDir, opts) + const storeDir = await storePath({ + pkgRoot: process.cwd(), + storePath: opts.storeDir, + pnpmHomeDir: opts.pnpmHomeDir, + }) + const cafsDir = path.join(storeDir, 'files') + await fetchNode(fetch, opts.useNodeVersion, versionDir, { + ...opts, + cafsDir, + retry: { + maxTimeout: opts.fetchRetryMaxtimeout, + minTimeout: opts.fetchRetryMintimeout, + retries: opts.fetchRetries, + factor: opts.fetchRetryFactor, + }, + }) } return versionDir } -async function installNode (fetch: FetchFromRegistry, wantedNodeVersion: string, versionDir: string, opts: NvmNodeCommandOptions & { nodeMirrorBaseUrl: string }) { - const { tarball, pkgName } = getNodeJSTarball(wantedNodeVersion, opts.nodeMirrorBaseUrl) - if (tarball.endsWith('.zip')) { - await downloadAndUnpackZip(fetch, tarball, versionDir, pkgName) - return - } - const getCredentials = () => ({ authHeaderValue: undefined, alwaysAuth: undefined }) - const { tarball: fetchTarball } = createFetcher(fetch, getCredentials, { - retry: { - maxTimeout: opts.fetchRetryMaxtimeout, - minTimeout: opts.fetchRetryMintimeout, - retries: opts.fetchRetries, - factor: opts.fetchRetryFactor, - }, - timeout: opts.fetchTimeout, - }) - const storeDir = await storePath({ - pkgRoot: process.cwd(), - storePath: opts.storeDir, - pnpmHomeDir: opts.pnpmHomeDir, - }) - const cafsDir = path.join(storeDir, 'files') - const cafs = createCafsStore(cafsDir) - const { filesIndex } = await fetchTarball(cafs, { tarball }, { - lockfileDir: process.cwd(), - }) - await cafs.importPackage(versionDir, { - filesResponse: { - filesIndex: await waitForFilesIndex(filesIndex as FilesIndex), - fromStore: false, - }, - force: true, - }) -} - -async function downloadAndUnpackZip ( - fetchFromRegistry: FetchFromRegistry, - zipUrl: string, - targetDir: string, - pkgName: string -) { - const response = await fetchFromRegistry(zipUrl) - const tmp = path.join(tempy.directory(), 'pnpm.zip') - const dest = fs.createWriteStream(tmp) - await new Promise((resolve, reject) => { - response.body!.pipe(dest).on('error', reject).on('close', resolve) - }) - const zip = new AdmZip(tmp) - const nodeDir = path.dirname(targetDir) - zip.extractAllTo(nodeDir, true) - await renameOverwrite(path.join(nodeDir, pkgName), targetDir) - await fs.promises.unlink(tmp) -} - -function getNodeJSTarball (nodeVersion: string, nodeMirror: string) { - const platform = process.platform === 'win32' ? 'win' : process.platform - const arch = normalizeArch(process.platform, process.arch) - const extension = platform === 'win' ? 'zip' : 'tar.gz' - const pkgName = `node-v${nodeVersion}-${platform}-${arch}` - return { - pkgName, - tarball: `${nodeMirror}v${nodeVersion}/${pkgName}.${extension}`, - } -} - async function readNodeVersionsManifest (nodesDir: string): Promise<{ default?: string }> { try { return await loadJsonFile<{ default?: string }>(path.join(nodesDir, 'versions.json')) diff --git a/packages/plugin-commands-env/tsconfig.json b/packages/plugin-commands-env/tsconfig.json index 8b114f95d9..c7fb3acaf4 100644 --- a/packages/plugin-commands-env/tsconfig.json +++ b/packages/plugin-commands-env/tsconfig.json @@ -18,9 +18,6 @@ { "path": "../config" }, - { - "path": "../create-cafs-store" - }, { "path": "../error" }, @@ -31,10 +28,10 @@ "path": "../fetcher-base" }, { - "path": "../store-path" + "path": "../node.fetcher" }, { - "path": "../tarball-fetcher" + "path": "../store-path" } ] } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 050c191824..50d85ade17 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1710,6 +1710,33 @@ importers: '@types/normalize-path': 3.0.0 rimraf: 3.0.2 + packages/node.fetcher: + specifiers: + '@pnpm/create-cafs-store': workspace:1.1.0 + '@pnpm/fetcher-base': workspace:12.1.0 + '@pnpm/fetching-types': workspace:3.0.0 + '@pnpm/node.fetcher': workspace:0.0.0 + '@pnpm/prepare': workspace:* + '@pnpm/tarball-fetcher': workspace:10.0.5 + '@types/adm-zip': ^0.4.34 + adm-zip: ^0.5.5 + node-fetch: 3.0.0-beta.9 + rename-overwrite: ^4.0.2 + tempy: ^1.0.0 + dependencies: + '@pnpm/create-cafs-store': link:../create-cafs-store + '@pnpm/fetcher-base': link:../fetcher-base + '@pnpm/fetching-types': link:../fetching-types + '@pnpm/tarball-fetcher': link:../tarball-fetcher + adm-zip: 0.5.9 + rename-overwrite: 4.0.2 + tempy: 1.0.1 + devDependencies: + '@pnpm/node.fetcher': 'link:' + '@pnpm/prepare': link:../../privatePackages/prepare + '@types/adm-zip': 0.4.34 + node-fetch: 3.0.0-beta.9 + packages/normalize-registries: specifiers: '@pnpm/normalize-registries': workspace:3.0.3 @@ -2103,14 +2130,13 @@ importers: specifiers: '@pnpm/cli-utils': workspace:0.7.13 '@pnpm/config': workspace:15.3.0 - '@pnpm/create-cafs-store': workspace:1.1.0 '@pnpm/error': workspace:3.0.1 '@pnpm/fetch': workspace:5.0.3 '@pnpm/fetcher-base': workspace:12.1.0 + '@pnpm/node.fetcher': workspace:0.0.0 '@pnpm/plugin-commands-env': workspace:2.1.12 '@pnpm/prepare': workspace:* '@pnpm/store-path': workspace:6.0.0 - '@pnpm/tarball-fetcher': workspace:10.0.5 '@types/adm-zip': ^0.4.34 '@types/semver': ^7.3.4 '@zkochan/cmd-shim': ^5.2.2 @@ -2120,28 +2146,22 @@ importers: nock: 12.0.3 node-fetch: 3.0.0-beta.9 path-name: ^1.0.0 - rename-overwrite: ^4.0.2 render-help: ^1.0.1 semver: ^7.3.4 - tempy: ^1.0.0 version-selector-type: ^3.0.0 write-json-file: ^4.3.0 dependencies: '@pnpm/cli-utils': link:../cli-utils '@pnpm/config': link:../config - '@pnpm/create-cafs-store': link:../create-cafs-store '@pnpm/error': link:../error '@pnpm/fetch': link:../fetch '@pnpm/fetcher-base': link:../fetcher-base + '@pnpm/node.fetcher': link:../node.fetcher '@pnpm/store-path': link:../store-path - '@pnpm/tarball-fetcher': link:../tarball-fetcher '@zkochan/cmd-shim': 5.2.2 - adm-zip: 0.5.9 load-json-file: 6.2.0 - rename-overwrite: 4.0.2 render-help: 1.0.2 semver: 7.3.7 - tempy: 1.0.1 version-selector-type: 3.0.0 write-json-file: 4.3.0 devDependencies: @@ -2149,6 +2169,7 @@ importers: '@pnpm/prepare': link:../../privatePackages/prepare '@types/adm-zip': 0.4.34 '@types/semver': 7.3.9 + adm-zip: 0.5.9 execa: /safe-execa/0.1.1 nock: 12.0.3 node-fetch: 3.0.0-beta.9 @@ -5441,7 +5462,7 @@ packages: /@types/adm-zip/0.4.34: resolution: {integrity: sha512-8ToYLLAYhkRfcmmljrKi22gT2pqu7hGMDtORP1emwIEGmgUTZOsaDjzWFzW5N2frcFRz/50CWt4zA1CxJ73pmQ==} dependencies: - '@types/node': 17.0.41 + '@types/node': 18.0.0 dev: true /@types/archy/0.0.31: @@ -6390,7 +6411,6 @@ packages: /adm-zip/0.5.9: resolution: {integrity: sha512-s+3fXLkeeLjZ2kLjCBwQufpI5fuN+kIGBxu6530nVQZGVol0d7Y/M88/xw9HGGUcJjKf8LutN3VPRUBq6N7Ajg==} engines: {node: '>=6.0'} - dev: false /agent-base/6.0.2: resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==}