refactor: create @pnpm/node.fetcher (#4908)

.changeset/eleven-swans-switch.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/node.fetcher": major
+---
+
+Initial release.

packages/node.fetcher/README.md

@@ -0,0 +1,17 @@
+# @pnpm/node.fetcher
+
+> Node.js artifacts fetcher
+
+Downloads and extracts the right Node.js artifact for the active platform and architecture.
+
+[![npm version](https://img.shields.io/npm/v/@pnpm/node.fetcher.svg)](https://www.npmjs.com/package/@pnpm/node.fetcher)
+
+## Installation
+
+```sh
+pnpm add @pnpm/node.fetcher
+```
+
+## License
+
+MIT

packages/node.fetcher/jest.config.js

@@ -0,0 +1,3 @@
+const config = require('../../jest.config.js')
+
+module.exports = config

packages/node.fetcher/package.json

@@ -0,0 +1,52 @@
+{
+  "name": "@pnpm/node.fetcher",
+  "version": "0.0.0",
+  "description": "Node.js artifacts fetcher",
+  "funding": "https://opencollective.com/pnpm",
+  "main": "lib/index.js",
+  "types": "lib/index.d.ts",
+  "files": [
+    "lib",
+    "!*.map"
+  ],
+  "exports": {
+    ".": "./lib/index.js"
+  },
+  "engines": {
+    "node": ">=14.6"
+  },
+  "scripts": {
+    "lint": "eslint src/**/*.ts test/**/*.ts",
+    "_test": "jest",
+    "test": "pnpm run compile && pnpm run _test",
+    "prepublishOnly": "pnpm run compile",
+    "compile": "tsc --build && pnpm run lint --fix"
+  },
+  "repository": "https://github.com/pnpm/pnpm/blob/main/packages/node.fetcher",
+  "keywords": [
+    "pnpm7",
+    "pnpm",
+    "env",
+    "node.js"
+  ],
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/pnpm/pnpm/issues"
+  },
+  "homepage": "https://github.com/pnpm/pnpm/blob/main/packages/node.fetcher#readme",
+  "dependencies": {
+    "@pnpm/create-cafs-store": "workspace:1.1.0",
+    "@pnpm/fetching-types": "workspace:3.0.0",
+    "@pnpm/fetcher-base": "workspace:12.1.0",
+    "@pnpm/tarball-fetcher": "workspace:10.0.5",
+    "adm-zip": "^0.5.5",
+    "rename-overwrite": "^4.0.2",
+    "tempy": "^1.0.0"
+  },
+  "devDependencies": {
+    "@pnpm/node.fetcher": "workspace:0.0.0",
+    "@pnpm/prepare": "workspace:*",
+    "@types/adm-zip": "^0.4.34",
+    "node-fetch": "3.0.0-beta.9"
+  }
+}

packages/node.fetcher/src/getNodeTarball.ts

@@ -0,0 +1,17 @@
+import normalizeArch from './normalizeArch'
+
+export function getNodeTarball (
+  nodeVersion: string,
+  nodeMirror: string,
+  processPlatform: string,
+  processArch: string
+) {
+  const platform = processPlatform === 'win32' ? 'win' : processPlatform
+  const arch = normalizeArch(processPlatform, processArch)
+  const extension = platform === 'win' ? 'zip' : 'tar.gz'
+  const pkgName = `node-v${nodeVersion}-${platform}-${arch}`
+  return {
+    pkgName,
+    tarball: `${nodeMirror}v${nodeVersion}/${pkgName}.${extension}`,
+  }
+}

packages/node.fetcher/src/index.ts

@@ -0,0 +1,63 @@
+import fs from 'fs'
+import path from 'path'
+import {
+  FetchFromRegistry,
+  RetryTimeoutOptions,
+} from '@pnpm/fetching-types'
+import { FilesIndex } from '@pnpm/fetcher-base'
+import createCafsStore from '@pnpm/create-cafs-store'
+import createFetcher, { waitForFilesIndex } from '@pnpm/tarball-fetcher'
+import AdmZip from 'adm-zip'
+import renameOverwrite from 'rename-overwrite'
+import tempy from 'tempy'
+import { getNodeTarball } from './getNodeTarball'
+
+export interface FetchNodeOptions {
+  cafsDir: string
+  fetchTimeout?: number
+  nodeMirrorBaseUrl: string
+  retry?: RetryTimeoutOptions
+}
+
+export async function fetchNode (fetch: FetchFromRegistry, version: string, targetDir: string, opts: FetchNodeOptions) {
+  const { tarball, pkgName } = getNodeTarball(version, opts.nodeMirrorBaseUrl, process.platform, process.arch)
+  if (tarball.endsWith('.zip')) {
+    await downloadAndUnpackZip(fetch, tarball, targetDir, pkgName)
+    return
+  }
+  const getCredentials = () => ({ authHeaderValue: undefined, alwaysAuth: undefined })
+  const { tarball: fetchTarball } = createFetcher(fetch, getCredentials, {
+    retry: opts.retry,
+    timeout: opts.fetchTimeout,
+  })
+  const cafs = createCafsStore(opts.cafsDir)
+  const { filesIndex } = await fetchTarball(cafs, { tarball }, {
+    lockfileDir: process.cwd(),
+  })
+  await cafs.importPackage(targetDir, {
+    filesResponse: {
+      filesIndex: await waitForFilesIndex(filesIndex as FilesIndex),
+      fromStore: false,
+    },
+    force: true,
+  })
+}
+
+async function downloadAndUnpackZip (
+  fetchFromRegistry: FetchFromRegistry,
+  zipUrl: string,
+  targetDir: string,
+  pkgName: string
+) {
+  const response = await fetchFromRegistry(zipUrl)
+  const tmp = path.join(tempy.directory(), 'pnpm.zip')
+  const dest = fs.createWriteStream(tmp)
+  await new Promise((resolve, reject) => {
+    response.body!.pipe(dest).on('error', reject).on('close', resolve)
+  })
+  const zip = new AdmZip(tmp)
+  const nodeDir = path.dirname(targetDir)
+  zip.extractAllTo(nodeDir, true)
+  await renameOverwrite(path.join(nodeDir, pkgName), targetDir)
+  await fs.promises.unlink(tmp)
+}

packages/node.fetcher/test/getNodeTarball.test.ts

@@ -0,0 +1,36 @@
+import { getNodeTarball } from '../lib/getNodeTarball'
+
+test.each([
+  [
+    '16.0.0',
+    'https://nodejs.org/download/release/',
+    'win32',
+    'ia32',
+    {
+      pkgName: 'node-v16.0.0-win-x86',
+      tarball: 'https://nodejs.org/download/release/v16.0.0/node-v16.0.0-win-x86.zip',
+    },
+  ],
+  [
+    '16.0.0',
+    'https://nodejs.org/download/release/',
+    'linux',
+    'arm',
+    {
+      pkgName: 'node-v16.0.0-linux-armv7l',
+      tarball: 'https://nodejs.org/download/release/v16.0.0/node-v16.0.0-linux-armv7l.tar.gz',
+    },
+  ],
+  [
+    '16.0.0',
+    'https://nodejs.org/download/release/',
+    'linux',
+    'x64',
+    {
+      pkgName: 'node-v16.0.0-linux-x64',
+      tarball: 'https://nodejs.org/download/release/v16.0.0/node-v16.0.0-linux-x64.tar.gz',
+    },
+  ],
+])('getNodeTarball', (version, nodeMirrorBaseUrl, platform, arch, tarball) => {
+  expect(getNodeTarball(version, nodeMirrorBaseUrl, platform, arch)).toStrictEqual(tarball)
+})

packages/node.fetcher/test/node.test.ts

@@ -0,0 +1,40 @@
+import AdmZip from 'adm-zip'
+import { Response } from 'node-fetch'
+import path from 'path'
+import { Readable } from 'stream'
+import { fetchNode, FetchNodeOptions } from '@pnpm/node.fetcher'
+import { tempDir } from '@pnpm/prepare'
+
+const fetchMock = jest.fn(async (url: string) => {
+  if (url.endsWith('.zip')) {
+    // The Windows code path for pnpm's node bootstrapping expects a subdir
+    // within the .zip file.
+    const pkgName = path.basename(url, '.zip')
+    const zip = new AdmZip()
+    zip.addFile(`${pkgName}/dummy-file`, Buffer.from('test'))
+
+    return new Response(Readable.from(zip.toBuffer()))
+  }
+
+  return new Response(Readable.from(Buffer.alloc(0)))
+})
+
+beforeEach(() => {
+  fetchMock.mockClear()
+})
+
+test('install Node uses node-mirror:release option', async () => {
+  tempDir()
+
+  const nodeMirrorBaseUrl = 'https://pnpm-node-mirror-test.localhost/download/release/'
+  const opts: FetchNodeOptions = {
+    nodeMirrorBaseUrl,
+    cafsDir: path.resolve('files'),
+  }
+
+  await fetchNode(fetchMock, '16.4.0', path.resolve('node'), opts)
+
+  for (const call of fetchMock.mock.calls) {
+    expect(call[0]).toMatch(nodeMirrorBaseUrl)
+  }
+})

packages/node.fetcher/test/normalizeArch.test.ts

@@ -0,0 +1,9 @@
+import normalizeArch from '../lib/normalizeArch'
+
+test.each([
+  ['win32', 'ia32', 'x86'],
+  ['linux', 'arm', 'armv7l'], // Raspberry Pi 4
+  ['linux', 'x64', 'x64'],
+])('normalizedArch(%s, %s)', (platform, arch, normalizedArch) => {
+  expect(normalizeArch(platform, arch)).toBe(normalizedArch)
+})

packages/node.fetcher/tsconfig.json

@@ -0,0 +1,28 @@
+{
+  "extends": "@pnpm/tsconfig",
+  "compilerOptions": {
+    "outDir": "lib",
+    "rootDir": "src"
+  },
+  "include": [
+    "src/**/*.ts",
+    "../../typings/**/*.d.ts"
+  ],
+  "references": [
+    {
+      "path": "../../privatePackages/prepare"
+    },
+    {
+      "path": "../create-cafs-store"
+    },
+    {
+      "path": "../fetcher-base"
+    },
+    {
+      "path": "../fetching-types"
+    },
+    {
+      "path": "../tarball-fetcher"
+    }
+  ]
+}

packages/node.fetcher/tsconfig.lint.json

@@ -0,0 +1,8 @@
+{
+  "extends": "./tsconfig.json",
+  "include": [
+    "src/**/*.ts",
+    "test/**/*.ts",
+    "../../typings/**/*.d.ts"
+  ]
+}

packages/plugin-commands-env/package.json

@@ -32,19 +32,15 @@
   "dependencies": {
     "@pnpm/cli-utils": "workspace:0.7.13",
     "@pnpm/config": "workspace:15.3.0",
-    "@pnpm/create-cafs-store": "workspace:1.1.0",
     "@pnpm/error": "workspace:3.0.1",
     "@pnpm/fetch": "workspace:5.0.3",
     "@pnpm/fetcher-base": "workspace:12.1.0",
+    "@pnpm/node.fetcher": "workspace:0.0.0",
     "@pnpm/store-path": "workspace:6.0.0",
-    "@pnpm/tarball-fetcher": "workspace:10.0.5",
     "@zkochan/cmd-shim": "^5.2.2",
-    "adm-zip": "^0.5.5",
     "load-json-file": "^6.2.0",
-    "rename-overwrite": "^4.0.2",
     "render-help": "^1.0.1",
     "semver": "^7.3.4",
-    "tempy": "^1.0.0",
     "version-selector-type": "^3.0.0",
     "write-json-file": "^4.3.0"
   },
@@ -54,6 +50,7 @@
     "@pnpm/prepare": "workspace:*",
     "@types/adm-zip": "^0.4.34",
     "@types/semver": "^7.3.4",
+    "adm-zip": "^0.5.5",
     "execa": "npm:safe-execa@^0.1.1",
     "nock": "12.0.3",
     "node-fetch": "3.0.0-beta.9",

packages/plugin-commands-env/src/node.ts

@@ -2,16 +2,10 @@ import fs from 'fs'
 import path from 'path'
 import { Config } from '@pnpm/config'
 import { createFetchFromRegistry, FetchFromRegistry } from '@pnpm/fetch'
-import { FilesIndex } from '@pnpm/fetcher-base'
-import createCafsStore from '@pnpm/create-cafs-store'
+import { fetchNode } from '@pnpm/node.fetcher'
 import storePath from '@pnpm/store-path'
-import createFetcher, { waitForFilesIndex } from '@pnpm/tarball-fetcher'
-import AdmZip from 'adm-zip'
-import renameOverwrite from 'rename-overwrite'
-import tempy from 'tempy'
 import loadJsonFile from 'load-json-file'
 import writeJsonFile from 'write-json-file'
-import normalizeArch from './normalizeArch'
 import getNodeMirror from './getNodeMirror'
 import { parseNodeEditionSpecifier } from './parseNodeEditionSpecifier'
 
@@ -71,76 +65,26 @@ export async function getNodeDir (fetch: FetchFromRegistry, opts: NvmNodeCommand
   await fs.promises.mkdir(nodesDir, { recursive: true })
   const versionDir = path.join(nodesDir, opts.useNodeVersion)
   if (!fs.existsSync(versionDir)) {
-    await installNode(fetch, opts.useNodeVersion, versionDir, opts)
+    const storeDir = await storePath({
+      pkgRoot: process.cwd(),
+      storePath: opts.storeDir,
+      pnpmHomeDir: opts.pnpmHomeDir,
+    })
+    const cafsDir = path.join(storeDir, 'files')
+    await fetchNode(fetch, opts.useNodeVersion, versionDir, {
+      ...opts,
+      cafsDir,
+      retry: {
+        maxTimeout: opts.fetchRetryMaxtimeout,
+        minTimeout: opts.fetchRetryMintimeout,
+        retries: opts.fetchRetries,
+        factor: opts.fetchRetryFactor,
+      },
+    })
   }
   return versionDir
 }
 
-async function installNode (fetch: FetchFromRegistry, wantedNodeVersion: string, versionDir: string, opts: NvmNodeCommandOptions & { nodeMirrorBaseUrl: string }) {
-  const { tarball, pkgName } = getNodeJSTarball(wantedNodeVersion, opts.nodeMirrorBaseUrl)
-  if (tarball.endsWith('.zip')) {
-    await downloadAndUnpackZip(fetch, tarball, versionDir, pkgName)
-    return
-  }
-  const getCredentials = () => ({ authHeaderValue: undefined, alwaysAuth: undefined })
-  const { tarball: fetchTarball } = createFetcher(fetch, getCredentials, {
-    retry: {
-      maxTimeout: opts.fetchRetryMaxtimeout,
-      minTimeout: opts.fetchRetryMintimeout,
-      retries: opts.fetchRetries,
-      factor: opts.fetchRetryFactor,
-    },
-    timeout: opts.fetchTimeout,
-  })
-  const storeDir = await storePath({
-    pkgRoot: process.cwd(),
-    storePath: opts.storeDir,
-    pnpmHomeDir: opts.pnpmHomeDir,
-  })
-  const cafsDir = path.join(storeDir, 'files')
-  const cafs = createCafsStore(cafsDir)
-  const { filesIndex } = await fetchTarball(cafs, { tarball }, {
-    lockfileDir: process.cwd(),
-  })
-  await cafs.importPackage(versionDir, {
-    filesResponse: {
-      filesIndex: await waitForFilesIndex(filesIndex as FilesIndex),
-      fromStore: false,
-    },
-    force: true,
-  })
-}
-
-async function downloadAndUnpackZip (
-  fetchFromRegistry: FetchFromRegistry,
-  zipUrl: string,
-  targetDir: string,
-  pkgName: string
-) {
-  const response = await fetchFromRegistry(zipUrl)
-  const tmp = path.join(tempy.directory(), 'pnpm.zip')
-  const dest = fs.createWriteStream(tmp)
-  await new Promise((resolve, reject) => {
-    response.body!.pipe(dest).on('error', reject).on('close', resolve)
-  })
-  const zip = new AdmZip(tmp)
-  const nodeDir = path.dirname(targetDir)
-  zip.extractAllTo(nodeDir, true)
-  await renameOverwrite(path.join(nodeDir, pkgName), targetDir)
-  await fs.promises.unlink(tmp)
-}
-
-function getNodeJSTarball (nodeVersion: string, nodeMirror: string) {
-  const platform = process.platform === 'win32' ? 'win' : process.platform
-  const arch = normalizeArch(process.platform, process.arch)
-  const extension = platform === 'win' ? 'zip' : 'tar.gz'
-  const pkgName = `node-v${nodeVersion}-${platform}-${arch}`
-  return {
-    pkgName,
-    tarball: `${nodeMirror}v${nodeVersion}/${pkgName}.${extension}`,
-  }
-}
-
 async function readNodeVersionsManifest (nodesDir: string): Promise<{ default?: string }> {
   try {
     return await loadJsonFile<{ default?: string }>(path.join(nodesDir, 'versions.json'))

packages/plugin-commands-env/tsconfig.json

@@ -18,9 +18,6 @@
     {
       "path": "../config"
     },
-    {
-      "path": "../create-cafs-store"
-    },
     {
       "path": "../error"
     },
@@ -31,10 +28,10 @@
       "path": "../fetcher-base"
     },
     {
-      "path": "../store-path"
+      "path": "../node.fetcher"
     },
     {
-      "path": "../tarball-fetcher"
+      "path": "../store-path"
     }
   ]
 }

pnpm-lock.yaml

@@ -1710,6 +1710,33 @@ importers:
       '@types/normalize-path': 3.0.0
       rimraf: 3.0.2
 
+  packages/node.fetcher:
+    specifiers:
+      '@pnpm/create-cafs-store': workspace:1.1.0
+      '@pnpm/fetcher-base': workspace:12.1.0
+      '@pnpm/fetching-types': workspace:3.0.0
+      '@pnpm/node.fetcher': workspace:0.0.0
+      '@pnpm/prepare': workspace:*
+      '@pnpm/tarball-fetcher': workspace:10.0.5
+      '@types/adm-zip': ^0.4.34
+      adm-zip: ^0.5.5
+      node-fetch: 3.0.0-beta.9
+      rename-overwrite: ^4.0.2
+      tempy: ^1.0.0
+    dependencies:
+      '@pnpm/create-cafs-store': link:../create-cafs-store
+      '@pnpm/fetcher-base': link:../fetcher-base
+      '@pnpm/fetching-types': link:../fetching-types
+      '@pnpm/tarball-fetcher': link:../tarball-fetcher
+      adm-zip: 0.5.9
+      rename-overwrite: 4.0.2
+      tempy: 1.0.1
+    devDependencies:
+      '@pnpm/node.fetcher': 'link:'
+      '@pnpm/prepare': link:../../privatePackages/prepare
+      '@types/adm-zip': 0.4.34
+      node-fetch: 3.0.0-beta.9
+
   packages/normalize-registries:
     specifiers:
       '@pnpm/normalize-registries': workspace:3.0.3
@@ -2103,14 +2130,13 @@ importers:
     specifiers:
       '@pnpm/cli-utils': workspace:0.7.13
       '@pnpm/config': workspace:15.3.0
-      '@pnpm/create-cafs-store': workspace:1.1.0
       '@pnpm/error': workspace:3.0.1
       '@pnpm/fetch': workspace:5.0.3
       '@pnpm/fetcher-base': workspace:12.1.0
+      '@pnpm/node.fetcher': workspace:0.0.0
       '@pnpm/plugin-commands-env': workspace:2.1.12
       '@pnpm/prepare': workspace:*
       '@pnpm/store-path': workspace:6.0.0
-      '@pnpm/tarball-fetcher': workspace:10.0.5
       '@types/adm-zip': ^0.4.34
       '@types/semver': ^7.3.4
       '@zkochan/cmd-shim': ^5.2.2
@@ -2120,28 +2146,22 @@ importers:
       nock: 12.0.3
       node-fetch: 3.0.0-beta.9
       path-name: ^1.0.0
-      rename-overwrite: ^4.0.2
       render-help: ^1.0.1
       semver: ^7.3.4
-      tempy: ^1.0.0
       version-selector-type: ^3.0.0
       write-json-file: ^4.3.0
     dependencies:
       '@pnpm/cli-utils': link:../cli-utils
       '@pnpm/config': link:../config
-      '@pnpm/create-cafs-store': link:../create-cafs-store
       '@pnpm/error': link:../error
       '@pnpm/fetch': link:../fetch
       '@pnpm/fetcher-base': link:../fetcher-base
+      '@pnpm/node.fetcher': link:../node.fetcher
       '@pnpm/store-path': link:../store-path
-      '@pnpm/tarball-fetcher': link:../tarball-fetcher
       '@zkochan/cmd-shim': 5.2.2
-      adm-zip: 0.5.9
       load-json-file: 6.2.0
-      rename-overwrite: 4.0.2
       render-help: 1.0.2
       semver: 7.3.7
-      tempy: 1.0.1
       version-selector-type: 3.0.0
       write-json-file: 4.3.0
     devDependencies:
@@ -2149,6 +2169,7 @@ importers:
       '@pnpm/prepare': link:../../privatePackages/prepare
       '@types/adm-zip': 0.4.34
       '@types/semver': 7.3.9
+      adm-zip: 0.5.9
       execa: /safe-execa/0.1.1
       nock: 12.0.3
       node-fetch: 3.0.0-beta.9
@@ -5441,7 +5462,7 @@ packages:
   /@types/adm-zip/0.4.34:
     resolution: {integrity: sha512-8ToYLLAYhkRfcmmljrKi22gT2pqu7hGMDtORP1emwIEGmgUTZOsaDjzWFzW5N2frcFRz/50CWt4zA1CxJ73pmQ==}
     dependencies:
-      '@types/node': 17.0.41
+      '@types/node': 18.0.0
     dev: true
 
   /@types/archy/0.0.31:
@@ -6390,7 +6411,6 @@ packages:
   /adm-zip/0.5.9:
     resolution: {integrity: sha512-s+3fXLkeeLjZ2kLjCBwQufpI5fuN+kIGBxu6530nVQZGVol0d7Y/M88/xw9HGGUcJjKf8LutN3VPRUBq6N7Ajg==}
     engines: {node: '>=6.0'}
-    dev: false
 
   /agent-base/6.0.2:
     resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==}