From 4cf7ef367264492f04fbda50643698616b93ec1d Mon Sep 17 00:00:00 2001 From: Zoltan Kochan Date: Thu, 4 Jun 2020 02:34:23 +0300 Subject: [PATCH] perf(npm-resolver): don't use write-json-file --- .changeset/famous-schools-flow.md | 5 +++++ packages/npm-resolver/package.json | 7 +++++-- packages/npm-resolver/src/pickPackage.ts | 25 ++++++++++++++++++++---- pnpm-lock.yaml | 15 +++++++++++--- 4 files changed, 43 insertions(+), 9 deletions(-) create mode 100644 .changeset/famous-schools-flow.md diff --git a/.changeset/famous-schools-flow.md b/.changeset/famous-schools-flow.md new file mode 100644 index 0000000000..f3e72b900d --- /dev/null +++ b/.changeset/famous-schools-flow.md @@ -0,0 +1,5 @@ +--- +"@pnpm/npm-resolver": minor +--- + +Reducing filesystem operations required to write the metadata file to the cache. diff --git a/packages/npm-resolver/package.json b/packages/npm-resolver/package.json index dd2b5e3de4..ac8227a05d 100644 --- a/packages/npm-resolver/package.json +++ b/packages/npm-resolver/package.json @@ -43,18 +43,21 @@ "fetch-from-npm-registry": "workspace:4.0.3", "load-json-file": "6.2.0", "mem": "^6.1.0", + "mz": "^2.7.0", "normalize-path": "3.0.0", "p-limit": "2.3.0", "p-memoize": "4.0.0", "parse-npm-tarball-url": "3.0.0", + "path-temp": "^2.0.0", + "rename-overwrite": "^3.0.0", "semver": "^7.3.2", "ssri": "6.0.1", - "version-selector-type": "^3.0.0", - "write-json-file": "4.0.0" + "version-selector-type": "^3.0.0" }, "devDependencies": { "@pnpm/logger": "3.2.2", "@pnpm/npm-resolver": "link:", + "@types/mz": "^2.7.1", "@types/normalize-path": "^3.0.0", "@types/semver": "^7.2.0", "@types/ssri": "^6.0.3", diff --git a/packages/npm-resolver/src/pickPackage.ts b/packages/npm-resolver/src/pickPackage.ts index b66cc67e1d..d7e6524564 100644 --- a/packages/npm-resolver/src/pickPackage.ts +++ b/packages/npm-resolver/src/pickPackage.ts @@ -4,9 +4,11 @@ import { VersionSelectors } from '@pnpm/resolver-base' import { PackageManifest } from '@pnpm/types' import getRegistryName = require('encode-registry') import loadJsonFile = require('load-json-file') +import fs = require('mz/fs') import pLimit, { Limit } from 'p-limit' import path = require('path') -import writeJsonFile = require('write-json-file') +import pathTemp = require('path-temp') +import renameOverwrite = require('rename-overwrite') import { RegistryPackageSpec } from './parsePref' import pickPackageFromMeta from './pickPackageFromMeta' import toRaw from './toRaw' @@ -119,7 +121,13 @@ export default async ( ctx.metaCache.set(spec.name, meta) if (!opts.dryRun) { // tslint:disable-next-line:no-floating-promises - limit(() => saveMeta(pkgMirror, meta)) + limit(async () => { + try { + await saveMeta(pkgMirror, meta) + } catch (err) { + // We don't care if this file was not written to the cache + } + }) } return { meta, @@ -145,8 +153,17 @@ async function loadMeta (pkgMirror: string): Promise { } } -function saveMeta (pkgMirror: string, meta: PackageMeta): Promise { - return writeJsonFile(pkgMirror, meta) +const createdDirs = new Set() + +async function saveMeta (pkgMirror: string, meta: PackageMeta): Promise { + const dir = path.dirname(pkgMirror) + if (!createdDirs.has(dir)) { + await fs.mkdir(dir, { recursive: true }) + createdDirs.add(dir) + } + const temp = pathTemp(dir) + await fs.writeFile(temp, JSON.stringify(meta)) + await renameOverwrite(temp, pkgMirror) } function validatePackageName (pkgName: string) { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f0b78511e9..759f2172da 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1089,17 +1089,20 @@ importers: fetch-from-npm-registry: 'link:../fetch-from-npm-registry' load-json-file: 6.2.0 mem: 6.1.0 + mz: 2.7.0 normalize-path: 3.0.0 p-limit: 2.3.0 p-memoize: 4.0.0 parse-npm-tarball-url: 3.0.0 + path-temp: 2.0.0 + rename-overwrite: 3.0.0 semver: 7.3.2 ssri: 6.0.1 version-selector-type: 3.0.0 - write-json-file: 4.0.0 devDependencies: '@pnpm/logger': 3.2.2 '@pnpm/npm-resolver': 'link:' + '@types/mz': 2.7.1 '@types/normalize-path': 3.0.0 '@types/semver': 7.2.0 '@types/ssri': 6.0.3 @@ -1113,6 +1116,7 @@ importers: '@pnpm/resolve-workspace-range': 'workspace:1.0.1' '@pnpm/resolver-base': 'workspace:7.0.1' '@pnpm/types': 'workspace:6.0.0' + '@types/mz': ^2.7.1 '@types/normalize-path': ^3.0.0 '@types/semver': ^7.2.0 '@types/ssri': ^6.0.3 @@ -1121,17 +1125,19 @@ importers: fetch-from-npm-registry: 'workspace:4.0.3' load-json-file: 6.2.0 mem: ^6.1.0 + mz: ^2.7.0 nock: 12.0.3 normalize-path: 3.0.0 p-limit: 2.3.0 p-memoize: 4.0.0 parse-npm-tarball-url: 3.0.0 path-exists: 4.0.0 + path-temp: ^2.0.0 + rename-overwrite: ^3.0.0 semver: ^7.3.2 ssri: 6.0.1 tempy: 0.5.0 version-selector-type: ^3.0.0 - write-json-file: 4.0.0 packages/outdated: dependencies: '@pnpm/config': 'link:../config' @@ -3645,7 +3651,7 @@ packages: integrity: sha1-aaI6OtKcrwCX8G7aWbNh7i8GOfY= /@types/mz/2.7.1: dependencies: - '@types/node': 14.0.6 + '@types/node': 14.0.9 resolution: integrity: sha512-H86h7KmRDVs9UeSiQvtUeVhS+WYpJSYSsZrRvNYpGWGiytEqxwEtvgRnINESQtCgnojIH2wS2WgaMTJP0firBw== /@types/ncp/2.0.4: @@ -3671,6 +3677,9 @@ packages: /@types/node/14.0.6: resolution: integrity: sha512-FbNmu4F67d3oZMWBV6Y4MaPER+0EpE9eIYf2yaHhCWovc1dlXCZkqGX4NLHfVVr6umt20TNBdRzrNJIzIKfdbw== + /@types/node/14.0.9: + resolution: + integrity: sha512-0sCTiXKXELOBxvZLN4krQ0FPOAA7ij+6WwvD0k/PHd9/KAkr4dXel5J9fh6F4x1FwAQILqAWkmpeuS6mjf1iKA== /@types/nopt/3.0.29: resolution: integrity: sha1-8Z3z20yX7hRZonQAKDIKcdcJZM4=