diff --git a/.changeset/long-masks-burn.md b/.changeset/long-masks-burn.md new file mode 100644 index 0000000000..e22a6bc998 --- /dev/null +++ b/.changeset/long-masks-burn.md @@ -0,0 +1,5 @@ +--- +"@pnpm/package-store": minor +--- + +Prune unreferenced files from the store. diff --git a/packages/package-store/package.json b/packages/package-store/package.json index 9bd6821ccc..21bf27a369 100644 --- a/packages/package-store/package.json +++ b/packages/package-store/package.json @@ -35,6 +35,7 @@ "path-temp": "2.0.0", "ramda": "0.27.0", "rename-overwrite": "^3.0.0", + "ssri": "^8.0.0", "write-json-file": "4.0.0" }, "devDependencies": { @@ -46,6 +47,7 @@ "@types/proxyquire": "1.3.28", "@types/ramda": "^0.27.4", "@types/sinon": "^9.0.0", + "@types/ssri": "^6.0.2", "proxyquire": "2.1.3", "sinon": "9.0.2", "tempy": "0.5.0" diff --git a/packages/package-store/src/storeController/index.ts b/packages/package-store/src/storeController/index.ts index 0ff7a03946..467705a5c4 100644 --- a/packages/package-store/src/storeController/index.ts +++ b/packages/package-store/src/storeController/index.ts @@ -1,7 +1,7 @@ import { getFilePathByModeInCafs as _getFilePathByModeInCafs } from '@pnpm/cafs' import { FetchFunction } from '@pnpm/fetcher-base' import lock from '@pnpm/fs-locker' -import { globalInfo, globalWarn } from '@pnpm/logger' +import { globalWarn } from '@pnpm/logger' import createPackageRequester, { getCacheByEngine } from '@pnpm/package-requester' import pkgIdToFilename from '@pnpm/pkgid-to-filename' import { ResolveFunction } from '@pnpm/resolver-base' @@ -17,14 +17,14 @@ import pLimit from 'p-limit' import path = require('path') import exists = require('path-exists') import R = require('ramda') -import { promisify } from 'util' import writeJsonFile = require('write-json-file') import { read as readStore, save as saveStore, saveSync as saveStoreSync, } from '../fs/storeIndex' -import createImportPackage, { copyPkg } from './createImportPackage' +import createImportPackage from './createImportPackage' +import prune from './prune' export default async function ( resolve: ResolveFunction, @@ -75,7 +75,7 @@ export default async function ( findPackageUsages, getPackageLocation, importPackage, - prune, + prune: prune.bind(null, storeDir), requestPackage: packageRequester.requestPackage, saveState: saveStore.bind(null, initOpts.storeDir, storeIndex), saveStateSync: saveStoreSync.bind(null, initOpts.storeDir, storeIndex), @@ -131,21 +131,6 @@ export default async function ( }) } - async function prune () { - const removedProjects = await getRemovedProject(storeIndex) - for (const pkgId in storeIndex) { - if (storeIndex.hasOwnProperty(pkgId)) { - storeIndex[pkgId] = R.difference(storeIndex[pkgId], removedProjects) - - if (!storeIndex[pkgId].length) { - delete storeIndex[pkgId] - await rimraf(path.join(storeDir, pkgId)) - globalInfo(`- ${pkgId}`) - } - } - } - } - async function findPackageUsages (searchQueries: string[]): Promise { const results = {} as PackageUsagesBySearchQueries diff --git a/packages/package-store/src/storeController/prune.ts b/packages/package-store/src/storeController/prune.ts new file mode 100644 index 0000000000..5cd1df5c18 --- /dev/null +++ b/packages/package-store/src/storeController/prune.ts @@ -0,0 +1,44 @@ +import { globalInfo } from '@pnpm/logger' +import rimraf = require('@zkochan/rimraf') +import loadJsonFile = require('load-json-file') +import fs = require('mz/fs') +import path = require('path') +import ssri = require('ssri') + +export default async function prune (storeDir: string) { + const cafsDir = path.join(storeDir, 'files') + await rimraf(path.join(storeDir, 'metadata')) + await rimraf(path.join(storeDir, 'metadata-full')) + globalInfo('Removed all cached metadata files') + const pkgIndexFiles = [] as string[] + const removedHashes = new Set() + const dirs = await fs.readdir(cafsDir) + let fileCounter = 0 + for (const dir of dirs) { + const subdir = path.join(cafsDir, dir) + for (const fileName of await fs.readdir(subdir)) { + const filePath = path.join(subdir, fileName) + if (fileName.endsWith('-index.json')) { + pkgIndexFiles.push(filePath) + continue + } + const stat = await fs.stat(filePath) + if (stat.nlink === 1) { + await fs.unlink(filePath) + fileCounter++ + removedHashes.add(ssri.fromHex(`${dir}${fileName}`, 'sha512').toString()) + } + } + } + globalInfo(`Removed ${fileCounter} file${fileCounter === 1 ? '' : 's'}`) + + let pkgCounter = 0 + for (const pkgIndexFilePath of pkgIndexFiles) { + const pkgFilesIndex = await loadJsonFile(pkgIndexFilePath) + if (removedHashes.has(pkgFilesIndex['package.json'].integrity)) { + await fs.unlink(pkgIndexFilePath) + pkgCounter++ + } + } + globalInfo(`Removed ${pkgCounter} package${pkgCounter === 1 ? '' : 's'}`) +} diff --git a/packages/plugin-commands-store/test/storePrune.ts b/packages/plugin-commands-store/test/storePrune.ts index 72248c8f75..59f007ef01 100644 --- a/packages/plugin-commands-store/test/storePrune.ts +++ b/packages/plugin-commands-store/test/storePrune.ts @@ -37,7 +37,7 @@ test('remove unreferenced packages', async (t) => { t.ok(reporter.calledWithMatch({ level: 'info', - message: `- localhost+${REGISTRY_MOCK_PORT}/is-negative/2.1.0`, + message: 'Removed 1 package', }), 'report removal') await project.storeHasNot('is-negative', '2.1.0') @@ -56,7 +56,7 @@ test('remove unreferenced packages', async (t) => { t.notOk(reporter.calledWithMatch({ level: 'info', - message: `- localhost+${REGISTRY_MOCK_PORT}/is-negative/2.1.0`, + message: 'Removed 1 package', })) t.end() }) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 94ac64db2c..2a5c3cde61 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1313,6 +1313,7 @@ importers: path-temp: 2.0.0 ramda: 0.27.0 rename-overwrite: 3.0.0 + ssri: 8.0.0 write-json-file: 4.0.0 devDependencies: '@pnpm/logger': 3.2.2 @@ -1323,6 +1324,7 @@ importers: '@types/proxyquire': 1.3.28 '@types/ramda': 0.27.4 '@types/sinon': 9.0.0 + '@types/ssri': 6.0.2 proxyquire: 2.1.3 sinon: 9.0.2 tempy: 0.5.0 @@ -1344,6 +1346,7 @@ importers: '@types/proxyquire': 1.3.28 '@types/ramda': ^0.27.4 '@types/sinon': ^9.0.0 + '@types/ssri': ^6.0.2 '@zkochan/rimraf': 1.0.0 load-json-file: 6.2.0 make-empty-dir: ^1.0.0 @@ -1356,6 +1359,7 @@ importers: ramda: 0.27.0 rename-overwrite: ^3.0.0 sinon: 9.0.2 + ssri: ^8.0.0 tempy: 0.5.0 write-json-file: 4.0.0 packages/parse-cli-args: @@ -8771,7 +8775,6 @@ packages: /minipass/3.1.1: dependencies: yallist: 4.0.0 - dev: true engines: node: '>=8' resolution: @@ -11253,7 +11256,6 @@ packages: /ssri/8.0.0: dependencies: minipass: 3.1.1 - dev: true engines: node: '>= 8' resolution: @@ -12694,7 +12696,6 @@ packages: resolution: integrity: sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g== /yallist/4.0.0: - dev: true resolution: integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== /yaml-tag/1.1.0: