feat: pruning of the content-addressable store

ref #2470
PR #2533

Co-authored-by: Aparajita Fishman <aparajita@aparajita.com>

.changeset/long-masks-burn.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/package-store": minor
+---
+
+Prune unreferenced files from the store.

packages/package-store/package.json

@@ -35,6 +35,7 @@
     "path-temp": "2.0.0",
     "ramda": "0.27.0",
     "rename-overwrite": "^3.0.0",
+    "ssri": "^8.0.0",
     "write-json-file": "4.0.0"
   },
   "devDependencies": {
@@ -46,6 +47,7 @@
     "@types/proxyquire": "1.3.28",
     "@types/ramda": "^0.27.4",
     "@types/sinon": "^9.0.0",
+    "@types/ssri": "^6.0.2",
     "proxyquire": "2.1.3",
     "sinon": "9.0.2",
     "tempy": "0.5.0"

packages/package-store/src/storeController/index.ts

@@ -1,7 +1,7 @@
 import { getFilePathByModeInCafs as _getFilePathByModeInCafs } from '@pnpm/cafs'
 import { FetchFunction } from '@pnpm/fetcher-base'
 import lock from '@pnpm/fs-locker'
-import { globalInfo, globalWarn } from '@pnpm/logger'
+import { globalWarn } from '@pnpm/logger'
 import createPackageRequester, { getCacheByEngine } from '@pnpm/package-requester'
 import pkgIdToFilename from '@pnpm/pkgid-to-filename'
 import { ResolveFunction } from '@pnpm/resolver-base'
@@ -17,14 +17,14 @@ import pLimit from 'p-limit'
 import path = require('path')
 import exists = require('path-exists')
 import R = require('ramda')
-import { promisify } from 'util'
 import writeJsonFile = require('write-json-file')
 import {
   read as readStore,
   save as saveStore,
   saveSync as saveStoreSync,
 } from '../fs/storeIndex'
-import createImportPackage, { copyPkg } from './createImportPackage'
+import createImportPackage from './createImportPackage'
+import prune from './prune'
 
 export default async function (
   resolve: ResolveFunction,
@@ -75,7 +75,7 @@ export default async function (
     findPackageUsages,
     getPackageLocation,
     importPackage,
-    prune,
+    prune: prune.bind(null, storeDir),
     requestPackage: packageRequester.requestPackage,
     saveState: saveStore.bind(null, initOpts.storeDir, storeIndex),
     saveStateSync: saveStoreSync.bind(null, initOpts.storeDir, storeIndex),
@@ -131,21 +131,6 @@ export default async function (
     })
   }
 
-  async function prune () {
-    const removedProjects = await getRemovedProject(storeIndex)
-    for (const pkgId in storeIndex) {
-      if (storeIndex.hasOwnProperty(pkgId)) {
-        storeIndex[pkgId] = R.difference(storeIndex[pkgId], removedProjects)
-
-        if (!storeIndex[pkgId].length) {
-          delete storeIndex[pkgId]
-          await rimraf(path.join(storeDir, pkgId))
-          globalInfo(`- ${pkgId}`)
-        }
-      }
-    }
-  }
-
   async function findPackageUsages (searchQueries: string[]): Promise<PackageUsagesBySearchQueries> {
     const results = {} as PackageUsagesBySearchQueries
 

packages/package-store/src/storeController/prune.ts

@@ -0,0 +1,44 @@
+import { globalInfo } from '@pnpm/logger'
+import rimraf = require('@zkochan/rimraf')
+import loadJsonFile = require('load-json-file')
+import fs = require('mz/fs')
+import path = require('path')
+import ssri = require('ssri')
+
+export default async function prune (storeDir: string) {
+  const cafsDir = path.join(storeDir, 'files')
+  await rimraf(path.join(storeDir, 'metadata'))
+  await rimraf(path.join(storeDir, 'metadata-full'))
+  globalInfo('Removed all cached metadata files')
+  const pkgIndexFiles = [] as string[]
+  const removedHashes = new Set<string>()
+  const dirs = await fs.readdir(cafsDir)
+  let fileCounter = 0
+  for (const dir of dirs) {
+    const subdir = path.join(cafsDir, dir)
+    for (const fileName of await fs.readdir(subdir)) {
+      const filePath = path.join(subdir, fileName)
+      if (fileName.endsWith('-index.json')) {
+        pkgIndexFiles.push(filePath)
+        continue
+      }
+      const stat = await fs.stat(filePath)
+      if (stat.nlink === 1) {
+        await fs.unlink(filePath)
+        fileCounter++
+        removedHashes.add(ssri.fromHex(`${dir}${fileName}`, 'sha512').toString())
+      }
+    }
+  }
+  globalInfo(`Removed ${fileCounter} file${fileCounter === 1 ? '' : 's'}`)
+
+  let pkgCounter = 0
+  for (const pkgIndexFilePath of pkgIndexFiles) {
+    const pkgFilesIndex = await loadJsonFile<object>(pkgIndexFilePath)
+    if (removedHashes.has(pkgFilesIndex['package.json'].integrity)) {
+      await fs.unlink(pkgIndexFilePath)
+      pkgCounter++
+    }
+  }
+  globalInfo(`Removed ${pkgCounter} package${pkgCounter === 1 ? '' : 's'}`)
+}

packages/plugin-commands-store/test/storePrune.ts

@@ -37,7 +37,7 @@ test('remove unreferenced packages', async (t) => {
 
   t.ok(reporter.calledWithMatch({
     level: 'info',
-    message: `- localhost+${REGISTRY_MOCK_PORT}/is-negative/2.1.0`,
+    message: 'Removed 1 package',
   }), 'report removal')
 
   await project.storeHasNot('is-negative', '2.1.0')
@@ -56,7 +56,7 @@ test('remove unreferenced packages', async (t) => {
 
   t.notOk(reporter.calledWithMatch({
     level: 'info',
-    message: `- localhost+${REGISTRY_MOCK_PORT}/is-negative/2.1.0`,
+    message: 'Removed 1 package',
   }))
   t.end()
 })

pnpm-lock.yaml

@@ -1313,6 +1313,7 @@ importers:
       path-temp: 2.0.0
       ramda: 0.27.0
       rename-overwrite: 3.0.0
+      ssri: 8.0.0
       write-json-file: 4.0.0
     devDependencies:
       '@pnpm/logger': 3.2.2
@@ -1323,6 +1324,7 @@ importers:
       '@types/proxyquire': 1.3.28
       '@types/ramda': 0.27.4
       '@types/sinon': 9.0.0
+      '@types/ssri': 6.0.2
       proxyquire: 2.1.3
       sinon: 9.0.2
       tempy: 0.5.0
@@ -1344,6 +1346,7 @@ importers:
       '@types/proxyquire': 1.3.28
       '@types/ramda': ^0.27.4
       '@types/sinon': ^9.0.0
+      '@types/ssri': ^6.0.2
       '@zkochan/rimraf': 1.0.0
       load-json-file: 6.2.0
       make-empty-dir: ^1.0.0
@@ -1356,6 +1359,7 @@ importers:
       ramda: 0.27.0
       rename-overwrite: ^3.0.0
       sinon: 9.0.2
+      ssri: ^8.0.0
       tempy: 0.5.0
       write-json-file: 4.0.0
   packages/parse-cli-args:
@@ -8771,7 +8775,6 @@ packages:
   /minipass/3.1.1:
     dependencies:
       yallist: 4.0.0
-    dev: true
     engines:
       node: '>=8'
     resolution:
@@ -11253,7 +11256,6 @@ packages:
   /ssri/8.0.0:
     dependencies:
       minipass: 3.1.1
-    dev: true
     engines:
       node: '>= 8'
     resolution:
@@ -12694,7 +12696,6 @@ packages:
     resolution:
       integrity: sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==
   /yallist/4.0.0:
-    dev: true
     resolution:
       integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==
   /yaml-tag/1.1.0: