feat: executable are saved into a separate dir

Executables are saved into a separate subdirectory
inside the content-addressable filesystem.

ref #2470

.changeset/late-dolphins-destroy.md

@@ -0,0 +1,12 @@
+---
+"@pnpm/cafs": minor
+"@pnpm/fetcher-base": minor
+"@pnpm/package-requester": minor
+"@pnpm/package-store": minor
+"@pnpm/plugin-commands-installation": minor
+"@pnpm/store-controller-types": minor
+"supi": minor
+"@pnpm/tarball-fetcher": minor
+---
+
+Executables are saved into a separate directory inside the content-addressable storage.

packages/cafs/src/addFilesFromDir.ts

@@ -10,8 +10,8 @@ const MAX_BULK_SIZE = 1 * 1024 * 1024 // 1MB
 
 export default async function (
   cafs: {
-    addStream: (stream: NodeJS.ReadableStream) => Promise<ssri.Integrity>,
-    addBuffer: (buffer: Buffer) => Promise<ssri.Integrity>,
+    addStream: (stream: NodeJS.ReadableStream, mode: number) => Promise<ssri.Integrity>,
+    addBuffer: (buffer: Buffer, mode: number) => Promise<ssri.Integrity>,
   },
   dirname: string,
 ) {
@@ -22,8 +22,8 @@ export default async function (
 
 async function _retrieveFileIntegrities (
   cafs: {
-    addStream: (stream: NodeJS.ReadableStream) => Promise<ssri.Integrity>,
-    addBuffer: (buffer: Buffer) => Promise<ssri.Integrity>,
+    addStream: (stream: NodeJS.ReadableStream, mode: number) => Promise<ssri.Integrity>,
+    addBuffer: (buffer: Buffer, mode: number) => Promise<ssri.Integrity>,
   },
   rootDir: string,
   currDir: string,
@@ -43,9 +43,10 @@ async function _retrieveFileIntegrities (
         index[relativePath] = {
           generatingIntegrity: limit(() => {
             return stat.size < MAX_BULK_SIZE
-              ? fs.readFile(fullPath).then(cafs.addBuffer)
-              : cafs.addStream(fs.createReadStream(fullPath))
+              ? fs.readFile(fullPath).then((buffer) => cafs.addBuffer(buffer, stat.mode))
+              : cafs.addStream(fs.createReadStream(fullPath), stat.mode)
           }),
+          mode: stat.mode,
           size: stat.size,
         }
       }

packages/cafs/src/addFilesFromTarball.ts

@@ -5,7 +5,7 @@ import { Duplex, PassThrough } from 'stream'
 import tar = require('tar-stream')
 
 export default async function (
-  addStreamToCafs: (fileStream: PassThrough) => Promise<ssri.Integrity>,
+  addStreamToCafs: (fileStream: PassThrough, mode: number) => Promise<ssri.Integrity>,
   _ignore: null | ((filename: string) => Boolean),
   stream: NodeJS.ReadableStream,
 ): Promise<FilesIndex> {
@@ -20,9 +20,10 @@ export default async function (
         next()
         return
       }
-      const generatingIntegrity = addStreamToCafs(fileStream)
+      const generatingIntegrity = addStreamToCafs(fileStream, header.mode!)
       filesIndex[filename] = {
         generatingIntegrity,
+        mode: header.mode!,
         size: header.size,
       }
       next()

packages/cafs/src/checkFilesIntegrity.ts

@@ -9,7 +9,7 @@ const MAX_BULK_SIZE = 1 * 1024 * 1024 // 1MB
 
 export default async function (
   cafsDir: string,
-  integrityObj: Record<string, { size: number, integrity: string }>,
+  integrityObj: Record<string, { size: number, mode: number, integrity: string }>,
 ) {
   let verified = true
   await Promise.all(
@@ -22,7 +22,7 @@ export default async function (
           }
           if (
             !await verifyFile(
-              getFilePathInCafs(cafsDir, fstat.integrity),
+              getFilePathInCafs(cafsDir, fstat),
               fstat,
             )
           ) {

packages/cafs/src/index.ts

@@ -26,18 +26,23 @@ async function addStreamToCafs (
   locker: Map<string, Promise<void>>,
   cafsDir: string,
   fileStream: NodeJS.ReadableStream,
+  mode: number,
 ): Promise<ssri.Integrity> {
   const buffer = await getStream.buffer(fileStream)
-  return addBufferToCafs(locker, cafsDir, buffer)
+  return addBufferToCafs(locker, cafsDir, buffer, mode)
 }
 
+const modeIsExecutable = (mode: number) => (mode & 0o111) === 0o111
+
 async function addBufferToCafs (
   locker: Map<string, Promise<void>>,
   cafsDir: string,
   buffer: Buffer,
+  mode: number,
 ): Promise<ssri.Integrity> {
   const integrity = ssri.fromData(buffer)
-  const fileDest = contentPathFromHex(cafsDir, integrity.hexDigest())
+  const isExecutable = modeIsExecutable(mode)
+  const fileDest = contentPathFromHex(cafsDir, isExecutable, integrity.hexDigest())
   if (locker.has(fileDest)) {
     await locker.get(fileDest)
     return integrity
@@ -55,7 +60,7 @@ async function addBufferToCafs (
     // If we don't allow --no-verify-store-integrity then we probably can write
     // to the final file directly.
     const temp = pathTemp(path.dirname(fileDest))
-    await writeFile(temp, buffer)
+    await writeFile(temp, buffer, isExecutable ? 0o755 : undefined)
     await renameOverwrite(temp, fileDest)
   })()
   locker.set(fileDest, p)
@@ -63,18 +68,29 @@ async function addBufferToCafs (
   return integrity
 }
 
-export function getFilePathInCafs (cafsDir: string, integrity: string | Hash) {
-  return contentPathFromIntegrity(cafsDir, integrity)
+export function getFilePathInCafs (
+  cafsDir: string,
+  file: {
+    integrity: string | Hash,
+    mode: number,
+  },
+) {
+  return contentPathFromIntegrity(cafsDir, file.integrity, file.mode)
 }
 
-function contentPathFromIntegrity (cafsDir: string, integrity: string | Hash) {
+function contentPathFromIntegrity (
+  cafsDir: string,
+  integrity: string | Hash,
+  mode: number,
+) {
   const sri = ssri.parse(integrity, { single: true })
-  return contentPathFromHex(cafsDir, sri.hexDigest())
+  const isExecutable = modeIsExecutable(mode)
+  return contentPathFromHex(cafsDir, isExecutable, sri.hexDigest())
 }
 
-function contentPathFromHex (cafsDir: string, hex: string) {
+function contentPathFromHex (cafsDir: string, isExecutable: boolean, hex: string) {
   return path.join(
-    cafsDir,
+    isExecutable ? path.join(cafsDir, 'x') : cafsDir,
     hex.slice(0, 2),
     hex.slice(2),
   )

packages/cafs/src/writeFile.ts

@@ -7,13 +7,12 @@ const dirs = new Set()
 export default async function (
   fileDest: string,
   buffer: Buffer,
+  mode?: number,
 ) {
   const dir = path.dirname(fileDest)
   if (!dirs.has(dir)) {
     await fs.mkdir(dir, { recursive: true })
     dirs.add(dir)
   }
-  const fd = await fs.open(fileDest, 'w')
-  await fs.write(fd, buffer, 0, buffer.length, 0)
-  await fs.close(fd)
+  await fs.writeFile(fileDest, buffer, { mode })
 }

packages/cafs/test/index.ts

@@ -9,7 +9,7 @@ test('unpack', async (t) => {
   t.comment(dest)
   const cafs = createCafs(dest)
   await cafs.addFilesFromTarball(
-    fs.createReadStream(path.join(__dirname, '../__fixtures__/babel-helper-hoist-variables-6.24.1.tgz')),
+    fs.createReadStream(path.join(__dirname, '../__fixtures__/node-gyp-6.1.0.tgz')),
   )
   t.end()
 })

packages/fetcher-base/src/index.ts

@@ -25,6 +25,7 @@ export interface FetchResult {
 
 export interface FilesIndex {
   [filename: string]: {
+    mode: number,
     size: number,
     generatingIntegrity: Promise<Integrity>,
   },

packages/package-requester/src/packageRequester.ts

@@ -241,7 +241,7 @@ async function resolveAndFetch (
 
 function fetchToStore (
   ctx: {
-    checkFilesIntegrity: (integrity: Record<string, { size: number, integrity: string }>) => Promise<boolean>,
+    checkFilesIntegrity: (integrity: Record<string, { size: number, mode: number, integrity: string }>) => Promise<boolean>,
     fetch: (
       packageId: string,
       resolution: Resolution,
@@ -253,7 +253,7 @@ function fetchToStore (
       bundledManifest?: Promise<BundledManifest>,
       inStoreLocation: string,
     }>,
-    getFilePathInCafs: (integrity: string) => string,
+    getFilePathInCafs: (file: { mode: number, integrity: string }) => string,
     requestsQueue: {add: <T>(fn: () => Promise<T>, opts: {priority: number}) => Promise<T>},
     storeIndex: StoreIndex,
     storeDir: string,
@@ -345,7 +345,7 @@ function fetchToStore (
 
   if (opts.fetchRawManifest && !result.bundledManifest) {
     result.bundledManifest = removeKeyOnFail(
-      result.files.then(({ filesIndex }) => readBundledManifest(ctx.getFilePathInCafs(filesIndex['package.json'].integrity))),
+      result.files.then(({ filesIndex }) => readBundledManifest(ctx.getFilePathInCafs(filesIndex['package.json']))),
     )
   }
 
@@ -380,7 +380,7 @@ function fetchToStore (
       ) {
         let integrity
         try {
-          integrity = await loadJsonFile<Record<string, { size: number, integrity: string }>>(path.join(target, 'integrity.json'))
+          integrity = await loadJsonFile<Record<string, { size: number, mode: number, integrity: string }>>(path.join(target, 'integrity.json'))
         } catch (err) {
           // ignoring. It is fine if the integrity file is not present. Just refetch the package
         }
@@ -391,7 +391,7 @@ function fetchToStore (
             fromStore: true,
           })
           if (opts.fetchRawManifest) {
-            readBundledManifest(ctx.getFilePathInCafs(integrity['package.json'].integrity))
+            readBundledManifest(ctx.getFilePathInCafs(integrity['package.json']))
               .then(bundledManifest.resolve)
               .catch(bundledManifest.reject)
           }
@@ -449,6 +449,7 @@ function fetchToStore (
             const fileIntegrity = await filesIndex[filename].generatingIntegrity
             integrity[filename] = {
               integrity: fileIntegrity.toString(), // TODO: use the raw Integrity object
+              mode: filesIndex[filename].mode,
               size: filesIndex[filename].size,
             }
           }),
@@ -458,7 +459,7 @@ function fetchToStore (
 
       let pkgName: string | undefined = opts.pkgName
       if (!pkgName || opts.fetchRawManifest) {
-        const manifest = await readPackage(ctx.getFilePathInCafs(integrity['package.json'].integrity)) as DependencyManifest
+        const manifest = await readPackage(ctx.getFilePathInCafs(integrity['package.json'])) as DependencyManifest
         bundledManifest.resolve(pickBundledManifest(manifest))
         if (!pkgName) {
           pkgName = manifest.name

packages/package-requester/test/index.ts

@@ -458,7 +458,7 @@ test('fetchPackageToStore() concurrency check', async (t) => {
     const fetchResult = fetchResults[0]
     const files = await fetchResult.files()
 
-    ino1 = fs.statSync(getFilePathInCafs(cafsDir, files.filesIndex['package.json'].integrity)).ino
+    ino1 = fs.statSync(getFilePathInCafs(cafsDir, files.filesIndex['package.json'])).ino
 
     t.deepEqual(Object.keys(files.filesIndex).sort(),
       ['package.json', 'index.js', 'license', 'readme.md'].sort(),
@@ -473,7 +473,7 @@ test('fetchPackageToStore() concurrency check', async (t) => {
     const fetchResult = fetchResults[1]
     const files = await fetchResult.files()
 
-    ino2 = fs.statSync(getFilePathInCafs(cafsDir, files.filesIndex['package.json'].integrity)).ino
+    ino2 = fs.statSync(getFilePathInCafs(cafsDir, files.filesIndex['package.json'])).ino
 
     t.deepEqual(Object.keys(files.filesIndex).sort(),
       ['package.json', 'index.js', 'license', 'readme.md'].sort(),
@@ -682,7 +682,7 @@ test('refetch package to store if it has been modified', async (t) => {
     })
 
     const { filesIndex } = await fetchResult.files()
-    indexJsFile = getFilePathInCafs(cafsDir, filesIndex['index.js'].integrity)
+    indexJsFile = getFilePathInCafs(cafsDir, filesIndex['index.js'])
   }
 
   // Adding some content to the file to change its integrity

packages/package-store/src/storeController/index.ts

@@ -62,8 +62,8 @@ export default async function (
   const getFilePathInCafs = _getFilePathInCafs.bind(null, cafsDir)
   const importPackage: ImportPackageFunction = (to, opts) => {
     const filesMap = {} as Record<string, string>
-    for (const [fileName, { integrity }] of Object.entries(opts.filesResponse.filesIndex)) {
-      filesMap[fileName] = getFilePathInCafs(integrity)
+    for (const [fileName, fileMeta] of Object.entries(opts.filesResponse.filesIndex)) {
+      filesMap[fileName] = getFilePathInCafs(fileMeta)
     }
     return impPkg(to, { filesMap, fromStore: opts.filesResponse.fromStore, force: opts.force })
   }
@@ -177,6 +177,7 @@ export default async function (
           const fileIntegrity = await filesIndex[filename].generatingIntegrity
           integrity[filename] = {
             integrity: fileIntegrity.toString(), // TODO: use the raw Integrity object
+            mode: filesIndex[filename].mode,
             size: filesIndex[filename].size,
           }
         }),

packages/plugin-commands-installation/test/update/interactive.ts

@@ -53,11 +53,13 @@ test('interactively update', async (t) => {
     },
   })
 
+  const storeDir = path.resolve('pnpm-store')
   await add.handler({
     ...DEFAULT_OPTIONS,
     dir: process.cwd(),
     linkWorkspacePackages: true,
     save: false,
+    storeDir,
   }, [
     'is-negative@1.0.0',
     'is-positive@2.0.0',
@@ -74,6 +76,7 @@ test('interactively update', async (t) => {
     dir: process.cwd(),
     interactive: true,
     linkWorkspacePackages: true,
+    storeDir,
   })
 
   t.ok(prompt.calledWithMatch({
@@ -111,6 +114,7 @@ test('interactively update', async (t) => {
     interactive: true,
     latest: true,
     linkWorkspacePackages: true,
+    storeDir,
   })
 
   t.ok(prompt.calledWithMatch({

packages/store-controller-types/src/index.ts

@@ -85,7 +85,7 @@ export type ImportPackageFunction = (
 
 export interface PackageFilesResponse {
   fromStore: boolean,
-  filesIndex: Record<string, { integrity: string }>,
+  filesIndex: Record<string, { mode: number, integrity: string }>,
 }
 
 export type RequestPackageFunction = (

packages/supi/test/install/lifecycleScripts.ts

@@ -236,7 +236,7 @@ test("reports child's close event", async (t: tape.Test) => {
   }
 })
 
-test.skip('lifecycle scripts have access to node-gyp', async (t: tape.Test) => {
+test('lifecycle scripts have access to node-gyp', async (t: tape.Test) => {
   prepareEmpty(t)
 
   // `npm test` adds node-gyp to the PATH

packages/supi/test/install/misc.ts

@@ -554,7 +554,7 @@ test('bin specified in the directories property linked to .bin folder', async (t
   await project.isExecutable('.bin/pkg-with-directories-bin')
 })
 
-test.skip('building native addons', async (t: tape.Test) => {
+test('building native addons', async (t: tape.Test) => {
   const project = prepareEmpty(t)
 
   await addDependenciesToPackage({}, ['diskusage@1.1.3'], await testDefaults({ fastUnpack: false }))

packages/supi/test/install/sideEffects.ts

@@ -123,7 +123,7 @@ test.skip('readonly side effects cache', async (t) => {
   t.notOk(await exists(path.join(opts2.storeDir, `localhost+${REGISTRY_MOCK_PORT}/diskusage/1.1.2/side_effects/${ENGINE_DIR}/package/build`)), 'cache folder not created')
 })
 
-test.skip('uploading errors do not interrupt installation', async (t) => {
+test('uploading errors do not interrupt installation', async (t) => {
   prepareEmpty(t)
 
   const opts = await testDefaults({

packages/tarball-fetcher/test/download.ts

@@ -108,7 +108,7 @@ test('redownload the tarball when the one in cache does not satisfy integrity',
   streamParser.removeListener('data', reporter as any) // tslint:disable-line:no-any
 
   const pkgJsonIntegrity = await filesIndex['package.json'].generatingIntegrity
-  t.equal((await readPackage(getFilePathInCafs(pkgJsonIntegrity))).version, '6.24.1')
+  t.equal((await readPackage(getFilePathInCafs({ integrity: pkgJsonIntegrity, ...filesIndex['package.json'] }))).version, '6.24.1')
 
   t.ok(scope.isDone())
   t.end()