fix: the file protocol should link all files by default (#4532)

close #4510

.changeset/shaggy-grapes-give.md

@@ -0,0 +1,5 @@
+---
+"@pnpm/directory-fetcher": major
+---
+
+Fetch all files from the directory by default.

packages/directory-fetcher/src/index.ts

@@ -1,3 +1,4 @@
+import { promises as fs } from 'fs'
 import path from 'path'
 import { Cafs, DeferredManifestPromise } from '@pnpm/fetcher-base'
 import { safeReadProjectManifestOnly } from '@pnpm/read-project-manifest'
@@ -10,7 +11,14 @@ export interface DirectoryFetcherOptions {
   manifest?: DeferredManifestPromise
 }
 
-export default () => {
+export interface CreateDirectoryFetcherOptions {
+  includeOnlyPackageFiles?: boolean
+}
+
+export default (
+  opts?: CreateDirectoryFetcherOptions
+) => {
+  const fetchFromDir = opts?.includeOnlyPackageFiles ? fetchPackageFilesFromDir : fetchAllFilesFromDir
   return {
     directory: (
       cafs: Cafs,
@@ -23,9 +31,63 @@ export default () => {
   }
 }
 
+type FetchFromDirOpts = Omit<DirectoryFetcherOptions, 'lockfileDir'>
+
 export async function fetchFromDir (
   dir: string,
-  opts: Omit<DirectoryFetcherOptions, 'lockfileDir'>
+  opts: FetchFromDirOpts & CreateDirectoryFetcherOptions
+) {
+  if (opts.includeOnlyPackageFiles) {
+    return fetchPackageFilesFromDir(dir, opts)
+  }
+  return fetchAllFilesFromDir(dir, opts)
+}
+
+async function fetchAllFilesFromDir (
+  dir: string,
+  opts: FetchFromDirOpts
+) {
+  const filesIndex = await _fetchAllFilesFromDir(dir)
+  if (opts.manifest) {
+    // In a regular pnpm workspace it will probably never happen that a dependency has no package.json file.
+    // Safe read was added to support the Bit workspace in which the components have no package.json files.
+    // Related PR in Bit: https://github.com/teambit/bit/pull/5251
+    const manifest = await safeReadProjectManifestOnly(dir) ?? {}
+    opts.manifest.resolve(manifest as any) // eslint-disable-line @typescript-eslint/no-explicit-any
+  }
+  return {
+    local: true as const,
+    filesIndex,
+    packageImportMethod: 'hardlink' as const,
+  }
+}
+
+async function _fetchAllFilesFromDir (
+  dir: string,
+  relativeDir = ''
+): Promise<Record<string, string>> {
+  const filesIndex: Record<string, string> = {}
+  const files = await fs.readdir(dir)
+  await Promise.all(files
+    .filter((file) => file !== 'node_modules')
+    .map(async (file) => {
+      const filePath = path.join(dir, file)
+      const stat = await fs.stat(filePath)
+      const relativeSubdir = `${relativeDir}${relativeDir ? '/' : ''}${file}`
+      if (stat.isDirectory()) {
+        const subFilesIndex = await _fetchAllFilesFromDir(filePath, relativeSubdir)
+        Object.assign(filesIndex, subFilesIndex)
+      } else {
+        filesIndex[relativeSubdir] = filePath
+      }
+    })
+  )
+  return filesIndex
+}
+
+async function fetchPackageFilesFromDir (
+  dir: string,
+  opts: FetchFromDirOpts
 ) {
   const files = await packlist({ path: dir })
   const filesIndex: Record<string, string> = fromPairs(files.map((file) => [file, path.join(dir, file)]))

packages/directory-fetcher/test/index.ts

@@ -5,7 +5,30 @@ import fixtures from '@pnpm/test-fixtures'
 
 const f = fixtures(__dirname)
 
-test('fetch', async () => {
+test('fetch including only package files', async () => {
+  process.chdir(f.find('simple-pkg'))
+  const fetcher = createFetcher({ includeOnlyPackageFiles: true })
+
+  // eslint-disable-next-line
+  const fetchResult = await fetcher.directory({} as any, {
+    directory: '.',
+    type: 'directory',
+  }, {
+    lockfileDir: process.cwd(),
+  })
+
+  expect(fetchResult.local).toBe(true)
+  expect(fetchResult.packageImportMethod).toBe('hardlink')
+  expect(fetchResult.filesIndex['package.json']).toBe(path.resolve('package.json'))
+
+  // Only those files are included which would get published
+  expect(Object.keys(fetchResult.filesIndex).sort()).toStrictEqual([
+    'index.js',
+    'package.json',
+  ])
+})
+
+test('fetch including all files', async () => {
   process.chdir(f.find('simple-pkg'))
   const fetcher = createFetcher()
 
@@ -25,6 +48,7 @@ test('fetch', async () => {
   expect(Object.keys(fetchResult.filesIndex).sort()).toStrictEqual([
     'index.js',
     'package.json',
+    'test.js',
   ])
 })