fix: solve circular dependencies issue in the workspace

This commit is contained in:
Zoltan Kochan
2022-04-30 03:44:06 +03:00
parent 41f8281637
commit 1ceb632b1c
29 changed files with 282 additions and 126 deletions

View File

@@ -0,0 +1,5 @@
---
"@pnpm/create-cafs-store": major
---
Project created.

1
.gitignore vendored
View File

@@ -42,3 +42,4 @@ yarn.lock
RELEASE.md
.jest-cache
.turbo

View File

@@ -0,0 +1,13 @@
# @pnpm/create-cafs-store
> Create a CAFS store controller
## Installation
```
pnpm add @pnpm/create-cafs-store
```
## License
[MIT](LICENSE)

View File

@@ -0,0 +1,3 @@
const config = require('../../jest.config.js');
module.exports = Object.assign({}, config, {});

View File

@@ -0,0 +1,69 @@
{
"name": "@pnpm/create-cafs-store",
"description": "Create a CAFS store controller",
"version": "0.0.0",
"bugs": {
"url": "https://github.com/pnpm/pnpm/issues"
},
"main": "lib/index.js",
"types": "lib/index.d.ts",
"files": [
"lib",
"!*.map"
],
"peerDependencies": {
"@pnpm/logger": "^4.0.0"
},
"dependencies": {
"@pnpm/cafs": "workspace:4.0.0",
"@pnpm/core-loggers": "workspace:7.0.0",
"@pnpm/fetcher-base": "workspace:12.0.0",
"@pnpm/store-controller-types": "workspace:13.0.0",
"@zkochan/rimraf": "^2.1.2",
"make-empty-dir": "^2.0.0",
"mem": "^8.0.0",
"p-limit": "^3.1.0",
"path-exists": "^4.0.0",
"path-temp": "^2.0.0",
"rename-overwrite": "^4.0.2",
"sanitize-filename": "^1.6.3"
},
"devDependencies": {
"@pnpm/logger": "^4.0.0",
"@pnpm/create-cafs-store": "workspace:0.0.0",
"@pnpm/prepare": "workspace:*"
},
"directories": {
"test": "test"
},
"homepage": "https://github.com/pnpm/pnpm/blob/main/packages/create-cafs-store#readme",
"keywords": [
"pnpm7",
"store",
"storage",
"global store",
"maching store",
"central storage",
"cache",
"packages"
],
"license": "MIT",
"engines": {
"node": ">=14.19"
},
"repository": "https://github.com/pnpm/pnpm/blob/main/packages/create-cafs-store",
"scripts": {
"start": "tsc --watch",
"fix": "tslint -c tslint.json src/**/*.ts test/**/*.ts --fix",
"lint": "eslint src/**/*.ts test/**/*.ts",
"pretest": "rimraf .tmp",
"_test": "pnpm pretest && jest",
"test": "pnpm run compile && pnpm run _test",
"prepublishOnly": "pnpm run compile",
"compile": "tsc --build && pnpm run lint --fix"
},
"funding": "https://opencollective.com/pnpm",
"exports": {
".": "./lib/index.js"
}
}

View File

@@ -4,7 +4,7 @@ import { globalInfo, globalWarn } from '@pnpm/logger'
import { packageImportMethodLogger } from '@pnpm/core-loggers'
import pLimit from 'p-limit'
import exists from 'path-exists'
import importIndexedDir, { ImportFile } from '../fs/importIndexedDir'
import importIndexedDir, { ImportFile } from './importIndexedDir'
const limitLinking = pLimit(16)

View File

@@ -0,0 +1,81 @@
import { promises as fs } from 'fs'
import path from 'path'
import createCafs, {
getFilePathByModeInCafs,
} from '@pnpm/cafs'
import { PackageFilesResponse } from '@pnpm/fetcher-base'
import {
ImportPackageFunction,
PackageFileInfo,
} from '@pnpm/store-controller-types'
import memoize from 'mem'
import pathTemp from 'path-temp'
import createImportPackage from './createImportPackage'
function createPackageImporter (
opts: {
packageImportMethod?: 'auto' | 'hardlink' | 'copy' | 'clone'
cafsDir: string
}
): ImportPackageFunction {
const cachedImporterCreator = memoize(createImportPackage)
const packageImportMethod = opts.packageImportMethod
const gfm = getFlatMap.bind(null, opts.cafsDir)
return async (to, opts) => {
const { filesMap, isBuilt } = gfm(opts.filesResponse, opts.targetEngine)
const impPkg = cachedImporterCreator(opts.filesResponse.packageImportMethod ?? packageImportMethod)
const importMethod = await impPkg(to, { filesMap, fromStore: opts.filesResponse.fromStore, force: opts.force })
return { importMethod, isBuilt }
}
}
function getFlatMap (
cafsDir: string,
filesResponse: PackageFilesResponse,
targetEngine?: string
): { filesMap: Record<string, string>, isBuilt: boolean } {
if (filesResponse.local) {
return {
filesMap: filesResponse.filesIndex,
isBuilt: false,
}
}
let isBuilt!: boolean
let filesIndex!: Record<string, PackageFileInfo>
if (targetEngine && ((filesResponse.sideEffects?.[targetEngine]) != null)) {
filesIndex = filesResponse.sideEffects?.[targetEngine]
isBuilt = true
} else {
filesIndex = filesResponse.filesIndex
isBuilt = false
}
const filesMap = {}
for (const [fileName, fileMeta] of Object.entries(filesIndex)) {
filesMap[fileName] = getFilePathByModeInCafs(cafsDir, fileMeta.integrity, fileMeta.mode)
}
return { filesMap, isBuilt }
}
export default function createCafsStore (
storeDir: string,
opts?: {
ignoreFile?: (filename: string) => boolean
packageImportMethod?: 'auto' | 'hardlink' | 'copy' | 'clone'
}
) {
const cafsDir = path.join(storeDir, 'files')
const baseTempDir = path.join(storeDir, 'tmp')
const importPackage = createPackageImporter({
packageImportMethod: opts?.packageImportMethod,
cafsDir,
})
return {
...createCafs(cafsDir, opts?.ignoreFile),
importPackage,
tempDir: async () => {
const tmpDir = pathTemp(baseTempDir)
await fs.mkdir(tmpDir, { recursive: true })
return tmpDir
},
}
}

View File

@@ -19,7 +19,7 @@ baseLogger['globalWarn'] = globalWarn
jest.mock('@pnpm/logger', () => baseLogger)
// eslint-disable-next-line
import createImportPackage from '@pnpm/package-store/lib/storeController/createImportPackage'
import createImportPackage from '@pnpm/create-cafs-store/lib/createImportPackage'
test('packageImportMethod=auto: clone files by default', async () => {
const importPackage = createImportPackage('auto')

View File

@@ -1,6 +1,6 @@
import { promises as fs } from 'fs'
import path from 'path'
import createImportPackage from '@pnpm/package-store/lib/storeController/createImportPackage'
import createImportPackage from '@pnpm/create-cafs-store/lib/createImportPackage'
import { prepareEmpty } from '@pnpm/prepare'
test('importing a package with invalid files', async () => {

View File

@@ -0,0 +1,28 @@
{
"extends": "@pnpm/tsconfig",
"compilerOptions": {
"outDir": "lib",
"rootDir": "src"
},
"include": [
"src/**/*.ts",
"../../typings/**/*.d.ts"
],
"references": [
{
"path": "../../privatePackages/prepare"
},
{
"path": "../cafs"
},
{
"path": "../core-loggers"
},
{
"path": "../fetcher-base"
},
{
"path": "../store-controller-types"
}
]
}

View File

@@ -0,0 +1,8 @@
{
"extends": "./tsconfig.json",
"include": [
"src/**/*.ts",
"test/**/*.ts",
"../../typings/**/*.d.ts"
]
}

View File

@@ -38,7 +38,7 @@
"devDependencies": {
"@pnpm/cafs": "workspace:4.0.0",
"@pnpm/git-fetcher": "workspace:5.1.0",
"@pnpm/package-store": "workspace:13.0.1",
"@pnpm/create-cafs-store": "workspace:0.0.0",
"@pnpm/types": "workspace:8.0.0",
"p-defer": "^3.0.0",
"tempy": "^1.0.0"

View File

@@ -1,6 +1,6 @@
/// <reference path="../../../typings/index.d.ts"/>
import path from 'path'
import { createCafsStore } from '@pnpm/package-store'
import createCafsStore from '@pnpm/create-cafs-store'
import createFetcher from '@pnpm/git-fetcher'
import { DependencyManifest } from '@pnpm/types'
import pDefer from 'p-defer'

View File

@@ -12,6 +12,9 @@
{
"path": "../cafs"
},
{
"path": "../create-cafs-store"
},
{
"path": "../fetcher-base"
},

View File

@@ -64,7 +64,7 @@
"@pnpm/client": "workspace:7.1.0",
"@pnpm/logger": "^4.0.0",
"@pnpm/package-requester": "workspace:18.0.1",
"@pnpm/package-store": "workspace:13.0.1",
"@pnpm/create-cafs-store": "workspace:0.0.0",
"@pnpm/registry-mock": "2.15.1",
"@pnpm/test-fixtures": "workspace:*",
"@types/normalize-path": "^3.0.0",

View File

@@ -5,7 +5,7 @@ import { getFilePathInCafs, PackageFilesIndex } from '@pnpm/cafs'
import createClient from '@pnpm/client'
import { streamParser } from '@pnpm/logger'
import createPackageRequester, { PackageFilesResponse, PackageResponse } from '@pnpm/package-requester'
import { createCafsStore } from '@pnpm/package-store'
import createCafsStore from '@pnpm/create-cafs-store'
import { REGISTRY_MOCK_PORT } from '@pnpm/registry-mock'
import fixtures from '@pnpm/test-fixtures'
import { DependencyManifest } from '@pnpm/types'

View File

@@ -21,6 +21,9 @@
{
"path": "../core-loggers"
},
{
"path": "../create-cafs-store"
},
{
"path": "../dependency-path"
},

View File

@@ -16,7 +16,7 @@
},
"dependencies": {
"@pnpm/cafs": "workspace:4.0.0",
"@pnpm/core-loggers": "workspace:7.0.0",
"@pnpm/create-cafs-store": "workspace:0.0.0",
"@pnpm/fetcher-base": "workspace:12.0.0",
"@pnpm/package-requester": "workspace:18.0.1",
"@pnpm/resolver-base": "workspace:9.0.0",
@@ -24,14 +24,7 @@
"@pnpm/types": "workspace:8.0.0",
"@zkochan/rimraf": "^2.1.2",
"load-json-file": "^6.2.0",
"make-empty-dir": "^2.0.0",
"mem": "^8.0.0",
"p-limit": "^3.1.0",
"path-exists": "^4.0.0",
"path-temp": "^2.0.0",
"ramda": "^0.27.1",
"rename-overwrite": "^4.0.2",
"sanitize-filename": "^1.6.3",
"ssri": "^8.0.1",
"write-json-file": "^4.3.0"
},

View File

@@ -1,6 +1,5 @@
import createStore, { createCafsStore } from './storeController'
import createStore from './storeController'
export default createStore
export { createCafsStore }
export * from '@pnpm/store-controller-types'

View File

@@ -1,92 +1,18 @@
import { promises as fs } from 'fs'
import path from 'path'
import createCafs, {
getFilePathByModeInCafs,
import {
PackageFilesIndex,
} from '@pnpm/cafs'
import { FetchFunction, PackageFilesResponse } from '@pnpm/fetcher-base'
import createCafsStore from '@pnpm/create-cafs-store'
import { FetchFunction } from '@pnpm/fetcher-base'
import createPackageRequester from '@pnpm/package-requester'
import { ResolveFunction } from '@pnpm/resolver-base'
import {
ImportPackageFunction,
PackageFileInfo,
StoreController,
} from '@pnpm/store-controller-types'
import loadJsonFile from 'load-json-file'
import memoize from 'mem'
import pathTemp from 'path-temp'
import writeJsonFile from 'write-json-file'
import createImportPackage from './createImportPackage'
import prune from './prune'
function createPackageImporter (
opts: {
packageImportMethod?: 'auto' | 'hardlink' | 'copy' | 'clone'
cafsDir: string
}
): ImportPackageFunction {
const cachedImporterCreator = memoize(createImportPackage)
const packageImportMethod = opts.packageImportMethod
const gfm = getFlatMap.bind(null, opts.cafsDir)
return async (to, opts) => {
const { filesMap, isBuilt } = gfm(opts.filesResponse, opts.targetEngine)
const impPkg = cachedImporterCreator(opts.filesResponse.packageImportMethod ?? packageImportMethod)
const importMethod = await impPkg(to, { filesMap, fromStore: opts.filesResponse.fromStore, force: opts.force })
return { importMethod, isBuilt }
}
}
function getFlatMap (
cafsDir: string,
filesResponse: PackageFilesResponse,
targetEngine?: string
): { filesMap: Record<string, string>, isBuilt: boolean } {
if (filesResponse.local) {
return {
filesMap: filesResponse.filesIndex,
isBuilt: false,
}
}
let isBuilt!: boolean
let filesIndex!: Record<string, PackageFileInfo>
if (targetEngine && ((filesResponse.sideEffects?.[targetEngine]) != null)) {
filesIndex = filesResponse.sideEffects?.[targetEngine]
isBuilt = true
} else {
filesIndex = filesResponse.filesIndex
isBuilt = false
}
const filesMap = {}
for (const [fileName, fileMeta] of Object.entries(filesIndex)) {
filesMap[fileName] = getFilePathByModeInCafs(cafsDir, fileMeta.integrity, fileMeta.mode)
}
return { filesMap, isBuilt }
}
export function createCafsStore (
storeDir: string,
opts?: {
ignoreFile?: (filename: string) => boolean
packageImportMethod?: 'auto' | 'hardlink' | 'copy' | 'clone'
}
) {
const cafsDir = path.join(storeDir, 'files')
const baseTempDir = path.join(storeDir, 'tmp')
const importPackage = createPackageImporter({
packageImportMethod: opts?.packageImportMethod,
cafsDir,
})
return {
...createCafs(cafsDir, opts?.ignoreFile),
importPackage,
tempDir: async () => {
const tmpDir = pathTemp(baseTempDir)
await fs.mkdir(tmpDir, { recursive: true })
return tmpDir
},
}
}
export default async function (
resolve: ResolveFunction,
fetchers: {[type: string]: FetchFunction},

View File

@@ -19,7 +19,7 @@
"path": "../client"
},
{
"path": "../core-loggers"
"path": "../create-cafs-store"
},
{
"path": "../fetcher-base"

View File

@@ -35,7 +35,7 @@
"@pnpm/error": "workspace:3.0.1",
"@pnpm/fetch": "workspace:5.0.0",
"@pnpm/fetcher-base": "workspace:12.0.0",
"@pnpm/package-store": "workspace:13.0.1",
"@pnpm/create-cafs-store": "workspace:0.0.0",
"@pnpm/store-path": "workspace:6.0.0",
"@pnpm/tarball-fetcher": "workspace:10.0.1",
"@zkochan/cmd-shim": "^5.2.2",

View File

@@ -3,7 +3,7 @@ import path from 'path'
import { Config } from '@pnpm/config'
import { createFetchFromRegistry, FetchFromRegistry } from '@pnpm/fetch'
import { FilesIndex } from '@pnpm/fetcher-base'
import { createCafsStore } from '@pnpm/package-store'
import createCafsStore from '@pnpm/create-cafs-store'
import storePath from '@pnpm/store-path'
import createFetcher, { waitForFilesIndex } from '@pnpm/tarball-fetcher'
import AdmZip from 'adm-zip'

View File

@@ -18,6 +18,9 @@
{
"path": "../config"
},
{
"path": "../create-cafs-store"
},
{
"path": "../error"
},
@@ -27,9 +30,6 @@
{
"path": "../fetcher-base"
},
{
"path": "../package-store"
},
{
"path": "../store-path"
},

View File

@@ -47,7 +47,7 @@
"devDependencies": {
"@pnpm/fetch": "workspace:5.0.0",
"@pnpm/logger": "^4.0.0",
"@pnpm/package-store": "workspace:13.0.1",
"@pnpm/create-cafs-store": "workspace:0.0.0",
"@pnpm/tarball-fetcher": "workspace:10.0.1",
"@pnpm/test-fixtures": "workspace:*",
"@types/ramda": "0.27.39",

View File

@@ -3,7 +3,7 @@ import fs from 'fs'
import path from 'path'
import PnpmError, { FetchError } from '@pnpm/error'
import { createFetchFromRegistry } from '@pnpm/fetch'
import { createCafsStore } from '@pnpm/package-store'
import createCafsStore from '@pnpm/create-cafs-store'
import fixtures from '@pnpm/test-fixtures'
import createFetcher, {
BadTarballError,

View File

@@ -15,6 +15,9 @@
{
"path": "../core-loggers"
},
{
"path": "../create-cafs-store"
},
{
"path": "../error"
},

69
pnpm-lock.yaml generated
View File

@@ -559,6 +559,41 @@ importers:
'@pnpm/core-loggers': 'link:'
'@pnpm/logger': 4.0.0
packages/create-cafs-store:
specifiers:
'@pnpm/cafs': workspace:4.0.0
'@pnpm/core-loggers': workspace:7.0.0
'@pnpm/create-cafs-store': workspace:0.0.0
'@pnpm/fetcher-base': workspace:12.0.0
'@pnpm/logger': ^4.0.0
'@pnpm/prepare': workspace:*
'@pnpm/store-controller-types': workspace:13.0.0
'@zkochan/rimraf': ^2.1.2
make-empty-dir: ^2.0.0
mem: ^8.0.0
p-limit: ^3.1.0
path-exists: ^4.0.0
path-temp: ^2.0.0
rename-overwrite: ^4.0.2
sanitize-filename: ^1.6.3
dependencies:
'@pnpm/cafs': link:../cafs
'@pnpm/core-loggers': link:../core-loggers
'@pnpm/fetcher-base': link:../fetcher-base
'@pnpm/store-controller-types': link:../store-controller-types
'@zkochan/rimraf': 2.1.2
make-empty-dir: 2.0.0
mem: 8.1.1
p-limit: 3.1.0
path-exists: 4.0.0
path-temp: 2.0.0
rename-overwrite: 4.0.2
sanitize-filename: 1.6.3
devDependencies:
'@pnpm/create-cafs-store': 'link:'
'@pnpm/logger': 4.0.0
'@pnpm/prepare': link:../../privatePackages/prepare
packages/default-reporter:
specifiers:
'@pnpm/config': workspace:15.1.0
@@ -986,9 +1021,9 @@ importers:
packages/git-fetcher:
specifiers:
'@pnpm/cafs': workspace:4.0.0
'@pnpm/create-cafs-store': workspace:0.0.0
'@pnpm/fetcher-base': workspace:12.0.0
'@pnpm/git-fetcher': workspace:5.1.0
'@pnpm/package-store': workspace:13.0.1
'@pnpm/prepare-package': workspace:2.0.1
'@pnpm/types': workspace:8.0.0
'@zkochan/rimraf': ^2.1.2
@@ -1002,8 +1037,8 @@ importers:
execa: /safe-execa/0.1.1
devDependencies:
'@pnpm/cafs': link:../cafs
'@pnpm/create-cafs-store': link:../create-cafs-store
'@pnpm/git-fetcher': 'link:'
'@pnpm/package-store': link:../package-store
'@pnpm/types': link:../types
p-defer: 3.0.0
tempy: 1.0.1
@@ -1816,13 +1851,13 @@ importers:
'@pnpm/cafs': workspace:4.0.0
'@pnpm/client': workspace:7.1.0
'@pnpm/core-loggers': workspace:7.0.0
'@pnpm/create-cafs-store': workspace:0.0.0
'@pnpm/error': workspace:3.0.1
'@pnpm/fetcher-base': workspace:12.0.0
'@pnpm/graceful-fs': workspace:2.0.0
'@pnpm/logger': ^4.0.0
'@pnpm/package-is-installable': workspace:6.0.1
'@pnpm/package-requester': workspace:18.0.1
'@pnpm/package-store': workspace:13.0.1
'@pnpm/read-package-json': workspace:6.0.1
'@pnpm/registry-mock': 2.15.1
'@pnpm/resolver-base': workspace:9.0.0
@@ -1872,9 +1907,9 @@ importers:
ssri: 8.0.1
devDependencies:
'@pnpm/client': link:../client
'@pnpm/create-cafs-store': link:../create-cafs-store
'@pnpm/logger': 4.0.0
'@pnpm/package-requester': 'link:'
'@pnpm/package-store': link:../package-store
'@pnpm/registry-mock': 2.15.1
'@pnpm/test-fixtures': link:../../privatePackages/test-fixtures
'@types/normalize-path': 3.0.0
@@ -1890,7 +1925,7 @@ importers:
specifiers:
'@pnpm/cafs': workspace:4.0.0
'@pnpm/client': workspace:7.1.0
'@pnpm/core-loggers': workspace:7.0.0
'@pnpm/create-cafs-store': workspace:0.0.0
'@pnpm/fetcher-base': workspace:12.0.0
'@pnpm/logger': ^4.0.0
'@pnpm/package-requester': workspace:18.0.1
@@ -1903,20 +1938,13 @@ importers:
'@types/ssri': ^7.1.0
'@zkochan/rimraf': ^2.1.2
load-json-file: ^6.2.0
make-empty-dir: ^2.0.0
mem: ^8.0.0
p-limit: ^3.1.0
path-exists: ^4.0.0
path-temp: ^2.0.0
ramda: ^0.27.1
rename-overwrite: ^4.0.2
sanitize-filename: ^1.6.3
ssri: ^8.0.1
tempy: ^1.0.0
write-json-file: ^4.3.0
dependencies:
'@pnpm/cafs': link:../cafs
'@pnpm/core-loggers': link:../core-loggers
'@pnpm/create-cafs-store': link:../create-cafs-store
'@pnpm/fetcher-base': link:../fetcher-base
'@pnpm/package-requester': link:../package-requester
'@pnpm/resolver-base': link:../resolver-base
@@ -1924,14 +1952,7 @@ importers:
'@pnpm/types': link:../types
'@zkochan/rimraf': 2.1.2
load-json-file: 6.2.0
make-empty-dir: 2.0.0
mem: 8.1.1
p-limit: 3.1.0
path-exists: 4.0.0
path-temp: 2.0.0
ramda: 0.27.2
rename-overwrite: 4.0.2
sanitize-filename: 1.6.3
ssri: 8.0.1
write-json-file: 4.3.0
devDependencies:
@@ -2057,10 +2078,10 @@ importers:
specifiers:
'@pnpm/cli-utils': workspace:0.7.2
'@pnpm/config': workspace:15.1.0
'@pnpm/create-cafs-store': workspace:0.0.0
'@pnpm/error': workspace:3.0.1
'@pnpm/fetch': workspace:5.0.0
'@pnpm/fetcher-base': workspace:12.0.0
'@pnpm/package-store': workspace:13.0.1
'@pnpm/plugin-commands-env': workspace:2.1.1
'@pnpm/prepare': workspace:*
'@pnpm/store-path': workspace:6.0.0
@@ -2083,10 +2104,10 @@ importers:
dependencies:
'@pnpm/cli-utils': link:../cli-utils
'@pnpm/config': link:../config
'@pnpm/create-cafs-store': link:../create-cafs-store
'@pnpm/error': link:../error
'@pnpm/fetch': link:../fetch
'@pnpm/fetcher-base': link:../fetcher-base
'@pnpm/package-store': link:../package-store
'@pnpm/store-path': link:../store-path
'@pnpm/tarball-fetcher': link:../tarball-fetcher
'@zkochan/cmd-shim': 5.2.2
@@ -3402,13 +3423,13 @@ importers:
packages/tarball-fetcher:
specifiers:
'@pnpm/core-loggers': workspace:7.0.0
'@pnpm/create-cafs-store': workspace:0.0.0
'@pnpm/error': workspace:3.0.1
'@pnpm/fetch': workspace:5.0.0
'@pnpm/fetcher-base': workspace:12.0.0
'@pnpm/fetching-types': workspace:3.0.0
'@pnpm/graceful-fs': workspace:2.0.0
'@pnpm/logger': ^4.0.0
'@pnpm/package-store': workspace:13.0.1
'@pnpm/prepare-package': workspace:2.0.1
'@pnpm/tarball-fetcher': workspace:10.0.1
'@pnpm/test-fixtures': workspace:*
@@ -3431,9 +3452,9 @@ importers:
ramda: 0.27.2
ssri: 8.0.1
devDependencies:
'@pnpm/create-cafs-store': link:../create-cafs-store
'@pnpm/fetch': link:../fetch
'@pnpm/logger': 4.0.0
'@pnpm/package-store': link:../package-store
'@pnpm/tarball-fetcher': 'link:'
'@pnpm/test-fixtures': link:../../privatePackages/test-fixtures
'@types/ramda': 0.27.39