mirror of
https://github.com/pnpm/pnpm.git
synced 2025-12-24 07:38:12 -05:00
refactor: package requester (#7036)
ref https://github.com/pnpm/pnpm/issues/6808
This commit is contained in:
9
.changeset/mean-hotels-raise.md
Normal file
9
.changeset/mean-hotels-raise.md
Normal file
@@ -0,0 +1,9 @@
|
||||
---
|
||||
"@pnpm/package-requester": major
|
||||
"@pnpm/store-controller-types": major
|
||||
"@pnpm/package-store": major
|
||||
"@pnpm/deps.graph-builder": minor
|
||||
"@pnpm/server": major
|
||||
---
|
||||
|
||||
Breaking changes to the API.
|
||||
10
deps/graph-builder/src/lockfileToDepGraph.ts
vendored
10
deps/graph-builder/src/lockfileToDepGraph.ts
vendored
@@ -18,8 +18,8 @@ import { type IncludedDependencies } from '@pnpm/modules-yaml'
|
||||
import { packageIsInstallable } from '@pnpm/package-is-installable'
|
||||
import { type PatchFile, type Registries } from '@pnpm/types'
|
||||
import {
|
||||
type PkgRequestFetchResult,
|
||||
type FetchPackageToStoreFunction,
|
||||
type PackageFilesResponse,
|
||||
type StoreController,
|
||||
} from '@pnpm/store-controller-types'
|
||||
import * as dp from '@pnpm/dependency-path'
|
||||
@@ -33,8 +33,7 @@ export interface DependenciesGraphNode {
|
||||
hasBundledDependencies: boolean
|
||||
modules: string
|
||||
name: string
|
||||
fetchingFiles: () => Promise<PackageFilesResponse>
|
||||
finishing: () => Promise<void>
|
||||
fetching: () => Promise<PkgRequestFetchResult>
|
||||
dir: string
|
||||
children: Record<string, string>
|
||||
optionalDependencies: Set<string>
|
||||
@@ -161,7 +160,7 @@ export async function lockfileToDepGraph (
|
||||
name: pkgName,
|
||||
version: pkgVersion,
|
||||
},
|
||||
})
|
||||
}) as any // eslint-disable-line
|
||||
if (fetchResponse instanceof Promise) fetchResponse = await fetchResponse
|
||||
} catch (err: any) { // eslint-disable-line
|
||||
if (pkgSnapshot.optional) return
|
||||
@@ -171,9 +170,8 @@ export async function lockfileToDepGraph (
|
||||
children: {},
|
||||
depPath,
|
||||
dir,
|
||||
fetchingFiles: fetchResponse.files,
|
||||
fetching: fetchResponse.fetching,
|
||||
filesIndexFile: fetchResponse.filesIndexFile,
|
||||
finishing: fetchResponse.finishing,
|
||||
hasBin: pkgSnapshot.hasBin === true,
|
||||
hasBundledDependencies: pkgSnapshot.bundledDependencies != null,
|
||||
modules,
|
||||
|
||||
@@ -20,9 +20,9 @@ export async function writePackage (dep: ParseWantedDependencyResult, dest: stri
|
||||
projectDir: opts.dir,
|
||||
registry: (dep.alias && pickRegistryForPackage(opts.registries, dep.alias)) ?? opts.registries.default,
|
||||
})
|
||||
const filesResponse = await pkgResponse.files!()
|
||||
const { files } = await pkgResponse.fetching!()
|
||||
await store.ctrl.importPackage(dest, {
|
||||
filesResponse,
|
||||
filesResponse: files,
|
||||
force: true,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1226,14 +1226,14 @@ const _installInContext: InstallFunction = async (projects, ctx, opts) => {
|
||||
),
|
||||
...linkedDependenciesByProjectId[project.id].map(({ pkgId }) => ({
|
||||
dir: path.join(project.rootDir, pkgId.substring(5)),
|
||||
fetchingBundledManifest: undefined,
|
||||
fetching: undefined,
|
||||
})),
|
||||
]
|
||||
linkedPackages = await linkBinsOfPackages(
|
||||
(
|
||||
await Promise.all(
|
||||
directPkgs.map(async (dep) => {
|
||||
const manifest = await dep.fetchingBundledManifest?.() ?? await safeReadProjectManifestOnly(dep.dir)
|
||||
const manifest = (await dep.fetching?.())?.bundledManifest ?? await safeReadProjectManifestOnly(dep.dir)
|
||||
let nodeExecPath: string | undefined
|
||||
if (manifest?.name) {
|
||||
nodeExecPath = project.manifest.dependenciesMeta?.[manifest.name]?.node
|
||||
|
||||
@@ -425,20 +425,20 @@ async function linkAllPkgs (
|
||||
) {
|
||||
return Promise.all(
|
||||
depNodes.map(async (depNode) => {
|
||||
const filesResponse = await depNode.fetchingFiles()
|
||||
const { files } = await depNode.fetching()
|
||||
|
||||
if (typeof depNode.requiresBuild === 'function') {
|
||||
depNode.requiresBuild = await depNode.requiresBuild()
|
||||
}
|
||||
let sideEffectsCacheKey: string | undefined
|
||||
if (opts.sideEffectsCacheRead && filesResponse.sideEffects && !isEmpty(filesResponse.sideEffects)) {
|
||||
if (opts.sideEffectsCacheRead && files.sideEffects && !isEmpty(files.sideEffects)) {
|
||||
sideEffectsCacheKey = calcDepState(opts.depGraph, opts.depsStateCache, depNode.depPath, {
|
||||
isBuilt: !opts.ignoreScripts && depNode.requiresBuild,
|
||||
patchFileHash: depNode.patchFile?.hash,
|
||||
})
|
||||
}
|
||||
const { importMethod, isBuilt } = await storeController.importPackage(depNode.dir, {
|
||||
filesResponse,
|
||||
filesResponse: files,
|
||||
force: opts.force,
|
||||
sideEffectsCacheKey,
|
||||
requiresBuild: depNode.requiresBuild || depNode.patchFile != null,
|
||||
|
||||
@@ -583,7 +583,11 @@ export async function headlessInstall (opts: HeadlessOptions): Promise<Installat
|
||||
}
|
||||
|
||||
// waiting till package requests are finished
|
||||
await Promise.all(depNodes.map(({ finishing }) => finishing))
|
||||
await Promise.all(depNodes.map(async ({ fetching }) => {
|
||||
try {
|
||||
await fetching?.()
|
||||
} catch {}
|
||||
}))
|
||||
|
||||
summaryLogger.debug({ prefix: lockfileDir })
|
||||
|
||||
@@ -776,7 +780,7 @@ async function linkAllPkgs (
|
||||
depNodes.map(async (depNode) => {
|
||||
let filesResponse!: PackageFilesResponse
|
||||
try {
|
||||
filesResponse = await depNode.fetchingFiles()
|
||||
filesResponse = (await depNode.fetching()).files
|
||||
} catch (err: any) { // eslint-disable-line
|
||||
if (depNode.optional) return
|
||||
throw err
|
||||
|
||||
@@ -98,10 +98,10 @@ async function linkAllPkgsInOrder (
|
||||
await Promise.all(
|
||||
Object.entries(hierarchy).map(async ([dir, deps]) => {
|
||||
const depNode = graph[dir]
|
||||
if (depNode.fetchingFiles) {
|
||||
if (depNode.fetching) {
|
||||
let filesResponse!: PackageFilesResponse
|
||||
try {
|
||||
filesResponse = await depNode.fetchingFiles()
|
||||
filesResponse = (await depNode.fetching()).files
|
||||
} catch (err: any) { // eslint-disable-line
|
||||
if (depNode.optional) return
|
||||
throw err
|
||||
|
||||
@@ -216,7 +216,7 @@ async function fetchDeps (
|
||||
name: pkgName,
|
||||
version: pkgVersion,
|
||||
},
|
||||
})
|
||||
}) as any // eslint-disable-line
|
||||
if (fetchResponse instanceof Promise) fetchResponse = await fetchResponse
|
||||
} catch (err: any) { // eslint-disable-line
|
||||
if (pkgSnapshot.optional) return
|
||||
@@ -228,9 +228,8 @@ async function fetchDeps (
|
||||
children: {},
|
||||
depPath,
|
||||
dir,
|
||||
fetchingFiles: fetchResponse.files,
|
||||
fetching: fetchResponse.fetching,
|
||||
filesIndexFile: fetchResponse.filesIndexFile,
|
||||
finishing: fetchResponse.finishing,
|
||||
hasBin: pkgSnapshot.hasBin === true,
|
||||
hasBundledDependencies: pkgSnapshot.bundledDependencies != null,
|
||||
modules,
|
||||
|
||||
@@ -15,7 +15,7 @@ import {
|
||||
type FetchOptions,
|
||||
type FetchResult,
|
||||
} from '@pnpm/fetcher-base'
|
||||
import { type Cafs, type PackageFilesResponse } from '@pnpm/cafs-types'
|
||||
import { type Cafs } from '@pnpm/cafs-types'
|
||||
import gfs from '@pnpm/graceful-fs'
|
||||
import { logger } from '@pnpm/logger'
|
||||
import { packageIsInstallable } from '@pnpm/package-is-installable'
|
||||
@@ -29,7 +29,7 @@ import {
|
||||
} from '@pnpm/resolver-base'
|
||||
import {
|
||||
type BundledManifest,
|
||||
type BundledManifestFunction,
|
||||
type PkgRequestFetchResult,
|
||||
type FetchPackageToStoreFunction,
|
||||
type FetchPackageToStoreOptions,
|
||||
type GetFilesIndexFilePath,
|
||||
@@ -284,18 +284,15 @@ async function resolveAndFetch (
|
||||
updated,
|
||||
publishedAt,
|
||||
},
|
||||
bundledManifest: fetchResult.bundledManifest,
|
||||
files: fetchResult.files,
|
||||
fetching: fetchResult.fetching,
|
||||
filesIndexFile: fetchResult.filesIndexFile,
|
||||
finishing: fetchResult.finishing,
|
||||
}
|
||||
}
|
||||
|
||||
interface FetchLock {
|
||||
bundledManifest?: Promise<BundledManifest | undefined>
|
||||
files: Promise<PackageFilesResponse>
|
||||
fetching: Promise<PkgRequestFetchResult>
|
||||
filesIndexFile: string
|
||||
finishing: Promise<void>
|
||||
fetchRawManifest?: boolean
|
||||
}
|
||||
|
||||
function getFilesIndexFilePath (
|
||||
@@ -337,37 +334,24 @@ function fetchToStore (
|
||||
},
|
||||
opts: FetchPackageToStoreOptions
|
||||
): {
|
||||
bundledManifest?: BundledManifestFunction
|
||||
filesIndexFile: string
|
||||
files: () => Promise<PackageFilesResponse>
|
||||
finishing: () => Promise<void>
|
||||
fetching: () => Promise<PkgRequestFetchResult>
|
||||
} {
|
||||
if (!opts.pkg.name) {
|
||||
opts.fetchRawManifest = true
|
||||
}
|
||||
|
||||
if (!ctx.fetchingLocker.has(opts.pkg.id)) {
|
||||
const bundledManifest = pDefer<BundledManifest>()
|
||||
const files = pDefer<PackageFilesResponse>()
|
||||
const finishing = pDefer<undefined>()
|
||||
const fetching = pDefer<PkgRequestFetchResult>()
|
||||
const { filesIndexFile, target } = getFilesIndexFilePath(ctx, opts)
|
||||
|
||||
doFetchToStore(filesIndexFile, bundledManifest, files, finishing, target) // eslint-disable-line
|
||||
doFetchToStore(filesIndexFile, fetching, target) // eslint-disable-line
|
||||
|
||||
if (opts.fetchRawManifest) {
|
||||
ctx.fetchingLocker.set(opts.pkg.id, {
|
||||
bundledManifest: removeKeyOnFail(bundledManifest.promise),
|
||||
files: removeKeyOnFail(files.promise),
|
||||
filesIndexFile,
|
||||
finishing: removeKeyOnFail(finishing.promise),
|
||||
})
|
||||
} else {
|
||||
ctx.fetchingLocker.set(opts.pkg.id, {
|
||||
files: removeKeyOnFail(files.promise),
|
||||
filesIndexFile,
|
||||
finishing: removeKeyOnFail(finishing.promise),
|
||||
})
|
||||
}
|
||||
ctx.fetchingLocker.set(opts.pkg.id, {
|
||||
fetching: removeKeyOnFail(fetching.promise),
|
||||
filesIndexFile,
|
||||
fetchRawManifest: opts.fetchRawManifest,
|
||||
})
|
||||
|
||||
// When files resolves, the cached result has to set fromStore to true, without
|
||||
// affecting previous invocations: so we need to replace the cache.
|
||||
@@ -375,17 +359,17 @@ function fetchToStore (
|
||||
// Changing the value of fromStore is needed for correct reporting of `pnpm server`.
|
||||
// Otherwise, if a package was not in store when the server started, it will always be
|
||||
// reported as "downloaded" instead of "reused".
|
||||
files.promise.then((cache) => {
|
||||
fetching.promise.then((cache) => {
|
||||
progressLogger.debug({
|
||||
packageId: opts.pkg.id,
|
||||
requester: opts.lockfileDir,
|
||||
status: cache.fromStore
|
||||
status: cache.files.fromStore
|
||||
? 'found_in_store'
|
||||
: 'fetched',
|
||||
})
|
||||
|
||||
// If it's already in the store, we don't need to update the cache
|
||||
if (cache.fromStore) {
|
||||
if (cache.files.fromStore) {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -397,9 +381,12 @@ function fetchToStore (
|
||||
|
||||
ctx.fetchingLocker.set(opts.pkg.id, {
|
||||
...tmp,
|
||||
files: Promise.resolve({
|
||||
fetching: Promise.resolve({
|
||||
...cache,
|
||||
fromStore: true,
|
||||
files: {
|
||||
...cache.files,
|
||||
fromStore: true,
|
||||
},
|
||||
}),
|
||||
})
|
||||
})
|
||||
@@ -410,25 +397,33 @@ function fetchToStore (
|
||||
|
||||
const result = ctx.fetchingLocker.get(opts.pkg.id)!
|
||||
|
||||
if (opts.fetchRawManifest && (result.bundledManifest == null)) {
|
||||
result.bundledManifest = removeKeyOnFail(
|
||||
result.files.then(async (filesResult) => {
|
||||
if (!filesResult.filesIndex['package.json']) return undefined
|
||||
if (filesResult.unprocessed) {
|
||||
const { integrity, mode } = filesResult.filesIndex['package.json']
|
||||
const manifestPath = ctx.getFilePathByModeInCafs(integrity, mode)
|
||||
return readBundledManifest(manifestPath)
|
||||
if (opts.fetchRawManifest && !result.fetchRawManifest) {
|
||||
result.fetching = removeKeyOnFail(
|
||||
result.fetching.then(async ({ files }) => {
|
||||
if (!files.filesIndex['package.json']) return {
|
||||
files,
|
||||
bundledManifest: undefined,
|
||||
}
|
||||
if (files.unprocessed) {
|
||||
const { integrity, mode } = files.filesIndex['package.json']
|
||||
const manifestPath = ctx.getFilePathByModeInCafs(integrity, mode)
|
||||
return {
|
||||
files,
|
||||
bundledManifest: await readBundledManifest(manifestPath),
|
||||
}
|
||||
}
|
||||
return {
|
||||
files,
|
||||
bundledManifest: await readBundledManifest(files.filesIndex['package.json']),
|
||||
}
|
||||
return readBundledManifest(filesResult.filesIndex['package.json'])
|
||||
})
|
||||
)
|
||||
result.fetchRawManifest = true
|
||||
}
|
||||
|
||||
return {
|
||||
bundledManifest: (result.bundledManifest != null) ? pShare(result.bundledManifest) : undefined,
|
||||
files: pShare(result.files),
|
||||
fetching: pShare(result.fetching),
|
||||
filesIndexFile: result.filesIndexFile,
|
||||
finishing: pShare(result.finishing),
|
||||
}
|
||||
|
||||
async function removeKeyOnFail<T> (p: Promise<T>): Promise<T> {
|
||||
@@ -442,9 +437,7 @@ function fetchToStore (
|
||||
|
||||
async function doFetchToStore (
|
||||
filesIndexFile: string,
|
||||
bundledManifest: pDefer.DeferredPromise<BundledManifest>,
|
||||
files: pDefer.DeferredPromise<PackageFilesResponse>,
|
||||
finishing: pDefer.DeferredPromise<void>,
|
||||
fetching: pDefer.DeferredPromise<PkgRequestFetchResult>,
|
||||
target: string
|
||||
) {
|
||||
try {
|
||||
@@ -482,16 +475,15 @@ Package name mismatch found while reading ${JSON.stringify(opts.pkg.resolution)}
|
||||
This means that the lockfile is broken. Expected package: ${opts.expectedPkg.name}@${opts.expectedPkg.version}. \
|
||||
Actual package in the store by the given integrity: ${pkgFilesIndex.name}@${pkgFilesIndex.version}.`)
|
||||
}
|
||||
files.resolve({
|
||||
unprocessed: true,
|
||||
filesIndex: pkgFilesIndex.files,
|
||||
fromStore: true,
|
||||
sideEffects: pkgFilesIndex.sideEffects,
|
||||
fetching.resolve({
|
||||
files: {
|
||||
unprocessed: true,
|
||||
filesIndex: pkgFilesIndex.files,
|
||||
fromStore: true,
|
||||
sideEffects: pkgFilesIndex.sideEffects,
|
||||
},
|
||||
bundledManifest: manifest == null ? manifest : normalizeBundledManifest(manifest),
|
||||
})
|
||||
if (opts.fetchRawManifest) {
|
||||
bundledManifest.resolve(manifest == null ? manifest : normalizeBundledManifest(manifest))
|
||||
}
|
||||
finishing.resolve(undefined)
|
||||
return
|
||||
}
|
||||
if ((pkgFilesIndex?.files) != null) {
|
||||
@@ -539,34 +531,23 @@ Actual package in the store by the given integrity: ${pkgFilesIndex.name}@${pkgF
|
||||
},
|
||||
}
|
||||
), { priority })
|
||||
if (opts.fetchRawManifest) {
|
||||
bundledManifest.resolve(fetchedPackage.manifest == null ? fetchedPackage.manifest : normalizeBundledManifest(fetchedPackage.manifest))
|
||||
}
|
||||
|
||||
const filesResult: PackageFilesResponse = {
|
||||
local: fetchedPackage.local,
|
||||
fromStore: !fetchedPackage.local ? false : !ctx.relinkLocalDirDeps,
|
||||
filesIndex: fetchedPackage.filesIndex,
|
||||
packageImportMethod: (fetchedPackage as DirectoryFetcherResult).packageImportMethod,
|
||||
}
|
||||
if (fetchedPackage.filesIndex['package.json'] != null) {
|
||||
readBundledManifest(fetchedPackage.filesIndex['package.json'])
|
||||
.then(bundledManifest.resolve)
|
||||
.catch(bundledManifest.reject)
|
||||
}
|
||||
|
||||
if (isLocalTarballDep && (opts.pkg.resolution as TarballResolution).integrity) {
|
||||
await fs.mkdir(target, { recursive: true })
|
||||
await gfs.writeFile(path.join(target, TARBALL_INTEGRITY_FILENAME), (opts.pkg.resolution as TarballResolution).integrity!, 'utf8')
|
||||
}
|
||||
|
||||
files.resolve(filesResult)
|
||||
finishing.resolve(undefined)
|
||||
fetching.resolve({
|
||||
files: {
|
||||
local: fetchedPackage.local,
|
||||
fromStore: !fetchedPackage.local ? false : !ctx.relinkLocalDirDeps,
|
||||
filesIndex: fetchedPackage.filesIndex,
|
||||
packageImportMethod: (fetchedPackage as DirectoryFetcherResult).packageImportMethod,
|
||||
},
|
||||
bundledManifest: fetchedPackage.manifest == null ? fetchedPackage.manifest : normalizeBundledManifest(fetchedPackage.manifest),
|
||||
})
|
||||
} catch (err: any) { // eslint-disable-line
|
||||
files.reject(err)
|
||||
if (opts.fetchRawManifest) {
|
||||
bundledManifest.reject(err)
|
||||
}
|
||||
fetching.reject(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,11 +5,9 @@ import { type PackageFilesIndex } from '@pnpm/store.cafs'
|
||||
import { createClient } from '@pnpm/client'
|
||||
import { streamParser } from '@pnpm/logger'
|
||||
import { createPackageRequester, type PackageResponse } from '@pnpm/package-requester'
|
||||
import type { PackageFilesResponse } from '@pnpm/cafs-types'
|
||||
import { createCafsStore } from '@pnpm/create-cafs-store'
|
||||
import { REGISTRY_MOCK_PORT } from '@pnpm/registry-mock'
|
||||
import { fixtures } from '@pnpm/test-fixtures'
|
||||
import { type DependencyManifest } from '@pnpm/types'
|
||||
import delay from 'delay'
|
||||
import { depPathToFilename } from '@pnpm/dependency-path'
|
||||
import { restartWorkerPool } from '@pnpm/worker'
|
||||
@@ -17,6 +15,7 @@ import loadJsonFile from 'load-json-file'
|
||||
import nock from 'nock'
|
||||
import normalize from 'normalize-path'
|
||||
import tempy from 'tempy'
|
||||
import { type PkgRequestFetchResult } from '@pnpm/store-controller-types'
|
||||
|
||||
const registry = `http://localhost:${REGISTRY_MOCK_PORT}`
|
||||
const f = fixtures(__dirname)
|
||||
@@ -67,11 +66,9 @@ test('request package', async () => {
|
||||
tarball: `http://localhost:${REGISTRY_MOCK_PORT}/is-positive/-/is-positive-1.0.0.tgz`,
|
||||
})
|
||||
|
||||
const files = await pkgResponse.files!()
|
||||
const { files } = await pkgResponse.fetching!()
|
||||
expect(Object.keys(files.filesIndex).sort()).toStrictEqual(['package.json', 'index.js', 'license', 'readme.md'].sort())
|
||||
expect(files.fromStore).toBeFalsy()
|
||||
|
||||
expect(pkgResponse.finishing!()).toBeTruthy()
|
||||
})
|
||||
|
||||
test('request package but skip fetching', async () => {
|
||||
@@ -111,8 +108,7 @@ test('request package but skip fetching', async () => {
|
||||
tarball: `http://localhost:${REGISTRY_MOCK_PORT}/is-positive/-/is-positive-1.0.0.tgz`,
|
||||
})
|
||||
|
||||
expect(pkgResponse.files).toBeFalsy()
|
||||
expect(pkgResponse.finishing).toBeFalsy()
|
||||
expect(pkgResponse.fetching).toBeFalsy()
|
||||
})
|
||||
|
||||
test('request package but skip fetching, when resolution is already available', async () => {
|
||||
@@ -150,8 +146,6 @@ test('request package but skip fetching, when resolution is already available',
|
||||
latest: string
|
||||
manifest: { name: string }
|
||||
}
|
||||
files: () => Promise<object>
|
||||
finishing: () => Promise<void>
|
||||
}
|
||||
|
||||
expect(pkgResponse).toBeTruthy()
|
||||
@@ -168,8 +162,7 @@ test('request package but skip fetching, when resolution is already available',
|
||||
tarball: `http://localhost:${REGISTRY_MOCK_PORT}/is-positive/-/is-positive-1.0.0.tgz`,
|
||||
})
|
||||
|
||||
expect(pkgResponse.files).toBeFalsy()
|
||||
expect(pkgResponse.finishing).toBeFalsy()
|
||||
expect(pkgResponse.fetching).toBeFalsy()
|
||||
})
|
||||
|
||||
test('refetch local tarball if its integrity has changed', async () => {
|
||||
@@ -211,15 +204,13 @@ test('refetch local tarball if its integrity has changed', async () => {
|
||||
},
|
||||
},
|
||||
}) as PackageResponse & {
|
||||
files: () => Promise<PackageFilesResponse>
|
||||
finishing: () => Promise<void>
|
||||
fetching: () => Promise<PkgRequestFetchResult>
|
||||
}
|
||||
await response.files()
|
||||
await response.finishing()
|
||||
const { files, bundledManifest } = await response.fetching()
|
||||
|
||||
expect(response.body.updated).toBeFalsy()
|
||||
expect((await response.files()).fromStore).toBeFalsy()
|
||||
expect(await response.bundledManifest!()).toBeTruthy()
|
||||
expect(files.fromStore).toBeFalsy()
|
||||
expect(bundledManifest).toBeTruthy()
|
||||
}
|
||||
|
||||
f.copy('pnpm-package-requester-4.1.2.tgz', tarballPath)
|
||||
@@ -244,12 +235,11 @@ test('refetch local tarball if its integrity has changed', async () => {
|
||||
},
|
||||
},
|
||||
})
|
||||
await response.files!()
|
||||
await response.finishing!()
|
||||
const { files, bundledManifest } = await response.fetching!()
|
||||
|
||||
expect(response.body.updated).toBeTruthy()
|
||||
expect((await response.files!()).fromStore).toBeFalsy()
|
||||
expect(await response.bundledManifest!()).toBeTruthy()
|
||||
expect(files.fromStore).toBeFalsy()
|
||||
expect(bundledManifest).toBeTruthy()
|
||||
}
|
||||
|
||||
{
|
||||
@@ -271,15 +261,13 @@ test('refetch local tarball if its integrity has changed', async () => {
|
||||
},
|
||||
},
|
||||
}) as PackageResponse & {
|
||||
files: () => Promise<PackageFilesResponse>
|
||||
finishing: () => Promise<void>
|
||||
fetching: () => Promise<PkgRequestFetchResult>
|
||||
}
|
||||
await response.files()
|
||||
await response.finishing()
|
||||
const { files, bundledManifest } = await response.fetching()
|
||||
|
||||
expect(response.body.updated).toBeFalsy()
|
||||
expect((await response.files()).fromStore).toBeTruthy()
|
||||
expect(await response.bundledManifest!()).toBeTruthy()
|
||||
expect(files.fromStore).toBeTruthy()
|
||||
expect(bundledManifest).toBeTruthy()
|
||||
}
|
||||
})
|
||||
|
||||
@@ -289,7 +277,7 @@ test('refetch local tarball if its integrity has changed. The requester does not
|
||||
f.copy('pnpm-package-requester-0.8.1.tgz', tarballPath)
|
||||
const tarball = `file:${tarballPath}`
|
||||
const wantedPackage = { pref: tarball }
|
||||
const storeDir = path.join(__dirname, '..', '.store')
|
||||
const storeDir = path.join(projectDir, 'store')
|
||||
const cafs = createCafsStore(storeDir)
|
||||
const requestPackageOpts = {
|
||||
downloadPriority: 0,
|
||||
@@ -310,15 +298,13 @@ test('refetch local tarball if its integrity has changed. The requester does not
|
||||
})
|
||||
|
||||
const response = await requestPackage(wantedPackage, requestPackageOpts) as PackageResponse & {
|
||||
files: () => Promise<PackageFilesResponse>
|
||||
finishing: () => Promise<void>
|
||||
fetching: () => Promise<PkgRequestFetchResult>
|
||||
}
|
||||
await response.files()
|
||||
await response.finishing()
|
||||
const { files, bundledManifest } = await response.fetching()
|
||||
|
||||
expect(response.body.updated).toBeTruthy()
|
||||
expect((await response.files()).fromStore).toBeFalsy()
|
||||
expect(await response.bundledManifest!()).toBeTruthy()
|
||||
expect(files.fromStore).toBeFalsy()
|
||||
expect(bundledManifest).toBeTruthy()
|
||||
}
|
||||
|
||||
f.copy('pnpm-package-requester-4.1.2.tgz', tarballPath)
|
||||
@@ -334,15 +320,13 @@ test('refetch local tarball if its integrity has changed. The requester does not
|
||||
})
|
||||
|
||||
const response = await requestPackage(wantedPackage, requestPackageOpts) as PackageResponse & {
|
||||
files: () => Promise<PackageFilesResponse>
|
||||
finishing: () => Promise<void>
|
||||
fetching: () => Promise<PkgRequestFetchResult>
|
||||
}
|
||||
await response.files()
|
||||
await response.finishing()
|
||||
const { files, bundledManifest } = await response.fetching()
|
||||
|
||||
expect(response.body.updated).toBeTruthy()
|
||||
expect((await response.files()).fromStore).toBeFalsy()
|
||||
expect(await response.bundledManifest!()).toBeTruthy()
|
||||
expect(files.fromStore).toBeFalsy()
|
||||
expect(bundledManifest).toBeTruthy()
|
||||
}
|
||||
|
||||
{
|
||||
@@ -355,14 +339,12 @@ test('refetch local tarball if its integrity has changed. The requester does not
|
||||
})
|
||||
|
||||
const response = await requestPackage(wantedPackage, requestPackageOpts) as PackageResponse & {
|
||||
files: () => Promise<PackageFilesResponse>
|
||||
finishing: () => Promise<void>
|
||||
fetching: () => Promise<PkgRequestFetchResult>
|
||||
}
|
||||
await response.files()
|
||||
await response.finishing()
|
||||
const { files, bundledManifest } = await response.fetching()
|
||||
|
||||
expect((await response.files()).fromStore).toBeTruthy()
|
||||
expect(await response.bundledManifest!()).toBeTruthy()
|
||||
expect(files.fromStore).toBeTruthy()
|
||||
expect(bundledManifest).toBeTruthy()
|
||||
}
|
||||
})
|
||||
|
||||
@@ -394,9 +376,8 @@ test('fetchPackageToStore()', async () => {
|
||||
},
|
||||
})
|
||||
|
||||
expect(fetchResult.bundledManifest).toBeFalsy()
|
||||
|
||||
const files = await fetchResult.files()
|
||||
const { files, bundledManifest } = await fetchResult.fetching()
|
||||
expect(bundledManifest).toBeFalsy()
|
||||
expect(Object.keys(files.filesIndex).sort()).toStrictEqual(['package.json', 'index.js', 'license', 'readme.md'].sort())
|
||||
expect(files.fromStore).toBeFalsy()
|
||||
|
||||
@@ -404,8 +385,6 @@ test('fetchPackageToStore()', async () => {
|
||||
expect(indexFile).toBeTruthy()
|
||||
expect(typeof indexFile.files['package.json'].checkedAt).toBeTruthy()
|
||||
|
||||
expect(fetchResult.finishing()).toBeTruthy()
|
||||
|
||||
const fetchResult2 = packageRequester.fetchPackageToStore({
|
||||
fetchRawManifest: true,
|
||||
force: false,
|
||||
@@ -425,7 +404,7 @@ test('fetchPackageToStore()', async () => {
|
||||
// This verifies that when a package has been cached with no full manifest
|
||||
// the full manifest is requested and added to the cache
|
||||
expect(
|
||||
await fetchResult2.bundledManifest!()
|
||||
(await fetchResult2.fetching()).bundledManifest
|
||||
).toStrictEqual(
|
||||
{
|
||||
engines: { node: '>=0.10.0' },
|
||||
@@ -487,26 +466,22 @@ test('fetchPackageToStore() concurrency check', async () => {
|
||||
|
||||
{
|
||||
const fetchResult = fetchResults[0]
|
||||
const files = await fetchResult.files()
|
||||
const { files } = await fetchResult.fetching()
|
||||
|
||||
ino1 = statSync(files.filesIndex['package.json'] as string).ino
|
||||
|
||||
expect(Object.keys(files.filesIndex).sort()).toStrictEqual(['package.json', 'index.js', 'license', 'readme.md'].sort())
|
||||
expect(files.fromStore).toBeFalsy()
|
||||
|
||||
expect(fetchResult.finishing).toBeTruthy()
|
||||
}
|
||||
|
||||
{
|
||||
const fetchResult = fetchResults[1]
|
||||
const files = await fetchResult.files()
|
||||
const { files } = await fetchResult.fetching()
|
||||
|
||||
ino2 = statSync(files.filesIndex['package.json'] as string).ino
|
||||
|
||||
expect(Object.keys(files.filesIndex).sort()).toStrictEqual(['package.json', 'index.js', 'license', 'readme.md'].sort())
|
||||
expect(files.fromStore).toBeFalsy()
|
||||
|
||||
expect(fetchResult.finishing()).toBeTruthy()
|
||||
}
|
||||
|
||||
expect(ino1).toBe(ino2)
|
||||
@@ -555,7 +530,7 @@ test('fetchPackageToStore() does not cache errors', async () => {
|
||||
},
|
||||
},
|
||||
})
|
||||
await expect(badRequest.files()).rejects.toThrow()
|
||||
await expect(badRequest.fetching()).rejects.toThrow()
|
||||
|
||||
const fetchResult = packageRequester.fetchPackageToStore({
|
||||
force: false,
|
||||
@@ -571,11 +546,10 @@ test('fetchPackageToStore() does not cache errors', async () => {
|
||||
},
|
||||
},
|
||||
})
|
||||
const files = await fetchResult.files()
|
||||
const { files } = await fetchResult.fetching()
|
||||
expect(Object.keys(files.filesIndex).sort()).toStrictEqual(['package.json', 'index.js', 'license', 'readme.md'].sort())
|
||||
expect(files.fromStore).toBeFalsy()
|
||||
|
||||
expect(fetchResult.finishing()).toBeTruthy()
|
||||
expect(nock.isDone()).toBeTruthy()
|
||||
})
|
||||
|
||||
@@ -623,11 +597,11 @@ test('always return a package manifest in the response', async () => {
|
||||
preferredVersions: {},
|
||||
projectDir,
|
||||
registry,
|
||||
}) as PackageResponse & { bundledManifest: () => Promise<DependencyManifest> }
|
||||
}) as PackageResponse & { fetching: () => Promise<PkgRequestFetchResult> }
|
||||
|
||||
expect(pkgResponse.body).toBeTruthy()
|
||||
expect(
|
||||
await pkgResponse.bundledManifest()
|
||||
(await pkgResponse.fetching()).bundledManifest
|
||||
).toEqual(
|
||||
{
|
||||
engines: { node: '>=0.10.0' },
|
||||
@@ -686,7 +660,7 @@ test('fetchPackageToStore() fetch raw manifest of cached package', async () => {
|
||||
}),
|
||||
])
|
||||
|
||||
expect(await fetchResults[1].bundledManifest!()).toBeTruthy()
|
||||
expect((await fetchResults[1].fetching()).bundledManifest).toBeTruthy()
|
||||
})
|
||||
|
||||
test('refetch package to store if it has been modified', async () => {
|
||||
@@ -724,7 +698,7 @@ test('refetch package to store if it has been modified', async () => {
|
||||
},
|
||||
})
|
||||
|
||||
const { filesIndex } = await fetchResult.files()
|
||||
const { filesIndex } = (await fetchResult.fetching()).files
|
||||
indexJsFile = filesIndex['index.js'] as string
|
||||
}
|
||||
|
||||
@@ -762,7 +736,7 @@ test('refetch package to store if it has been modified', async () => {
|
||||
},
|
||||
})
|
||||
|
||||
await fetchResult.files()
|
||||
await fetchResult.fetching()
|
||||
}
|
||||
|
||||
streamParser.removeListener('data', reporter)
|
||||
@@ -805,8 +779,7 @@ test('do not fetch an optional package that is not installable', async () => {
|
||||
expect(pkgResponse.body.isInstallable).toBe(false)
|
||||
expect(pkgResponse.body.id).toBe(`localhost+${REGISTRY_MOCK_PORT}/@pnpm.e2e/not-compatible-with-any-os/1.0.0`)
|
||||
|
||||
expect(pkgResponse.files).toBeFalsy()
|
||||
expect(pkgResponse.finishing).toBeFalsy()
|
||||
expect(pkgResponse.fetching).toBeFalsy()
|
||||
})
|
||||
|
||||
// Test case for https://github.com/pnpm/pnpm/issues/1866
|
||||
@@ -868,7 +841,7 @@ test('throw exception if the package data in the store differs from the expected
|
||||
projectDir,
|
||||
registry,
|
||||
})
|
||||
await pkgResponse.finishing!()
|
||||
await pkgResponse.fetching!()
|
||||
}
|
||||
|
||||
// Fail when the name of the package is different in the store
|
||||
@@ -881,7 +854,7 @@ test('throw exception if the package data in the store differs from the expected
|
||||
storeDir,
|
||||
verifyStoreIntegrity: true,
|
||||
})
|
||||
const { files } = requestPackage.fetchPackageToStore({
|
||||
const { fetching } = requestPackage.fetchPackageToStore({
|
||||
force: false,
|
||||
lockfileDir: tempy.directory(),
|
||||
pkg: {
|
||||
@@ -895,7 +868,7 @@ test('throw exception if the package data in the store differs from the expected
|
||||
version: '1.0.0',
|
||||
},
|
||||
})
|
||||
await expect(files()).rejects.toThrow(/Package name mismatch found while reading/)
|
||||
await expect(fetching()).rejects.toThrow(/Package name mismatch found while reading/)
|
||||
}
|
||||
|
||||
// Fail when the version of the package is different in the store
|
||||
@@ -908,7 +881,7 @@ test('throw exception if the package data in the store differs from the expected
|
||||
storeDir,
|
||||
verifyStoreIntegrity: true,
|
||||
})
|
||||
const { files } = requestPackage.fetchPackageToStore({
|
||||
const { fetching } = requestPackage.fetchPackageToStore({
|
||||
force: false,
|
||||
lockfileDir: tempy.directory(),
|
||||
pkg: {
|
||||
@@ -922,7 +895,7 @@ test('throw exception if the package data in the store differs from the expected
|
||||
version: '2.0.0',
|
||||
},
|
||||
})
|
||||
await expect(files()).rejects.toThrow(/Package name mismatch found while reading/)
|
||||
await expect(fetching()).rejects.toThrow(/Package name mismatch found while reading/)
|
||||
}
|
||||
|
||||
// Do not fail when the versions are the same but written in a differnt format (1.0.0 is the same as v1.0.0)
|
||||
@@ -935,7 +908,7 @@ test('throw exception if the package data in the store differs from the expected
|
||||
storeDir,
|
||||
verifyStoreIntegrity: true,
|
||||
})
|
||||
const { files } = requestPackage.fetchPackageToStore({
|
||||
const { fetching } = requestPackage.fetchPackageToStore({
|
||||
force: false,
|
||||
lockfileDir: tempy.directory(),
|
||||
pkg: {
|
||||
@@ -949,7 +922,7 @@ test('throw exception if the package data in the store differs from the expected
|
||||
version: 'v1.0.0',
|
||||
},
|
||||
})
|
||||
await expect(files()).resolves.toStrictEqual(expect.anything())
|
||||
await expect(fetching()).resolves.toStrictEqual(expect.anything())
|
||||
}
|
||||
|
||||
{
|
||||
@@ -961,7 +934,7 @@ test('throw exception if the package data in the store differs from the expected
|
||||
storeDir,
|
||||
verifyStoreIntegrity: true,
|
||||
})
|
||||
const { files } = requestPackage.fetchPackageToStore({
|
||||
const { fetching } = requestPackage.fetchPackageToStore({
|
||||
force: false,
|
||||
lockfileDir: tempy.directory(),
|
||||
pkg: {
|
||||
@@ -975,7 +948,7 @@ test('throw exception if the package data in the store differs from the expected
|
||||
version: 'v1.0.0',
|
||||
},
|
||||
})
|
||||
await expect(files()).resolves.toStrictEqual(expect.anything())
|
||||
await expect(fetching()).resolves.toStrictEqual(expect.anything())
|
||||
}
|
||||
})
|
||||
|
||||
@@ -1000,7 +973,7 @@ test("don't throw an error if the package was updated, so the expectedPkg has a
|
||||
projectDir,
|
||||
registry,
|
||||
})
|
||||
await pkgResponse.finishing!()
|
||||
await pkgResponse.fetching!()
|
||||
}
|
||||
const requestPackage = createPackageRequester({
|
||||
resolve,
|
||||
@@ -1022,7 +995,7 @@ test("don't throw an error if the package was updated, so the expectedPkg has a
|
||||
version: '3.0.0',
|
||||
},
|
||||
})
|
||||
await expect(pkgResponse.files!()).resolves.toStrictEqual(expect.anything())
|
||||
await expect(pkgResponse.fetching!()).resolves.toStrictEqual(expect.anything())
|
||||
})
|
||||
|
||||
test('the version in the bundled manifest should be normalized', async () => {
|
||||
@@ -1045,10 +1018,9 @@ test('the version in the bundled manifest should be normalized', async () => {
|
||||
projectDir: tempy.directory(),
|
||||
registry,
|
||||
})
|
||||
await expect(pkgResponse.bundledManifest!()).resolves.toStrictEqual(expect.objectContaining({
|
||||
expect((await pkgResponse.fetching!()).bundledManifest).toStrictEqual(expect.objectContaining({
|
||||
version: '1.2.1',
|
||||
}))
|
||||
await pkgResponse.finishing!()
|
||||
})
|
||||
|
||||
test('should skip store integrity check and resolve manifest if fetchRawManifest is true', async () => {
|
||||
@@ -1077,7 +1049,7 @@ test('should skip store integrity check and resolve manifest if fetchRawManifest
|
||||
registry,
|
||||
})
|
||||
|
||||
await pkgResponse.finishing!()
|
||||
await pkgResponse.fetching!()
|
||||
}
|
||||
|
||||
{
|
||||
@@ -1106,9 +1078,9 @@ test('should skip store integrity check and resolve manifest if fetchRawManifest
|
||||
},
|
||||
})
|
||||
|
||||
await fetchResult.finishing()
|
||||
await fetchResult.fetching()
|
||||
|
||||
await expect(fetchResult.bundledManifest!()).resolves.toStrictEqual(expect.objectContaining({
|
||||
expect((await fetchResult.fetching!()).bundledManifest).toStrictEqual(expect.objectContaining({
|
||||
name: 'is-positive',
|
||||
version: '1.0.0',
|
||||
}))
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import path from 'path'
|
||||
import { WANTED_LOCKFILE } from '@pnpm/constants'
|
||||
import { PnpmError } from '@pnpm/error'
|
||||
import {
|
||||
packageManifestLogger,
|
||||
@@ -277,7 +276,11 @@ export async function resolveDependencies (
|
||||
}
|
||||
|
||||
// waiting till package requests are finished
|
||||
const waitTillAllFetchingsFinish = async () => Promise.all(Object.values(resolvedPackagesByDepPath).map(async ({ finishing }) => finishing?.()))
|
||||
const waitTillAllFetchingsFinish = async () => Promise.all(Object.values(resolvedPackagesByDepPath).map(async ({ fetching }) => {
|
||||
try {
|
||||
await fetching?.()
|
||||
} catch {}
|
||||
}))
|
||||
|
||||
return {
|
||||
dependenciesByProjectId,
|
||||
@@ -322,36 +325,39 @@ async function finishLockfileUpdates (
|
||||
) {
|
||||
return Promise.all(pendingRequiresBuilds.map(async (depPath) => {
|
||||
const depNode = dependenciesGraph[depPath]
|
||||
let requiresBuild!: boolean
|
||||
if (depNode.optional) {
|
||||
// We assume that all optional dependencies have to be built.
|
||||
// Optional dependencies are not always downloaded, so there is no way to know whether they need to be built or not.
|
||||
requiresBuild = true
|
||||
} else if (depNode.fetchingBundledManifest != null) {
|
||||
const filesResponse = await depNode.fetchingFiles()
|
||||
// The npm team suggests to always read the package.json for deciding whether the package has lifecycle scripts
|
||||
const pkgJson = await depNode.fetchingBundledManifest()
|
||||
requiresBuild = Boolean(
|
||||
pkgJson?.scripts != null && (
|
||||
Boolean(pkgJson.scripts.preinstall) ||
|
||||
Boolean(pkgJson.scripts.install) ||
|
||||
Boolean(pkgJson.scripts.postinstall)
|
||||
) ||
|
||||
filesResponse.filesIndex['binding.gyp'] ||
|
||||
Object.keys(filesResponse.filesIndex).some((filename) => !(filename.match(/^[.]hooks[\\/]/) == null)) // TODO: optimize this
|
||||
)
|
||||
} else {
|
||||
// This should never ever happen
|
||||
throw new Error(`Cannot create ${WANTED_LOCKFILE} because raw manifest (aka package.json) wasn't fetched for "${depPath}"`)
|
||||
}
|
||||
if (typeof depNode.requiresBuild === 'function') {
|
||||
depNode.requiresBuild['resolve'](requiresBuild)
|
||||
}
|
||||
if (!depNode) return
|
||||
try {
|
||||
let requiresBuild!: boolean
|
||||
if (depNode.optional) {
|
||||
// We assume that all optional dependencies have to be built.
|
||||
// Optional dependencies are not always downloaded, so there is no way to know whether they need to be built or not.
|
||||
requiresBuild = true
|
||||
} else {
|
||||
// The npm team suggests to always read the package.json for deciding whether the package has lifecycle scripts
|
||||
const { files, bundledManifest: pkgJson } = await depNode.fetching()
|
||||
requiresBuild = Boolean(
|
||||
pkgJson?.scripts != null && (
|
||||
Boolean(pkgJson.scripts.preinstall) ||
|
||||
Boolean(pkgJson.scripts.install) ||
|
||||
Boolean(pkgJson.scripts.postinstall)
|
||||
) ||
|
||||
files.filesIndex['binding.gyp'] ||
|
||||
Object.keys(files.filesIndex).some((filename) => !(filename.match(/^[.]hooks[\\/]/) == null)) // TODO: optimize this
|
||||
)
|
||||
}
|
||||
if (typeof depNode.requiresBuild === 'function') {
|
||||
depNode.requiresBuild['resolve'](requiresBuild)
|
||||
}
|
||||
|
||||
// TODO: try to cover with unit test the case when entry is no longer available in lockfile
|
||||
// It is an edge that probably happens if the entry is removed during lockfile prune
|
||||
if (requiresBuild && newLockfile.packages?.[depPath]) {
|
||||
newLockfile.packages[depPath].requiresBuild = true
|
||||
// TODO: try to cover with unit test the case when entry is no longer available in lockfile
|
||||
// It is an edge that probably happens if the entry is removed during lockfile prune
|
||||
if (requiresBuild && newLockfile.packages?.[depPath]) {
|
||||
newLockfile.packages[depPath].requiresBuild = true
|
||||
}
|
||||
} catch (err: any) { // eslint-disable-line
|
||||
if (typeof depNode.requiresBuild === 'function') {
|
||||
depNode.requiresBuild['reject'](err)
|
||||
}
|
||||
}
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -25,14 +25,13 @@ import {
|
||||
type WorkspacePackages,
|
||||
} from '@pnpm/resolver-base'
|
||||
import {
|
||||
type PackageFilesResponse,
|
||||
type PkgRequestFetchResult,
|
||||
type PackageResponse,
|
||||
type StoreController,
|
||||
} from '@pnpm/store-controller-types'
|
||||
import {
|
||||
type AllowedDeprecatedVersions,
|
||||
type Dependencies,
|
||||
type DependencyManifest,
|
||||
type PackageManifest,
|
||||
type PatchFile,
|
||||
type PeerDependenciesMeta,
|
||||
@@ -205,10 +204,8 @@ export interface ResolvedPackage {
|
||||
prod: boolean
|
||||
dev: boolean
|
||||
optional: boolean
|
||||
fetchingFiles: () => Promise<PackageFilesResponse>
|
||||
fetchingBundledManifest?: () => Promise<DependencyManifest | undefined>
|
||||
fetching: () => Promise<PkgRequestFetchResult>
|
||||
filesIndexFile: string
|
||||
finishing: () => Promise<void>
|
||||
name: string
|
||||
version: string
|
||||
peerDependencies: Dependencies
|
||||
@@ -1124,7 +1121,7 @@ async function resolveDependency (
|
||||
}
|
||||
|
||||
if (pkgResponse.body.isLocal) {
|
||||
const manifest = pkgResponse.body.manifest ?? await pkgResponse.bundledManifest!()
|
||||
const manifest = pkgResponse.body.manifest ?? (await pkgResponse.fetching!()).bundledManifest
|
||||
if (!manifest) {
|
||||
// This should actually never happen because the local-resolver returns a manifest
|
||||
// even if no real manifest exists in the filesystem.
|
||||
@@ -1301,11 +1298,9 @@ async function resolveDependency (
|
||||
!ctx.resolvedPackagesByDepPath[depPath].parentImporterIds.has(parentImporterId)
|
||||
ctx.resolvedPackagesByDepPath[depPath].parentImporterIds.add(parentImporterId)
|
||||
}
|
||||
if (ctx.resolvedPackagesByDepPath[depPath].fetchingFiles == null && pkgResponse.files != null) {
|
||||
ctx.resolvedPackagesByDepPath[depPath].fetchingFiles = pkgResponse.files
|
||||
if (ctx.resolvedPackagesByDepPath[depPath].fetching == null && pkgResponse.fetching != null) {
|
||||
ctx.resolvedPackagesByDepPath[depPath].fetching = pkgResponse.fetching
|
||||
ctx.resolvedPackagesByDepPath[depPath].filesIndexFile = pkgResponse.filesIndexFile!
|
||||
ctx.resolvedPackagesByDepPath[depPath].finishing = pkgResponse.finishing!
|
||||
ctx.resolvedPackagesByDepPath[depPath].fetchingBundledManifest = pkgResponse.bundledManifest!
|
||||
}
|
||||
|
||||
if (ctx.dependenciesTree.has(nodeId)) {
|
||||
@@ -1369,7 +1364,7 @@ async function getManifestFromResponse (
|
||||
pkgResponse: PackageResponse,
|
||||
wantedDependency: WantedDependency
|
||||
): Promise<PackageManifest> {
|
||||
const pkg = pkgResponse.body.manifest ?? await pkgResponse.bundledManifest!()
|
||||
const pkg = pkgResponse.body.manifest ?? (await pkgResponse.fetching!()).bundledManifest
|
||||
if (pkg) return pkg
|
||||
return {
|
||||
name: wantedDependency.pref.split('/').pop()!,
|
||||
@@ -1431,10 +1426,8 @@ function getResolvedPackage (
|
||||
parentImporterIds: new Set([options.parentImporterId]),
|
||||
depPath: options.depPath,
|
||||
dev: options.wantedDependency.dev,
|
||||
fetchingBundledManifest: options.pkgResponse.bundledManifest,
|
||||
fetchingFiles: options.pkgResponse.files!,
|
||||
fetching: options.pkgResponse.fetching!,
|
||||
filesIndexFile: options.pkgResponse.filesIndexFile!,
|
||||
finishing: options.pkgResponse.finishing!,
|
||||
hasBin: options.hasBin,
|
||||
hasBundledDependencies: !((options.pkg.bundledDependencies ?? options.pkg.bundleDependencies) == null),
|
||||
id: options.pkgResponse.body.id,
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
import path from 'path'
|
||||
import { createClient } from '@pnpm/client'
|
||||
import { createPackageStore } from '@pnpm/package-store'
|
||||
import { type FetchPackageToStoreFunction } from '@pnpm/store-controller-types'
|
||||
import tempy from 'tempy'
|
||||
|
||||
describe('store.importPackage()', () => {
|
||||
@@ -22,7 +23,7 @@ describe('store.importPackage()', () => {
|
||||
verifyStoreIntegrity: true,
|
||||
})
|
||||
const pkgId = 'registry.npmjs.org/is-positive/1.0.0'
|
||||
const fetchResponse = storeController.fetchPackage({
|
||||
const fetchResponse = (storeController.fetchPackage as FetchPackageToStoreFunction)({
|
||||
force: false,
|
||||
lockfileDir: tempy.directory(),
|
||||
pkg: {
|
||||
@@ -36,7 +37,7 @@ describe('store.importPackage()', () => {
|
||||
})
|
||||
const importTo = tempy.directory()
|
||||
const { importMethod } = await storeController.importPackage(importTo, {
|
||||
filesResponse: await fetchResponse.files(),
|
||||
filesResponse: (await fetchResponse.fetching()).files,
|
||||
force: false,
|
||||
})
|
||||
expect(typeof importMethod).toBe('string')
|
||||
@@ -61,7 +62,7 @@ describe('store.importPackage()', () => {
|
||||
verifyStoreIntegrity: true,
|
||||
})
|
||||
const pkgId = 'registry.npmjs.org/is-positive/1.0.0'
|
||||
const fetchResponse = storeController.fetchPackage({
|
||||
const fetchResponse = (storeController.fetchPackage as FetchPackageToStoreFunction)({
|
||||
force: false,
|
||||
lockfileDir: tempy.directory(),
|
||||
pkg: {
|
||||
@@ -75,7 +76,7 @@ describe('store.importPackage()', () => {
|
||||
})
|
||||
const importTo = tempy.directory()
|
||||
const { importMethod } = await storeController.importPackage(importTo, {
|
||||
filesResponse: await fetchResponse.files(),
|
||||
filesResponse: (await fetchResponse.fetching()).files,
|
||||
force: false,
|
||||
})
|
||||
expect(importMethod).toBe('copy')
|
||||
|
||||
@@ -37,7 +37,7 @@ export async function storeAdd (
|
||||
projectDir: prefix,
|
||||
registry: (dep.alias && pickRegistryForPackage(registries, dep.alias)) ?? registries.default,
|
||||
})
|
||||
await pkgResponse.files!()
|
||||
await pkgResponse.fetching!()
|
||||
globalInfo(`+ ${pkgResponse.body.id}`)
|
||||
} catch (e: any) { // eslint-disable-line
|
||||
hasFailures = true
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { fetch } from '@pnpm/fetch'
|
||||
import {
|
||||
type PkgRequestFetchResult,
|
||||
type FetchPackageToStoreOptions,
|
||||
type PackageFilesResponse,
|
||||
type PackageResponse,
|
||||
@@ -7,7 +8,6 @@ import {
|
||||
type StoreController,
|
||||
type WantedDependency,
|
||||
} from '@pnpm/store-controller-types'
|
||||
import { type DependencyManifest } from '@pnpm/types'
|
||||
|
||||
import pLimit from 'p-limit'
|
||||
import pShare from 'promise-share'
|
||||
@@ -90,70 +90,41 @@ async function requestPackage (
|
||||
options: RequestPackageOptions
|
||||
): Promise<PackageResponse> {
|
||||
const msgId = uuidv4()
|
||||
|
||||
return limitedFetch(`${remotePrefix}/requestPackage`, {
|
||||
const packageResponseBody = await limitedFetch(`${remotePrefix}/requestPackage`, {
|
||||
msgId,
|
||||
options,
|
||||
wantedDependency,
|
||||
})
|
||||
.then((packageResponseBody: any) => { // eslint-disable-line @typescript-eslint/no-explicit-any
|
||||
const fetchingBundledManifest = !packageResponseBody['fetchingBundledManifestInProgress'] // eslint-disable-line
|
||||
? undefined
|
||||
: limitedFetch(`${remotePrefix}/rawManifestResponse`, {
|
||||
msgId,
|
||||
})
|
||||
delete packageResponseBody['fetchingBundledManifestInProgress'] // eslint-disable-line
|
||||
|
||||
if (options.skipFetch) {
|
||||
return {
|
||||
body: packageResponseBody,
|
||||
bundledManifest: fetchingBundledManifest && pShare(fetchingBundledManifest),
|
||||
}
|
||||
}
|
||||
|
||||
const fetchingFiles = limitedFetch(`${remotePrefix}/packageFilesResponse`, {
|
||||
msgId,
|
||||
})
|
||||
return {
|
||||
body: packageResponseBody,
|
||||
bundledManifest: fetchingBundledManifest && pShare(fetchingBundledManifest),
|
||||
files: pShare(fetchingFiles),
|
||||
finishing: pShare(Promise.all([fetchingBundledManifest, fetchingFiles]).then(() => undefined)),
|
||||
}
|
||||
})
|
||||
const fetchingFiles = limitedFetch(`${remotePrefix}/packageFilesResponse`, {
|
||||
msgId,
|
||||
})
|
||||
return {
|
||||
body: packageResponseBody,
|
||||
fetching: pShare(fetchingFiles),
|
||||
}
|
||||
}
|
||||
|
||||
function fetchPackage (
|
||||
async function fetchPackage (
|
||||
remotePrefix: string,
|
||||
limitedFetch: (url: string, body: object) => any, // eslint-disable-line
|
||||
options: FetchPackageToStoreOptions
|
||||
): {
|
||||
bundledManifest?: () => Promise<DependencyManifest>
|
||||
files: () => Promise<PackageFilesResponse>
|
||||
): Promise<{
|
||||
fetching: () => Promise<PkgRequestFetchResult>
|
||||
filesIndexFile: string
|
||||
finishing: () => Promise<void>
|
||||
inStoreLocation: string
|
||||
} {
|
||||
}> {
|
||||
const msgId = uuidv4()
|
||||
|
||||
return limitedFetch(`${remotePrefix}/fetchPackage`, {
|
||||
const fetchResponseBody = await limitedFetch(`${remotePrefix}/fetchPackage`, {
|
||||
msgId,
|
||||
options,
|
||||
}) as object & { filesIndexFile: string, inStoreLocation: string }
|
||||
const fetching = limitedFetch(`${remotePrefix}/packageFilesResponse`, {
|
||||
msgId,
|
||||
})
|
||||
.then((fetchResponseBody: object & { filesIndexFile: string, inStoreLocation: string }) => {
|
||||
const fetchingBundledManifest = options.fetchRawManifest
|
||||
? limitedFetch(`${remotePrefix}/rawManifestResponse`, { msgId })
|
||||
: undefined
|
||||
|
||||
const fetchingFiles = limitedFetch(`${remotePrefix}/packageFilesResponse`, {
|
||||
msgId,
|
||||
})
|
||||
return {
|
||||
bundledManifest: fetchingBundledManifest && pShare(fetchingBundledManifest),
|
||||
files: pShare(fetchingFiles),
|
||||
filesIndexFile: fetchResponseBody.filesIndexFile,
|
||||
finishing: pShare(Promise.all([fetchingBundledManifest, fetchingFiles]).then(() => undefined)),
|
||||
inStoreLocation: fetchResponseBody.inStoreLocation,
|
||||
}
|
||||
})
|
||||
return {
|
||||
fetching: pShare(fetching),
|
||||
filesIndexFile: fetchResponseBody.filesIndexFile,
|
||||
inStoreLocation: fetchResponseBody.inStoreLocation,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
import http, { type IncomingMessage, type Server, type ServerResponse } from 'http'
|
||||
import { globalInfo } from '@pnpm/logger'
|
||||
import {
|
||||
type BundledManifestFunction,
|
||||
type PackageFilesResponse,
|
||||
type PkgRequestFetchResult,
|
||||
type RequestPackageOptions,
|
||||
type StoreController,
|
||||
type WantedDependency,
|
||||
type FetchPackageToStoreFunction,
|
||||
} from '@pnpm/store-controller-types'
|
||||
import { locking } from './lock'
|
||||
|
||||
@@ -34,8 +34,7 @@ export function createServer (
|
||||
ignoreUploadRequests?: boolean
|
||||
}
|
||||
) {
|
||||
const rawManifestPromises: Record<string, BundledManifestFunction> = {}
|
||||
const filesPromises: Record<string, () => Promise<PackageFilesResponse>> = {}
|
||||
const filesPromises: Record<string, () => Promise<PkgRequestFetchResult>> = {}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-invalid-void-type
|
||||
const lock = locking<void>()
|
||||
@@ -75,13 +74,8 @@ export function createServer (
|
||||
try {
|
||||
body = await bodyPromise
|
||||
const pkgResponse = await store.requestPackage(body.wantedDependency, body.options)
|
||||
if (pkgResponse['bundledManifest']) {
|
||||
rawManifestPromises[body.msgId] = pkgResponse['bundledManifest']
|
||||
// @ts-expect-error
|
||||
pkgResponse.body['fetchingBundledManifestInProgress'] = true
|
||||
}
|
||||
if (pkgResponse['files']) {
|
||||
filesPromises[body.msgId] = pkgResponse['files']
|
||||
if (pkgResponse.fetching) {
|
||||
filesPromises[body.msgId] = pkgResponse.fetching
|
||||
}
|
||||
res.end(JSON.stringify(pkgResponse.body))
|
||||
} catch (err: any) { // eslint-disable-line
|
||||
@@ -97,13 +91,8 @@ export function createServer (
|
||||
case '/fetchPackage': {
|
||||
try {
|
||||
body = await bodyPromise
|
||||
const pkgResponse = store.fetchPackage(body.options as any) // eslint-disable-line
|
||||
if (pkgResponse['bundledManifest']) { // eslint-disable-line
|
||||
rawManifestPromises[body.msgId] = pkgResponse['bundledManifest'] // eslint-disable-line
|
||||
}
|
||||
if (pkgResponse['files']) { // eslint-disable-line
|
||||
filesPromises[body.msgId] = pkgResponse['files'] // eslint-disable-line
|
||||
}
|
||||
const pkgResponse = (store.fetchPackage as FetchPackageToStoreFunction)(body.options as any) // eslint-disable-line
|
||||
filesPromises[body.msgId] = pkgResponse.fetching
|
||||
res.end(JSON.stringify({ filesIndexFile: pkgResponse.filesIndexFile }))
|
||||
} catch (err: any) { // eslint-disable-line
|
||||
res.end(JSON.stringify({
|
||||
@@ -122,13 +111,6 @@ export function createServer (
|
||||
res.end(JSON.stringify(filesResponse))
|
||||
break
|
||||
}
|
||||
case '/rawManifestResponse': {
|
||||
body = await bodyPromise
|
||||
const manifestResponse = await rawManifestPromises[body.msgId]()
|
||||
delete rawManifestPromises[body.msgId]
|
||||
res.end(JSON.stringify(manifestResponse))
|
||||
break
|
||||
}
|
||||
case '/prune':
|
||||
// Disable store pruning when a server is running
|
||||
res.statusCode = 403
|
||||
|
||||
@@ -56,18 +56,15 @@ test('server', async () => {
|
||||
}
|
||||
)
|
||||
|
||||
expect((await response.bundledManifest!())?.name).toBe('is-positive')
|
||||
const { bundledManifest, files } = await response.fetching!()
|
||||
expect(bundledManifest?.name).toBe('is-positive')
|
||||
expect(response.body.id).toBe('registry.npmjs.org/is-positive/1.0.0')
|
||||
|
||||
expect(response.body.manifest!.name).toBe('is-positive')
|
||||
expect(response.body.manifest!.version).toBe('1.0.0')
|
||||
|
||||
const files = await response.files!()
|
||||
expect(files.fromStore).toBeFalsy()
|
||||
expect(files.filesIndex).toHaveProperty(['package.json'])
|
||||
expect(response.finishing).toBeTruthy()
|
||||
|
||||
await response.finishing!()
|
||||
|
||||
await server.close()
|
||||
await storeCtrl.close()
|
||||
@@ -86,7 +83,7 @@ test('fetchPackage', async () => {
|
||||
const storeCtrl = await connectStoreController({ remotePrefix, concurrency: 100 })
|
||||
const pkgId = 'registry.npmjs.org/is-positive/1.0.0'
|
||||
// This should be fixed
|
||||
// eslint-disable-next-line
|
||||
|
||||
const response = await storeCtrl.fetchPackage({
|
||||
fetchRawManifest: true,
|
||||
force: false,
|
||||
@@ -103,14 +100,11 @@ test('fetchPackage', async () => {
|
||||
|
||||
expect(typeof response.filesIndexFile).toBe('string')
|
||||
|
||||
expect(await response.bundledManifest!()).toBeTruthy()
|
||||
const { bundledManifest, files } = await response.fetching!()
|
||||
expect(bundledManifest).toBeTruthy()
|
||||
|
||||
const files = await response['files']()
|
||||
expect(files.fromStore).toBeFalsy()
|
||||
expect(files.filesIndex).toHaveProperty(['package.json'])
|
||||
expect(response).toHaveProperty(['finishing'])
|
||||
|
||||
await response['finishing']()
|
||||
|
||||
await server.close()
|
||||
await storeCtrl.close()
|
||||
|
||||
@@ -45,7 +45,7 @@ export type UploadPkgToStore = (builtPkgLocation: string, opts: UploadPkgToStore
|
||||
|
||||
export interface StoreController {
|
||||
requestPackage: RequestPackageFunction
|
||||
fetchPackage: FetchPackageToStoreFunction
|
||||
fetchPackage: FetchPackageToStoreFunction | FetchPackageToStoreFunctionAsync
|
||||
getFilesIndexFilePath: GetFilesIndexFilePath
|
||||
importPackage: ImportPackageFunctionAsync
|
||||
close: () => Promise<void>
|
||||
@@ -53,15 +53,25 @@ export interface StoreController {
|
||||
upload: UploadPkgToStore
|
||||
}
|
||||
|
||||
export interface PkgRequestFetchResult {
|
||||
bundledManifest?: BundledManifest
|
||||
files: PackageFilesResponse
|
||||
}
|
||||
|
||||
export type FetchPackageToStoreFunction = (
|
||||
opts: FetchPackageToStoreOptions
|
||||
) => {
|
||||
bundledManifest?: BundledManifestFunction
|
||||
filesIndexFile: string
|
||||
files: () => Promise<PackageFilesResponse>
|
||||
finishing: () => Promise<void>
|
||||
fetching: () => Promise<PkgRequestFetchResult>
|
||||
}
|
||||
|
||||
export type FetchPackageToStoreFunctionAsync = (
|
||||
opts: FetchPackageToStoreOptions
|
||||
) => Promise<{
|
||||
filesIndexFile: string
|
||||
fetching: () => Promise<PkgRequestFetchResult>
|
||||
}>
|
||||
|
||||
export type GetFilesIndexFilePath = (opts: Pick<FetchPackageToStoreOptions, 'pkg' | 'ignoreScripts'>) => {
|
||||
filesIndexFile: string
|
||||
target: string
|
||||
@@ -122,10 +132,8 @@ export interface RequestPackageOptions {
|
||||
export type BundledManifestFunction = () => Promise<BundledManifest | undefined>
|
||||
|
||||
export interface PackageResponse {
|
||||
bundledManifest?: BundledManifestFunction
|
||||
files?: () => Promise<PackageFilesResponse>
|
||||
fetching?: () => Promise<PkgRequestFetchResult>
|
||||
filesIndexFile?: string
|
||||
finishing?: () => Promise<void> // a package request is finished once its integrity is generated and saved
|
||||
body: {
|
||||
isLocal: boolean
|
||||
isInstallable?: boolean
|
||||
|
||||
Reference in New Issue
Block a user