diff --git a/.changeset/fix-minimum-release-age-cached-abbreviated-metadata.md b/.changeset/fix-minimum-release-age-cached-abbreviated-metadata.md new file mode 100644 index 0000000000..3016a5fd62 --- /dev/null +++ b/.changeset/fix-minimum-release-age-cached-abbreviated-metadata.md @@ -0,0 +1,6 @@ +--- +"@pnpm/npm-resolver": patch +"pnpm": patch +--- + +Fixed `minimumReleaseAge` handling when cached metadata is abbreviated. The npm registry returns abbreviated package metadata (without the per-version `time` field) by default, which made the maturity check throw `ERR_PNPM_MISSING_TIME` whenever cached abbreviated metadata was reused. pnpm now upgrades cached abbreviated metadata to the full document via a follow-up fetch when `minimumReleaseAge` is active, persists the upgrade to the on-disk cache so subsequent installs skip the extra fetch, and lets `ERR_PNPM_MISSING_TIME` from the cache fast-path fall through to the network fetch even under strict mode. diff --git a/resolving/npm-resolver/src/pickPackage.ts b/resolving/npm-resolver/src/pickPackage.ts index 7a564a3f16..b30503ad9c 100644 --- a/resolving/npm-resolver/src/pickPackage.ts +++ b/resolving/npm-resolver/src/pickPackage.ts @@ -1,5 +1,6 @@ import { promises as fs } from 'fs' import path from 'path' +import util from 'util' import { ABBREVIATED_META_DIR, FULL_META_DIR, FULL_FILTERED_META_DIR } from '@pnpm/constants' import { createHexHash } from '@pnpm/crypto.hash' import { PnpmError } from '@pnpm/error' @@ -135,16 +136,25 @@ export async function pickPackage ( : ABBREVIATED_META_DIR // Cache key includes fullMetadata to avoid returning abbreviated metadata when full metadata is requested. const cacheKey = fullMetadata ? `${spec.name}:full` : spec.name - const cachedMeta = ctx.metaCache.get(cacheKey) - if (cachedMeta != null) { - return { - meta: cachedMeta, - pickedPackage: _pickPackageFromMeta(cachedMeta), - } - } - const registryName = getRegistryName(opts.registry) const pkgMirror = path.join(ctx.cacheDir, metaDir, registryName, `${encodePkgName(spec.name)}.json`) + const cachedMeta = ctx.metaCache.get(cacheKey) + if (cachedMeta != null) { + // The in-memory cache may hold abbreviated metadata from an earlier call + // that didn't need `time` (no publishedBy then). If this call has + // publishedBy, upgrade to full metadata so the maturity check can run on + // real time data instead of throwing ERR_PNPM_MISSING_TIME. + const upgraded = await maybeUpgradeAbbreviatedMetaForReleaseAge(ctx, spec, opts, cachedMeta) + let metaForCache = upgraded.meta + if (upgraded.upgraded) { + metaForCache = persistUpgradedMeta(ctx, pkgMirror, metaForCache, opts.dryRun) + ctx.metaCache.set(cacheKey, metaForCache) + } + return { + meta: metaForCache, + pickedPackage: _pickPackageFromMeta(metaForCache), + } + } return runLimited(pkgMirror, async (limit) => { let metaCachedInStore: PackageMeta | null | undefined @@ -161,6 +171,14 @@ export async function pickPackage ( } if (metaCachedInStore != null) { + // Disk-cached meta may be abbreviated; upgrade for the maturity check + // instead of letting the picker throw ERR_PNPM_MISSING_TIME. + const upgraded = await maybeUpgradeAbbreviatedMetaForReleaseAge(ctx, spec, opts, metaCachedInStore) + metaCachedInStore = upgraded.meta + if (upgraded.upgraded) { + metaCachedInStore = persistUpgradedMeta(ctx, pkgMirror, metaCachedInStore, opts.dryRun) + ctx.metaCache.set(cacheKey, metaCachedInStore) + } const pickedPackage = _pickPackageFromMeta(metaCachedInStore) if (pickedPackage) { return { @@ -184,8 +202,12 @@ export async function pickPackage ( pickedPackage, } } - } catch (err) { - if (ctx.strictPublishedByCheck) { + } catch (err: unknown) { + // MISSING_TIME from cached abbreviated metadata should fall through + // to the network fetch path even under strictPublishedByCheck — + // the fetch will upgrade to full metadata and run the maturity check + // on real `time` data. + if (shouldRethrowFromFastPathCache(err, ctx.strictPublishedByCheck)) { throw err } } @@ -202,8 +224,8 @@ export async function pickPackage ( pickedPackage, } } - } catch (err) { - if (ctx.strictPublishedByCheck) { + } catch (err: unknown) { + if (shouldRethrowFromFastPathCache(err, ctx.strictPublishedByCheck)) { throw err } } @@ -216,6 +238,22 @@ export async function pickPackage ( fullMetadata, registry: opts.registry, }) + // When publishedBy is active but the registry returned abbreviated + // metadata (no per-version `time`), re-fetch with `fullMetadata: true` + // so the maturity check can run properly. Without this, abbreviated + // metadata + publishedBy would throw ERR_PNPM_MISSING_TIME. + if ( + opts.publishedBy && + !fullMetadata && + meta.time == null && + opts.publishedByExclude?.(spec.name) !== true + ) { + meta = await ctx.fetch(spec.name, { + authHeaderValue: opts.authHeaderValue, + fullMetadata: true, + registry: opts.registry, + }) + } if (ctx.filterMetadata) { meta = clearMeta(meta) } @@ -252,6 +290,80 @@ export async function pickPackage ( }) } +// When `publishedBy` is active and the cached metadata is abbreviated (no +// per-version `time`), the maturity check can't run on the data we have and +// `pickPackageFromMeta` will throw ERR_PNPM_MISSING_TIME. Upgrade to full +// metadata via a follow-up fetch so the check can proceed on real `time` data. +async function maybeUpgradeAbbreviatedMetaForReleaseAge ( + ctx: { + fetch: (pkgName: string, opts: { registry: string, authHeaderValue?: string, fullMetadata?: boolean }) => Promise + offline?: boolean + }, + spec: RegistryPackageSpec, + opts: { + publishedBy?: Date + publishedByExclude?: PickPackageFromMetaOptions['publishedByExclude'] + authHeaderValue?: string + registry: string + }, + meta: PackageMeta +): Promise<{ meta: PackageMeta, upgraded: boolean }> { + if ( + ctx.offline === true || + !opts.publishedBy || + meta.time != null || + opts.publishedByExclude?.(spec.name) === true + ) { + return { meta, upgraded: false } + } + const fullMeta = await ctx.fetch(spec.name, { + authHeaderValue: opts.authHeaderValue, + fullMetadata: true, + registry: opts.registry, + }) + return { meta: fullMeta, upgraded: true } +} + +// Returns true when a fast-path cache catch should rethrow. MISSING_TIME is +// excluded so callers fall through to the network fetch path, which can +// upgrade abbreviated cached metadata to full and run the maturity check on +// real `time` data. +function shouldRethrowFromFastPathCache (err: unknown, strictPublishedByCheck: boolean | undefined): boolean { + if (isMissingTimeError(err)) return false + return strictPublishedByCheck === true +} + +function isMissingTimeError (err: unknown): boolean { + return util.types.isNativeError(err) && 'code' in err && err.code === 'ERR_PNPM_MISSING_TIME' +} + +// Persists upgraded full metadata to the on-disk cache mirror and returns the +// meta to store in the in-memory cache. When `filterMetadata` is on the +// returned meta is stripped via `clearMeta`. Without persisting here, a fresh +// process would re-trigger the upgrade fetch on its next install since the +// on-disk cache still holds the abbreviated form. +function persistUpgradedMeta ( + ctx: { filterMetadata?: boolean }, + pkgMirror: string, + meta: PackageMeta, + dryRun: boolean +): PackageMeta { + const metaForCache = ctx.filterMetadata ? clearMeta(meta) : meta + metaForCache.cachedAt = Date.now() + if (!dryRun) { + const stringifiedMeta = JSON.stringify(metaForCache) + // eslint-disable-next-line @typescript-eslint/no-floating-promises + runLimited(pkgMirror, (l) => l(async () => { + try { + await saveMeta(pkgMirror, stringifiedMeta) + } catch (err: any) { // eslint-disable-line + // We don't care if this file was not written to the cache + } + })) + } + return metaForCache +} + function clearMeta (pkg: PackageMeta): PackageMeta { const versions: PackageMeta['versions'] = {} for (const [version, info] of Object.entries(pkg.versions)) { diff --git a/resolving/npm-resolver/test/publishedBy.test.ts b/resolving/npm-resolver/test/publishedBy.test.ts index 0f7848b9b8..38711816e3 100644 --- a/resolving/npm-resolver/test/publishedBy.test.ts +++ b/resolving/npm-resolver/test/publishedBy.test.ts @@ -1,6 +1,6 @@ import fs from 'fs' import path from 'path' -import { FULL_FILTERED_META_DIR } from '@pnpm/constants' +import { ABBREVIATED_META_DIR, FULL_FILTERED_META_DIR } from '@pnpm/constants' import { createFetchFromRegistry } from '@pnpm/fetch' import { createNpmResolver } from '@pnpm/npm-resolver' import { type Registries } from '@pnpm/types' @@ -18,6 +18,7 @@ const registries: Registries = { /* eslint-disable @typescript-eslint/no-explicit-any */ const badDatesMeta = loadJsonFile.sync(f.find('bad-dates.json')) const isPositiveMeta = loadJsonFile.sync(f.find('is-positive-full.json')) +const isPositiveAbbreviatedMeta = loadJsonFile.sync(f.find('is-positive.json')) /* eslint-enable @typescript-eslint/no-explicit-any */ const fetch = createFetchFromRegistry({}) @@ -147,3 +148,97 @@ test('should skip time field validation for excluded packages', async () => { expect(resolveResult!.resolvedVia).toBe('npm-registry') expect(resolveResult!.manifest.version).toBe('3.1.0') }) + +test('re-fetch full metadata when registry returns abbreviated metadata and publishedBy is set', async () => { + // The npm registry returns abbreviated metadata by default (no per-version `time` field). + // When publishedBy is set, pnpm needs `time` for the maturity check, so it should + // automatically re-fetch the full metadata document. + nock(registries.default) + .get('/is-positive') + .reply(200, isPositiveAbbreviatedMeta) + nock(registries.default) + .get('/is-positive') + .reply(200, isPositiveMeta) + + const cacheDir = tempy.directory() + const { resolveFromNpm } = createResolveFromNpm({ + cacheDir, + registries, + }) + // 3.0.0 was published 2015-07-10 (mature relative to publishedBy 2016-01-01); + // 3.1.0 was published 2016-01-11 (not yet mature). So resolution must pick 3.0.0. + const resolveResult = await resolveFromNpm({ alias: 'is-positive', bareSpecifier: '^3.0.0' }, { + publishedBy: new Date('2016-01-01T00:00:00.000Z'), + }) + + expect(resolveResult!.resolvedVia).toBe('npm-registry') + expect(resolveResult!.id).toBe('is-positive@3.0.0') +}) + +test('upgrade disk-cached abbreviated metadata to full when publishedBy is set', async () => { + // The disk cache holds abbreviated metadata (no per-version `time`). When a + // later install uses publishedBy, pnpm needs to upgrade to full metadata so + // the maturity check has real `time` data. + const cacheDir = tempy.directory() + fs.mkdirSync(path.join(cacheDir, `${ABBREVIATED_META_DIR}/registry.npmjs.org`), { recursive: true }) + fs.writeFileSync( + path.join(cacheDir, `${ABBREVIATED_META_DIR}/registry.npmjs.org/is-positive.json`), + JSON.stringify(isPositiveAbbreviatedMeta), + 'utf8' + ) + + // The upgrade fetch goes to the registry asking for full metadata. + nock(registries.default) + .get('/is-positive') + .reply(200, isPositiveMeta) + + const { resolveFromNpm } = createResolveFromNpm({ + cacheDir, + registries, + preferOffline: true, + }) + const resolveResult = await resolveFromNpm({ alias: 'is-positive', bareSpecifier: '^3.0.0' }, { + publishedBy: new Date('2016-01-01T00:00:00.000Z'), + }) + + expect(resolveResult!.resolvedVia).toBe('npm-registry') + expect(resolveResult!.id).toBe('is-positive@3.0.0') +}) + +test('strictPublishedByCheck=true does not rethrow ERR_PNPM_MISSING_TIME from the version-spec cache path', async () => { + // Regression test: the version-spec fast path + // (`!opts.updateToLatest && spec.type === 'version'`) in pickPackage used to + // rethrow ERR_PNPM_MISSING_TIME under strictPublishedByCheck, instead of + // falling through to the registry-fetch path. The fix lets MISSING_TIME from + // cached abbreviated meta fall through so the fetch can upgrade to full + // metadata and run the maturity check on real `time` data. + const cacheDir = tempy.directory() + fs.mkdirSync(path.join(cacheDir, `${ABBREVIATED_META_DIR}/registry.npmjs.org`), { recursive: true }) + // Stash abbreviated meta on disk so the version-spec fast path loads it and + // pickPackageFromMeta throws MISSING_TIME on the maturity check. + fs.writeFileSync( + path.join(cacheDir, `${ABBREVIATED_META_DIR}/registry.npmjs.org/is-positive.json`), + JSON.stringify(isPositiveAbbreviatedMeta), + 'utf8' + ) + + // The fall-through fetch returns full metadata with `time`. + nock(registries.default) + .get('/is-positive') + .reply(200, isPositiveMeta) + + const { resolveFromNpm } = createResolveFromNpm({ + cacheDir, + registries, + strictPublishedByCheck: true, + }) + + // Exact-version specifier hits the version-spec cache path. 3.0.0 was + // published 2015-07-10, mature relative to publishedBy 2015-08-17. + const resolveResult = await resolveFromNpm({ alias: 'is-positive', bareSpecifier: '3.0.0' }, { + publishedBy: new Date('2015-08-17T19:26:00.508Z'), + }) + + expect(resolveResult!.resolvedVia).toBe('npm-registry') + expect(resolveResult!.id).toBe('is-positive@3.0.0') +})