refactor: create a separate package for filtering package documents by publish date (#10006)

.changeset/tough-buses-exist.md

@@ -0,0 +1,6 @@
+---
+"@pnpm/registry.pkg-doc-filter": major
+"@pnpm/registry.types": major
+---
+
+Initial release.

pnpm-lock.yaml

@@ -6706,6 +6706,35 @@ importers:
         specifier: workspace:*
         version: 'link:'
 
+  registry/pkg-doc-filter:
+    dependencies:
+      '@pnpm/registry.types':
+        specifier: workspace:*
+        version: link:../types
+      semver:
+        specifier: 'catalog:'
+        version: 7.7.1
+    devDependencies:
+      '@pnpm/logger':
+        specifier: workspace:*
+        version: link:../../packages/logger
+      '@pnpm/registry.pkg-doc-filter':
+        specifier: workspace:*
+        version: 'link:'
+      '@types/semver':
+        specifier: 'catalog:'
+        version: 7.5.3
+
+  registry/types:
+    dependencies:
+      '@pnpm/types':
+        specifier: workspace:*
+        version: link:../../packages/types
+    devDependencies:
+      '@pnpm/registry.types':
+        specifier: workspace:*
+        version: 'link:'
+
   releasing/plugin-commands-deploy:
     dependencies:
       '@pnpm/cli-utils':
@@ -7199,6 +7228,12 @@ importers:
       '@pnpm/pick-registry-for-package':
         specifier: workspace:*
         version: link:../../config/pick-registry-for-package
+      '@pnpm/registry.pkg-doc-filter':
+        specifier: workspace:*
+        version: link:../../registry/pkg-doc-filter
+      '@pnpm/registry.types':
+        specifier: workspace:*
+        version: link:../../registry/types
       '@pnpm/resolve-workspace-range':
         specifier: workspace:*
         version: link:../../workspace/resolve-workspace-range

pnpm-workspace.yaml

@@ -31,6 +31,7 @@ packages:
   - tools/*
   - worker
   - pnpm/artifacts/*
+  - registry/*
   - releasing/*
   - resolving/*
   - reviewing/*

registry/pkg-doc-filter/README.md

@@ -0,0 +1,28 @@
+# @pnpm/registry.pkg-doc-filter
+
+> Filters the package document from the registry
+
+<!--@shields('npm')-->
+[![npm version](https://img.shields.io/npm/v/@pnpm/registry.pkg-doc-filter.svg)](https://www.npmjs.com/package/@pnpm/registry.pkg-doc-filter)
+<!--/@-->
+
+## Installation
+
+```
+pnpm add @pnpm/registry.pkg-doc-filter
+```
+
+## Usage
+
+```ts
+import { filterPkgDocByPublishDate } from '@pnpm/registry.pkg-doc-filter'
+
+const pkgDoc = await (await fetch('https://registry.npmjs.org/is-odd')).json()
+
+// Keep only those versions in the document that were published before Jan 1 2023.
+filterPkgDocByPublishDate(pkgDoc, new Date('2023-01-01'))
+```
+
+## License
+
+MIT

registry/pkg-doc-filter/package.json

@@ -0,0 +1,53 @@
+{
+  "name": "@pnpm/registry.pkg-doc-filter",
+  "version": "1000.0.0-0",
+  "description": "Filters the package document from the registry",
+  "keywords": [
+    "pnpm",
+    "pnpm10",
+    "npm",
+    "pkg-doc"
+  ],
+  "license": "MIT",
+  "funding": "https://opencollective.com/pnpm",
+  "repository": "https://github.com/pnpm/pnpm/blob/main/registry/pkg-doc-filter",
+  "homepage": "https://github.com/pnpm/pnpm/blob/main/registry/pkg-doc-filter#readme",
+  "bugs": {
+    "url": "https://github.com/pnpm/pnpm/issues"
+  },
+  "type": "commonjs",
+  "main": "lib/index.js",
+  "types": "lib/index.d.ts",
+  "exports": {
+    ".": "./lib/index.js"
+  },
+  "files": [
+    "lib",
+    "!*.map"
+  ],
+  "scripts": {
+    "lint": "eslint \"src/**/*.ts\" \"test/**/*.ts\"",
+    "_test": "jest",
+    "test": "pnpm run compile && pnpm run _test",
+    "prepublishOnly": "pnpm run compile",
+    "compile": "tsc --build && pnpm run lint --fix"
+  },
+  "dependencies": {
+    "@pnpm/registry.types": "workspace:*",
+    "semver": "catalog:"
+  },
+  "peerDependencies": {
+    "@pnpm/logger": "catalog:"
+  },
+  "devDependencies": {
+    "@pnpm/logger": "workspace:*",
+    "@pnpm/registry.pkg-doc-filter": "workspace:*",
+    "@types/semver": "catalog:"
+  },
+  "engines": {
+    "node": ">=18.12"
+  },
+  "jest": {
+    "preset": "@pnpm/jest-config"
+  }
+}

registry/pkg-doc-filter/src/index.ts

@@ -0,0 +1,77 @@
+import { globalWarn } from '@pnpm/logger'
+import { type PackageDocumentWithTime } from '@pnpm/registry.types'
+import semver from 'semver'
+
+export function filterPkgDocByPublishDate (pkgDoc: PackageDocumentWithTime, publishedBy: Date): PackageDocumentWithTime {
+  const versionsWithinDate: PackageDocumentWithTime['versions'] = {}
+  for (const version in pkgDoc.versions) {
+    if (!Object.hasOwn(pkgDoc.versions, version)) continue
+    const timeStr = pkgDoc.time[version]
+    if (timeStr && new Date(timeStr) <= publishedBy) {
+      versionsWithinDate[version] = pkgDoc.versions[version]
+    }
+  }
+
+  const distTagsWithinDate: PackageDocumentWithTime['dist-tags'] = {}
+  const allDistTags = pkgDoc['dist-tags'] ?? {}
+  const parsedSemverCache = new Map<string, semver.SemVer>()
+  function tryParseSemver (semverStr: string): semver.SemVer | null {
+    let parsedSemver = parsedSemverCache.get(semverStr)
+    if (!parsedSemver) {
+      try {
+        parsedSemver = new semver.SemVer(semverStr, true)
+      } catch {
+        return null
+      }
+      parsedSemverCache.set(semverStr, parsedSemver)
+    }
+    return parsedSemver
+  }
+  for (const tag in allDistTags) {
+    if (!Object.hasOwn(allDistTags, tag)) continue
+    const distTagVersion = allDistTags[tag]
+    if (versionsWithinDate[distTagVersion]) {
+      distTagsWithinDate[tag] = distTagVersion
+      continue
+    }
+    // Repopulate the tag to the highest version available within date that has the same major as the original tag's version
+    const originalSemVer = tryParseSemver(distTagVersion)
+    if (!originalSemVer) continue
+    const originalIsPrerelease = (originalSemVer.prerelease.length > 0)
+    let bestVersion: string | undefined
+    for (const candidate in versionsWithinDate) {
+      if (!Object.hasOwn(versionsWithinDate, candidate)) continue
+      const candidateParsed = tryParseSemver(candidate)
+      if (
+        !candidateParsed ||
+        candidateParsed.major !== originalSemVer.major ||
+        (candidateParsed.prerelease.length > 0) !== originalIsPrerelease
+      ) continue
+      if (!bestVersion) {
+        bestVersion = candidate
+      } else {
+        try {
+          const candidateIsDeprecated = pkgDoc.versions[candidate].deprecated != null
+          const bestVersionIsDeprecated = pkgDoc.versions[bestVersion].deprecated != null
+          if (
+            (semver.gt(candidate, bestVersion, true) && (bestVersionIsDeprecated === candidateIsDeprecated)) ||
+            (bestVersionIsDeprecated && !candidateIsDeprecated)
+          ) {
+            bestVersion = candidate
+          }
+        } catch (err) {
+          globalWarn(`Failed to compare semver versions ${candidate} and ${bestVersion} from packument of ${pkgDoc.name}, skipping candidate version.`)
+        }
+      }
+    }
+    if (bestVersion) {
+      distTagsWithinDate[tag] = bestVersion
+    }
+  }
+
+  return {
+    ...pkgDoc,
+    versions: versionsWithinDate,
+    'dist-tags': distTagsWithinDate,
+  }
+}

registry/pkg-doc-filter/test/__snapshots__/index.ts.snap

@@ -0,0 +1,43 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`filterPkgDocByPublishDate 1`] = `
+{
+  "dist-tags": {
+    "latest": "3.0.0",
+  },
+  "name": "dist-tag-date",
+  "time": {
+    "2.9.9": "2020-01-01T00:00:00.000Z",
+    "3.0.0": "2020-02-01T00:00:00.000Z",
+    "3.1.0": "2020-03-01T00:00:00.000Z",
+    "3.2.0": "2020-05-01T00:00:00.000Z",
+  },
+  "versions": {
+    "2.9.9": {
+      "dist": {
+        "shasum": "",
+        "tarball": "https://registry.npmjs.org/dist-tag-date/-/dist-tag-date-2.9.9.tgz",
+      },
+      "name": "dist-tag-date",
+      "version": "2.9.9",
+    },
+    "3.0.0": {
+      "dist": {
+        "shasum": "",
+        "tarball": "https://registry.npmjs.org/dist-tag-date/-/dist-tag-date-3.0.0.tgz",
+      },
+      "name": "dist-tag-date",
+      "version": "3.0.0",
+    },
+    "3.1.0": {
+      "deprecated": "This version is deprecated",
+      "dist": {
+        "shasum": "",
+        "tarball": "https://registry.npmjs.org/dist-tag-date/-/dist-tag-date-3.1.0.tgz",
+      },
+      "name": "dist-tag-date",
+      "version": "3.1.0",
+    },
+  },
+}
+`;

registry/pkg-doc-filter/test/index.ts

@@ -0,0 +1,41 @@
+import { filterPkgDocByPublishDate } from '@pnpm/registry.pkg-doc-filter'
+
+test('filterPkgDocByPublishDate', () => {
+  const cutoff = new Date('2020-04-01T00:00:00.000Z')
+  const name = 'dist-tag-date'
+  expect(filterPkgDocByPublishDate({
+    name,
+    versions: {
+      '3.0.0': {
+        name,
+        version: '3.0.0',
+        dist: { tarball: `https://registry.npmjs.org/${name}/-/${name}-3.0.0.tgz`, shasum: '' },
+      },
+      '3.1.0': {
+        name,
+        version: '3.1.0',
+        dist: { tarball: `https://registry.npmjs.org/${name}/-/${name}-3.1.0.tgz`, shasum: '' },
+        deprecated: 'This version is deprecated',
+      },
+      '3.2.0': {
+        name,
+        version: '3.2.0',
+        dist: { tarball: `https://registry.npmjs.org/${name}/-/${name}-3.2.0.tgz`, shasum: '' },
+      },
+      '2.9.9': {
+        name,
+        version: '2.9.9',
+        dist: { tarball: `https://registry.npmjs.org/${name}/-/${name}-2.9.9.tgz`, shasum: '' },
+      },
+    },
+    'dist-tags': {
+      latest: '3.2.0',
+    },
+    time: {
+      '2.9.9': '2020-01-01T00:00:00.000Z',
+      '3.0.0': '2020-02-01T00:00:00.000Z',
+      '3.1.0': '2020-03-01T00:00:00.000Z',
+      '3.2.0': '2020-05-01T00:00:00.000Z',
+    },
+  }, cutoff)).toMatchSnapshot()
+})

registry/pkg-doc-filter/test/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "extends": "../tsconfig.json",
+  "compilerOptions": {
+    "noEmit": false,
+    "outDir": "../test.lib",
+    "rootDir": "."
+  },
+  "include": [
+    "**/*.ts",
+    "../../../__typings__/**/*.d.ts"
+  ],
+  "references": [
+    {
+      "path": ".."
+    }
+  ]
+}

registry/pkg-doc-filter/tsconfig.json

@@ -0,0 +1,19 @@
+{
+  "extends": "@pnpm/tsconfig",
+  "compilerOptions": {
+    "outDir": "lib",
+    "rootDir": "src"
+  },
+  "include": [
+    "src/**/*.ts",
+    "../../__typings__/**/*.d.ts"
+  ],
+  "references": [
+    {
+      "path": "../../packages/logger"
+    },
+    {
+      "path": "../types"
+    }
+  ]
+}

registry/pkg-doc-filter/tsconfig.lint.json

@@ -0,0 +1,8 @@
+{
+  "extends": "./tsconfig.json",
+  "include": [
+    "src/**/*.ts",
+    "test/**/*.ts",
+    "../../__typings__/**/*.d.ts"
+  ]
+}

registry/types/README.md

@@ -0,0 +1,17 @@
+# @pnpm/registry.types
+
+> Types related to the npm registry
+
+<!--@shields('npm')-->
+[![npm version](https://img.shields.io/npm/v/@pnpm/registry.types.svg)](https://www.npmjs.com/package/@pnpm/registry.types)
+<!--/@-->
+
+## Installation
+
+```
+pnpm add @pnpm/registry.types
+```
+
+## License
+
+MIT

registry/types/package.json

@@ -0,0 +1,45 @@
+{
+  "name": "@pnpm/registry.types",
+  "version": "1000.0.0-0",
+  "description": "Types related to the npm registry",
+  "keywords": [
+    "pnpm",
+    "pnpm10",
+    "npm"
+  ],
+  "license": "MIT",
+  "funding": "https://opencollective.com/pnpm",
+  "repository": "https://github.com/pnpm/pnpm/blob/main/registry/types",
+  "homepage": "https://github.com/pnpm/pnpm/blob/main/registry/types#readme",
+  "bugs": {
+    "url": "https://github.com/pnpm/pnpm/issues"
+  },
+  "type": "commonjs",
+  "main": "lib/index.js",
+  "types": "lib/index.d.ts",
+  "exports": {
+    ".": "./lib/index.js"
+  },
+  "files": [
+    "lib",
+    "!*.map"
+  ],
+  "scripts": {
+    "lint": "eslint \"src/**/*.ts\"",
+    "test": "pnpm run compile",
+    "prepublishOnly": "pnpm run compile",
+    "compile": "tsc --build && pnpm run lint --fix"
+  },
+  "dependencies": {
+    "@pnpm/types": "workspace:*"
+  },
+  "devDependencies": {
+    "@pnpm/registry.types": "workspace:*"
+  },
+  "engines": {
+    "node": ">=18.12"
+  },
+  "jest": {
+    "preset": "@pnpm/jest-config"
+  }
+}

registry/types/src/index.ts

@@ -0,0 +1,33 @@
+import { type PackageManifest } from '@pnpm/types'
+
+export type PackageDocument = PackageMeta
+
+export type PackageDocumentWithTime = PackageMetaWithTime
+
+export interface PackageMeta {
+  name: string
+  'dist-tags': Record<string, string>
+  versions: Record<string, PackageInRegistry>
+  time?: PackageMetaTime
+  cachedAt?: number
+}
+
+export interface PackageMetaWithTime extends PackageMeta {
+  time: PackageMetaTime
+}
+
+export type PackageMetaTime = Record<string, string> & {
+  unpublished?: {
+    time: string
+    versions: string[]
+  }
+}
+
+export interface PackageInRegistry extends PackageManifest {
+  hasInstallScript?: boolean
+  dist: {
+    integrity?: string
+    shasum: string
+    tarball: string
+  }
+}

registry/types/tsconfig.json

@@ -0,0 +1,16 @@
+{
+  "extends": "@pnpm/tsconfig",
+  "compilerOptions": {
+    "outDir": "lib",
+    "rootDir": "src"
+  },
+  "include": [
+    "src/**/*.ts",
+    "../../__typings__/**/*.d.ts"
+  ],
+  "references": [
+    {
+      "path": "../../packages/types"
+    }
+  ]
+}

registry/types/tsconfig.lint.json

@@ -0,0 +1,8 @@
+{
+  "extends": "./tsconfig.json",
+  "include": [
+    "src/**/*.ts",
+    "test/**/*.ts",
+    "../../__typings__/**/*.d.ts"
+  ]
+}

resolving/npm-resolver/package.json

@@ -40,6 +40,8 @@
     "@pnpm/fetching-types": "workspace:*",
     "@pnpm/graceful-fs": "workspace:*",
     "@pnpm/pick-registry-for-package": "workspace:*",
+    "@pnpm/registry.pkg-doc-filter": "workspace:*",
+    "@pnpm/registry.types": "workspace:*",
     "@pnpm/resolve-workspace-range": "workspace:*",
     "@pnpm/resolver-base": "workspace:*",
     "@pnpm/resolving.jsr-specifier-parser": "workspace:*",

resolving/npm-resolver/src/fetch.ts

@@ -7,8 +7,8 @@ import {
   PnpmError,
 } from '@pnpm/error'
 import { type FetchFromRegistry, type RetryTimeoutOptions } from '@pnpm/fetching-types'
+import { type PackageMeta } from '@pnpm/registry.types'
 import * as retry from '@zkochan/retry'
-import { type PackageMeta } from './pickPackage.js'
 
 interface RegistryResponse {
   status: number

resolving/npm-resolver/src/index.ts

@@ -7,6 +7,7 @@ import {
   type RetryTimeoutOptions,
 } from '@pnpm/fetching-types'
 import { pickRegistryForPackage } from '@pnpm/pick-registry-for-package'
+import { type PackageMeta, type PackageInRegistry } from '@pnpm/registry.types'
 import { resolveWorkspaceRange } from '@pnpm/resolve-workspace-range'
 import {
   type DirectoryResolution,
@@ -28,8 +29,6 @@ import semver from 'semver'
 import ssri from 'ssri'
 import versionSelectorType from 'version-selector-type'
 import {
-  type PackageInRegistry,
-  type PackageMeta,
   type PackageMetaCache,
   type PickPackageOptions,
   pickPackage,

resolving/npm-resolver/src/pickPackage.ts

@@ -5,7 +5,7 @@ import { PnpmError } from '@pnpm/error'
 import { logger } from '@pnpm/logger'
 import gfs from '@pnpm/graceful-fs'
 import { type VersionSelectors } from '@pnpm/resolver-base'
-import { type PackageManifest } from '@pnpm/types'
+import { type PackageMeta, type PackageInRegistry } from '@pnpm/registry.types'
 import getRegistryName from 'encode-registry'
 import loadJsonFile from 'load-json-file'
 import pLimit from 'p-limit'
@@ -17,40 +17,12 @@ import { toRaw } from './toRaw.js'
 import { pickPackageFromMeta, pickVersionByVersionRange, pickLowestVersionByVersionRange } from './pickPackageFromMeta.js'
 import { type RegistryPackageSpec } from './parseBareSpecifier.js'
 
-export interface PackageMeta {
-  name: string
-  'dist-tags': Record<string, string>
-  versions: Record<string, PackageInRegistry>
-  time?: PackageMetaTime
-  cachedAt?: number
-}
-
-export interface PackageMetaWithTime extends PackageMeta {
-  time: PackageMetaTime
-}
-
-export type PackageMetaTime = Record<string, string> & {
-  unpublished?: {
-    time: string
-    versions: string[]
-  }
-}
-
 export interface PackageMetaCache {
   get: (key: string) => PackageMeta | undefined
   set: (key: string, meta: PackageMeta) => void
   has: (key: string) => boolean
 }
 
-export interface PackageInRegistry extends PackageManifest {
-  hasInstallScript?: boolean
-  dist: {
-    integrity?: string
-    shasum: string
-    tarball: string
-  }
-}
-
 interface RefCountedLimiter {
   count: number
   limit: pLimit.Limit

resolving/npm-resolver/src/pickPackageFromMeta.ts

@@ -1,10 +1,10 @@
 import { PnpmError } from '@pnpm/error'
-import { globalWarn } from '@pnpm/logger'
+import { filterPkgDocByPublishDate } from '@pnpm/registry.pkg-doc-filter'
+import { type PackageInRegistry, type PackageMeta, type PackageMetaWithTime } from '@pnpm/registry.types'
 import { type VersionSelectors } from '@pnpm/resolver-base'
 import semver from 'semver'
 import util from 'util'
 import { type RegistryPackageSpec } from './parseBareSpecifier.js'
-import { type PackageInRegistry, type PackageMeta, type PackageMetaWithTime } from './pickPackage.js'
 
 export interface PickVersionByVersionRangeOptions {
   meta: PackageMeta
@@ -24,7 +24,7 @@ export function pickPackageFromMeta (
 ): PackageInRegistry | null {
   if (publishedBy) {
     assertMetaHasTime(meta)
-    meta = filterMetaByPublishedDate(meta, publishedBy)
+    meta = filterPkgDocByPublishDate(meta, publishedBy)
   }
   if ((!meta.versions || Object.keys(meta.versions).length === 0) && !publishedBy) {
     // Unfortunately, the npm registry doesn't return the time field in the abbreviated metadata.
@@ -227,77 +227,3 @@ class PreferredVersionsPrioritizer {
       .map((weight) => versionsByWeight[parseInt(weight, 10)])
   }
 }
-
-function filterMetaByPublishedDate (meta: PackageMetaWithTime, publishedBy: Date): PackageMeta {
-  const versionsWithinDate: PackageMeta['versions'] = {}
-  for (const version in meta.versions) {
-    if (!Object.hasOwn(meta.versions, version)) continue
-    const timeStr = meta.time[version]
-    if (timeStr && new Date(timeStr) <= publishedBy) {
-      versionsWithinDate[version] = meta.versions[version]
-    }
-  }
-
-  const distTagsWithinDate: PackageMeta['dist-tags'] = {}
-  const allDistTags = meta['dist-tags'] ?? {}
-  const parsedSemverCache = new Map<string, semver.SemVer>()
-  function tryParseSemver (semverStr: string): semver.SemVer | null {
-    let parsedSemver = parsedSemverCache.get(semverStr)
-    if (!parsedSemver) {
-      try {
-        parsedSemver = new semver.SemVer(semverStr, true)
-      } catch {
-        return null
-      }
-      parsedSemverCache.set(semverStr, parsedSemver)
-    }
-    return parsedSemver
-  }
-  for (const tag in allDistTags) {
-    if (!Object.hasOwn(allDistTags, tag)) continue
-    const distTagVersion = allDistTags[tag]
-    if (versionsWithinDate[distTagVersion]) {
-      distTagsWithinDate[tag] = distTagVersion
-      continue
-    }
-    // Repopulate the tag to the highest version available within date that has the same major as the original tag's version
-    const originalSemVer = tryParseSemver(distTagVersion)
-    if (!originalSemVer) continue
-    const originalIsPrerelease = (originalSemVer.prerelease.length > 0)
-    let bestVersion: string | undefined
-    for (const candidate in versionsWithinDate) {
-      if (!Object.hasOwn(versionsWithinDate, candidate)) continue
-      const candidateParsed = tryParseSemver(candidate)
-      if (
-        !candidateParsed ||
-        candidateParsed.major !== originalSemVer.major ||
-        (candidateParsed.prerelease.length > 0) !== originalIsPrerelease
-      ) continue
-      if (!bestVersion) {
-        bestVersion = candidate
-      } else {
-        try {
-          const candidateIsDeprecated = meta.versions[candidate].deprecated != null
-          const bestVersionIsDeprecated = meta.versions[bestVersion].deprecated != null
-          if (
-            (semver.gt(candidate, bestVersion, true) && (bestVersionIsDeprecated === candidateIsDeprecated)) ||
-            (bestVersionIsDeprecated && !candidateIsDeprecated)
-          ) {
-            bestVersion = candidate
-          }
-        } catch (err) {
-          globalWarn(`Failed to compare semver versions ${candidate} and ${bestVersion} from packument of ${meta.name}, skipping candidate version.`)
-        }
-      }
-    }
-    if (bestVersion) {
-      distTagsWithinDate[tag] = bestVersion
-    }
-  }
-
-  return {
-    ...meta,
-    versions: versionsWithinDate,
-    'dist-tags': distTagsWithinDate,
-  }
-}

resolving/npm-resolver/tsconfig.json

@@ -42,6 +42,12 @@
     {
       "path": "../../packages/types"
     },
+    {
+      "path": "../../registry/pkg-doc-filter"
+    },
+    {
+      "path": "../../registry/types"
+    },
     {
       "path": "../../workspace/resolve-workspace-range"
     },