diff --git a/.changeset/fix-git-dep-license-resolution.md b/.changeset/fix-git-dep-license-resolution.md new file mode 100644 index 0000000000..fc88261141 --- /dev/null +++ b/.changeset/fix-git-dep-license-resolution.md @@ -0,0 +1,6 @@ +--- +"@pnpm/store.pkg-finder": patch +"pnpm": patch +--- + +Fixed `pnpm sbom` and `pnpm licenses` failing to resolve license information for git-sourced dependencies (`git+https://`, `git+ssh://`, `github:` shorthand). These commands now correctly read the package manifest from the content-addressable store for `type: 'git'` resolutions [#11260](https://github.com/pnpm/pnpm/issues/11260). diff --git a/deps/compliance/license-scanner/package.json b/deps/compliance/license-scanner/package.json index d566bf014e..c15ec2db42 100644 --- a/deps/compliance/license-scanner/package.json +++ b/deps/compliance/license-scanner/package.json @@ -58,6 +58,7 @@ "@pnpm/constants": "workspace:*", "@pnpm/deps.compliance.license-scanner": "workspace:*", "@pnpm/logger": "workspace:*", + "@pnpm/store.cafs": "workspace:*", "@types/ramda": "catalog:", "@types/semver": "catalog:" }, diff --git a/deps/compliance/license-scanner/test/getPkgInfo.spec.ts b/deps/compliance/license-scanner/test/getPkgInfo.spec.ts index 9683bcb107..151123f50b 100644 --- a/deps/compliance/license-scanner/test/getPkgInfo.spec.ts +++ b/deps/compliance/license-scanner/test/getPkgInfo.spec.ts @@ -2,7 +2,8 @@ import fs from 'node:fs' import os from 'node:os' import path from 'node:path' -import { StoreIndex } from '@pnpm/store.index' +import type { PackageFilesIndex } from '@pnpm/store.cafs' +import { gitHostedStoreIndexKey, StoreIndex, storeIndexKey } from '@pnpm/store.index' import { getPkgInfo } from '../lib/getPkgInfo.js' @@ -11,7 +12,13 @@ export const DEFAULT_REGISTRIES = { '@jsr': 'https://npm.jsr.io/', } -describe('licences', () => { +function writeCafsFile (storeDir: string, digest: string, content: string): void { + const dir = path.join(storeDir, 'files', digest.slice(0, 2)) + fs.mkdirSync(dir, { recursive: true }) + fs.writeFileSync(path.join(dir, digest.slice(2)), content) +} + +describe('getPkgInfo', () => { let storeDir: string let storeIndex: StoreIndex @@ -25,7 +32,16 @@ describe('licences', () => { fs.rmSync(storeDir, { recursive: true, force: true }) }) - test('getPkgInfo() should throw error when package info can not be fetched', async () => { + const defaultGetOpts = () => ({ + storeDir, + storeIndex, + virtualStoreDir: 'virtual-store-dir', + modulesDir: 'modules-dir', + dir: 'workspace-dir', + virtualStoreDirMaxLength: 120, + }) + + test('should throw when registry package is not in the store', async () => { await expect( getPkgInfo( { @@ -40,15 +56,119 @@ describe('licences', () => { }, registries: DEFAULT_REGISTRIES, }, - { - storeDir, - storeIndex, - virtualStoreDir: 'virtual-store-dir', - modulesDir: 'modules-dir', - dir: 'workspace-dir', - virtualStoreDirMaxLength: 120, - } + defaultGetOpts() ) ).rejects.toThrow(/Failed to find package index file for bogus-package@1\.0\.0 \(at .*\), please consider running 'pnpm install'/) }) + + test('should throw when git dependency is not in the store', async () => { + const depPath = 'left-pad@git+https://github.com/stevemao/left-pad.git#2fca6157' + await expect( + getPkgInfo( + { + name: 'left-pad', + version: '1.3.0', + id: depPath, + depPath, + snapshot: { + resolution: { + type: 'git', + repo: 'https://github.com/stevemao/left-pad.git', + commit: '2fca6157', + }, + }, + registries: DEFAULT_REGISTRIES, + }, + defaultGetOpts() + ) + ).rejects.toThrow(/Failed to find package index file for/) + }) + + test('should extract license from a registry package in the store', async () => { + const digest = 'ee00ff1122334455' + writeCafsFile(storeDir, digest, JSON.stringify({ + name: 'express', + version: '4.18.2', + license: 'MIT', + description: 'Fast web framework', + author: { name: 'Test Author' }, + homepage: 'https://expressjs.com/', + repository: { url: 'https://github.com/expressjs/express' }, + })) + + const pkgId = 'express@4.18.2' + const integrity = 'sha512-test/integrity001' + const filesIndex: PackageFilesIndex = { + algo: 'sha256', + files: new Map([ + ['package.json', { digest, mode: 0o644, size: 0 }], + ]), + } + storeIndex.set(storeIndexKey(integrity, pkgId), filesIndex) + + const result = await getPkgInfo( + { + name: 'express', + version: '4.18.2', + id: pkgId, + depPath: pkgId, + snapshot: { + resolution: { integrity }, + }, + registries: DEFAULT_REGISTRIES, + }, + defaultGetOpts() + ) + + expect(result.license).toBe('MIT') + expect(result.author).toBe('Test Author') + expect(result.description).toBe('Fast web framework') + }) + + test('should extract license from a git dependency in the store', async () => { + const digest = 'ff99aa8877665544' + writeCafsFile(storeDir, digest, JSON.stringify({ + name: 'left-pad', + version: '1.3.0', + license: 'MIT', + description: 'String left pad', + author: 'Steve Mao', + repository: { url: 'https://github.com/stevemao/left-pad' }, + })) + + // The installer stores git packages under just the git URL, without the + // package name prefix. packageIdFromSnapshot strips the prefix when the + // caller (lockfileToLicenseNodeTree) builds the id for getPkgInfo. + const gitUrl = 'git+https://github.com/stevemao/left-pad.git#2fca6157fcca165438e0f9495cf0e5a4e6f71349' + const depPath = `left-pad@${gitUrl}` + const filesIndex: PackageFilesIndex = { + algo: 'sha256', + files: new Map([ + ['package.json', { digest, mode: 0o644, size: 0 }], + ]), + } + storeIndex.set(gitHostedStoreIndexKey(gitUrl, { built: true }), filesIndex) + + const result = await getPkgInfo( + { + name: 'left-pad', + version: '1.3.0', + id: gitUrl, + depPath, + snapshot: { + resolution: { + type: 'git', + repo: 'https://github.com/stevemao/left-pad.git', + commit: '2fca6157fcca165438e0f9495cf0e5a4e6f71349', + }, + }, + registries: DEFAULT_REGISTRIES, + }, + defaultGetOpts() + ) + + expect(result.license).toBe('MIT') + expect(result.author).toBe('Steve Mao') + expect(result.description).toBe('String left pad') + }) }) diff --git a/deps/compliance/license-scanner/tsconfig.json b/deps/compliance/license-scanner/tsconfig.json index 5ac407caa5..41c64330b7 100644 --- a/deps/compliance/license-scanner/tsconfig.json +++ b/deps/compliance/license-scanner/tsconfig.json @@ -42,6 +42,9 @@ { "path": "../../../pkg-manifest/reader" }, + { + "path": "../../../store/cafs" + }, { "path": "../../../store/index" }, diff --git a/deps/compliance/sbom/package.json b/deps/compliance/sbom/package.json index 06c0f61f20..1b47a8d5ce 100644 --- a/deps/compliance/sbom/package.json +++ b/deps/compliance/sbom/package.json @@ -54,6 +54,7 @@ "@jest/globals": "catalog:", "@pnpm/deps.compliance.sbom": "workspace:*", "@pnpm/logger": "workspace:*", + "@pnpm/store.cafs": "workspace:*", "@types/ssri": "catalog:" }, "engines": { diff --git a/deps/compliance/sbom/src/getPkgMetadata.ts b/deps/compliance/sbom/src/getPkgMetadata.ts index 78117fe600..a373538860 100644 --- a/deps/compliance/sbom/src/getPkgMetadata.ts +++ b/deps/compliance/sbom/src/getPkgMetadata.ts @@ -1,9 +1,9 @@ import { isSpdxLicenseExpression, resolveLicense } from '@pnpm/deps.compliance.license-resolver' -import { type PackageSnapshot, pkgSnapshotToResolution } from '@pnpm/lockfile.utils' +import { packageIdFromSnapshot, type PackageSnapshot, pkgSnapshotToResolution } from '@pnpm/lockfile.utils' import { readPackageJson } from '@pnpm/pkg-manifest.reader' import type { StoreIndex } from '@pnpm/store.index' import { readPackageFileMap } from '@pnpm/store.pkg-finder' -import type { PackageManifest, Registries } from '@pnpm/types' +import type { DepPath, PackageManifest, Registries } from '@pnpm/types' import pLimit from 'p-limit' const limitMetadataReads = pLimit(4) @@ -24,7 +24,7 @@ export interface GetPkgMetadataOptions { } export async function getPkgMetadata ( - depPath: string, + depPath: DepPath, snapshot: PackageSnapshot, registries: Registries, opts: GetPkgMetadataOptions @@ -33,12 +33,12 @@ export async function getPkgMetadata ( } async function getPkgMetadataUnclamped ( - depPath: string, + depPath: DepPath, snapshot: PackageSnapshot, registries: Registries, opts: GetPkgMetadataOptions ): Promise { - const id = snapshot.id ?? depPath + const id = packageIdFromSnapshot(depPath, snapshot) const resolution = pkgSnapshotToResolution(depPath, snapshot, registries) let files: Map diff --git a/deps/compliance/sbom/test/getPkgMetadata.test.ts b/deps/compliance/sbom/test/getPkgMetadata.test.ts new file mode 100644 index 0000000000..4643feb306 --- /dev/null +++ b/deps/compliance/sbom/test/getPkgMetadata.test.ts @@ -0,0 +1,135 @@ +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' + +import { describe, expect, it } from '@jest/globals' +import type { PackageFilesIndex } from '@pnpm/store.cafs' +import { gitHostedStoreIndexKey, StoreIndex, storeIndexKey } from '@pnpm/store.index' +import type { DepPath } from '@pnpm/types' + +import { getPkgMetadata } from '../lib/getPkgMetadata.js' + +const DEFAULT_REGISTRIES = { + default: 'https://registry.npmjs.org/', + '@jsr': 'https://npm.jsr.io/', +} + +function writeCafsFile (storeDir: string, digest: string, content: string): void { + const filePath = path.join(storeDir, 'files', digest.slice(0, 2), digest.slice(2)) + fs.mkdirSync(path.dirname(filePath), { recursive: true }) + fs.writeFileSync(filePath, content) +} + +describe('getPkgMetadata', () => { + let storeDir: string + let storeIndex: StoreIndex + + beforeAll(() => { + storeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'pnpm-sbom-metadata-test-')) + storeIndex = new StoreIndex(storeDir) + }) + + afterAll(() => { + storeIndex.close() + fs.rmSync(storeDir, { recursive: true, force: true }) + }) + + const defaultOpts = () => ({ + storeDir, + storeIndex, + lockfileDir: '/tmp/project', + virtualStoreDirMaxLength: 120, + }) + + it('should extract metadata from a registry package', async () => { + const digest = 'aa11bb22cc33dd44' + writeCafsFile(storeDir, digest, JSON.stringify({ + name: 'express', + version: '4.18.2', + license: 'MIT', + description: 'Fast web framework', + author: { name: 'Test Author' }, + })) + + const integrity = 'sha512-sbom/test001' + const pkgId = 'express@4.18.2' + const filesIndex: PackageFilesIndex = { + algo: 'sha256', + files: new Map([ + ['package.json', { digest, mode: 0o644, size: 0 }], + ]), + } + storeIndex.set(storeIndexKey(integrity, pkgId), filesIndex) + + const result = await getPkgMetadata( + pkgId as DepPath, + { resolution: { integrity } }, + DEFAULT_REGISTRIES, + defaultOpts() + ) + + expect(result.license).toBe('MIT') + expect(result.description).toBe('Fast web framework') + expect(result.author).toBe('Test Author') + }) + + it('should extract metadata from a git dependency using the real store key format', async () => { + const digest = 'dd44ee55ff660011' + writeCafsFile(storeDir, digest, JSON.stringify({ + name: 'left-pad', + version: '1.3.0', + license: 'MIT', + description: 'String left pad', + author: 'Steve Mao', + })) + + // The installer stores git packages under just the git URL, without the + // package name prefix. getPkgMetadata must strip the prefix from depPath + // via packageIdFromSnapshot to match. + const gitUrl = 'git+https://github.com/stevemao/left-pad.git#2fca6157fcca165438e0f9495cf0e5a4e6f71349' + const depPath = `left-pad@${gitUrl}` as DepPath + const filesIndex: PackageFilesIndex = { + algo: 'sha256', + files: new Map([ + ['package.json', { digest, mode: 0o644, size: 0 }], + ]), + } + storeIndex.set(gitHostedStoreIndexKey(gitUrl, { built: true }), filesIndex) + + const result = await getPkgMetadata( + depPath, + { + resolution: { + type: 'git', + repo: 'https://github.com/stevemao/left-pad.git', + commit: '2fca6157fcca165438e0f9495cf0e5a4e6f71349', + }, + }, + DEFAULT_REGISTRIES, + defaultOpts() + ) + + expect(result.license).toBe('MIT') + expect(result.description).toBe('String left pad') + expect(result.author).toBe('Steve Mao') + }) + + it('should return empty metadata when store entry is missing', async () => { + const depPath = 'missing@git+https://github.com/user/missing.git#deadbeef' as DepPath + + const result = await getPkgMetadata( + depPath, + { + resolution: { + type: 'git', + repo: 'https://github.com/user/missing.git', + commit: 'deadbeef', + }, + }, + DEFAULT_REGISTRIES, + defaultOpts() + ) + + expect(result).toEqual({}) + }) +}) diff --git a/deps/compliance/sbom/tsconfig.json b/deps/compliance/sbom/tsconfig.json index d463607772..269bee0a95 100644 --- a/deps/compliance/sbom/tsconfig.json +++ b/deps/compliance/sbom/tsconfig.json @@ -30,6 +30,9 @@ { "path": "../../../pkg-manifest/reader" }, + { + "path": "../../../store/cafs" + }, { "path": "../../../store/index" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 937d385990..13e0940e27 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -3045,6 +3045,9 @@ importers: '@pnpm/logger': specifier: workspace:* version: link:../../../core/logger + '@pnpm/store.cafs': + specifier: workspace:* + version: link:../../../store/cafs '@types/ramda': specifier: 'catalog:' version: 0.31.1 @@ -3100,6 +3103,9 @@ importers: '@pnpm/logger': specifier: workspace:* version: link:../../../core/logger + '@pnpm/store.cafs': + specifier: workspace:* + version: link:../../../store/cafs '@types/ssri': specifier: 'catalog:' version: 7.1.5 diff --git a/store/pkg-finder/package.json b/store/pkg-finder/package.json index ffc625e86b..633f5a8575 100644 --- a/store/pkg-finder/package.json +++ b/store/pkg-finder/package.json @@ -24,10 +24,11 @@ "!*.map" ], "scripts": { - "lint": "eslint \"src/**/*.ts\"", + "lint": "eslint \"src/**/*.ts\" \"test/**/*.ts\"", "compile": "tsgo --build && pn lint --fix", "prepublishOnly": "pn compile", - "test": "pn compile" + "test": "pn compile && pn .test", + ".test": "cross-env NODE_OPTIONS=\"$NODE_OPTIONS --experimental-vm-modules --disable-warning=ExperimentalWarning --disable-warning=DEP0169\" jest" }, "dependencies": { "@pnpm/deps.path": "workspace:*", diff --git a/store/pkg-finder/src/index.ts b/store/pkg-finder/src/index.ts index 920d2e082c..7912a18eb0 100644 --- a/store/pkg-finder/src/index.ts +++ b/store/pkg-finder/src/index.ts @@ -55,6 +55,8 @@ export async function readPackageFileMap ( ) } else if (!packageResolution.type && 'tarball' in packageResolution && packageResolution.tarball) { pkgIndexFilePath = gitHostedStoreIndexKey(packageId, { built: true }) + } else if (packageResolution.type === 'git') { + pkgIndexFilePath = gitHostedStoreIndexKey(packageId, { built: true }) } else { return undefined } diff --git a/store/pkg-finder/test/readPackageFileMap.test.ts b/store/pkg-finder/test/readPackageFileMap.test.ts new file mode 100644 index 0000000000..0743934a98 --- /dev/null +++ b/store/pkg-finder/test/readPackageFileMap.test.ts @@ -0,0 +1,139 @@ +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' + +import type { GitResolution, Resolution, TarballResolution } from '@pnpm/resolving.resolver-base' +import type { PackageFilesIndex } from '@pnpm/store.cafs' +import { gitHostedStoreIndexKey, StoreIndex, storeIndexKey } from '@pnpm/store.index' +import { readPackageFileMap } from '@pnpm/store.pkg-finder' + +function createFilesIndex (): PackageFilesIndex { + return { + algo: 'sha256', + files: new Map([ + ['package.json', { digest: 'abc123', mode: 0o644, size: 0 }], + ['index.js', { digest: 'def456', mode: 0o644, size: 0 }], + ]), + } +} + +function writeCafsFile (storeDir: string, digest: string, content: string): void { + const dir = path.join(storeDir, 'files', digest.slice(0, 2)) + fs.mkdirSync(dir, { recursive: true }) + fs.writeFileSync(path.join(dir, digest.slice(2)), content) +} + +describe('readPackageFileMap', () => { + let storeDir: string + let storeIndex: StoreIndex + + beforeAll(() => { + storeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'pnpm-pkg-finder-test-')) + storeIndex = new StoreIndex(storeDir) + }) + + afterAll(() => { + storeIndex.close() + fs.rmSync(storeDir, { recursive: true, force: true }) + }) + + const defaultOpts = () => ({ + storeDir, + storeIndex, + lockfileDir: '/tmp/project', + virtualStoreDirMaxLength: 120, + }) + + it('should resolve registry packages by integrity hash', async () => { + const integrity = 'sha512-abc123registry' + const pkgId = 'express@4.18.2' + const key = storeIndexKey(integrity, pkgId) + + storeIndex.set(key, createFilesIndex()) + + const resolution: TarballResolution = { + integrity, + tarball: 'https://registry.npmjs.org/express/-/express-4.18.2.tgz', + } + + const result = await readPackageFileMap(resolution, pkgId, defaultOpts()) + + expect(result).toBeDefined() + expect(result!.has('package.json')).toBe(true) + expect(result!.has('index.js')).toBe(true) + }) + + it('should resolve git-hosted tarball packages (no type, has tarball)', async () => { + const pkgId = 'left-pad@https://codeload.github.com/stevemao/left-pad/tar.gz/abc123' + const key = gitHostedStoreIndexKey(pkgId, { built: true }) + + storeIndex.set(key, createFilesIndex()) + + const resolution = { + tarball: 'https://codeload.github.com/stevemao/left-pad/tar.gz/abc123', + } as TarballResolution + + const result = await readPackageFileMap(resolution, pkgId, defaultOpts()) + + expect(result).toBeDefined() + expect(result!.has('package.json')).toBe(true) + expect(result!.has('index.js')).toBe(true) + }) + + it('should resolve git dependencies with type "git" and return readable file paths', async () => { + const digest = 'aabbccdd001122' + const manifestContent = JSON.stringify({ + name: 'left-pad', + version: '1.3.0', + license: 'MIT', + }) + writeCafsFile(storeDir, digest, manifestContent) + + const pkgId = 'left-pad@git+https://github.com/stevemao/left-pad.git#2fca6157fcca165438e0f9495cf0e5a4e6f71349' + const filesIndex: PackageFilesIndex = { + algo: 'sha256', + files: new Map([ + ['package.json', { digest, mode: 0o644, size: 0 }], + ]), + } + storeIndex.set(gitHostedStoreIndexKey(pkgId, { built: true }), filesIndex) + + const resolution: GitResolution = { + type: 'git', + repo: 'https://github.com/stevemao/left-pad.git', + commit: '2fca6157fcca165438e0f9495cf0e5a4e6f71349', + } + + const result = await readPackageFileMap(resolution, pkgId, defaultOpts()) + + expect(result).toBeDefined() + const manifestPath = result!.get('package.json')! + expect(fs.existsSync(manifestPath)).toBe(true) + + const parsed = JSON.parse(fs.readFileSync(manifestPath, 'utf8')) + expect(parsed.name).toBe('left-pad') + expect(parsed.license).toBe('MIT') + }) + + it('should throw ENOENT when store index has no entry for a git dependency', async () => { + const pkgId = 'missing-pkg@git+https://github.com/user/missing-pkg.git#deadbeef' + + const resolution: GitResolution = { + type: 'git', + repo: 'https://github.com/user/missing-pkg.git', + commit: 'deadbeef', + } + + await expect( + readPackageFileMap(resolution, pkgId, defaultOpts()) + ).rejects.toThrow(/package index not found/) + }) + + it('should return undefined for unknown resolution types', async () => { + const resolution = { type: 'unknown-type' } as unknown as Resolution + + const result = await readPackageFileMap(resolution, 'some-pkg@1.0.0', defaultOpts()) + + expect(result).toBeUndefined() + }) +}) diff --git a/store/pkg-finder/test/tsconfig.json b/store/pkg-finder/test/tsconfig.json new file mode 100644 index 0000000000..67ce5e1d0e --- /dev/null +++ b/store/pkg-finder/test/tsconfig.json @@ -0,0 +1,18 @@ +{ + "extends": "../tsconfig.json", + "compilerOptions": { + "noEmit": false, + "outDir": "../node_modules/.test.lib", + "rootDir": "..", + "isolatedModules": true + }, + "include": [ + "**/*.ts", + "../../../__typings__/**/*.d.ts" + ], + "references": [ + { + "path": ".." + } + ] +}