chore: move all tarball files out from Git LFS to a package dependency (#8885)

This commit is contained in:
Zoltan Kochan
2024-12-18 03:06:51 +01:00
committed by GitHub
parent 5c48047e12
commit 7ec017e13f
52 changed files with 140 additions and 86 deletions

View File

Binary file not shown.

View File

@@ -1 +0,0 @@
module.exports = () => require('./package.json').name

View File

@@ -1,4 +0,0 @@
{
"name": "tar-pkg",
"version": "1.0.0"
}

View File

@@ -1,9 +0,0 @@
lockfileVersion: '9.0'
settings:
autoInstallPeers: true
excludeLinksFromLockfile: false
importers:
.: {}

View File

Binary file not shown.

View File

@@ -0,0 +1,17 @@
{
"name": "@pnpm/prepare-temp-dir",
"version": "0.0.0",
"main": "lib/index.js",
"types": "lib/index.d.ts",
"dependencies": {},
"devDependencies": {
"@pnpm/prepare-temp-dir": "workspace:*",
"@types/node": "catalog:"
},
"scripts": {
"lint": "eslint src/**/*.ts",
"prepublishOnly": "pnpm run compile",
"test": "pnpm run compile",
"compile": "rimraf tsconfig.tsbuildinfo lib && tsc --build"
}
}

View File

@@ -0,0 +1,29 @@
import fs from 'fs'
import path from 'path'
// The testing folder should be outside of the project to avoid lookup in the project's node_modules
// Not using the OS temp directory due to issues on Windows CI.
const tmpBaseDir = path.join(__dirname, '../../../../pnpm_tmp')
const tmpPath = path.join(tmpBaseDir, `${getFilesCountInDir(tmpBaseDir).toString()}_${process.pid.toString()}`)
let dirNumber = 0
export function tempDir (chdir: boolean = true): string {
dirNumber++
const dirname = dirNumber.toString()
const tmpDir = path.join(tmpPath, dirname)
fs.mkdirSync(tmpDir, { recursive: true })
if (chdir) process.chdir(tmpDir)
return tmpDir
}
function getFilesCountInDir (dir: string): number {
try {
return fs.readdirSync(dir).length
} catch {
return 0
}
}

View File

@@ -0,0 +1,12 @@
{
"extends": "@pnpm/tsconfig",
"compilerOptions": {
"outDir": "lib",
"rootDir": "src"
},
"include": [
"src/**/*.ts",
"../../__typings__/**/*.d.ts"
],
"references": []
}

View File

@@ -0,0 +1,8 @@
{
"extends": "./tsconfig.json",
"include": [
"src/**/*.ts",
"test/**/*.ts",
"../../__typings__/**/*.d.ts"
]
}

View File

@@ -5,6 +5,7 @@
"types": "lib/index.d.ts",
"dependencies": {
"@pnpm/assert-project": "workspace:*",
"@pnpm/prepare-temp-dir": "workspace:*",
"@pnpm/types": "workspace:*",
"write-json5-file": "^3.1.0",
"write-pkg": "catalog:",

View File

@@ -2,39 +2,14 @@ import fs from 'fs'
import path from 'path'
import { assertProject, type Modules, type Project } from '@pnpm/assert-project'
import { type ProjectManifest } from '@pnpm/types'
import { tempDir } from '@pnpm/prepare-temp-dir'
import { sync as writeJson5File } from 'write-json5-file'
import { sync as writeYamlFile } from 'write-yaml-file'
import writePkg from 'write-pkg'
export type { Modules, Project }
export type ManifestFormat = 'JSON' | 'JSON5' | 'YAML'
// The testing folder should be outside of the project to avoid lookup in the project's node_modules
// Not using the OS temp directory due to issues on Windows CI.
const tmpBaseDir = path.join(__dirname, '../../../../pnpm_tmp')
function getFilesCountInDir (dir: string): number {
try {
return fs.readdirSync(dir).length
} catch {
return 0
}
}
const tmpPath = path.join(tmpBaseDir, `${getFilesCountInDir(tmpBaseDir).toString()}_${process.pid.toString()}`)
let dirNumber = 0
export function tempDir (chdir: boolean = true): string {
dirNumber++
const dirname = dirNumber.toString()
const tmpDir = path.join(tmpPath, dirname)
fs.mkdirSync(tmpDir, { recursive: true })
if (chdir) process.chdir(tmpDir)
return tmpDir
}
export { tempDir }
interface LocationAndManifest {
location: string

View File

@@ -14,6 +14,9 @@
},
{
"path": "../assert-project"
},
{
"path": "../prepare-temp-dir"
}
]
}

View File

@@ -32,7 +32,7 @@
"prepublishOnly": "pnpm run compile"
},
"dependencies": {
"@pnpm/prepare": "workspace:*",
"@pnpm/prepare-temp-dir": "workspace:*",
"fs-extra": "catalog:"
},
"devDependencies": {

View File

@@ -1,6 +1,6 @@
import fs from 'fs'
import path from 'path'
import { tempDir } from '@pnpm/prepare'
import { tempDir } from '@pnpm/prepare-temp-dir'
export interface FixturesHandle {
copy: (name: string, dest: string) => void
@@ -62,6 +62,8 @@ function findFixture (dir: string, name: string): string {
if (fs.existsSync(checkDir)) return checkDir
checkDir = path.join(dir, '__fixtures__', name)
if (fs.existsSync(checkDir)) return checkDir
checkDir = path.join(dir, 'node_modules/@pnpm/tgz-fixtures/tgz', name)
if (fs.existsSync(checkDir)) return checkDir
if (dir === root) throw new Error(`Local package "${name}" not found`)
dir = path.dirname(dir)
}

View File

@@ -10,7 +10,7 @@
],
"references": [
{
"path": "../prepare"
"path": "../prepare-temp-dir"
}
]
}

View File

@@ -39,6 +39,7 @@
"@pnpm/eslint-config": "workspace:*",
"@pnpm/jest-config": "workspace:*",
"@pnpm/meta-updater": "catalog:",
"@pnpm/tgz-fixtures": "0.0.0",
"@pnpm/tsconfig": "workspace:*",
"@types/jest": "^29.5.12",
"@types/node": "catalog:",

View File

Binary file not shown.

View File

@@ -1 +0,0 @@
{}

View File

Binary file not shown.

View File

@@ -1,7 +0,0 @@
{
"name": "tar-pkg-with-dep",
"version": "1.0.0",
"dependencies": {
"is-positive": "^1.0.0"
}
}

View File

@@ -1,7 +0,0 @@
{
"name": "tar-pkg-with-dep",
"version": "1.0.0",
"dependencies": {
"is-positive": "^2.0.0"
}
}

View File

@@ -28,7 +28,7 @@ test('bundledDependencies (pkg-with-bundled-dependencies@1.0.0)', async () => {
test('local tarball with bundledDependencies', async () => {
const project = prepareEmpty()
f.copy('pkg-with-bundled-dependencies/pkg-with-bundled-dependencies-1.0.0.tgz', 'pkg.tgz')
f.copy('pkg-with-bundled-dependencies-1.0.0.tgz', 'pkg.tgz')
await addDependenciesToPackage({}, ['file:pkg.tgz'], testDefaults({ fastUnpack: false }))
const lockfile = project.readLockfile()
@@ -45,7 +45,7 @@ test('local tarball with bundledDependencies', async () => {
test('local tarball with bundledDependencies true', async () => {
const project = prepareEmpty()
f.copy('pkg-with-bundle-dependencies-true/pkg-with-bundle-dependencies-true-1.0.0.tgz', 'pkg.tgz')
f.copy('pkg-with-bundle-dependencies-true-1.0.0.tgz', 'pkg.tgz')
await addDependenciesToPackage({}, ['file:pkg.tgz'], testDefaults({ fastUnpack: false }))
const lockfile = project.readLockfile()

View File

@@ -164,7 +164,7 @@ test('local file with symlinked node_modules', async () => {
test('package with a broken symlink', async () => {
const project = prepareEmpty()
await addDependenciesToPackage({}, [f.find('has-broken-symlink/has-broken-symlink.tar.gz')], testDefaults({ fastUnpack: false }))
await addDependenciesToPackage({}, [f.find('has-broken-symlink.tar.gz')], testDefaults({ fastUnpack: false }))
const m = project.requireModule('has-broken-symlink')
@@ -173,20 +173,20 @@ test('package with a broken symlink', async () => {
test('tarball local package', async () => {
const project = prepareEmpty()
const manifest = await addDependenciesToPackage({}, [f.find('tar-pkg/tar-pkg-1.0.0.tgz')], testDefaults({ fastUnpack: false }))
const manifest = await addDependenciesToPackage({}, [f.find('tar-pkg-1.0.0.tgz')], testDefaults({ fastUnpack: false }))
const m = project.requireModule('tar-pkg')
expect(m()).toBe('tar-pkg')
const pkgSpec = `file:${normalizePath(f.find('tar-pkg/tar-pkg-1.0.0.tgz'))}`
const pkgSpec = `file:${normalizePath(f.find('tar-pkg-1.0.0.tgz'))}`
expect(manifest.dependencies).toStrictEqual({ 'tar-pkg': pkgSpec })
const lockfile = project.readLockfile()
expect(lockfile.packages[`tar-pkg@${lockfile.importers['.'].dependencies!['tar-pkg'].version}`]).toStrictEqual({
resolution: {
integrity: 'sha512-HP/5Rgt3pVFLzjmN9qJJ6vZMgCwoCIl/m2bPndYT283CUqnmFiMx0GeeIJ7SyK6TYoJM78SEvFEOQie++caHqw==',
tarball: `file:${normalizePath(path.relative(process.cwd(), f.find('tar-pkg/tar-pkg-1.0.0.tgz')))}`,
tarball: `file:${normalizePath(path.relative(process.cwd(), f.find('tar-pkg-1.0.0.tgz')))}`,
},
version: '1.0.0',
})
@@ -195,7 +195,7 @@ test('tarball local package', async () => {
test('tarball local package from project directory', async () => {
const project = prepareEmpty()
f.copy('tar-pkg/tar-pkg-1.0.0.tgz', path.resolve('tar-pkg-1.0.0.tgz'))
f.copy('tar-pkg-1.0.0.tgz', path.resolve('tar-pkg-1.0.0.tgz'))
const manifest = await install({
dependencies: {

View File

@@ -1176,7 +1176,7 @@ test('local tarball dependency with peer dependency', async () => {
const reporter = sinon.spy()
const manifest = await addDependenciesToPackage({}, [
`file:${f.find('tar-pkg-with-peers/tar-pkg-with-peers-1.0.0.tgz')}`,
`file:${f.find('tar-pkg-with-peers-1.0.0.tgz')}`,
'bar@npm:@pnpm.e2e/bar@100.0.0',
'foo@npm:@pnpm.e2e/foo@100.0.0',
], testDefaults({ reporter }))

View File

@@ -1062,7 +1062,7 @@ const REGISTRY_MIRROR_DIR = path.join(__dirname, './registry-mirror')
/* eslint-disable @typescript-eslint/no-explicit-any */
const isPositiveMeta = loadJsonFile.sync<any>(path.join(REGISTRY_MIRROR_DIR, 'is-positive.json'))
/* eslint-enable @typescript-eslint/no-explicit-any */
const tarballPath = path.join(REGISTRY_MIRROR_DIR, 'is-positive-3.1.0.tgz')
const tarballPath = f.find('is-positive-3.1.0.tgz')
test('tarball domain differs from registry domain', async () => {
nock('https://registry.example.com', { allowUnmocked: true })

View File

Binary file not shown.

View File

@@ -472,6 +472,7 @@ test('available packages are relinked during forced install', async () => {
test('installing local dependency', async () => {
let prefix = f.prepare('has-local-dep')
f.copy('tar-pkg-1.0.0.tgz', path.join(prefix, 'tar-pkg-1.0.0.tgz'))
prefix = path.join(prefix, 'pkg')
const reporter = sinon.spy()

View File

Binary file not shown.

27
pnpm-lock.yaml generated
View File

@@ -720,6 +720,9 @@ importers:
'@pnpm/meta-updater':
specifier: 'catalog:'
version: 2.0.3
'@pnpm/tgz-fixtures':
specifier: 0.0.0
version: 0.0.0
'@pnpm/tsconfig':
specifier: workspace:*
version: link:__utils__/tsconfig
@@ -972,6 +975,9 @@ importers:
'@pnpm/assert-project':
specifier: workspace:*
version: link:../assert-project
'@pnpm/prepare-temp-dir':
specifier: workspace:*
version: link:../prepare-temp-dir
'@pnpm/types':
specifier: workspace:*
version: link:../../packages/types
@@ -992,6 +998,15 @@ importers:
specifier: 'catalog:'
version: 18.19.34
__utils__/prepare-temp-dir:
devDependencies:
'@pnpm/prepare-temp-dir':
specifier: workspace:*
version: 'link:'
'@types/node':
specifier: 'catalog:'
version: 18.19.34
__utils__/scripts:
dependencies:
'@pnpm/workspace.find-packages':
@@ -1025,9 +1040,9 @@ importers:
__utils__/test-fixtures:
dependencies:
'@pnpm/prepare':
'@pnpm/prepare-temp-dir':
specifier: workspace:*
version: link:../prepare
version: link:../prepare-temp-dir
fs-extra:
specifier: 'catalog:'
version: 11.2.0
@@ -7060,6 +7075,9 @@ importers:
'@pnpm/store.cafs':
specifier: workspace:*
version: 'link:'
'@pnpm/test-fixtures':
specifier: workspace:*
version: link:../../__utils__/test-fixtures
'@pnpm/types':
specifier: workspace:*
version: link:../../packages/types
@@ -9091,6 +9109,9 @@ packages:
resolution: {integrity: sha512-BSGvYd59kPKVTUk1InekEp+TiPnJ8650/bQyiOUFSvqHi61YipcR+E4H2i3xTnk2e+GHdGbXvEtAZbQmyxb0/g==}
engines: {node: '>=18.12'}
'@pnpm/tgz-fixtures@0.0.0':
resolution: {integrity: sha512-6YlfA/aWpeYbX9ADtSv3kKJYjTUE8rXw3gKzLPuO8hc4S7fP6sZwQXaYP7uwyWieU45TR3u0V/g8esQQYZrGMA==}
'@pnpm/types@11.1.0':
resolution: {integrity: sha512-wnlOhu7hjv9/qsf2cbK0YqpaV9c4LS69Utxd+r8hq/GWhyrOHcM1QOlfQb0Mzci0q4DDgB8VXT4dhBnEBL4c5g==}
engines: {node: '>=18.12'}
@@ -16066,6 +16087,8 @@ snapshots:
dependencies:
strip-comments-strings: 1.2.0
'@pnpm/tgz-fixtures@0.0.0': {}
'@pnpm/types@11.1.0': {}
'@pnpm/types@12.0.0': {}

View File

@@ -5,6 +5,8 @@ import { type DirectoryResolution } from '@pnpm/resolver-base'
import normalize from 'normalize-path'
import { logger } from '@pnpm/logger'
const TEST_DIR = path.dirname(require.resolve('@pnpm/tgz-fixtures/tgz/pnpm-local-resolver-0.1.1.tgz'))
test('resolve directory', async () => {
const resolveResult = await resolveFromLocal({ pref: '..' }, { projectDir: __dirname })
expect(resolveResult!.id).toEqual('link:..')
@@ -63,7 +65,7 @@ test('resolve directory specified using the link: protocol', async () => {
test('resolve file', async () => {
const wantedDependency = { pref: './pnpm-local-resolver-0.1.1.tgz' }
const resolveResult = await resolveFromLocal(wantedDependency, { projectDir: __dirname })
const resolveResult = await resolveFromLocal(wantedDependency, { projectDir: TEST_DIR })
expect(resolveResult).toEqual({
id: 'file:pnpm-local-resolver-0.1.1.tgz',
@@ -79,16 +81,16 @@ test('resolve file', async () => {
test("resolve file when lockfile directory differs from the package's dir", async () => {
const wantedDependency = { pref: './pnpm-local-resolver-0.1.1.tgz' }
const resolveResult = await resolveFromLocal(wantedDependency, {
lockfileDir: path.join(__dirname, '..'),
projectDir: __dirname,
lockfileDir: path.join(TEST_DIR, '..'),
projectDir: TEST_DIR,
})
expect(resolveResult).toEqual({
id: 'file:test/pnpm-local-resolver-0.1.1.tgz',
id: 'file:tgz/pnpm-local-resolver-0.1.1.tgz',
normalizedPref: 'file:pnpm-local-resolver-0.1.1.tgz',
resolution: {
integrity: 'sha512-UHd2zKRT/w70KKzFlj4qcT81A1Q0H7NM9uKxLzIZ/VZqJXzt5Hnnp2PYPb5Ezq/hAamoYKIn5g7fuv69kP258w==',
tarball: 'file:test/pnpm-local-resolver-0.1.1.tgz',
tarball: 'file:tgz/pnpm-local-resolver-0.1.1.tgz',
},
resolvedVia: 'local-filesystem',
})
@@ -96,7 +98,7 @@ test("resolve file when lockfile directory differs from the package's dir", asyn
test('resolve tarball specified with file: protocol', async () => {
const wantedDependency = { pref: 'file:./pnpm-local-resolver-0.1.1.tgz' }
const resolveResult = await resolveFromLocal(wantedDependency, { projectDir: __dirname })
const resolveResult = await resolveFromLocal(wantedDependency, { projectDir: TEST_DIR })
expect(resolveResult).toEqual({
id: 'file:pnpm-local-resolver-0.1.1.tgz',
@@ -112,7 +114,7 @@ test('resolve tarball specified with file: protocol', async () => {
test('fail when resolving tarball specified with the link: protocol', async () => {
try {
const wantedDependency = { pref: 'link:./pnpm-local-resolver-0.1.1.tgz' }
await resolveFromLocal(wantedDependency, { projectDir: __dirname })
await resolveFromLocal(wantedDependency, { projectDir: TEST_DIR })
fail()
} catch (err: any) { // eslint-disable-line
expect(err).toBeDefined()

View File

Binary file not shown.

View File

Binary file not shown.

View File

@@ -29,6 +29,7 @@
"devDependencies": {
"@pnpm/cafs-types": "workspace:*",
"@pnpm/store.cafs": "workspace:*",
"@pnpm/test-fixtures": "workspace:*",
"@pnpm/types": "workspace:*",
"@types/is-gzip": "catalog:",
"@types/node": "catalog:",

View File

Binary file not shown.

View File

Binary file not shown.

View File

Binary file not shown.

View File

Binary file not shown.

View File

@@ -2,18 +2,21 @@ import fs from 'fs'
import path from 'path'
import symlinkDir from 'symlink-dir'
import tempy from 'tempy'
import { fixtures } from '@pnpm/test-fixtures'
import {
createCafs,
checkPkgFilesIntegrity,
getFilePathByModeInCafs,
} from '../src'
const f = fixtures(__dirname)
describe('cafs', () => {
it('unpack', () => {
const dest = tempy.directory()
const cafs = createCafs(dest)
const { filesIndex } = cafs.addFilesFromTarball(
fs.readFileSync(path.join(__dirname, '../__fixtures__/node-gyp-6.1.0.tgz'))
fs.readFileSync(f.find('node-gyp-6.1.0.tgz'))
)
expect(Object.keys(filesIndex)).toHaveLength(121)
const pkgFile = filesIndex['package.json']
@@ -86,7 +89,7 @@ test('file names are normalized when unpacking a tarball', () => {
const dest = tempy.directory()
const cafs = createCafs(dest)
const { filesIndex } = cafs.addFilesFromTarball(
fs.readFileSync(path.join(__dirname, 'fixtures/colorize-semver-diff.tgz'))
fs.readFileSync(f.find('colorize-semver-diff.tgz'))
)
expect(Object.keys(filesIndex).sort()).toStrictEqual([
'LICENSE',
@@ -101,7 +104,7 @@ test('broken magic in tarball headers is handled gracefully', () => {
const dest = tempy.directory()
const cafs = createCafs(dest)
cafs.addFilesFromTarball(
fs.readFileSync(path.join(__dirname, 'fixtures/jquery.dirtyforms-2.0.0.tgz'))
fs.readFileSync(f.find('jquery.dirtyforms-2.0.0.tgz'))
)
})
@@ -109,7 +112,7 @@ test('unpack an older version of tar that prefixes with spaces', () => {
const dest = tempy.directory()
const cafs = createCafs(dest)
const { filesIndex } = cafs.addFilesFromTarball(
fs.readFileSync(path.join(__dirname, 'fixtures/parsers-3.0.0-rc.48.1.tgz'))
fs.readFileSync(f.find('parsers-3.0.0-rc.48.1.tgz'))
)
expect(Object.keys(filesIndex).sort()).toStrictEqual([
'lib/grammars/resolution.d.ts',
@@ -137,7 +140,7 @@ test('unpack a tarball that contains hard links', () => {
const dest = tempy.directory()
const cafs = createCafs(dest)
const { filesIndex } = cafs.addFilesFromTarball(
fs.readFileSync(path.join(__dirname, 'fixtures/vue.examples.todomvc.todo-store-0.0.1.tgz'))
fs.readFileSync(f.find('vue.examples.todomvc.todo-store-0.0.1.tgz'))
)
expect(Object.keys(filesIndex).length).toBeGreaterThan(0)
})
@@ -147,7 +150,7 @@ test('unpack should not fail when the tarball format seems to be not USTAR or GN
const dest = tempy.directory()
const cafs = createCafs(dest)
const { filesIndex } = cafs.addFilesFromTarball(
fs.readFileSync(path.join(__dirname, '../__fixtures__/devextreme-17.1.6.tgz'))
fs.readFileSync(f.find('devextreme-17.1.6.tgz'))
)
expect(Object.keys(filesIndex).length).toBeGreaterThan(0)
})

View File

@@ -9,6 +9,9 @@
"../../__typings__/**/*.d.ts"
],
"references": [
{
"path": "../../__utils__/test-fixtures"
},
{
"path": "../../fetching/fetcher-base"
},

View File

Binary file not shown.

View File

@@ -6,6 +6,8 @@ import { selfUpdate } from '@pnpm/tools.plugin-commands-self-updater'
import spawn from 'cross-spawn'
import nock from 'nock'
const pnpmTarballPath = require.resolve('@pnpm/tgz-fixtures/tgz/pnpm-9.1.0.tgz')
jest.mock('@pnpm/cli-meta', () => {
const actualModule = jest.requireActual('@pnpm/cli-meta')
@@ -89,7 +91,7 @@ test('self-update', async () => {
.reply(200, createMetadata('9.1.0', opts.registries.default))
nock(opts.registries.default)
.get('/pnpm/-/pnpm-9.1.0.tgz')
.replyWithFile(200, path.join(__dirname, 'pnpm-9.1.0.tgz'))
.replyWithFile(200, pnpmTarballPath)
await selfUpdate.handler(opts, [])
@@ -114,7 +116,7 @@ test('self-update by exact version', async () => {
.reply(200, createMetadata('9.2.0', opts.registries.default, ['9.1.0']))
nock(opts.registries.default)
.get('/pnpm/-/pnpm-9.1.0.tgz')
.replyWithFile(200, path.join(__dirname, 'pnpm-9.1.0.tgz'))
.replyWithFile(200, pnpmTarballPath)
await selfUpdate.handler(opts, ['9.1.0'])
@@ -186,7 +188,7 @@ test('self-update updates the packageManager field in package.json', async () =>
.reply(200, createMetadata('9.1.0', opts.registries.default))
nock(opts.registries.default)
.get('/pnpm/-/pnpm-9.1.0.tgz')
.replyWithFile(200, path.join(__dirname, 'pnpm-9.1.0.tgz'))
.replyWithFile(200, pnpmTarballPath)
const output = await selfUpdate.handler(opts, [])