/// import { jest } from '@jest/globals' import fs from 'fs' import path from 'path' import { FetchError, PnpmError } from '@pnpm/error' import { createFetchFromRegistry } from '@pnpm/fetch' import { createCafsStore } from '@pnpm/create-cafs-store' import { fixtures } from '@pnpm/test-fixtures' import { lexCompare } from '@pnpm/util.lex-comparator' import nock from 'nock' import ssri from 'ssri' import { temporaryDirectory } from 'tempy' const originalModule = await import('@pnpm/logger') jest.unstable_mockModule('@pnpm/logger', async () => { return { ...originalModule, globalWarn: jest.fn(), } }) const { globalWarn } = await import('@pnpm/logger') const { createTarballFetcher, BadTarballError, TarballIntegrityError, } = await import('@pnpm/tarball-fetcher') beforeEach(() => { jest.mocked(globalWarn).mockClear() }) const storeDir = temporaryDirectory() const filesIndexFile = path.join(storeDir, 'index.json') const cafs = createCafsStore(storeDir) const f = fixtures(import.meta.dirname) const tarballPath = f.find('babel-helper-hoist-variables-6.24.1.tgz') const tarballSize = 1279 const tarballIntegrity = 'sha1-HssnaJydJVE+rbyZFKc/VAi+enY=' const registry = 'http://example.com/' const fetchFromRegistry = createFetchFromRegistry({}) const getAuthHeader = () => undefined const fetch = createTarballFetcher(fetchFromRegistry, getAuthHeader, { rawConfig: {}, retry: { maxTimeout: 100, minTimeout: 0, retries: 1, }, }) const pkg = {} test('fail when tarball size does not match content-length', async () => { const scope = nock(registry) .get('/foo.tgz') .times(2) .replyWithFile(200, tarballPath, { 'Content-Length': (1024 * 1024).toString(), }) process.chdir(temporaryDirectory()) const resolution = { // Even though the integrity of the downloaded tarball // will not match this value, the error will be about // Content-Length mismatch, // which indicates bad network connection. (see https://github.com/pnpm/pnpm/issues/1235) integrity: 'sha1-HssnaJydJVE+rbzZFKc/VAi+enY=', tarball: `${registry}foo.tgz`, } await expect( fetch.remoteTarball(cafs, resolution, { filesIndexFile, lockfileDir: process.cwd(), pkg, }) ).rejects.toThrow( new BadTarballError({ expectedSize: 1048576, receivedSize: tarballSize, tarballUrl: resolution.tarball, }) ) expect(scope.isDone()).toBeTruthy() }) test('retry when tarball size does not match content-length', async () => { nock(registry) .get('/foo.tgz') .replyWithFile(200, tarballPath, { 'Content-Length': (1024 * 1024).toString(), }) nock(registry) .get('/foo.tgz') .replyWithFile(200, tarballPath, { 'Content-Length': tarballSize.toString(), }) process.chdir(temporaryDirectory()) const resolution = { tarball: 'http://example.com/foo.tgz' } const result = await fetch.remoteTarball(cafs, resolution, { filesIndexFile, lockfileDir: process.cwd(), pkg, }) expect(result.filesMap).toBeTruthy() expect(nock.isDone()).toBeTruthy() }) test('fail when integrity check fails two times in a row', async () => { const scope = nock(registry) .get('/foo.tgz') .times(2) .replyWithFile(200, f.find('babel-helper-hoist-variables-7.0.0-alpha.10.tgz'), { 'Content-Length': '1194', }) process.chdir(temporaryDirectory()) const resolution = { integrity: tarballIntegrity, tarball: 'http://example.com/foo.tgz', } await expect( fetch.remoteTarball(cafs, resolution, { filesIndexFile, lockfileDir: process.cwd(), pkg, }) ).rejects.toThrow( new TarballIntegrityError({ algorithm: 'sha512', expected: 'sha1-HssnaJydJVE+rbyZFKc/VAi+enY=', found: 'sha1-ACjKMFA7S6uRFXSDFfH4aT+4B4Y=', sri: '', url: resolution.tarball, }) ) expect(scope.isDone()).toBeTruthy() }) test('retry when integrity check fails', async () => { const scope = nock(registry) .get('/foo.tgz') .replyWithFile(200, f.find('babel-helper-hoist-variables-7.0.0-alpha.10.tgz'), { 'Content-Length': '1194', }) .get('/foo.tgz') .replyWithFile(200, tarballPath, { 'Content-Length': tarballSize.toString(), }) process.chdir(temporaryDirectory()) const resolution = { integrity: tarballIntegrity, tarball: 'http://example.com/foo.tgz', } const params: Array<[number | null, number]> = [] await fetch.remoteTarball(cafs, resolution, { filesIndexFile, lockfileDir: process.cwd(), onStart (size, attempts) { params.push([size, attempts]) }, pkg, }) expect(params[0]).toStrictEqual([1194, 1]) expect(params[1]).toStrictEqual([tarballSize, 2]) expect(scope.isDone()).toBeTruthy() }) test('fail when integrity check of local file fails', async () => { const storeDir = temporaryDirectory() process.chdir(storeDir) f.copy('babel-helper-hoist-variables-7.0.0-alpha.10.tgz', 'tar.tgz') const resolution = { integrity: tarballIntegrity, tarball: 'file:tar.tgz', } await expect( fetch.localTarball(cafs, resolution, { filesIndexFile, lockfileDir: process.cwd(), pkg, }) ).rejects.toThrow( new TarballIntegrityError({ algorithm: 'sha512', expected: 'sha1-HssnaJydJVE+rbyZFKc/VAi+enY=', found: 'sha1-ACjKMFA7S6uRFXSDFfH4aT+4B4Y=', sri: '', url: path.join(storeDir, 'tar.tgz'), }) ) }) test("don't fail when integrity check of local file succeeds", async () => { process.chdir(temporaryDirectory()) const localTarballLocation = path.resolve('tar.tgz') f.copy('babel-helper-hoist-variables-7.0.0-alpha.10.tgz', localTarballLocation) const resolution = { integrity: await getFileIntegrity(localTarballLocation), tarball: 'file:tar.tgz', } const { filesMap } = await fetch.localTarball(cafs, resolution, { filesIndexFile, lockfileDir: process.cwd(), pkg, }) expect(typeof filesMap.get('package.json')).toBe('string') }) test("don't fail when fetching a local tarball in offline mode", async () => { process.chdir(temporaryDirectory()) const tarballAbsoluteLocation = f.find('babel-helper-hoist-variables-7.0.0-alpha.10.tgz') const resolution = { integrity: await getFileIntegrity(tarballAbsoluteLocation), tarball: `file:${tarballAbsoluteLocation}`, } const fetch = createTarballFetcher(fetchFromRegistry, getAuthHeader, { offline: true, rawConfig: {}, retry: { maxTimeout: 100, minTimeout: 0, retries: 1, }, }) const { filesMap } = await fetch.localTarball(cafs, resolution, { filesIndexFile, lockfileDir: process.cwd(), pkg, }) expect(typeof filesMap.get('package.json')).toBe('string') }) test('fail when trying to fetch a non-local tarball in offline mode', async () => { process.chdir(temporaryDirectory()) const tarballAbsoluteLocation = f.find('babel-helper-hoist-variables-7.0.0-alpha.10.tgz') const resolution = { integrity: await getFileIntegrity(tarballAbsoluteLocation), tarball: `${registry}foo.tgz`, } const fetch = createTarballFetcher(fetchFromRegistry, getAuthHeader, { offline: true, rawConfig: {}, retry: { maxTimeout: 100, minTimeout: 0, retries: 1, }, }) await expect( fetch.remoteTarball(cafs, resolution, { filesIndexFile, lockfileDir: process.cwd(), pkg, }) ).rejects.toThrow( new PnpmError('NO_OFFLINE_TARBALL', `A package is missing from the store but cannot download it in offline mode. \ The missing package may be downloaded from ${resolution.tarball}.`) ) }) test('retry on server error', async () => { const scope = nock(registry) .get('/foo.tgz') .reply(500) .get('/foo.tgz') .replyWithFile(200, tarballPath, { 'Content-Length': tarballSize.toString(), }) process.chdir(temporaryDirectory()) const resolution = { integrity: tarballIntegrity, tarball: 'http://example.com/foo.tgz', } const index = await fetch.remoteTarball(cafs, resolution, { filesIndexFile, lockfileDir: process.cwd(), pkg, }) expect(index).toBeTruthy() expect(scope.isDone()).toBeTruthy() }) test('throw error when accessing private package w/o authorization', async () => { const scope = nock(registry) .get('/foo.tgz') .reply(403) process.chdir(temporaryDirectory()) const resolution = { integrity: tarballIntegrity, tarball: 'http://example.com/foo.tgz', } await expect( fetch.remoteTarball(cafs, resolution, { filesIndexFile, lockfileDir: process.cwd(), pkg, }) ).rejects.toThrow( new FetchError( { url: resolution.tarball, }, { status: 403, // statusText: 'Forbidden', statusText: '', } ) ) expect(scope.isDone()).toBeTruthy() }) test('do not retry when package does not exist', async () => { const scope = nock(registry) .get('/foo.tgz') .reply(404) process.chdir(temporaryDirectory()) const resolution = { integrity: tarballIntegrity, tarball: 'http://example.com/foo.tgz', } await expect( fetch.remoteTarball(cafs, resolution, { filesIndexFile, lockfileDir: process.cwd(), pkg, }) ).rejects.toThrow( new FetchError( { url: resolution.tarball, }, { status: 404, statusText: '', } ) ) expect(scope.isDone()).toBeTruthy() }) test('accessing private packages', async () => { const scope = nock( registry, { reqheaders: { authorization: 'Bearer ofjergrg349gj3f2', }, } ) .get('/foo.tgz') .replyWithFile(200, tarballPath, { 'Content-Length': tarballSize.toString(), }) process.chdir(temporaryDirectory()) const getAuthHeader = () => 'Bearer ofjergrg349gj3f2' const fetch = createTarballFetcher(fetchFromRegistry, getAuthHeader, { rawConfig: {}, retry: { maxTimeout: 100, minTimeout: 0, retries: 1, }, }) const resolution = { integrity: tarballIntegrity, registry, tarball: 'http://example.com/foo.tgz', } const index = await fetch.remoteTarball(cafs, resolution, { filesIndexFile, lockfileDir: process.cwd(), pkg, }) expect(index).toBeTruthy() expect(scope.isDone()).toBeTruthy() }) async function getFileIntegrity (filename: string) { return (await ssri.fromStream(fs.createReadStream(filename))).toString() } // Covers the regression reported in https://github.com/pnpm/pnpm/issues/4064 test('fetch a big repository', async () => { process.chdir(temporaryDirectory()) const resolution = { tarball: 'https://codeload.github.com/sveltejs/action-deploy-docs/tar.gz/a65fbf5a90f53c9d72fed4daaca59da50f074355' } const result = await fetch.gitHostedTarball(cafs, resolution, { filesIndexFile, lockfileDir: process.cwd(), pkg, }) expect(result.filesMap).toBeTruthy() }) test('fail when preparing a git-hosted package', async () => { process.chdir(temporaryDirectory()) const resolution = { tarball: 'https://codeload.github.com/pnpm-e2e/prepare-script-fails/tar.gz/ba58874aae1210a777eb309dd01a9fdacc7e54e7' } await expect( fetch.gitHostedTarball(cafs, resolution, { allowBuild: (pkgName) => pkgName === '@pnpm.e2e/prepare-script-fails', filesIndexFile, lockfileDir: process.cwd(), pkg, }) ).rejects.toThrow('Failed to prepare git-hosted package fetched from "https://codeload.github.com/pnpm-e2e/prepare-script-fails/tar.gz/ba58874aae1210a777eb309dd01a9fdacc7e54e7": @pnpm.e2e/prepare-script-fails@1.0.0 npm-install: `npm install`') }) test('take only the files included in the package, when fetching a git-hosted package', async () => { process.chdir(temporaryDirectory()) const resolution = { tarball: 'https://codeload.github.com/pnpm-e2e/pkg-with-ignored-files/tar.gz/958d6d487217512bb154d02836e9b5b922a600d8' } const result = await fetch.gitHostedTarball(cafs, resolution, { filesIndexFile, lockfileDir: process.cwd(), pkg, }) expect(Array.from(result.filesMap.keys()).sort(lexCompare)).toStrictEqual([ 'README.md', 'dist/index.js', 'package.json', ]) }) test('fail when extracting a broken tarball', async () => { const scope = nock(registry) .get('/foo.tgz') .times(2) .reply(200, 'this is not a valid tarball') process.chdir(temporaryDirectory()) const resolution = { tarball: `${registry}foo.tgz`, } await expect( fetch.remoteTarball(cafs, resolution, { filesIndexFile, lockfileDir: process.cwd(), pkg, }) ).rejects.toThrow(`Failed to add tarball from "${registry}foo.tgz" to store: Invalid checksum for TAR header at offset 0. Expected 0, got NaN` ) expect(scope.isDone()).toBeTruthy() }) test('do not build the package when scripts are ignored', async () => { process.chdir(temporaryDirectory()) const tarball = 'https://codeload.github.com/pnpm-e2e/prepare-script-works/tar.gz/55416a9c468806a935636c0ad0371a14a64df8c9' const resolution = { tarball } const fetch = createTarballFetcher(fetchFromRegistry, getAuthHeader, { ignoreScripts: true, rawConfig: {}, retry: { maxTimeout: 100, minTimeout: 0, retries: 1, }, }) const { filesMap } = await fetch.gitHostedTarball(cafs, resolution, { filesIndexFile, lockfileDir: process.cwd(), pkg, }) expect(filesMap.has('package.json')).toBeTruthy() expect(filesMap.has('prepare.txt')).toBeFalsy() expect(globalWarn).toHaveBeenCalledWith(`The git-hosted package fetched from "${tarball}" has to be built but the build scripts were ignored.`) }) test('when extracting files with the same name, pick the last ones', async () => { const tar = f.find('tarball-with-duplicate-files/archive.tar') const resolution = { tarball: `file:${tar}`, } const { filesMap, manifest } = await fetch.localTarball(cafs, resolution, { filesIndexFile, lockfileDir: process.cwd(), readManifest: true, pkg, }) const pkgJson = JSON.parse(fs.readFileSync(filesMap.get('package.json')!, 'utf8')) expect(pkgJson.name).toBe('pkg2') expect(manifest?.name).toBe('pkg2') }) test('use the subfolder when path is present', async () => { process.chdir(temporaryDirectory()) const resolution = { tarball: 'https://codeload.github.com/RexSkz/test-git-subfolder-fetch/tar.gz/2b42a57a945f19f8ffab8ecbd2021fdc2c58ee22', path: '/packages/simple-react-app', } const fetch = createTarballFetcher(fetchFromRegistry, getAuthHeader, { ignoreScripts: true, rawConfig: {}, retry: { maxTimeout: 100, minTimeout: 0, retries: 1, }, }) const { filesMap } = await fetch.gitHostedTarball(cafs, resolution, { filesIndexFile, lockfileDir: process.cwd(), pkg, }) expect(filesMap.has('package.json')).toBeTruthy() expect(filesMap.has('lerna.json')).toBeFalsy() }) test('prevent directory traversal attack when path is present', async () => { process.chdir(temporaryDirectory()) const tarball = 'https://codeload.github.com/RexSkz/test-git-subfolder-fetch/tar.gz/2b42a57a945f19f8ffab8ecbd2021fdc2c58ee22' const path = '../../etc' const resolution = { tarball, path } const fetch = createTarballFetcher(fetchFromRegistry, getAuthHeader, { ignoreScripts: true, rawConfig: {}, retry: { maxTimeout: 100, minTimeout: 0, retries: 1, }, }) await expect(() => fetch.gitHostedTarball(cafs, resolution, { filesIndexFile, lockfileDir: process.cwd(), pkg, })).rejects.toThrow(`Failed to prepare git-hosted package fetched from "${tarball}": Path "${path}" should be a sub directory`) }) test('fail when path is not exists', async () => { process.chdir(temporaryDirectory()) const tarball = 'https://codeload.github.com/RexSkz/test-git-subfolder-fetch/tar.gz/2b42a57a945f19f8ffab8ecbd2021fdc2c58ee22' const path = '/not-exists' const resolution = { tarball, path } const fetch = createTarballFetcher(fetchFromRegistry, getAuthHeader, { ignoreScripts: true, rawConfig: {}, retry: { maxTimeout: 100, minTimeout: 0, retries: 1, }, }) await expect(() => fetch.gitHostedTarball(cafs, resolution, { filesIndexFile, lockfileDir: process.cwd(), pkg, })).rejects.toThrow(`Failed to prepare git-hosted package fetched from "${tarball}": Path "${path}" is not a directory`) })