mirror/pnpm
1
0
Fork 0
mirror of https://github.com/pnpm/pnpm.git synced 2026-01-08 15:08:27 -05:00
Code Issues Packages Projects Releases Wiki Activity

feat!: using a content-addressable filesystem

Browse Source 
ref #2470

BREAKING CHANGE:

New global store structure
This commit is contained in:
Zoltan Kochan
2020-04-25 21:34:07 +03:00
committed by Zoltan Kochan
parent f385c82cf7
commit b6a82072ea
59 changed files with 896 additions and 634 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
.changeset/pretty-elephants-warn.md Normal file
View File

@@ -0,0 +1,5 @@
---
"@pnpm/cafs": major
---
Project created.

18
.changeset/sharp-coins-enjoy.md Normal file
View File

@@ -0,0 +1,18 @@
---
"@pnpm/cafs": major
"@pnpm/default-fetcher": major
"@pnpm/fetcher-base": major
"@pnpm/git-fetcher": major
"@pnpm/headless": major
"@pnpm/package-requester": major
"@pnpm/package-store": major
"@pnpm/plugin-commands-store": major
"pnpm": major
"@pnpm/server": major
"@pnpm/store-controller-types": major
"@pnpm/tarball-fetcher": major
"@pnpm/store-connection-manager": minor
"supi": minor
---
Using a content-addressable filesystem for storing packages.

17
packages/cafs/README.md Normal file
View File

@@ -0,0 +1,17 @@
# @pnpm/cafs
> A content-addressable filesystem for the packages storage
<!--@shields('npm')-->
[![npm version](https://img.shields.io/npm/v/@pnpm/cafs.svg)](https://www.npmjs.com/package/@pnpm/cafs)
<!--/@-->
## Installation
```sh
<pnpm|npm|yarn> add @pnpm/cafs
```
## License
MIT © [Zoltan Kochan](https://www.kochan.io/)

BIN
packages/cafs/__fixtures__/babel-helper-hoist-variables-6.24.1.tgz Normal file
View File

Binary file not shown.

48
packages/cafs/package.json Normal file
View File

@@ -0,0 +1,48 @@
{
"name": "@pnpm/cafs",
"version": "0.0.0",
"description": "A content-addressable filesystem for the packages storage",
"main": "lib/index.js",
"typings": "lib/index.d.ts",
"scripts": {
"_test": "cd ../.. && c8 --reporter lcov --reports-dir packages/cafs/coverage ts-node packages/cafs/test --type-check",
"test": "pnpm run compile && pnpm run _test",
"compile": "rimraf lib tsconfig.tsbuildinfo && tsc --build",
"prepublishOnly": "pnpm run compile"
},
"keywords": [],
"author": "Zoltan Kochan <z@kochan.io> (https://www.kochan.io/)",
"license": "MIT",
"dependencies": {
"@pnpm/fetcher-base": "workspace:6.0.0",
"@zkochan/rimraf": "1.0.0",
"decompress-maybe": "^1.0.0",
"get-stream": "5.1.0",
"mz": "2.7.0",
"p-limit": "2.3.0",
"path-exists": "4.0.0",
"path-temp": "2.0.0",
"rename-overwrite": "2.0.2",
"ssri": "6.0.1",
"tar-stream": "^2.1.2"
},
"devDependencies": {
"@types/mz": "2.7.0",
"@types/node": "^13.13.2",
"@types/ssri": "^6.0.2",
"@types/tar-stream": "^2.1.0",
"tempy": "^0.5.0"
},
"bugs": {
"url": "https://github.com/pnpm/pnpm/issues"
},
"engines": {
"node": ">=10.13"
},
"files": [
"lib",
"!*.map"
],
"homepage": "https://github.com/pnpm/pnpm/blob/master/packages/cafs#readme",
"repository": "https://github.com/pnpm/pnpm/blob/master/packages/cafs"
}

58
packages/cafs/src/addFilesFromDir.ts Normal file
View File

@@ -0,0 +1,58 @@
import { FilesIndex } from '@pnpm/fetcher-base'
import fs = require('mz/fs')
import pLimit from 'p-limit'
import path = require('path')
import ssri = require('ssri')
const limit = pLimit(20)
const MAX_BULK_SIZE = 1 * 1024 * 1024 // 1MB
export default async function (
cafs: {
addStream: (stream: NodeJS.ReadableStream) => Promise<ssri.Integrity>,
addBuffer: (buffer: Buffer) => Promise<ssri.Integrity>,
},
dirname: string,
) {
const index = {}
await _retrieveFileIntegrities(cafs, dirname, dirname, index)
return index
}
async function _retrieveFileIntegrities (
cafs: {
addStream: (stream: NodeJS.ReadableStream) => Promise<ssri.Integrity>,
addBuffer: (buffer: Buffer) => Promise<ssri.Integrity>,
},
rootDir: string,
currDir: string,
index: FilesIndex,
) {
try {
const files = await fs.readdir(currDir)
await Promise.all(files.map(async (file) => {
const fullPath = path.join(currDir, file)
const stat = await fs.stat(fullPath)
if (stat.isDirectory()) {
await _retrieveFileIntegrities(cafs, rootDir, fullPath, index)
return
}
if (stat.isFile()) {
const relativePath = path.relative(rootDir, fullPath)
index[relativePath] = {
generatingIntegrity: limit(() => {
return stat.size < MAX_BULK_SIZE
? fs.readFile(fullPath).then(cafs.addBuffer)
: cafs.addStream(fs.createReadStream(fullPath))
}),
size: stat.size,
}
}
}))
} catch (err) {
if (err.code !== 'ENOENT') {
throw err
}
}
}

41
packages/cafs/src/addFilesFromTarball.ts Normal file
View File

@@ -0,0 +1,41 @@
import { FilesIndex } from '@pnpm/fetcher-base'
import decompress = require('decompress-maybe')
import ssri = require('ssri')
import { Duplex, PassThrough } from 'stream'
import tar = require('tar-stream')
export default async function (
addStreamToCafs: (fileStream: PassThrough) => Promise<ssri.Integrity>,
_ignore: null | ((filename: string) => Boolean),
stream: NodeJS.ReadableStream,
): Promise<FilesIndex> {
const ignore = _ignore ? _ignore : () => false
const extract = tar.extract()
const filesIndex = {}
await new Promise((resolve, reject) => {
extract.on('entry', async (header, fileStream, next) => {
const filename = header.name.substr(header.name.indexOf('/') + 1)
if (header.type !== 'file' || ignore(filename)) {
fileStream.resume()
next()
return
}
const generatingIntegrity = addStreamToCafs(fileStream)
filesIndex[filename] = {
generatingIntegrity,
size: header.size,
}
next()
})
// listener
extract.on('finish', () => resolve())
extract.on('error', reject)
// pipe through extractor
stream
.on('error', reject)
.pipe(decompress() as Duplex)
.on('error', reject).pipe(extract)
})
return filesIndex
}

76
packages/cafs/src/checkFilesIntegrity.ts Normal file
View File

@@ -0,0 +1,76 @@
import rimraf = require('@zkochan/rimraf')
import fs = require('mz/fs')
import pLimit from 'p-limit'
import ssri = require('ssri')
import { getFilePathInCafs } from '.'
const limit = pLimit(20)
const MAX_BULK_SIZE = 1 * 1024 * 1024 // 1MB
export default async function (
cafsDir: string,
integrityObj: Record<string, { size: number, integrity: string }>,
) {
let verified = true
await Promise.all(
Object.keys(integrityObj)
.map((f) =>
limit(async () => {
const fstat = integrityObj[f]
if (!fstat.integrity) {
throw new Error(`Integrity checksum is missing for ${f}`)
}
if (
!await verifyFile(
getFilePathInCafs(cafsDir, fstat.integrity),
fstat,
)
) {
verified = false
}
}),
),
)
return verified
}
async function verifyFile (filename: string, fstat: { size: number, integrity: string }) {
if (fstat.size > MAX_BULK_SIZE) {
try {
const ok = Boolean(await ssri.checkStream(fs.createReadStream(filename), fstat.integrity))
if (!ok) {
await rimraf(filename)
}
return ok
} catch (err) {
switch (err.code) {
case 'ENOENT': return false
case 'EINTEGRITY': {
// Broken files are removed from the store
await rimraf(filename)
return false
}
}
throw err
}
}
try {
const data = await fs.readFile(filename)
const ok = Boolean(ssri.checkData(data, fstat.integrity))
if (!ok) {
await rimraf(filename)
}
return ok
} catch (err) {
switch (err.code) {
case 'ENOENT': return false
case 'EINTEGRITY': {
// Broken files are removed from the store
await rimraf(filename)
return false
}
}
throw err
}
}

81
packages/cafs/src/index.ts Normal file
View File

@@ -0,0 +1,81 @@
import getStream = require('get-stream')
import path = require('path')
import exists = require('path-exists')
import pathTemp = require('path-temp')
import renameOverwrite = require('rename-overwrite')
import ssri = require('ssri')
import { Hash } from 'ssri'
import addFilesFromDir from './addFilesFromDir'
import addFilesFromTarball from './addFilesFromTarball'
import checkFilesIntegrity from './checkFilesIntegrity'
import writeFile from './writeFile'
export { checkFilesIntegrity }
export default function createCafs (cafsDir: string, ignore?: ((filename: string) => Boolean)) {
const locker = new Map()
const addStream = addStreamToCafs.bind(null, locker, cafsDir)
const addBuffer = addBufferToCafs.bind(null, locker, cafsDir)
return {
addFilesFromDir: addFilesFromDir.bind(null, { addBuffer, addStream }),
addFilesFromTarball: addFilesFromTarball.bind(null, addStream, ignore ?? null),
}
}
async function addStreamToCafs (
locker: Map<string, Promise<void>>,
cafsDir: string,
fileStream: NodeJS.ReadableStream,
): Promise<ssri.Integrity> {
const buffer = await getStream.buffer(fileStream)
return addBufferToCafs(locker, cafsDir, buffer)
}
async function addBufferToCafs (
locker: Map<string, Promise<void>>,
cafsDir: string,
buffer: Buffer,
): Promise<ssri.Integrity> {
const integrity = ssri.fromData(buffer)
const fileDest = contentPathFromHex(cafsDir, integrity.hexDigest())
if (locker.has(fileDest)) {
await locker.get(fileDest)
return integrity
}
const p = (async () => {
// This is a slow operation. Should be rewritten
if (await exists(fileDest)) return
// This might be too cautious.
// The write is atomic, so in case pnpm crashes, no broken file
// will be added to the store.
// It might be a redundant step though, as we verify the contents of the
// files before linking
//
// If we don't allow --no-verify-store-integrity then we probably can write
// to the final file directly.
const temp = pathTemp(path.dirname(fileDest))
await writeFile(temp, buffer)
await renameOverwrite(temp, fileDest)
})()
locker.set(fileDest, p)
await p
return integrity
}
export function getFilePathInCafs (cafsDir: string, integrity: string | Hash) {
return contentPathFromIntegrity(cafsDir, integrity)
}
function contentPathFromIntegrity (cafsDir: string, integrity: string | Hash) {
const sri = ssri.parse(integrity, { single: true })
return contentPathFromHex(cafsDir, sri.hexDigest())
}
function contentPathFromHex (cafsDir: string, hex: string) {
return path.join(
cafsDir,
hex.slice(0, 2),
hex.slice(2),
)
}

19
packages/cafs/src/writeFile.ts Normal file
View File

@@ -0,0 +1,19 @@
import fs = require('mz/fs')
import path = require('path')
const dirs = new Set()
// write a stream to destination file
export default async function (
fileDest: string,
buffer: Buffer,
) {
const dir = path.dirname(fileDest)
if (!dirs.has(dir)) {
await fs.mkdir(dir, { recursive: true })
dirs.add(dir)
}
const fd = await fs.open(fileDest, 'w')
await fs.write(fd, buffer, 0, buffer.length, 0)
await fs.close(fd)
}

15
packages/cafs/test/index.ts Normal file
View File

@@ -0,0 +1,15 @@
import fs = require('fs')
import path = require('path')
import test = require('tape')
import tempy = require('tempy')
import createCafs from '../src'
test('unpack', async (t) => {
const dest = tempy.directory()
t.comment(dest)
const cafs = createCafs(dest)
await cafs.addFilesFromTarball(
fs.createReadStream(path.join(__dirname, '../__fixtures__/babel-helper-hoist-variables-6.24.1.tgz')),
)
t.end()
})

16
packages/cafs/tsconfig.json Normal file
View File

@@ -0,0 +1,16 @@
{
"extends": "@pnpm/tsconfig",
"compilerOptions": {
"outDir": "lib",
"rootDir": "src"
},
"include": [
"src/**/*.ts",
"../../typings/**/*.d.ts"
],
"references": [
{
"path": "../fetcher-base"
}
]
}

10
packages/default-fetcher/src/index.ts
View File

@@ -1,16 +1,9 @@
import fetchFromGit from '@pnpm/git-fetcher'
import {
DirectoryResolution,
TarballResolution,
} from '@pnpm/resolver-base'
import createTarballFetcher, {
IgnoreFunction,
} from '@pnpm/tarball-fetcher'
import createTarballFetcher from '@pnpm/tarball-fetcher'
export default function (
opts: {
alwaysAuth?: boolean,
fsIsCaseSensitive?: boolean,
registry: string,
rawConfig: object,
strictSsl?: boolean,
@@ -25,7 +18,6 @@ export default function (
fetchRetryMintimeout?: number,
fetchRetryMaxtimeout?: number,
userAgent?: string,
ignoreFile?: IgnoreFunction,
offline?: boolean,
},
) {

12
packages/fetcher-base/src/index.ts
View File

@@ -1,5 +1,10 @@
import { Resolution } from '@pnpm/resolver-base'
import { IntegrityMap } from 'ssri'
import { Integrity } from 'ssri'
export type Cafs = {
addFilesFromDir: (dir: string) => Promise<FilesIndex>,
addFilesFromTarball: (stream: NodeJS.ReadableStream) => Promise<FilesIndex>,
}
export interface FetchOptions {
cachedTarballLocation: string,
@@ -9,19 +14,18 @@ export interface FetchOptions {
}
export type FetchFunction = (
cafs: Cafs,
resolution: Resolution,
targetFolder: string,
opts: FetchOptions,
) => Promise<FetchResult>
export interface FetchResult {
filesIndex: FilesIndex,
tempLocation: string,
}
export interface FilesIndex {
[filename: string]: {
size: number,
generatingIntegrity?: Promise<IntegrityMap>,
generatingIntegrity: Promise<Integrity>,
},
}

8
packages/git-fetcher/package.json
View File

@@ -30,13 +30,13 @@
},
"homepage": "https://github.com/pnpm/pnpm/blob/master/packages/git-fetcher#readme",
"dependencies": {
"@pnpm/fetcher-base": "workspace:6.0.0",
"@zkochan/rimraf": "1.0.0",
"dint": "4.0.0",
"execa": "4.0.0",
"path-temp": "2.0.0"
"tempy": "0.5.0"
},
"devDependencies": {
"@pnpm/git-fetcher": "link:",
"tempy": "0.5.0"
"@pnpm/cafs": "workspace:0.0.0",
"@pnpm/git-fetcher": "link:"
}
}

13
packages/git-fetcher/src/index.ts
View File

@@ -1,8 +1,8 @@
import { Cafs } from '@pnpm/fetcher-base'
import rimraf = require('@zkochan/rimraf')
import dint = require('dint')
import execa = require('execa')
import path = require('path')
import pathTemp = require('path-temp')
import tempy = require('tempy')
export default () => {
return {
@@ -11,16 +11,17 @@ export default () => {
repo: string,
commit: string,
},
targetFolder: string,
opts: {
cafs: Cafs,
},
) {
const tempLocation = pathTemp(targetFolder)
const tempLocation = tempy.directory()
await execGit(['clone', resolution.repo, tempLocation])
await execGit(['checkout', resolution.commit], { cwd: tempLocation })
// removing /.git to make directory integrity calculation faster
await rimraf(path.join(tempLocation, '.git'))
return {
filesIndex: await dint.from(tempLocation),
tempLocation,
filesIndex: await opts.cafs.addFilesFromDir(tempLocation),
}
},
}

9
packages/git-fetcher/test/index.ts
View File

@@ -1,15 +1,20 @@
///<reference path="../../../typings/index.d.ts"/>
import createCafs from '@pnpm/cafs'
import createFetcher from '@pnpm/git-fetcher'
import test = require('tape')
import tempy = require('tempy')
test('fetch', async t => {
const cafsDir = tempy.directory()
t.comment(`cafs at ${cafsDir}`)
const fetch = createFetcher().git
const fetchResult = await fetch({
commit: 'c9b30e71d704cd30fa71f2edd1ecc7dcc4985493',
repo: 'https://github.com/kevva/is-positive.git',
}, tempy.directory())
t.ok(fetchResult.tempLocation)
}, {
cafs: createCafs(cafsDir),
})
t.ok(fetchResult.filesIndex['package.json'])
t.ok(await fetchResult.filesIndex['package.json'].generatingIntegrity)
t.end()
})

9
packages/git-fetcher/tsconfig.json
View File

@@ -8,5 +8,12 @@
"src/**/*.ts",
"../../typings/**/*.d.ts"
],
"references": []
"references": [
{
"path": "../fetcher-base"
},
{
"path": "../cafs"
}
]
}

4
packages/headless/package.json
View File

@@ -34,12 +34,14 @@
"fs-extra": "9.0.0",
"is-windows": "1.0.2",
"isexe": "2.0.0",
"load-json-file": "6.2.0",
"mz": "2.7.0",
"npm-run-all": "4.1.5",
"read-yaml-file": "2.0.0",
"sinon": "9.0.2",
"tape-promise": "4.0.0",
"tempy": "0.5.0"
"tempy": "0.5.0",
"write-json-file": "4.3.0"
},
"directories": {
"test": "test"

5
packages/headless/src/index.ts
View File

@@ -542,7 +542,6 @@ async function lockfileToDepGraph (
// ignore
})
graph[peripheralLocation] = {
centralLocation: pkgLocation.dir,
children: {},
fetchingFiles: fetchResponse.files,
finishing: fetchResponse.finishing,
@@ -649,7 +648,6 @@ async function getChildrenPaths (
export interface DependenciesGraphNode {
hasBundledDependencies: boolean,
centralLocation: string,
modules: string,
name: string,
fetchingFiles: () => Promise<PackageFilesResponse>,
@@ -686,8 +684,7 @@ async function linkAllPkgs (
depNodes.map(async (depNode) => {
const filesResponse = await depNode.fetchingFiles()
if (depNode.independent) return
return storeController.importPackage(depNode.centralLocation, depNode.peripheralLocation, {
return storeController.importPackage(depNode.peripheralLocation, {
filesResponse,
force: opts.force,
})

18
packages/headless/test/index.ts
View File

@@ -15,12 +15,14 @@ import readprojectsContext from '@pnpm/read-projects-context'
import { REGISTRY_MOCK_PORT } from '@pnpm/registry-mock'
import rimraf = require('@zkochan/rimraf')
import fse = require('fs-extra')
import loadJsonFile = require('load-json-file')
import path = require('path')
import exists = require('path-exists')
import readYamlFile from 'read-yaml-file'
import sinon = require('sinon')
import test = require('tape')
import tempy = require('tempy')
import writeJsonFile = require('write-json-file')
import testDefaults from './utils/testDefaults'
const fixtures = path.join(__dirname, 'fixtures')
@@ -710,7 +712,7 @@ test('installing with hoistPattern=* and shamefullyHoist=true', async (t) => {
const ENGINE_DIR = `${process.platform}-${process.arch}-node-${process.version.split('.')[0]}`
test('using side effects cache', async (t) => {
test.skip('using side effects cache', async (t) => {
const prefix = path.join(fixtures, 'side-effects')
// Right now, hardlink does not work with side effects, so we specify copy as the packageImportMethod
@@ -723,18 +725,24 @@ test('using side effects cache', async (t) => {
}, {}, {}, { packageImportMethod: 'copy' })
await headless(opts)
const cacheBuildDir = path.join(opts.storeDir, `localhost+${REGISTRY_MOCK_PORT}/diskusage/1.1.3/side_effects/${ENGINE_DIR}/package/build`)
fse.writeFileSync(path.join(cacheBuildDir, 'new-file.txt'), 'some new content')
const cacheIntegrityPath = path.join(opts.storeDir, `localhost+${REGISTRY_MOCK_PORT}/diskusage/1.1.3/side_effects/${ENGINE_DIR}/integrity.json`)
const cacheIntegrity = await loadJsonFile(cacheIntegrityPath)
t.ok(cacheIntegrity['build/Makefile'])
delete cacheIntegrity['build/Makefile']
t.ok(cacheIntegrity['build/binding.Makefile'])
await writeJsonFile(cacheIntegrityPath, cacheIntegrity)
await rimraf(path.join(prefix, 'node_modules'))
await headless(opts)
t.ok(await exists(path.join(prefix, 'node_modules/diskusage/build/new-file.txt')), 'side effects cache correctly used')
t.notOk(await exists(path.join(prefix, 'node_modules/diskusage/build/Makefile')), 'side effects cache correctly used')
t.ok(await exists(path.join(prefix, 'node_modules/diskusage/build/binding.Makefile')), 'side effects cache correctly used')
t.end()
})
test('using side effects cache and hoistPattern=*', async (t) => {
test.skip('using side effects cache and hoistPattern=*', async (t) => {
const lockfileDir = path.join(fixtures, 'side-effects-of-subdep')
const { projects } = await readprojectsContext(

4
packages/package-requester/package.json
View File

@@ -35,7 +35,7 @@
"@pnpm/logger": "^3.1.0"
},
"dependencies": {
"@pnpm/check-package": "3.0.1",
"@pnpm/cafs": "workspace:0.0.0",
"@pnpm/core-loggers": "workspace:4.0.1",
"@pnpm/fetcher-base": "workspace:6.0.0",
"@pnpm/pkgid-to-filename": "2.0.0",
@@ -43,13 +43,11 @@
"@pnpm/resolver-base": "workspace:7.0.0",
"@pnpm/store-controller-types": "workspace:7.0.0",
"@pnpm/types": "workspace:5.0.0",
"@zkochan/rimraf": "1.0.0",
"load-json-file": "6.2.0",
"make-dir": "^3.1.0",
"mz": "2.7.0",
"p-limit": "2.3.0",
"p-queue": "6.3.0",
"path-exists": "4.0.0",
"promise-share": "1.0.0",
"ramda": "0.27.0",
"rename-overwrite": "2.0.2",

180
packages/package-requester/src/packageRequester.ts
View File

@@ -1,13 +1,17 @@
import checkPackage from '@pnpm/check-package'
import createCafs, {
checkFilesIntegrity as _checkFilesIntegrity,
getFilePathInCafs as _getFilePathInCafs,
} from '@pnpm/cafs'
import { fetchingProgressLogger } from '@pnpm/core-loggers'
import {
Cafs,
FetchFunction,
FetchOptions,
FetchResult,
} from '@pnpm/fetcher-base'
import logger from '@pnpm/logger'
import pkgIdToFilename from '@pnpm/pkgid-to-filename'
import { fromDir as readPkgFromDir } from '@pnpm/read-package-json'
import readPackage from '@pnpm/read-package-json'
import {
DirectoryResolution,
Resolution,
@@ -27,18 +31,13 @@ import {
DependencyManifest,
StoreIndex,
} from '@pnpm/types'
import rimraf = require('@zkochan/rimraf')
import loadJsonFile = require('load-json-file')
import makeDir = require('make-dir')
import * as fs from 'mz/fs'
import PQueue from 'p-queue'
import path = require('path')
import exists = require('path-exists')
import pShare = require('promise-share')
import R = require('ramda')
import renameOverwrite = require('rename-overwrite')
import ssri = require('ssri')
import symlinkDir = require('symlink-dir')
import writeJsonFile = require('write-json-file')
const TARBALL_INTEGRITY_FILENAME = 'tarball-integrity'
@@ -64,12 +63,14 @@ export default function (
resolve: ResolveFunction,
fetchers: {[type: string]: FetchFunction},
opts: {
ignoreFile?: (filename: string) => boolean,
networkConcurrency?: number,
storeDir: string,
storeIndex: StoreIndex,
verifyStoreIntegrity: boolean,
},
): RequestPackageFunction & {
cafs: Cafs,
fetchPackageToStore: FetchPackageToStoreFunction,
requestPackage: RequestPackageFunction,
} {
@@ -82,10 +83,15 @@ export default function (
requestsQueue['counter'] = 0 // tslint:disable-line
requestsQueue['concurrency'] = networkConcurrency // tslint:disable-line
const fetch = fetcher.bind(null, fetchers)
const cafsDir = path.join(opts.storeDir, 'files')
const cafs = createCafs(cafsDir, opts.ignoreFile)
const getFilePathInCafs = _getFilePathInCafs.bind(null, cafsDir)
const fetch = fetcher.bind(null, fetchers, cafs)
const fetchPackageToStore = fetchToStore.bind(null, {
checkFilesIntegrity: _checkFilesIntegrity.bind(null, cafsDir),
fetch,
fetchingLocker: new Map(),
getFilePathInCafs,
requestsQueue,
storeDir: opts.storeDir,
storeIndex: opts.storeIndex,
@@ -99,7 +105,7 @@ export default function (
verifyStoreIntegrity: opts.verifyStoreIntegrity,
})
return Object.assign(requestPackage, { fetchPackageToStore, requestPackage })
return Object.assign(requestPackage, { cafs, fetchPackageToStore, requestPackage })
}
async function resolveAndFetch (
@@ -235,10 +241,10 @@ async function resolveAndFetch (
function fetchToStore (
ctx: {
checkFilesIntegrity: (integrity: Record<string, { size: number, integrity: string }>) => Promise<boolean>,
fetch: (
packageId: string,
resolution: Resolution,
target: string,
opts: FetchOptions,
) => Promise<FetchResult>,
fetchingLocker: Map<string, {
@@ -247,6 +253,7 @@ function fetchToStore (
bundledManifest?: Promise<BundledManifest>,
inStoreLocation: string,
}>,
getFilePathInCafs: (integrity: string) => string,
requestsQueue: {add: <T>(fn: () => Promise<T>, opts: {priority: number}) => Promise<T>},
storeIndex: StoreIndex,
storeDir: string,
@@ -297,7 +304,7 @@ function fetchToStore (
// Changing the value of fromStore is needed for correct reporting of `pnpm server`.
// Otherwise, if a package was not in store when the server started, it will be always
// reported as "downloaded" instead of "reused".
files.promise.then(({ filenames, fromStore }) => { // tslint:disable-line
files.promise.then(({ filesIndex, fromStore }) => { // tslint:disable-line
// If it's already in the store, we don't need to update the cache
if (fromStore) {
return
@@ -317,7 +324,7 @@ function fetchToStore (
ctx.fetchingLocker.set(opts.pkgId, {
bundledManifest: tmp.bundledManifest,
files: Promise.resolve({
filenames,
filesIndex,
fromStore: true,
}),
finishing: tmp.finishing,
@@ -338,7 +345,7 @@ function fetchToStore (
if (opts.fetchRawManifest && !result.bundledManifest) {
result.bundledManifest = removeKeyOnFail(
result.files.then(() => readBundledManifest(path.join(result.inStoreLocation, 'package'))),
result.files.then(({ filesIndex }) => readBundledManifest(ctx.getFilePathInCafs(filesIndex['package.json'].integrity))),
)
}
@@ -363,31 +370,28 @@ function fetchToStore (
) {
try {
const isLocalTarballDep = opts.pkgId.startsWith('file:')
const linkToUnpacked = path.join(target, 'package')
// We can safely assume that if there is no data about the package in `store.json` then
// it is not in the store yet.
// In case there is record about the package in `store.json`, we check it in the file system just in case
const targetExists = ctx.storeIndex[targetRelative] && await exists(path.join(linkToUnpacked, 'package.json'))
if (
!opts.force && targetExists &&
!opts.force &&
(
isLocalTarballDep === false ||
await tarballIsUpToDate(opts.resolution as any, target, opts.lockfileDir) // tslint:disable-line
)
) {
let integrity
try {
integrity = await loadJsonFile<Record<string, { size: number, integrity: string }>>(path.join(target, 'integrity.json'))
} catch (err) {
// ignoring. It is fine if the integrity file is not present. Just refetch the package
}
// if target exists and it wasn't modified, then no need to refetch it
const satisfiedIntegrity = ctx.verifyStoreIntegrity
? await checkPackage(linkToUnpacked)
: await loadJsonFile<object>(path.join(path.dirname(linkToUnpacked), 'integrity.json'))
if (satisfiedIntegrity) {
if (integrity && await ctx.checkFilesIntegrity(integrity)) {
files.resolve({
filenames: Object.keys(satisfiedIntegrity).filter((f) => !satisfiedIntegrity[f].isDir), // Filtering can be removed for store v3
filesIndex: integrity,
fromStore: true,
})
if (opts.fetchRawManifest) {
readBundledManifest(linkToUnpacked)
readBundledManifest(ctx.getFilePathInCafs(integrity['package.json'].integrity))
.then(bundledManifest.resolve)
.catch(bundledManifest.reject)
}
@@ -403,101 +407,71 @@ function fetchToStore (
// We fetch into targetStage directory first and then fs.rename() it to the
// target directory.
let filesIndex!: {}
let tempLocation!: string
await Promise.all([
(async () => {
// Tarballs are requested first because they are bigger than metadata files.
// However, when one line is left available, allow it to be picked up by a metadata request.
// This is done in order to avoid situations when tarballs are downloaded in chunks
// As much tarballs should be downloaded simultaneously as possible.
const priority = (++ctx.requestsQueue['counter'] % ctx.requestsQueue['concurrency'] === 0 ? -1 : 1) * 1000 // tslint:disable-line
// Tarballs are requested first because they are bigger than metadata files.
// However, when one line is left available, allow it to be picked up by a metadata request.
// This is done in order to avoid situations when tarballs are downloaded in chunks
// As much tarballs should be downloaded simultaneously as possible.
const priority = (++ctx.requestsQueue['counter'] % ctx.requestsQueue['concurrency'] === 0 ? -1 : 1) * 1000 // tslint:disable-line
const fetchedPackage = await ctx.requestsQueue.add(() => ctx.fetch(
opts.pkgId,
opts.resolution,
target,
{
cachedTarballLocation: path.join(ctx.storeDir, opts.pkgId, 'packed.tgz'),
lockfileDir: opts.lockfileDir,
onProgress: (downloaded) => {
fetchingProgressLogger.debug({
downloaded,
packageId: opts.pkgId,
status: 'in_progress',
})
},
onStart: (size, attempt) => {
fetchingProgressLogger.debug({
attempt,
packageId: opts.pkgId,
size,
status: 'started',
})
},
},
), { priority })
const fetchedPackage = await ctx.requestsQueue.add(() => ctx.fetch(
opts.pkgId,
opts.resolution,
{
cachedTarballLocation: path.join(ctx.storeDir, opts.pkgId, 'packed.tgz'),
lockfileDir: opts.lockfileDir,
onProgress: (downloaded) => {
fetchingProgressLogger.debug({
downloaded,
packageId: opts.pkgId,
status: 'in_progress',
})
},
onStart: (size, attempt) => {
fetchingProgressLogger.debug({
attempt,
packageId: opts.pkgId,
size,
status: 'started',
})
},
},
), { priority })
filesIndex = fetchedPackage.filesIndex
tempLocation = fetchedPackage.tempLocation
})(),
// removing only the folder with the unpacked files
// not touching tarball and integrity.json
targetExists && await rimraf(path.join(target, 'node_modules')),
])
const filesIndex = fetchedPackage.filesIndex
// Ideally, files wouldn't care about when integrity is calculated.
// However, we can only rename the temp folder once we know the package name.
// And we cannot rename the temp folder till we're calculating integrities.
if (ctx.verifyStoreIntegrity) {
const fileIntegrities = await Promise.all(
Object.keys(filesIndex)
.map((filename) =>
filesIndex[filename].generatingIntegrity
.then((fileIntegrity: object) => ({
[filename]: {
integrity: fileIntegrity,
size: filesIndex[filename].size,
},
})),
),
)
const integrity = fileIntegrities
.reduce((acc, info) => {
Object.assign(acc, info)
return acc
}, {})
await writeJsonFile(path.join(target, 'integrity.json'), integrity, { indent: undefined })
} else {
// TODO: save only filename: {size}
await writeJsonFile(path.join(target, 'integrity.json'), filesIndex, { indent: undefined })
}
const integrity = {}
await Promise.all(
Object.keys(filesIndex)
.map(async (filename) => {
const fileIntegrity = await filesIndex[filename].generatingIntegrity
integrity[filename] = {
integrity: fileIntegrity.toString(), // TODO: use the raw Integrity object
size: filesIndex[filename].size,
}
}),
)
await writeJsonFile(path.join(target, 'integrity.json'), integrity, { indent: undefined })
finishing.resolve(undefined)
let pkgName: string | undefined = opts.pkgName
if (!pkgName || opts.fetchRawManifest) {
const manifest = await readPkgFromDir(tempLocation) as DependencyManifest
const manifest = await readPackage(ctx.getFilePathInCafs(integrity['package.json'].integrity)) as DependencyManifest
bundledManifest.resolve(pickBundledManifest(manifest))
if (!pkgName) {
pkgName = manifest.name
}
}
const unpacked = path.join(target, 'node_modules', pkgName)
await makeDir(path.dirname(unpacked))
// rename(oldPath, newPath) is an atomic operation, so we do it at the
// end
await renameOverwrite(tempLocation, unpacked)
await symlinkDir(unpacked, linkToUnpacked)
if (isLocalTarballDep && opts.resolution['integrity']) { // tslint:disable-line:no-string-literal
await fs.writeFile(path.join(target, TARBALL_INTEGRITY_FILENAME), opts.resolution['integrity'], 'utf8') // tslint:disable-line:no-string-literal
}
ctx.storeIndex[targetRelative] = ctx.storeIndex[targetRelative] || []
files.resolve({
filenames: Object.keys(filesIndex).filter((f) => !filesIndex[f].isDir), // Filtering can be removed for store v3
filesIndex: integrity,
fromStore: false,
})
} catch (err) {
@@ -509,8 +483,8 @@ function fetchToStore (
}
}
async function readBundledManifest (dir: string): Promise<BundledManifest> {
return pickBundledManifest(await readPkgFromDir(dir) as DependencyManifest)
async function readBundledManifest (pkgJsonPath: string): Promise<BundledManifest> {
return pickBundledManifest(await readPackage(pkgJsonPath) as DependencyManifest)
}
async function tarballIsUpToDate (
@@ -564,9 +538,9 @@ function differed<T> (): PromiseContainer<T> {
async function fetcher (
fetcherByHostingType: {[hostingType: string]: FetchFunction},
cafs: Cafs,
packageId: string,
resolution: Resolution,
target: string,
opts: FetchOptions,
): Promise<FetchResult> {
const fetch = fetcherByHostingType[resolution.type || 'tarball']
@@ -574,7 +548,7 @@ async function fetcher (
throw new Error(`Fetching for dependency type "${resolution.type}" is not supported`)
}
try {
return await fetch(resolution, target, opts)
return await fetch(cafs, resolution, opts)
} catch (err) {
packageRequestLogger.warn({
message: `Fetching ${packageId} failed!`,

187
packages/package-requester/test/index.ts
View File

@@ -1,4 +1,5 @@
///<reference path="../../../typings/index.d.ts" />
import { getFilePathInCafs } from '@pnpm/cafs'
import localResolver from '@pnpm/local-resolver'
import { streamParser } from '@pnpm/logger'
import createResolver from '@pnpm/npm-resolver'
@@ -6,9 +7,7 @@ import createPackageRequester, { PackageFilesResponse, PackageResponse } from '@
import { ResolveFunction } from '@pnpm/resolver-base'
import createFetcher from '@pnpm/tarball-fetcher'
import { DependencyManifest } from '@pnpm/types'
import rimraf = require('@zkochan/rimraf')
import delay from 'delay'
import loadJsonFile = require('load-json-file')
import fs = require('mz/fs')
import ncpCB = require('ncp')
import nock = require('nock')
@@ -38,10 +37,11 @@ const fetch = createFetcher({
})
test('request package', async t => {
const storeDir = tempy.directory()
const storeIndex = {}
const requestPackage = createPackageRequester(resolve, fetch, {
networkConcurrency: 1,
storeDir: '.store',
storeDir,
storeIndex,
verifyStoreIntegrity: true,
})
@@ -54,18 +54,14 @@ test('request package', async t => {
preferredVersions: {},
projectDir,
registry,
}) as PackageResponse & {
body: {inStoreLocation: string, latest: string, manifest: {name: string}},
files: () => Promise<{filenames: string[], fromStore: boolean}>,
finishing: () => Promise<void>,
}
}) as PackageResponse
t.ok(pkgResponse, 'response received')
t.ok(pkgResponse.body, 'response has body')
t.equal(pkgResponse.body.id, 'registry.npmjs.org/is-positive/1.0.0', 'responded with correct package ID')
t.equal(pkgResponse.body.resolvedVia, 'npm-registry', 'responded with correct resolvedVia')
t.equal(pkgResponse.body.inStoreLocation, path.join('.store', 'registry.npmjs.org', 'is-positive', '1.0.0'), 'package location in store returned')
t.equal(pkgResponse.body.inStoreLocation, path.join(storeDir, 'registry.npmjs.org', 'is-positive', '1.0.0'), 'package location in store returned')
t.equal(pkgResponse.body.isLocal, false, 'package is not local')
t.equal(typeof pkgResponse.body.latest, 'string', 'latest is returned')
t.equal(pkgResponse.body.manifest.name, 'is-positive', 'package manifest returned')
@@ -77,10 +73,9 @@ test('request package', async t => {
}, 'resolution returned')
const files = await pkgResponse.files!()
t.deepEqual(files, {
filenames: [ 'package.json', 'index.js', 'license', 'readme.md' ],
fromStore: false,
}, 'returned info about files after fetch completed')
t.deepEqual(Object.keys(files.filesIndex).sort(),
['package.json', 'index.js', 'license', 'readme.md'].sort())
t.notOk(files.fromStore)
t.ok(pkgResponse.finishing!())
@@ -106,11 +101,7 @@ test('request package but skip fetching', async t => {
projectDir,
registry,
skipFetch: true,
}) as PackageResponse & {
body: {inStoreLocation: string, latest: string, manifest: {name: string}},
files: () => Promise<object>,
finishing: () => Promise<void>,
}
}) as PackageResponse
t.ok(pkgResponse, 'response received')
t.ok(pkgResponse.body, 'response has body')
@@ -195,7 +186,7 @@ test('refetch local tarball if its integrity has changed', async t => {
await ncp(path.join(__dirname, 'pnpm-package-requester-0.8.1.tgz'), tarballPath)
const tarball = `file:${tarballRelativePath}`
const wantedPackage = { pref: tarball }
const storeDir = path.join(__dirname, '..', '.store')
const storeDir = tempy.directory()
const pkgId = `file:${normalize(tarballRelativePath)}`
const requestPackageOpts = {
currentPackageId: pkgId,
@@ -369,13 +360,13 @@ test('refetch local tarball if its integrity has changed. The requester does not
test('fetchPackageToStore()', async (t) => {
const packageRequester = createPackageRequester(resolve, fetch, {
networkConcurrency: 1,
storeDir: '.store',
storeDir: tempy.directory(),
storeIndex: {},
verifyStoreIntegrity: true,
})
const pkgId = 'registry.npmjs.org/is-positive/1.0.0'
const fetchResult = await packageRequester.fetchPackageToStore({
const fetchResult = packageRequester.fetchPackageToStore({
force: false,
lockfileDir: tempy.directory(),
pkgId,
@@ -389,14 +380,14 @@ test('fetchPackageToStore()', async (t) => {
t.notOk(fetchResult.bundledManifest, 'full manifest not returned')
const files = await fetchResult.files()
t.deepEqual(files, {
filenames: [ 'package.json', 'index.js', 'license', 'readme.md' ],
fromStore: false,
}, 'returned info about files after fetch completed')
t.deepEqual(Object.keys(files.filesIndex).sort(),
['package.json', 'index.js', 'license', 'readme.md'].sort(),
'returned info about files after fetch completed')
t.notOk(files.fromStore)
t.ok(fetchResult.finishing())
const fetchResult2 = await packageRequester.fetchPackageToStore({
const fetchResult2 = packageRequester.fetchPackageToStore({
fetchRawManifest: true,
force: false,
lockfileDir: tempy.directory(),
@@ -425,9 +416,11 @@ test('fetchPackageToStore()', async (t) => {
})
test('fetchPackageToStore() concurrency check', async (t) => {
const storeDir = tempy.directory()
const cafsDir = path.join(storeDir, 'files')
const packageRequester = createPackageRequester(resolve, fetch, {
networkConcurrency: 1,
storeDir: '.store',
storeDir,
storeIndex: {},
verifyStoreIntegrity: true,
})
@@ -462,29 +455,31 @@ test('fetchPackageToStore() concurrency check', async (t) => {
let ino2!: Number
{
const fetchResult = await fetchResults[0]
const fetchResult = fetchResults[0]
const files = await fetchResult.files()
ino1 = fs.statSync(path.join(fetchResult.inStoreLocation, 'package', 'package.json')).ino
ino1 = fs.statSync(getFilePathInCafs(cafsDir, files.filesIndex['package.json'].integrity)).ino
t.deepEqual(files, {
filenames: [ 'package.json', 'index.js', 'license', 'readme.md' ],
fromStore: false,
}, 'returned info about files after fetch completed')
t.deepEqual(Object.keys(files.filesIndex).sort(),
['package.json', 'index.js', 'license', 'readme.md'].sort(),
'returned info about files after fetch completed',
)
t.notOk(files.fromStore)
t.ok(fetchResult.finishing)
}
{
const fetchResult = await fetchResults[1]
const fetchResult = fetchResults[1]
const files = await fetchResult.files()
ino2 = fs.statSync(path.join(fetchResult.inStoreLocation, 'package', 'package.json')).ino
ino2 = fs.statSync(getFilePathInCafs(cafsDir, files.filesIndex['package.json'].integrity)).ino
t.deepEqual(files, {
filenames: [ 'package.json', 'index.js', 'license', 'readme.md' ],
fromStore: false,
}, 'returned info about files after fetch completed')
t.deepEqual(Object.keys(files.filesIndex).sort(),
['package.json', 'index.js', 'license', 'readme.md'].sort(),
'returned info about files after fetch completed',
)
t.notOk(files.fromStore)
t.ok(fetchResult.finishing())
}
@@ -521,7 +516,7 @@ test('fetchPackageToStore() does not cache errors', async (t) => {
const pkgId = 'registry.npmjs.org/is-positive/1.0.0'
try {
const badRequest = await packageRequester.fetchPackageToStore({
const badRequest = packageRequester.fetchPackageToStore({
force: false,
lockfileDir: tempy.directory(),
pkgId,
@@ -537,7 +532,7 @@ test('fetchPackageToStore() does not cache errors', async (t) => {
t.pass('first fetch failed')
}
const fetchResult = await packageRequester.fetchPackageToStore({
const fetchResult = packageRequester.fetchPackageToStore({
force: false,
lockfileDir: tempy.directory(),
pkgId,
@@ -548,10 +543,11 @@ test('fetchPackageToStore() does not cache errors', async (t) => {
},
})
const files = await fetchResult.files()
t.deepEqual(files, {
filenames: [ 'package.json', 'index.js', 'license', 'readme.md' ],
fromStore: false,
}, 'returned info about files after fetch completed')
t.deepEqual(Object.keys(files.filesIndex).sort(),
[ 'package.json', 'index.js', 'license', 'readme.md' ].sort(),
'returned info about files after fetch completed',
)
t.notOk(files.fromStore)
t.ok(fetchResult.finishing())
t.ok(nock.isDone())
@@ -561,6 +557,7 @@ test('fetchPackageToStore() does not cache errors', async (t) => {
// This test was added to cover the issue described here: https://github.com/pnpm/supi/issues/65
test('always return a package manifest in the response', async t => {
nock.cleanAll()
const requestPackage = createPackageRequester(resolve, fetch, {
networkConcurrency: 1,
storeDir: '.store',
@@ -656,7 +653,9 @@ test('fetchPackageToStore() fetch raw manifest of cached package', async (t) =>
test('refetch package to store if it has been modified', async (t) => {
nock.cleanAll()
const storeDir = tempy.directory()
const cafsDir = path.join(storeDir, 'files')
const storeIndex = {}
const lockfileDir = tempy.directory()
t.comment(`store location: ${storeDir}`)
const pkgId = 'registry.npmjs.org/magic-hook/2.0.0'
@@ -665,6 +664,7 @@ test('refetch package to store if it has been modified', async (t) => {
tarball: 'https://registry.npmjs.org/magic-hook/-/magic-hook-2.0.0.tgz',
}
let indexJsFile!: string
{
const packageRequester = createPackageRequester(resolve, fetch, {
networkConcurrency: 1,
@@ -673,28 +673,23 @@ test('refetch package to store if it has been modified', async (t) => {
verifyStoreIntegrity: true,
})
const fetchResult = await packageRequester.fetchPackageToStore({
const fetchResult = packageRequester.fetchPackageToStore({
fetchRawManifest: false,
force: false,
lockfileDir: tempy.directory(),
lockfileDir,
pkgId,
resolution,
})
await fetchResult.files()
const { filesIndex } = await fetchResult.files()
indexJsFile = getFilePathInCafs(cafsDir, filesIndex['index.js'].integrity)
}
const distPathInStore = await path.join(storeDir, pkgId, 'node_modules', 'magic-hook', 'dist')
t.ok(await fs.exists(distPathInStore), `${distPathInStore} exists`)
await rimraf(distPathInStore)
t.notOk(await fs.exists(distPathInStore), `${distPathInStore} not exists`)
// Adding some content to the file to change its integrity
await fs.appendFile(indexJsFile, '// foobar')
const reporter = sinon.spy()
streamParser.on('data', reporter)
const projectDir = tempy.directory()
{
const packageRequester = createPackageRequester(resolve, fetch, {
@@ -704,10 +699,10 @@ test('refetch package to store if it has been modified', async (t) => {
verifyStoreIntegrity: true,
})
const fetchResult = await packageRequester.fetchPackageToStore({
const fetchResult = packageRequester.fetchPackageToStore({
fetchRawManifest: false,
force: false,
lockfileDir: projectDir,
lockfileDir,
pkgId,
resolution,
})
@@ -717,85 +712,13 @@ test('refetch package to store if it has been modified', async (t) => {
streamParser.removeListener('data', reporter)
t.ok(await fs.exists(distPathInStore), `${distPathInStore} exists`)
t.notOk((await fs.readFile(indexJsFile, 'utf8')).includes('// foobar'))
t.ok(reporter.calledWithMatch({
level: 'warn',
message: `Refetching ${path.join(storeDir, pkgId)} to store. It was either modified or had no integrity checksums`,
name: 'pnpm:package-requester',
prefix: projectDir,
}), 'refetch logged')
t.end()
})
test('refetch package to store if it has no integrity checksums and verification is needed', async (t) => {
nock.cleanAll()
const storeDir = tempy.directory()
const storeIndex = {}
t.comment(`store location: ${storeDir}`)
const pkgId = 'registry.npmjs.org/magic-hook/2.0.0'
const resolution = {
registry: 'https://registry.npmjs.org/',
tarball: 'https://registry.npmjs.org/magic-hook/-/magic-hook-2.0.0.tgz',
}
{
const packageRequester = createPackageRequester(resolve, fetch, {
networkConcurrency: 1,
storeDir,
storeIndex,
verifyStoreIntegrity: false,
})
const fetchResult = await packageRequester.fetchPackageToStore({
fetchRawManifest: false,
force: false,
lockfileDir: tempy.directory(),
pkgId,
resolution,
})
await fetchResult.files()
const integrityJson = await loadJsonFile<object>(path.join(storeDir, pkgId, 'integrity.json'))
t.notOk(integrityJson['package.json'].integrity, 'no integrity hash generated')
}
const reporter = sinon.spy()
streamParser.on('data', reporter)
const projectDir = tempy.directory()
{
const packageRequester = createPackageRequester(resolve, fetch, {
networkConcurrency: 1,
storeDir,
storeIndex,
verifyStoreIntegrity: true,
})
const fetchResult = await packageRequester.fetchPackageToStore({
fetchRawManifest: false,
force: false,
lockfileDir: projectDir,
pkgId,
resolution,
})
await fetchResult.files()
const integrityJson = await loadJsonFile<object>(path.join(storeDir, pkgId, 'integrity.json'))
t.ok(integrityJson['package.json'].integrity, 'integrity hash generated')
}
streamParser.removeListener('data', reporter)
t.ok(reporter.calledWithMatch({
level: 'warn',
message: `Refetching ${path.join(storeDir, pkgId)} to store. It was either modified or had no integrity checksums`,
name: 'pnpm:package-requester',
prefix: projectDir,
prefix: lockfileDir,
}), 'refetch logged')
t.end()

3
packages/package-requester/tsconfig.json
View File

@@ -9,6 +9,9 @@
"../../typings/**/*.d.ts"
],
"references": [
{
"path": "../cafs"
},
{
"path": "../core-loggers"
},

3
packages/package-store/package.json
View File

@@ -16,6 +16,7 @@
"@pnpm/logger": "^3.1.0"
},
"dependencies": {
"@pnpm/cafs": "workspace:0.0.0",
"@pnpm/core-loggers": "workspace:4.0.1",
"@pnpm/fetcher-base": "workspace:6.0.0",
"@pnpm/fs-locker": "3.0.1",
@@ -28,7 +29,6 @@
"load-json-file": "6.2.0",
"make-empty-dir": "^1.0.0",
"mz": "2.7.0",
"ncp": "2.0.0",
"p-filter": "2.1.0",
"p-limit": "2.3.0",
"path-exists": "4.0.0",
@@ -43,7 +43,6 @@
"@pnpm/package-store": "link:",
"@pnpm/tarball-fetcher": "workspace:*",
"@types/mz": "^2.7.0",
"@types/ncp": "2.0.3",
"@types/proxyquire": "1.3.28",
"@types/ramda": "^0.27.4",
"@types/sinon": "^9.0.0",

74
packages/package-store/src/fs/importIndexedDir.ts
View File

@@ -6,25 +6,43 @@ import path = require('path')
import pathTemp = require('path-temp')
import renameOverwrite = require('rename-overwrite')
const importingLogger = pnpmLogger('_package-file-already-exists')
const filenameConflictsLogger = pnpmLogger('_filename-conflicts')
type ImportFile = (src: string, dest: string) => Promise<void>
export default async function importIndexedDir (importFile: ImportFile, existingDir: string, newDir: string, filenames: string[]) {
export default async function importIndexedDir (
importFile: ImportFile,
newDir: string,
filenames: Record<string, string>,
) {
const stage = pathTemp(path.dirname(newDir))
try {
await tryImportIndexedDir(importFile, existingDir, stage, filenames)
await tryImportIndexedDir(importFile, stage, filenames)
await renameOverwrite(stage, newDir)
} catch (err) {
try { await rimraf(stage) } catch (err) {} // tslint:disable-line:no-empty
throw err
if (err['code'] !== 'EEXIST') throw err
const { uniqueFileMap, conflictingFileNames } = getUniqueFileMap(filenames)
if (Object.keys(conflictingFileNames).length === 0) throw err
filenameConflictsLogger.debug({
conflicts: conflictingFileNames,
writingTo: newDir,
})
globalWarn(
`Not all files were linked to "${path.relative(process.cwd(), newDir)}". ` +
'Some of the files have equal names in different case, ' +
'which is an issue on case-insensitive filesystems. ' +
`The conflicting file names are: ${JSON.stringify(conflictingFileNames)}`,
)
await importIndexedDir(importFile, newDir, uniqueFileMap)
}
}
async function tryImportIndexedDir (importFile: ImportFile, existingDir: string, newDir: string, filenames: string[]) {
async function tryImportIndexedDir (importFile: ImportFile, newDir: string, filenames: Record<string, string>) {
await makeEmptyDir(newDir, { recursive: true })
const alldirs = new Set<string>()
filenames
Object.keys(filenames)
.forEach((f) => {
const dir = path.join(newDir, path.dirname(f))
if (dir === '.') return
@@ -35,32 +53,30 @@ async function tryImportIndexedDir (importFile: ImportFile, existingDir: string,
.sort((d1, d2) => d1.length - d2.length)
.map((dir) => fs.mkdir(dir, { recursive: true })),
)
let allLinked = true
await Promise.all(
filenames
.map(async (f: string) => {
const src = path.join(existingDir, f)
Object.entries(filenames)
.map(async ([f, src]: [string, string]) => {
const dest = path.join(newDir, f)
try {
await importFile(src, dest)
} catch (err) {
if (err['code'] !== 'EEXIST') throw err
// If the file is already linked, we ignore the error.
// This is an extreme edge case that may happen only in one case,
// when the store folder is case sensitive and the project's node_modules
// is case insensitive.
// So, for instance, foo.js and Foo.js could be unpacked to the store
// but they cannot be both linked to node_modules.
// More details at https://github.com/pnpm/pnpm/issues/1685
allLinked = false
importingLogger.debug({ src, dest })
}
await importFile(src, dest)
}),
)
if (!allLinked) {
globalWarn(
`Not all files from "${existingDir}" were linked to "${newDir}". ` +
'This happens when the store is case sensitive while the target directory is case insensitive.',
)
}
function getUniqueFileMap (fileMap: Record<string, string>) {
const lowercaseFiles = new Map<string, string>()
const conflictingFileNames = {}
const uniqueFileMap = {}
for (const filename of Object.keys(fileMap).sort()) {
const lowercaseFilename = filename.toLowerCase()
if (lowercaseFiles.has(lowercaseFilename)) {
conflictingFileNames[filename] = lowercaseFiles.get(lowercaseFilename)
continue
}
lowercaseFiles.set(lowercaseFilename, filename)
uniqueFileMap[filename] = fileMap[filename]
}
return {
conflictingFileNames,
uniqueFileMap,
}
}

59
packages/package-store/src/storeController/createImportPackage.ts
View File

@@ -1,25 +1,19 @@
import { importingLogger } from '@pnpm/core-loggers'
import { globalInfo, globalWarn } from '@pnpm/logger'
import {
ImportPackageFunction,
PackageFilesResponse,
} from '@pnpm/store-controller-types'
import { ImportPackageFunction } from '@pnpm/store-controller-types'
import fs = require('mz/fs')
import ncpCB = require('ncp')
import pLimit from 'p-limit'
import path = require('path')
import exists = require('path-exists')
import pathTemp = require('path-temp')
import renameOverwrite = require('rename-overwrite')
import { promisify } from 'util'
import importIndexedDir from '../fs/importIndexedDir'
const ncp = promisify(ncpCB)
const limitLinking = pLimit(16)
export default (packageImportMethod?: 'auto' | 'hardlink' | 'copy' | 'clone'): ImportPackageFunction => {
export default (packageImportMethod?: 'auto' | 'hardlink' | 'copy' | 'clone'): (to: string, opts: { filesMap: Record<string, string>, fromStore: boolean, force: boolean }) => ReturnType<ImportPackageFunction> => {
const importPackage = createImportPackage(packageImportMethod)
return (from, to, opts) => limitLinking(() => importPackage(from, to, opts))
return (to, opts) => limitLinking(() => importPackage(to, opts))
}
function createImportPackage (packageImportMethod?: 'auto' | 'hardlink' | 'copy' | 'clone') {
@@ -49,22 +43,22 @@ function createAutoImporter () {
return auto
async function initialAuto (
from: string,
to: string,
opts: {
filesResponse: PackageFilesResponse,
filesMap: Record<string, string>,
force: boolean,
fromStore: boolean,
},
) {
try {
await clonePkg(from, to, opts)
await clonePkg(to, opts)
auto = clonePkg
return
} catch (err) {
// ignore
}
try {
await hardlinkPkg(from, to, opts)
await hardlinkPkg(to, opts)
auto = hardlinkPkg
return
} catch (err) {
@@ -72,24 +66,24 @@ function createAutoImporter () {
globalWarn(err.message)
globalInfo('Falling back to copying packages from store')
auto = copyPkg
await auto(from, to, opts)
await auto(to, opts)
}
}
}
async function clonePkg (
from: string,
to: string,
opts: {
filesResponse: PackageFilesResponse,
filesMap: Record<string, string>,
fromStore: boolean,
force: boolean,
},
) {
const pkgJsonPath = path.join(to, 'package.json')
if (!opts.filesResponse.fromStore || opts.force || !await exists(pkgJsonPath)) {
importingLogger.debug({ from, to, method: 'clone' })
await importIndexedDir(cloneFile, from, to, opts.filesResponse.filenames)
if (!opts.fromStore || opts.force || !await exists(pkgJsonPath)) {
importingLogger.debug({ to, method: 'clone' })
await importIndexedDir(cloneFile, to, opts.filesMap)
}
}
@@ -98,27 +92,26 @@ async function cloneFile (from: string, to: string) {
}
async function hardlinkPkg (
from: string,
to: string,
opts: {
filesResponse: PackageFilesResponse,
filesMap: Record<string, string>,
force: boolean,
fromStore: boolean,
},
) {
const pkgJsonPath = path.join(to, 'package.json')
if (!opts.filesResponse.fromStore || opts.force || !await exists(pkgJsonPath) || !await pkgLinkedToStore(pkgJsonPath, from, to)) {
importingLogger.debug({ from, to, method: 'hardlink' })
await importIndexedDir(fs.link, from, to, opts.filesResponse.filenames)
if (!opts.fromStore || opts.force || !await exists(pkgJsonPath) || !await pkgLinkedToStore(pkgJsonPath, opts.filesMap['package.json'], to)) {
importingLogger.debug({ to, method: 'hardlink' })
await importIndexedDir(fs.link, to, opts.filesMap)
}
}
async function pkgLinkedToStore (
pkgJsonPath: string,
from: string,
pkgJsonPathInStore: string,
to: string,
) {
const pkgJsonPathInStore = path.join(from, 'package.json')
if (await isSameFile(pkgJsonPath, pkgJsonPathInStore)) return true
globalInfo(`Relinking ${to} from the store`)
return false
@@ -130,19 +123,17 @@ async function isSameFile (file1: string, file2: string) {
}
export async function copyPkg (
from: string,
to: string,
opts: {
filesResponse: PackageFilesResponse,
filesMap: Record<string, string>,
fromStore: boolean,
force: boolean,
},
) {
const pkgJsonPath = path.join(to, 'package.json')
if (!opts.filesResponse.fromStore || opts.force || !await exists(pkgJsonPath)) {
importingLogger.debug({ from, to, method: 'copy' })
const staging = pathTemp(path.dirname(to))
await fs.mkdir(staging, { recursive: true })
await ncp(from + '/.', staging)
await renameOverwrite(staging, to)
if (!opts.fromStore || opts.force || !await exists(pkgJsonPath)) {
importingLogger.debug({ to, method: 'copy' })
await importIndexedDir(fs.copyFile, to, opts.filesMap)
}
}

39
packages/package-store/src/storeController/index.ts
View File

@@ -1,3 +1,4 @@
import { getFilePathInCafs as _getFilePathInCafs } from '@pnpm/cafs'
import { FetchFunction } from '@pnpm/fetcher-base'
import lock from '@pnpm/fs-locker'
import { globalInfo, globalWarn } from '@pnpm/logger'
@@ -5,6 +6,7 @@ import createPackageRequester, { getCacheByEngine } from '@pnpm/package-requeste
import pkgIdToFilename from '@pnpm/pkgid-to-filename'
import { ResolveFunction } from '@pnpm/resolver-base'
import {
ImportPackageFunction,
PackageUsagesBySearchQueries,
StoreController,
} from '@pnpm/store-controller-types'
@@ -15,6 +17,8 @@ import pLimit from 'p-limit'
import path = require('path')
import exists = require('path-exists')
import R = require('ramda')
import { promisify } from 'util'
import writeJsonFile = require('write-json-file')
import {
read as readStore,
save as saveStore,
@@ -26,6 +30,7 @@ export default async function (
resolve: ResolveFunction,
fetchers: {[type: string]: FetchFunction},
initOpts: {
ignoreFile?: (filename: string) => boolean,
locks?: string,
lockStaleDuration?: number,
storeDir: string,
@@ -45,19 +50,31 @@ export default async function (
const storeIndex = await readStore(initOpts.storeDir) || {}
const packageRequester = createPackageRequester(resolve, fetchers, {
ignoreFile: initOpts.ignoreFile,
networkConcurrency: initOpts.networkConcurrency,
storeDir: initOpts.storeDir,
storeIndex,
verifyStoreIntegrity: initOpts.verifyStoreIntegrity,
})
const impPkg = createImportPackage(initOpts.packageImportMethod)
const cafsDir = path.join(storeDir, 'files')
const getFilePathInCafs = _getFilePathInCafs.bind(null, cafsDir)
const importPackage: ImportPackageFunction = (to, opts) => {
const filesMap = {} as Record<string, string>
for (const [fileName, { integrity }] of Object.entries(opts.filesResponse.filesIndex)) {
filesMap[fileName] = getFilePathInCafs(integrity)
}
return impPkg(to, { filesMap, fromStore: opts.filesResponse.fromStore, force: opts.force })
}
return {
close: unlock ? async () => { await unlock() } : () => Promise.resolve(undefined),
closeSync: unlock ? () => unlock.sync() : () => undefined,
fetchPackage: packageRequester.fetchPackageToStore,
findPackageUsages,
getPackageLocation,
importPackage: createImportPackage(initOpts.packageImportMethod),
importPackage,
prune,
requestPackage: packageRequester.requestPackage,
saveState: saveStore.bind(null, initOpts.storeDir, storeIndex),
@@ -150,10 +167,22 @@ export default async function (
}
async function upload (builtPkgLocation: string, opts: {packageId: string, engine: string}) {
const cachePath = path.join(storeDir, opts.packageId, 'side_effects', opts.engine, 'package')
// TODO calculate integrity.json here
const filenames: string[] = []
await copyPkg(builtPkgLocation, cachePath, { filesResponse: { fromStore: true, filenames }, force: true })
const filesIndex = await packageRequester.cafs.addFilesFromDir(builtPkgLocation)
// TODO: move this to a function
// This is duplicated in @pnpm/package-requester
const integrity = {}
await Promise.all(
Object.keys(filesIndex)
.map(async (filename) => {
const fileIntegrity = await filesIndex[filename].generatingIntegrity
integrity[filename] = {
integrity: fileIntegrity.toString(), // TODO: use the raw Integrity object
size: filesIndex[filename].size,
}
}),
)
const cachePath = path.join(storeDir, opts.packageId, 'side_effects', opts.engine)
await writeJsonFile(path.join(cachePath, 'integrity.json'), integrity, { indent: undefined })
}
}

26
packages/package-store/test/createImportPackage.spec.ts
View File

@@ -21,15 +21,16 @@ test('packageImportMethod=auto: clone files by default', async (t) => {
const importPackage = createImportPackage('auto')
fsMock.copyFile = sinon.spy()
fsMock.rename = sinon.spy()
await importPackage('store/package', 'project/package', {
filesResponse: {
filenames: ['package.json', 'index.js'],
fromStore: false,
await importPackage('project/package', {
filesMap: {
'index.js': 'hash2',
'package.json': 'hash1',
},
force: false,
fromStore: false,
})
t.ok(fsMock.copyFile.calledWith(path.join('store', 'package', 'package.json'), path.join('project', '_tmp', 'package.json'), fs.constants.COPYFILE_FICLONE_FORCE))
t.ok(fsMock.copyFile.calledWith(path.join('store', 'package', 'index.js'), path.join('project', '_tmp', 'index.js'), fs.constants.COPYFILE_FICLONE_FORCE))
t.ok(fsMock.copyFile.calledWith(path.join('hash1'), path.join('project', '_tmp', 'package.json'), fs.constants.COPYFILE_FICLONE_FORCE))
t.ok(fsMock.copyFile.calledWith(path.join('hash2'), path.join('project', '_tmp', 'index.js'), fs.constants.COPYFILE_FICLONE_FORCE))
t.end()
})
@@ -38,14 +39,15 @@ test('packageImportMethod=auto: link files if cloning fails', async (t) => {
fsMock.copyFile = () => { throw new Error('This file system does not support cloning') }
fsMock.link = sinon.spy()
fsMock.rename = sinon.spy()
await importPackage('store/package', 'project/package', {
filesResponse: {
filenames: ['package.json', 'index.js'],
fromStore: false,
await importPackage('project/package', {
filesMap: {
'index.js': 'hash2',
'package.json': 'hash1',
},
force: false,
fromStore: false,
})
t.ok(fsMock.link.calledWith(path.join('store', 'package', 'package.json'), path.join('project', '_tmp', 'package.json')))
t.ok(fsMock.link.calledWith(path.join('store', 'package', 'index.js'), path.join('project', '_tmp', 'index.js')))
t.ok(fsMock.link.calledWith(path.join('hash1'), path.join('project', '_tmp', 'package.json')))
t.ok(fsMock.link.calledWith(path.join('hash2'), path.join('project', '_tmp', 'index.js')))
t.end()
})

10
packages/package-store/test/index.ts
View File

@@ -31,7 +31,7 @@ test('store.importPackage()', async (t) => {
verifyStoreIntegrity: true,
})
const pkgId = 'registry.npmjs.org/is-positive/1.0.0'
const fetchResponse = await storeController.fetchPackage({
const fetchResponse = storeController.fetchPackage({
force: false,
lockfileDir: tempy.directory(),
pkgId,
@@ -42,8 +42,7 @@ test('store.importPackage()', async (t) => {
},
})
const importTo = tempy.directory()
const importFrom = path.join(fetchResponse.inStoreLocation, 'node_modules', 'is-positive')
await storeController.importPackage(importFrom, importTo, {
await storeController.importPackage(importTo, {
filesResponse: await fetchResponse.files(),
force: false,
})
@@ -70,7 +69,7 @@ test('store.importPackage() by copying', async (t) => {
verifyStoreIntegrity: true,
})
const pkgId = 'registry.npmjs.org/is-positive/1.0.0'
const fetchResponse = await storeController.fetchPackage({
const fetchResponse = storeController.fetchPackage({
force: false,
lockfileDir: tempy.directory(),
pkgId,
@@ -81,8 +80,7 @@ test('store.importPackage() by copying', async (t) => {
},
})
const importTo = tempy.directory()
const importFrom = path.join(fetchResponse.inStoreLocation, 'node_modules', 'is-positive')
await storeController.importPackage(importFrom, importTo, {
await storeController.importPackage(importTo, {
filesResponse: await fetchResponse.files(),
force: false,
})

3
packages/package-store/tsconfig.json
View File

@@ -9,6 +9,9 @@
"../../typings/**/*.d.ts"
],
"references": [
{
"path": "../cafs"
},
{
"path": "../core-loggers"
},

2
packages/plugin-commands-store/package.json
View File

@@ -55,6 +55,7 @@
"@pnpm/config": "workspace:8.3.0",
"@pnpm/error": "workspace:1.2.0",
"@pnpm/get-context": "workspace:1.2.1",
"@pnpm/lockfile-utils": "workspace:2.0.11",
"@pnpm/normalize-registries": "workspace:1.0.0",
"@pnpm/parse-wanted-dependency": "workspace:1.0.0",
"@pnpm/pick-registry-for-package": "workspace:1.0.0",
@@ -65,6 +66,7 @@
"archy": "1.0.0",
"common-tags": "1.8.0",
"dependency-path": "workspace:4.0.6",
"dint": "4.0.0",
"p-filter": "2.1.0",
"ramda": "0.27.0",
"render-help": "1.0.0"

25
packages/plugin-commands-store/src/storeStatus/index.ts
View File

@@ -1,7 +1,9 @@
import checkPackage from '@pnpm/check-package'
import { getContextForSingleImporter } from '@pnpm/get-context'
import { nameVerFromPkgSnapshot } from '@pnpm/lockfile-utils'
import { streamParser } from '@pnpm/logger'
import * as dp from 'dependency-path'
import dint = require('dint')
import loadJsonFile = require('load-json-file')
import pFilter = require('p-filter')
import path = require('path')
import extendOptions, {
@@ -18,6 +20,7 @@ export default async function (maybeOpts: StoreStatusOptions) {
registries,
storeDir,
skipped,
virtualStoreDir,
wantedLockfile,
} = await getContextForSingleImporter({}, {
...opts,
@@ -25,19 +28,23 @@ export default async function (maybeOpts: StoreStatusOptions) {
})
if (!wantedLockfile) return []
const pkgPaths = (Object.keys(wantedLockfile.packages || {})
.map((id) => {
if (id === '/') return null
return dp.resolve(registries, id)
const pkgs = Object.keys(wantedLockfile.packages || {})
.filter((relDepPath) => !skipped.has(relDepPath))
.map((relDepPath) => {
return {
pkgPath: dp.resolve(registries, relDepPath),
...nameVerFromPkgSnapshot(relDepPath, wantedLockfile.packages![relDepPath]),
}
})
.filter((pkgId) => pkgId && !skipped.has(pkgId)) as string[])
.map((pkgPath: string) => path.join(storeDir, pkgPath))
const modified = await pFilter(pkgPaths, async (pkgPath: string) => !await checkPackage(path.join(pkgPath, 'package')))
const modified = await pFilter(pkgs, async ({ pkgPath, name }) => {
const integrity = await loadJsonFile(path.join(storeDir, pkgPath, 'integrity.json'))
return (await dint.check(path.join(virtualStoreDir, pkgPath, 'node_modules', name), integrity)) === false
})
if (reporter) {
streamParser.removeListener('data', reporter)
}
return modified
return modified.map(({ pkgPath }) => pkgPath)
}

3
packages/plugin-commands-store/test/storeStatus.ts
View File

@@ -16,8 +16,7 @@ test('CLI fails when store status finds modified packages', async function (t) {
await execa('pnpm', ['add', 'is-positive@3.1.0', '--store-dir', storeDir, '--registry', REGISTRY, '--verify-store-integrity'])
const isPositive = await project.resolve('is-positive', '3.1.0', 'index.js')
await rimraf(isPositive)
await rimraf(`node_modules/.pnpm/localhost+${REGISTRY_MOCK_PORT}/is-positive/3.1.0/node_modules/is-positive/index.js`)
let err!: PnpmError
try {

3
packages/plugin-commands-store/tsconfig.json
View File

@@ -21,6 +21,9 @@
{
"path": "../get-context"
},
{
"path": "../lockfile-utils"
},
{
"path": "../normalize-registries"
},

6
packages/pnpm/test/install/misc.ts
View File

@@ -164,10 +164,12 @@ test("don't fail on case insensitive filesystems when package has 2 files with s
const integrityFile = await import(path.join(storeDir, `localhost+${REGISTRY_MOCK_PORT}`, 'with-same-file-in-different-cases', '1.0.0', 'integrity.json'))
const packageFiles = Object.keys(integrityFile).sort()
t.deepEqual(packageFiles, ['Foo.js', 'foo.js', 'package.json'])
const files = await fs.readdir('node_modules/with-same-file-in-different-cases')
if (await dirIsCaseSensitive(storeDir)) {
t.deepEqual(packageFiles, ['Foo.js', 'foo.js', 'package.json'])
t.deepEqual(files, ['Foo.js', 'foo.js', 'package.json'])
} else {
t.deepEqual(packageFiles, ['foo.js', 'package.json'])
t.deepEqual(files, ['Foo.js', 'package.json'])
}
})

6
packages/pnpm/test/install/sideEffects.ts
View File

@@ -11,7 +11,7 @@ const test = promisifyTape(tape)
const ENGINE_DIR = `${process.platform}-${process.arch}-node-${process.version.split('.')[0]}`
test('caching side effects of native package', async function (t) {
test.skip('caching side effects of native package', async function (t) {
const project = prepare(t)
await execPnpm(['add', '--side-effects-cache', 'diskusage@1.1.3'])
@@ -33,7 +33,7 @@ test('caching side effects of native package', async function (t) {
t.end()
})
test('using side effects cache', async function (t) {
test.skip('using side effects cache', async function (t) {
const project = prepare(t)
// Right now, hardlink does not work with side effects, so we specify copy as the packageImportMethod
@@ -52,7 +52,7 @@ test('using side effects cache', async function (t) {
t.end()
})
test('readonly side effects cache', async function (t) {
test.skip('readonly side effects cache', async function (t) {
const project = prepare(t)
await execPnpm(['add', 'diskusage@1.1.2', '--side-effects-cache', '--no-verify-store-integrity'])

1
packages/server/package.json
View File

@@ -44,6 +44,7 @@
"@types/uuid": "^7.0.3",
"@zkochan/rimraf": "1.0.0",
"is-port-reachable": "3.0.0",
"load-json-file": "6.2.0",
"mz": "2.7.0",
"node-fetch": "2.6.0",
"tempy": "0.5.0"

3
packages/server/src/connectStoreController.ts
View File

@@ -48,12 +48,11 @@ export default function (
packageName,
}) as { dir: string, isBuilt: boolean }
},
importPackage: async (from: string, to: string, opts: {
importPackage: async (to: string, opts: {
filesResponse: PackageFilesResponse,
force: boolean,
}) => {
await limitedFetch(`${remotePrefix}/importPackage`, {
from,
opts,
to,
})

2
packages/server/src/createServer.ts
View File

@@ -142,7 +142,7 @@ export default function (
break
case '/importPackage':
const importPackageBody = (await bodyPromise) as any // tslint:disable-line:no-any
await store.importPackage(importPackageBody.from, importPackageBody.to, importPackageBody.opts)
await store.importPackage(importPackageBody.to, importPackageBody.opts)
res.end(JSON.stringify('OK'))
break
case '/upload':

23
packages/server/test/index.ts
View File

@@ -6,6 +6,7 @@ import { PackageFilesResponse, ResolveFunction } from '@pnpm/store-controller-ty
import createFetcher from '@pnpm/tarball-fetcher'
import rimraf = require('@zkochan/rimraf')
import isPortReachable = require('is-port-reachable')
import loadJsonFile = require('load-json-file')
import fs = require('mz/fs')
import fetch from 'node-fetch'
import path = require('path')
@@ -14,9 +15,10 @@ import tempy = require('tempy')
const registry = 'https://registry.npmjs.org/'
const storeDir = tempy.directory()
async function createStoreController () {
async function createStoreController (storeDir?: string) {
if (!storeDir) {
storeDir = tempy.directory()
}
const rawConfig = { registry }
const resolve = createResolver({
metaCache: new Map<string, object>() as PackageMetaCache,
@@ -69,7 +71,7 @@ test('server', async t => {
const files = await response.files!()
t.notOk(files.fromStore)
t.ok(files.filenames.includes('package.json'))
t.ok(files.filesIndex['package.json'])
t.ok(response.finishing)
await response.finishing!()
@@ -83,7 +85,8 @@ test('fetchPackage', async t => {
const port = 5813
const hostname = '127.0.0.1'
const remotePrefix = `http://${hostname}:${port}`
const storeCtrlForServer = await createStoreController()
const storeDir = tempy.directory()
const storeCtrlForServer = await createStoreController(storeDir)
const server = createServer(storeCtrlForServer, {
hostname,
port,
@@ -108,7 +111,7 @@ test('fetchPackage', async t => {
const files = await response['files']() as PackageFilesResponse
t.notOk(files.fromStore)
t.ok(files.filenames.includes('package.json'))
t.ok(files.filesIndex['package.json'])
t.ok(response['finishing'])
await response['finishing']()
@@ -173,7 +176,8 @@ test('server upload', async t => {
const port = 5813
const hostname = '127.0.0.1'
const remotePrefix = `http://${hostname}:${port}`
const storeCtrlForServer = await createStoreController()
const storeDir = tempy.directory()
const storeCtrlForServer = await createStoreController(storeDir)
const server = createServer(storeCtrlForServer, {
hostname,
port,
@@ -188,9 +192,8 @@ test('server upload', async t => {
packageId: fakePkgId,
})
const cachePath = path.join(storeDir, fakePkgId, 'side_effects', fakeEngine, 'package')
t.ok(await fs.exists(cachePath), 'cache directory created')
t.deepEqual(await fs.readdir(cachePath), ['side-effect.js', 'side-effect.txt'], 'all files uploaded to cache')
const cacheIntegrity = await loadJsonFile(path.join(storeDir, fakePkgId, 'side_effects', fakeEngine, 'integrity.json'))
t.deepEqual(Object.keys(cacheIntegrity), ['side-effect.js', 'side-effect.txt'], 'all files uploaded to cache')
await server.close()
await storeCtrl.close()

13
packages/store-connection-manager/src/createNewStoreController.ts
View File

@@ -1,8 +1,6 @@
import { Config } from '@pnpm/config'
import createFetcher from '@pnpm/default-fetcher'
import logger from '@pnpm/logger'
import createStore from '@pnpm/package-store'
import dirIsCaseSensitive from 'dir-is-case-sensitive'
import fs = require('mz/fs')
import path = require('path')
import createResolver, { CreateResolverOptions } from './createResolver'
@@ -29,17 +27,10 @@ export default async (
})
const resolve = createResolver(sopts)
await fs.mkdir(sopts.storeDir, { recursive: true })
const fsIsCaseSensitive = await dirIsCaseSensitive(sopts.storeDir)
logger.debug({
// An undefined field would cause a crash of the logger
// so converting it to null
isCaseSensitive: typeof fsIsCaseSensitive === 'boolean'
? fsIsCaseSensitive : null,
store: sopts.storeDir,
})
const fetchers = createFetcher({ ...sopts, fsIsCaseSensitive })
const fetchers = createFetcher(sopts)
return {
ctrl: await createStore(resolve, fetchers as {}, {
ignoreFile: sopts.ignoreFile,
locks: sopts.locks,
lockStaleDuration: sopts.lockStaleDuration,
networkConcurrency: sopts.networkConcurrency,

3
packages/store-controller-types/src/index.ts
View File

@@ -76,7 +76,6 @@ export interface FetchPackageToStoreOptions {
}
export type ImportPackageFunction = (
from: string,
to: string,
opts: {
filesResponse: PackageFilesResponse,
@@ -86,7 +85,7 @@ export type ImportPackageFunction = (
export interface PackageFilesResponse {
fromStore: boolean,
filenames: string[],
filesIndex: Record<string, { integrity: string }>,
}
export type RequestPackageFunction = (

10
packages/supi/src/install/link.ts
View File

@@ -253,8 +253,8 @@ export default async function linkPackages (
const pkgJson = await depNode.fetchingBundledManifest()
depNode.requiresBuild = Boolean(
pkgJson.scripts && (pkgJson.scripts.preinstall || pkgJson.scripts.install || pkgJson.scripts.postinstall) ||
filesResponse.filenames.includes('binding.gyp') ||
filesResponse.filenames.some((filename) => !!filename.match(/^[.]hooks[\\/]/)), // TODO: optimize this
filesResponse.filesIndex['binding.gyp'] ||
Object.keys(filesResponse.filesIndex).some((filename) => !!filename.match(/^[.]hooks[\\/]/)), // TODO: optimize this
)
// TODO: try to cover with unit test the case when entry is no longer available in lockfile
@@ -451,7 +451,6 @@ async function selectNewFromWantedDeps (
const depNode = depGraph[depPath]
if (!depNode) return
if (prevRelDepPaths.has(wantedRelDepPath)) {
if (depNode.independent) return
if (await fs.exists(depNode.peripheralLocation)) {
return
}
@@ -476,11 +475,10 @@ async function linkAllPkgs (
},
) {
return Promise.all(
depNodes.map(async ({ centralLocation, fetchingFiles, independent, peripheralLocation }) => {
depNodes.map(async ({ fetchingFiles, independent, peripheralLocation }) => {
const filesResponse = await fetchingFiles()
if (independent) return
return storeController.importPackage(centralLocation, peripheralLocation, {
return storeController.importPackage(peripheralLocation, {
filesResponse,
force: opts.force,
})

2
packages/supi/src/install/resolvePeers.ts
View File

@@ -23,7 +23,6 @@ export interface DependenciesGraphNode {
version: string,
hasBin: boolean,
hasBundledDependencies: boolean,
centralLocation: string,
modules: string,
fetchingBundledManifest?: () => Promise<DependencyManifest>,
fetchingFiles: () => Promise<PackageFilesResponse>,
@@ -226,7 +225,6 @@ function resolvePeersOfNode (
ctx.depGraph[absolutePath] = {
absolutePath,
additionalInfo: node.resolvedPackage.additionalInfo,
centralLocation,
children: Object.assign(children, resolvedPeers),
depth: node.depth,
dev: node.resolvedPackage.dev,

14
packages/supi/test/install/misc.ts
View File

@@ -408,20 +408,6 @@ test('refetch package to store if it has been modified', async (t) => {
t.ok(distPathExists, 'magic-hook@2.0.0 dist folder reinstalled')
})
test("don't refetch package to store if it has been modified and verify-store-integrity = false", async (t: tape.Test) => {
const project = prepareEmpty(t)
const opts = await testDefaults({ verifyStoreIntegrity: false })
const manifest = await addDependenciesToPackage({}, ['magic-hook@2.0.0'], opts)
await writeJsonFile(path.join(await project.getStorePath(), `localhost+${REGISTRY_MOCK_PORT}`, 'magic-hook', '2.0.0', 'node_modules', 'magic-hook', 'package.json'), {})
await rimraf('node_modules')
await addDependenciesToPackage(manifest, ['magic-hook@2.0.0'], opts)
t.deepEqual(project.requireModule('magic-hook/package.json'), {}, 'package.json not refetched even though it was mutated')
})
// TODO: decide what to do with this case
// tslint:disable-next-line:no-string-literal
test.skip('relink package to project if the dependency is not linked from store', async (t: tape.Test) => {

8
packages/supi/test/install/sideEffects.ts
View File

@@ -13,7 +13,7 @@ const test = promisifyTape(tape)
const ENGINE_DIR = `${process.platform}-${process.arch}-node-${process.version.split('.')[0]}`
test('caching side effects of native package', async (t) => {
test.skip('caching side effects of native package', async (t) => {
prepareEmpty(t)
const opts = await testDefaults({
@@ -38,7 +38,7 @@ test('caching side effects of native package', async (t) => {
t.notEqual(stat1.ino, stat3.ino, 'cache is overridden when force is true')
})
test('caching side effects of native package when hoisting is used', async (t) => {
test.skip('caching side effects of native package when hoisting is used', async (t) => {
const project = prepareEmpty(t)
const opts = await testDefaults({
@@ -67,7 +67,7 @@ test('caching side effects of native package when hoisting is used', async (t) =
await project.has('.pnpm/node_modules/es6-promise') // verifying that a flat node_modules was created
})
test('using side effects cache', async (t) => {
test.skip('using side effects cache', async (t) => {
prepareEmpty(t)
// Right now, hardlink does not work with side effects, so we specify copy as the packageImportMethod
@@ -89,7 +89,7 @@ test('using side effects cache', async (t) => {
t.ok(await exists('node_modules/diskusage/build/new-file.txt'), 'side effects cache correctly used')
})
test('readonly side effects cache', async (t) => {
test.skip('readonly side effects cache', async (t) => {
prepareEmpty(t)
const opts1 = await testDefaults({

2
packages/supi/test/utils/testDefaults.ts
View File

@@ -50,13 +50,13 @@ export default async function testDefaults<T> (
}),
createFetcher({
alwaysAuth: true,
ignoreFile: opts?.fastUnpack === false ? undefined : (filename) => filename !== 'package.json',
rawConfig,
registry,
...retryOpts,
...fetchOpts,
}) as {},
{
ignoreFile: opts?.fastUnpack === false ? undefined : (filename) => filename !== 'package.json',
locks: path.join(storeDir, '_locks'),
storeDir,
verifyStoreIntegrity: true,

1
packages/tarball-fetcher/example/.gitignore vendored
View File

@@ -1 +0,0 @@
dist

25
packages/tarball-fetcher/example/index.js
View File

@@ -1,25 +0,0 @@
'use strict'
const createFetcher = require('@pnpm/tarball-fetcher').default
process.chdir(__dirname)
const registry = 'https://registry.npmjs.org/'
const fetch = createFetcher({
registry,
rawConfig: {
registry,
},
})
const resolution = {
tarball: 'https://registry.npmjs.org/is-positive/-/is-positive-1.0.0.tgz',
}
fetch.tarball(resolution, 'dist/unpacked', {
cachedTarballLocation: 'dist/cache.tgz',
prefix: process.cwd(),
})
.then(index => console.log(Object.keys(index)))
.catch(err => {
console.error(err)
process.exit(1)
})

6
packages/tarball-fetcher/package.json
View File

@@ -12,7 +12,6 @@
"lint": "tslint -c ../../tslint.json src/**/*.ts test/**/*.ts",
"prepublishOnly": "pnpm run compile",
"_test": "cd ../.. && c8 --reporter lcov --reports-dir packages/tarball-fetcher/coverage ts-node packages/tarball-fetcher/test --type-check",
"post_test": "node example",
"test": "pnpm run compile && pnpm run _test",
"compile": "rimraf lib tsconfig.tsbuildinfo && tsc --build"
},
@@ -46,11 +45,12 @@
"path-temp": "2.0.0",
"retry": "0.12.0",
"rimraf": "3.0.2",
"ssri": "6.0.1",
"unpack-stream": "6.0.0"
"ssri": "6.0.1"
},
"devDependencies": {
"@pnpm/cafs": "workspace:0.0.0",
"@pnpm/logger": "3.2.2",
"@pnpm/read-package-json": "workspace:3.1.0",
"@pnpm/tarball-fetcher": "link:",
"@types/graceful-fs": "^4.1.3",
"@types/mz": "^2.7.0",

62
packages/tarball-fetcher/src/createDownloader.ts
View File

@@ -1,6 +1,5 @@
import PnpmError from '@pnpm/error'
import { FetchResult, FilesIndex } from '@pnpm/fetcher-base'
import logger from '@pnpm/logger'
import { Cafs, FetchResult, FilesIndex } from '@pnpm/fetcher-base'
import createFetcher from 'fetch-from-npm-registry'
import fs = require('graceful-fs')
import { IncomingMessage } from 'http'
@@ -10,12 +9,9 @@ import pathTemp = require('path-temp')
import retry = require('retry')
import rimraf = require('rimraf')
import ssri = require('ssri')
import unpackStream = require('unpack-stream')
import urlLib = require('url')
import { BadTarballError } from './errorTypes'
const ignorePackageFileLogger = logger('_ignore-package-file')
class TarballFetchError extends PnpmError {
public readonly httpStatusCode: number
public readonly uri: string
@@ -61,13 +57,11 @@ export type DownloadFunction = (url: string, saveto: string, opts: {
authHeaderValue: string | undefined,
alwaysAuth: boolean | undefined,
},
unpackTo: string,
cafs: Cafs,
registry?: string,
onStart?: (totalSize: number | null, attempt: number) => void,
onProgress?: (downloaded: number) => void,
ignore?: (filename: string) => boolean,
integrity?: string
generatePackageIntegrity?: boolean,
integrity?: string,
}) => Promise<FetchResult>
export interface NpmRegistryClient {
@@ -78,7 +72,6 @@ export interface NpmRegistryClient {
export default (
gotOpts: {
alwaysAuth: boolean,
fsIsCaseSensitive: boolean,
registry: string,
// proxy
proxy?: string,
@@ -114,13 +107,11 @@ export default (
authHeaderValue: string | undefined,
alwaysAuth: boolean | undefined,
},
unpackTo: string,
cafs: Cafs,
registry?: string,
onStart?: (totalSize: number | null, attempt: number) => void,
onProgress?: (downloaded: number) => void,
ignore?: (filename: string) => boolean,
integrity?: string,
generatePackageIntegrity?: boolean,
}): Promise<FetchResult> {
const saveToDir = path.dirname(saveto)
await makeDir(saveToDir)
@@ -185,15 +176,10 @@ export default (
.pipe(writeStream)
.on('error', reject)
const tempLocation = pathTemp(opts.unpackTo)
const ignore = gotOpts.fsIsCaseSensitive ? opts.ignore : createIgnorer(url, opts.ignore)
try {
const [integrityCheckResult, filesIndex] = await Promise.all([
opts.integrity && safeCheckStream(res.body, opts.integrity, url) || true,
unpackStream.local(res.body, tempLocation, {
generateIntegrity: opts.generatePackageIntegrity,
ignore,
}),
opts.cafs.addFilesFromTarball(res.body),
waitTillClosed({ stream, size, getDownloaded: () => downloaded, url }),
])
if (integrityCheckResult !== true) {
@@ -202,15 +188,11 @@ export default (
fs.rename(tempTarballLocation, saveto, () => {
// ignore errors
})
resolve({ tempLocation, filesIndex: filesIndex as FilesIndex })
resolve({ filesIndex: filesIndex as FilesIndex })
} catch (err) {
rimraf(tempTarballLocation, () => {
// ignore errors
})
rimraf(tempLocation, () => {
// Just ignoring this error
// A redundant stage folder won't break anything
})
reject(err)
}
})
@@ -223,38 +205,6 @@ export default (
}
}
function createIgnorer (tarballUrl: string, ignore?: (filename: string) => boolean) {
const lowercaseFiles = new Set<string>()
if (ignore) {
return (filename: string) => {
const lowercaseFilename = filename.toLowerCase()
if (lowercaseFiles.has(lowercaseFilename)) {
ignorePackageFileLogger.debug({
reason: 'case-insensitive-duplicate',
skippedFilename: filename,
tarballUrl,
})
return true
}
lowercaseFiles.add(lowercaseFilename)
return ignore(filename)
}
}
return (filename: string) => {
const lowercaseFilename = filename.toLowerCase()
if (lowercaseFiles.has(lowercaseFilename)) {
ignorePackageFileLogger.debug({
reason: 'case-insensitive-duplicate',
skippedFilename: filename,
tarballUrl,
})
return true
}
lowercaseFiles.add(lowercaseFilename)
return false
}
}
async function safeCheckStream (stream: any, integrity: string, url: string): Promise<true | Error> { // tslint:disable-line:no-any
try {
await ssri.checkStream(stream, integrity)

54
packages/tarball-fetcher/src/index.ts
View File

@@ -1,5 +1,6 @@
import PnpmError from '@pnpm/error'
import {
Cafs,
FetchFunction,
FetchOptions,
FetchResult,
@@ -9,14 +10,9 @@ import getCredentialsByURI = require('credentials-by-uri')
import mem = require('mem')
import fs = require('mz/fs')
import path = require('path')
import pathTemp = require('path-temp')
import rimraf = require('rimraf')
import ssri = require('ssri')
import * as unpackStream from 'unpack-stream'
import createDownloader, { DownloadFunction } from './createDownloader'
export type IgnoreFunction = (filename: string) => boolean
export default function (
opts: {
registry: string,
@@ -34,18 +30,13 @@ export default function (
fetchRetryMintimeout?: number,
fetchRetryMaxtimeout?: number,
userAgent?: string,
ignoreFile?: IgnoreFunction,
offline?: boolean,
fsIsCaseSensitive?: boolean,
},
): { tarball: FetchFunction } {
const download = createDownloader({
alwaysAuth: opts.alwaysAuth || false,
ca: opts.ca,
cert: opts.cert,
fsIsCaseSensitive: typeof opts.fsIsCaseSensitive === 'boolean'
? opts.fsIsCaseSensitive
: false,
key: opts.key,
localAddress: opts.localAddress,
proxy: opts.httpsProxy || opts.proxy,
@@ -69,10 +60,8 @@ export default function (
fetchFromRemoteTarball: fetchFromRemoteTarball.bind(null, {
download,
getCredentialsByURI: mem((registry: string) => getCreds(registry)),
ignoreFile: opts.ignoreFile,
offline: opts.offline,
}),
ignore: opts.ignoreFile,
}) as FetchFunction,
}
}
@@ -80,7 +69,7 @@ export default function (
function fetchFromTarball (
ctx: {
fetchFromRemoteTarball: (
dir: string,
cafs: Cafs,
dist: {
integrity?: string,
registry?: string,
@@ -88,37 +77,34 @@ function fetchFromTarball (
},
opts: FetchOptions,
) => Promise<FetchResult>,
ignore?: IgnoreFunction,
},
cafs: Cafs,
resolution: {
integrity?: string,
registry?: string,
tarball: string,
},
target: string,
opts: FetchOptions,
) {
if (resolution.tarball.startsWith('file:')) {
const tarball = path.join(opts.lockfileDir, resolution.tarball.slice(5))
return fetchFromLocalTarball(tarball, target, {
ignore: ctx.ignore,
return fetchFromLocalTarball(cafs, tarball, {
integrity: resolution.integrity,
})
}
return ctx.fetchFromRemoteTarball(target, resolution, opts)
return ctx.fetchFromRemoteTarball(cafs, resolution, opts)
}
async function fetchFromRemoteTarball (
ctx: {
offline?: boolean,
download: DownloadFunction,
ignoreFile?: IgnoreFunction,
getCredentialsByURI: (registry: string) => {
authHeaderValue: string | undefined,
alwaysAuth: boolean | undefined,
},
},
unpackTo: string,
cafs: Cafs,
dist: {
integrity?: string,
registry?: string,
@@ -127,7 +113,7 @@ async function fetchFromRemoteTarball (
opts: FetchOptions,
) {
try {
return await fetchFromLocalTarball(opts.cachedTarballLocation, unpackTo, {
return await fetchFromLocalTarball(cafs, opts.cachedTarballLocation, {
integrity: dist.integrity,
})
} catch (err) {
@@ -163,44 +149,32 @@ async function fetchFromRemoteTarball (
const auth = dist.registry ? ctx.getCredentialsByURI(dist.registry) : undefined
return ctx.download(dist.tarball, opts.cachedTarballLocation, {
auth,
ignore: ctx.ignoreFile,
cafs,
integrity: dist.integrity,
onProgress: opts.onProgress,
onStart: opts.onStart,
registry: dist.registry,
unpackTo,
})
}
}
async function fetchFromLocalTarball (
cafs: Cafs,
tarball: string,
dir: string,
opts: {
ignore?: IgnoreFunction,
integrity?: string,
},
): Promise<FetchResult> {
const tarballStream = fs.createReadStream(tarball)
const tempLocation = pathTemp(dir)
try {
const filesIndex = (
const tarballStream = fs.createReadStream(tarball)
const [filesIndex] = (
await Promise.all([
unpackStream.local(
tarballStream,
tempLocation,
{
ignore: opts.ignore,
},
),
cafs.addFilesFromTarball(tarballStream),
opts.integrity && (ssri.checkStream(tarballStream, opts.integrity) as any), // tslint:disable-line
])
)[0]
return { filesIndex, tempLocation }
)
return { filesIndex }
} catch (err) {
rimraf(tempLocation, () => {
// ignore errors
})
err.attempts = 1
err.resource = tarball
throw err

54
packages/tarball-fetcher/test/download.ts
View File

@@ -1,5 +1,7 @@
///<reference path="../../../typings/index.d.ts" />
import createCafs, { getFilePathInCafs as _getFilePathInCafs } from '@pnpm/cafs'
import { LogBase, streamParser } from '@pnpm/logger'
import readPackage from '@pnpm/read-package-json'
import createFetcher from '@pnpm/tarball-fetcher'
import cpFile = require('cp-file')
import { existsSync } from 'fs'
@@ -10,6 +12,11 @@ import ssri = require('ssri')
import test = require('tape')
import tempy = require('tempy')
const cafsDir = tempy.directory()
console.log(cafsDir)
const cafs = createCafs(cafsDir)
const getFilePathInCafs = _getFilePathInCafs.bind(_getFilePathInCafs, cafsDir)
const tarballPath = path.join(__dirname, 'tars', 'babel-helper-hoist-variables-6.24.1.tgz')
const tarballSize = 1279
const tarballIntegrity = 'sha1-HssnaJydJVE+rbyZFKc/VAi+enY='
@@ -35,7 +42,6 @@ test('fail when tarball size does not match content-length', async t => {
process.chdir(tempy.directory())
t.comment(`temp dir ${process.cwd()}`)
const unpackTo = path.resolve('unpacked')
const cachedTarballLocation = path.resolve('cached')
const resolution = {
// Even though the integrity of the downloaded tarball
@@ -47,7 +53,7 @@ test('fail when tarball size does not match content-length', async t => {
}
try {
await fetch.tarball(resolution, unpackTo, {
await fetch.tarball(cafs, resolution, {
cachedTarballLocation,
lockfileDir: process.cwd(),
})
@@ -95,13 +101,14 @@ test('redownload the tarball when the one in cache does not satisfy integrity',
}
}
streamParser.on('data', reporter as any) // tslint:disable-line:no-any
const { tempLocation } = await fetch.tarball(resolution, path.join(cacheDir, 'unpacked'), {
const { filesIndex } = await fetch.tarball(cafs, resolution, {
cachedTarballLocation,
lockfileDir: process.cwd(),
})
streamParser.removeListener('data', reporter as any) // tslint:disable-line:no-any
t.equal((await import(path.join(tempLocation, 'package.json'))).version, '6.24.1')
const pkgJsonIntegrity = await filesIndex['package.json'].generatingIntegrity
t.equal((await readPackage(getFilePathInCafs(pkgJsonIntegrity))).version, '6.24.1')
t.ok(scope.isDone())
t.end()
@@ -134,7 +141,7 @@ test('fail when the tarball in the cache does not pass integrity check in offlin
},
registry,
})
await fetch.tarball(resolution, path.join(cacheDir, 'unpacked'), {
await fetch.tarball(cafs, resolution, {
cachedTarballLocation,
lockfileDir: process.cwd(),
})
@@ -163,17 +170,14 @@ test('retry when tarball size does not match content-length', async t => {
process.chdir(tempy.directory())
t.comment(`testing in ${process.cwd()}`)
const unpackTo = path.resolve('unpacked')
const cachedTarballLocation = path.resolve('cached.tgz')
const resolution = { tarball: 'http://example.com/foo.tgz' }
const result = await fetch.tarball(resolution, unpackTo, {
const result = await fetch.tarball(cafs, resolution, {
cachedTarballLocation,
lockfileDir: process.cwd(),
})
t.equal(typeof result.tempLocation, 'string')
// fetch.tarball() doesn't wait till the cached tarball is renamed.
// So this may happen a bit later
setTimeout(() => {
@@ -200,7 +204,6 @@ test('redownload incomplete cached tarballs', async t => {
process.chdir(tempy.directory())
t.comment(`testing in ${process.cwd()}`)
const unpackTo = path.resolve('unpacked')
const cachedTarballLocation = path.resolve('cached')
const cachedTarballFd = await fs.open(cachedTarballLocation, 'w')
const tarballData = await fs.readFile(tarballPath)
@@ -217,7 +220,7 @@ test('redownload incomplete cached tarballs', async t => {
}
streamParser.on('data', reporter as any) // tslint:disable-line:no-any
try {
await fetch.tarball(resolution, unpackTo, {
await fetch.tarball(cafs, resolution, {
cachedTarballLocation,
lockfileDir: process.cwd(),
})
@@ -242,7 +245,6 @@ test('fail when integrity check fails two times in a row', async t => {
process.chdir(tempy.directory())
t.comment(`testing in ${process.cwd()}`)
const unpackTo = path.resolve('unpacked')
const cachedTarballLocation = path.resolve('cached')
const resolution = {
integrity: tarballIntegrity,
@@ -250,7 +252,7 @@ test('fail when integrity check fails two times in a row', async t => {
}
try {
await fetch.tarball(resolution, unpackTo, {
await fetch.tarball(cafs, resolution, {
cachedTarballLocation,
lockfileDir: process.cwd(),
})
@@ -281,7 +283,6 @@ test('retry when integrity check fails', async t => {
process.chdir(tempy.directory())
t.comment(`testing in ${process.cwd()}`)
const unpackTo = path.resolve('unpacked')
const cachedTarballLocation = path.resolve('cached')
const resolution = {
integrity: tarballIntegrity,
@@ -289,7 +290,7 @@ test('retry when integrity check fails', async t => {
}
const params: Array<[number | null, number]> = []
await fetch.tarball(resolution, unpackTo, {
await fetch.tarball(cafs, resolution, {
cachedTarballLocation,
lockfileDir: process.cwd(),
onStart (size, attempts) {
@@ -308,7 +309,6 @@ test('fail when integrity check of local file fails', async (t) => {
process.chdir(tempy.directory())
t.comment(`testing in ${process.cwd()}`)
const unpackTo = path.resolve('unpacked')
const cachedTarballLocation = path.resolve('cached')
const tarballAbsoluteLocation = path.join(__dirname, 'tars', 'babel-helper-hoist-variables-7.0.0-alpha.10.tgz')
const tarball = path.relative(process.cwd(), tarballAbsoluteLocation)
@@ -319,7 +319,7 @@ test('fail when integrity check of local file fails', async (t) => {
let err: Error | null = null
try {
await fetch.tarball(resolution, unpackTo, {
await fetch.tarball(cafs, resolution, {
cachedTarballLocation,
lockfileDir: process.cwd(),
})
@@ -341,7 +341,6 @@ test("don't fail when integrity check of local file succeeds", async (t) => {
process.chdir(tempy.directory())
t.comment(`testing in ${process.cwd()}`)
const unpackTo = path.resolve('unpacked')
const cachedTarballLocation = path.resolve('cached')
const tarballAbsoluteLocation = path.join(__dirname, 'tars', 'babel-helper-hoist-variables-7.0.0-alpha.10.tgz')
const tarball = path.relative(process.cwd(), tarballAbsoluteLocation)
@@ -350,13 +349,12 @@ test("don't fail when integrity check of local file succeeds", async (t) => {
tarball: `file:${tarball}`,
}
const { filesIndex, tempLocation } = await fetch.tarball(resolution, unpackTo, {
const { filesIndex } = await fetch.tarball(cafs, resolution, {
cachedTarballLocation,
lockfileDir: process.cwd(),
})
t.equal(typeof filesIndex['package.json'], 'object', 'files index returned')
t.equal(typeof tempLocation, 'string', 'temp location returned')
t.end()
})
@@ -372,7 +370,6 @@ test("don't fail when the cache tarball does not exist", async (t) => {
process.chdir(tempy.directory())
t.comment(`testing in ${process.cwd()}`)
const unpackTo = path.resolve('unpacked')
const cachedTarballLocation = path.resolve('dir', 'cached')
const tarballAbsoluteLocation = path.join(__dirname, 'tars', 'babel-helper-hoist-variables-7.0.0-alpha.10.tgz')
const resolution = {
@@ -380,13 +377,12 @@ test("don't fail when the cache tarball does not exist", async (t) => {
tarball: `${registry}foo.tgz`,
}
const { filesIndex, tempLocation } = await fetch.tarball(resolution, unpackTo, {
const { filesIndex } = await fetch.tarball(cafs, resolution, {
cachedTarballLocation,
lockfileDir: process.cwd(),
})
t.equal(typeof filesIndex['package.json'], 'object', 'files index returned')
t.equal(typeof tempLocation, 'string', 'temp location returned')
t.end()
})
@@ -395,7 +391,6 @@ test('fail when the cache tarball does not exist in offline mode', async (t) =>
process.chdir(tempy.directory())
t.comment(`testing in ${process.cwd()}`)
const unpackTo = path.resolve('unpacked')
const cachedTarballLocation = path.resolve('dir', 'cached')
const tarballAbsoluteLocation = path.join(__dirname, 'tars', 'babel-helper-hoist-variables-7.0.0-alpha.10.tgz')
const resolution = {
@@ -415,7 +410,7 @@ test('fail when the cache tarball does not exist in offline mode', async (t) =>
},
registry,
})
await fetch.tarball(resolution, unpackTo, {
await fetch.tarball(cafs, resolution, {
cachedTarballLocation,
lockfileDir: process.cwd(),
})
@@ -441,14 +436,13 @@ test('retry on server error', async t => {
process.chdir(tempy.directory())
t.comment(`testing in ${process.cwd()}`)
const unpackTo = path.resolve('unpacked')
const cachedTarballLocation = path.resolve('cached')
const resolution = {
integrity: tarballIntegrity,
tarball: 'http://example.com/foo.tgz',
}
const index = await fetch.tarball(resolution, unpackTo, {
const index = await fetch.tarball(cafs, resolution, {
cachedTarballLocation,
lockfileDir: process.cwd(),
})
@@ -467,7 +461,6 @@ test('throw error when accessing private package w/o authorization', async t =>
process.chdir(tempy.directory())
t.comment(`testing in ${process.cwd()}`)
const unpackTo = path.resolve('unpacked')
const cachedTarballLocation = path.resolve('cached')
const resolution = {
integrity: tarballIntegrity,
@@ -477,7 +470,7 @@ test('throw error when accessing private package w/o authorization', async t =>
let err!: Error
try {
await fetch.tarball(resolution, unpackTo, {
await fetch.tarball(cafs, resolution, {
cachedTarballLocation,
lockfileDir: process.cwd(),
})
@@ -525,7 +518,6 @@ test('accessing private packages', async t => {
registry,
})
const unpackTo = path.resolve('unpacked')
const cachedTarballLocation = path.resolve('cached')
const resolution = {
integrity: tarballIntegrity,
@@ -533,7 +525,7 @@ test('accessing private packages', async t => {
tarball: 'http://example.com/foo.tgz',
}
const index = await fetch.tarball(resolution, unpackTo, {
const index = await fetch.tarball(cafs, resolution, {
cachedTarballLocation,
lockfileDir: process.cwd(),
})

6
packages/tarball-fetcher/tsconfig.json
View File

@@ -17,6 +17,12 @@
},
{
"path": "../fetch-from-npm-registry"
},
{
"path": "../cafs"
},
{
"path": "../read-package-json"
}
]
}

112
pnpm-lock.yaml generated
View File

@@ -92,6 +92,42 @@ importers:
graph-sequencer: 2.0.0
ramda: 0.27.0
run-groups: 2.0.3
packages/cafs:
dependencies:
'@pnpm/fetcher-base': 'link:../fetcher-base'
'@zkochan/rimraf': 1.0.0
decompress-maybe: 1.0.0
get-stream: 5.1.0
mz: 2.7.0
p-limit: 2.3.0
path-exists: 4.0.0
path-temp: 2.0.0
rename-overwrite: 2.0.2
ssri: 6.0.1
tar-stream: 2.1.2
devDependencies:
'@types/mz': 2.7.0
'@types/node': 13.13.2
'@types/ssri': 6.0.2
'@types/tar-stream': 2.1.0
tempy: 0.5.0
specifiers:
'@pnpm/fetcher-base': 'workspace:6.0.0'
'@types/mz': 2.7.0
'@types/node': ^13.13.2
'@types/ssri': ^6.0.2
'@types/tar-stream': ^2.1.0
'@zkochan/rimraf': 1.0.0
decompress-maybe: ^1.0.0
get-stream: 5.1.0
mz: 2.7.0
p-limit: 2.3.0
path-exists: 4.0.0
path-temp: 2.0.0
rename-overwrite: 2.0.2
ssri: 6.0.1
tar-stream: ^2.1.2
tempy: ^0.5.0
packages/cli-meta:
dependencies:
load-json-file: 6.2.0
@@ -504,19 +540,19 @@ importers:
ramda: 0.27.0
packages/git-fetcher:
dependencies:
'@pnpm/fetcher-base': 'link:../fetcher-base'
'@zkochan/rimraf': 1.0.0
dint: 4.0.0
execa: 4.0.0
path-temp: 2.0.0
devDependencies:
'@pnpm/git-fetcher': 'link:'
tempy: 0.5.0
devDependencies:
'@pnpm/cafs': 'link:../cafs'
'@pnpm/git-fetcher': 'link:'
specifiers:
'@pnpm/cafs': 'workspace:0.0.0'
'@pnpm/fetcher-base': 'workspace:6.0.0'
'@pnpm/git-fetcher': 'link:'
'@zkochan/rimraf': 1.0.0
dint: 4.0.0
execa: 4.0.0
path-temp: 2.0.0
tempy: 0.5.0
packages/git-resolver:
dependencies:
@@ -589,12 +625,14 @@ importers:
fs-extra: 9.0.0
is-windows: 1.0.2
isexe: 2.0.0
load-json-file: 6.2.0
mz: 2.7.0
npm-run-all: 4.1.5
read-yaml-file: 2.0.0
sinon: 9.0.2
tape-promise: 4.0.0
tempy: 0.5.0
write-json-file: 4.3.0
specifiers:
'@pnpm/assert-project': 'workspace:*'
'@pnpm/build-modules': 'workspace:4.1.13'
@@ -635,6 +673,7 @@ importers:
fs-extra: 9.0.0
is-windows: 1.0.2
isexe: 2.0.0
load-json-file: 6.2.0
mz: 2.7.0
npm-run-all: 4.1.5
p-limit: 2.3.0
@@ -646,6 +685,7 @@ importers:
sinon: 9.0.2
tape-promise: 4.0.0
tempy: 0.5.0
write-json-file: 4.3.0
packages/hoist:
dependencies:
'@pnpm/constants': 'link:../constants'
@@ -1166,7 +1206,7 @@ importers:
semver: ^7.3.2
packages/package-requester:
dependencies:
'@pnpm/check-package': 3.0.1
'@pnpm/cafs': 'link:../cafs'
'@pnpm/core-loggers': 'link:../core-loggers'
'@pnpm/fetcher-base': 'link:../fetcher-base'
'@pnpm/pkgid-to-filename': 2.0.0
@@ -1174,13 +1214,11 @@ importers:
'@pnpm/resolver-base': 'link:../resolver-base'
'@pnpm/store-controller-types': 'link:../store-controller-types'
'@pnpm/types': 'link:../types'
'@zkochan/rimraf': 1.0.0
load-json-file: 6.2.0
make-dir: 3.1.0
mz: 2.7.0
p-limit: 2.3.0
p-queue: 6.3.0
path-exists: 4.0.0
promise-share: 1.0.0
ramda: 0.27.0
rename-overwrite: 2.0.2
@@ -1206,7 +1244,7 @@ importers:
sinon: 9.0.2
tempy: 0.5.0
specifiers:
'@pnpm/check-package': 3.0.1
'@pnpm/cafs': 'workspace:0.0.0'
'@pnpm/core-loggers': 'workspace:4.0.1'
'@pnpm/fetcher-base': 'workspace:6.0.0'
'@pnpm/local-resolver': 'workspace:*'
@@ -1225,7 +1263,6 @@ importers:
'@types/ramda': ^0.27.4
'@types/sinon': ^9.0.0
'@types/ssri': ^6.0.2
'@zkochan/rimraf': 1.0.0
delay: 4.3.0
load-json-file: 6.2.0
make-dir: ^3.1.0
@@ -1235,7 +1272,6 @@ importers:
normalize-path: 3.0.0
p-limit: 2.3.0
p-queue: 6.3.0
path-exists: 4.0.0
promise-share: 1.0.0
ramda: 0.27.0
rename-overwrite: 2.0.2
@@ -1246,6 +1282,7 @@ importers:
write-json-file: 4.0.0
packages/package-store:
dependencies:
'@pnpm/cafs': 'link:../cafs'
'@pnpm/core-loggers': 'link:../core-loggers'
'@pnpm/fetcher-base': 'link:../fetcher-base'
'@pnpm/fs-locker': 3.0.1
@@ -1258,7 +1295,6 @@ importers:
load-json-file: 6.2.0
make-empty-dir: 1.0.0
mz: 2.7.0
ncp: 2.0.0
p-filter: 2.1.0
p-limit: 2.3.0
path-exists: 4.0.0
@@ -1272,7 +1308,6 @@ importers:
'@pnpm/package-store': 'link:'
'@pnpm/tarball-fetcher': 'link:../tarball-fetcher'
'@types/mz': 2.7.0
'@types/ncp': 2.0.3
'@types/proxyquire': 1.3.28
'@types/ramda': 0.27.4
'@types/sinon': 9.0.0
@@ -1280,6 +1315,7 @@ importers:
sinon: 9.0.2
tempy: 0.5.0
specifiers:
'@pnpm/cafs': 'workspace:0.0.0'
'@pnpm/core-loggers': 'workspace:4.0.1'
'@pnpm/fetcher-base': 'workspace:6.0.0'
'@pnpm/fs-locker': 3.0.1
@@ -1293,7 +1329,6 @@ importers:
'@pnpm/tarball-fetcher': 'workspace:*'
'@pnpm/types': 'workspace:5.0.0'
'@types/mz': ^2.7.0
'@types/ncp': 2.0.3
'@types/proxyquire': 1.3.28
'@types/ramda': ^0.27.4
'@types/sinon': ^9.0.0
@@ -1301,7 +1336,6 @@ importers:
load-json-file: 6.2.0
make-empty-dir: ^1.0.0
mz: 2.7.0
ncp: 2.0.0
p-filter: 2.1.0
p-limit: 2.3.0
path-exists: 4.0.0
@@ -1959,6 +1993,7 @@ importers:
'@pnpm/config': 'link:../config'
'@pnpm/error': 'link:../error'
'@pnpm/get-context': 'link:../get-context'
'@pnpm/lockfile-utils': 'link:../lockfile-utils'
'@pnpm/normalize-registries': 'link:../normalize-registries'
'@pnpm/parse-wanted-dependency': 'link:../parse-wanted-dependency'
'@pnpm/pick-registry-for-package': 'link:../pick-registry-for-package'
@@ -1969,6 +2004,7 @@ importers:
archy: 1.0.0
common-tags: 1.8.0
dependency-path: 'link:../dependency-path'
dint: 4.0.0
p-filter: 2.1.0
ramda: 0.27.0
render-help: 1.0.0
@@ -1996,6 +2032,7 @@ importers:
'@pnpm/error': 'workspace:1.2.0'
'@pnpm/get-context': 'workspace:1.2.1'
'@pnpm/lockfile-file': 'workspace:3.0.8'
'@pnpm/lockfile-utils': 'workspace:2.0.11'
'@pnpm/logger': 3.2.2
'@pnpm/normalize-registries': 'workspace:1.0.0'
'@pnpm/parse-wanted-dependency': 'workspace:1.0.0'
@@ -2014,6 +2051,7 @@ importers:
archy: 1.0.0
common-tags: 1.8.0
dependency-path: 'workspace:4.0.6'
dint: 4.0.0
execa: 4.0.0
load-json-file: 6.2.0
p-filter: 2.1.0
@@ -2415,6 +2453,7 @@ importers:
'@types/uuid': 7.0.3
'@zkochan/rimraf': 1.0.0
is-port-reachable: 3.0.0
load-json-file: 6.2.0
mz: 2.7.0
node-fetch: 2.6.0
tempy: 0.5.0
@@ -2433,6 +2472,7 @@ importers:
'@types/uuid': ^7.0.3
'@zkochan/rimraf': 1.0.0
is-port-reachable: 3.0.0
load-json-file: 6.2.0
mz: 2.7.0
node-fetch: 2.6.0
p-limit: 2.3.0
@@ -2724,9 +2764,10 @@ importers:
retry: 0.12.0
rimraf: 3.0.2
ssri: 6.0.1
unpack-stream: 6.0.0
devDependencies:
'@pnpm/cafs': 'link:../cafs'
'@pnpm/logger': 3.2.2
'@pnpm/read-package-json': 'link:../read-package-json'
'@pnpm/tarball-fetcher': 'link:'
'@types/graceful-fs': 4.1.3
'@types/mz': 2.7.0
@@ -2737,9 +2778,11 @@ importers:
nock: 12.0.3
tempy: 0.5.0
specifiers:
'@pnpm/cafs': 'workspace:0.0.0'
'@pnpm/error': 'workspace:1.2.0'
'@pnpm/fetcher-base': 'workspace:6.0.0'
'@pnpm/logger': 3.2.2
'@pnpm/read-package-json': 'workspace:3.1.0'
'@pnpm/tarball-fetcher': 'link:'
'@types/graceful-fs': ^4.1.3
'@types/mz': ^2.7.0
@@ -2759,7 +2802,6 @@ importers:
rimraf: 3.0.2
ssri: 6.0.1
tempy: 0.5.0
unpack-stream: 6.0.0
packages/tarball-resolver:
dependencies:
'@pnpm/resolver-base': 'link:../resolver-base'
@@ -3701,6 +3743,12 @@ packages:
dev: true
resolution:
integrity: sha512-0V8cKowBdsiA9nbxAg7531sF2cdPZNiUogcfIUeUGm+bejUBE/bvibz3rH36iQP9bQjO/sOzFwU97/uC5mCyoA==
/@types/tar-stream/2.1.0:
dependencies:
'@types/node': 13.13.2
dev: true
resolution:
integrity: sha512-s1UQxQUVMHbSkCC0X4qdoiWgHF8DoyY1JjQouFsnk/8ysoTdBaiCHud/exoAZzKDbzAXVc+ah6sczxGVMAohFw==
/@types/touch/3.1.1:
dependencies:
'@types/node': 13.13.2
@@ -5928,7 +5976,7 @@ packages:
integrity: sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==
/dir-is-case-sensitive/1.0.2:
dependencies:
graceful-fs: 4.2.1
graceful-fs: 4.2.3
path-temp: 2.0.0
engines:
node: '>=8.15'
@@ -8725,10 +8773,6 @@ packages:
node: '>= 8.0.0'
resolution:
integrity: sha512-SyV9uPETRig5ZmYev0ANfiGeB+g6N2EnqqEfBbCGmmJ6MgZ3E4qv5aPbnHVdZ60KAHHXV+T3sXopdrnIXQdmjQ==
/mkdirp-classic/0.5.2:
dev: false
resolution:
integrity: sha512-ejdnDQcR75gwknmMw/tx02AuRs8jCtqFoFqDZMjiNxsu85sRIJVXDKHuLYvUUPRBUtV2FpSZa9bL1BUa3BdR2g==
/mkdirp/0.5.5:
dependencies:
minimist: 1.2.5
@@ -11036,7 +11080,6 @@ packages:
/sort-keys/4.0.0:
dependencies:
is-plain-obj: 2.1.0
dev: false
engines:
node: '>=8'
resolution:
@@ -11490,15 +11533,6 @@ packages:
hasBin: true
resolution:
integrity: sha512-waWwC/OqYVE9TS6r1IynlP2sEdk4Lfo6jazlgkuNkPTHIbuG2BTABIaKdlQWwPeB6Oo4ksZ1j33Yt0NTOAlYMQ==
/tar-fs/2.0.1:
dependencies:
chownr: 1.1.4
mkdirp-classic: 0.5.2
pump: 3.0.0
tar-stream: 2.1.2
dev: false
resolution:
integrity: sha512-6tzWDMeroL87uF/+lin46k+Q+46rAJ0SyPGz7OW7wTgblI273hsBqk2C1j0/xNadNLKDTUL9BukSjB7cwgmlPA==
/tar-stream/2.1.2:
dependencies:
bl: 4.0.2
@@ -12033,17 +12067,6 @@ packages:
dev: true
resolution:
integrity: sha512-8rMeVYWSIyccIJscb9NdCfZKSRBKYTeVnwmiRYT2ulE3qd1RaDQ0xQDP+rI3ccIWbhu/zuo5cgN8z73belNZgw==
/unpack-stream/6.0.0:
dependencies:
'@types/ssri': 6.0.2
decompress-maybe: 1.0.0
ssri: 6.0.1
tar-fs: 2.0.1
dev: false
engines:
node: '>=8.15'
resolution:
integrity: sha512-BJVLP4Vo+BE5r3mZdcioGFA1XHFUI1wmhy7ytlCicItn815lWZXGj4uxip3H7ou1UWBLFUDFBnX6t4ScvOVSEA==
/unpipe/1.0.0:
dev: true
engines:
@@ -12533,7 +12556,6 @@ packages:
make-dir: 3.1.0
sort-keys: 4.0.0
write-file-atomic: 3.0.3
dev: false
engines:
node: '>=8.3'
resolution:

4
privatePackages/assert-store/src/index.ts
View File

@@ -17,9 +17,9 @@ export default (t: Test, storePath: string | Promise<string>, encodedRegistryNam
async resolve (pkgName: string, version?: string, relativePath?: string): Promise<string> {
const pkgFolder = version ? path.join(ern, pkgName, version) : pkgName
if (relativePath) {
return path.join(await storePath, pkgFolder, 'package', relativePath)
return path.join(await storePath, pkgFolder, relativePath)
}
return path.join(await storePath, pkgFolder, 'package')
return path.join(await storePath, pkgFolder)
},
}
return store

10
typings/local.d.ts vendored
View File

@@ -142,3 +142,13 @@ declare module 'yaml-tag' {
const anything: any;
export = anything;
}
declare module 'decompress-maybe' {
const anything: any;
export = anything;
}
declare module 'stream-cache' {
const anything: any;
export = anything;
}