diff --git a/example/client.js b/example/client.js index 010ab864b4..376bfe9b9e 100644 --- a/example/client.js +++ b/example/client.js @@ -1,5 +1,5 @@ 'use strict' -const connectPackageRequester = require('@pnpm/server').connectPackageRequester +const connectStoreController = require('@pnpm/server').connectStoreController main() .then(() => console.log('Done')) @@ -9,8 +9,8 @@ main() const port = 5813 const hostname = '127.0.0.1' const registry = 'https://registry.npmjs.org/' - const requestPackage = await connectPackageRequester({port, hostname}) - const response = await requestPackage( + const storeCtrl = await connectStoreController({port, hostname}) + const response = await storeCtrl.requestPackage( {alias: 'is-positive', pref: '1.0.0'}, { downloadPriority: 0, @@ -26,5 +26,5 @@ main() console.log(await response.fetchingManifest) console.log(await response['fetchingFiles']) - requestPackage.close() + storeCtrl.close() } diff --git a/example/server.js b/example/server.js index 1b07078c76..af947be5da 100644 --- a/example/server.js +++ b/example/server.js @@ -1,6 +1,6 @@ 'use strict' const createServer = require('@pnpm/server').createServer -const createPackageRequester = require('@pnpm/package-requester').default +const createStore = require('package-store').default const createResolver = require('@pnpm/npm-resolver').default const createFetcher = require('@pnpm/tarball-fetcher').default @@ -23,15 +23,14 @@ async function main() { strictSsl: true, rawNpmConfig, }) - const requestPackageForServer = createPackageRequester(resolve, fetchers, { + const storeCtrl = await createStore(resolve, fetchers, { networkConcurrency: 1, - storePath: store, - storeIndex: {}, + store, }) const port = 5813 const hostname = '127.0.0.1'; - const server = createServer(requestPackageForServer, { + const server = createServer(storeCtrl, { port, hostname, }) diff --git a/package.json b/package.json index da96fe4852..83663984ed 100644 --- a/package.json +++ b/package.json @@ -62,6 +62,7 @@ "@types/node": "^8.0.57", "@types/uuid": "^3.4.3", "json-socket": "^0.3.0", + "package-store": "^0.11.0", "uuid": "^3.1.0" } } diff --git a/shrinkwrap.yaml b/shrinkwrap.yaml index 61c6b852b6..47b715527d 100644 --- a/shrinkwrap.yaml +++ b/shrinkwrap.yaml @@ -4,6 +4,7 @@ dependencies: '@types/node': 8.0.57 '@types/uuid': 3.4.3 json-socket: 0.3.0 + package-store: 0.11.0 uuid: 3.1.0 devDependencies: '@pnpm/logger': 1.0.0 @@ -19,6 +20,16 @@ devDependencies: tslint: 5.8.0 typescript: 2.6.2 packages: + /@pnpm/check-package/1.0.0: + dependencies: + '@types/load-json-file': 2.0.7 + dint: 2.0.0 + load-json-file: 4.0.0 + dev: false + engines: + node: '>=4' + resolution: + integrity: sha512-rN+hiOjGtxwwus7c/E0xkCs9prBeXzZCSCW/Q3k2WdHGSRNLAepr777iv+i2eqTfz9vXlTF0rLZOGi62ujzwYA== /@pnpm/exec/1.1.2: dependencies: '@types/got': 7.1.5 @@ -31,6 +42,16 @@ packages: node: '>=4' resolution: integrity: sha512-ZNaOAOvl9J45nhrnMcRxT9wd2G6hPzC444nRmo8Wk/GmY6dd8FVN/XG/nEdHPjcYPyyrW2UA3McNi+klB9oBHw== + /@pnpm/fs-locker/1.0.0: + dependencies: + '@types/node': 8.0.57 + mkdirp-promise: 5.0.1 + proper-lockfile: 2.0.1 + dev: false + engines: + node: '>=4' + resolution: + integrity: sha512-JRIhC8lBFJjuaDnTMgCfCaO8IEa04JHCrNMW4RAjNrCeWBdcw2sczxp7qjqZPDc90z1VgUMlkpkGzXePcS9flQ== /@pnpm/logger/1.0.0: dependencies: '@types/node': 8.0.57 @@ -93,6 +114,43 @@ packages: '@pnpm/logger': ^1.0.0 resolution: integrity: sha512-x1h+ezzikqrwW3pq+vOudeQTowjkLG8dvr/4CzDyqpvmJZgZ6DLO+3h8Y0NwzKCPcnDQF7OUwy1VQYXYtgb5mw== + /@pnpm/package-requester/0.5.0: + dependencies: + '@pnpm/check-package': 1.0.0 + '@pnpm/pkgid-to-filename': 1.0.0 + '@pnpm/types': 1.3.0 + '@types/load-json-file': 2.0.7 + '@types/mz': 0.0.32 + '@types/p-queue': 1.1.0 + '@types/write-json-file': 2.2.1 + load-json-file: 4.0.0 + mkdirp-promise: 5.0.1 + mz: 2.7.0 + p-limit: 1.1.0 + p-queue: 2.3.0 + path-exists: 3.0.0 + read-package-json: 2.0.12 + rename-overwrite: 1.0.0 + rimraf-then: 1.0.1 + symlink-dir: 1.1.0 + unpack-stream: 3.0.0 + util.promisify: 1.0.0 + write-json-file: 2.3.0 + dev: false + engines: + node: '>=4' + peerDependencies: + '@pnpm/logger': ^1.0.0 + resolution: + integrity: sha512-cqT435oyGBhQGMJhOJoexO+A7nucyuUf3fzznaTxSo1MMLjku8shntdcdY/t6rjCi0tHKIKqZEAHoWGX0/c9NQ== + /@pnpm/pkgid-to-filename/1.0.0: + dependencies: + normalize-path: 2.1.1 + dev: false + engines: + node: '>=4' + resolution: + integrity: sha512-HnWDy3ecIKBsbRR9bwf+GJpV1CWLQ9Injj7eLSEQIsXQ+CLXU0h1NaxBCW7zcJm+2kbwnm0E1aFdFIEJXhHiTg== /@pnpm/tarball-fetcher/0.3.1: dependencies: '@types/mem': 1.1.2 @@ -117,6 +175,10 @@ packages: /@pnpm/types/1.2.1: resolution: integrity: sha512-gvOa3EP2z7O+IFiYsRBB+SBdnyE+SHTVFM5bPr2WGHhPhrScMUJ2jqgzlC79FgRP9c7uoe8Wbp3LYClEDh/U6A== + /@pnpm/types/1.3.0: + dev: false + resolution: + integrity: sha512-dhW0tHPWT19r1lwgv2RMnt31oC6PcsNeGcUlIDYRmC+J7g7Rcm19tZ0xmIAqh9R+U2Uw3PecXsuz+jkC/r5h+A== /@sindresorhus/is/0.6.0: dev: true engines: @@ -2423,6 +2485,14 @@ packages: node: '>=4' resolution: integrity: sha1-TgHYXCPaGe1xpK+6MZvbTZTIXgA= + /p-filter/1.0.0: + dependencies: + p-map: 1.2.0 + dev: false + engines: + node: '>=4' + resolution: + integrity: sha1-Yp0xcVAgnI/VCLoTdxPvS7kg6ds= /p-finally/1.0.0: dev: true engines: @@ -2524,6 +2594,32 @@ packages: '@pnpm/logger': ^1.0.0 resolution: integrity: sha512-42OC/ytnVm+Tpzl3L8hdEAy+oAHd2wLcu85fnAL/4CEe7Zq/PjWHmrfmeJ3+YC6isMDD0avcIHaEhsbhEnyhvA== + /package-store/0.11.0: + dependencies: + '@pnpm/fs-locker': 1.0.0 + '@pnpm/package-requester': 0.5.0 + '@pnpm/types': 1.3.0 + '@types/load-json-file': 2.0.7 + '@types/node': 8.0.57 + '@types/ramda': 0.25.8 + drive-by-path: 1.0.0 + encode-registry: 1.1.0 + load-json-file: 4.0.0 + os-homedir: 1.0.2 + p-filter: 1.0.0 + p-limit: 1.1.0 + path-absolute: 1.0.0 + path-exists: 3.0.0 + ramda: 0.25.0 + rimraf-then: 1.0.1 + write-json-file: 2.3.0 + dev: false + engines: + node: '>=4' + peerDependencies: + '@pnpm/logger': ^1.0.0 + resolution: + integrity: sha512-sfw2GEpX7ZNfpOa0Ihy2d9ZpPROctLQsPDeUBJYMaer/SAIzdJK3B0XBFKWvvJX2jbOTrS+bBBWEw4KDFszXHQ== /pako/0.2.9: resolution: integrity: sha1-8/dSL073gjSNqBYbrZ7P1Rv4OnU= @@ -2726,6 +2822,15 @@ packages: dev: true resolution: integrity: sha1-kRgvkckkhplXQPoF4NqUKsmGvvo= + /proper-lockfile/2.0.1: + dependencies: + graceful-fs: 4.1.11 + retry: 0.10.1 + dev: false + engines: + node: '>=4.0.0' + resolution: + integrity: sha1-FZ+wYZPTIAP0s2kd0uwaY0qoDR0= /pseudomap/1.0.2: dev: true resolution: @@ -3005,7 +3110,6 @@ packages: resolution: integrity: sha1-8ej0YeQGS6Oegq883CqMiT0HZ1k= /retry/0.10.1: - dev: true resolution: integrity: sha1-52OI0heZLCUnUCQdPTlW/tmNj/Q= /rimraf-then/1.0.1: @@ -3603,7 +3707,6 @@ packages: decompress-maybe: 1.0.0 ssri: 5.0.0 tar-fs: 1.16.0 - dev: true engines: node: '>=4' resolution: @@ -3798,6 +3901,7 @@ specifiers: mos: ^2.0.0-alpha.3 mos-plugin-readme: ^1.0.4 package-preview: ^1.0.1 + package-store: ^0.11.0 rimraf: ^2.6.2 tape: ^4.8.0 ts-node: ^3.3.0 diff --git a/src/connectPackageRequester.ts b/src/connectStoreController.ts similarity index 64% rename from src/connectPackageRequester.ts rename to src/connectStoreController.ts index cf979419e9..533069d911 100644 --- a/src/connectPackageRequester.ts +++ b/src/connectStoreController.ts @@ -7,16 +7,17 @@ import { } from '@pnpm/package-requester' import JsonSocket = require('json-socket') import net = require('net') +import {StoreController} from 'package-store' import uuid = require('uuid') export default function ( - opts: { + initOpts: { port: number, hostname?: string, }, -): Promise void }> { +): Promise void }> { const socket = new JsonSocket(new net.Socket()); - socket.connect(opts.port, opts.hostname || '127.0.0.1') + socket.connect(initOpts.port, initOpts.hostname || '127.0.0.1') return new Promise((resolve, reject) => { socket.on('connect', () => { @@ -26,9 +27,31 @@ export default function ( waiters.resolve(message.action, message.body) }) - const result = requestPackage.bind(null, socket, waiters) - result['close'] = () => socket.end() // tslint:disable-line - resolve(result) + resolve({ + close: () => socket.end(), + prune: async () => { + socket.sendMessage({ + action: 'prune', + }, (err) => err && console.error(err)) + }, + requestPackage: requestPackage.bind(null, socket, waiters), + saveState: async () => { + socket.sendMessage({ + action: 'saveState', + }, (err) => err && console.error(err)) + }, + saveStateAndClose: async () => { + socket.sendMessage({ + action: 'saveStateAndClose', + }, (err) => err && console.error(err)) + }, + updateConnections: async (prefix: string, opts: {addDependencies: string[], removeDependencies: string[], prune: boolean}) => { + socket.sendMessage({ + action: 'updateConnections', + args: [prefix, opts], + }, (err) => err && console.error(err)) + }, + }) }) }) } @@ -89,9 +112,9 @@ function requestPackage ( }) socket.sendMessage({ + action: 'requestPackage', + args: [wantedDependency, options], msgId, - options, - wantedDependency, }, (err) => err && console.error(err)) return response diff --git a/src/createServer.ts b/src/createServer.ts index d258f32f86..f59c7157db 100644 --- a/src/createServer.ts +++ b/src/createServer.ts @@ -1,9 +1,14 @@ -import {RequestPackageFunction} from '@pnpm/package-requester' +import { + RequestPackageFunction, + RequestPackageOptions, + WantedDependency, +} from '@pnpm/package-requester' import JsonSocket = require('json-socket') import net = require('net') +import {StoreController} from 'package-store' export default function ( - requestPackage: RequestPackageFunction, + store: StoreController, opts: { port: number, hostname?: string, @@ -13,27 +18,27 @@ export default function ( server.listen(opts.port, opts.hostname); server.on('connection', (socket) => { const jsonSocket = new JsonSocket(socket) + const requestPackage = requestPackageWithCtx.bind(null, {jsonSocket, store}) + jsonSocket.on('message', async (message) => { - const packageResponse = await requestPackage(message.wantedDependency, message.options) - jsonSocket.sendMessage({ - action: `packageResponse:${message.msgId}`, - body: packageResponse, - }, (err) => err && console.error(err)) - - if (!packageResponse.isLocal) { - packageResponse.fetchingFiles.then((packageFilesResponse) => { - jsonSocket.sendMessage({ - action: `packageFilesResponse:${message.msgId}`, - body: packageFilesResponse, - }, (err) => err && console.error(err)) - }) - - packageResponse.fetchingManifest.then((manifestResponse) => { - jsonSocket.sendMessage({ - action: `manifestResponse:${message.msgId}`, - body: manifestResponse, - }, (err) => err && console.error(err)) - }) + switch (message.action) { + case 'requestPackage': { + await requestPackage(message.msgId, message.args[0], message.args[1]) + return + } + case 'prune': { + await store.prune() + return + } + case 'updateConnections': { + await store.updateConnections(message.args[0], message.args[1]) + return + } + case 'saveState': + case 'saveStateAndClose': { + await store.saveState() + return + } } }) }) @@ -42,3 +47,35 @@ export default function ( close: () => server.close(), } } + +async function requestPackageWithCtx ( + ctx: { + jsonSocket: JsonSocket, + store: StoreController, + }, + msgId: string, + wantedDependency: WantedDependency, + options: RequestPackageOptions, +) { + const packageResponse = await ctx.store.requestPackage(wantedDependency, options) + ctx.jsonSocket.sendMessage({ + action: `packageResponse:${msgId}`, + body: packageResponse, + }, (err) => err && console.error(err)) + + if (!packageResponse.isLocal) { + packageResponse.fetchingFiles.then((packageFilesResponse) => { + ctx.jsonSocket.sendMessage({ + action: `packageFilesResponse:${msgId}`, + body: packageFilesResponse, + }, (err) => err && console.error(err)) + }) + + packageResponse.fetchingManifest.then((manifestResponse) => { + ctx.jsonSocket.sendMessage({ + action: `manifestResponse:${msgId}`, + body: manifestResponse, + }, (err) => err && console.error(err)) + }) + } +} diff --git a/src/index.ts b/src/index.ts index a159277e90..f73556193e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,7 +1,7 @@ -import connectPackageRequester from './connectPackageRequester' +import connectStoreController from './connectStoreController' import createServer from './createServer' export { createServer, - connectPackageRequester, + connectStoreController, } diff --git a/test/index.ts b/test/index.ts index 415ad908a6..9ad5114315 100644 --- a/test/index.ts +++ b/test/index.ts @@ -1,13 +1,14 @@ import test = require('tape') import { createServer, - connectPackageRequester, + connectStoreController, } from '@pnpm/server' -import createPackageRequester, { +import { PackageFilesResponse, } from '@pnpm/package-requester' import createResolver from '@pnpm/npm-resolver' import createFetcher from '@pnpm/tarball-fetcher' +import createStore from 'package-store' import net = require('net') import JsonSocket = require('json-socket') @@ -26,20 +27,21 @@ test('server', async t => { strictSsl: true, rawNpmConfig, }) - const requestPackageForServer = createPackageRequester(resolve, fetchers, { + const storeCtrlForServer = await createStore(resolve, fetchers, { networkConcurrency: 1, - storePath: store, - storeIndex: {}, + store: store, + locks: undefined, + lockStaleDuration: 100, }) const port = 5813 const hostname = '127.0.0.1'; - const server = createServer(requestPackageForServer, { + const server = createServer(storeCtrlForServer, { port, hostname, }) - const requestPackage = await connectPackageRequester({port, hostname}) - const response = await requestPackage( + const storeCtrl = await connectStoreController({port, hostname}) + const response = await storeCtrl.requestPackage( {alias: 'is-positive', pref: '1.0.0'}, { downloadPriority: 0, @@ -65,6 +67,6 @@ test('server', async t => { await response['finishing'] server.close() - requestPackage.close() + storeCtrl.close() t.end() })