diff --git a/docs/api/plugins.md b/docs/api/plugins.md index 82633244f..9485d196f 100644 --- a/docs/api/plugins.md +++ b/docs/api/plugins.md @@ -153,6 +153,12 @@ event, e.g., `server.on('after', plugins.metrics());`: * `res` {Object} the response obj * `route` {Object} the route obj that serviced the request +The module includes the following plugins to be used with restify's `pre` event: +* `inflightRequestThrottle(options)` - limits the max number of inflight requests + * `options.limit` {Number} the maximum number of inflight requests the server will handle before returning an error + * `options.err` {Error} opts.err A restify error used as a response when the inflight request limit is exceeded + * `options.server` {Object} The restify server that this module will throttle + ## Accept Parser Parses out the `Accept` header, and ensures that the server can respond to what @@ -439,6 +445,31 @@ uniform request distribution. To enable this, you can pass in `options.tokensTable`, which is simply any Object that supports `put` and `get` with a `String` key, and an `Object` value. +## Inflight Request Throttling + +```js +var errors = require('restify-errors'); +var restify = require('restify'); + +var server = restify.createServer(); +const options = { limit: 600, server: server }; +options.res = new errors.InternalServerError(); +server.pre(restify.plugins.inflightRequestThrottle(options)); +``` + +The `inflightRequestThrottle` module allows you to specify an upper limit to +the maximum number of inflight requests your server is able to handle. This +is a simple heuristic for protecting against event loop contention between +requests causing unacceptable latencies. + +The custom error is optional, and allows you to specify your own response +and status code when rejecting incoming requests due to too many inflight +requests. It defaults to `503 ServiceUnavailableError`. + +This plugin should be registered as early as possibly in the middleware stack +using `pre` to avoid performing unnecessary work. + + ## Conditional Request Handler ```js diff --git a/lib/plugins/index.js b/lib/plugins/index.js index b14b7a5df..cd4b020b6 100644 --- a/lib/plugins/index.js +++ b/lib/plugins/index.js @@ -14,6 +14,7 @@ module.exports = { dateParser: require('./date'), fullResponse: require('./fullResponse'), gzipResponse: require('./gzip'), + inflightRequestThrottle: require('./inflightRequestThrottle'), jsonBodyParser: require('./jsonBodyParser'), jsonp: require('./jsonp'), multipartBodyParser: require('./multipartBodyParser'), diff --git a/lib/plugins/inflightRequestThrottle.js b/lib/plugins/inflightRequestThrottle.js new file mode 100644 index 000000000..04096a0fe --- /dev/null +++ b/lib/plugins/inflightRequestThrottle.js @@ -0,0 +1,64 @@ +'use strict'; + +var assert = require('assert-plus'); +var ServiceUnavailableError = require('restify-errors').ServiceUnavailableError; +var defaultResponse = new ServiceUnavailableError('resource exhausted'); + +/** + * inflightRequestThrottle + * + * Place an upper limit on the number of inlfight requests restify will accept. + * For every request that exceeds this threshold, restify will respond with an + * error. This plugin should be registered as early as possible in the + * middleware stack using `pre` to avoid performing unnecessary work. + * + * @param {Object} opts configure this plugin + * @param {Number} opts.limit maximum number of inflight requests the server + * will handle before returning an error + * @param {Error} opts.err A restify error used as a response when the inflight + * request limit is exceeded + * @param {Function} opts.server the instance of the restify server this plugin + * will throttle. + * @returns {Function} middleware to be registered on server.pre + */ +function inflightRequestThrottle (opts) { + + // Scrub input and populate our configuration + assert.object(opts, 'opts'); + assert.number(opts.limit, 'opts.limit'); + assert.object(opts.server, 'opts.server'); + assert.func(opts.server.inflightRequests, 'opts.server.inflightRequests'); + + if (opts.err !== undefined && opts.err !== null) { + assert.ok(opts.err instanceof Error, 'opts.res must be an error'); + assert.optionalNumber(opts.err.statusCode, 'opts.err.statusCode'); + } + + var self = {}; + self._err = opts.err || defaultResponse; + self._limit = opts.limit; + self._server = opts.server; + + function onRequest (req, res, next) { + var inflightRequests = self._server.inflightRequests(); + + if (inflightRequests > self._limit) { + req.log.trace({ + plugin: 'inflightRequestThrottle', + inflightRequests: inflightRequests, + limit: self._limit + }, 'maximum inflight requests exceeded, rejecting request'); + return res.send(self._err); + } + + return next(); + } + + // We need to bind in order to keep our `this` context when passed back + // out of the constructor. + return onRequest; +} + +inflightRequestThrottle.prototype.onRequest = + +module.exports = inflightRequestThrottle; diff --git a/test/plugins/inflightRequestThrottle.test.js b/test/plugins/inflightRequestThrottle.test.js new file mode 100644 index 000000000..ee28b8b63 --- /dev/null +++ b/test/plugins/inflightRequestThrottle.test.js @@ -0,0 +1,123 @@ +'use strict'; + +var assert = require('chai').assert; +var restify = require('../../lib/index.js'); +var restifyClients = require('restify-clients'); +var inflightRequestThrottle = restify.plugins.inflightRequestThrottle; + +function fakeServer(count) { + return { + inflightRequests: function () { + return count; + } + }; +} + +describe('inlfightRequestThrottle', function () { + + it('Unit: Should shed load', function (done) { + var logged = false; + var opts = { server: fakeServer(10), limit: 1 }; + var plugin = inflightRequestThrottle(opts); + function send (body) { + assert(logged, 'Should have emitted a log'); + assert.equal(body.statusCode, 503, 'Defaults to 503 status'); + assert(body instanceof Error, 'Defaults to error body'); + done(); + } + function next () { + assert(false, 'Should not call next'); + done(); + } + function trace () { + logged = true; + } + var log = { trace: trace }; + var fakeReq = { log: log }; + plugin(fakeReq, { send: send }, next); + }); + + it('Unit: Should support custom response', function (done) { + var server = fakeServer(10); + var err = new Error('foo'); + var opts = { server: server, limit: 1, err: err }; + var plugin = inflightRequestThrottle(opts); + function send (body) { + assert.equal(body, err, 'Overrides body'); + done(); + } + function next () { + assert(false, 'Should not call next'); + done(); + } + var fakeReq = { log : { trace: function () {} } }; + plugin(fakeReq, { send: send }, next); + }); + + it('Unit: Should let request through when not under load', function (done) { + var opts = { server: fakeServer(1), limit: 2 }; + var plugin = inflightRequestThrottle(opts); + function send () { + assert(false, 'Should not call send'); + done(); + } + function next () { + assert(true, 'Should call next'); + done(); + } + var fakeReq = { log : { trace: function () {} } }; + plugin(fakeReq, { send: send }, next); + }); + + it('Integration: Should shed load', function (done) { + var server = restify.createServer(); + var client = { + close: function () {} + }; + var isDone = false; + var to; + function finish() { + if (isDone) { + return null; + } + clearTimeout(to); + isDone = true; + client.close(); + server.close(); + return done(); + } + to = setTimeout(finish, 2000); + var err = new Error('foo'); + err.statusCode = 555; + var opts = { server: server, limit: 1, err: err }; + server.pre(inflightRequestThrottle(opts)); + var RES; + server.get('/foo', function (req, res) { + if (RES) { + res.send(999); + } else { + RES = res; + } + }); + server.listen(0, '127.0.0.1', function () { + client = restifyClients.createJsonClient({ + url: 'http://127.0.0.1:' + server.address().port, + retry: false + }); + client.get({ path: '/foo' }, function (e, _, res) { + assert(e === null || e === undefined, + 'First request isnt shed'); + assert.equal(res.statusCode, 200, '200 returned on success'); + finish(); + }); + client.get({ path: '/foo' }, function (e, _, res) { + assert(e, 'Second request is shed'); + assert.equal(e.name, + 'InternalServerError', 'Default err returned'); + assert.equal(res.statusCode, 555, + 'Default shed status code returned'); + RES.send(200); + }); + }); + }); +});