Skip to content

Commit

Permalink
Initial code drop.
Browse files Browse the repository at this point in the history
  • Loading branch information
ricmoo committed Feb 24, 2016
0 parents commit d1c1825
Show file tree
Hide file tree
Showing 5 changed files with 324 additions and 0 deletions.
165 changes: 165 additions & 0 deletions README.md
@@ -0,0 +1,165 @@
Promise Rationing
=================

Promises are quite useful, but `Promise.all()` executes all promises in parallel, which is fine most of the time, but sometimes there are scarce resources that need to rationed (such as file descriptors, database connections, et cetera).

The `promise-rationing` library provides a familiar API to execute promise-compatible functions as promises, while limiting the number of concurrent functions running.

## Promise Compatible Function

A promise requires a function which takes in two functions, a `resolve` function and a `reject` function.

The returned object from `.all()` is a completely bona fide `Promise`, which can then be `.then`-ed and such.

**NOTE:** Why must we pass in a function and not a promise? Once a promise is created it begins executing immediately, which is what is trying to be prevented.

Example: Waiters
----------------

Try this out. There are never more than 4 waiters *doing there thing* at a time, and once one finishes, the next gets its turn.

```javascript
var promiseRationing = require('./index.js');

function waiter(i) {
return function(resolve, reject) {
console.log('Start ' + i);
var t0 = (new Date()).getTime()
setTimeout(function () {
resolve('r' + i);
console.log('End ' + i + ' after ' + ((new Date()).getTime() - t0) + 'ms');
}, 5000 * Math.random());
};
}

var waiters = [];
for (var i = 0; i < 10; i++) {
waiters.push(waiter(i));
}

var promise = promiseRationing.all(waiters, 4);

promise.then(function (values) {
console.log("Resolve", values);
}, function (error) {
console.log("Error", error);
});
```

### The output
```
/Users/promise-rationing> node test-waiters.js
Start 0
Start 1
Start 2
Start 3
End 0 after 904ms
Start 4
End 2 after 1791ms
Start 5
End 3 after 3039ms
Start 6
End 1 after 4382ms
Start 7
End 4 after 3565ms
Start 8
End 5 after 3095ms
Start 9
End 7 after 2484ms
End 8 after 2425ms
End 9 after 2200ms
End 6 after 4650ms
Resolve [ 'r0', 'r1', 'r2', 'r3', 'r4', 'r5', 'r6', 'r7', 'r8', 'r9' ]
```

Example: Processing MANY files
------------------------------

If you have a directory of 1000's of files, each of which you wish to process, simply creating a promise for each file would likely result in your operating system running out of file descriptors, as each promise begins immediately.

To handle this situation, we queue up all the files for processing, but limit to 20 functions at a time.

```javascript
var fs = require('fs');
var util = require('util');

var promiseRationing = require('./index.js');

// This function will return a promize-compatible function that will return the filesize
function badFileSizeFunction(path) {
return function(resolve, reject) {
fs.readFile(path, function(error, data) {
if (error) {
reject(error.message);

} else {
console.log(util.format('Read "%s"... (%d bytes)', path, data.length));
resolve(data.length);
}
});
}
}

// Find all the files in /etc that end in .conf
var filenames = fs.readdirSync('/etc/');
var readers = [];
for (var i = 0; i < filenames.length; i++) {
var path = '/etc/' + filenames[i];
if (path.match(/\.conf$/)) {
readers.push(badFileSizeFunction(path));
}
}

// Do our promise magic
var promise = promiseRationing.all(readers, 4);

promise.then(function (values) {
console.log("Resolve", values);
}, function (error) {
console.log("Error", error);
});
```

### The Output

```
/Users/promise-rationing> node test-count.js
Read "/etc/asl.conf"... (1051 bytes)
Read "/etc/autofs.conf"... (1935 bytes)
Read "/etc/dnsextd.conf"... (2378 bytes)
Read "/etc/ftpd.conf"... (54 bytes)
Read "/etc/kern_loader.conf"... (0 bytes)
Read "/etc/newsyslog.conf"... (1318 bytes)
Read "/etc/man.conf"... (4574 bytes)
Read "/etc/iscsid.conf"... (296 bytes)
Read "/etc/nfs.conf"... (43 bytes)
Read "/etc/notify.conf"... (303 bytes)
Read "/etc/ntp-restrict.conf"... (414 bytes)
Read "/etc/ntp.conf"... (23 bytes)
Read "/etc/ntp_opendirectory.conf"... (23 bytes)
Read "/etc/pf.conf"... (1027 bytes)
Read "/etc/resolv.conf"... (241 bytes)
Read "/etc/rtadvd.conf"... (891 bytes)
Read "/etc/syslog.conf"... (96 bytes)
Resolve [ 1051, 1935, 2378, 54, 296, 0, 4574, 1318, 43, 303, 414, 23,
23, 1027, 241, 891, 96 ]
```


Testing
-------

There are the two demo files provided above, but better testing must be added at some point...


Donations
---------

Obviously, it's all licensed under the MIT license, so use it as you wish; but if you'd like to buy me a coffee, I won't complain. =)

- Bitcoin - `18UDs4qV1shu2CgTS2tKojhCtM69kpnWg9`
- Dogecoin - `DQe5fTQWzKsd2hBpEoRh8ubmq5eTJ5HjXz`
- Testnet3 - `mmr3CkfqSjbgbzzLfuhZw7sEH5VbEi2vJt`

61 changes: 61 additions & 0 deletions index.js
@@ -0,0 +1,61 @@
function all(iterable, limit) {

// Must be able to run at least one at a time
if (!limit || limit < 1 || typeof(limit) !== 'number') {
return Promise.reject('limit must be greater than 0');
}

limit = parseInt(limit);


// We launch all the promises, stalling them with the pending functions
var pending = [];
var error = false;
var queueMe = function(func) {
return new Promise(function(resolve, reject) {
new Promise(function(blockResolve, neverReject) {
pending.push(blockResolve)

}).then(function() {
if (error) { return; }

// after unblocking, we call the original function
func(function() {
if (error) { return; }

// Provide the original function with its arguments
resolve.apply(this, Array.prototype.slice.apply(arguments));

// Nudge next promise to begin
if (pending.length) {
pending.shift()();
}
}, function () {
// Handle errors
reject.apply(this, Array.apply(null, arguments));
error = true;
});
});
});
}

// Create and queue ever callback as a stalled promise
var callbacks = [];
for (var k in iterable) {
callbacks.push(queueMe(iterable[k]));
}

// Bootstrap the initial limit promises
for (var i = 0; i < limit; i++) {
if (pending.length === 0) { break; }
pending.shift()();
}

// Wait for all of them to return
return Promise.all(callbacks);
}

module.exports = {
all: all,
}

28 changes: 28 additions & 0 deletions package.json
@@ -0,0 +1,28 @@
{
"name": "promise-rationing",
"version": "0.0.1",
"description": "Promise-like interface to limit how many concurrent promises are executed.",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"repository": {
"type": "git",
"url": "https://github.com/ricmoo/promise-rationing"
},
"keywords": [
"promise",
"limit",
"limited",
"cap",
"ration",
"rationing",
"concurrent"
],
"author": "Richard Moore <me@ricmoo.com>",
"license": "MIT",
"bugs": {
"url": "https://github.com/ricmoo/promise-rationing/issues"
},
"homepage": "https://github.com/ricmoo/promise-rationing"
}
39 changes: 39 additions & 0 deletions test-count.js
@@ -0,0 +1,39 @@
var fs = require('fs');
var util = require('util');

var promiseRationing = require('./index.js');

// This function will return a promize-compatible function that will return the filesize
function badFileSizeFunction(path) {
return function(resolve, reject) {
fs.readFile(path, function(error, data) {
if (error) {
reject(error.message);

} else {
console.log(util.format('Read "%s"... (%d bytes)', path, data.length));
resolve(data.length);
}
});
}
}

// Find all the files in /etc that end in .conf
var filenames = fs.readdirSync('/etc/');
var readers = [];
for (var i = 0; i < filenames.length; i++) {
var path = '/etc/' + filenames[i];
if (path.match(/\.conf$/)) {
readers.push(badFileSizeFunction(path));
}
}

// Do our promise magic
var promise = promiseRationing.all(readers, 4);

promise.then(function (values) {
console.log("Resolve", values);
}, function (error) {
console.log("Error", error);
});

31 changes: 31 additions & 0 deletions test-waiters.js
@@ -0,0 +1,31 @@
var promiseRationing = require('./index.js');

function waiter(i) {
return function(resolve, reject) {
console.log('Start ' + i);
var t0 = (new Date()).getTime()
setTimeout(function () {
//if (i == 3) {
// reject('r' + i);
//} else {
resolve('r' + i);
//}
console.log('End ' + i + ' after ' + ((new Date()).getTime() - t0) + 'ms');
}, 5000 * Math.random());
};
}

var waiters = [];
for (var i = 0; i < 10; i++) {
waiters.push(waiter(i));
}

var promise = promiseRationing.all(waiters, 4);
console.log(promise);

promise.then(function (values) {
console.log("Resolve", values);
}, function (error) {
console.log("Error", error);
});

0 comments on commit d1c1825

Please sign in to comment.