queue.js
7.21 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
'use strict';
Object.defineProperty(exports, "__esModule", {
value: true
});
exports.default = function (worker, concurrency) {
var _worker = (0, _wrapAsync2.default)(worker);
return (0, _queue2.default)((items, cb) => {
_worker(items[0], cb);
}, concurrency, 1);
};
var _queue = require('./internal/queue.js');
var _queue2 = _interopRequireDefault(_queue);
var _wrapAsync = require('./internal/wrapAsync.js');
var _wrapAsync2 = _interopRequireDefault(_wrapAsync);
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
module.exports = exports['default'];
/**
* A queue of tasks for the worker function to complete.
* @typedef {Iterable} QueueObject
* @memberOf module:ControlFlow
* @property {Function} length - a function returning the number of items
* waiting to be processed. Invoke with `queue.length()`.
* @property {boolean} started - a boolean indicating whether or not any
* items have been pushed and processed by the queue.
* @property {Function} running - a function returning the number of items
* currently being processed. Invoke with `queue.running()`.
* @property {Function} workersList - a function returning the array of items
* currently being processed. Invoke with `queue.workersList()`.
* @property {Function} idle - a function returning false if there are items
* waiting or being processed, or true if not. Invoke with `queue.idle()`.
* @property {number} concurrency - an integer for determining how many `worker`
* functions should be run in parallel. This property can be changed after a
* `queue` is created to alter the concurrency on-the-fly.
* @property {number} payload - an integer that specifies how many items are
* passed to the worker function at a time. only applies if this is a
* [cargo]{@link module:ControlFlow.cargo} object
* @property {AsyncFunction} push - add a new task to the `queue`. Calls `callback`
* once the `worker` has finished processing the task. Instead of a single task,
* a `tasks` array can be submitted. The respective callback is used for every
* task in the list. Invoke with `queue.push(task, [callback])`,
* @property {AsyncFunction} unshift - add a new task to the front of the `queue`.
* Invoke with `queue.unshift(task, [callback])`.
* @property {AsyncFunction} pushAsync - the same as `q.push`, except this returns
* a promise that rejects if an error occurs.
* @property {AsyncFunction} unshiftAsync - the same as `q.unshift`, except this returns
* a promise that rejects if an error occurs.
* @property {Function} remove - remove items from the queue that match a test
* function. The test function will be passed an object with a `data` property,
* and a `priority` property, if this is a
* [priorityQueue]{@link module:ControlFlow.priorityQueue} object.
* Invoked with `queue.remove(testFn)`, where `testFn` is of the form
* `function ({data, priority}) {}` and returns a Boolean.
* @property {Function} saturated - a function that sets a callback that is
* called when the number of running workers hits the `concurrency` limit, and
* further tasks will be queued. If the callback is omitted, `q.saturated()`
* returns a promise for the next occurrence.
* @property {Function} unsaturated - a function that sets a callback that is
* called when the number of running workers is less than the `concurrency` &
* `buffer` limits, and further tasks will not be queued. If the callback is
* omitted, `q.unsaturated()` returns a promise for the next occurrence.
* @property {number} buffer - A minimum threshold buffer in order to say that
* the `queue` is `unsaturated`.
* @property {Function} empty - a function that sets a callback that is called
* when the last item from the `queue` is given to a `worker`. If the callback
* is omitted, `q.empty()` returns a promise for the next occurrence.
* @property {Function} drain - a function that sets a callback that is called
* when the last item from the `queue` has returned from the `worker`. If the
* callback is omitted, `q.drain()` returns a promise for the next occurrence.
* @property {Function} error - a function that sets a callback that is called
* when a task errors. Has the signature `function(error, task)`. If the
* callback is omitted, `error()` returns a promise that rejects on the next
* error.
* @property {boolean} paused - a boolean for determining whether the queue is
* in a paused state.
* @property {Function} pause - a function that pauses the processing of tasks
* until `resume()` is called. Invoke with `queue.pause()`.
* @property {Function} resume - a function that resumes the processing of
* queued tasks when the queue is paused. Invoke with `queue.resume()`.
* @property {Function} kill - a function that removes the `drain` callback and
* empties remaining tasks from the queue forcing it to go idle. No more tasks
* should be pushed to the queue after calling this function. Invoke with `queue.kill()`.
*
* @example
* const q = async.queue(worker, 2)
* q.push(item1)
* q.push(item2)
* q.push(item3)
* // queues are iterable, spread into an array to inspect
* const items = [...q] // [item1, item2, item3]
* // or use for of
* for (let item of q) {
* console.log(item)
* }
*
* q.drain(() => {
* console.log('all done')
* })
* // or
* await q.drain()
*/
/**
* Creates a `queue` object with the specified `concurrency`. Tasks added to the
* `queue` are processed in parallel (up to the `concurrency` limit). If all
* `worker`s are in progress, the task is queued until one becomes available.
* Once a `worker` completes a `task`, that `task`'s callback is called.
*
* @name queue
* @static
* @memberOf module:ControlFlow
* @method
* @category Control Flow
* @param {AsyncFunction} worker - An async function for processing a queued task.
* If you want to handle errors from an individual task, pass a callback to
* `q.push()`. Invoked with (task, callback).
* @param {number} [concurrency=1] - An `integer` for determining how many
* `worker` functions should be run in parallel. If omitted, the concurrency
* defaults to `1`. If the concurrency is `0`, an error is thrown.
* @returns {module:ControlFlow.QueueObject} A queue object to manage the tasks. Callbacks can be
* attached as certain properties to listen for specific events during the
* lifecycle of the queue.
* @example
*
* // create a queue object with concurrency 2
* var q = async.queue(function(task, callback) {
* console.log('hello ' + task.name);
* callback();
* }, 2);
*
* // assign a callback
* q.drain(function() {
* console.log('all items have been processed');
* });
* // or await the end
* await q.drain()
*
* // assign an error callback
* q.error(function(err, task) {
* console.error('task experienced an error');
* });
*
* // add some items to the queue
* q.push({name: 'foo'}, function(err) {
* console.log('finished processing foo');
* });
* // callback is optional
* q.push({name: 'bar'});
*
* // add some items to the queue (batch-wise)
* q.push([{name: 'baz'},{name: 'bay'},{name: 'bax'}], function(err) {
* console.log('finished processing item');
* });
*
* // add some items to the front of the queue
* q.unshift({name: 'bar'}, function (err) {
* console.log('finished processing bar');
* });
*/