mirror of
https://github.com/NodeBB/NodeBB.git
synced 2025-11-13 09:25:45 +01:00
Async refactor in place (#7736)
* feat: allow both callback&and await
* feat: ignore async key
* feat: callbackify and promisify in same file
* Revert "feat: callbackify and promisify in same file"
This reverts commit cea206a9b8.
* feat: no need to store .callbackify
* feat: change getTopics to async
* feat: remove .async
* fix: byScore
* feat: rewrite topics/index and social with async/await
* fix: rewrite topics/data.js
fix issue with async.waterfall, only pass result if its not undefined
* feat: add callbackify to redis/psql
* feat: psql use await
* fix: redis 🌋
* feat: less returns
* feat: more await rewrite
* fix: redis tests
* feat: convert sortedSetAdd
rewrite psql transaction to async/await
* feat: 🐶
* feat: test
* feat: log client and query
* feat: log bind
* feat: more logs
* feat: more logs
* feat: check perform
* feat: dont callbackify transaction
* feat: remove logs
* fix: main functions
* feat: more logs
* fix: increment
* fix: rename
* feat: remove cls
* fix: remove console.log
* feat: add deprecation message to .async usage
* feat: update more dbal methods
* fix: redis :voodoo:
* feat: fix redis zrem, convert setObject
* feat: upgrade getObject methods
* fix: psql getObjectField
* fix: redis tests
* feat: getObjectKeys
* feat: getObjectValues
* feat: isObjectField
* fix: add missing return
* feat: delObjectField
* feat: incrObjectField
* fix: add missing await
* feat: remove exposed helpers
* feat: list methods
* feat: flush/empty
* feat: delete
* fix: redis delete all
* feat: get/set
* feat: incr/rename
* feat: type
* feat: expire
* feat: setAdd
* feat: setRemove
* feat: isSetMember
* feat: getSetMembers
* feat: setCount, setRemoveRandom
* feat: zcard,zcount
* feat: sortedSetRank
* feat: isSortedSetMember
* feat: zincrby
* feat: sortedSetLex
* feat: processSortedSet
* fix: add mising await
* feat: debug psql
* fix: psql test
* fix: test
* fix: another test
* fix: test fix
* fix: psql tests
* feat: remove logs
* feat: user arrow func
use builtin async promises
* feat: topic bookmarks
* feat: topic.delete
* feat: topic.restore
* feat: topics.purge
* feat: merge
* feat: suggested
* feat: topics/user.js
* feat: topics modules
* feat: topics/follow
* fix: deprecation msg
* feat: fork
* feat: topics/posts
* feat: sorted/recent
* feat: topic/teaser
* feat: topics/tools
* feat: topics/unread
* feat: add back node versions
disable deprecation notice
wrap async controllers in try/catch
* feat: use db directly
* feat: promisify in place
* fix: redis/psql
* feat: deprecation message
logs for psql
* feat: more logs
* feat: more logs
* feat: logs again
* feat: more logs
* fix: call release
* feat: restore travis, remove logs
* fix: loops
* feat: remove .async. usage
This commit is contained in:
Barış Soner Uşaklı
committed by
GitHub
GitHub
parent
43ce5f8af3
commit
805dcd7ca2
133
src/batch.js
133
src/batch.js
@@ -2,39 +2,32 @@
|
||||
|
||||
'use strict';
|
||||
|
||||
var async = require('async');
|
||||
const util = require('util');
|
||||
|
||||
var db = require('./database');
|
||||
var utils = require('./utils');
|
||||
|
||||
var DEFAULT_BATCH_SIZE = 100;
|
||||
|
||||
exports.processSortedSet = function (setKey, process, options, callback) {
|
||||
if (typeof options === 'function') {
|
||||
callback = options;
|
||||
options = {};
|
||||
}
|
||||
const sleep = util.promisify(setTimeout);
|
||||
|
||||
callback = typeof callback === 'function' ? callback : function () {};
|
||||
exports.processSortedSet = async function (setKey, process, options) {
|
||||
options = options || {};
|
||||
|
||||
if (typeof process !== 'function') {
|
||||
return callback(new Error('[[error:process-not-a-function]]'));
|
||||
throw new Error('[[error:process-not-a-function]]');
|
||||
}
|
||||
|
||||
// Progress bar handling (upgrade scripts)
|
||||
if (options.progress) {
|
||||
db.sortedSetCard(setKey, function (err, total) {
|
||||
if (!err) {
|
||||
options.progress.total = total;
|
||||
}
|
||||
});
|
||||
options.progress.total = await db.sortedSetCard(setKey);
|
||||
}
|
||||
|
||||
options.batch = options.batch || DEFAULT_BATCH_SIZE;
|
||||
|
||||
// use the fast path if possible
|
||||
if (db.processSortedSet && typeof options.doneIf !== 'function' && !utils.isNumber(options.alwaysStartAt)) {
|
||||
return db.processSortedSet(setKey, process, options, callback);
|
||||
return await db.processSortedSet(setKey, process, options);
|
||||
}
|
||||
|
||||
// custom done condition
|
||||
@@ -42,90 +35,58 @@ exports.processSortedSet = function (setKey, process, options, callback) {
|
||||
|
||||
var start = 0;
|
||||
var stop = options.batch;
|
||||
var done = false;
|
||||
|
||||
async.whilst(
|
||||
function (next) {
|
||||
next(null, !done);
|
||||
},
|
||||
function (next) {
|
||||
async.waterfall([
|
||||
function (next) {
|
||||
db['getSortedSetRange' + (options.withScores ? 'WithScores' : '')](setKey, start, stop, next);
|
||||
},
|
||||
function (ids, _next) {
|
||||
if (!ids.length || options.doneIf(start, stop, ids)) {
|
||||
done = true;
|
||||
return next();
|
||||
}
|
||||
process(ids, function (err) {
|
||||
_next(err);
|
||||
});
|
||||
},
|
||||
function (next) {
|
||||
start += utils.isNumber(options.alwaysStartAt) ? options.alwaysStartAt : options.batch + 1;
|
||||
stop = start + options.batch;
|
||||
|
||||
if (options.interval) {
|
||||
setTimeout(next, options.interval);
|
||||
} else {
|
||||
next();
|
||||
}
|
||||
},
|
||||
], next);
|
||||
},
|
||||
callback
|
||||
);
|
||||
};
|
||||
|
||||
exports.processArray = function (array, process, options, callback) {
|
||||
if (typeof options === 'function') {
|
||||
callback = options;
|
||||
options = {};
|
||||
if (process && process.constructor && process.constructor.name !== 'AsyncFunction') {
|
||||
process = util.promisify(process);
|
||||
}
|
||||
|
||||
callback = typeof callback === 'function' ? callback : function () {};
|
||||
while (true) {
|
||||
/* eslint-disable no-await-in-loop */
|
||||
const ids = await db['getSortedSetRange' + (options.withScores ? 'WithScores' : '')](setKey, start, stop);
|
||||
if (!ids.length || options.doneIf(start, stop, ids)) {
|
||||
return;
|
||||
}
|
||||
await process(ids);
|
||||
|
||||
start += utils.isNumber(options.alwaysStartAt) ? options.alwaysStartAt : options.batch + 1;
|
||||
stop = start + options.batch;
|
||||
|
||||
if (options.interval) {
|
||||
await sleep(options.interval);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
exports.processArray = async function (array, process, options) {
|
||||
options = options || {};
|
||||
|
||||
if (!Array.isArray(array) || !array.length) {
|
||||
return callback();
|
||||
return;
|
||||
}
|
||||
if (typeof process !== 'function') {
|
||||
return callback(new Error('[[error:process-not-a-function]]'));
|
||||
throw new Error('[[error:process-not-a-function]]');
|
||||
}
|
||||
|
||||
var batch = options.batch || DEFAULT_BATCH_SIZE;
|
||||
var start = 0;
|
||||
var done = false;
|
||||
if (process && process.constructor && process.constructor.name !== 'AsyncFunction') {
|
||||
process = util.promisify(process);
|
||||
}
|
||||
|
||||
async.whilst(
|
||||
function (next) {
|
||||
next(null, !done);
|
||||
},
|
||||
function (next) {
|
||||
var currentBatch = array.slice(start, start + batch);
|
||||
if (!currentBatch.length) {
|
||||
done = true;
|
||||
return next();
|
||||
}
|
||||
async.waterfall([
|
||||
function (next) {
|
||||
process(currentBatch, function (err) {
|
||||
next(err);
|
||||
});
|
||||
},
|
||||
function (next) {
|
||||
start += batch;
|
||||
if (options.interval) {
|
||||
setTimeout(next, options.interval);
|
||||
} else {
|
||||
next();
|
||||
}
|
||||
},
|
||||
], next);
|
||||
},
|
||||
function (err) {
|
||||
callback(err);
|
||||
while (true) {
|
||||
var currentBatch = array.slice(start, start + batch);
|
||||
|
||||
if (!currentBatch.length) {
|
||||
return;
|
||||
}
|
||||
);
|
||||
|
||||
await process(currentBatch);
|
||||
|
||||
start += batch;
|
||||
|
||||
if (options.interval) {
|
||||
await sleep(options.interval);
|
||||
}
|
||||
}
|
||||
};
|
||||
require('./promisify')(exports);
|
||||
|
||||
Reference in New Issue
Block a user