Async refactor in place (#7736)

* feat: allow both callback&and await

* feat: ignore async key

* feat: callbackify and promisify in same file

* Revert "feat: callbackify and promisify in same file"

This reverts commit cea206a9b8.

* feat: no need to store .callbackify

* feat: change getTopics to async

* feat: remove .async

* fix: byScore

* feat: rewrite topics/index and social with async/await

* fix: rewrite topics/data.js

fix issue with async.waterfall, only pass result if its not undefined

* feat: add callbackify to redis/psql

* feat: psql use await

* fix: redis 🌋

* feat: less returns

* feat: more await rewrite

* fix: redis tests

* feat: convert sortedSetAdd

rewrite psql transaction to async/await

* feat: 🐶

* feat: test

* feat: log client and query

* feat: log bind

* feat: more logs

* feat: more logs

* feat: check perform

* feat: dont callbackify transaction

* feat: remove logs

* fix: main functions

* feat: more logs

* fix: increment

* fix: rename

* feat: remove cls

* fix: remove console.log

* feat: add deprecation message to .async usage

* feat: update more dbal methods

* fix: redis :voodoo:

* feat:  fix redis zrem, convert setObject

* feat: upgrade getObject methods

* fix: psql getObjectField

* fix: redis tests

* feat: getObjectKeys

* feat: getObjectValues

* feat: isObjectField

* fix: add missing return

* feat: delObjectField

* feat: incrObjectField

* fix: add missing await

* feat: remove exposed helpers

* feat: list methods

* feat: flush/empty

* feat: delete

* fix: redis delete all

* feat: get/set

* feat: incr/rename

* feat: type

* feat: expire

* feat: setAdd

* feat: setRemove

* feat: isSetMember

* feat: getSetMembers

* feat: setCount, setRemoveRandom

* feat: zcard,zcount

* feat: sortedSetRank

* feat: isSortedSetMember

* feat: zincrby

* feat: sortedSetLex

* feat: processSortedSet

* fix: add mising await

* feat: debug psql

* fix: psql test

* fix: test

* fix: another test

* fix: test fix

* fix: psql tests

* feat: remove logs

* feat: user arrow func

use builtin async promises

* feat: topic bookmarks

* feat: topic.delete

* feat: topic.restore

* feat: topics.purge

* feat: merge

* feat: suggested

* feat: topics/user.js

* feat: topics modules

* feat: topics/follow

* fix: deprecation msg

* feat: fork

* feat: topics/posts

* feat: sorted/recent

* feat: topic/teaser

* feat: topics/tools

* feat: topics/unread

* feat: add back node versions

disable deprecation notice
wrap async controllers in try/catch

* feat: use db directly

* feat: promisify in place

* fix: redis/psql

* feat: deprecation message

logs for psql

* feat: more logs

* feat: more logs

* feat: logs again

* feat: more logs

* fix: call release

* feat: restore travis, remove logs

* fix: loops

* feat: remove .async. usage
This commit is contained in:
Barış Soner Uşaklı
2019-07-09 12:46:49 -04:00
committed by GitHub
parent 43ce5f8af3
commit 805dcd7ca2
73 changed files with 4030 additions and 6110 deletions

View File

@@ -1,7 +1,6 @@
'use strict';
var async = require('async');
var _ = require('lodash');
var validator = require('validator');
@@ -13,114 +12,71 @@ var plugins = require('../plugins');
var utils = require('../../public/src/utils');
module.exports = function (Topics) {
Topics.onNewPostMade = function (postData, callback) {
async.series([
function (next) {
Topics.updateLastPostTime(postData.tid, postData.timestamp, next);
},
function (next) {
Topics.addPostToTopic(postData.tid, postData, next);
},
], callback);
Topics.onNewPostMade = async function (postData) {
await Topics.updateLastPostTime(postData.tid, postData.timestamp);
await Topics.addPostToTopic(postData.tid, postData);
};
Topics.getTopicPosts = function (tid, set, start, stop, uid, reverse, callback) {
async.waterfall([
function (next) {
posts.getPostsFromSet(set, start, stop, uid, reverse, next);
},
function (posts, next) {
Topics.calculatePostIndices(posts, start);
Topics.getTopicPosts = async function (tid, set, start, stop, uid, reverse) {
const postData = await posts.getPostsFromSet(set, start, stop, uid, reverse);
Topics.calculatePostIndices(postData, start);
Topics.addPostData(posts, uid, next);
},
], callback);
return await Topics.addPostData(postData, uid);
};
Topics.addPostData = function (postData, uid, callback) {
Topics.addPostData = async function (postData, uid) {
if (!Array.isArray(postData) || !postData.length) {
return callback(null, []);
return [];
}
var pids = postData.map(post => post && post.pid);
if (!Array.isArray(pids) || !pids.length) {
return callback(null, []);
async function getPostUserData(field, method) {
const uids = _.uniq(postData.filter(p => p && parseInt(p[field], 10) >= 0).map(p => p[field]));
const userData = await method(uids);
return _.zipObject(uids, userData);
}
const [
bookmarks,
voteData,
userData,
editors,
replies,
] = await Promise.all([
posts.hasBookmarked(pids, uid),
posts.getVoteStatusByPostIDs(pids, uid),
getPostUserData('uid', async function (uids) {
return await posts.getUserInfoForPosts(uids, uid);
}),
getPostUserData('editor', async function (uids) {
return await user.getUsersFields(uids, ['uid', 'username', 'userslug']);
}),
getPostReplies(pids, uid),
Topics.addParentPosts(postData),
]);
function getPostUserData(field, method, callback) {
var uidsMap = {};
postData.forEach(function (postObj, i) {
if (postObj) {
postObj.user = postObj.uid ? userData[postObj.uid] : _.clone(userData[postObj.uid]);
postObj.editor = postObj.editor ? editors[postObj.editor] : null;
postObj.bookmarked = bookmarks[i];
postObj.upvoted = voteData.upvotes[i];
postObj.downvoted = voteData.downvotes[i];
postObj.votes = postObj.votes || 0;
postObj.replies = replies[i];
postObj.selfPost = parseInt(uid, 10) > 0 && parseInt(uid, 10) === postObj.uid;
postData.forEach((post) => {
if (post && parseInt(post[field], 10) >= 0) {
uidsMap[post[field]] = 1;
// Username override for guests, if enabled
if (meta.config.allowGuestHandles && postObj.uid === 0 && postObj.handle) {
postObj.user.username = validator.escape(String(postObj.handle));
}
});
const uids = Object.keys(uidsMap);
}
});
async.waterfall([
function (next) {
method(uids, next);
},
function (users, next) {
next(null, _.zipObject(uids, users));
},
], callback);
}
async.waterfall([
function (next) {
async.parallel({
bookmarks: function (next) {
posts.hasBookmarked(pids, uid, next);
},
voteData: function (next) {
posts.getVoteStatusByPostIDs(pids, uid, next);
},
userData: function (next) {
getPostUserData('uid', function (uids, next) {
posts.getUserInfoForPosts(uids, uid, next);
}, next);
},
editors: function (next) {
getPostUserData('editor', function (uids, next) {
user.getUsersFields(uids, ['uid', 'username', 'userslug'], next);
}, next);
},
parents: function (next) {
Topics.addParentPosts(postData, next);
},
replies: function (next) {
getPostReplies(pids, uid, next);
},
}, next);
},
function (results, next) {
postData.forEach(function (postObj, i) {
if (postObj) {
postObj.user = postObj.uid ? results.userData[postObj.uid] : _.clone(results.userData[postObj.uid]);
postObj.editor = postObj.editor ? results.editors[postObj.editor] : null;
postObj.bookmarked = results.bookmarks[i];
postObj.upvoted = results.voteData.upvotes[i];
postObj.downvoted = results.voteData.downvotes[i];
postObj.votes = postObj.votes || 0;
postObj.replies = results.replies[i];
postObj.selfPost = parseInt(uid, 10) > 0 && parseInt(uid, 10) === postObj.uid;
// Username override for guests, if enabled
if (meta.config.allowGuestHandles && postObj.uid === 0 && postObj.handle) {
postObj.user.username = validator.escape(String(postObj.handle));
}
}
});
plugins.fireHook('filter:topics.addPostData', {
posts: postData,
uid: uid,
}, next);
},
function (data, next) {
next(null, data.posts);
},
], callback);
const result = await plugins.fireHook('filter:topics.addPostData', {
posts: postData,
uid: uid,
});
return result.posts;
};
Topics.modifyPostsByPrivilege = function (topicData, topicPrivileges) {
@@ -142,40 +98,31 @@ module.exports = function (Topics) {
});
};
Topics.addParentPosts = function (postData, callback) {
Topics.addParentPosts = async function (postData) {
var parentPids = postData.map(function (postObj) {
return postObj && postObj.hasOwnProperty('toPid') ? parseInt(postObj.toPid, 10) : null;
}).filter(Boolean);
if (!parentPids.length) {
return setImmediate(callback);
return;
}
parentPids = _.uniq(parentPids);
var parentPosts;
async.waterfall([
async.apply(posts.getPostsFields, parentPids, ['uid']),
function (_parentPosts, next) {
parentPosts = _parentPosts;
var parentUids = _.uniq(parentPosts.map(postObj => postObj && postObj.uid));
const parentPosts = await posts.getPostsFields(parentPids, ['uid']);
const parentUids = _.uniq(parentPosts.map(postObj => postObj && postObj.uid));
const userData = await user.getUsersFields(parentUids, ['username']);
user.getUsersFields(parentUids, ['username'], next);
},
function (userData, next) {
var usersMap = {};
userData.forEach(function (user) {
usersMap[user.uid] = user.username;
});
var parents = {};
parentPosts.forEach(function (post, i) {
parents[parentPids[i]] = { username: usersMap[post.uid] };
});
var usersMap = {};
userData.forEach(function (user) {
usersMap[user.uid] = user.username;
});
var parents = {};
parentPosts.forEach(function (post, i) {
parents[parentPids[i]] = { username: usersMap[post.uid] };
});
postData.forEach(function (post) {
post.parent = parents[post.toPid];
});
next();
},
], callback);
postData.forEach(function (post) {
post.parent = parents[post.toPid];
});
};
Topics.calculatePostIndices = function (posts, start) {
@@ -186,251 +133,151 @@ module.exports = function (Topics) {
});
};
Topics.getLatestUndeletedPid = function (tid, callback) {
async.waterfall([
function (next) {
Topics.getLatestUndeletedReply(tid, next);
},
function (pid, next) {
if (pid) {
return callback(null, pid);
}
Topics.getTopicField(tid, 'mainPid', next);
},
function (mainPid, next) {
posts.getPostFields(mainPid, ['pid', 'deleted'], next);
},
function (mainPost, next) {
next(null, mainPost.pid && !mainPost.deleted ? mainPost.pid : null);
},
], callback);
Topics.getLatestUndeletedPid = async function (tid) {
const pid = await Topics.getLatestUndeletedReply(tid);
if (pid) {
return pid;
}
const mainPid = await Topics.getTopicField(tid, 'mainPid');
const mainPost = await posts.getPostFields(mainPid, ['pid', 'deleted']);
return mainPost.pid && !mainPost.deleted ? mainPost.pid : null;
};
Topics.getLatestUndeletedReply = function (tid, callback) {
Topics.getLatestUndeletedReply = async function (tid) {
var isDeleted = false;
var done = false;
var latestPid = null;
var index = 0;
var pids;
async.doWhilst(
function (next) {
async.waterfall([
function (_next) {
db.getSortedSetRevRange('tid:' + tid + ':posts', index, index, _next);
},
function (_pids, _next) {
pids = _pids;
if (!pids.length) {
done = true;
return next();
}
posts.getPostField(pids[0], 'deleted', _next);
},
function (deleted, _next) {
isDeleted = deleted;
if (!isDeleted) {
latestPid = pids[0];
}
index += 1;
_next();
},
], next);
},
function (next) {
next(null, isDeleted && !done);
},
function (err) {
callback(err, parseInt(latestPid, 10));
do {
/* eslint-disable no-await-in-loop */
const pids = await db.getSortedSetRevRange('tid:' + tid + ':posts', index, index);
if (!pids.length) {
return null;
}
);
isDeleted = await posts.getPostField(pids[0], 'deleted');
if (!isDeleted) {
return parseInt(pids[0], 10);
}
index += 1;
} while (isDeleted);
};
Topics.addPostToTopic = function (tid, postData, callback) {
async.waterfall([
function (next) {
Topics.getTopicField(tid, 'mainPid', next);
},
function (mainPid, next) {
if (!parseInt(mainPid, 10)) {
Topics.setTopicField(tid, 'mainPid', postData.pid, next);
} else {
const upvotes = parseInt(postData.upvotes, 10) || 0;
const downvotes = parseInt(postData.downvotes, 10) || 0;
const votes = upvotes - downvotes;
db.sortedSetsAdd([
'tid:' + tid + ':posts', 'tid:' + tid + ':posts:votes',
], [postData.timestamp, votes], postData.pid, next);
}
},
function (next) {
Topics.increasePostCount(tid, next);
},
function (next) {
db.sortedSetIncrBy('tid:' + tid + ':posters', 1, postData.uid, next);
},
function (count, next) {
Topics.updateTeaser(tid, next);
},
], callback);
Topics.addPostToTopic = async function (tid, postData) {
const mainPid = await Topics.getTopicField(tid, 'mainPid');
if (!parseInt(mainPid, 10)) {
await Topics.setTopicField(tid, 'mainPid', postData.pid);
} else {
const upvotes = parseInt(postData.upvotes, 10) || 0;
const downvotes = parseInt(postData.downvotes, 10) || 0;
const votes = upvotes - downvotes;
await db.sortedSetsAdd([
'tid:' + tid + ':posts', 'tid:' + tid + ':posts:votes',
], [postData.timestamp, votes], postData.pid);
}
await Topics.increasePostCount(tid);
await db.sortedSetIncrBy('tid:' + tid + ':posters', 1, postData.uid);
await Topics.updateTeaser(tid);
};
Topics.removePostFromTopic = function (tid, postData, callback) {
async.waterfall([
function (next) {
db.sortedSetsRemove([
'tid:' + tid + ':posts',
'tid:' + tid + ':posts:votes',
], postData.pid, next);
},
function (next) {
Topics.decreasePostCount(tid, next);
},
function (next) {
db.sortedSetIncrBy('tid:' + tid + ':posters', -1, postData.uid, next);
},
function (count, next) {
Topics.updateTeaser(tid, next);
},
], callback);
Topics.removePostFromTopic = async function (tid, postData) {
await db.sortedSetsRemove([
'tid:' + tid + ':posts',
'tid:' + tid + ':posts:votes',
], postData.pid);
await Topics.decreasePostCount(tid);
await db.sortedSetIncrBy('tid:' + tid + ':posters', -1, postData.uid);
await Topics.updateTeaser(tid);
};
Topics.getPids = function (tid, callback) {
async.waterfall([
function (next) {
async.parallel({
mainPid: function (next) {
Topics.getTopicField(tid, 'mainPid', next);
},
pids: function (next) {
db.getSortedSetRange('tid:' + tid + ':posts', 0, -1, next);
},
}, next);
},
function (results, next) {
if (parseInt(results.mainPid, 10)) {
results.pids = [results.mainPid].concat(results.pids);
}
next(null, results.pids);
},
], callback);
Topics.getPids = async function (tid) {
var [mainPid, pids] = await Promise.all([
Topics.getTopicField(tid, 'mainPid'),
db.getSortedSetRange('tid:' + tid + ':posts', 0, -1),
]);
if (parseInt(mainPid, 10)) {
pids = [mainPid].concat(pids);
}
return pids;
};
Topics.increasePostCount = function (tid, callback) {
incrementFieldAndUpdateSortedSet(tid, 'postcount', 1, 'topics:posts', callback);
Topics.increasePostCount = async function (tid) {
incrementFieldAndUpdateSortedSet(tid, 'postcount', 1, 'topics:posts');
};
Topics.decreasePostCount = function (tid, callback) {
incrementFieldAndUpdateSortedSet(tid, 'postcount', -1, 'topics:posts', callback);
Topics.decreasePostCount = async function (tid) {
incrementFieldAndUpdateSortedSet(tid, 'postcount', -1, 'topics:posts');
};
Topics.increaseViewCount = function (tid, callback) {
incrementFieldAndUpdateSortedSet(tid, 'viewcount', 1, 'topics:views', callback);
Topics.increaseViewCount = async function (tid) {
incrementFieldAndUpdateSortedSet(tid, 'viewcount', 1, 'topics:views');
};
function incrementFieldAndUpdateSortedSet(tid, field, by, set, callback) {
callback = callback || function () {};
async.waterfall([
function (next) {
db.incrObjectFieldBy('topic:' + tid, field, by, next);
},
function (value, next) {
db.sortedSetAdd(set, value, tid, next);
},
], callback);
async function incrementFieldAndUpdateSortedSet(tid, field, by, set) {
const value = await db.incrObjectFieldBy('topic:' + tid, field, by);
await db.sortedSetAdd(set, value, tid);
}
Topics.getTitleByPid = function (pid, callback) {
Topics.getTopicFieldByPid('title', pid, callback);
Topics.getTitleByPid = async function (pid) {
return await Topics.getTopicFieldByPid('title', pid);
};
Topics.getTopicFieldByPid = function (field, pid, callback) {
async.waterfall([
function (next) {
posts.getPostField(pid, 'tid', next);
},
function (tid, next) {
Topics.getTopicField(tid, field, next);
},
], callback);
Topics.getTopicFieldByPid = async function (field, pid) {
const tid = await posts.getPostField(pid, 'tid');
return await Topics.getTopicField(tid, field);
};
Topics.getTopicDataByPid = function (pid, callback) {
async.waterfall([
function (next) {
posts.getPostField(pid, 'tid', next);
},
function (tid, next) {
Topics.getTopicData(tid, next);
},
], callback);
Topics.getTopicDataByPid = async function (pid) {
const tid = await posts.getPostField(pid, 'tid');
return await Topics.getTopicData(tid);
};
Topics.getPostCount = function (tid, callback) {
db.getObjectField('topic:' + tid, 'postcount', callback);
Topics.getPostCount = async function (tid) {
return await db.getObjectField('topic:' + tid, 'postcount');
};
function getPostReplies(pids, callerUid, callback) {
var arrayOfReplyPids;
var replyData;
var uniqueUids;
var uniquePids;
async.waterfall([
function (next) {
const keys = pids.map(pid => 'pid:' + pid + ':replies');
db.getSortedSetsMembers(keys, next);
},
function (arrayOfPids, next) {
arrayOfReplyPids = arrayOfPids;
async function getPostReplies(pids, callerUid) {
const keys = pids.map(pid => 'pid:' + pid + ':replies');
const arrayOfReplyPids = await db.getSortedSetsMembers(keys);
uniquePids = _.uniq(_.flatten(arrayOfPids));
const uniquePids = _.uniq(_.flatten(arrayOfReplyPids));
posts.getPostsFields(uniquePids, ['pid', 'uid', 'timestamp'], next);
},
function (_replyData, next) {
replyData = _replyData;
const uids = replyData.map(replyData => replyData && replyData.uid);
const replyData = await posts.getPostsFields(uniquePids, ['pid', 'uid', 'timestamp']);
uniqueUids = _.uniq(uids);
const uids = replyData.map(replyData => replyData && replyData.uid);
user.getUsersWithFields(uniqueUids, ['uid', 'username', 'userslug', 'picture'], callerUid, next);
},
function (userData, next) {
var uidMap = _.zipObject(uniqueUids, userData);
var pidMap = _.zipObject(uniquePids, replyData);
const uniqueUids = _.uniq(uids);
var returnData = arrayOfReplyPids.map(function (replyPids) {
var uidsUsed = {};
var currentData = {
hasMore: false,
users: [],
text: replyPids.length > 1 ? '[[topic:replies_to_this_post, ' + replyPids.length + ']]' : '[[topic:one_reply_to_this_post]]',
count: replyPids.length,
timestampISO: replyPids.length ? utils.toISOString(pidMap[replyPids[0]].timestamp) : undefined,
};
const userData = await user.getUsersWithFields(uniqueUids, ['uid', 'username', 'userslug', 'picture'], callerUid);
replyPids.sort(function (a, b) {
return parseInt(a, 10) - parseInt(b, 10);
});
var uidMap = _.zipObject(uniqueUids, userData);
var pidMap = _.zipObject(uniquePids, replyData);
replyPids.forEach(function (replyPid) {
var replyData = pidMap[replyPid];
if (!uidsUsed[replyData.uid] && currentData.users.length < 6) {
currentData.users.push(uidMap[replyData.uid]);
uidsUsed[replyData.uid] = true;
}
});
var returnData = arrayOfReplyPids.map(function (replyPids) {
var uidsUsed = {};
var currentData = {
hasMore: false,
users: [],
text: replyPids.length > 1 ? '[[topic:replies_to_this_post, ' + replyPids.length + ']]' : '[[topic:one_reply_to_this_post]]',
count: replyPids.length,
timestampISO: replyPids.length ? utils.toISOString(pidMap[replyPids[0]].timestamp) : undefined,
};
if (currentData.users.length > 5) {
currentData.users.pop();
currentData.hasMore = true;
}
replyPids.sort((a, b) => parseInt(a, 10) - parseInt(b, 10));
return currentData;
});
replyPids.forEach(function (replyPid) {
var replyData = pidMap[replyPid];
if (!uidsUsed[replyData.uid] && currentData.users.length < 6) {
currentData.users.push(uidMap[replyData.uid]);
uidsUsed[replyData.uid] = true;
}
});
next(null, returnData);
},
], callback);
if (currentData.users.length > 5) {
currentData.users.pop();
currentData.hasMore = true;
}
return currentData;
});
return returnData;
}
};