perf: update old upgrade scripts to use bulkSet/Add

fix a missing await

src/upgrades/1.10.2/upgrade_bans_to_hashes.js

@@ -11,14 +11,23 @@ module.exports = {
 	method: async function () {
 		const { progress } = this;
 
+		progress.total = await db.sortedSetCard('users:joindate');
+
 		await batch.processSortedSet('users:joindate', async (uids) => {
-			for (const uid of uids) {
-				progress.incr();
-				const [bans, reasons, userData] = await Promise.all([
-					db.getSortedSetRevRangeWithScores(`uid:${uid}:bans`, 0, -1),
-					db.getSortedSetRevRangeWithScores(`banned:${uid}:reasons`, 0, -1),
-					db.getObjectFields(`user:${uid}`, ['banned', 'banned:expire', 'joindate', 'lastposttime', 'lastonline']),
-				]);
+			progress.incr(uids.length);
+			const [allUserData, allBans] = await Promise.all([
+				db.getObjectsFields(
+					uids.map(uid => `user:${uid}`),
+					['banned', 'banned:expire', 'joindate', 'lastposttime', 'lastonline'],
+				),
+				db.getSortedSetsMembersWithScores(
+					uids.map(uid => `uid:${uid}:bans`)
+				),
+			]);
+
+			await Promise.all(uids.map(async (uid, index) => {
+				const userData = allUserData[index];
+				const bans = allBans[index] || [];
 
 				// has no history, but is banned, create plain object with just uid and timestmap
 				if (!bans.length && parseInt(userData.banned, 10)) {
@@ -31,6 +40,7 @@ module.exports = {
 					const banKey = `uid:${uid}:ban:${banTimestamp}`;
 					await addBan(uid, banKey, { uid: uid, timestamp: banTimestamp });
 				} else if (bans.length) {
+					const reasons = await db.getSortedSetRevRangeWithScores(`banned:${uid}:reasons`, 0, -1);
 					// process ban history
 					for (const ban of bans) {
 						const reasonData = reasons.find(reasonData => reasonData.score === ban.score);
@@ -46,14 +56,16 @@ module.exports = {
 						await addBan(uid, banKey, data);
 					}
 				}
-			}
+			}));
 		}, {
-			progress: this.progress,
+			batch: 500,
 		});
 	},
 };
 
 async function addBan(uid, key, data) {
-	await db.setObject(key, data);
-	await db.sortedSetAdd(`uid:${uid}:bans:timestamp`, data.timestamp, key);
+	await Promise.all([
+		db.setObject(key, data),
+		db.sortedSetAdd(`uid:${uid}:bans:timestamp`, data.timestamp, key),
+	]);
 }

src/upgrades/1.10.2/username_email_history.js

@@ -11,27 +11,34 @@ module.exports = {
 	method: async function () {
 		const { progress } = this;
 
-		await batch.processSortedSet('users:joindate', async (uids) => {
-			async function updateHistory(uid, set, fieldName) {
-				const count = await db.sortedSetCard(set);
-				if (count <= 0) {
-					// User has not changed their username/email before, record original username
-					const userData = await user.getUserFields(uid, [fieldName, 'joindate']);
-					if (userData && userData.joindate && userData[fieldName]) {
-						await db.sortedSetAdd(set, userData.joindate, [userData[fieldName], userData.joindate].join(':'));
-					}
-				}
-			}
+		progress.total = await db.sortedSetCard('users:joindate');
 
-			await Promise.all(uids.map(async (uid) => {
-				await Promise.all([
-					updateHistory(uid, `user:${uid}:usernames`, 'username'),
-					updateHistory(uid, `user:${uid}:emails`, 'email'),
-				]);
-				progress.incr();
-			}));
+		await batch.processSortedSet('users:joindate', async (uids) => {
+			const [usernameHistory, emailHistory, userData] = await Promise.all([
+				db.sortedSetsCard(uids.map(uid => `user:${uid}:usernames`)),
+				db.sortedSetsCard(uids.map(uid => `user:${uid}:emails`)),
+				user.getUsersFields(uids, ['uid', 'username', 'email', 'joindate']),
+			]);
+
+			const bulkAdd = [];
+			userData.forEach((data, index) => {
+				const thisUsernameHistory = usernameHistory[index];
+				const thisEmailHistory = emailHistory[index];
+				if (thisUsernameHistory <= 0 && data && data.joindate && data.username) {
+					bulkAdd.push([
+						`user:${data.uid}:usernames`, data.joindate, [data.username, data.joindate].join(':'),
+					]);
+				}
+				if (thisEmailHistory <= 0 && data && data.joindate && data.email) {
+					bulkAdd.push([
+						`user:${data.uid}:emails`, data.joindate, [data.email, data.joindate].join(':'),
+					]);
+				}
+			});
+			await db.sortedSetAddBulk(bulkAdd);
+			progress.incr(uids.length);
 		}, {
-			progress: this.progress,
+			batch: 500,
 		});
 	},
 };

src/upgrades/1.12.1/clear_username_email_history.js

@@ -1,45 +1,32 @@
 'use strict';
 
-const async = require('async');
+
 const db = require('../../database');
 const user = require('../../user');
+const batch = require('../../batch');
 
 module.exports = {
 	name: 'Delete username email history for deleted users',
 	timestamp: Date.UTC(2019, 2, 25),
-	method: function (callback) {
+	method: async function () {
 		const { progress } = this;
-		let currentUid = 1;
-		db.getObjectField('global', 'nextUid', (err, nextUid) => {
-			if (err) {
-				return callback(err);
-			}
-			progress.total = nextUid;
-			async.whilst((next) => {
-				next(null, currentUid < nextUid);
-			},
-			(next) => {
-				progress.incr();
-				user.exists(currentUid, (err, exists) => {
-					if (err) {
-						return next(err);
-					}
-					if (exists) {
-						currentUid += 1;
-						return next();
-					}
-					db.deleteAll([`user:${currentUid}:usernames`, `user:${currentUid}:emails`], (err) => {
-						if (err) {
-							return next(err);
-						}
-						currentUid += 1;
-						next();
-					});
-				});
-			},
-			(err) => {
-				callback(err);
-			});
+
+		progress.total = await db.getObjectField('global', 'nextUid');
+		const allUids = [];
+		for (let i = 1; i < progress.total; i += 1) {
+			allUids.push(i);
+		}
+		await batch.processArray(allUids, async (uids) => {
+			const exists = await user.exists(uids);
+			const missingUids = uids.filter((uid, index) => !exists[index]);
+			const keysToDelete = [
+				...missingUids.map(uid => `user:${uid}:usernames`),
+				...missingUids.map(uid => `user:${uid}:emails`),
+			];
+			await db.deleteAll(keysToDelete);
+			progress.incr(uids.length);
+		}, {
+			batch: 500,
 		});
 	},
 };

src/upgrades/1.12.1/moderation_notes_refactor.js

@@ -12,10 +12,12 @@ module.exports = {
 		const { progress } = this;
 
 		await batch.processSortedSet('users:joindate', async (uids) => {
-			await Promise.all(uids.map(async (uid) => {
-				progress.incr();
-
-				const notes = await db.getSortedSetRevRange(`uid:${uid}:moderation:notes`, 0, -1);
+			progress.incr(uids.length);
+			const allNotes = await db.getSortedSetsMembers(
+				uids.map(uid => `uid:${uid}:moderation:notes`)
+			);
+			await Promise.all(uids.map(async (uid, index) => {
+				const notes = allNotes[index];
 				for (const note of notes) {
 					const noteData = JSON.parse(note);
 					noteData.timestamp = noteData.timestamp || Date.now();

src/upgrades/1.13.0/clean_post_topic_hash.js

@@ -8,6 +8,7 @@ module.exports = {
 	timestamp: Date.UTC(2019, 9, 7),
 	method: async function () {
 		const { progress } = this;
+		progress.total = await db.sortedSetCard('posts:pid') + await db.sortedSetCard('topics:tid');
 		await cleanPost(progress);
 		await cleanTopic(progress);
 	},
@@ -51,7 +52,6 @@ async function cleanPost(progress) {
 		}));
 	}, {
 		batch: 500,
-		progress: progress,
 	});
 }
 
@@ -90,6 +90,5 @@ async function cleanTopic(progress) {
 		}));
 	}, {
 		batch: 500,
-		progress: progress,
 	});
 }

src/upgrades/1.6.2/topics_lastposttime_zset.js

@@ -1,29 +1,30 @@
 'use strict';
 
-const async = require('async');
-
 const db = require('../../database');
+const batch = require('../../batch');
 
 module.exports = {
 	name: 'New sorted set cid:<cid>:tids:lastposttime',
 	timestamp: Date.UTC(2017, 9, 30),
-	method: function (callback) {
+	method: async function () {
 		const { progress } = this;
+		progress.total = await db.sortedSetCard('topics:tid');
 
-		require('../../batch').processSortedSet('topics:tid', (tids, next) => {
-			async.eachSeries(tids, (tid, next) => {
-				db.getObjectFields(`topic:${tid}`, ['cid', 'timestamp', 'lastposttime'], (err, topicData) => {
-					if (err || !topicData) {
-						return next(err);
-					}
-					progress.incr();
-
-					const timestamp = topicData.lastposttime || topicData.timestamp || Date.now();
-					db.sortedSetAdd(`cid:${topicData.cid}:tids:lastposttime`, timestamp, tid, next);
-				}, next);
-			}, next);
+		await batch.processSortedSet('topics:tid', async (tids) => {
+			const topicData = await db.getObjectsFields(
+				tids.map(tid => `topic:${tid}`), ['tid', 'cid', 'timestamp', 'lastposttime']
+			);
+			const bulkAdd = [];
+			topicData.forEach((data) => {
+				if (data && data.cid && data.tid) {
+					const timestamp = data.lastposttime || data.timestamp || Date.now();
+					bulkAdd.push([`cid:${data.cid}:tids:lastposttime`, timestamp, data.tid]);
+				}
+			});
+			await db.sortedSetAddBulk(bulkAdd);
+			progress.incr(tids.length);
 		}, {
-			progress: this.progress,
-		}, callback);
+			batch: 500,
+		});
 	},
 };

src/upgrades/1.7.1/notification-settings.js

@@ -8,23 +8,38 @@ module.exports = {
 	timestamp: Date.UTC(2017, 10, 15),
 	method: async function () {
 		const { progress } = this;
-
+		progress.total = await db.sortedSetCard('users:joindate');
 		await batch.processSortedSet('users:joindate', async (uids) => {
-			await Promise.all(uids.map(async (uid) => {
-				progress.incr();
-				const userSettings = await db.getObjectFields(`user:${uid}:settings`, ['sendChatNotifications', 'sendPostNotifications']);
-				if (userSettings) {
+
+			const userSettings = await db.getObjectsFields(
+				uids.map(uid => `user:${uid}:settings`),
+				['sendChatNotifications', 'sendPostNotifications'],
+			);
+
+			const bulkSet = [];
+			userSettings.forEach((settings, index) => {
+				const set = {};
+				if (settings) {
 					if (parseInt(userSettings.sendChatNotifications, 10) === 1) {
-						await db.setObjectField(`user:${uid}:settings`, 'notificationType_new-chat', 'notificationemail');
+						set['notificationType_new-chat'] = 'notificationemail';
 					}
 					if (parseInt(userSettings.sendPostNotifications, 10) === 1) {
-						await db.setObjectField(`user:${uid}:settings`, 'notificationType_new-reply', 'notificationemail');
+						set['notificationType_new-reply'] = 'notificationemail';
+					}
+					if (Object.keys(set).length) {
+						bulkSet.push([`user:${uids[index]}:settings`, set]);
 					}
 				}
-				await db.deleteObjectFields(`user:${uid}:settings`, ['sendChatNotifications', 'sendPostNotifications']);
-			}));
+			});
+			await db.setObjectBulk(bulkSet);
+
+			await db.deleteObjectFields(
+				uids.map(uid => `user:${uid}:settings`),
+				['sendChatNotifications', 'sendPostNotifications'],
+			);
+
+			progress.incr(uids.length);
 		}, {
-			progress: progress,
 			batch: 500,
 		});
 	},

src/upgrades/1.7.3/topic_votes.js

@@ -10,32 +10,42 @@ module.exports = {
 	method: async function () {
 		const { progress } = this;
 
-		batch.processSortedSet('topics:tid', async (tids) => {
-			await Promise.all(tids.map(async (tid) => {
-				progress.incr();
-				const topicData = await db.getObjectFields(`topic:${tid}`, ['mainPid', 'cid', 'pinned']);
-				if (topicData.mainPid && topicData.cid) {
-					const postData = await db.getObject(`post:${topicData.mainPid}`);
-					if (postData) {
-						const upvotes = parseInt(postData.upvotes, 10) || 0;
-						const downvotes = parseInt(postData.downvotes, 10) || 0;
-						const data = {
-							upvotes: upvotes,
-							downvotes: downvotes,
-						};
-						const votes = upvotes - downvotes;
-						await Promise.all([
-							db.setObject(`topic:${tid}`, data),
-							db.sortedSetAdd('topics:votes', votes, tid),
-						]);
-						if (parseInt(topicData.pinned, 10) !== 1) {
-							await db.sortedSetAdd(`cid:${topicData.cid}:tids:votes`, votes, tid);
-						}
+		progress.total = await db.sortedSetCard('topics:tid');
+
+		await batch.processSortedSet('topics:tid', async (tids) => {
+			const topicsData = await db.getObjectsFields(
+				tids.map(tid => `topic:${tid}`),
+				['tid', 'mainPid', 'cid', 'pinned'],
+			);
+			const mainPids = topicsData.map(topicData => topicData && topicData.mainPid);
+			const mainPosts = await db.getObjects(mainPids.map(pid => `post:${pid}`));
+
+			const bulkSet = [];
+			const bulkAdd = [];
+
+			topicsData.forEach((topicData, index) => {
+				const mainPost = mainPosts[index];
+				if (mainPost && topicData && topicData.cid) {
+					const upvotes = parseInt(mainPost.upvotes, 10) || 0;
+					const downvotes = parseInt(mainPost.downvotes, 10) || 0;
+					const data = {
+						upvotes: upvotes,
+						downvotes: downvotes,
+					};
+					const votes = upvotes - downvotes;
+					bulkSet.push([`topic:${topicData.tid}`, data]);
+					bulkAdd.push(['topics:votes', votes, topicData.tid]);
+					if (parseInt(topicData.pinned, 10) !== 1) {
+						bulkAdd.push([`cid:${topicData.cid}:tids:votes`, votes, topicData.tid]);
 					}
 				}
-			}));
+			});
+
+			await db.setObjectBulk(bulkSet);
+			await db.sortedSetAddBulk('topics:votes', bulkAdd);
+
+			progress.incr(tids.length);
 		}, {
-			progress: progress,
 			batch: 500,
 		});
 	},

src/upgrades/1.8.1/diffs_zset_to_listhash.js

@@ -1,57 +1,40 @@
 'use strict';
 
-const async = require('async');
 const db = require('../../database');
 const batch = require('../../batch');
 
-
 module.exports = {
 	name: 'Reformatting post diffs to be stored in lists and hash instead of single zset',
 	timestamp: Date.UTC(2018, 2, 15),
-	method: function (callback) {
+	method: async function () {
 		const { progress } = this;
 
-		batch.processSortedSet('posts:pid', (pids, next) => {
-			async.each(pids, (pid, next) => {
-				db.getSortedSetRangeWithScores(`post:${pid}:diffs`, 0, -1, (err, diffs) => {
-					if (err) {
-						return next(err);
-					}
+		progress.total = await db.sortedSetCard('posts:pid');
 
-					if (!diffs || !diffs.length) {
-						progress.incr();
-						return next();
-					}
+		await batch.processSortedSet('posts:pid', async (pids) => {
+			const postDiffs = await db.getSortedSetsMembersWithScores(
+				pids.map(pid => `post:${pid}:diffs`),
+			);
 
-					// For each diff, push to list
-					async.each(diffs, (diff, next) => {
-						async.series([
-							async.apply(db.delete.bind(db), `post:${pid}:diffs`),
-							async.apply(db.listPrepend.bind(db), `post:${pid}:diffs`, diff.score),
-							async.apply(db.setObject.bind(db), `diff:${pid}.${diff.score}`, {
-								pid: pid,
-								patch: diff.value,
-							}),
-						], next);
-					}, (err) => {
-						if (err) {
-							return next(err);
-						}
+			await db.deleteAll(pids.map(pid => `post:${pid}:diffs`));
 
-						progress.incr();
-						return next();
-					});
-				});
-			}, (err) => {
-				if (err) {
-					// Probably type error, ok to incr and continue
-					progress.incr();
+			await Promise.all(postDiffs.map(async (diffs, index) => {
+				if (!diffs || !diffs.length) {
+					return;
 				}
-
-				return next();
-			});
+				diffs.reverse();
+				const pid = pids[index];
+				await db.listAppend(`post:${pid}:diffs`, diffs.map(d => d.score));
+				await db.setObjectBulk(
+					diffs.map(d => ([`diff:${pid}.${d.score}`, {
+						pid: pid,
+						patch: d.value,
+					}]))
+				);
+			}));
+			progress.incr(pids.length);
 		}, {
-			progress: progress,
-		}, callback);
+			batch: 500,
+		});
 	},
 };

src/upgrades/1.9.0/refresh_post_upload_associations.js

@@ -1,21 +1,20 @@
 'use strict';
 
-const async = require('async');
+const db = require('../../database');
 const posts = require('../../posts');
+const batch = require('../../batch');
 
 module.exports = {
 	name: 'Refresh post-upload associations',
 	timestamp: Date.UTC(2018, 3, 16),
-	method: function (callback) {
+	method: async function () {
 		const { progress } = this;
-
-		require('../../batch').processSortedSet('posts:pid', (pids, next) => {
-			async.each(pids, (pid, next) => {
-				posts.uploads.sync(pid, next);
-				progress.incr();
-			}, next);
+		progress.total = await db.sortedSetCard('posts:pid');
+		await batch.processSortedSet('posts:pid', async (pids) => {
+			await Promise.all(pids.map(pid => posts.uploads.sync(pid)));
+			progress.incr(pids.length);
 		}, {
-			progress: this.progress,
-		}, callback);
+			batch: 500,
+		});
 	},
 };

src/upgrades/2.8.7/fix-email-sorted-sets.js

@@ -26,7 +26,7 @@ module.exports = {
 				}
 
 				// user has email but doesn't match whats stored in user hash, gh#11259
-				if (userData.email && userData.email.toLowerCase() !== email.toLowerCase()) {
+				if (userData.email && email && String(userData.email).toLowerCase() !== email.toLowerCase()) {
 					bulkRemove.push(['email:uid', email]);
 					bulkRemove.push(['email:sorted', `${email.toLowerCase()}:${uid}`]);
 				}

src/upgrades/3.7.0/category-read-by-uid.js

@@ -9,6 +9,7 @@ module.exports = {
 	method: async function () {
 		const { progress } = this;
 		const nextCid = await db.getObjectField('global', 'nextCid');
+		progress.total = nextCid;
 		const allCids = [];
 		for (let i = 1; i <= nextCid; i++) {
 			allCids.push(i);
@@ -18,7 +19,6 @@ module.exports = {
 			progress.incr(cids.length);
 		}, {
 			batch: 500,
-			progress,
 		});
 	},
 };