From 2b987d09ce487bb97891c8c38d639a2e6c216537 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bar=C4=B1=C5=9F=20Soner=20U=C5=9Fakl=C4=B1?= Date: Thu, 25 Sep 2025 02:03:14 -0400 Subject: [PATCH] perf: update old upgrade scripts to use bulkSet/Add fix a missing await --- src/upgrades/1.10.2/upgrade_bans_to_hashes.js | 34 ++++++---- src/upgrades/1.10.2/username_email_history.js | 45 +++++++------ .../1.12.1/clear_username_email_history.js | 53 ++++++---------- .../1.12.1/moderation_notes_refactor.js | 10 +-- src/upgrades/1.13.0/clean_post_topic_hash.js | 3 +- .../1.6.2/topics_lastposttime_zset.js | 35 ++++++----- src/upgrades/1.7.1/notification-settings.js | 35 ++++++++--- src/upgrades/1.7.3/topic_votes.js | 56 ++++++++++------- src/upgrades/1.8.1/diffs_zset_to_listhash.js | 63 +++++++------------ .../1.9.0/refresh_post_upload_associations.js | 19 +++--- src/upgrades/2.8.7/fix-email-sorted-sets.js | 2 +- src/upgrades/3.7.0/category-read-by-uid.js | 2 +- 12 files changed, 186 insertions(+), 171 deletions(-) diff --git a/src/upgrades/1.10.2/upgrade_bans_to_hashes.js b/src/upgrades/1.10.2/upgrade_bans_to_hashes.js index 84c7a0ed4d..2bc55b4667 100644 --- a/src/upgrades/1.10.2/upgrade_bans_to_hashes.js +++ b/src/upgrades/1.10.2/upgrade_bans_to_hashes.js @@ -11,14 +11,23 @@ module.exports = { method: async function () { const { progress } = this; + progress.total = await db.sortedSetCard('users:joindate'); + await batch.processSortedSet('users:joindate', async (uids) => { - for (const uid of uids) { - progress.incr(); - const [bans, reasons, userData] = await Promise.all([ - db.getSortedSetRevRangeWithScores(`uid:${uid}:bans`, 0, -1), - db.getSortedSetRevRangeWithScores(`banned:${uid}:reasons`, 0, -1), - db.getObjectFields(`user:${uid}`, ['banned', 'banned:expire', 'joindate', 'lastposttime', 'lastonline']), - ]); + progress.incr(uids.length); + const [allUserData, allBans] = await Promise.all([ + db.getObjectsFields( + uids.map(uid => `user:${uid}`), + ['banned', 'banned:expire', 'joindate', 'lastposttime', 'lastonline'], + ), + db.getSortedSetsMembersWithScores( + uids.map(uid => `uid:${uid}:bans`) + ), + ]); + + await Promise.all(uids.map(async (uid, index) => { + const userData = allUserData[index]; + const bans = allBans[index] || []; // has no history, but is banned, create plain object with just uid and timestmap if (!bans.length && parseInt(userData.banned, 10)) { @@ -31,6 +40,7 @@ module.exports = { const banKey = `uid:${uid}:ban:${banTimestamp}`; await addBan(uid, banKey, { uid: uid, timestamp: banTimestamp }); } else if (bans.length) { + const reasons = await db.getSortedSetRevRangeWithScores(`banned:${uid}:reasons`, 0, -1); // process ban history for (const ban of bans) { const reasonData = reasons.find(reasonData => reasonData.score === ban.score); @@ -46,14 +56,16 @@ module.exports = { await addBan(uid, banKey, data); } } - } + })); }, { - progress: this.progress, + batch: 500, }); }, }; async function addBan(uid, key, data) { - await db.setObject(key, data); - await db.sortedSetAdd(`uid:${uid}:bans:timestamp`, data.timestamp, key); + await Promise.all([ + db.setObject(key, data), + db.sortedSetAdd(`uid:${uid}:bans:timestamp`, data.timestamp, key), + ]); } diff --git a/src/upgrades/1.10.2/username_email_history.js b/src/upgrades/1.10.2/username_email_history.js index 3b03568a69..8ee4306e3d 100644 --- a/src/upgrades/1.10.2/username_email_history.js +++ b/src/upgrades/1.10.2/username_email_history.js @@ -11,27 +11,34 @@ module.exports = { method: async function () { const { progress } = this; - await batch.processSortedSet('users:joindate', async (uids) => { - async function updateHistory(uid, set, fieldName) { - const count = await db.sortedSetCard(set); - if (count <= 0) { - // User has not changed their username/email before, record original username - const userData = await user.getUserFields(uid, [fieldName, 'joindate']); - if (userData && userData.joindate && userData[fieldName]) { - await db.sortedSetAdd(set, userData.joindate, [userData[fieldName], userData.joindate].join(':')); - } - } - } + progress.total = await db.sortedSetCard('users:joindate'); - await Promise.all(uids.map(async (uid) => { - await Promise.all([ - updateHistory(uid, `user:${uid}:usernames`, 'username'), - updateHistory(uid, `user:${uid}:emails`, 'email'), - ]); - progress.incr(); - })); + await batch.processSortedSet('users:joindate', async (uids) => { + const [usernameHistory, emailHistory, userData] = await Promise.all([ + db.sortedSetsCard(uids.map(uid => `user:${uid}:usernames`)), + db.sortedSetsCard(uids.map(uid => `user:${uid}:emails`)), + user.getUsersFields(uids, ['uid', 'username', 'email', 'joindate']), + ]); + + const bulkAdd = []; + userData.forEach((data, index) => { + const thisUsernameHistory = usernameHistory[index]; + const thisEmailHistory = emailHistory[index]; + if (thisUsernameHistory <= 0 && data && data.joindate && data.username) { + bulkAdd.push([ + `user:${data.uid}:usernames`, data.joindate, [data.username, data.joindate].join(':'), + ]); + } + if (thisEmailHistory <= 0 && data && data.joindate && data.email) { + bulkAdd.push([ + `user:${data.uid}:emails`, data.joindate, [data.email, data.joindate].join(':'), + ]); + } + }); + await db.sortedSetAddBulk(bulkAdd); + progress.incr(uids.length); }, { - progress: this.progress, + batch: 500, }); }, }; diff --git a/src/upgrades/1.12.1/clear_username_email_history.js b/src/upgrades/1.12.1/clear_username_email_history.js index 822b500884..0d36534502 100644 --- a/src/upgrades/1.12.1/clear_username_email_history.js +++ b/src/upgrades/1.12.1/clear_username_email_history.js @@ -1,45 +1,32 @@ 'use strict'; -const async = require('async'); + const db = require('../../database'); const user = require('../../user'); +const batch = require('../../batch'); module.exports = { name: 'Delete username email history for deleted users', timestamp: Date.UTC(2019, 2, 25), - method: function (callback) { + method: async function () { const { progress } = this; - let currentUid = 1; - db.getObjectField('global', 'nextUid', (err, nextUid) => { - if (err) { - return callback(err); - } - progress.total = nextUid; - async.whilst((next) => { - next(null, currentUid < nextUid); - }, - (next) => { - progress.incr(); - user.exists(currentUid, (err, exists) => { - if (err) { - return next(err); - } - if (exists) { - currentUid += 1; - return next(); - } - db.deleteAll([`user:${currentUid}:usernames`, `user:${currentUid}:emails`], (err) => { - if (err) { - return next(err); - } - currentUid += 1; - next(); - }); - }); - }, - (err) => { - callback(err); - }); + + progress.total = await db.getObjectField('global', 'nextUid'); + const allUids = []; + for (let i = 1; i < progress.total; i += 1) { + allUids.push(i); + } + await batch.processArray(allUids, async (uids) => { + const exists = await user.exists(uids); + const missingUids = uids.filter((uid, index) => !exists[index]); + const keysToDelete = [ + ...missingUids.map(uid => `user:${uid}:usernames`), + ...missingUids.map(uid => `user:${uid}:emails`), + ]; + await db.deleteAll(keysToDelete); + progress.incr(uids.length); + }, { + batch: 500, }); }, }; diff --git a/src/upgrades/1.12.1/moderation_notes_refactor.js b/src/upgrades/1.12.1/moderation_notes_refactor.js index 390273d74a..85118a9a0c 100644 --- a/src/upgrades/1.12.1/moderation_notes_refactor.js +++ b/src/upgrades/1.12.1/moderation_notes_refactor.js @@ -12,10 +12,12 @@ module.exports = { const { progress } = this; await batch.processSortedSet('users:joindate', async (uids) => { - await Promise.all(uids.map(async (uid) => { - progress.incr(); - - const notes = await db.getSortedSetRevRange(`uid:${uid}:moderation:notes`, 0, -1); + progress.incr(uids.length); + const allNotes = await db.getSortedSetsMembers( + uids.map(uid => `uid:${uid}:moderation:notes`) + ); + await Promise.all(uids.map(async (uid, index) => { + const notes = allNotes[index]; for (const note of notes) { const noteData = JSON.parse(note); noteData.timestamp = noteData.timestamp || Date.now(); diff --git a/src/upgrades/1.13.0/clean_post_topic_hash.js b/src/upgrades/1.13.0/clean_post_topic_hash.js index caa6dbd8f6..20cfd78c22 100644 --- a/src/upgrades/1.13.0/clean_post_topic_hash.js +++ b/src/upgrades/1.13.0/clean_post_topic_hash.js @@ -8,6 +8,7 @@ module.exports = { timestamp: Date.UTC(2019, 9, 7), method: async function () { const { progress } = this; + progress.total = await db.sortedSetCard('posts:pid') + await db.sortedSetCard('topics:tid'); await cleanPost(progress); await cleanTopic(progress); }, @@ -51,7 +52,6 @@ async function cleanPost(progress) { })); }, { batch: 500, - progress: progress, }); } @@ -90,6 +90,5 @@ async function cleanTopic(progress) { })); }, { batch: 500, - progress: progress, }); } diff --git a/src/upgrades/1.6.2/topics_lastposttime_zset.js b/src/upgrades/1.6.2/topics_lastposttime_zset.js index 1dee9feb1a..f299b19c01 100644 --- a/src/upgrades/1.6.2/topics_lastposttime_zset.js +++ b/src/upgrades/1.6.2/topics_lastposttime_zset.js @@ -1,29 +1,30 @@ 'use strict'; -const async = require('async'); - const db = require('../../database'); +const batch = require('../../batch'); module.exports = { name: 'New sorted set cid::tids:lastposttime', timestamp: Date.UTC(2017, 9, 30), - method: function (callback) { + method: async function () { const { progress } = this; + progress.total = await db.sortedSetCard('topics:tid'); - require('../../batch').processSortedSet('topics:tid', (tids, next) => { - async.eachSeries(tids, (tid, next) => { - db.getObjectFields(`topic:${tid}`, ['cid', 'timestamp', 'lastposttime'], (err, topicData) => { - if (err || !topicData) { - return next(err); - } - progress.incr(); - - const timestamp = topicData.lastposttime || topicData.timestamp || Date.now(); - db.sortedSetAdd(`cid:${topicData.cid}:tids:lastposttime`, timestamp, tid, next); - }, next); - }, next); + await batch.processSortedSet('topics:tid', async (tids) => { + const topicData = await db.getObjectsFields( + tids.map(tid => `topic:${tid}`), ['tid', 'cid', 'timestamp', 'lastposttime'] + ); + const bulkAdd = []; + topicData.forEach((data) => { + if (data && data.cid && data.tid) { + const timestamp = data.lastposttime || data.timestamp || Date.now(); + bulkAdd.push([`cid:${data.cid}:tids:lastposttime`, timestamp, data.tid]); + } + }); + await db.sortedSetAddBulk(bulkAdd); + progress.incr(tids.length); }, { - progress: this.progress, - }, callback); + batch: 500, + }); }, }; diff --git a/src/upgrades/1.7.1/notification-settings.js b/src/upgrades/1.7.1/notification-settings.js index fed592effb..e3693d4f04 100644 --- a/src/upgrades/1.7.1/notification-settings.js +++ b/src/upgrades/1.7.1/notification-settings.js @@ -8,23 +8,38 @@ module.exports = { timestamp: Date.UTC(2017, 10, 15), method: async function () { const { progress } = this; - + progress.total = await db.sortedSetCard('users:joindate'); await batch.processSortedSet('users:joindate', async (uids) => { - await Promise.all(uids.map(async (uid) => { - progress.incr(); - const userSettings = await db.getObjectFields(`user:${uid}:settings`, ['sendChatNotifications', 'sendPostNotifications']); - if (userSettings) { + + const userSettings = await db.getObjectsFields( + uids.map(uid => `user:${uid}:settings`), + ['sendChatNotifications', 'sendPostNotifications'], + ); + + const bulkSet = []; + userSettings.forEach((settings, index) => { + const set = {}; + if (settings) { if (parseInt(userSettings.sendChatNotifications, 10) === 1) { - await db.setObjectField(`user:${uid}:settings`, 'notificationType_new-chat', 'notificationemail'); + set['notificationType_new-chat'] = 'notificationemail'; } if (parseInt(userSettings.sendPostNotifications, 10) === 1) { - await db.setObjectField(`user:${uid}:settings`, 'notificationType_new-reply', 'notificationemail'); + set['notificationType_new-reply'] = 'notificationemail'; + } + if (Object.keys(set).length) { + bulkSet.push([`user:${uids[index]}:settings`, set]); } } - await db.deleteObjectFields(`user:${uid}:settings`, ['sendChatNotifications', 'sendPostNotifications']); - })); + }); + await db.setObjectBulk(bulkSet); + + await db.deleteObjectFields( + uids.map(uid => `user:${uid}:settings`), + ['sendChatNotifications', 'sendPostNotifications'], + ); + + progress.incr(uids.length); }, { - progress: progress, batch: 500, }); }, diff --git a/src/upgrades/1.7.3/topic_votes.js b/src/upgrades/1.7.3/topic_votes.js index 008aaece0a..d5f6b9fd57 100644 --- a/src/upgrades/1.7.3/topic_votes.js +++ b/src/upgrades/1.7.3/topic_votes.js @@ -10,32 +10,42 @@ module.exports = { method: async function () { const { progress } = this; - batch.processSortedSet('topics:tid', async (tids) => { - await Promise.all(tids.map(async (tid) => { - progress.incr(); - const topicData = await db.getObjectFields(`topic:${tid}`, ['mainPid', 'cid', 'pinned']); - if (topicData.mainPid && topicData.cid) { - const postData = await db.getObject(`post:${topicData.mainPid}`); - if (postData) { - const upvotes = parseInt(postData.upvotes, 10) || 0; - const downvotes = parseInt(postData.downvotes, 10) || 0; - const data = { - upvotes: upvotes, - downvotes: downvotes, - }; - const votes = upvotes - downvotes; - await Promise.all([ - db.setObject(`topic:${tid}`, data), - db.sortedSetAdd('topics:votes', votes, tid), - ]); - if (parseInt(topicData.pinned, 10) !== 1) { - await db.sortedSetAdd(`cid:${topicData.cid}:tids:votes`, votes, tid); - } + progress.total = await db.sortedSetCard('topics:tid'); + + await batch.processSortedSet('topics:tid', async (tids) => { + const topicsData = await db.getObjectsFields( + tids.map(tid => `topic:${tid}`), + ['tid', 'mainPid', 'cid', 'pinned'], + ); + const mainPids = topicsData.map(topicData => topicData && topicData.mainPid); + const mainPosts = await db.getObjects(mainPids.map(pid => `post:${pid}`)); + + const bulkSet = []; + const bulkAdd = []; + + topicsData.forEach((topicData, index) => { + const mainPost = mainPosts[index]; + if (mainPost && topicData && topicData.cid) { + const upvotes = parseInt(mainPost.upvotes, 10) || 0; + const downvotes = parseInt(mainPost.downvotes, 10) || 0; + const data = { + upvotes: upvotes, + downvotes: downvotes, + }; + const votes = upvotes - downvotes; + bulkSet.push([`topic:${topicData.tid}`, data]); + bulkAdd.push(['topics:votes', votes, topicData.tid]); + if (parseInt(topicData.pinned, 10) !== 1) { + bulkAdd.push([`cid:${topicData.cid}:tids:votes`, votes, topicData.tid]); } } - })); + }); + + await db.setObjectBulk(bulkSet); + await db.sortedSetAddBulk('topics:votes', bulkAdd); + + progress.incr(tids.length); }, { - progress: progress, batch: 500, }); }, diff --git a/src/upgrades/1.8.1/diffs_zset_to_listhash.js b/src/upgrades/1.8.1/diffs_zset_to_listhash.js index 370242fba1..277418a79e 100644 --- a/src/upgrades/1.8.1/diffs_zset_to_listhash.js +++ b/src/upgrades/1.8.1/diffs_zset_to_listhash.js @@ -1,57 +1,40 @@ 'use strict'; -const async = require('async'); const db = require('../../database'); const batch = require('../../batch'); - module.exports = { name: 'Reformatting post diffs to be stored in lists and hash instead of single zset', timestamp: Date.UTC(2018, 2, 15), - method: function (callback) { + method: async function () { const { progress } = this; - batch.processSortedSet('posts:pid', (pids, next) => { - async.each(pids, (pid, next) => { - db.getSortedSetRangeWithScores(`post:${pid}:diffs`, 0, -1, (err, diffs) => { - if (err) { - return next(err); - } + progress.total = await db.sortedSetCard('posts:pid'); - if (!diffs || !diffs.length) { - progress.incr(); - return next(); - } + await batch.processSortedSet('posts:pid', async (pids) => { + const postDiffs = await db.getSortedSetsMembersWithScores( + pids.map(pid => `post:${pid}:diffs`), + ); - // For each diff, push to list - async.each(diffs, (diff, next) => { - async.series([ - async.apply(db.delete.bind(db), `post:${pid}:diffs`), - async.apply(db.listPrepend.bind(db), `post:${pid}:diffs`, diff.score), - async.apply(db.setObject.bind(db), `diff:${pid}.${diff.score}`, { - pid: pid, - patch: diff.value, - }), - ], next); - }, (err) => { - if (err) { - return next(err); - } + await db.deleteAll(pids.map(pid => `post:${pid}:diffs`)); - progress.incr(); - return next(); - }); - }); - }, (err) => { - if (err) { - // Probably type error, ok to incr and continue - progress.incr(); + await Promise.all(postDiffs.map(async (diffs, index) => { + if (!diffs || !diffs.length) { + return; } - - return next(); - }); + diffs.reverse(); + const pid = pids[index]; + await db.listAppend(`post:${pid}:diffs`, diffs.map(d => d.score)); + await db.setObjectBulk( + diffs.map(d => ([`diff:${pid}.${d.score}`, { + pid: pid, + patch: d.value, + }])) + ); + })); + progress.incr(pids.length); }, { - progress: progress, - }, callback); + batch: 500, + }); }, }; diff --git a/src/upgrades/1.9.0/refresh_post_upload_associations.js b/src/upgrades/1.9.0/refresh_post_upload_associations.js index 44acfc079f..6183529641 100644 --- a/src/upgrades/1.9.0/refresh_post_upload_associations.js +++ b/src/upgrades/1.9.0/refresh_post_upload_associations.js @@ -1,21 +1,20 @@ 'use strict'; -const async = require('async'); +const db = require('../../database'); const posts = require('../../posts'); +const batch = require('../../batch'); module.exports = { name: 'Refresh post-upload associations', timestamp: Date.UTC(2018, 3, 16), - method: function (callback) { + method: async function () { const { progress } = this; - - require('../../batch').processSortedSet('posts:pid', (pids, next) => { - async.each(pids, (pid, next) => { - posts.uploads.sync(pid, next); - progress.incr(); - }, next); + progress.total = await db.sortedSetCard('posts:pid'); + await batch.processSortedSet('posts:pid', async (pids) => { + await Promise.all(pids.map(pid => posts.uploads.sync(pid))); + progress.incr(pids.length); }, { - progress: this.progress, - }, callback); + batch: 500, + }); }, }; diff --git a/src/upgrades/2.8.7/fix-email-sorted-sets.js b/src/upgrades/2.8.7/fix-email-sorted-sets.js index fcab69a8f4..84919e6774 100644 --- a/src/upgrades/2.8.7/fix-email-sorted-sets.js +++ b/src/upgrades/2.8.7/fix-email-sorted-sets.js @@ -26,7 +26,7 @@ module.exports = { } // user has email but doesn't match whats stored in user hash, gh#11259 - if (userData.email && userData.email.toLowerCase() !== email.toLowerCase()) { + if (userData.email && email && String(userData.email).toLowerCase() !== email.toLowerCase()) { bulkRemove.push(['email:uid', email]); bulkRemove.push(['email:sorted', `${email.toLowerCase()}:${uid}`]); } diff --git a/src/upgrades/3.7.0/category-read-by-uid.js b/src/upgrades/3.7.0/category-read-by-uid.js index 4ef564f53a..971620613e 100644 --- a/src/upgrades/3.7.0/category-read-by-uid.js +++ b/src/upgrades/3.7.0/category-read-by-uid.js @@ -9,6 +9,7 @@ module.exports = { method: async function () { const { progress } = this; const nextCid = await db.getObjectField('global', 'nextCid'); + progress.total = nextCid; const allCids = []; for (let i = 1; i <= nextCid; i++) { allCids.push(i); @@ -18,7 +19,6 @@ module.exports = { progress.incr(cids.length); }, { batch: 500, - progress, }); }, };