Nemcio/NodeBB
1
0
Fork 0
mirror of https://github.com/NodeBB/NodeBB.git synced 2025-10-26 08:36:12 +01:00
Code Issues Packages Projects Releases Wiki Activity

perf: update old upgrade scripts to use bulkSet/Add

Browse Source 
fix a missing await
This commit is contained in:
Barış Soner Uşaklı
2025-09-25 02:03:14 -04:00
parent 6055b345e1
commit 2b987d09ce
12 changed files with 186 additions and 171 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
src/upgrades/1.10.2/upgrade_bans_to_hashes.js
View File

@@ -11,15 +11,24 @@ module.exports = {
method: async function () { method: async function () {
const { progress } = this; const { progress } = this;
progress.total = await db.sortedSetCard('users:joindate');
await batch.processSortedSet('users:joindate', async (uids) => { await batch.processSortedSet('users:joindate', async (uids) => {
for (const uid of uids) { progress.incr(uids.length);
progress.incr(); const [allUserData, allBans] = await Promise.all([
const [bans, reasons, userData] = await Promise.all([ db.getObjectsFields(
db.getSortedSetRevRangeWithScores(`uid:${uid}:bans`, 0, -1), uids.map(uid => `user:${uid}`),
db.getSortedSetRevRangeWithScores(`banned:${uid}:reasons`, 0, -1), ['banned', 'banned:expire', 'joindate', 'lastposttime', 'lastonline'],
db.getObjectFields(`user:${uid}`, ['banned', 'banned:expire', 'joindate', 'lastposttime', 'lastonline']), ),
db.getSortedSetsMembersWithScores(
uids.map(uid => `uid:${uid}:bans`)
),
]); ]);
await Promise.all(uids.map(async (uid, index) => {
const userData = allUserData[index];
const bans = allBans[index] || [];
// has no history, but is banned, create plain object with just uid and timestmap // has no history, but is banned, create plain object with just uid and timestmap
if (!bans.length && parseInt(userData.banned, 10)) { if (!bans.length && parseInt(userData.banned, 10)) {
const banTimestamp = ( const banTimestamp = (
@@ -31,6 +40,7 @@ module.exports = {
const banKey = `uid:${uid}:ban:${banTimestamp}`; const banKey = `uid:${uid}:ban:${banTimestamp}`;
await addBan(uid, banKey, { uid: uid, timestamp: banTimestamp }); await addBan(uid, banKey, { uid: uid, timestamp: banTimestamp });
} else if (bans.length) { } else if (bans.length) {
const reasons = await db.getSortedSetRevRangeWithScores(`banned:${uid}:reasons`, 0, -1);
// process ban history // process ban history
for (const ban of bans) { for (const ban of bans) {
const reasonData = reasons.find(reasonData => reasonData.score === ban.score); const reasonData = reasons.find(reasonData => reasonData.score === ban.score);
@@ -46,14 +56,16 @@ module.exports = {
await addBan(uid, banKey, data); await addBan(uid, banKey, data);
} }
} }
} }));
}, { }, {
progress: this.progress, batch: 500,
}); });
}, },
}; };
async function addBan(uid, key, data) { async function addBan(uid, key, data) {
await db.setObject(key, data); await Promise.all([
await db.sortedSetAdd(`uid:${uid}:bans:timestamp`, data.timestamp, key); db.setObject(key, data),
db.sortedSetAdd(`uid:${uid}:bans:timestamp`, data.timestamp, key),
]);
} }

43
src/upgrades/1.10.2/username_email_history.js
View File

@@ -11,27 +11,34 @@ module.exports = {
method: async function () { method: async function () {
const { progress } = this; const { progress } = this;
await batch.processSortedSet('users:joindate', async (uids) => { progress.total = await db.sortedSetCard('users:joindate');
async function updateHistory(uid, set, fieldName) {
const count = await db.sortedSetCard(set);
if (count <= 0) {
// User has not changed their username/email before, record original username
const userData = await user.getUserFields(uid, [fieldName, 'joindate']);
if (userData && userData.joindate && userData[fieldName]) {
await db.sortedSetAdd(set, userData.joindate, [userData[fieldName], userData.joindate].join(':'));
}
}
}
await Promise.all(uids.map(async (uid) => { await batch.processSortedSet('users:joindate', async (uids) => {
await Promise.all([ const [usernameHistory, emailHistory, userData] = await Promise.all([
updateHistory(uid, `user:${uid}:usernames`, 'username'), db.sortedSetsCard(uids.map(uid => `user:${uid}:usernames`)),
updateHistory(uid, `user:${uid}:emails`, 'email'), db.sortedSetsCard(uids.map(uid => `user:${uid}:emails`)),
user.getUsersFields(uids, ['uid', 'username', 'email', 'joindate']),
]); ]);
progress.incr();
})); const bulkAdd = [];
userData.forEach((data, index) => {
const thisUsernameHistory = usernameHistory[index];
const thisEmailHistory = emailHistory[index];
if (thisUsernameHistory <= 0 && data && data.joindate && data.username) {
bulkAdd.push([
`user:${data.uid}:usernames`, data.joindate, [data.username, data.joindate].join(':'),
]);
}
if (thisEmailHistory <= 0 && data && data.joindate && data.email) {
bulkAdd.push([
`user:${data.uid}:emails`, data.joindate, [data.email, data.joindate].join(':'),
]);
}
});
await db.sortedSetAddBulk(bulkAdd);
progress.incr(uids.length);
}, { }, {
progress: this.progress, batch: 500,
}); });
}, },
}; };

51
src/upgrades/1.12.1/clear_username_email_history.js
View File

@@ -1,45 +1,32 @@
'use strict'; 'use strict';
const async = require('async');
const db = require('../../database'); const db = require('../../database');
const user = require('../../user'); const user = require('../../user');
const batch = require('../../batch');
module.exports = { module.exports = {
name: 'Delete username email history for deleted users', name: 'Delete username email history for deleted users',
timestamp: Date.UTC(2019, 2, 25), timestamp: Date.UTC(2019, 2, 25),
method: function (callback) { method: async function () {
const { progress } = this; const { progress } = this;
let currentUid = 1;
db.getObjectField('global', 'nextUid', (err, nextUid) => { progress.total = await db.getObjectField('global', 'nextUid');
if (err) { const allUids = [];
return callback(err); for (let i = 1; i < progress.total; i += 1) {
allUids.push(i);
} }
progress.total = nextUid; await batch.processArray(allUids, async (uids) => {
async.whilst((next) => { const exists = await user.exists(uids);
next(null, currentUid < nextUid); const missingUids = uids.filter((uid, index) => !exists[index]);
}, const keysToDelete = [
(next) => { ...missingUids.map(uid => `user:${uid}:usernames`),
progress.incr(); ...missingUids.map(uid => `user:${uid}:emails`),
user.exists(currentUid, (err, exists) => { ];
if (err) { await db.deleteAll(keysToDelete);
return next(err); progress.incr(uids.length);
} }, {
if (exists) { batch: 500,
currentUid += 1;
return next();
}
db.deleteAll([`user:${currentUid}:usernames`, `user:${currentUid}:emails`], (err) => {
if (err) {
return next(err);
}
currentUid += 1;
next();
});
});
},
(err) => {
callback(err);
});
}); });
}, },
}; };

10
src/upgrades/1.12.1/moderation_notes_refactor.js
View File

@@ -12,10 +12,12 @@ module.exports = {
const { progress } = this; const { progress } = this;
await batch.processSortedSet('users:joindate', async (uids) => { await batch.processSortedSet('users:joindate', async (uids) => {
await Promise.all(uids.map(async (uid) => { progress.incr(uids.length);
progress.incr(); const allNotes = await db.getSortedSetsMembers(
uids.map(uid => `uid:${uid}:moderation:notes`)
const notes = await db.getSortedSetRevRange(`uid:${uid}:moderation:notes`, 0, -1); );
await Promise.all(uids.map(async (uid, index) => {
const notes = allNotes[index];
for (const note of notes) { for (const note of notes) {
const noteData = JSON.parse(note); const noteData = JSON.parse(note);
noteData.timestamp = noteData.timestamp || Date.now(); noteData.timestamp = noteData.timestamp || Date.now();

3
src/upgrades/1.13.0/clean_post_topic_hash.js
View File

@@ -8,6 +8,7 @@ module.exports = {
timestamp: Date.UTC(2019, 9, 7), timestamp: Date.UTC(2019, 9, 7),
method: async function () { method: async function () {
const { progress } = this; const { progress } = this;
progress.total = await db.sortedSetCard('posts:pid') + await db.sortedSetCard('topics:tid');
await cleanPost(progress); await cleanPost(progress);
await cleanTopic(progress); await cleanTopic(progress);
}, },
@@ -51,7 +52,6 @@ async function cleanPost(progress) {
})); }));
}, { }, {
batch: 500, batch: 500,
progress: progress,
}); });
} }
@@ -90,6 +90,5 @@ async function cleanTopic(progress) {
})); }));
}, { }, {
batch: 500, batch: 500,
progress: progress,
}); });
} }

33
src/upgrades/1.6.2/topics_lastposttime_zset.js
View File

@@ -1,29 +1,30 @@
'use strict'; 'use strict';
const async = require('async');
const db = require('../../database'); const db = require('../../database');
const batch = require('../../batch');
module.exports = { module.exports = {
name: 'New sorted set cid:<cid>:tids:lastposttime', name: 'New sorted set cid:<cid>:tids:lastposttime',
timestamp: Date.UTC(2017, 9, 30), timestamp: Date.UTC(2017, 9, 30),
method: function (callback) { method: async function () {
const { progress } = this; const { progress } = this;
progress.total = await db.sortedSetCard('topics:tid');
require('../../batch').processSortedSet('topics:tid', (tids, next) => { await batch.processSortedSet('topics:tid', async (tids) => {
async.eachSeries(tids, (tid, next) => { const topicData = await db.getObjectsFields(
db.getObjectFields(`topic:${tid}`, ['cid', 'timestamp', 'lastposttime'], (err, topicData) => { tids.map(tid => `topic:${tid}`), ['tid', 'cid', 'timestamp', 'lastposttime']
if (err || !topicData) { );
return next(err); const bulkAdd = [];
topicData.forEach((data) => {
if (data && data.cid && data.tid) {
const timestamp = data.lastposttime || data.timestamp || Date.now();
bulkAdd.push([`cid:${data.cid}:tids:lastposttime`, timestamp, data.tid]);
} }
progress.incr(); });
await db.sortedSetAddBulk(bulkAdd);
const timestamp = topicData.lastposttime || topicData.timestamp || Date.now(); progress.incr(tids.length);
db.sortedSetAdd(`cid:${topicData.cid}:tids:lastposttime`, timestamp, tid, next);
}, next);
}, next);
}, { }, {
progress: this.progress, batch: 500,
}, callback); });
}, },
}; };

35
src/upgrades/1.7.1/notification-settings.js
View File

@@ -8,23 +8,38 @@ module.exports = {
timestamp: Date.UTC(2017, 10, 15), timestamp: Date.UTC(2017, 10, 15),
method: async function () { method: async function () {
const { progress } = this; const { progress } = this;
progress.total = await db.sortedSetCard('users:joindate');
await batch.processSortedSet('users:joindate', async (uids) => { await batch.processSortedSet('users:joindate', async (uids) => {
await Promise.all(uids.map(async (uid) => {
progress.incr(); const userSettings = await db.getObjectsFields(
const userSettings = await db.getObjectFields(`user:${uid}:settings`, ['sendChatNotifications', 'sendPostNotifications']); uids.map(uid => `user:${uid}:settings`),
if (userSettings) { ['sendChatNotifications', 'sendPostNotifications'],
);
const bulkSet = [];
userSettings.forEach((settings, index) => {
const set = {};
if (settings) {
if (parseInt(userSettings.sendChatNotifications, 10) === 1) { if (parseInt(userSettings.sendChatNotifications, 10) === 1) {
await db.setObjectField(`user:${uid}:settings`, 'notificationType_new-chat', 'notificationemail'); set['notificationType_new-chat'] = 'notificationemail';
} }
if (parseInt(userSettings.sendPostNotifications, 10) === 1) { if (parseInt(userSettings.sendPostNotifications, 10) === 1) {
await db.setObjectField(`user:${uid}:settings`, 'notificationType_new-reply', 'notificationemail'); set['notificationType_new-reply'] = 'notificationemail';
}
if (Object.keys(set).length) {
bulkSet.push([`user:${uids[index]}:settings`, set]);
} }
} }
await db.deleteObjectFields(`user:${uid}:settings`, ['sendChatNotifications', 'sendPostNotifications']); });
})); await db.setObjectBulk(bulkSet);
await db.deleteObjectFields(
uids.map(uid => `user:${uid}:settings`),
['sendChatNotifications', 'sendPostNotifications'],
);
progress.incr(uids.length);
}, { }, {
progress: progress,
batch: 500, batch: 500,
}); });
}, },

44
src/upgrades/1.7.3/topic_votes.js
View File

@@ -10,32 +10,42 @@ module.exports = {
method: async function () { method: async function () {
const { progress } = this; const { progress } = this;
batch.processSortedSet('topics:tid', async (tids) => { progress.total = await db.sortedSetCard('topics:tid');
await Promise.all(tids.map(async (tid) => {
progress.incr(); await batch.processSortedSet('topics:tid', async (tids) => {
const topicData = await db.getObjectFields(`topic:${tid}`, ['mainPid', 'cid', 'pinned']); const topicsData = await db.getObjectsFields(
if (topicData.mainPid && topicData.cid) { tids.map(tid => `topic:${tid}`),
const postData = await db.getObject(`post:${topicData.mainPid}`); ['tid', 'mainPid', 'cid', 'pinned'],
if (postData) { );
const upvotes = parseInt(postData.upvotes, 10) || 0; const mainPids = topicsData.map(topicData => topicData && topicData.mainPid);
const downvotes = parseInt(postData.downvotes, 10) || 0; const mainPosts = await db.getObjects(mainPids.map(pid => `post:${pid}`));
const bulkSet = [];
const bulkAdd = [];
topicsData.forEach((topicData, index) => {
const mainPost = mainPosts[index];
if (mainPost && topicData && topicData.cid) {
const upvotes = parseInt(mainPost.upvotes, 10) || 0;
const downvotes = parseInt(mainPost.downvotes, 10) || 0;
const data = { const data = {
upvotes: upvotes, upvotes: upvotes,
downvotes: downvotes, downvotes: downvotes,
}; };
const votes = upvotes - downvotes; const votes = upvotes - downvotes;
await Promise.all([ bulkSet.push([`topic:${topicData.tid}`, data]);
db.setObject(`topic:${tid}`, data), bulkAdd.push(['topics:votes', votes, topicData.tid]);
db.sortedSetAdd('topics:votes', votes, tid),
]);
if (parseInt(topicData.pinned, 10) !== 1) { if (parseInt(topicData.pinned, 10) !== 1) {
await db.sortedSetAdd(`cid:${topicData.cid}:tids:votes`, votes, tid); bulkAdd.push([`cid:${topicData.cid}:tids:votes`, votes, topicData.tid]);
} }
} }
} });
}));
await db.setObjectBulk(bulkSet);
await db.sortedSetAddBulk('topics:votes', bulkAdd);
progress.incr(tids.length);
}, { }, {
progress: progress,
batch: 500, batch: 500,
}); });
}, },

63
src/upgrades/1.8.1/diffs_zset_to_listhash.js
View File

@@ -1,57 +1,40 @@
'use strict'; 'use strict';
const async = require('async');
const db = require('../../database'); const db = require('../../database');
const batch = require('../../batch'); const batch = require('../../batch');
module.exports = { module.exports = {
name: 'Reformatting post diffs to be stored in lists and hash instead of single zset', name: 'Reformatting post diffs to be stored in lists and hash instead of single zset',
timestamp: Date.UTC(2018, 2, 15), timestamp: Date.UTC(2018, 2, 15),
method: function (callback) { method: async function () {
const { progress } = this; const { progress } = this;
batch.processSortedSet('posts:pid', (pids, next) => { progress.total = await db.sortedSetCard('posts:pid');
async.each(pids, (pid, next) => {
db.getSortedSetRangeWithScores(`post:${pid}:diffs`, 0, -1, (err, diffs) => {
if (err) {
return next(err);
}
await batch.processSortedSet('posts:pid', async (pids) => {
const postDiffs = await db.getSortedSetsMembersWithScores(
pids.map(pid => `post:${pid}:diffs`),
);
await db.deleteAll(pids.map(pid => `post:${pid}:diffs`));
await Promise.all(postDiffs.map(async (diffs, index) => {
if (!diffs || !diffs.length) { if (!diffs || !diffs.length) {
progress.incr(); return;
return next();
} }
diffs.reverse();
// For each diff, push to list const pid = pids[index];
async.each(diffs, (diff, next) => { await db.listAppend(`post:${pid}:diffs`, diffs.map(d => d.score));
async.series([ await db.setObjectBulk(
async.apply(db.delete.bind(db), `post:${pid}:diffs`), diffs.map(d => ([`diff:${pid}.${d.score}`, {
async.apply(db.listPrepend.bind(db), `post:${pid}:diffs`, diff.score),
async.apply(db.setObject.bind(db), `diff:${pid}.${diff.score}`, {
pid: pid, pid: pid,
patch: diff.value, patch: d.value,
}), }]))
], next); );
}, (err) => { }));
if (err) { progress.incr(pids.length);
return next(err);
}
progress.incr();
return next();
});
});
}, (err) => {
if (err) {
// Probably type error, ok to incr and continue
progress.incr();
}
return next();
});
}, { }, {
progress: progress, batch: 500,
}, callback); });
}, },
}; };

19
src/upgrades/1.9.0/refresh_post_upload_associations.js
View File

@@ -1,21 +1,20 @@
'use strict'; 'use strict';
const async = require('async'); const db = require('../../database');
const posts = require('../../posts'); const posts = require('../../posts');
const batch = require('../../batch');
module.exports = { module.exports = {
name: 'Refresh post-upload associations', name: 'Refresh post-upload associations',
timestamp: Date.UTC(2018, 3, 16), timestamp: Date.UTC(2018, 3, 16),
method: function (callback) { method: async function () {
const { progress } = this; const { progress } = this;
progress.total = await db.sortedSetCard('posts:pid');
require('../../batch').processSortedSet('posts:pid', (pids, next) => { await batch.processSortedSet('posts:pid', async (pids) => {
async.each(pids, (pid, next) => { await Promise.all(pids.map(pid => posts.uploads.sync(pid)));
posts.uploads.sync(pid, next); progress.incr(pids.length);
progress.incr();
}, next);
}, { }, {
progress: this.progress, batch: 500,
}, callback); });
}, },
}; };

2
src/upgrades/2.8.7/fix-email-sorted-sets.js
View File

@@ -26,7 +26,7 @@ module.exports = {
} }
// user has email but doesn't match whats stored in user hash, gh#11259 // user has email but doesn't match whats stored in user hash, gh#11259
if (userData.email && userData.email.toLowerCase() !== email.toLowerCase()) { if (userData.email && email && String(userData.email).toLowerCase() !== email.toLowerCase()) {
bulkRemove.push(['email:uid', email]); bulkRemove.push(['email:uid', email]);
bulkRemove.push(['email:sorted', `${email.toLowerCase()}:${uid}`]); bulkRemove.push(['email:sorted', `${email.toLowerCase()}:${uid}`]);
} }

2
src/upgrades/3.7.0/category-read-by-uid.js
View File

@@ -9,6 +9,7 @@ module.exports = {
method: async function () { method: async function () {
const { progress } = this; const { progress } = this;
const nextCid = await db.getObjectField('global', 'nextCid'); const nextCid = await db.getObjectField('global', 'nextCid');
progress.total = nextCid;
const allCids = []; const allCids = [];
for (let i = 1; i <= nextCid; i++) { for (let i = 1; i <= nextCid; i++) {
allCids.push(i); allCids.push(i);
@@ -18,7 +19,6 @@ module.exports = {
progress.incr(cids.length); progress.incr(cids.length);
}, { }, {
batch: 500, batch: 500,
progress,
}); });
}, },
}; };