Nemcio/NodeBB
1
0
Fork 0
mirror of https://github.com/NodeBB/NodeBB.git synced 2025-10-27 00:56:13 +01:00
Code Issues Packages Projects Releases Wiki Activity

refactor: closes #13547, process user uploads via batch

Browse Source 
reduce processed user count to 100 per batch
This commit is contained in:
Barış Soner Uşaklı
2025-07-15 13:02:46 -04:00
parent 97a5d54387
commit 1ad97ac194
1 changed files with 22 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
src/upgrades/4.3.0/normalize_thumbs_uploads.js
View File

@@ -89,34 +89,36 @@ module.exports = {
const keys = uids.map(uid => `uid:${uid}:uploads`); const keys = uids.map(uid => `uid:${uid}:uploads`);
const userUploadData = await db.getSortedSetsMembersWithScores(keys); const userUploadData = await db.getSortedSetsMembersWithScores(keys);
const bulkAdd = [];
const bulkRemove = [];
const promises = [];
userUploadData.forEach((userUploads, idx) => { await Promise.all(userUploadData.map(async (allUserUploads, idx) => {
const uid = uids[idx]; const uid = uids[idx];
if (Array.isArray(userUploads)) { if (Array.isArray(allUserUploads)) {
userUploads.forEach((userUpload) => { await batch.processArray(allUserUploads, async (userUploads) => {
const normalizedPath = normalizePath(userUpload.value); const bulkAdd = [];
if (normalizedPath !== userUpload.value) { const bulkRemove = [];
bulkAdd.push([`uid:${uid}:uploads`, userUpload.score, normalizedPath]); const promises = [];
promises.push(db.setObjectField(`upload:${md5(normalizedPath)}`, 'uid', uid)); userUploads.forEach((userUpload) => {
const normalizedPath = normalizePath(userUpload.value);
if (normalizedPath !== userUpload.value) {
bulkAdd.push([`uid:${uid}:uploads`, userUpload.score, normalizedPath]);
promises.push(db.setObjectField(`upload:${md5(normalizedPath)}`, 'uid', uid));
bulkRemove.push([`uid:${uid}:uploads`, userUpload.value]); bulkRemove.push([`uid:${uid}:uploads`, userUpload.value]);
promises.push(db.delete(`upload:${md5(userUpload.value)}`)); promises.push(db.delete(`upload:${md5(userUpload.value)}`));
} }
});
await Promise.all(promises);
await db.sortedSetRemoveBulk(bulkRemove);
await db.sortedSetAddBulk(bulkAdd);
}, {
batch: 500,
}); });
} }
}); }));
await Promise.all(promises);
await db.sortedSetRemoveBulk(bulkRemove);
await db.sortedSetAddBulk(bulkAdd);
progress.incr(uids.length); progress.incr(uids.length);
}, { }, {
batch: 500, batch: 100,
}); });
}, },
}; };