Nemcio/NodeBB
1
0
Fork 0
mirror of https://github.com/NodeBB/NodeBB.git synced 2025-10-26 08:36:12 +01:00
Code Issues Packages Projects Releases Wiki Activity

refactor: closes #13547, process user uploads via batch

Browse Source 
reduce processed user count to 100 per batch
This commit is contained in:
Barış Soner Uşaklı
2025-07-15 13:02:46 -04:00
parent 97a5d54387
commit 1ad97ac194
1 changed files with 22 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
src/upgrades/4.3.0/normalize_thumbs_uploads.js
View File

@@ -89,34 +89,36 @@ module.exports = {
const keys = uids.map(uid => `uid:${uid}:uploads`);
const userUploadData = await db.getSortedSetsMembersWithScores(keys);
const bulkAdd = [];
const bulkRemove = [];
const promises = [];
userUploadData.forEach((userUploads, idx) => {
await Promise.all(userUploadData.map(async (allUserUploads, idx) => {
const uid = uids[idx];
if (Array.isArray(userUploads)) {
userUploads.forEach((userUpload) => {
const normalizedPath = normalizePath(userUpload.value);
if (normalizedPath !== userUpload.value) {
bulkAdd.push([`uid:${uid}:uploads`, userUpload.score, normalizedPath]);
promises.push(db.setObjectField(`upload:${md5(normalizedPath)}`, 'uid', uid));
if (Array.isArray(allUserUploads)) {
await batch.processArray(allUserUploads, async (userUploads) => {
const bulkAdd = [];
const bulkRemove = [];
const promises = [];
userUploads.forEach((userUpload) => {
const normalizedPath = normalizePath(userUpload.value);
if (normalizedPath !== userUpload.value) {
bulkAdd.push([`uid:${uid}:uploads`, userUpload.score, normalizedPath]);
promises.push(db.setObjectField(`upload:${md5(normalizedPath)}`, 'uid', uid));
bulkRemove.push([`uid:${uid}:uploads`, userUpload.value]);
promises.push(db.delete(`upload:${md5(userUpload.value)}`));
}
bulkRemove.push([`uid:${uid}:uploads`, userUpload.value]);
promises.push(db.delete(`upload:${md5(userUpload.value)}`));
}
});
await Promise.all(promises);
await db.sortedSetRemoveBulk(bulkRemove);
await db.sortedSetAddBulk(bulkAdd);
}, {
batch: 500,
});
}
});
await Promise.all(promises);
await db.sortedSetRemoveBulk(bulkRemove);
await db.sortedSetAddBulk(bulkAdd);
}));
progress.incr(uids.length);
}, {
batch: 500,
batch: 100,
});
},
};