refactor: closes #13547, process user uploads via batch

reduce processed user count to 100 per batch
This commit is contained in:
Barış Soner Uşaklı
2025-07-15 13:02:46 -04:00
parent 97a5d54387
commit 1ad97ac194

View File

@@ -89,13 +89,14 @@ module.exports = {
const keys = uids.map(uid => `uid:${uid}:uploads`); const keys = uids.map(uid => `uid:${uid}:uploads`);
const userUploadData = await db.getSortedSetsMembersWithScores(keys); const userUploadData = await db.getSortedSetsMembersWithScores(keys);
await Promise.all(userUploadData.map(async (allUserUploads, idx) => {
const uid = uids[idx];
if (Array.isArray(allUserUploads)) {
await batch.processArray(allUserUploads, async (userUploads) => {
const bulkAdd = []; const bulkAdd = [];
const bulkRemove = []; const bulkRemove = [];
const promises = []; const promises = [];
userUploadData.forEach((userUploads, idx) => {
const uid = uids[idx];
if (Array.isArray(userUploads)) {
userUploads.forEach((userUpload) => { userUploads.forEach((userUpload) => {
const normalizedPath = normalizePath(userUpload.value); const normalizedPath = normalizePath(userUpload.value);
if (normalizedPath !== userUpload.value) { if (normalizedPath !== userUpload.value) {
@@ -106,17 +107,18 @@ module.exports = {
promises.push(db.delete(`upload:${md5(userUpload.value)}`)); promises.push(db.delete(`upload:${md5(userUpload.value)}`));
} }
}); });
}
});
await Promise.all(promises); await Promise.all(promises);
await db.sortedSetRemoveBulk(bulkRemove); await db.sortedSetRemoveBulk(bulkRemove);
await db.sortedSetAddBulk(bulkAdd); await db.sortedSetAddBulk(bulkAdd);
progress.incr(uids.length);
}, { }, {
batch: 500, batch: 500,
}); });
}
}));
progress.incr(uids.length);
}, {
batch: 100,
});
}, },
}; };