mirror of
https://github.com/NodeBB/NodeBB.git
synced 2025-10-27 00:56:13 +01:00
refactor: closes #13547, process user uploads via batch
reduce processed user count to 100 per batch
This commit is contained in:
@@ -89,13 +89,14 @@ module.exports = {
|
|||||||
const keys = uids.map(uid => `uid:${uid}:uploads`);
|
const keys = uids.map(uid => `uid:${uid}:uploads`);
|
||||||
|
|
||||||
const userUploadData = await db.getSortedSetsMembersWithScores(keys);
|
const userUploadData = await db.getSortedSetsMembersWithScores(keys);
|
||||||
|
|
||||||
|
await Promise.all(userUploadData.map(async (allUserUploads, idx) => {
|
||||||
|
const uid = uids[idx];
|
||||||
|
if (Array.isArray(allUserUploads)) {
|
||||||
|
await batch.processArray(allUserUploads, async (userUploads) => {
|
||||||
const bulkAdd = [];
|
const bulkAdd = [];
|
||||||
const bulkRemove = [];
|
const bulkRemove = [];
|
||||||
const promises = [];
|
const promises = [];
|
||||||
|
|
||||||
userUploadData.forEach((userUploads, idx) => {
|
|
||||||
const uid = uids[idx];
|
|
||||||
if (Array.isArray(userUploads)) {
|
|
||||||
userUploads.forEach((userUpload) => {
|
userUploads.forEach((userUpload) => {
|
||||||
const normalizedPath = normalizePath(userUpload.value);
|
const normalizedPath = normalizePath(userUpload.value);
|
||||||
if (normalizedPath !== userUpload.value) {
|
if (normalizedPath !== userUpload.value) {
|
||||||
@@ -106,17 +107,18 @@ module.exports = {
|
|||||||
promises.push(db.delete(`upload:${md5(userUpload.value)}`));
|
promises.push(db.delete(`upload:${md5(userUpload.value)}`));
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
await Promise.all(promises);
|
await Promise.all(promises);
|
||||||
await db.sortedSetRemoveBulk(bulkRemove);
|
await db.sortedSetRemoveBulk(bulkRemove);
|
||||||
await db.sortedSetAddBulk(bulkAdd);
|
await db.sortedSetAddBulk(bulkAdd);
|
||||||
|
|
||||||
progress.incr(uids.length);
|
|
||||||
}, {
|
}, {
|
||||||
batch: 500,
|
batch: 500,
|
||||||
});
|
});
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
|
||||||
|
progress.incr(uids.length);
|
||||||
|
}, {
|
||||||
|
batch: 100,
|
||||||
|
});
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|||||||
Reference in New Issue
Block a user