From 528ee335d50c88a9a163c60f0b2be884ca6f42e0 Mon Sep 17 00:00:00 2001
From: barisusakli <barisusakli@gmail.com>
Date: Wed, 6 Aug 2014 21:30:06 -0400
Subject: [PATCH] optimize getTopicsByTids and getTeasers

---
 src/categories.js             |   7 ++
 src/controllers/categories.js |  25 +----
 src/database/level/sets.js    |   4 +
 src/database/mongo/sets.js    |  18 +++
 src/database/redis/sets.js    |   8 ++
 src/privileges/categories.js  |  22 ++++
 src/topics.js                 | 206 +++++++++++++++++++++-------------
 src/topics/tags.js            |  19 ++++
 8 files changed, 210 insertions(+), 99 deletions(-)

diff --git a/src/categories.js b/src/categories.js
index 59c0829473..4011f459e0 100644
--- a/src/categories.js
+++ b/src/categories.js
@@ -296,6 +296,13 @@ var db = require('./database'),
 		db.getObjectField('category:' + cid, field, callback);
 	};
 
+	Categories.getMultipleCategoryFields = function(cids, fields, callback) {
+		var keys = cids.map(function(cid) {
+			return 'category:' + cid;
+		});
+		db.getObjectsFields(keys, fields, callback);
+	};
+
 	Categories.getCategoryFields = function(cid, fields, callback) {
 		db.getObjectFields('category:' + cid, fields, callback);
 	};
diff --git a/src/controllers/categories.js b/src/controllers/categories.js
index dffe49bcbc..506747f741 100644
--- a/src/controllers/categories.js
+++ b/src/controllers/categories.js
@@ -74,22 +74,16 @@ categoriesController.get = function(req, res, next) {
 		},
 		function(disabled, next) {
 			if (parseInt(disabled, 10) === 1) {
-				return next(new Error('category-disabled'));
+				return next(new Error('[[error:category-disabled]]'));
 			}
 
-			privileges.categories.get(cid, uid, function(err, categoryPrivileges) {
-				if (err) {
-					return next(err);
-				}
-
-				if (!categoryPrivileges.read) {
-					return next(new Error('[[error:no-privileges]]'));
-				}
-
-				next(null, categoryPrivileges);
-			});
+			privileges.categories.get(cid, uid, next);
 		},
 		function (privileges, next) {
+			if (!privileges.read) {
+				return next(new Error('[[error:no-privileges]]'));
+			}
+
 			user.getSettings(uid, function(err, settings) {
 				if (err) {
 					return next(err);
@@ -111,12 +105,6 @@ categoriesController.get = function(req, res, next) {
 						return next(err);
 					}
 
-					if (categoryData) {
-						if (parseInt(categoryData.disabled, 10) === 1) {
-							return next(new Error('[[error:category-disabled]]'));
-						}
-					}
-
 					categoryData.privileges = privileges;
 					next(err, categoryData);
 				});
@@ -189,7 +177,6 @@ categoriesController.get = function(req, res, next) {
 				active: x === parseInt(page, 10)
 			});
 		}
-
 		res.render('category', data);
 	});
 };
diff --git a/src/database/level/sets.js b/src/database/level/sets.js
index bc5aacf1b5..5d6ffabf6c 100644
--- a/src/database/level/sets.js
+++ b/src/database/level/sets.js
@@ -51,6 +51,10 @@ module.exports = function(db, module) {
 		});
 	};
 
+	module.getSetsMembers = function(keys, callback) {
+		throw new Error('not-implemented');
+	};
+
 	module.setCount = function(key, callback) {
 		module.getListRange(key, 0, -1, function(err, set) {
 			callback(err, set.length);
diff --git a/src/database/mongo/sets.js b/src/database/mongo/sets.js
index fb5f16343a..6b00ec5142 100644
--- a/src/database/mongo/sets.js
+++ b/src/database/mongo/sets.js
@@ -91,6 +91,24 @@ module.exports = function(db, module) {
 		});
 	};
 
+	module.getSetsMembers = function(keys, callback) {
+		db.collection('objects').find({_key: {$in: keys}}, {_key: 1, members: 1}).toArray(function(err, data) {
+			if (err) {
+				return callback(err);
+			}
+			var sets = {};
+			data.forEach(function(set) {
+				sets[set._key] = set.members || [];
+			});
+
+			var returnData = new Array(keys.length);
+			for(var i=0; i<keys.length; ++i) {
+				returnData[i] = sets[keys[i]] || [];
+			}
+			callback(null, returnData);
+		});
+	};
+
 	module.setCount = function(key, callback) {
 		db.collection('objects').findOne({_key:key}, function(err, data) {
 			return callback(err, data ? data.members.length : 0);
diff --git a/src/database/redis/sets.js b/src/database/redis/sets.js
index 79aca1c0b0..88db7ad36a 100644
--- a/src/database/redis/sets.js
+++ b/src/database/redis/sets.js
@@ -54,6 +54,14 @@ module.exports = function(redisClient, module) {
 		redisClient.smembers(key, callback);
 	};
 
+	module.getSetsMembers = function(keys, callback) {
+		var multi = redisClient.multi();
+		for (var i=0; i<keys.length; ++i) {
+			multi.smembers(keys[i]);
+		}
+		multi.exec(callback);
+	};
+
 	module.setCount = function(key, callback) {
 		redisClient.scard(key, callback);
 	};
diff --git a/src/privileges/categories.js b/src/privileges/categories.js
index d2527b1fa8..e7572888f7 100644
--- a/src/privileges/categories.js
+++ b/src/privileges/categories.js
@@ -104,6 +104,28 @@ module.exports = function(privileges) {
 		});
 	};
 
+	privileges.categories.isAdminOrMod = function(cids, uid, callback) {
+		async.parallel({
+			isModerators: function(next) {
+				user.isModerator(uid, cids, next);
+			},
+			isAdmin: function(next) {
+				user.isAdministrator(uid, next);
+			}
+		}, function(err, results) {
+			if (err) {
+				return callback(err);
+			}
+
+			var returnData = new Array(cids.length);
+			for (var i=0; i<cids.length; ++i) {
+				returnData[i] = results.isAdmin || results.isModerators[i];
+			}
+
+			callback(null, returnData);
+		});
+	};
+
 	privileges.categories.canMoveAllTopics = function(currentCid, targetCid, uid, callback) {
 		async.parallel({
 			isAdministrator: function(next) {
diff --git a/src/topics.js b/src/topics.js
index dcee9e4063..990d09832b 100644
--- a/src/topics.js
+++ b/src/topics.js
@@ -170,96 +170,83 @@ var async = require('async'),
 	};
 
 	Topics.getTopicsByTids = function(tids, uid, callback) {
-		if (!Array.isArray(tids) || tids.length === 0) {
+		if (!Array.isArray(tids) || !tids.length) {
 			return callback(null, []);
 		}
 
-		var categoryCache = {},
-			privilegeCache = {},
-			userCache = {};
-
-
-		function loadTopicInfo(topicData, next) {
-			if (!topicData) {
-				return next(null, null);
-			}
-
-			function isTopicVisible(topicData, topicInfo) {
-				if (parseInt(topicInfo.categoryData.disabled, 10) === 1) {
-					return false;
-				}
-				var deleted = parseInt(topicData.deleted, 10) !== 0;
-				return !deleted || (deleted && topicInfo.privileges.view_deleted) || parseInt(topicData.uid, 10) === parseInt(uid, 10);
-			}
-
-			async.parallel({
-				hasread: function(next) {
-					Topics.hasReadTopic(topicData.tid, uid, next);
-				},
-				teaser: function(next) {
-					Topics.getTeaser(topicData.tid, next);
-				},
-				privileges: function(next) {
-					if (privilegeCache[topicData.cid]) {
-						return next(null, privilegeCache[topicData.cid]);
-					}
-					privileges.categories.get(topicData.cid, uid, next);
-				},
-				categoryData: function(next) {
-					if (categoryCache[topicData.cid]) {
-						return next(null, categoryCache[topicData.cid]);
-					}
-					categories.getCategoryFields(topicData.cid, ['name', 'slug', 'icon', 'bgColor', 'color', 'disabled'], next);
-				},
-				user: function(next) {
-					if (userCache[topicData.uid]) {
-						return next(null, userCache[topicData.uid]);
-					}
-					user.getUserFields(topicData.uid, ['username', 'userslug', 'picture'], next);
-				},
-				tags: function(next) {
-					Topics.getTopicTagsObjects(topicData.tid, next);
-				}
-			}, function(err, topicInfo) {
-				if(err) {
-					return next(err);
-				}
-
-				privilegeCache[topicData.cid] = topicInfo.privileges;
-				categoryCache[topicData.cid] = topicInfo.categoryData;
-				userCache[topicData.uid] = topicInfo.user;
-
-				if (!isTopicVisible(topicData, topicInfo)) {
-					return next(null, null);
-				}
-
-				topicData.pinned = parseInt(topicData.pinned, 10) === 1;
-				topicData.locked = parseInt(topicData.locked, 10) === 1;
-				topicData.deleted = parseInt(topicData.deleted, 10) === 1;
-				topicData.unread = !(topicInfo.hasread && parseInt(uid, 10) !== 0);
-				topicData.unreplied = parseInt(topicData.postcount, 10) <= 1;
-
-				topicData.category = topicInfo.categoryData;
-				topicData.teaser = topicInfo.teaser;
-				topicData.user = topicInfo.user;
-				topicData.tags = topicInfo.tags;
-
-				next(null, topicData);
-			});
-		}
-
 		Topics.getTopicsData(tids, function(err, topics) {
+			function mapFilter(array, field) {
+				return array.map(function(topic) {
+					return topic[field];
+				}).filter(function(value, index, array) {
+					return array.indexOf(value) === index;
+				});
+			}
+
 			if (err) {
 				return callback(err);
 			}
 
-			async.mapSeries(topics, loadTopicInfo, function(err, topics) {
-				if(err) {
+			var uids = mapFilter(topics, 'uid');
+			var cids = mapFilter(topics, 'cid');
+
+			async.parallel({
+				users: function(next) {
+					user.getMultipleUserFields(uids, ['uid', 'username', 'userslug', 'picture'], next);
+				},
+				categories: function(next) {
+					categories.getMultipleCategoryFields(cids, ['cid', 'name', 'slug', 'icon', 'bgColor', 'color', 'disabled'], next);
+				},
+				hasRead: function(next) {
+					Topics.hasReadTopics(tids, uid, next);
+				},
+				isAdminOrMod: function(next) {
+					privileges.categories.isAdminOrMod(cids, uid, next);
+				},
+				teasers: function(next) {
+					Topics.getTeasers(tids, next);
+				},
+				tags: function(next) {
+					Topics.getTopicsTagsObjects(tids, next);
+				}
+			}, function(err, results) {
+				function arrayToObject(array, field) {
+					var obj = {};
+					for (var i=0; i<array.length; ++i) {
+						obj[array[i][field]] = array[i];
+					}
+					return obj;
+				}
+
+				if (err) {
 					return callback(err);
 				}
 
+				var users = arrayToObject(results.users, 'uid');
+				var categories = arrayToObject(results.categories, 'cid');
+				var isAdminOrMod = {};
+				cids.forEach(function(cid, index) {
+					isAdminOrMod[cid] = results.isAdminOrMod[index];
+				});
+
+				for (var i=0; i<topics.length; ++i) {
+					topics[i].category = categories[topics[i].cid];
+					topics[i].category.disabled = parseInt(topics[i].category.disabled, 10) === 1;
+					topics[i].user = users[topics[i].uid];
+					topics[i].teaser = results.teasers[i];
+					topics[i].tags = results.tags[i];
+
+					topics[i].pinned = parseInt(topics[i].pinned, 10) === 1;
+					topics[i].locked = parseInt(topics[i].locked, 10) === 1;
+					topics[i].deleted = parseInt(topics[i].deleted, 10) === 1;
+					topics[i].unread = !(results.hasRead[i] && parseInt(uid, 10) !== 0);
+					topics[i].unreplied = parseInt(topics[i].postcount, 10) <= 1;
+				}
+
 				topics = topics.filter(function(topic) {
-					return !!topic;
+					return !topic.category.disabled ||
+						!topic.deleted || (topic.deleted && isAdminOrMod[topic.cid]) ||
+						parseInt(topic.uid, 10) === parseInt(uid, 10);
 				});
 
 				plugins.fireHook('filter:topics.get', topics, callback);
@@ -336,12 +323,71 @@ var async = require('async'),
 	};
 
 	Topics.getTeasers = function(tids, callback) {
-
 		if(!Array.isArray(tids)) {
 			return callback(null, []);
 		}
 
-		async.map(tids, Topics.getTeaser, callback);
+		async.map(tids, function(tid, next) {
+			db.getSortedSetRevRange('tid:' + tid + ':posts', 0, 0, function(err, data) {
+				next(err, Array.isArray(data) && data.length ? data[0] : null);
+			});
+		}, function(err, pids) {
+			if (err) {
+				return callback(err);
+			}
+
+			var postKeys = pids.map(function(pid) {
+				return 'post:' + pid;
+			});
+
+			async.parallel({
+				indices: function(next) {
+					var sets = tids.map(function(tid) {
+						return 'tid:' + tid + ':posts';
+					});
+					db.sortedSetsRanks(sets, pids, next);
+				},
+				posts: function(next) {
+					db.getObjectsFields(postKeys, ['pid', 'uid', 'timestamp'], next);
+				}
+			}, function(err, results) {
+				if (err) {
+					return callback(err);
+				}
+
+				var indices = results.indices.map(function(index) {
+					if (!utils.isNumber(index)) {
+						return 1;
+					}
+					return parseInt(index, 10) + 2;
+				});
+
+				var uids = results.posts.map(function(post) {
+					return post.uid;
+				}).filter(function(uid, index, array) {
+					return array.indexOf(uid) === index;
+				});
+
+				user.getMultipleUserFields(uids, ['uid', 'username', 'userslug', 'picture'], function(err, userData) {
+					if (err) {
+						return callback(err);
+					}
+
+					var users = {};
+					userData.forEach(function(user) {
+						users[user.uid] = user;
+					});
+
+					results.posts.forEach(function(post, index) {
+						post.user = users[post.uid];
+						post.index = indices[index];
+						post.timestamp = utils.toISOString(post.timestamp);
+					});
+
+					callback(err, results.posts);
+				});
+			});
+		});
 	};
 
 	Topics.getTeaser = function(tid, callback) {
diff --git a/src/topics/tags.js b/src/topics/tags.js
index 15fd437a26..afc2b3cb28 100644
--- a/src/topics/tags.js
+++ b/src/topics/tags.js
@@ -92,6 +92,25 @@ module.exports = function(Topics) {
 		});
 	};
 
+	Topics.getTopicsTagsObjects = function(tids, callback) {
+		var sets = tids.map(function(tid) {
+			return 'topic:' + tid + ':tags';
+		});
+
+		db.getSetsMembers(sets, function(err, members) {
+			if (err) {
+				return callback(err);
+			}
+
+			members.forEach(function(tags, index) {
+				if (Array.isArray(tags)) {
+					members[index] = mapToObject(tags);
+				}
+			})
+			callback(null, members);
+		});
+	};
+
 	function mapToObject(tags) {
 		if (!tags) {
 			return tags;