cache categories:cid and cid:<cid>:children

these rarely change, no need to go to db for them

src/cache.js

@@ -0,0 +1,56 @@
+'use strict';
+
+var LRU = require('lru-cache');
+var pubsub = require('./pubsub');
+
+var cache = LRU({
+	max: 1000,
+	maxAge: 0,
+});
+cache.hits = 0;
+cache.misses = 0;
+
+const cacheGet = cache.get;
+const cacheDel = cache.del;
+const cacheReset = cache.reset;
+
+cache.get = function (key) {
+	const data = cacheGet.apply(cache, [key]);
+	if (data === undefined) {
+		cache.misses += 1;
+	} else {
+		cache.hits += 1;
+	}
+	return data;
+};
+
+cache.del = function (key) {
+	if (!Array.isArray(key)) {
+		key = [key];
+	}
+	pubsub.publish('local:cache:del', key);
+	key.forEach(key => cacheDel.apply(cache, [key]));
+};
+
+cache.reset = function () {
+	pubsub.publish('local:cache:reset');
+	localReset();
+};
+
+function localReset() {
+	cacheReset.apply(cache);
+	cache.hits = 0;
+	cache.misses = 0;
+}
+
+pubsub.on('local:cache:reset', function () {
+	localReset();
+});
+
+pubsub.on('local:cache:del', function (keys) {
+	if (Array.isArray(keys)) {
+		keys.forEach(key => cacheDel.apply(cache, [key]));
+	}
+});
+
+module.exports = cache;

src/categories/create.js

@@ -7,6 +7,7 @@ var groups = require('../groups');
 var plugins = require('../plugins');
 var privileges = require('../privileges');
 var utils = require('../utils');
+var cache = require('../cache');
 
 module.exports = function (Categories) {
 	Categories.create = function (data, callback) {
@@ -82,6 +83,7 @@ module.exports = function (Categories) {
 				], next);
 			},
 			function (results, next) {
+				cache.del(['categories:cid', 'cid:' + parentCid + ':children']);
 				if (data.cloneFromCid && parseInt(data.cloneFromCid, 10)) {
 					return Categories.copySettingsFrom(data.cloneFromCid, category.cid, !data.parentCid, next);
 				}
@@ -153,9 +155,11 @@ module.exports = function (Categories) {
 				const newParent = parseInt(results.source.parentCid, 10) || 0;
 				if (copyParent) {
 					tasks.push(async.apply(db.sortedSetRemove, 'cid:' + oldParent + ':children', toCid));
-				}
-				if (copyParent) {
 					tasks.push(async.apply(db.sortedSetAdd, 'cid:' + newParent + ':children', results.source.order, toCid));
+					tasks.push(function (next) {
+						cache.del(['cid:' + oldParent + ':children', 'cid:' + newParent + ':children']);
+						setImmediate(next);
+					});
 				}
 
 				destination.description = results.source.description;

src/categories/data.js

@@ -56,7 +56,7 @@ module.exports = function (Categories) {
 
 	Categories.getAllCategoryFields = function (fields, callback) {
 		async.waterfall([
-			async.apply(db.getSortedSetRange, 'categories:cid', 0, -1),
+			async.apply(Categories.getAllCidsFromSet, 'categories:cid'),
 			function (cids, next) {
 				Categories.getCategoriesFields(cids, fields, next);
 			},

src/categories/delete.js

@@ -7,6 +7,7 @@ var plugins = require('../plugins');
 var topics = require('../topics');
 var groups = require('../groups');
 var privileges = require('../privileges');
+var cache = require('../cache');
 
 module.exports = function (Categories) {
 	Categories.purge = function (cid, uid, callback) {
@@ -94,7 +95,18 @@ module.exports = function (Categories) {
 							], next);
 						}, next);
 					},
-				], next);
+				], function (err) {
+					if (err) {
+						return next(err);
+					}
+					cache.del([
+						'categories:cid',
+						'cid:0:children',
+						'cid:' + results.parentCid + ':children',
+						'cid:' + cid + ':children',
+					]);
+					next();
+				});
 			},
 		], function (err) {
 			callback(err);

src/categories/index.js

@@ -9,6 +9,7 @@ var user = require('../user');
 var Groups = require('../groups');
 var plugins = require('../plugins');
 var privileges = require('../privileges');
+const cache = require('../cache');
 
 var Categories = module.exports;
 
@@ -82,10 +83,25 @@ Categories.isIgnored = function (cids, uid, callback) {
 	db.isSortedSetMembers('uid:' + uid + ':ignored:cids', cids, callback);
 };
 
+Categories.getAllCidsFromSet = function (key, callback) {
+	const cids = cache.get(key);
+	if (cids) {
+		return setImmediate(callback, null, cids.slice());
+	}
+
+	db.getSortedSetRange(key, 0, -1, function (err, cids) {
+		if (err) {
+			return callback(err);
+		}
+		cache.set(key, cids);
+		callback(null, cids.slice());
+	});
+};
+
 Categories.getAllCategories = function (uid, callback) {
 	async.waterfall([
 		function (next) {
-			db.getSortedSetRange('categories:cid', 0, -1, next);
+			Categories.getAllCids(next);
 		},
 		function (cids, next) {
 			Categories.getCategories(cids, uid, next);
@@ -96,7 +112,7 @@ Categories.getAllCategories = function (uid, callback) {
 Categories.getCidsByPrivilege = function (set, uid, privilege, callback) {
 	async.waterfall([
 		function (next) {
-			db.getSortedSetRange(set, 0, -1, next);
+			Categories.getAllCidsFromSet(set, next);
 		},
 		function (cids, next) {
 			privileges.categories.filterCids(privilege, cids, uid, next);
@@ -268,7 +284,7 @@ function getChildrenTree(category, uid, callback) {
 }
 
 Categories.getChildrenCids = function (rootCid, callback) {
-	var allCids = [];
+	let allCids = [];
 	function recursive(keys, callback) {
 		db.getSortedSetRange(keys, 0, -1, function (err, childrenCids) {
 			if (err) {
@@ -283,9 +299,19 @@ Categories.getChildrenCids = function (rootCid, callback) {
 			recursive(keys, callback);
 		});
 	}
+	const key = 'cid:' + rootCid + ':children';
+	const childrenCids = cache.get(key);
+	if (childrenCids) {
+		return setImmediate(callback, null, childrenCids.slice());
+	}
 
-	recursive('cid:' + rootCid + ':children', function (err) {
+	recursive(key, function (err) {
-		callback(err, _.uniq(allCids));
+		if (err) {
+			return callback(err);
+		}
+		allCids = _.uniq(allCids);
+		cache.set(key, allCids);
+		callback(null, allCids.slice());
 	});
 };
 

src/categories/update.js

@@ -1,4 +1,3 @@
-
 'use strict';
 
 var async = require('async');
@@ -8,6 +7,7 @@ var meta = require('../meta');
 var utils = require('../utils');
 var translator = require('../translator');
 var plugins = require('../plugins');
+var cache = require('../cache');
 
 module.exports = function (Categories) {
 	Categories.update = function (modified, callback) {
@@ -112,6 +112,10 @@ module.exports = function (Categories) {
 					function (next) {
 						db.setObjectField('category:' + cid, 'parentCid', newParent, next);
 					},
+					function (next) {
+						cache.del(['cid:' + oldParent + ':children', 'cid:' + newParent + ':children']);
+						next();
+					},
 				], next);
 			},
 		], function (err) {
@@ -149,6 +153,10 @@ module.exports = function (Categories) {
 					function (next) {
 						db.sortedSetAdd('cid:' + parentCid + ':children', order, cid, next);
 					},
+					function (next) {
+						cache.del(['categories:cid', 'cid:' + parentCid + ':children']);
+						next();
+					},
 				], next);
 			},
 		], function (err) {

src/controllers/admin/cache.js

@@ -8,6 +8,7 @@ cacheController.get = function (req, res) {
 	var postCache = require('../../posts/cache');
 	var groupCache = require('../../groups').cache;
 	var objectCache = require('../../database').objectCache;
+	var localCache = require('../../cache');
 
 	var avgPostSize = 0;
 	var percentFull = 0;
@@ -32,11 +33,20 @@ cacheController.get = function (req, res) {
 			max: groupCache.max,
 			itemCount: groupCache.itemCount,
 			percentFull: ((groupCache.length / groupCache.max) * 100).toFixed(2),
-			dump: req.query.debug ? JSON.stringify(groupCache.dump(), null, 4) : false,
 			hits: utils.addCommas(String(groupCache.hits)),
 			misses: utils.addCommas(String(groupCache.misses)),
 			hitRatio: (groupCache.hits / (groupCache.hits + groupCache.misses)).toFixed(4),
 		},
+		localCache: {
+			length: localCache.length,
+			max: localCache.max,
+			itemCount: localCache.itemCount,
+			percentFull: ((localCache.length / localCache.max) * 100).toFixed(2),
+			dump: req.query.debug ? JSON.stringify(localCache.dump(), null, 4) : false,
+			hits: utils.addCommas(String(localCache.hits)),
+			misses: utils.addCommas(String(localCache.misses)),
+			hitRatio: (localCache.hits / (localCache.hits + localCache.misses)).toFixed(4),
+		},
 	};
 
 	if (objectCache) {
@@ -45,7 +55,6 @@ cacheController.get = function (req, res) {
 			max: objectCache.max,
 			itemCount: objectCache.itemCount,
 			percentFull: ((objectCache.length / objectCache.max) * 100).toFixed(2),
-			dump: req.query.debug ? JSON.stringify(objectCache.dump(), null, 4) : false,
 			hits: utils.addCommas(String(objectCache.hits)),
 			misses: utils.addCommas(String(objectCache.misses)),
 			hitRatio: (objectCache.hits / (objectCache.hits + objectCache.misses)).toFixed(4),

src/controllers/admin/homepage.js

@@ -2,7 +2,6 @@
 
 var async = require('async');
 
-var db = require('../../database');
 var categories = require('../../categories');
 var privileges = require('../../privileges');
 var plugins = require('../../plugins');
@@ -12,7 +11,7 @@ var homePageController = module.exports;
 homePageController.get = function (req, res, next) {
 	async.waterfall([
 		function (next) {
-			db.getSortedSetRange('categories:cid', 0, -1, next);
+			categories.getAllCidsFromSet('categories:cid', next);
 		},
 		function (cids, next) {
 			privileges.categories.filterCids('find', cids, 0, next);

src/controllers/admin/privileges.js

@@ -2,7 +2,6 @@
 
 var async = require('async');
 
-var db = require('../../database');
 var categories = require('../../categories');
 var privileges = require('../../privileges');
 
@@ -23,7 +22,7 @@ privilegesController.get = function (req, res, callback) {
 				allCategories: function (next) {
 					async.waterfall([
 						function (next) {
-							db.getSortedSetRange('categories:cid', 0, -1, next);
+							categories.getAllCidsFromSet('categories:cid', next);
 						},
 						function (cids, next) {
 							categories.getCategories(cids, req.uid, next);

src/socket.io/admin/categories.js

@@ -2,7 +2,6 @@
 
 var async = require('async');
 
-var db = require('../../database');
 var groups = require('../../groups');
 var categories = require('../../categories');
 var privileges = require('../../privileges');
@@ -21,7 +20,7 @@ Categories.create = function (socket, data, callback) {
 
 Categories.getAll = function (socket, data, callback) {
 	async.waterfall([
-		async.apply(db.getSortedSetRange, 'categories:cid', 0, -1),
+		async.apply(categories.getAllCidsFromSet, 'categories:cid'),
 		async.apply(categories.getCategoriesData),
 		function (categories, next) {
 			// Hook changes, there is no req, and res

src/socket.io/categories.js

@@ -1,7 +1,7 @@
 'use strict';
 
 var async = require('async');
-var db = require('../database');
+
 var categories = require('../categories');
 var privileges = require('../privileges');
 var user = require('../user');
@@ -21,7 +21,7 @@ SocketCategories.get = function (socket, data, callback) {
 				isAdmin: async.apply(user.isAdministrator, socket.uid),
 				categories: function (next) {
 					async.waterfall([
-						async.apply(db.getSortedSetRange, 'categories:cid', 0, -1),
+						async.apply(categories.getAllCidsFromSet, 'categories:cid'),
 						async.apply(categories.getCategoriesData),
 					], next);
 				},
@@ -139,7 +139,7 @@ SocketCategories.getMoveCategories = function (socket, data, callback) {
 				categories: function (next) {
 					async.waterfall([
 						function (next) {
-							db.getSortedSetRange('categories:cid', 0, -1, next);
+							categories.getAllCidsFromSet('categories:cid', next);
 						},
 						function (cids, next) {
 							categories.getCategories(cids, socket.uid, next);
@@ -183,7 +183,7 @@ function ignoreOrWatch(fn, socket, cid, callback) {
 			user.isAdminOrGlobalModOrSelf(socket.uid, targetUid, next);
 		},
 		function (next) {
-			db.getSortedSetRange('categories:cid', 0, -1, next);
+			categories.getAllCidsFromSet('categories:cid', next);
 		},
 		function (cids, next) {
 			categories.getCategoriesFields(cids, ['cid', 'parentCid'], next);

src/user/categories.js

@@ -14,9 +14,6 @@ module.exports = function (User) {
 	};
 
 	User.getWatchedCategories = function (uid, callback) {
-		if (parseInt(uid, 10) <= 0) {
-			return setImmediate(callback, null, []);
-		}
 		async.waterfall([
 			function (next) {
 				async.parallel({
@@ -24,16 +21,13 @@ module.exports = function (User) {
 						User.getIgnoredCategories(uid, next);
 					},
 					all: function (next) {
-						db.getSortedSetRange('categories:cid', 0, -1, next);
+						categories.getAllCidsFromSet('categories:cid', next);
 					},
 				}, next);
 			},
 			function (results, next) {
 				const ignored = new Set(results.ignored);
-
+				const watched = results.all.filter(cid => cid && !ignored.has(String(cid)));
-				var watched = results.all.filter(function (cid) {
-					return cid && !ignored.has(String(cid));
-				});
 				next(null, watched);
 			},
 		], callback);

src/user/index.js

@@ -7,6 +7,7 @@ var groups = require('../groups');
 var plugins = require('../plugins');
 var db = require('../database');
 var privileges = require('../privileges');
+var categories = require('../categories');
 var meta = require('../meta');
 
 var User = module.exports;
@@ -288,7 +289,7 @@ User.getAdminsandGlobalModsandModerators = function (callback) {
 
 User.getModeratorUids = function (callback) {
 	async.waterfall([
-		async.apply(db.getSortedSetRange, 'categories:cid', 0, -1),
+		async.apply(categories.getAllCidsFromSet, 'categories:cid'),
 		function (cids, next) {
 			var groupNames = cids.reduce(function (memo, cid) {
 				memo.push('cid:' + cid + ':privileges:moderate');
@@ -321,19 +322,20 @@ User.getModeratorUids = function (callback) {
 };
 
 User.getModeratedCids = function (uid, callback) {
+	if (parseInt(uid, 10) <= 0) {
+		return setImmediate(callback, null, []);
+	}
 	var cids;
 	async.waterfall([
 		function (next) {
-			db.getSortedSetRange('categories:cid', 0, -1, next);
+			categories.getAllCidsFromSet('categories:cid', next);
 		},
 		function (_cids, next) {
 			cids = _cids;
 			User.isModerator(uid, cids, next);
 		},
 		function (isMods, next) {
-			cids = cids.filter(function (cid, index) {
+			cids = cids.filter((cid, index) => cid && isMods[index]);
-				return cid && isMods[index];
-			});
 			next(null, cids);
 		},
 	], callback);

src/views/admin/advanced/cache.tpl

@@ -4,7 +4,6 @@
 		<div class="panel panel-default">
 			<div class="panel-heading"><i class="fa fa-calendar-o"></i> [[admin/advanced/cache:post-cache]]</div>
 			<div class="panel-body">
-
 				<label>[[admin/advanced/cache:posts-in-cache]]</label><br/>
 				<span>{postCache.itemCount}</span><br/>
 
@@ -30,6 +29,7 @@
 				</div>
 			</div>
 		</div>
+
 		<!-- IF objectCache -->
 		<div class="panel panel-default">
 			<div class="panel-heading"><i class="fa fa-calendar-o"></i> Object Cache</div>
@@ -45,10 +45,6 @@
 				<label>Hits:</label> <span>{objectCache.hits}</span><br/>
 				<label>Misses:</label> <span>{objectCache.misses}</span><br/>
 				<label>Hit Ratio:</label> <span>{objectCache.hitRatio}</span><br/>
-
-				<!-- IF objectCache.dump -->
-				<pre>{objectCache.dump}</pre>
-				<!-- ENDIF objectCache.dump -->
 			</div>
 		</div>
 		<!-- ENDIF objectCache -->
@@ -69,10 +65,29 @@
 				<label>Hits:</label> <span>{groupCache.hits}</span><br/>
 				<label>Misses:</label> <span>{groupCache.misses}</span><br/>
 				<label>Hit Ratio:</label> <span>{groupCache.hitRatio}</span><br/>
+			</div>
+		</div>
 
-				<!-- IF groupCache.dump -->
+		<div class="panel panel-default">
-				<pre>{groupCache.dump}</pre>
+			<div class="panel-heading"><i class="fa fa-calendar-o"></i> Local Cache</div>
-				<!-- ENDIF groupCache.dump -->
+			<div class="panel-body">
+
+				<label>[[admin/advanced/cache:length-to-max]]</label><br/>
+				<span>{localCache.length} / {localCache.max}</span><br/>
+
+				<div class="progress">
+					<div class="progress-bar" role="progressbar" aria-valuenow="{localCache.percentFull}" aria-valuemin="0" aria-valuemax="100" style="width: {localCache.percentFull}%;">
+						[[admin/advanced/cache:percent-full, {localCache.percentFull}]]
+					</div>
+				</div>
+
+				<label>Hits:</label> <span>{localCache.hits}</span><br/>
+				<label>Misses:</label> <span>{localCache.misses}</span><br/>
+				<label>Hit Ratio:</label> <span>{localCache.hitRatio}</span><br/>
+
+				<!-- IF localCache.dump -->
+				<pre>{localCache.dump}</pre>
+				<!-- ENDIF localCache.dump -->
 
 			</div>
 		</div>

test/mocks/databasemock.js

@@ -175,6 +175,8 @@ function setupMockDefaults(callback) {
 			groups.resetCache();
 			var postCache = require('../../src/posts/cache');
 			postCache.reset();
+			var localCache = require('../../src/cache');
+			localCache.reset();
 			next();
 		},
 		function (next) {