convert uid mappings to sorted sets

email:uid, username:uid, userslug:uid, fullname:uid all converted to
sorted sets
prevents hitting mongodb document size limit

src/upgrade.js

@@ -21,7 +21,7 @@ var db = require('./database'),
 	schemaDate, thisSchemaDate,
 
 	// IMPORTANT: REMEMBER TO UPDATE VALUE OF latestSchema
-	latestSchema = Date.UTC(2015, 1, 25, 6);
+	latestSchema = Date.UTC(2015, 4, 7);
 
 Upgrade.check = function(callback) {
 	db.get('schemaDate', function(err, value) {
@@ -938,7 +938,7 @@ Upgrade.upgrade = function(callback) {
 						}
 						winston.info('[2015/02/24] Upgrading privilege groups to system groups done');
 						Upgrade.update(thisSchemaDate, next);
-					})
+					});
 				});
 			} else {
 				winston.info('[2015/02/24] Upgrading privilege groups to system groups skipped');
@@ -963,6 +963,48 @@ Upgrade.upgrade = function(callback) {
 				winston.info('[2015/02/25] Upgrading menu items to dynamic navigation system skipped');
 				next();
 			}
+		},
+		function(next) {
+			function upgradeHashToSortedSet(hash, callback) {
+				db.getObject(hash, function(err, oldHash) {
+					if (err) {
+						return callback(err);
+					}
+					db.rename(hash, hash + '_old', function(err) {
+						if (err) {
+							return callback(err);
+						}
+						var keys = Object.keys(oldHash);
+						async.each(keys, function(key, next) {
+							db.sortedSetAdd(hash, oldHash[key], key, next);
+						}, callback);
+					});
+				});
+			}
+
+			thisSchemaDate = Date.UTC(2015, 4, 7);
+			if (schemaDate < thisSchemaDate) {
+				updatesMade = true;
+				winston.info('[2015/02/25] Upgrading uid mappings to sorted set');
+
+				async.series([
+					async.apply(upgradeHashToSortedSet, 'email:uid'),
+					async.apply(upgradeHashToSortedSet, 'fullname:uid'),
+					async.apply(upgradeHashToSortedSet, 'username:uid'),
+					async.apply(upgradeHashToSortedSet, 'userslug:uid'),
+				], function(err) {
+					if (err) {
+						return next(err);
+					}
+
+					winston.info('[2015/05/07] Upgrading uid mappings to sorted set done');
+					Upgrade.update(thisSchemaDate, next);
+				});
+
+			} else {
+				winston.info('[2015/05/07] Upgrading uid mappings to sorted set skipped');
+				next();
+			}
 		}
 
 		// Add new schema updates here

src/user.js

@@ -334,26 +334,18 @@ var	async = require('async'),
 		if (!username) {
 			return callback();
 		}
-		db.getObjectField('username:uid', username, callback);
+		db.sortedSetScore('username:uid', username, callback);
 	};
 
 	User.getUidsByUsernames = function(usernames, callback) {
-		db.getObjectFields('username:uid', usernames, function(err, users) {
+		db.sortedSetScores('username:uid', usernames, callback);
-			if (err) {
-				return callback(err);
-			}
-			var uids = usernames.map(function(username) {
-				return users[username];
-			});
-			callback(null, uids);
-		});
 	};
 
 	User.getUidByUserslug = function(userslug, callback) {
 		if (!userslug) {
 			return callback();
 		}
-		db.getObjectField('userslug:uid', userslug, callback);
+		db.sortedSetScore('userslug:uid', userslug, callback);
 	};
 
 	User.getUsernamesByUids = function(uids, callback) {
@@ -382,11 +374,11 @@ var	async = require('async'),
 	};
 
 	User.getUidByEmail = function(email, callback) {
-		db.getObjectField('email:uid', email.toLowerCase(), callback);
+		db.sortedSetScore('email:uid', email.toLowerCase(), callback);
 	};
 
 	User.getUsernameByEmail = function(email, callback) {
-		db.getObjectField('email:uid', email.toLowerCase(), function(err, uid) {
+		db.sortedSetScore('email:uid', email.toLowerCase(), function(err, uid) {
 			if (err) {
 				return callback(err);
 			}

src/user/admin.js

@@ -32,7 +32,7 @@ module.exports = function(User) {
 
 		async.waterfall([
 			function(next) {
-				db.getObjectValues('username:uid', next);
+				db.getSortedSetRange('username:uid', 0, -1, next);
 			},
 			function(uids, next) {
 				User.getMultipleUserFields(uids, ['uid', 'email', 'username'], next);

src/user/create.js

@@ -91,10 +91,10 @@ module.exports = function(User) {
 					function(next) {
 						async.parallel([
 							function(next) {
-								db.setObjectField('username:uid', userData.username, userData.uid, next);
+								db.sortedSetAdd('username:uid', userData.uid, userData.username, next);
 							},
 							function(next) {
-								db.setObjectField('userslug:uid', userData.userslug, userData.uid, next);
+								db.sortedSetAdd('userslug:uid', userData.uid, userData.userslug, next);
 							},
 							function(next) {
 								db.sortedSetAdd('users:joindate', timestamp, userData.uid, next);
@@ -107,7 +107,7 @@ module.exports = function(User) {
 							},
 							function(next) {
 								if (userData.email) {
-									db.setObjectField('email:uid', userData.email.toLowerCase(), userData.uid, next);
+									db.sortedSetAdd('email:uid', userData.uid, userData.email.toLowerCase(), next);
 									if (parseInt(userData.uid, 10) !== 1 && parseInt(meta.config.requireEmailConfirmation, 10) === 1) {
 										User.email.sendValidationEmail(userData.uid, userData.email);
 									}

src/user/delete.js

@@ -49,17 +49,17 @@ module.exports = function(User) {
 
 			async.parallel([
 				function(next) {
-					db.deleteObjectField('username:uid', userData.username, next);
+					db.sortedSetRemove('username:uid', userData.username, next);
 				},
 				function(next) {
-					db.deleteObjectField('userslug:uid', userData.userslug, next);
+					db.sortedSetRemove('userslug:uid', userData.userslug, next);
 				},
 				function(next) {
-					db.deleteObjectField('fullname:uid', userData.fullname, next);
+					db.sortedSetRemove('fullname:uid', userData.fullname, next);
 				},
 				function(next) {
 					if (userData.email) {
-						db.deleteObjectField('email:uid', userData.email.toLowerCase(), next);
+						db.sortedSetRemove('email:uid', userData.email.toLowerCase(), next);
 					} else {
 						next();
 					}

src/user/email.js

@@ -22,7 +22,7 @@ var async = require('async'),
 	};
 
 	UserEmail.available = function(email, callback) {
-		db.isObjectField('email:uid', email.toLowerCase(), function(err, exists) {
+		db.isSortedSetMember('email:uid', email.toLowerCase(), function(err, exists) {
 			callback(err, !exists);
 		});
 	};

src/user/profile.js

@@ -165,7 +165,7 @@ module.exports = function(User) {
 				return callback();
 			}
 
-			db.deleteObjectField('email:uid', userData.email.toLowerCase(), function(err) {
+			db.sortedSetRemove('email:uid', userData.email.toLowerCase(), function(err) {
 				if (err) {
 					return callback(err);
 				}
@@ -176,7 +176,7 @@ module.exports = function(User) {
 						User.setUserField(uid, 'gravatarpicture', gravatarpicture, next);
 					},
 					function(next) {
-						db.setObjectField('email:uid', newEmail.toLowerCase(), uid, next);
+						db.sortedSetAdd('email:uid', uid, newEmail.toLowerCase(), next);
 					},
 					function(next) {
 						User.setUserField(uid, 'email', newEmail, next);
@@ -205,64 +205,51 @@ module.exports = function(User) {
 		}
 
 		User.getUserFields(uid, ['username', 'userslug'], function(err, userData) {
-			function update(field, object, value, callback) {
-				async.series([
-					function(next) {
-						db.deleteObjectField(field + ':uid', userData[field], next);
-					},
-					function(next) {
-						User.setUserField(uid, field, value, next);
-					},
-					function(next) {
-						db.setObjectField(object, value, uid, next);
-					}
-				], callback);
-			}
-
 			if (err) {
 				return callback(err);
 			}
 
 			async.parallel([
 				function(next) {
-					if (newUsername === userData.username) {
+					updateUidMapping('username', uid, newUsername, userData.username, next);
-						return next();
-					}
-
-					update('username', 'username:uid', newUsername, next);
 				},
 				function(next) {
 					var newUserslug = utils.slugify(newUsername);
-					if (newUserslug === userData.userslug) {
+					updateUidMapping('userslug', uid, newUserslug, userData.userslug, next);
-						return next();
-					}
-
-					update('userslug', 'userslug:uid', newUserslug, next);
 				}
 			], callback);
 		});
 	}
 
+	function updateUidMapping(field, uid, value, oldValue, callback) {
+		if (value === oldValue) {
+			return callback();
+		}
+
+		async.series([
+			function(next) {
+				db.sortedSetRemove(field + ':uid', oldValue, next);
+			},
+			function(next) {
+				User.setUserField(uid, field, value, next);
+			},
+			function(next) {
+				if (value) {
+					db.sortedSetAdd(field + ':uid', uid, value, next);
+				} else {
+					next();
+				}
+			}
+		], callback);
+	}
+
 	function updateFullname(uid, newFullname, callback) {
 		async.waterfall([
 			function(next) {
 				User.getUserField(uid, 'fullname', next);
 			},
 			function(fullname, next) {
-				if (newFullname === fullname) {
+				updateUidMapping('fullname', uid, newFullname, fullname, next);
-					return callback();
-				}
-				db.deleteObjectField('fullname:uid', fullname, next);
-			},
-			function(next) {
-				User.setUserField(uid, 'fullname', newFullname, next);
-			},
-			function(next) {
-				if (newFullname) {
-					db.setObjectField('fullname:uid', newFullname, uid, next);
-				} else {
-					next();
-				}
 			}
 		], callback);
 	}

src/user/search.js

@@ -84,7 +84,9 @@ module.exports = function(User) {
 			return searchBy + ':uid';
 		});
 
-		db.getObjects(keys, function(err, hashes) {
+		async.map(keys, function(key, next) {
+			db.getSortedSetRangeWithScores(key, 0, -1, next);
+		}, function(err, hashes) {
 			if (err || !hashes) {
 				return callback(err, []);
 			}
@@ -97,16 +99,16 @@ module.exports = function(User) {
 			var resultsPerPage = parseInt(meta.config.userSearchResultsPerPage, 10) || 20;
 			var hardCap = resultsPerPage * 10;
 
-			for(var i=0; i<hashes.length; ++i) {
+			for (var i=0; i<hashes.length; ++i) {
-				for(var field in hashes[i]) {
+				for (var k=0; k<hashes[i].length; ++k) {
+					var field = hashes[i][k].value;
 					if ((startsWith && field.toLowerCase().startsWith(query)) || (!startsWith && field.toLowerCase().indexOf(query) !== -1)) {
-						uids.push(hashes[i][field]);
+						uids.push(hashes[i][k].score);
 						if (uids.length >= hardCap) {
 							break;
 						}
 					}
 				}
-
 				if (uids.length >= hardCap) {
 					break;
 				}

tests/user.js

@@ -36,6 +36,7 @@ describe('User', function() {
 	beforeEach(function(){
 		userData = {
 			username: 'John Smith',
+			fullname: 'John Smith McNamara',
 			password: 'swordfish',
 			email: 'john@example.com',
 			callback: undefined
@@ -254,6 +255,33 @@ describe('User', function() {
 		});
 	});
 
+	describe('hash methods', function() {
+
+		it('should return uid from email', function(next) {
+			User.getUidByEmail('john@example.com', function(err, uid) {
+				assert.ifError(err);
+				assert.equal(parseInt(uid, 10), parseInt(testUid, 10));
+				done();
+			});
+		});
+
+		it('should return uid from username', function(next) {
+			User.getUidByUsername('John Smith', function(err, uid) {
+				assert.ifError(err);
+				assert.equal(parseInt(uid, 10), parseInt(testUid, 10));
+				done();
+			});
+		});
+
+		it('should return uid from userslug', function(next) {
+			User.getUidByUserslug('john-smith', function(err, uid) {
+				assert.ifError(err);
+				assert.equal(parseInt(uid, 10), parseInt(testUid, 10));
+				done();
+			});
+		});
+	});
+
 	after(function() {
 		db.flushdb();
 	});