Nemcio/NodeBB
1
0
Fork 0
mirror of https://github.com/NodeBB/NodeBB.git synced 2025-11-01 03:26:04 +01:00
Code Issues Packages Projects Releases Wiki Activity

convert uid mappings to sorted sets

Browse Source 
email:uid, username:uid, userslug:uid, fullname:uid all converted to
sorted sets
prevents hitting mongodb document size limit
This commit is contained in:
Barış Soner Uşaklı
2015-05-07 13:43:06 -04:00
parent 073afe4db0
commit c56b30ff60
9 changed files with 120 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
src/upgrade.js
View File

@@ -21,7 +21,7 @@ var db = require('./database'),
schemaDate, thisSchemaDate,
// IMPORTANT: REMEMBER TO UPDATE VALUE OF latestSchema
latestSchema = Date.UTC(2015, 1, 25, 6);
latestSchema = Date.UTC(2015, 4, 7);
Upgrade.check = function(callback) {
db.get('schemaDate', function(err, value) {
@@ -938,7 +938,7 @@ Upgrade.upgrade = function(callback) {
}
winston.info('[2015/02/24] Upgrading privilege groups to system groups done');
Upgrade.update(thisSchemaDate, next);
})
});
});
} else {
winston.info('[2015/02/24] Upgrading privilege groups to system groups skipped');
@@ -963,6 +963,48 @@ Upgrade.upgrade = function(callback) {
winston.info('[2015/02/25] Upgrading menu items to dynamic navigation system skipped');
next();
}
},
function(next) {
function upgradeHashToSortedSet(hash, callback) {
db.getObject(hash, function(err, oldHash) {
if (err) {
return callback(err);
}
db.rename(hash, hash + '_old', function(err) {
if (err) {
return callback(err);
}
var keys = Object.keys(oldHash);
async.each(keys, function(key, next) {
db.sortedSetAdd(hash, oldHash[key], key, next);
}, callback);
});
});
}
thisSchemaDate = Date.UTC(2015, 4, 7);
if (schemaDate < thisSchemaDate) {
updatesMade = true;
winston.info('[2015/02/25] Upgrading uid mappings to sorted set');
async.series([
async.apply(upgradeHashToSortedSet, 'email:uid'),
async.apply(upgradeHashToSortedSet, 'fullname:uid'),
async.apply(upgradeHashToSortedSet, 'username:uid'),
async.apply(upgradeHashToSortedSet, 'userslug:uid'),
], function(err) {
if (err) {
return next(err);
}
winston.info('[2015/05/07] Upgrading uid mappings to sorted set done');
Upgrade.update(thisSchemaDate, next);
});
} else {
winston.info('[2015/05/07] Upgrading uid mappings to sorted set skipped');
next();
}
}
// Add new schema updates here

18
src/user.js
View File

@@ -334,26 +334,18 @@ var async = require('async'),
if (!username) {
return callback();
}
db.getObjectField('username:uid', username, callback);
db.sortedSetScore('username:uid', username, callback);
};
User.getUidsByUsernames = function(usernames, callback) {
db.getObjectFields('username:uid', usernames, function(err, users) {
if (err) {
return callback(err);
}
var uids = usernames.map(function(username) {
return users[username];
});
callback(null, uids);
});
db.sortedSetScores('username:uid', usernames, callback);
};
User.getUidByUserslug = function(userslug, callback) {
if (!userslug) {
return callback();
}
db.getObjectField('userslug:uid', userslug, callback);
db.sortedSetScore('userslug:uid', userslug, callback);
};
User.getUsernamesByUids = function(uids, callback) {
@@ -382,11 +374,11 @@ var async = require('async'),
};
User.getUidByEmail = function(email, callback) {
db.getObjectField('email:uid', email.toLowerCase(), callback);
db.sortedSetScore('email:uid', email.toLowerCase(), callback);
};
User.getUsernameByEmail = function(email, callback) {
db.getObjectField('email:uid', email.toLowerCase(), function(err, uid) {
db.sortedSetScore('email:uid', email.toLowerCase(), function(err, uid) {
if (err) {
return callback(err);
}

2
src/user/admin.js
View File

@@ -32,7 +32,7 @@ module.exports = function(User) {
async.waterfall([
function(next) {
db.getObjectValues('username:uid', next);
db.getSortedSetRange('username:uid', 0, -1, next);
},
function(uids, next) {
User.getMultipleUserFields(uids, ['uid', 'email', 'username'], next);

6
src/user/create.js
View File

@@ -91,10 +91,10 @@ module.exports = function(User) {
function(next) {
async.parallel([
function(next) {
db.setObjectField('username:uid', userData.username, userData.uid, next);
db.sortedSetAdd('username:uid', userData.uid, userData.username, next);
},
function(next) {
db.setObjectField('userslug:uid', userData.userslug, userData.uid, next);
db.sortedSetAdd('userslug:uid', userData.uid, userData.userslug, next);
},
function(next) {
db.sortedSetAdd('users:joindate', timestamp, userData.uid, next);
@@ -107,7 +107,7 @@ module.exports = function(User) {
},
function(next) {
if (userData.email) {
db.setObjectField('email:uid', userData.email.toLowerCase(), userData.uid, next);
db.sortedSetAdd('email:uid', userData.uid, userData.email.toLowerCase(), next);
if (parseInt(userData.uid, 10) !== 1 && parseInt(meta.config.requireEmailConfirmation, 10) === 1) {
User.email.sendValidationEmail(userData.uid, userData.email);
}

8
src/user/delete.js
View File

@@ -49,17 +49,17 @@ module.exports = function(User) {
async.parallel([
function(next) {
db.deleteObjectField('username:uid', userData.username, next);
db.sortedSetRemove('username:uid', userData.username, next);
},
function(next) {
db.deleteObjectField('userslug:uid', userData.userslug, next);
db.sortedSetRemove('userslug:uid', userData.userslug, next);
},
function(next) {
db.deleteObjectField('fullname:uid', userData.fullname, next);
db.sortedSetRemove('fullname:uid', userData.fullname, next);
},
function(next) {
if (userData.email) {
db.deleteObjectField('email:uid', userData.email.toLowerCase(), next);
db.sortedSetRemove('email:uid', userData.email.toLowerCase(), next);
} else {
next();
}

2
src/user/email.js
View File

@@ -22,7 +22,7 @@ var async = require('async'),
};
UserEmail.available = function(email, callback) {
db.isObjectField('email:uid', email.toLowerCase(), function(err, exists) {
db.isSortedSetMember('email:uid', email.toLowerCase(), function(err, exists) {
callback(err, !exists);
});
};

67
src/user/profile.js
View File

@@ -165,7 +165,7 @@ module.exports = function(User) {
return callback();
}
db.deleteObjectField('email:uid', userData.email.toLowerCase(), function(err) {
db.sortedSetRemove('email:uid', userData.email.toLowerCase(), function(err) {
if (err) {
return callback(err);
}
@@ -176,7 +176,7 @@ module.exports = function(User) {
User.setUserField(uid, 'gravatarpicture', gravatarpicture, next);
},
function(next) {
db.setObjectField('email:uid', newEmail.toLowerCase(), uid, next);
db.sortedSetAdd('email:uid', uid, newEmail.toLowerCase(), next);
},
function(next) {
User.setUserField(uid, 'email', newEmail, next);
@@ -205,64 +205,51 @@ module.exports = function(User) {
}
User.getUserFields(uid, ['username', 'userslug'], function(err, userData) {
function update(field, object, value, callback) {
async.series([
function(next) {
db.deleteObjectField(field + ':uid', userData[field], next);
},
function(next) {
User.setUserField(uid, field, value, next);
},
function(next) {
db.setObjectField(object, value, uid, next);
}
], callback);
}
if (err) {
return callback(err);
}
async.parallel([
function(next) {
if (newUsername === userData.username) {
return next();
}
update('username', 'username:uid', newUsername, next);
updateUidMapping('username', uid, newUsername, userData.username, next);
},
function(next) {
var newUserslug = utils.slugify(newUsername);
if (newUserslug === userData.userslug) {
return next();
}
update('userslug', 'userslug:uid', newUserslug, next);
updateUidMapping('userslug', uid, newUserslug, userData.userslug, next);
}
], callback);
});
}
function updateUidMapping(field, uid, value, oldValue, callback) {
if (value === oldValue) {
return callback();
}
async.series([
function(next) {
db.sortedSetRemove(field + ':uid', oldValue, next);
},
function(next) {
User.setUserField(uid, field, value, next);
},
function(next) {
if (value) {
db.sortedSetAdd(field + ':uid', uid, value, next);
} else {
next();
}
}
], callback);
}
function updateFullname(uid, newFullname, callback) {
async.waterfall([
function(next) {
User.getUserField(uid, 'fullname', next);
},
function(fullname, next) {
if (newFullname === fullname) {
return callback();
}
db.deleteObjectField('fullname:uid', fullname, next);
},
function(next) {
User.setUserField(uid, 'fullname', newFullname, next);
},
function(next) {
if (newFullname) {
db.setObjectField('fullname:uid', newFullname, uid, next);
} else {
next();
}
updateUidMapping('fullname', uid, newFullname, fullname, next);
}
], callback);
}

12
src/user/search.js
View File

@@ -84,7 +84,9 @@ module.exports = function(User) {
return searchBy + ':uid';
});
db.getObjects(keys, function(err, hashes) {
async.map(keys, function(key, next) {
db.getSortedSetRangeWithScores(key, 0, -1, next);
}, function(err, hashes) {
if (err || !hashes) {
return callback(err, []);
}
@@ -97,16 +99,16 @@ module.exports = function(User) {
var resultsPerPage = parseInt(meta.config.userSearchResultsPerPage, 10) || 20;
var hardCap = resultsPerPage * 10;
for(var i=0; i<hashes.length; ++i) {
for(var field in hashes[i]) {
for (var i=0; i<hashes.length; ++i) {
for (var k=0; k<hashes[i].length; ++k) {
var field = hashes[i][k].value;
if ((startsWith && field.toLowerCase().startsWith(query)) || (!startsWith && field.toLowerCase().indexOf(query) !== -1)) {
uids.push(hashes[i][field]);
uids.push(hashes[i][k].score);
if (uids.length >= hardCap) {
break;
}
}
}
if (uids.length >= hardCap) {
break;
}

28
tests/user.js
View File

@@ -36,6 +36,7 @@ describe('User', function() {
beforeEach(function(){
userData = {
username: 'John Smith',
fullname: 'John Smith McNamara',
password: 'swordfish',
email: 'john@example.com',
callback: undefined
@@ -254,6 +255,33 @@ describe('User', function() {
});
});
describe('hash methods', function() {
it('should return uid from email', function(next) {
User.getUidByEmail('john@example.com', function(err, uid) {
assert.ifError(err);
assert.equal(parseInt(uid, 10), parseInt(testUid, 10));
done();
});
});
it('should return uid from username', function(next) {
User.getUidByUsername('John Smith', function(err, uid) {
assert.ifError(err);
assert.equal(parseInt(uid, 10), parseInt(testUid, 10));
done();
});
});
it('should return uid from userslug', function(next) {
User.getUidByUserslug('john-smith', function(err, uid) {
assert.ifError(err);
assert.equal(parseInt(uid, 10), parseInt(testUid, 10));
done();
});
});
});
after(function() {
db.flushdb();
});