Nemcio/NodeBB
1
0
Fork 0
mirror of https://github.com/NodeBB/NodeBB.git synced 2025-10-26 08:36:12 +01:00
Code Issues Packages Projects Releases Wiki Activity

fix: #5570, create per category user post zsets

Browse Source
This commit is contained in:
Barış Soner Uşaklı
2019-06-24 15:21:43 -04:00
parent 4e513cf38a
commit a39f0ef592
15 changed files with 316 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
public/src/client/account/topics.js
View File

@@ -3,15 +3,19 @@
define('forum/account/topics', ['forum/account/header', 'forum/infinitescroll'], function (header, infinitescroll) {
var AccountTopics = {};
var method;
var template;
var set;
AccountTopics.init = function () {
header.init();
AccountTopics.handleInfiniteScroll('account/topics', 'uid:' + ajaxify.data.theirid + ':topics');
AccountTopics.handleInfiniteScroll('topics.loadMoreUserTopics', 'account/topics');
};
AccountTopics.handleInfiniteScroll = function (_template, _set) {
AccountTopics.handleInfiniteScroll = function (_method, _template, _set) {
method = _method;
template = _template;
set = _set;
if (!config.usePagination) {
@@ -24,8 +28,9 @@ define('forum/account/topics', ['forum/account/header', 'forum/infinitescroll'],
return;
}
infinitescroll.loadMore('topics.loadMoreFromSet', {
infinitescroll.loadMore(method, {
set: set,
uid: ajaxify.data.theirid,
after: $('[component="category"]').attr('data-nextstart'),
count: config.topicsPerPage,
}, function (data, done) {
@@ -40,7 +45,7 @@ define('forum/account/topics', ['forum/account/header', 'forum/infinitescroll'],
}
function onTopicsLoaded(topics, callback) {
app.parseAndTranslate('account/topics', 'topics', { topics: topics }, function (html) {
app.parseAndTranslate(template, 'topics', { topics: topics }, function (html) {
$('[component="category"]').append(html);
html.find('.timeago').timeago();
app.createUserTooltips();

2
public/src/client/account/watched.js
View File

@@ -7,7 +7,7 @@ define('forum/account/watched', ['forum/account/header', 'forum/account/topics']
AccountWatched.init = function () {
header.init();
topics.handleInfiniteScroll('account/watched', 'uid:' + ajaxify.data.theirid + ':followed_tids');
topics.handleInfiniteScroll('topics.loadMoreFromSet', 'account/watched', 'uid:' + ajaxify.data.theirid + ':followed_tids');
};
return AccountWatched;

25
src/categories/recentreplies.js
View File

@@ -204,12 +204,10 @@ module.exports = function (Categories) {
batch.processArray(pids, function (pids, next) {
async.waterfall([
function (next) {
posts.getPostsFields(pids, ['timestamp'], next);
posts.getPostsFields(pids, ['pid', 'uid', 'timestamp', 'upvotes', 'downvotes'], next);
},
function (postData, next) {
var timestamps = postData.map(function (post) {
return post && post.timestamp;
});
var timestamps = postData.map(p => p && p.timestamp);
async.parallel([
function (next) {
@@ -218,6 +216,25 @@ module.exports = function (Categories) {
function (next) {
db.sortedSetAdd('cid:' + cid + ':pids', timestamps, pids, next);
},
function (next) {
async.each(postData, function (post, next) {
db.sortedSetRemove([
'cid:' + oldCid + ':uid:' + post.uid + ':pids',
'cid:' + oldCid + ':uid:' + post.uid + ':pids:votes',
], post.pid, next);
}, next);
},
function (next) {
async.each(postData, function (post, next) {
const keys = ['cid:' + cid + ':uid:' + post.uid + ':pids'];
const scores = [post.timestamp];
if (post.votes > 0) {
keys.push('cid:' + cid + ':uid:' + post.uid + ':pids:votes');
scores.push(post.votes);
}
db.sortedSetsAdd(keys, scores, post.pid, next);
}, next);
},
], next);
},
], next);

77
src/controllers/accounts/posts.js
View File

@@ -7,6 +7,7 @@ var db = require('../../database');
var user = require('../../user');
var posts = require('../../posts');
var topics = require('../../topics');
var categories = require('../../categories');
var pagination = require('../../pagination');
var helpers = require('../helpers');
var accountHelpers = require('./helpers');
@@ -15,52 +16,89 @@ var postsController = module.exports;
var templateToData = {
'account/bookmarks': {
set: 'bookmarks',
type: 'posts',
noItemsFoundKey: '[[topic:bookmarks.has_no_bookmarks]]',
crumb: '[[user:bookmarks]]',
getSets: function (callerUid, userData, calback) {
setImmediate(calback, null, 'uid:' + userData.uid + ':bookmarks');
},
},
'account/posts': {
set: 'posts',
type: 'posts',
noItemsFoundKey: '[[user:has_no_posts]]',
crumb: '[[global:posts]]',
getSets: function (callerUid, userData, callback) {
async.waterfall([
function (next) {
categories.getCidsByPrivilege('categories:cid', callerUid, 'topics:read', next);
},
function (cids, next) {
next(null, cids.map(c => 'cid:' + c + ':uid:' + userData.uid + ':pids'));
},
], callback);
},
},
'account/upvoted': {
set: 'upvote',
type: 'posts',
noItemsFoundKey: '[[user:has_no_upvoted_posts]]',
crumb: '[[global:upvoted]]',
getSets: function (callerUid, userData, calback) {
setImmediate(calback, null, 'uid:' + userData.uid + ':upvote');
},
},
'account/downvoted': {
set: 'downvote',
type: 'posts',
noItemsFoundKey: '[[user:has_no_downvoted_posts]]',
crumb: '[[global:downvoted]]',
getSets: function (callerUid, userData, calback) {
setImmediate(calback, null, 'uid:' + userData.uid + ':downvote');
},
},
'account/best': {
set: 'posts:votes',
type: 'posts',
noItemsFoundKey: '[[user:has_no_voted_posts]]',
crumb: '[[global:best]]',
getSets: function (callerUid, userData, callback) {
async.waterfall([
function (next) {
categories.getCidsByPrivilege('categories:cid', callerUid, 'topics:read', next);
},
function (cids, next) {
next(null, cids.map(c => 'cid:' + c + ':uid:' + userData.uid + ':pids:votes'));
},
], callback);
},
},
'account/watched': {
set: 'followed_tids',
type: 'topics',
noItemsFoundKey: '[[user:has_no_watched_topics]]',
crumb: '[[user:watched]]',
getSets: function (callerUid, userData, calback) {
setImmediate(calback, null, 'uid:' + userData.uid + ':followed_tids');
},
},
'account/ignored': {
set: 'ignored_tids',
type: 'topics',
noItemsFoundKey: '[[user:has_no_ignored_topics]]',
crumb: '[[user:ignored]]',
getSets: function (callerUid, userData, calback) {
setImmediate(calback, null, 'uid:' + userData.uid + ':ignored_tids');
},
},
'account/topics': {
set: 'topics',
type: 'topics',
noItemsFoundKey: '[[user:has_no_topics]]',
crumb: '[[global:topics]]',
getSets: function (callerUid, userData, callback) {
async.waterfall([
function (next) {
categories.getCidsByPrivilege('categories:cid', callerUid, 'topics:read', next);
},
function (cids, next) {
next(null, cids.map(c => 'cid:' + c + ':uid:' + userData.uid + ':tids'));
},
], callback);
},
},
};
@@ -98,9 +136,8 @@ postsController.getTopics = function (req, res, next) {
function getFromUserSet(template, req, res, callback) {
var data = templateToData[template];
data.template = template;
data.method = data.type === 'posts' ? posts.getPostSummariesFromSet : topics.getTopicsFromSet;
var userData;
var settings;
var itemsPerPage;
var page = Math.max(1, parseInt(req.query.page, 10) || 1);
@@ -121,15 +158,16 @@ function getFromUserSet(template, req, res, callback) {
}
userData = results.userData;
settings = results.settings;
itemsPerPage = data.type === 'topics' ? settings.topicsPerPage : settings.postsPerPage;
var setName = 'uid:' + userData.uid + ':' + data.set;
itemsPerPage = (data.template === 'account/topics' || data.template === 'account/watched') ? results.settings.topicsPerPage : results.settings.postsPerPage;
data.getSets(req.uid, userData, next);
},
function (sets, next) {
async.parallel({
itemCount: function (next) {
if (results.settings.usePagination) {
db.sortedSetCard(setName, next);
if (settings.usePagination) {
db.sortedSetsCardSum(sets, next);
} else {
next(null, 0);
}
@@ -137,7 +175,8 @@ function getFromUserSet(template, req, res, callback) {
data: function (next) {
var start = (page - 1) * itemsPerPage;
var stop = start + itemsPerPage - 1;
data.method(setName, req.uid, start, stop, next);
const method = data.type === 'topics' ? topics.getTopicsFromSet : posts.getPostSummariesFromSet;
method(sets, req.uid, start, stop, next);
},
}, next);
},
@@ -149,10 +188,10 @@ function getFromUserSet(template, req, res, callback) {
userData.pagination = pagination.create(page, pageCount);
userData.noItemsFoundKey = data.noItemsFoundKey;
userData.title = '[[pages:' + data.template + ', ' + userData.username + ']]';
userData.title = '[[pages:' + template + ', ' + userData.username + ']]';
userData.breadcrumbs = helpers.buildBreadcrumbs([{ text: userData.username, url: '/user/' + userData.userslug }, { text: data.crumb }]);
res.render(data.template, userData);
res.render(template, userData);
},
], callback);
}

29
src/controllers/accounts/profile.js
View File

@@ -4,9 +4,9 @@ var nconf = require('nconf');
var async = require('async');
const db = require('../../database');
const privileges = require('../../privileges');
var user = require('../../user');
var posts = require('../../posts');
const categories = require('../../categories');
var plugins = require('../../plugins');
var meta = require('../../meta');
var accountHelpers = require('./helpers');
@@ -103,34 +103,23 @@ profileController.get = function (req, res, callback) {
};
function getLatestPosts(callerUid, userData, callback) {
async.waterfall([
function (next) {
db.getSortedSetRevRange('uid:' + userData.uid + ':posts', 0, 99, next);
},
function (pids, next) {
getPosts(callerUid, pids, next);
},
], callback);
getPosts(callerUid, userData, 'pids', callback);
}
function getBestPosts(callerUid, userData, callback) {
async.waterfall([
function (next) {
db.getSortedSetRevRange('uid:' + userData.uid + ':posts:votes', 0, 99, next);
},
function (pids, next) {
getPosts(callerUid, pids, next);
},
], callback);
getPosts(callerUid, userData, 'pids:votes', callback);
}
function getPosts(callerUid, pids, callback) {
function getPosts(callerUid, userData, setSuffix, callback) {
async.waterfall([
function (next) {
privileges.posts.filter('topics:read', pids, callerUid, next);
categories.getCidsByPrivilege('categories:cid', callerUid, 'topics:read', next);
},
function (cids, next) {
const keys = cids.map(c => 'cid:' + c + ':uid:' + userData.uid + ':' + setSuffix);
db.getSortedSetRevRange(keys, 0, 9, next);
},
function (pids, next) {
pids = pids.slice(0, 10);
posts.getPostSummaryByPids(pids, callerUid, { stripTags: false }, next);
},
], callback);

11
src/database/mongo/sorted.js
View File

@@ -162,6 +162,17 @@ module.exports = function (db, module) {
async.map(keys, module.sortedSetCard, callback);
};
module.sortedSetsCardSum = function (keys, callback) {
if (!keys || (Array.isArray(keys) && !keys.length)) {
return callback(null, 0);
}
db.collection('objects').countDocuments({ _key: Array.isArray(keys) ? { $in: keys } : keys }, function (err, count) {
count = parseInt(count, 10);
callback(err, count || 0);
});
};
module.sortedSetRank = function (key, value, callback) {
getSortedSetRank(false, key, value, callback);
};

16
src/database/postgres/sorted.js
View File

@@ -259,6 +259,22 @@ SELECT o."_key" k,
});
};
module.sortedSetsCardSum = function (keys, callback) {
if (!keys || (Array.isArray(keys) && !keys.length)) {
return callback(null, 0);
}
if (!Array.isArray(keys)) {
keys = [keys];
}
module.sortedSetsCard(keys, function (err, counts) {
if (err) {
return callback(err);
}
const sum = counts.reduce(function (acc, val) { return acc + val; }, 0);
callback(null, sum);
});
};
module.sortedSetRank = function (key, value, callback) {
getSortedSetRank('ASC', [key], [value], function (err, result) {
callback(err, result ? result[0] : null);

16
src/database/redis/sorted.js
View File

@@ -129,6 +129,22 @@ module.exports = function (redisClient, module) {
batch.exec(callback);
};
module.sortedSetsCardSum = function (keys, callback) {
if (!keys || (Array.isArray(keys) && !keys.length)) {
return callback(null, 0);
}
if (!Array.isArray(keys)) {
keys = [keys];
}
module.sortedSetsCard(keys, function (err, counts) {
if (err) {
return callback(err);
}
const sum = counts.reduce(function (acc, val) { return acc + val; }, 0);
callback(null, sum);
});
};
module.sortedSetRank = function (key, value, callback) {
redisClient.zrank(key, value, callback);
};

2
src/posts/delete.js
View File

@@ -111,6 +111,8 @@ module.exports = function (Posts) {
const tasks = [
async.apply(db.decrObjectField, 'global', 'postCount'),
async.apply(db.decrObjectField, 'category:' + topicData.cid, 'post_count'),
async.apply(db.sortedSetRemove, 'cid:' + topicData.cid + ':uid:' + postData.uid + ':pids', postData.pid),
async.apply(db.sortedSetRemove, 'cid:' + topicData.cid + ':uid:' + postData.uid + ':pids:votes', postData.pid),
async.apply(topics.decreasePostCount, postData.tid),
async.apply(topics.updateTeaser, postData.tid),
async.apply(topics.updateLastPostTimeFromLastPid, postData.tid),

69
src/posts/index.js
View File

@@ -144,46 +144,47 @@ Posts.updatePostVoteCount = function (postData, callback) {
}
async.parallel([
function (next) {
if (postData.uid) {
if (postData.votes > 0) {
db.sortedSetAdd('uid:' + postData.uid + ':posts:votes', postData.votes, postData.pid, next);
} else {
db.sortedSetRemove('uid:' + postData.uid + ':posts:votes', postData.pid, next);
}
} else {
next();
}
},
function (next) {
let cid;
async.waterfall([
function (next) {
topics.getTopicFields(postData.tid, ['mainPid', 'cid', 'pinned'], next);
},
function (topicData, next) {
if (parseInt(topicData.mainPid, 10) === parseInt(postData.pid, 10)) {
async.parallel([
function (next) {
topics.setTopicFields(postData.tid, {
upvotes: postData.upvotes,
downvotes: postData.downvotes,
}, next);
},
function (next) {
db.sortedSetAdd('topics:votes', postData.votes, postData.tid, next);
},
function (next) {
if (!topicData.pinned) {
db.sortedSetAdd('cid:' + topicData.cid + ':tids:votes', postData.votes, postData.tid, next);
} else {
next();
}
},
], function (err) {
next(err);
});
return;
cid = topicData.cid;
if (parseInt(topicData.mainPid, 10) !== parseInt(postData.pid, 10)) {
return db.sortedSetAdd('tid:' + postData.tid + ':posts:votes', postData.votes, postData.pid, next);
}
async.parallel([
function (next) {
topics.setTopicFields(postData.tid, {
upvotes: postData.upvotes,
downvotes: postData.downvotes,
}, next);
},
function (next) {
db.sortedSetAdd('topics:votes', postData.votes, postData.tid, next);
},
function (next) {
if (!topicData.pinned) {
db.sortedSetAdd('cid:' + topicData.cid + ':tids:votes', postData.votes, postData.tid, next);
} else {
next();
}
},
], function (err) {
next(err);
});
},
function (next) {
if (postData.uid) {
if (postData.votes > 0) {
db.sortedSetAdd('cid:' + cid + ':uid:' + postData.uid + ':pids:votes', postData.votes, postData.pid, next);
} else {
db.sortedSetRemove('cid:' + cid + ':uid:' + postData.uid + ':pids:votes', postData.pid, next);
}
} else {
next();
}
db.sortedSetAdd('tid:' + postData.tid + ':posts:votes', postData.votes, postData.pid, next);
},
], next);
},

21
src/socket.io/posts.js
View File

@@ -7,6 +7,7 @@ var privileges = require('../privileges');
var plugins = require('../plugins');
var meta = require('../meta');
var topics = require('../topics');
const categories = require('../categories');
var user = require('../user');
var websockets = require('./index');
var socketHelpers = require('./helpers');
@@ -135,11 +136,27 @@ SocketPosts.loadMoreBookmarks = function (socket, data, callback) {
};
SocketPosts.loadMoreUserPosts = function (socket, data, callback) {
loadMorePosts('uid:' + data.uid + ':posts', socket.uid, data, callback);
async.waterfall([
function (next) {
categories.getCidsByPrivilege('categories:cid', socket.uid, 'topics:read', next);
},
function (cids, next) {
const keys = cids.map(c => 'cid:' + c + ':uid:' + data.uid + ':pids');
loadMorePosts(keys, socket.uid, data, next);
},
], callback);
};
SocketPosts.loadMoreBestPosts = function (socket, data, callback) {
loadMorePosts('uid:' + data.uid + ':posts:votes', socket.uid, data, callback);
async.waterfall([
function (next) {
categories.getCidsByPrivilege('categories:cid', socket.uid, 'topics:read', next);
},
function (cids, next) {
const keys = cids.map(c => 'cid:' + c + ':uid:' + data.uid + ':pids:votes');
loadMorePosts(keys, socket.uid, data, next);
},
], callback);
};
SocketPosts.loadMoreUpVotedPosts = function (socket, data, callback) {

13
src/socket.io/topics/infinitescroll.js
View File

@@ -3,6 +3,7 @@
var async = require('async');
var topics = require('../../topics');
const categories = require('../../categories');
var privileges = require('../../privileges');
var meta = require('../../meta');
var utils = require('../../utils');
@@ -117,6 +118,18 @@ module.exports = function (SocketTopics) {
topics.getTopicsFromSet(data.set, socket.uid, start, stop, callback);
};
SocketTopics.loadMoreUserTopics = function (socket, data, callback) {
async.waterfall([
function (next) {
categories.getCidsByPrivilege('categories:cid', socket.uid, 'topics:read', next);
},
function (cids, next) {
data.set = cids.map(c => 'cid:' + c + ':uid:' + data.uid + ':tids');
SocketTopics.loadMoreFromSet(socket, data, next);
},
], callback);
};
function calculateStartStop(data) {
var itemsPerPage = Math.min(meta.config.topicsPerPage || 20, parseInt(data.count, 10) || meta.config.topicsPerPage || 20);
var start = Math.max(0, parseInt(data.after, 10));

17
src/topics/fork.js
View File

@@ -158,13 +158,22 @@ module.exports = function (Topics) {
if (topicData[0].cid === topicData[1].cid) {
return callback();
}
async.parallel([
const removeFrom = [
'cid:' + topicData[0].cid + ':pids',
'cid:' + topicData[0].cid + ':uid:' + postData.uid + ':pids',
'cid:' + topicData[0].cid + ':uid:' + postData.uid + ':pids:votes',
];
const tasks = [
async.apply(db.incrObjectFieldBy, 'category:' + topicData[0].cid, 'post_count', -1),
async.apply(db.incrObjectFieldBy, 'category:' + topicData[1].cid, 'post_count', 1),
async.apply(db.sortedSetRemove, 'cid:' + topicData[0].cid + ':pids', postData.pid),
async.apply(db.sortedSetRemove, removeFrom, postData.pid),
async.apply(db.sortedSetAdd, 'cid:' + topicData[1].cid + ':pids', postData.timestamp, postData.pid),
], next);
async.apply(db.sortedSetAdd, 'cid:' + topicData[1].cid + ':uid:' + postData.uid + ':pids', postData.timestamp, postData.pid),
];
if (postData.votes > 0) {
tasks.push(async.apply(db.sortedSetAdd, 'cid:' + topicData[1].cid + ':uid:' + postData.uid + ':pids:votes', postData.votes, postData.pid));
}
async.parallel(tasks, next);
},
], callback);
}

54
src/upgrades/1.12.3/user_pid_sets.js Normal file
View File

@@ -0,0 +1,54 @@
'use strict';
const async = require('async');
const db = require('../../database');
const batch = require('../../batch');
const posts = require('../../posts');
const topics = require('../../topics');
module.exports = {
name: 'Create zsets for user posts per category',
timestamp: Date.UTC(2019, 5, 23),
method: function (callback) {
const progress = this.progress;
batch.processSortedSet('posts:pid', function (pids, next) {
async.eachSeries(pids, function (pid, _next) {
progress.incr();
let postData;
async.waterfall([
function (next) {
posts.getPostFields(pid, ['uid', 'tid', 'upvotes', 'downvotes', 'timestamp'], next);
},
function (_postData, next) {
if (!_postData.uid || !_postData.tid) {
return _next();
}
postData = _postData;
topics.getTopicField(postData.tid, 'cid', next);
},
function (cid, next) {
const keys = [
'cid:' + cid + ':uid:' + postData.uid + ':pids',
];
const scores = [
postData.timestamp,
];
if (postData.votes > 0) {
keys.push('cid:' + cid + ':uid:' + postData.uid + ':pids:votes');
scores.push(postData.votes);
}
db.sortedSetsAdd(keys, scores, pid, next);
},
function (next) {
db.sortedSetRemove('uid:' + postData.uid + ':posts:votes', pid, next);
},
], _next);
}, next);
}, {
progress: progress,
}, callback);
},
};

40
test/database/sorted.js
View File

@@ -392,7 +392,7 @@ describe('Sorted Set methods', function () {
describe('sortedSetsCard()', function () {
it('should return the number of elements in sorted sets', function (done) {
db.sortedSetsCard(['sortedSetTest1', 'sortedSetTest2', 'doesnotexist'], function (err, counts) {
assert.equal(err, null);
assert.ifError(err);
assert.equal(arguments.length, 2);
assert.deepEqual(counts, [3, 2, 0]);
done();
@@ -418,6 +418,44 @@ describe('Sorted Set methods', function () {
});
});
describe('sortedSetsCardSum()', function () {
it('should return the total number of elements in sorted sets', function (done) {
db.sortedSetsCardSum(['sortedSetTest1', 'sortedSetTest2', 'doesnotexist'], function (err, sum) {
assert.ifError(err);
assert.equal(arguments.length, 2);
assert.equal(sum, 5);
done();
});
});
it('should return 0 if keys is falsy', function (done) {
db.sortedSetsCardSum(undefined, function (err, counts) {
assert.ifError(err);
assert.equal(arguments.length, 2);
assert.deepEqual(counts, 0);
done();
});
});
it('should return 0 if keys is empty array', function (done) {
db.sortedSetsCardSum([], function (err, counts) {
assert.ifError(err);
assert.equal(arguments.length, 2);
assert.deepEqual(counts, 0);
done();
});
});
it('should return the total number of elements in sorted set', function (done) {
db.sortedSetsCardSum('sortedSetTest1', function (err, sum) {
assert.ifError(err);
assert.equal(arguments.length, 2);
assert.equal(sum, 3);
done();
});
});
});
describe('sortedSetRank()', function () {
it('should return falsy if sorted set does not exist', function (done) {
db.sortedSetRank('doesnotexist', 'value1', function (err, rank) {