Nemcio/NodeBB
1
0
Fork 0
mirror of https://github.com/NodeBB/NodeBB.git synced 2025-10-26 08:36:12 +01:00
Code Issues Packages Projects Releases Wiki Activity

refactor: only write analytics data on nbb that has runJobs=true

Browse Source 
prevents mongodb index error when lots of nodebbs try to write to the same key
This commit is contained in:
Barış Soner Uşaklı
2021-12-21 23:09:01 -05:00
parent a9f81215cf
commit 35fea58a57
1 changed files with 80 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

122
src/analytics.js
View File

@@ -5,27 +5,37 @@ const winston = require('winston');
const nconf = require('nconf'); const nconf = require('nconf');
const crypto = require('crypto'); const crypto = require('crypto');
const LRU = require('lru-cache'); const LRU = require('lru-cache');
const util = require('util');
const _ = require('lodash');
const sleep = util.promisify(setTimeout);
const db = require('./database'); const db = require('./database');
const utils = require('./utils'); const utils = require('./utils');
const plugins = require('./plugins'); const plugins = require('./plugins');
const meta = require('./meta'); const meta = require('./meta');
const pubsub = require('./pubsub');
const Analytics = module.exports; const Analytics = module.exports;
const secret = nconf.get('secret'); const secret = nconf.get('secret');
const counters = {}; let local = {
counters: {},
pageViews: 0,
pageViewsRegistered: 0,
pageViewsGuest: 0,
pageViewsBot: 0,
uniqueIPCount: 0,
uniquevisitors: 0,
};
const empty = _.cloneDeep(local);
const total = _.cloneDeep(local);
let pageViews = 0;
let pageViewsRegistered = 0;
let pageViewsGuest = 0;
let pageViewsBot = 0;
let uniqueIPCount = 0;
let uniquevisitors = 0;
let ipCache; let ipCache;
const runJobs = nconf.get('runJobs');
Analytics.init = async function () { Analytics.init = async function () {
ipCache = new LRU({ ipCache = new LRU({
max: parseInt(meta.config['analytics:maxCache'], 10) || 500, max: parseInt(meta.config['analytics:maxCache'], 10) || 500,
@@ -33,19 +43,47 @@ Analytics.init = async function () {
maxAge: 0, maxAge: 0,
}); });
new cronJob('*/10 * * * * *', (() => { new cronJob('*/10 * * * * *', (async () => {
Analytics.writeData(); publishLocalAnalytics();
if (runJobs) {
await sleep(2000);
await Analytics.writeData();
}
}), null, true); }), null, true);
if (runJobs) {
pubsub.on('analytics:publish', (data) => {
incrementProperties(total, data.local);
});
}
}; };
function publishLocalAnalytics() {
pubsub.publish('analytics:publish', {
local: local,
});
local = _.cloneDeep(empty);
}
function incrementProperties(obj1, obj2) {
for (const [key, value] of Object.entries(obj2)) {
if (typeof value === 'object') {
incrementProperties(obj1[key], value);
} else if (utils.isNumber(value)) {
obj1[key] = obj1[key] || 0;
obj1[key] += obj2[key];
}
}
}
Analytics.increment = function (keys, callback) { Analytics.increment = function (keys, callback) {
keys = Array.isArray(keys) ? keys : [keys]; keys = Array.isArray(keys) ? keys : [keys];
plugins.hooks.fire('action:analytics.increment', { keys: keys }); plugins.hooks.fire('action:analytics.increment', { keys: keys });
keys.forEach((key) => { keys.forEach((key) => {
counters[key] = counters[key] || 0; local.counters[key] = local.counters[key] || 0;
counters[key] += 1; local.counters[key] += 1;
}); });
if (typeof callback === 'function') { if (typeof callback === 'function') {
@@ -56,14 +94,14 @@ Analytics.increment = function (keys, callback) {
Analytics.getKeys = async () => db.getSortedSetRange('analyticsKeys', 0, -1); Analytics.getKeys = async () => db.getSortedSetRange('analyticsKeys', 0, -1);
Analytics.pageView = async function (payload) { Analytics.pageView = async function (payload) {
pageViews += 1; local.pageViews += 1;
if (payload.uid > 0) { if (payload.uid > 0) {
pageViewsRegistered += 1; local.pageViewsRegistered += 1;
} else if (payload.uid < 0) { } else if (payload.uid < 0) {
pageViewsBot += 1; local.pageViewsBot += 1;
} else { } else {
pageViewsGuest += 1; local.pageViewsGuest += 1;
} }
if (payload.ip) { if (payload.ip) {
@@ -76,12 +114,12 @@ Analytics.pageView = async function (payload) {
const score = await db.sortedSetScore('ip:recent', hash); const score = await db.sortedSetScore('ip:recent', hash);
if (!score) { if (!score) {
uniqueIPCount += 1; local.uniqueIPCount += 1;
} }
const today = new Date(); const today = new Date();
today.setHours(today.getHours(), 0, 0, 0); today.setHours(today.getHours(), 0, 0, 0);
if (!score || score < today.getTime()) { if (!score || score < today.getTime()) {
uniquevisitors += 1; local.uniquevisitors += 1;
await db.sortedSetAdd('ip:recent', Date.now(), hash); await db.sortedSetAdd('ip:recent', Date.now(), hash);
} }
} }
@@ -108,44 +146,44 @@ Analytics.writeData = async function () {
month.setMonth(month.getMonth(), 1); month.setMonth(month.getMonth(), 1);
month.setHours(0, 0, 0, 0); month.setHours(0, 0, 0, 0);
if (pageViews > 0) { if (total.pageViews > 0) {
incrByBulk.push(['analytics:pageviews', pageViews, today.getTime()]); incrByBulk.push(['analytics:pageviews', total.pageViews, today.getTime()]);
incrByBulk.push(['analytics:pageviews:month', pageViews, month.getTime()]); incrByBulk.push(['analytics:pageviews:month', total.pageViews, month.getTime()]);
pageViews = 0; total.pageViews = 0;
} }
if (pageViewsRegistered > 0) { if (total.pageViewsRegistered > 0) {
incrByBulk.push(['analytics:pageviews:registered', pageViewsRegistered, today.getTime()]); incrByBulk.push(['analytics:pageviews:registered', total.pageViewsRegistered, today.getTime()]);
incrByBulk.push(['analytics:pageviews:month:registered', pageViewsRegistered, month.getTime()]); incrByBulk.push(['analytics:pageviews:month:registered', total.pageViewsRegistered, month.getTime()]);
pageViewsRegistered = 0; total.pageViewsRegistered = 0;
} }
if (pageViewsGuest > 0) { if (total.pageViewsGuest > 0) {
incrByBulk.push(['analytics:pageviews:guest', pageViewsGuest, today.getTime()]); incrByBulk.push(['analytics:pageviews:guest', total.pageViewsGuest, today.getTime()]);
incrByBulk.push(['analytics:pageviews:month:guest', pageViewsGuest, month.getTime()]); incrByBulk.push(['analytics:pageviews:month:guest', total.pageViewsGuest, month.getTime()]);
pageViewsGuest = 0; total.pageViewsGuest = 0;
} }
if (pageViewsBot > 0) { if (total.pageViewsBot > 0) {
incrByBulk.push(['analytics:pageviews:bot', pageViewsBot, today.getTime()]); incrByBulk.push(['analytics:pageviews:bot', total.pageViewsBot, today.getTime()]);
incrByBulk.push(['analytics:pageviews:month:bot', pageViewsBot, month.getTime()]); incrByBulk.push(['analytics:pageviews:month:bot', total.pageViewsBot, month.getTime()]);
pageViewsBot = 0; total.pageViewsBot = 0;
} }
if (uniquevisitors > 0) { if (total.uniquevisitors > 0) {
incrByBulk.push(['analytics:uniquevisitors', uniquevisitors, today.getTime()]); incrByBulk.push(['analytics:uniquevisitors', total.uniquevisitors, today.getTime()]);
uniquevisitors = 0; total.uniquevisitors = 0;
} }
if (uniqueIPCount > 0) { if (total.uniqueIPCount > 0) {
dbQueue.push(db.incrObjectFieldBy('global', 'uniqueIPCount', uniqueIPCount)); dbQueue.push(db.incrObjectFieldBy('global', 'uniqueIPCount', total.uniqueIPCount));
uniqueIPCount = 0; total.uniqueIPCount = 0;
} }
for (const [key, value] of Object.entries(counters)) { for (const [key, value] of Object.entries(total.counters)) {
incrByBulk.push([`analytics:${key}`, value, today.getTime()]); incrByBulk.push([`analytics:${key}`, value, today.getTime()]);
metrics.push(key); metrics.push(key);
delete counters[key]; delete total.counters[key];
} }
if (incrByBulk.length) { if (incrByBulk.length) {
@@ -221,7 +259,7 @@ Analytics.getDailyStatsForSet = async function (set, day, numDays) {
}; };
Analytics.getUnwrittenPageviews = function () { Analytics.getUnwrittenPageviews = function () {
return pageViews; return local.pageViews;
}; };
Analytics.getSummary = async function () { Analytics.getSummary = async function () {