Nemcio/NodeBB
1
0
Fork 0
mirror of https://github.com/NodeBB/NodeBB.git synced 2025-10-26 08:36:12 +01:00
Code Issues Packages Projects Releases Wiki Activity

refactor: async/await analytics

Browse Source
This commit is contained in:
Barış Soner Uşaklı
2019-09-23 00:04:57 -04:00
parent f0ae2887b8
commit 223c108cc6
1 changed files with 92 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

212
src/analytics.js
View File

@@ -1,25 +1,26 @@
'use strict'; 'use strict';
var cronJob = require('cron').CronJob; const cronJob = require('cron').CronJob;
var async = require('async'); const winston = require('winston');
var winston = require('winston'); const nconf = require('nconf');
var nconf = require('nconf'); const crypto = require('crypto');
var crypto = require('crypto'); const LRU = require('lru-cache');
var LRU = require('lru-cache');
var db = require('./database');
var plugins = require('./plugins');
var Analytics = module.exports; const db = require('./database');
const utils = require('./utils');
const plugins = require('./plugins');
var counters = {}; const Analytics = module.exports;
var pageViews = 0; const counters = {};
var pageViewsRegistered = 0;
var pageViewsGuest = 0; let pageViews = 0;
var pageViewsBot = 0; let pageViewsRegistered = 0;
var uniqueIPCount = 0; let pageViewsGuest = 0;
var uniquevisitors = 0; let pageViewsBot = 0;
let uniqueIPCount = 0;
let uniquevisitors = 0;
/** /**
* TODO: allow the cache's max value to be configurable. On high-traffic installs, * TODO: allow the cache's max value to be configurable. On high-traffic installs,
@@ -87,177 +88,148 @@ Analytics.pageView = function (payload) {
} }
}; };
Analytics.writeData = function (callback) { Analytics.writeData = async function () {
callback = callback || function () {}; const today = new Date();
var today = new Date(); const month = new Date();
var month = new Date(); const dbQueue = [];
var dbQueue = [];
today.setHours(today.getHours(), 0, 0, 0); today.setHours(today.getHours(), 0, 0, 0);
month.setMonth(month.getMonth(), 1); month.setMonth(month.getMonth(), 1);
month.setHours(0, 0, 0, 0); month.setHours(0, 0, 0, 0);
if (pageViews > 0) { if (pageViews > 0) {
dbQueue.push(async.apply(db.sortedSetIncrBy, 'analytics:pageviews', pageViews, today.getTime())); dbQueue.push(db.sortedSetIncrBy('analytics:pageviews', pageViews, today.getTime()));
dbQueue.push(async.apply(db.sortedSetIncrBy, 'analytics:pageviews:month', pageViews, month.getTime())); dbQueue.push(db.sortedSetIncrBy('analytics:pageviews:month', pageViews, month.getTime()));
pageViews = 0; pageViews = 0;
} }
if (pageViewsRegistered > 0) { if (pageViewsRegistered > 0) {
dbQueue.push(async.apply(db.sortedSetIncrBy, 'analytics:pageviews:registered', pageViewsRegistered, today.getTime())); dbQueue.push(db.sortedSetIncrBy('analytics:pageviews:registered', pageViewsRegistered, today.getTime()));
dbQueue.push(async.apply(db.sortedSetIncrBy, 'analytics:pageviews:month:registered', pageViewsRegistered, month.getTime())); dbQueue.push(db.sortedSetIncrBy('analytics:pageviews:month:registered', pageViewsRegistered, month.getTime()));
pageViewsRegistered = 0; pageViewsRegistered = 0;
} }
if (pageViewsGuest > 0) { if (pageViewsGuest > 0) {
dbQueue.push(async.apply(db.sortedSetIncrBy, 'analytics:pageviews:guest', pageViewsGuest, today.getTime())); dbQueue.push(db.sortedSetIncrBy('analytics:pageviews:guest', pageViewsGuest, today.getTime()));
dbQueue.push(async.apply(db.sortedSetIncrBy, 'analytics:pageviews:month:guest', pageViewsGuest, month.getTime())); dbQueue.push(db.sortedSetIncrBy('analytics:pageviews:month:guest', pageViewsGuest, month.getTime()));
pageViewsGuest = 0; pageViewsGuest = 0;
} }
if (pageViewsBot > 0) { if (pageViewsBot > 0) {
dbQueue.push(async.apply(db.sortedSetIncrBy, 'analytics:pageviews:bot', pageViewsBot, today.getTime())); dbQueue.push(db.sortedSetIncrBy('analytics:pageviews:bot', pageViewsBot, today.getTime()));
dbQueue.push(async.apply(db.sortedSetIncrBy, 'analytics:pageviews:month:bot', pageViewsBot, month.getTime())); dbQueue.push(db.sortedSetIncrBy('analytics:pageviews:month:bot', pageViewsBot, month.getTime()));
pageViewsBot = 0; pageViewsBot = 0;
} }
if (uniquevisitors > 0) { if (uniquevisitors > 0) {
dbQueue.push(async.apply(db.sortedSetIncrBy, 'analytics:uniquevisitors', uniquevisitors, today.getTime())); dbQueue.push(db.sortedSetIncrBy('analytics:uniquevisitors', uniquevisitors, today.getTime()));
uniquevisitors = 0; uniquevisitors = 0;
} }
if (uniqueIPCount > 0) { if (uniqueIPCount > 0) {
dbQueue.push(async.apply(db.incrObjectFieldBy, 'global', 'uniqueIPCount', uniqueIPCount)); dbQueue.push(db.incrObjectFieldBy('global', 'uniqueIPCount', uniqueIPCount));
uniqueIPCount = 0; uniqueIPCount = 0;
} }
if (Object.keys(counters).length > 0) { if (Object.keys(counters).length > 0) {
for (var key in counters) { for (const key in counters) {
if (counters.hasOwnProperty(key)) { if (counters.hasOwnProperty(key)) {
dbQueue.push(async.apply(db.sortedSetIncrBy, 'analytics:' + key, counters[key], today.getTime())); dbQueue.push(db.sortedSetIncrBy('analytics:' + key, counters[key], today.getTime()));
delete counters[key]; delete counters[key];
} }
} }
} }
try {
async.parallel(dbQueue, function (err) { await Promise.all(dbQueue);
if (err) { } catch (err) {
winston.error('[analytics] Encountered error while writing analytics to data store', err); winston.error('[analytics] Encountered error while writing analytics to data store', err);
} throw err;
callback(err); }
});
}; };
Analytics.getHourlyStatsForSet = function (set, hour, numHours, callback) { Analytics.getHourlyStatsForSet = async function (set, hour, numHours) {
var terms = {}; const terms = {};
var hoursArr = []; const hoursArr = [];
hour = new Date(hour); hour = new Date(hour);
hour.setHours(hour.getHours(), 0, 0, 0); hour.setHours(hour.getHours(), 0, 0, 0);
for (var i = 0, ii = numHours; i < ii; i += 1) { for (let i = 0, ii = numHours; i < ii; i += 1) {
hoursArr.push(hour.getTime()); hoursArr.push(hour.getTime());
hour.setHours(hour.getHours() - 1, 0, 0, 0); hour.setHours(hour.getHours() - 1, 0, 0, 0);
} }
db.sortedSetScores(set, hoursArr, function (err, counts) { const counts = await db.sortedSetScores(set, hoursArr);
if (err) {
return callback(err);
}
hoursArr.forEach(function (term, index) { hoursArr.forEach(function (term, index) {
terms[term] = parseInt(counts[index], 10) || 0; terms[term] = parseInt(counts[index], 10) || 0;
});
var termsArr = [];
hoursArr.reverse();
hoursArr.forEach(function (hour) {
termsArr.push(terms[hour]);
});
callback(null, termsArr);
}); });
const termsArr = [];
hoursArr.reverse();
hoursArr.forEach(function (hour) {
termsArr.push(terms[hour]);
});
return termsArr;
}; };
Analytics.getDailyStatsForSet = function (set, day, numDays, callback) { Analytics.getDailyStatsForSet = async function (set, day, numDays) {
var daysArr = []; const daysArr = [];
day = new Date(day); day = new Date(day);
day.setDate(day.getDate() + 1); // set the date to tomorrow, because getHourlyStatsForSet steps *backwards* 24 hours to sum up the values day.setDate(day.getDate() + 1); // set the date to tomorrow, because getHourlyStatsForSet steps *backwards* 24 hours to sum up the values
day.setHours(0, 0, 0, 0); day.setHours(0, 0, 0, 0);
async.whilst(function (next) { while (numDays > 0) {
/* eslint-disable no-await-in-loop */
const dayData = await Analytics.getHourlyStatsForSet(set, day.getTime() - (1000 * 60 * 60 * 24 * numDays), 24);
daysArr.push(dayData.reduce((cur, next) => cur + next));
numDays -= 1; numDays -= 1;
next(null, numDays + 1); }
}, function (next) { return daysArr;
Analytics.getHourlyStatsForSet(set, day.getTime() - (1000 * 60 * 60 * 24 * numDays), 24, function (err, day) {
if (err) {
return next(err);
}
daysArr.push(day.reduce(function (cur, next) {
return cur + next;
}));
next();
});
}, function (err) {
callback(err, daysArr);
});
}; };
Analytics.getUnwrittenPageviews = function () { Analytics.getUnwrittenPageviews = function () {
return pageViews; return pageViews;
}; };
Analytics.getSummary = function (callback) { Analytics.getSummary = async function () {
var today = new Date(); const today = new Date();
today.setHours(0, 0, 0, 0); today.setHours(0, 0, 0, 0);
async.parallel({ const [seven, thirty] = await Promise.all([
seven: async.apply(Analytics.getDailyStatsForSet, 'analytics:pageviews', today, 7), Analytics.getDailyStatsForSet('analytics:pageviews', today, 7),
thirty: async.apply(Analytics.getDailyStatsForSet, 'analytics:pageviews', today, 30), Analytics.getDailyStatsForSet('analytics:pageviews', today, 30),
}, function (err, scores) { ]);
if (err) {
return callback(null, { return {
seven: 0, seven: seven.reduce((sum, cur) => sum + cur, 0),
thirty: 0, thirty: thirty.reduce((sum, cur) => sum + cur, 0),
}); };
} };
callback(null, {
seven: scores.seven.reduce(function (sum, cur) { Analytics.getCategoryAnalytics = async function (cid) {
sum += cur; return await utils.promiseParallel({
return sum; 'pageviews:hourly': Analytics.getHourlyStatsForSet('analytics:pageviews:byCid:' + cid, Date.now(), 24),
}, 0), 'pageviews:daily': Analytics.getDailyStatsForSet('analytics:pageviews:byCid:' + cid, Date.now(), 30),
thirty: scores.thirty.reduce(function (sum, cur) { 'topics:daily': Analytics.getDailyStatsForSet('analytics:topics:byCid:' + cid, Date.now(), 7),
sum += cur; 'posts:daily': Analytics.getDailyStatsForSet('analytics:posts:byCid:' + cid, Date.now(), 7),
return sum;
}, 0),
});
}); });
}; };
Analytics.getCategoryAnalytics = function (cid, callback) { Analytics.getErrorAnalytics = async function () {
async.parallel({ return await utils.promiseParallel({
'pageviews:hourly': async.apply(Analytics.getHourlyStatsForSet, 'analytics:pageviews:byCid:' + cid, Date.now(), 24), 'not-found': Analytics.getDailyStatsForSet('analytics:errors:404', Date.now(), 7),
'pageviews:daily': async.apply(Analytics.getDailyStatsForSet, 'analytics:pageviews:byCid:' + cid, Date.now(), 30), toobusy: Analytics.getDailyStatsForSet('analytics:errors:503', Date.now(), 7),
'topics:daily': async.apply(Analytics.getDailyStatsForSet, 'analytics:topics:byCid:' + cid, Date.now(), 7), });
'posts:daily': async.apply(Analytics.getDailyStatsForSet, 'analytics:posts:byCid:' + cid, Date.now(), 7),
}, callback);
}; };
Analytics.getErrorAnalytics = function (callback) { Analytics.getBlacklistAnalytics = async function () {
async.parallel({ return await utils.promiseParallel({
'not-found': async.apply(Analytics.getDailyStatsForSet, 'analytics:errors:404', Date.now(), 7), daily: Analytics.getDailyStatsForSet('analytics:blacklist', Date.now(), 7),
toobusy: async.apply(Analytics.getDailyStatsForSet, 'analytics:errors:503', Date.now(), 7), hourly: Analytics.getHourlyStatsForSet('analytics:blacklist', Date.now(), 24),
}, callback); });
};
Analytics.getBlacklistAnalytics = function (callback) {
async.parallel({
daily: async.apply(Analytics.getDailyStatsForSet, 'analytics:blacklist', Date.now(), 7),
hourly: async.apply(Analytics.getHourlyStatsForSet, 'analytics:blacklist', Date.now(), 24),
}, callback);
}; };
Analytics.async = require('./promisify')(Analytics); Analytics.async = require('./promisify')(Analytics);