zero downtime and reload support with cluster module

app.js

@@ -157,6 +157,13 @@ function start() {
 					process.on('SIGTERM', shutdown);
 					process.on('SIGINT', shutdown);
 					process.on('SIGHUP', restart);
+					process.on('message', function(message) {
+						switch(message) {
+							case 'reload':
+								meta.reload();
+							break;
+						}
+					})
 					process.on('uncaughtException', function(err) {
 						winston.error(err.message);
 						console.log(err.stack);

loader.js

@@ -95,7 +95,8 @@ var	nconf = require('nconf'),
 	// nbb, nbbOld;
 
 var Loader = {
-	timesStarted: 0
+	timesStarted: 0,
+	shutdown_queue: []
 };
 
 Loader.init = function() {
@@ -106,11 +107,6 @@ Loader.init = function() {
 		silent: process.env.NODE_ENV !== 'development' ? true : false
 	});
 
-	for(var x=0;x<numCPUs;x++) {
-		// Only the first worker sets up templates/sounds/jobs/etc
-		cluster.fork({ cluster_setup: x === 0 });
-	}
-
 	cluster.on('fork', function(worker) {
 		worker.on('message', function(message) {
 			if (message && typeof message === 'object' && message.action) {
@@ -118,6 +114,12 @@ Loader.init = function() {
 					case 'ready':
 						console.log('[cluster] Child Process (' + worker.process.pid + ') listening for connections.');
 						worker.send('bind');
+
+						// Kill an instance in the shutdown queue
+						var workerToKill = Loader.shutdown_queue.pop();
+						if (workerToKill) {
+							cluster.workers[workerToKill].kill();
+						}
 					break;
 					case 'restart':
 						console.log('[cluster] Restarting...');
@@ -125,6 +127,10 @@ Loader.init = function() {
 							console.log('[cluster] Restarting...');
 						});
 					break;
+					case 'reload':
+						console.log('[cluster] Reloading...');
+						Loader.reload();
+					break;
 				}
 			}
 		});
@@ -147,20 +153,37 @@ Loader.init = function() {
 		}
 
 		console.log('[cluster] Child Process (' + worker.process.pid + ') has exited (code: ' + code + ')');
-		cluster.fork();
+		if (!worker.suicide) {
+			console.log('[cluster] Spinning up another process...')
+			cluster.fork();
+		}
 	});
 
 	process.on('SIGHUP', Loader.restart);
+
+	Loader.start();
 	// fs.writeFile(__dirname + '/pidfile', process.pid);
 };
 
-Loader.restart = function(callback) {
-	async.eachSeries(Object.keys(cluster.workers), function(id, next) {
-		cluster.workers[id].kill();
-		next();
-	}, callback);
+Loader.start = function() {
+	for(var x=0;x<numCPUs;x++) {
+		// Only the first worker sets up templates/sounds/jobs/etc
+		cluster.fork({ cluster_setup: x === 0 });
+	}
 }
 
+Loader.restart = function(callback) {
+	// Slate existing workers for termination -- welcome to death row.
+	Loader.shutdown_queue = Loader.shutdown_queue.concat(Object.keys(cluster.workers));
+	Loader.start();
+};
+
+Loader.reload = function() {
+	Object.keys(cluster.workers).forEach(function(worker_id) {
+		cluster.workers[worker_id].send('reload');
+	});
+};
+
 Loader.init();
 
 // Start the daemon!

src/meta.js

@@ -40,7 +40,7 @@ var async = require('async'),
 					emitter.emit('nodebb:ready');
 				}
 
-				callback.apply(null, arguments);
+				if (callback) callback.apply(null, arguments);
 			});
 		});
 	};

src/meta/templates.js

@@ -5,6 +5,7 @@ var mkdirp = require('mkdirp'),
 	path = require('path'),
 	fs = require('fs'),
 	nconf = require('nconf'),
+	cluster = require('cluster'),
 
 	emitter = require('../emitter'),
 	plugins = require('../plugins'),
@@ -13,6 +14,16 @@ var mkdirp = require('mkdirp'),
 	Templates = {};
 
 Templates.compile = function(callback) {
+	if (cluster.isWorker && process.env.cluster_setup !== 'true') {
+		return setTimeout(function() {
+			console.log('FAKING TEMPLATE COMPILE');
+			emitter.emit('templates:compiled');
+			if (callback) callback();
+		}, 1000);
+	} else {
+		console.log('REAL TEMPLATE COMPILE');
+	}
+
 	var baseTemplatesPath = nconf.get('base_templates_path'),
 		viewsPath = nconf.get('views_dir'),
 		themeTemplatesPath = nconf.get('theme_templates_path');

src/middleware/index.js

@@ -128,14 +128,7 @@ module.exports = function(app, data) {
 
 	routeCurrentTheme(app, data.currentThemeId, data.themesData);
 	routeThemeScreenshots(app, data.themesData);
-
-	if (process.env.cluster_setup === 'true') {
-		meta.templates.compile();
-	} else {
-		setTimeout(function() {
-			emitter.emit('templates:compiled');
-		}, 1000);
-	}
+	meta.templates.compile();
 
 	return middleware;
 };

src/socket.io/admin.js

@@ -14,6 +14,7 @@ var	groups = require('../groups'),
 	async = require('async'),
 	winston = require('winston'),
 	index = require('./index'),
+	cluster = require('cluster'),
 
 	SocketAdmin = {
 		user: require('./admin/user'),
@@ -39,7 +40,13 @@ SocketAdmin.before = function(socket, method, next) {
 };
 
 SocketAdmin.reload = function(socket, data, callback) {
-	meta.reload(callback);
+	if (cluster.isWorker) {
+		process.send({
+			action: 'reload'
+		});
+	} else {
+		meta.reload(callback);
+	}
 };
 
 SocketAdmin.restart = function(socket, data, callback) {

src/webserver.js

@@ -41,7 +41,7 @@ if(nconf.get('ssl')) {
 	emailer.registerApp(app);
 	notifications.init();
 
-	if (process.env.cluster_setup === 'true') {
+	if (cluster.isWorker && process.env.cluster_setup === 'true') {
 		user.startJobs();
 	}
 
@@ -50,7 +50,7 @@ if(nconf.get('ssl')) {
 		meta.js.minify(app.enabled('minification'));
 		meta.css.minify();
 
-		if (process.env.cluster_setup === 'true') {
+		if (cluster.isWorker && process.env.cluster_setup === 'true') {
 			meta.sounds.init();
 		}
 	});