From 943874805d31651a8735e9af0985795567ac2965 Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Sun, 24 Aug 2014 12:30:49 -0400 Subject: [PATCH 1/6] proof-of-concept for hot-swapping of routes --- src/hotswap.js | 33 +++++++++++++++++ src/plugins.js | 34 ++++++++++++++++-- src/routes/debug.js | 6 +++- src/routes/index.js | 88 +++++++++++++++++++++++---------------------- src/webserver.js | 1 + 5 files changed, 117 insertions(+), 45 deletions(-) create mode 100644 src/hotswap.js diff --git a/src/hotswap.js b/src/hotswap.js new file mode 100644 index 0000000000..717c099052 --- /dev/null +++ b/src/hotswap.js @@ -0,0 +1,33 @@ +var HotSwap = {}, + winston = require('winston'), + stack; + +HotSwap.prepare = function(app) { + stack = app._router.stack; +}; + +HotSwap.find = function(id) { + if (stack) { + for(var x=0,numEntries=stack.length;x Date: Sun, 24 Aug 2014 14:25:26 -0400 Subject: [PATCH 2/6] framework for reloading --- src/meta.js | 5 +++++ src/routes/debug.js | 4 ---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/meta.js b/src/meta.js index 9c0dc8adad..fb4e5cac52 100644 --- a/src/meta.js +++ b/src/meta.js @@ -28,6 +28,11 @@ var async = require('async'), }); }; + Meta.reload = function(step) { + // 1. Reload plugins and associated routes + // 2. Minify scripts and css, update cache buster + }; + Meta.restart = function() { if (process.send) { process.send({ diff --git a/src/routes/debug.js b/src/routes/debug.js index 30f6eaf8b8..584a70a6b6 100644 --- a/src/routes/debug.js +++ b/src/routes/debug.js @@ -57,9 +57,5 @@ module.exports = function(app, middleware, controllers) { router.get('/test', function(req, res) { res.redirect(404); - var plugins = require('../plugins'); - plugins.reloadRoutes(function() { - res.send(200, 'routes replaced'); - }); }); }; From 841c755bb75a735a8b9b67f603387373dd1b242a Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Sun, 24 Aug 2014 17:46:22 -0400 Subject: [PATCH 3/6] on-demand reloading of client-side assets --- src/meta/js.js | 189 ++++++++++++++++++++++++--------------------- src/routes/meta.js | 4 +- 2 files changed, 103 insertions(+), 90 deletions(-) diff --git a/src/meta/js.js b/src/meta/js.js index 29c6f77880..0625738db9 100644 --- a/src/meta/js.js +++ b/src/meta/js.js @@ -19,35 +19,37 @@ module.exports = function(Meta) { hash: +new Date(), prepared: false, minFile: 'nodebb.min.js', - scripts: [ - 'vendor/jquery/js/jquery.js', - 'vendor/jquery/js/jquery-ui-1.10.4.custom.js', - 'vendor/jquery/timeago/jquery.timeago.min.js', - 'vendor/jquery/js/jquery.form.min.js', - 'vendor/jquery/serializeObject/jquery.ba-serializeobject.min.js', - 'vendor/jquery/deserialize/jquery.deserialize.min.js', - 'vendor/bootstrap/js/bootstrap.min.js', - 'vendor/jquery/bootstrap-tagsinput/bootstrap-tagsinput.min.js', - 'vendor/requirejs/require.js', - 'vendor/bootbox/bootbox.min.js', - 'vendor/tinycon/tinycon.js', - 'vendor/xregexp/xregexp.js', - 'vendor/xregexp/unicode/unicode-base.js', - 'vendor/buzz/buzz.min.js', - '../node_modules/templates.js/lib/templates.js', - 'src/utils.js', - 'src/app.js', - 'src/ajaxify.js', - 'src/variables.js', - 'src/widgets.js', - 'src/translator.js', - 'src/helpers.js', - 'src/overrides.js' - ] + scripts: { + base: [ + 'public/vendor/jquery/js/jquery.js', + 'public/vendor/jquery/js/jquery-ui-1.10.4.custom.js', + 'public/vendor/jquery/timeago/jquery.timeago.min.js', + 'public/vendor/jquery/js/jquery.form.min.js', + 'public/vendor/jquery/serializeObject/jquery.ba-serializeobject.min.js', + 'public/vendor/jquery/deserialize/jquery.deserialize.min.js', + 'public/vendor/bootstrap/js/bootstrap.min.js', + 'public/vendor/jquery/bootstrap-tagsinput/bootstrap-tagsinput.min.js', + 'public/vendor/requirejs/require.js', + 'public/vendor/bootbox/bootbox.min.js', + 'public/vendor/tinycon/tinycon.js', + 'public/vendor/xregexp/xregexp.js', + 'public/vendor/xregexp/unicode/unicode-base.js', + 'public/vendor/buzz/buzz.min.js', + './node_modules/templates.js/lib/templates.js', + 'public/src/utils.js', + 'public/src/app.js', + 'public/src/ajaxify.js', + 'public/src/variables.js', + 'public/src/widgets.js', + 'public/src/translator.js', + 'public/src/helpers.js', + 'public/src/overrides.js' + ] + } }; Meta.js.loadRJS = function(callback) { - var rjsPath = path.join(__dirname, '../..', '/public/src'); + var rjsPath = path.join(__dirname, '../../public/src'); async.parallel({ forum: function(next) { @@ -65,75 +67,53 @@ module.exports = function(Meta) { rjsFiles = rjsFiles.filter(function(file) { return file.match('admin') === null; }).map(function(file) { - return path.join('src', file.replace(rjsPath, '')); + return path.join('public/src', file.replace(rjsPath, '')); }); - Meta.js.scripts = Meta.js.scripts.concat(rjsFiles); + Meta.js.scripts.rjs = rjsFiles; callback(); }); }; Meta.js.prepare = function (callback) { - plugins.fireHook('filter:scripts.get', Meta.js.scripts, function(err, scripts) { - var jsPaths = scripts.map(function (jsPath) { - jsPath = path.normalize(jsPath); + async.parallel([ + async.apply(Meta.js.loadRJS), // Require.js scripts + async.apply(getPluginScripts), // plugin scripts via filter:scripts.get + function(next) { // client scripts via "scripts" config in plugin.json + var pluginsScripts = [], + pluginDirectories = [], + clientScripts = []; - if (jsPath.substring(0, 7) === 'plugins') { - var matches = _.map(plugins.staticDirs, function(realPath, mappedPath) { - if (jsPath.match(mappedPath)) { - return mappedPath; - } else { - return null; - } - }).filter(function(a) { return a; }); - - if (matches.length) { - var relPath = jsPath.slice(('plugins/' + matches[0]).length), - pluginId = matches[0].split(path.sep)[0]; - - return plugins.staticDirs[matches[0]] + relPath; - } else { - winston.warn('[meta.scripts.get] Could not resolve mapped path: ' + jsPath + '. Are you sure it is defined by a plugin?'); - return null; - } + pluginsScripts = plugins.clientScripts.filter(function(path) { + if (path.indexOf('.js') !== -1) { + return true; } else { - return path.join(__dirname, '../..', '/public', jsPath); + pluginDirectories.push(path); + return false; } }); - Meta.js.scripts = jsPaths.filter(function(path) { - return path !== null; - }); - - var pluginDirectories = []; - - plugins.clientScripts = plugins.clientScripts.filter(function(path) { - if (path.indexOf('.js') !== -1) { - return true; - } else { - pluginDirectories.push(path); - return false; - } - }); - - // Add plugin scripts - Meta.js.scripts = Meta.js.scripts.concat(plugins.clientScripts); - - async.each(pluginDirectories, function(directory, next) { - utils.walk(directory, function(err, scripts) { - Meta.js.scripts = Meta.js.scripts.concat(scripts); - next(err); - }); - }, function(err) { - // Translate into relative paths - Meta.js.scripts = Meta.js.scripts.map(function(script) { - return path.relative(path.resolve(__dirname, '../..'), script).replace(/\\/g, '/'); - }); - - Meta.js.prepared = true; - callback(err); + // Add plugin scripts + Meta.js.scripts.client = pluginsScripts; + + // Add plugin script directories + async.each(pluginDirectories, function(directory, next) { + utils.walk(directory, function(err, scripts) { + Meta.js.scripts.client = Meta.js.scripts.client.concat(scripts); + next(err); + }); + }, next); + } + ], function(err) { + if (err) return callback(err); + + // Convert all scripts to paths relative to the NodeBB base directory + var basePath = path.resolve(__dirname, '../..'); + Meta.js.scripts.all = Meta.js.scripts.base.concat(Meta.js.scripts.rjs, Meta.js.scripts.plugin, Meta.js.scripts.client).map(function(script) { + return path.relative(basePath, script).replace(/\\/g, '/'); }); + callback(); }); }; @@ -183,13 +163,11 @@ module.exports = function(Meta) { } }); - Meta.js.loadRJS(function() { - Meta.js.prepare(function() { - minifier.send({ - action: 'js', - minify: minify, - scripts: Meta.js.scripts - }); + Meta.js.prepare(function() { + minifier.send({ + action: 'js', + minify: minify, + scripts: Meta.js.scripts.all }); }); }; @@ -199,4 +177,39 @@ module.exports = function(Meta) { Meta.js.minifierProc.kill('SIGTERM'); } }; + + function getPluginScripts(callback) { + plugins.fireHook('filter:scripts.get', [], function(err, scripts) { + if (err) callback(err, []); + + var jsPaths = scripts.map(function (jsPath) { + jsPath = path.normalize(jsPath); + + // if (jsPath.substring(0, 7) === 'plugins') { + var matches = _.map(plugins.staticDirs, function(realPath, mappedPath) { + if (jsPath.match(mappedPath)) { + return mappedPath; + } else { + return null; + } + }).filter(function(a) { return a; }); + + if (matches.length) { + var relPath = jsPath.slice(('plugins/' + matches[0]).length), + pluginId = matches[0].split(path.sep)[0]; + + return plugins.staticDirs[matches[0]] + relPath; + } else { + winston.warn('[meta.scripts.get] Could not resolve mapped path: ' + jsPath + '. Are you sure it is defined by a plugin?'); + return null; + } + // } else { + // return path.join(__dirname, '../..', jsPath); + // } + }); + + Meta.js.scripts.plugin = jsPaths.filter(Boolean); + callback(); + }); + }; }; \ No newline at end of file diff --git a/src/routes/meta.js b/src/routes/meta.js index f0d9aa85b2..76cefadf0e 100644 --- a/src/routes/meta.js +++ b/src/routes/meta.js @@ -35,8 +35,8 @@ function setupPluginSourceMapping(app) { development mode (`./nodebb dev`) */ var routes = plugins.clientScripts, - mapping, - prefix = __dirname.split(path.sep).length - 1; + prefix = __dirname.split(path.sep).length - 1, + mapping; routes.forEach(function(route) { mapping = '/' + route.split(path.sep).slice(prefix).join('/'); From 013dfd0cebf3222c9c29a0db2bc0efd05cd888c0 Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Mon, 25 Aug 2014 10:13:01 -0400 Subject: [PATCH 4/6] added callbacks to css and js reloading, issue #2010 --- src/meta/css.js | 11 ++++++++++- src/meta/js.js | 23 +++++++++++++++-------- src/plugins.js | 8 +++++--- src/routes/plugins.js | 3 +-- src/webserver.js | 1 + 5 files changed, 32 insertions(+), 14 deletions(-) diff --git a/src/meta/css.js b/src/meta/css.js index f60e320703..737f203ee5 100644 --- a/src/meta/css.js +++ b/src/meta/css.js @@ -20,7 +20,7 @@ module.exports = function(Meta) { Meta.css.branding = {}; Meta.css.defaultBranding = {}; - Meta.css.minify = function() { + Meta.css.minify = function(callback) { winston.info('[meta/css] Minifying LESS/CSS'); db.getObjectFields('config', ['theme:type', 'theme:id'], function(err, themeData) { var themeId = (themeData['theme:id'] || 'nodebb-theme-vanilla'), @@ -54,6 +54,9 @@ module.exports = function(Meta) { parser.parse(source, function(err, tree) { if (err) { winston.error('[meta/css] Could not minify LESS/CSS: ' + err.message); + if (typeof callback === 'function') { + callback(err); + } return; } @@ -63,6 +66,9 @@ module.exports = function(Meta) { }); } catch (err) { winston.error('[meta/css] Syntax Error: ' + err.message + ' - ' + path.basename(err.filename) + ' on line ' + err.line); + if (typeof callback === 'function') { + callback(err); + } return; } @@ -89,6 +95,9 @@ module.exports = function(Meta) { winston.info('[meta/css] Done.'); emitter.emit('meta:css.compiled'); + if (typeof callback === 'function') { + callback(); + } }); }); }; diff --git a/src/meta/js.js b/src/meta/js.js index 0625738db9..14e3112f19 100644 --- a/src/meta/js.js +++ b/src/meta/js.js @@ -117,28 +117,35 @@ module.exports = function(Meta) { }); }; - Meta.js.minify = function(minify) { + Meta.js.minify = function(minify, callback) { var minifier = Meta.js.minifierProc = fork('minifier.js', { silent: true }), minifiedStream = minifier.stdio[1], + minifiedString = '', mapStream = minifier.stdio[2], + mapString = '', step = 0, onComplete = function() { if (step === 0) { return step++; } + Meta.js.cache = minifiedString; + Meta.js.map = mapString; winston.info('[meta/js] Compilation complete'); emitter.emit('meta:js.compiled'); minifier.kill(); + if (typeof callback === 'function') { + callback(); + } }; minifiedStream.on('data', function(buffer) { - Meta.js.cache += buffer.toString(); + minifiedString += buffer.toString(); }); mapStream.on('data', function(buffer) { - Meta.js.map += buffer.toString(); + mapString += buffer.toString(); }); minifier.on('message', function(message) { @@ -158,7 +165,11 @@ module.exports = function(Meta) { case 'error': winston.error('[meta/js] Could not compile client-side scripts! ' + message.payload.message); minifier.kill(); - process.exit(); + if (typeof callback === 'function') { + callback(err); + } else { + process.exit(0); + } break; } }); @@ -185,7 +196,6 @@ module.exports = function(Meta) { var jsPaths = scripts.map(function (jsPath) { jsPath = path.normalize(jsPath); - // if (jsPath.substring(0, 7) === 'plugins') { var matches = _.map(plugins.staticDirs, function(realPath, mappedPath) { if (jsPath.match(mappedPath)) { return mappedPath; @@ -203,9 +213,6 @@ module.exports = function(Meta) { winston.warn('[meta.scripts.get] Could not resolve mapped path: ' + jsPath + '. Are you sure it is defined by a plugin?'); return null; } - // } else { - // return path.join(__dirname, '../..', jsPath); - // } }); Meta.js.scripts.plugin = jsPaths.filter(Boolean); diff --git a/src/plugins.js b/src/plugins.js index b2578ed51e..427816c7bc 100644 --- a/src/plugins.js +++ b/src/plugins.js @@ -129,15 +129,18 @@ var fs = require('fs'), } else { var router = express.Router(); router.hotswapId = 'plugins'; + router.render = function() { + app.render.apply(app, arguments); + }; // Deprecated as of v0.5.0, remove this hook call for NodeBB v0.6.0-1 Plugins.fireHook('action:app.load', router, middleware, controllers); Plugins.fireHook('static:app.load', router, middleware, controllers, function() { hotswap.replace('plugins', router); + winston.info('[plugins] All plugins reloaded and rerouted'); + callback(); }); - - callback(); } }; @@ -527,7 +530,6 @@ var fs = require('fs'), // Reload meta data Plugins.reload(function() { - if(!active) { Plugins.fireHook('action:plugin.activate', id); } diff --git a/src/routes/plugins.js b/src/routes/plugins.js index fd195ca3f2..17efdb8c53 100644 --- a/src/routes/plugins.js +++ b/src/routes/plugins.js @@ -8,8 +8,7 @@ var _ = require('underscore'), async = require('async'), winston = require('winston'), - plugins = require('../plugins'), - pluginRoutes = []; + plugins = require('../plugins'); module.exports = function(app, middleware, controllers) { diff --git a/src/webserver.js b/src/webserver.js index f2595ddabc..5c6f571372 100644 --- a/src/webserver.js +++ b/src/webserver.js @@ -110,6 +110,7 @@ if(nconf.get('ssl')) { emitter.all(['templates:compiled', 'meta:js.compiled', 'meta:css.compiled'], function() { winston.info('NodeBB Ready'); emitter.emit('nodebb:ready'); + emitter.removeAllListeners('templates:compiled').removeAllListeners('meta:js.compiled').removeAllListeners('meta:css.compiled'); }); emitter.on('templates:compiled', function() { From 4e55707652b7dfa9fb18891c5a9d1366a6baf2bd Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Mon, 25 Aug 2014 10:46:48 -0400 Subject: [PATCH 5/6] closed #2011 --- public/src/forum/admin/index.js | 41 +++++++++++++++++++++++++++------ src/meta.js | 13 +++++++---- src/socket.io/admin.js | 4 ++++ 3 files changed, 47 insertions(+), 11 deletions(-) diff --git a/public/src/forum/admin/index.js b/public/src/forum/admin/index.js index c10706e257..58eb0d8eef 100644 --- a/public/src/forum/admin/index.js +++ b/public/src/forum/admin/index.js @@ -37,18 +37,45 @@ define('forum/admin/index', function() { }); $('.restart').on('click', function() { + bootbox.confirm('Are you sure you wish to restart NodeBB?', function(confirm) { + if (confirm) { + app.alert({ + timeout: 5000, + title: 'Restarting... ', + message: 'NodeBB is restarting.', + type: 'info' + }); + + $(window).one('action:reconnected', function() { + app.alertSuccess('NodeBB has successfully restarted.'); + }); + + socket.emit('admin.restart'); + } + }); + }); + + $('.reload').on('click', function() { app.alert({ - timeout: 5000, - title: 'Restarting...', + alert_id: 'instance_reload', + title: 'Reloading... ', message: 'NodeBB is restarting.', - type: 'info' + type: 'info', + timeout: 5000 }); - $(window).one('action:reconnected', function() { - app.alertSuccess('NodeBB has successfully restarted.'); + socket.emit('admin.reload', function(err) { + if (!err) { + app.alertSuccess('NodeBB has successfully reloaded.'); + } else { + app.alert({ + alert_id: 'instance_reload', + title: '[[global:alert.error]]', + message: err.message, + type: 'danger' + }); + } }); - - socket.emit('admin.restart'); }); }; diff --git a/src/meta.js b/src/meta.js index fb4e5cac52..9a616cc957 100644 --- a/src/meta.js +++ b/src/meta.js @@ -3,7 +3,8 @@ var async = require('async'), winston = require('winston'), user = require('./user'), - groups = require('./groups'); + groups = require('./groups'), + plugins = require('./plugins'); (function (Meta) { @@ -28,9 +29,13 @@ var async = require('async'), }); }; - Meta.reload = function(step) { - // 1. Reload plugins and associated routes - // 2. Minify scripts and css, update cache buster + Meta.reload = function(callback) { + plugins.reload(function() { + async.parallel([ + async.apply(Meta.js.minify, false), + async.apply(Meta.css.minify) + ], callback); + }); }; Meta.restart = function() { diff --git a/src/socket.io/admin.js b/src/socket.io/admin.js index bc8140a2a1..f51b0dd2a7 100644 --- a/src/socket.io/admin.js +++ b/src/socket.io/admin.js @@ -38,6 +38,10 @@ SocketAdmin.before = function(socket, method, next) { }); }; +SocketAdmin.reload = function(socket, data, callback) { + meta.reload(callback); +}; + SocketAdmin.restart = function(socket, data, callback) { meta.restart(); }; From c9e80b6f64b03587687900471529ef7a2f234ee9 Mon Sep 17 00:00:00 2001 From: Julian Lam Date: Mon, 25 Aug 2014 11:56:48 -0400 Subject: [PATCH 6/6] closed #2012 --- app.js | 10 +++++++++- loader.js | 29 ++++++++++++++++++++--------- src/webserver.js | 32 ++++++++++++++++++++------------ 3 files changed, 49 insertions(+), 22 deletions(-) diff --git a/app.js b/app.js index 26fe077f87..bd9cb02e0a 100644 --- a/app.js +++ b/app.js @@ -141,7 +141,13 @@ function start() { nconf.set('url', nconf.get('base_url') + (nconf.get('use_port') ? ':' + nconf.get('port') : '') + nconf.get('relative_path')); plugins.ready(function() { - webserver.init(); + webserver.init(function() { + // If this callback is called, this means that loader.js is used + process.on('SIGCONT', webserver.listen); + process.send({ + action: 'ready' + }); + }); }); process.on('SIGTERM', shutdown); @@ -313,6 +319,8 @@ function shutdown(code) { winston.info('[app] Shutdown (SIGTERM/SIGINT) Initialised.'); require('./src/database').close(); winston.info('[app] Database connection closed.'); + require('./src/webserver').server.close(); + winston.info('[app] Web server closed to connections.'); winston.info('[app] Shutdown complete.'); process.exit(code || 0); diff --git a/loader.js b/loader.js index 6448615348..a3ce275704 100644 --- a/loader.js +++ b/loader.js @@ -5,8 +5,7 @@ var nconf = require('nconf'), pidFilePath = __dirname + '/pidfile', output = fs.openSync(__dirname + '/logs/output.log', 'a'), start = function() { - var fork = require('child_process').fork, - nbb_start = function() { + var nbb_start = function(callback) { if (timesStarted > 3) { console.log('\n[loader] Experienced three start attempts in 10 seconds, most likely an error on startup. Halting.'); return nbb_stop(); @@ -18,14 +17,24 @@ var nconf = require('nconf'), } startTimer = setTimeout(resetTimer, 1000*10); - nbb = fork('./app', process.argv.slice(2), { + if (nbb) { + nbbOld = nbb; + } + + nbb = require('child_process').fork('./app', process.argv.slice(2), { env: process.env }); nbb.on('message', function(message) { if (message && typeof message === 'object' && message.action) { - if (message.action === 'restart') { - nbb_restart(); + switch (message.action) { + case 'ready': + if (!callback) return nbb.kill('SIGCONT'); + callback(); + break; + case 'restart': + nbb_restart(); + break; } } }); @@ -52,10 +61,12 @@ var nconf = require('nconf'), } }, nbb_restart = function() { - nbb.removeAllListeners('exit').on('exit', function() { - nbb_start(); + nbb_start(function() { + nbbOld.removeAllListeners('exit').on('exit', function() { + nbb.kill('SIGCONT'); + }); + nbbOld.kill(); }); - nbb.kill(); }, resetTimer = function() { clearTimeout(startTimer); @@ -70,7 +81,7 @@ var nconf = require('nconf'), nbb_start(); }, - nbb; + nbb, nbbOld; nconf.argv(); diff --git a/src/webserver.js b/src/webserver.js index 5c6f571372..3278713dd1 100644 --- a/src/webserver.js +++ b/src/webserver.js @@ -97,7 +97,7 @@ if(nconf.get('ssl')) { } module.exports.server = server; - module.exports.init = function () { + module.exports.init = function(callback) { server.on("error", function(err){ if (err.code === 'EADDRINUSE') { winston.error('NodeBB address in use, exiting...'); @@ -114,18 +114,26 @@ if(nconf.get('ssl')) { }); emitter.on('templates:compiled', function() { - var bind_address = ((nconf.get('bind_address') === "0.0.0.0" || !nconf.get('bind_address')) ? '0.0.0.0' : nconf.get('bind_address')) + ':' + port; - winston.info('NodeBB attempting to listen on: ' + bind_address); + if (process.send) { + callback(); + } else { + module.exports.listen(); + } + }); + }; - server.listen(port, nconf.get('bind_address'), function(){ - winston.info('NodeBB is now listening on: ' + bind_address); - if (process.send) { - process.send({ - action: 'ready', - bind_address: bind_address - }); - } - }); + module.exports.listen = function() { + var bind_address = ((nconf.get('bind_address') === "0.0.0.0" || !nconf.get('bind_address')) ? '0.0.0.0' : nconf.get('bind_address')) + ':' + port; + winston.info('NodeBB attempting to listen on: ' + bind_address); + + server.listen(port, nconf.get('bind_address'), function(){ + winston.info('NodeBB is now listening on: ' + bind_address); + if (process.send) { + process.send({ + action: 'listening', + bind_address: bind_address + }); + } }); };