Precompile all templates

- Benchpress compilation is 33x faster now
- Native module with JS fallback and pre-built binaries
- Dev template build is <1sec now
- Minified template build is ~5sec (uglify accounts for almost all)

install/package.json

@@ -22,7 +22,7 @@
         "async": "2.6.1",
         "autoprefixer": "^8.5.2",
         "bcryptjs": "2.4.3",
-        "benchpressjs": "^1.2.2",
+        "benchpressjs": "^1.2.3",
         "body-parser": "^1.18.2",
         "bootstrap": "^3.3.7",
         "chart.js": "^2.7.1",

src/emailer.js

@@ -289,22 +289,26 @@ Emailer.sendViaFallback = function (data, callback) {
 function buildCustomTemplates(config) {
 	async.waterfall([
 		function (next) {
-			Emailer.getTemplates(config, next);
+			async.parallel({
+				templates: function (cb) {
+					Emailer.getTemplates(config, cb);
+				},
+				paths: function (cb) {
+					file.walk(viewsDir, cb);
+				},
+			}, next);
 		},
-		function (templates, next) {
+		function (result, next) {
-			templates = templates.filter(function (template) {
+			var templates = result.templates.filter(function (template) {
 				return template.isCustom && template.text !== prevConfig['email:custom:' + path];
 			});
+			var paths = _.fromPairs(result.paths.map(function (p) {
+				var relative = path.relative(viewsDir, p).replace(/\\/g, '/');
+				return [relative, p];
+			}));
 			async.each(templates, function (template, next) {
 				async.waterfall([
 					function (next) {
-						file.walk(viewsDir, next);
-					},
-					function (paths, next) {
-						paths = _.fromPairs(paths.map(function (p) {
-							var relative = path.relative(viewsDir, p).replace(/\\/g, '/');
-							return [relative, p];
-						}));
 						meta.templates.processImports(paths, template.path, template.text, next);
 					},
 					function (source, next) {

src/meta/templates.js

@@ -8,6 +8,7 @@ var path = require('path');
 var fs = require('fs');
 var nconf = require('nconf');
 var _ = require('lodash');
+var Benchpress = require('benchpressjs');
 
 var plugins = require('../plugins');
 var file = require('../file');
@@ -113,6 +114,34 @@ function getTemplateFiles(dirs, callback) {
 	], callback);
 }
 
+function compileTemplate(filename, source, callback) {
+	async.waterfall([
+		function (next) {
+			file.walk(viewsPath, next);
+		},
+		function (paths, next) {
+			paths = _.fromPairs(paths.map(function (p) {
+				var relative = path.relative(viewsPath, p).replace(/\\/g, '/');
+				return [relative, p];
+			}));
+			async.waterfall([
+				function (next) {
+					processImports(paths, filename, source, next);
+				},
+				function (source, next) {
+					Benchpress.precompile(source, {
+						minify: global.env !== 'development',
+					}, next);
+				},
+				function (compiled, next) {
+					fs.writeFile(path.join(viewsPath, filename.replace(/\.tpl$/, '.js')), compiled, next);
+				},
+			], next);
+		},
+	], callback);
+}
+Templates.compileTemplate = compileTemplate;
+
 function compile(callback) {
 	callback = callback || function () {};
 
@@ -144,8 +173,22 @@ function compile(callback) {
 							next(err, source);
 						});
 					},
-					function (compiled, next) {
+					function (imported, next) {
-						fs.writeFile(path.join(viewsPath, name), compiled, next);
+						async.parallel([
+							function (cb) {
+								fs.writeFile(path.join(viewsPath, name), imported, cb);
+							},
+							function (cb) {
+								Benchpress.precompile(imported, { minify: global.env !== 'development' }, function (err, compiled) {
+									if (err) {
+										cb(err);
+										return;
+									}
+
+									fs.writeFile(path.join(viewsPath, name.replace(/\.tpl$/, '.js')), compiled, cb);
+								});
+							},
+						], next);
 					},
 				], next);
 			}, next);

src/middleware/index.js

@@ -2,13 +2,11 @@
 
 var async = require('async');
 var path = require('path');
-var fs = require('fs');
 var csrf = require('csurf');
 var validator = require('validator');
 var nconf = require('nconf');
 var ensureLoggedIn = require('connect-ensure-login');
 var toobusy = require('toobusy-js');
-var Benchpress = require('benchpressjs');
 var LRU = require('lru-cache');
 
 var plugins = require('../plugins');
@@ -207,58 +205,3 @@ middleware.delayLoading = function (req, res, next) {
 
 	setTimeout(next, 1000);
 };
-
-var viewsDir = nconf.get('views_dir');
-var workingCache = {};
-
-middleware.templatesOnDemand = function (req, res, next) {
-	var filePath = req.filePath || path.join(viewsDir, req.path);
-	if (!filePath.endsWith('.js')) {
-		return next();
-	}
-	var tplPath = filePath.replace(/\.js$/, '.tpl');
-	if (workingCache[filePath]) {
-		workingCache[filePath].push(next);
-		return;
-	}
-
-	async.waterfall([
-		function (cb) {
-			file.exists(filePath, cb);
-		},
-		function (exists, cb) {
-			if (exists) {
-				return next();
-			}
-
-			// need to check here again
-			// because compilation could have started since last check
-			if (workingCache[filePath]) {
-				workingCache[filePath].push(next);
-				return;
-			}
-
-			workingCache[filePath] = [next];
-			fs.readFile(tplPath, 'utf8', cb);
-		},
-		function (source, cb) {
-			Benchpress.precompile({
-				source: source,
-				minify: global.env !== 'development',
-			}, cb);
-		},
-		function (compiled, cb) {
-			if (!compiled) {
-				return cb(new Error('[[error:templatesOnDemand.compiled-template-empty, ' + tplPath + ']]'));
-			}
-			fs.writeFile(filePath, compiled, cb);
-		},
-	], function (err) {
-		var arr = workingCache[filePath];
-		workingCache[filePath] = null;
-
-		arr.forEach(function (callback) {
-			callback(err);
-		});
-	});
-};

src/routes/index.js

@@ -152,7 +152,6 @@ module.exports = function (app, middleware, hotswapIds, callback) {
 	}
 
 	app.use(middleware.privateUploads);
-	app.use(relativePath + '/assets/templates', middleware.templatesOnDemand);
 
 	var statics = [
 		{ route: '/assets', path: path.join(__dirname, '../../build/public') },

src/webserver.js

@@ -147,15 +147,7 @@ function setupExpressApp(app, callback) {
 	app.engine('tpl', function (filepath, data, next) {
 		filepath = filepath.replace(/\.tpl$/, '.js');
 
-		middleware.templatesOnDemand({
+		Benchpress.__express(filepath, data, next);
-			filePath: filepath,
-		}, null, function (err) {
-			if (err) {
-				return next(err);
-			}
-
-			Benchpress.__express(filepath, data, next);
-		});
 	});
 	app.set('view engine', 'tpl');
 	app.set('views', viewsDir);

test/controllers.js

@@ -72,15 +72,17 @@ describe('Controllers', function () {
 			});
 		}
 		var message = utils.generateUUID();
-		var tplPath = path.join(nconf.get('views_dir'), 'custom.tpl');
+		var name = 'custom.tpl';
+		var tplPath = path.join(nconf.get('views_dir'), name);
 
-		before(function () {
+		before(function (done) {
 			plugins.registerHook('myTestPlugin', {
 				hook: 'action:homepage.get:custom',
 				method: hookMethod,
 			});
 
 			fs.writeFileSync(tplPath, message);
+			meta.templates.compileTemplate(name, message, done);
 		});
 
 		it('should load default', function (done) {