Misc fixes and improvements (#6143)

* `setup` command fixes and improvements

- Enable using the `./nodebb setup` command for auto-setup with a JSON argument
- Change CLI so package-install and dependency install are separate steps
- Fix #6142

* Prevent compiling templates multiple times

- Multiple requests for same template get pooled
- Hopefully fixes the "templateFunction is not a function" error which happens if site is restarted during high-traffic times

* More helpful upgrade template

src/cli/index.js

@@ -9,14 +9,30 @@ var dirname = require('./paths').baseDir;
 // check to make sure dependencies are installed
 try {
 	fs.readFileSync(path.join(dirname, 'package.json'));
-	fs.readFileSync(path.join(dirname, 'node_modules/async/package.json'));
+} catch (e) {
+	if (e.code === 'ENOENT') {
+		console.warn('package.json not found.');
+		console.log('Populating package.json...\n');
+
+		packageInstall.updatePackageFile();
+		packageInstall.preserveExtraneousPlugins();
+
+		console.log('OK'.green + '\n'.reset);
+	} else {
+		throw e;
+	}
+}
+
+try {
+	fs.readFileSync(path.join(dirname, 'node_modules/async/package.json'), 'utf8');
+	fs.readFileSync(path.join(dirname, 'node_modules/commander/package.json'), 'utf8');
+	fs.readFileSync(path.join(dirname, 'node_modules/colors/package.json'), 'utf8');
+	fs.readFileSync(path.join(dirname, 'node_modules/nconf/package.json'), 'utf8');
 } catch (e) {
 	if (e.code === 'ENOENT') {
 		console.warn('Dependencies not yet installed.');
 		console.log('Installing them now...\n');
 
-		packageInstall.updatePackageFile();
-		packageInstall.preserveExtraneousPlugins();
 		packageInstall.npmInstallProduction();
 
 		require('colors');
@@ -121,10 +137,20 @@ program
 
 // management commands
 program
-	.command('setup')
-	.description('Run the NodeBB setup script')
-	.action(function () {
-		require('./setup').setup();
+	.command('setup [config]')
+	.description('Run the NodeBB setup script, or setup with an initial config')
+	.action(function (initConfig) {
+		if (initConfig) {
+			try {
+				initConfig = JSON.parse(initConfig);
+			} catch (e) {
+				console.warn('Invalid JSON passed as initial config value.'.red);
+				console.log('If you meant to pass in an initial config value, please try again.\n');
+
+				throw e;
+			}
+		}
+		require('./setup').setup(initConfig);
 	});
 
 program

src/cli/paths.js

@@ -6,10 +6,12 @@ var baseDir = path.join(__dirname, '../../');
 var loader = path.join(baseDir, 'loader.js');
 var app = path.join(baseDir, 'app.js');
 var pidfile = path.join(baseDir, 'pidfile');
+var config = path.join(baseDir, 'config.json');
 
 module.exports = {
 	baseDir: baseDir,
 	loader: loader,
 	app: app,
 	pidfile: pidfile,
+	config: config,
 };

src/cli/setup.js

@@ -2,10 +2,13 @@
 
 var winston = require('winston');
 var async = require('async');
+var path = require('path');
+var nconf = require('nconf');
 
 var install = require('../../install/web').install;
 
-function setup() {
+function setup(initConfig) {
+	var paths = require('./paths');
 	var install = require('../install');
 	var build = require('../meta/build');
 	var prestart = require('../prestart');
@@ -17,9 +20,19 @@ function setup() {
 	console.log('\nThis looks like a new installation, so you\'ll have to answer a few questions about your environment before we can proceed.');
 	console.log('Press enter to accept the default setting (shown in brackets).');
 
+	install.values = initConfig;
+
 	async.series([
 		install.setup,
-		prestart.loadConfig,
+		function (next) {
+			var configFile = paths.config;
+			if (nconf.get('config')) {
+				configFile = path.resolve(paths.baseDir, nconf.get('config'));
+			}
+
+			prestart.loadConfig(configFile);
+			next();
+		},
 		build.buildAll,
 	], function (err, data) {
 		// Disregard build step data

src/install.js

@@ -8,7 +8,7 @@ var winston = require('winston');
 var nconf = require('nconf');
 var utils = require('./utils.js');
 
-var install = {};
+var install = module.exports;
 var questions = {};
 
 questions.main = [
@@ -42,17 +42,15 @@ questions.optional = [
 ];
 
 function checkSetupFlag(next) {
-	var setupVal;
+	var setupVal = install.values;
 
 	try {
 		if (nconf.get('setup')) {
 			setupVal = JSON.parse(nconf.get('setup'));
 		}
-	} catch (err) {
-		setupVal = undefined;
-	}
+	} catch (err) {}
 
-	if (setupVal && setupVal instanceof Object) {
+	if (setupVal && typeof setupVal === 'object') {
 		if (setupVal['admin:username'] && setupVal['admin:password'] && setupVal['admin:password:confirm'] && setupVal['admin:email']) {
 			install.values = setupVal;
 			next();
@@ -74,9 +72,8 @@ function checkSetupFlag(next) {
 			process.exit();
 		}
 	} else if (nconf.get('database')) {
-		install.values = {
-			database: nconf.get('database'),
-		};
+		install.values = install.values || {};
+		install.values.database = nconf.get('database');
 		next();
 	} else {
 		next();
@@ -549,11 +546,9 @@ install.save = function (server_conf, callback) {
 		console.log('Configuration Saved OK');
 
 		nconf.file({
-			file: path.join(__dirname, '..', 'config.json'),
+			file: serverConfigPath,
 		});
 
 		callback();
 	});
 };
-
-module.exports = install;

src/middleware/index.js

@@ -203,12 +203,19 @@ middleware.delayLoading = function (req, res, next) {
 };
 
 var viewsDir = nconf.get('views_dir');
+var workingCache = {};
+
 middleware.templatesOnDemand = function (req, res, next) {
 	var filePath = req.filePath || path.join(viewsDir, req.path);
 	if (!filePath.endsWith('.js')) {
 		return next();
 	}
 
+	if (workingCache[filePath]) {
+		workingCache[filePath].push(next);
+		return;
+	}
+
 	async.waterfall([
 		function (cb) {
 			file.exists(filePath, cb);
@@ -218,6 +225,14 @@ middleware.templatesOnDemand = function (req, res, next) {
 				return next();
 			}
 
+			// need to check here again
+			// because compilation could have started since last check
+			if (workingCache[filePath]) {
+				workingCache[filePath].push(next);
+				return;
+			}
+
+			workingCache[filePath] = [next];
 			fs.readFile(filePath.replace(/\.js$/, '.tpl'), 'utf8', cb);
 		},
 		function (source, cb) {
@@ -229,5 +244,12 @@ middleware.templatesOnDemand = function (req, res, next) {
 		function (compiled, cb) {
 			fs.writeFile(filePath, compiled, cb);
 		},
-	], next);
+	], function (err) {
+		var arr = workingCache[filePath];
+		workingCache[filePath] = null;
+
+		arr.forEach(function (callback) {
+			callback(err);
+		});
+	});
 };

src/upgrades/TEMPLATE

@@ -6,7 +6,10 @@ var async = require('async');
 var winston = require('winston');
 
 module.exports = {
+	// you should use spaces
+	// the underscores are there so you can double click to select the whole thing
 	name: 'User_friendly_upgrade_script_name',
+	// remember, month is zero-indexed (so January is 0, December is 11)
 	timestamp: Date.UTC(2017, 0, 1),
 	method: function (callback) {
 		// Do stuff here...