Merge branch 'master' into develop

.gitignore

@@ -55,3 +55,5 @@ tx.exe
 
 ##Coverage output
 coverage
+
+build

Gruntfile.js

@@ -27,6 +27,8 @@ module.exports = function (grunt) {
 			compiling = 'js';
 		} else if (target === 'templatesUpdated') {
 			compiling = 'tpl';
+		} else if (target === 'langUpdated') {
+			compiling = 'lang';
 		} else if (target === 'serverUpdated') {
 			// Do nothing, just restart
 		}
@@ -93,7 +95,18 @@ module.exports = function (grunt) {
 					'!node_modules/nodebb-*/node_modules/**',
 					'!node_modules/nodebb-*/.git/**'
 				]
-			}
+			},
+			langUpdated: {
+				files: [
+					'public/language/**/*.json',
+					'node_modules/nodebb-*/**/*.json',
+					'!node_modules/nodebb-*/node_modules/**',
+					'!node_modules/nodebb-*/.git/**',
+					'!node_modules/nodebb-*/plugin.json',
+					'!node_modules/nodebb-*/package.json',
+					'!node_modules/nodebb-*/theme.json',
+				],
+			},
 		}
 	});
 

build.js

@@ -5,7 +5,7 @@ var winston = require('winston');
 
 var buildStart;
 
-var valid = ['js', 'clientCSS', 'acpCSS', 'tpl'];
+var valid = ['js', 'clientCSS', 'acpCSS', 'tpl', 'lang'];
 
 exports.buildAll = function (callback) {
 	exports.build(valid.join(','), callback);
@@ -89,6 +89,12 @@ exports.buildTargets = function (targets, callback) {
 						meta.templates.compile(step.bind(this, startTime, target, next));
 						break;
 					
+					case 'lang':
+						winston.info('[build] Building language files');
+						startTime = Date.now();
+						meta.languages.build(step.bind(this, startTime, target, next));
+						break;
+
 					default:
 						winston.warn('[build] Unknown build target: \'' + target + '\'');
 						setImmediate(next);

nodebb

@@ -375,7 +375,7 @@ switch(process.argv[2]) {
 		async.series([
 			function (next) {
 				process.stdout.write('1. '.bold + 'Bringing base dependencies up to date... '.yellow);
-				require('child_process').execFile('/usr/bin/env', ['npm', 'i', '--production'], { stdio: 'ignore' }, next);
+				cproc.exec('npm i --production', { cwd: __dirname, stdio: 'ignore' }, next);
 			},
 			function (next) {
 				process.stdout.write('OK\n'.green);

public/src/client/category.js

@@ -103,12 +103,9 @@ define('forum/category', [
 		return bottomIndex;
 	};
 
-	$(window).on('action:popstate', function (ev, data) {
-		if (data.url.startsWith('category/')) {
-			var cid = data.url.match(/^category\/(\d+)/);
-			if (cid && cid[1]) {
-				cid = cid[1];
-			}
+	$(window).on('action:ajaxify.contentLoaded', function (ev, data) {
+		if (ajaxify.data.template.category) {
+			var cid = ajaxify.data.cid;
 			if (!cid) {
 				return;
 			}
@@ -140,8 +137,10 @@ define('forum/category', [
 				$('[component="category"]').empty();
 
 				loadTopicsAfter(Math.max(0, bookmarkIndex - 1) + 1, 1, function () {
+					$(window).one('action:topics.loaded', function () {
 						Category.scrollToTopic(bookmarkIndex, clickedIndex, 0);
 					});
+				});
 			}
 		}
 	});
@@ -167,9 +166,8 @@ define('forum/category', [
 		}
 
 		var scrollTo = components.get('category/topic', 'index', bookmarkIndex);
-		var	cid = ajaxify.data.cid;
 
-		if (scrollTo.length && cid) {
+		if (scrollTo.length) {
 			$('html, body').animate({
 				scrollTop: (scrollTo.offset().top - offset) + 'px'
 			}, duration !== undefined ? duration : 400, function () {
@@ -272,7 +270,7 @@ define('forum/category', [
 			return callback();
 		}
 
-		$(window).trigger('action:categories.loading');
+		$(window).trigger('action:category.loading');
 		var params = utils.params();
 		infinitescroll.loadMore('categories.loadMore', {
 			cid: ajaxify.data.cid,
@@ -288,7 +286,7 @@ define('forum/category', [
 				done();
 			}
 
-			$(window).trigger('action:categories.loaded');
+			$(window).trigger('action:category.loaded');
 			callback();
 		});
 	}

public/src/modules/translator.js

@@ -3,7 +3,7 @@
 (function (factory) {
 	'use strict';
 	function loadClient(language, namespace) {
-		return Promise.resolve(jQuery.getJSON(config.relative_path + '/api/language/' + language + '/' + namespace));
+		return Promise.resolve(jQuery.getJSON(config.relative_path + '/assets/language/' + language + '/' + namespace + '.json?' + config['cache-buster']));
 	}
 	var warn = function () {};
 	if (typeof config === 'object' && config.environment === 'development') {
@@ -17,7 +17,6 @@
 	} else if (typeof module === 'object' && module.exports) {
 		// Node
 		(function () {
-			require('promise-polyfill');
 			var languages = require('../../../src/languages');
 
 			if (global.env === 'development') {
@@ -292,7 +291,7 @@
 				warn('[translator] Parameter `namespace` is ' + namespace + (namespace === '' ? '(empty string)' : ''));
 				translation = Promise.resolve({});
 			} else {
-				translation = this.translations[namespace] = this.translations[namespace] || this.load(this.lang, namespace);
+				translation = this.translations[namespace] = this.translations[namespace] || this.load(this.lang, namespace).catch(function () { return {}; });
 			}
 
 			if (key) {

public/src/require-config.js

@@ -1,7 +1,7 @@
 require.config({
 	baseUrl: config.relative_path + "/src/modules",
 	waitSeconds: 7,
-	urlArgs: "v=" + config['cache-buster'],
+	urlArgs: config['cache-buster'],
 	paths: {
 		'forum': '../client',
 		'admin': '../admin',

src/admin/search.js

@@ -5,7 +5,6 @@ var path = require('path');
 var async = require('async');
 var sanitizeHTML = require('sanitize-html');
 
-var languages = require('../languages');
 var utils = require('../../public/src/utils');
 var Translator = require('../../public/src/modules/translator').Translator;
 
@@ -19,7 +18,7 @@ function filterDirectories(directories) {
 		// exclude category.tpl, group.tpl, category-analytics.tpl
 		return !dir.includes('/partials/') &&
 			/\/.*\//.test(dir) &&
-			!/category|group|category\-analytics$/.test(dir);
+			!/manage\/(category|group|category\-analytics)$/.test(dir);
 	});
 }
 
@@ -107,6 +106,8 @@ function fallback(namespace, callback) {
 }
 
 function initDict(language, callback) {
+	var translator = Translator.create(language);
+
 	getAdminNamespaces(function (err, namespaces) {
 		if (err) {
 			return callback(err);
@@ -115,7 +116,9 @@ function initDict(language, callback) {
 		async.map(namespaces, function (namespace, cb) {
 			async.waterfall([
 				function (next) {
-					languages.get(language, namespace, next);
+					translator.getTranslation(namespace).then(function (translations) {
+						next(null, translations);
+					}, next);
 				},
 				function (translations, next) {
 					if (!translations || !Object.keys(translations).length) {
@@ -139,7 +142,7 @@ function initDict(language, callback) {
 							title[1] + '/' + title[2] + ']]') : '');
 					}
 
-					Translator.create(language).translate(title).then(function (title) {
+					translator.translate(title).then(function (title) {
 						next(null, {
 							namespace: namespace,
 							translations: str + '\n' + title,

src/controllers/index.js

@@ -353,7 +353,6 @@ Controllers.ping = function (req, res) {
 
 Controllers.handle404 = function (req, res) {
 	var relativePath = nconf.get('relative_path');
-	var isLanguage = new RegExp('^' + relativePath + '/api/language/.*/.*');
 	var isClientScript = new RegExp('^' + relativePath + '\\/src\\/.+\\.js');
 
 	if (plugins.hasListeners('action:meta.override404')) {
@@ -366,8 +365,6 @@ Controllers.handle404 = function (req, res) {
 
 	if (isClientScript.test(req.url)) {
 		res.type('text/javascript').status(200).send('');
-	} else if (isLanguage.test(req.url)) {
-		res.status(200).json({});
 	} else if (req.path.startsWith(relativePath + '/uploads') || (req.get('accept') && req.get('accept').indexOf('text/html') === -1) || req.path === '/favicon.ico') {
 		meta.errors.log404(req.path || '');
 		res.sendStatus(404);

src/languages.js

@@ -3,53 +3,27 @@
 var fs = require('fs');
 var path = require('path');
 var async = require('async');
-var LRU = require('lru-cache');
-
-var plugins = require('./plugins');
 
 var Languages = {};
-var	languagesPath = path.join(__dirname, '../public/language');
+var	languagesPath = path.join(__dirname, '../build/public/language');
 
 Languages.init = function (next) {
-	if (Languages.hasOwnProperty('_cache')) {
-		Languages._cache.reset();
-	} else {
-		Languages._cache = LRU(100);
-	}
-
 	next();
 };
 
 Languages.get = function (language, namespace, callback) {
-	var langNamespace = language + '/' + namespace;
-
-	if (Languages._cache && Languages._cache.has(langNamespace)) {
-		return callback(null, Languages._cache.get(langNamespace));
-	}
-
-	var languageData;
-
 	fs.readFile(path.join(languagesPath, language, namespace + '.json'), { encoding: 'utf-8' }, function (err, data) {
-		if (err && err.code !== 'ENOENT') {
+		if (err) {
 			return callback(err);
 		}
 
-		// If language file in core cannot be read, then no language file present
 		try {
-			languageData = JSON.parse(data) || {};
+			data = JSON.parse(data) || {};
 		} catch (e) {
-			languageData = {};
+			return callback(e);
 		}
 
-		if (plugins.customLanguages.hasOwnProperty(langNamespace)) {
-			Object.assign(languageData, plugins.customLanguages[langNamespace]);
-		}
-
-		if (Languages._cache) {
-			Languages._cache.set(langNamespace, languageData);
-		}
-
-		callback(null, languageData);
+		callback(null, data);
 	});
 };
 
@@ -73,11 +47,13 @@ Languages.list = function (callback) {
 
 				var configPath = path.join(languagesPath, folder, 'language.json');
 
-				fs.readFile(configPath, function (err, stream) {
-					if (err) {
+				fs.readFile(configPath, function (err, buffer) {
+					if (err && err.code !== 'ENOENT') {
 						return next(err);
 					}
-					languages.push(JSON.parse(stream.toString()));
+					if (buffer) {
+						languages.push(JSON.parse(buffer.toString()));
+					}
 					next();
 				});
 			});

src/meta.js

@@ -23,6 +23,7 @@ var utils = require('../public/src/utils');
 	require('./meta/dependencies')(Meta);
 	Meta.templates = require('./meta/templates');
 	Meta.blacklist = require('./meta/blacklist');
+	Meta.languages = require('./meta/languages');
 
 	/* Assorted */
 	Meta.userOrGroupExists = function (slug, callback) {

src/meta/configs.js

@@ -21,7 +21,7 @@ module.exports = function (Meta) {
 				Meta.configs.list(next);
 			},
 			function (config, next) {
-				config['cache-buster'] = utils.generateUUID();
+				config['cache-buster'] = 'v=' + utils.generateUUID();
 
 				Meta.config = config;
 				setImmediate(next);

src/meta/languages.js

@@ -0,0 +1,221 @@
+'use strict';
+
+var winston = require('winston');
+var path = require('path');
+var async = require('async');
+var fs = require('fs');
+var mkdirp = require('mkdirp');
+
+var file = require('../file');
+var utils = require('../../public/src/utils');
+var Plugins = require('../plugins');
+var db = require('../database');
+
+var buildLanguagesPath = path.join(__dirname, '../../build/public/language');
+var	coreLanguagesPath = path.join(__dirname, '../../public/language');
+
+function getTranslationTree(callback) {
+	async.waterfall([
+		// get plugin data
+		function (next) {
+			db.getSortedSetRange('plugins:active', 0, -1, next);
+		},
+		function (plugins, next) {
+			var pluginBasePath = path.join(__dirname, '../../node_modules');
+			var paths = plugins.map(function (plugin) {
+				return path.join(pluginBasePath, plugin);
+			});
+
+			// Filter out plugins with invalid paths
+			async.filter(paths, file.exists, function (paths) {
+				next(null, paths);
+			});
+		},
+		function (paths, next) {
+			async.map(paths, Plugins.loadPluginInfo, next);
+		},
+
+		// generate list of languages and namespaces
+		function (plugins, next) {
+			var languages = [], namespaces = [];
+
+			// pull languages and namespaces from paths
+			function extrude(languageDir, paths) {
+				paths.forEach(function (p) {
+					var rel = p.split(languageDir)[1].split(/[\/\\]/).slice(1);
+					var language = rel.shift().replace('_', '-').replace('@', '-x-');
+					var namespace = rel.join('/').replace(/\.json$/, '');
+
+					if (!language || !namespace) {
+						return;
+					}
+
+					if (languages.indexOf(language) === -1) {
+						languages.push(language);
+					}
+					if (namespaces.indexOf(namespace) === -1) {
+						namespaces.push(namespace);
+					}
+				});
+			}
+
+			plugins = plugins.filter(function (pluginData) {
+				return (typeof pluginData.languages === 'string');
+			});
+			async.parallel([
+				// get core languages and namespaces
+				function (nxt) {
+					utils.walk(coreLanguagesPath, function (err, paths) {
+						if (err) {
+							return nxt(err);
+						}
+
+						extrude(coreLanguagesPath, paths);
+						nxt();
+					});
+				},
+				// get plugin languages and namespaces
+				function (nxt) {
+					async.each(plugins, function (pluginData, cb) {
+						var pathToFolder = path.join(__dirname, '../../node_modules/', pluginData.id, pluginData.languages);
+						utils.walk(pathToFolder, function (err, paths) {
+							if (err) {
+								return cb(err);
+							}
+
+							extrude(pathToFolder, paths);
+							cb();
+						});
+					}, nxt);
+				},
+			], function (err) {
+				if (err) {
+					return next(err);
+				}
+
+				next(null, {
+					languages: languages,
+					namespaces: namespaces,
+					plugins: plugins,
+				});
+			});
+		},
+
+		// for each language and namespace combination,
+		// run through core and all plugins to generate
+		// a full translation hash
+		function (ref, next) {
+			var languages = ref.languages;
+			var namespaces = ref.namespaces;
+			var plugins = ref.plugins;
+
+			var tree = {};
+
+			async.eachLimit(languages, 10, function (lang, nxt) {
+				async.eachLimit(namespaces, 10, function (ns, cb) {
+					var translations = {};
+
+					async.series([
+						// core first
+						function (n) {
+							fs.readFile(path.join(coreLanguagesPath, lang, ns + '.json'), function (err, buffer) {
+								if (err) {
+									if (err.code === 'ENOENT') {
+										return n();
+									}
+									return n(err);
+								}
+
+								try {
+									Object.assign(translations, JSON.parse(buffer.toString()));
+									n();
+								} catch (err) {
+									n(err);
+								}
+							});
+						},
+						function (n) {
+							// for each plugin, fallback in this order:
+							//  1. correct language string (en-GB)
+							//  2. old language string (en_GB)
+							//  3. plugin defaultLang (en-US)
+							//  4. old plugin defaultLang (en_US)
+							async.eachLimit(plugins, 10, function (pluginData, call) {
+								var pluginLanguages = path.join(__dirname, '../../node_modules/', pluginData.id, pluginData.languages);
+								function tryLang(lang, onEnoent) {
+									fs.readFile(path.join(pluginLanguages, lang, ns + '.json'), function (err, buffer) {
+										if (err) {
+											if (err.code === 'ENOENT') {
+												return onEnoent();
+											}
+											return call(err);
+										}
+
+										try {
+											Object.assign(translations, JSON.parse(buffer.toString()));
+											call();
+										} catch (err) {
+											call(err);
+										}
+									});
+								}
+
+								tryLang(lang, function () {
+									tryLang(lang.replace('-', '_').replace('-x-', '@'), function () {
+										tryLang(pluginData.defaultLang, function () {
+											tryLang(pluginData.defaultLang.replace('-', '_').replace('-x-', '@'), call);
+										});
+									});
+								});
+							}, function (err) {
+								if (err) {
+									return n(err);
+								}
+
+								tree[lang] = tree[lang] || {};
+								tree[lang][ns] = translations;
+								n();
+							});
+						},
+					], cb);
+				}, nxt);
+			}, function (err) {
+				next(err, tree);
+			});
+		},
+	], callback);
+}
+
+// write translation hashes from the generated tree to language files
+function writeLanguageFiles(tree, callback) {
+	// iterate over languages and namespaces
+	async.eachLimit(Object.keys(tree), 10, function (language, cb) {
+		var namespaces = tree[language];
+		async.eachLimit(Object.keys(namespaces), 100, function (namespace, next) {
+			var translations = namespaces[namespace];
+
+			var filePath = path.join(buildLanguagesPath, language, namespace + '.json');
+
+			mkdirp(path.dirname(filePath), function (err) {
+				if (err) {
+					return next(err);
+				}
+
+				fs.writeFile(filePath, JSON.stringify(translations), next);
+			});
+		}, cb);
+	}, callback);
+}
+
+exports.build = function buildLanguages(callback) {
+	async.waterfall([
+		getTranslationTree,
+		writeLanguageFiles,
+	], function (err) {
+		if (err) {
+			winston.error('[build] Language build failed: ' + err.message);
+			throw err;
+		}
+		callback();
+	});
+};

src/middleware/index.js

@@ -207,23 +207,6 @@ middleware.applyBlacklist = function (req, res, next) {
 	});
 };
 
-middleware.getTranslation = function (req, res, next) {
-	var language = req.params.language;
-	var namespace = req.params[0];
-
-	if (language && namespace) {
-		languages.get(language, namespace, function (err, translations) {
-			if (err) {
-				return next(err);
-			}
-
-			res.status(200).json(translations);
-		});
-	} else {
-		res.status(404).json('{}');
-	}
-};
-
 middleware.processTimeagoLocales = function (req, res, next) {
 	var fallback = req.path.indexOf('-short') === -1 ? 'jquery.timeago.en.js' : 'jquery.timeago.en-short.js',
 		localPath = path.join(__dirname, '../../public/vendor/jquery/timeago/locales', req.path),

src/middleware/maintenance.js

@@ -24,7 +24,7 @@ module.exports = function (middleware) {
 			'^/templates/[\\w/]+.tpl',
 			'^/api/login',
 			'^/api/widgets/render',
-			'^/api/language/.+',
+			'^/public/language',
 			'^/uploads/system/site-logo.png'
 		];
 

src/plugins.js

@@ -29,8 +29,6 @@ var middleware;
 	Plugins.lessFiles = [];
 	Plugins.clientScripts = [];
 	Plugins.acpScripts = [];
-	Plugins.customLanguages = {};
-	Plugins.customLanguageFallbacks = {};
 	Plugins.libraryPaths = [];
 	Plugins.versionWarning = [];
 	Plugins.languageCodes = [];

src/plugins/load.js

@@ -9,8 +9,6 @@ var winston = require('winston');
 var nconf = require('nconf');
 var _ = require('underscore');
 var file = require('../file');
-
-var utils = require('../../public/src/utils');
 var meta = require('../meta');
 
 
@@ -91,9 +89,6 @@ module.exports = function (Plugins) {
 				function (next) {
 					mapClientModules(pluginData, next);
 				},
-				function (next) {
-					loadLanguages(pluginData, next);
-				}
 			], function (err) {
 				if (err) {
 					winston.verbose('[plugins] Could not load plugin : ' + pluginData.id);
@@ -252,60 +247,6 @@ module.exports = function (Plugins) {
 		callback();
 	}
 
-	function loadLanguages(pluginData, callback) {
-		if (typeof pluginData.languages !== 'string') {
-			return callback();
-		}
-
-		var pathToFolder = path.join(__dirname, '../../node_modules/', pluginData.id, pluginData.languages);
-		var defaultLang = (pluginData.defaultLang || 'en_GB').replace('_', '-').replace('@', '-x-');
-
-		utils.walk(pathToFolder, function (err, languages) {
-			if (err) {
-				return callback(err);
-			}
-
-			async.each(languages, function (pathToLang, next) {
-				fs.readFile(pathToLang, function (err, file) {
-					if (err) {
-						return next(err);
-					}
-					var data;
-					var language = path.dirname(pathToLang).split(/[\/\\]/).pop().replace('_', '-').replace('@', '-x-');
-					var namespace = path.basename(pathToLang, '.json');
-					var langNamespace = language + '/' + namespace;
-
-					try {
-						data = JSON.parse(file.toString());
-					} catch (err) {
-						winston.error('[plugins] Unable to parse custom language file: ' + pathToLang + '\r\n' + err.stack);
-						return next(err);
-					}
-
-					Plugins.customLanguages[langNamespace] = Plugins.customLanguages[langNamespace] || {};
-					Object.assign(Plugins.customLanguages[langNamespace], data);
-
-					if (defaultLang && defaultLang === language) {
-						Plugins.languageCodes.filter(function (lang) {
-							return defaultLang !== lang;
-						}).forEach(function (lang) {
-							var langNS = lang + '/' + namespace;
-							Plugins.customLanguages[langNS] = Object.assign(Plugins.customLanguages[langNS] || {}, data);
-						});
-					}
-
-					next();
-				});
-			}, function (err) {
-				if (err) {
-					return callback(err);
-				}
-
-				callback();
-			});
-		});
-	}
-
 	function resolveModulePath(fullPath, relPath) {
 		/**
 		  * With npm@3, dependencies can become flattened, and appear at the root level.
@@ -363,6 +304,7 @@ module.exports = function (Plugins) {
 
 				return callback(new Error('[[error:parse-error]]'));
 			}
+
 			callback(null, pluginData);
 		});
 	};

src/routes/index.js

@@ -4,6 +4,7 @@ var nconf = require('nconf');
 var winston = require('winston');
 var path = require('path');
 var async = require('async');
+var meta = require('../meta');
 var controllers = require('../controllers');
 var plugins = require('../plugins');
 var user = require('../user');
@@ -145,7 +146,17 @@ module.exports = function (app, middleware, hotswapIds) {
 	}
 
 	app.use(middleware.privateUploads);
-	app.use(relativePath + '/api/language/:language/(([a-zA-Z0-9\\-_.\\/]+))', middleware.getTranslation);
+	app.use(relativePath + '/assets', express.static(path.join(__dirname, '../../', 'build/public'), {
+		maxAge: app.enabled('cache') ? 5184000000 : 0
+	}));
+
+	// DEPRECATED
+	app.use(relativePath + '/api/language', function (req, res) {
+		winston.warn('[deprecated] Accessing language files from `/api/language` is deprecated. ' + 
+			'Use `/assets/language/[langCode]/[namespace].json` for prefetch paths.');
+		res.redirect(relativePath + '/assets/language' + req.path + '.json?' + meta.config['cache-buster']);
+	});
+
 	app.use(relativePath, express.static(path.join(__dirname, '../../', 'public'), {
 		maxAge: app.enabled('cache') ? 5184000000 : 0
 	}));