Minify modules in a batch

src/meta/js.js

@@ -88,44 +88,56 @@ module.exports = function (Meta) {
 		},
 	};
 
+	function copyFile(source, target, cb) {
+		var called = false;
+
+		var rd = fs.createReadStream(source);
+		rd.on('error', done);
+
+		var wr = fs.createWriteStream(target);
+		wr.on('error', done);
+		wr.on('close', function () {
+			done();
+		});
+		rd.pipe(wr);
+
+		function done(err) {
+			if (!called) {
+				cb(err);
+				called = true;
+			}
+		}
+	}
+
 	function minifyModules(modules, fork, callback) {
-		// for it to never fork
-		// otherwise it spawns way too many processes
-		// maybe eventually we can pool modules
-		// and pass the pools to the minifer
-		// to reduce the total number of threads
-		fork = false;
+		async.eachLimit(modules, 1000, function (mod, next) {
+			mkdirp(path.dirname(mod.destPath), next);
+		}, function (err) {
+			if (err) {
+				return callback(err);
+			}
 
-		async.eachLimit(modules, 500, function (mod, next) {
-			var srcPath = mod.srcPath;
-			var destPath = mod.destPath;
-
-			async.parallel({
-				dirped: function (cb) {
-					mkdirp(path.dirname(destPath), cb);
-				},
-				minified: function (cb) {
-					fs.readFile(srcPath, function (err, buffer) {
-						if (err) {
-							return cb(err);
-						}
-
-						if (srcPath.endsWith('.min.js') || path.dirname(srcPath).endsWith('min')) {
-							return cb(null, { code: buffer.toString() });
-						}
-
-						minifier.js.minify(buffer.toString(), fork, cb);
-					});
-				},
-			}, function (err, results) {
-				if (err) {
-					return next(err);
+			var filtered = modules.reduce(function (prev, mod) {
+				if (mod.srcPath.endsWith('.min.js') || path.dirname(mod.srcPath).endsWith('min')) {
+					prev.skip.push(mod);
+				} else {
+					prev.minify.push(mod);
 				}
 
-				var minified = results.minified;
-				fs.writeFile(destPath, minified.code, next);
-			});
-		}, callback);
+				return prev;
+			}, { minify: [], skip: [] });
+
+			async.parallel([
+				function (cb) {
+					minifier.js.minifyBatch(filtered.minify, fork, cb);
+				},
+				function (cb) {
+					async.eachLimit(filtered.skip, 500, function (mod, next) {
+						copyFile(mod.srcPath, mod.destPath, next);
+					}, cb);
+				},
+			], callback);
+		});
 	}
 
 	function linkModules(callback) {

src/meta/minifier.js

@@ -38,27 +38,26 @@ function setupDebugging() {
 	return forkProcessParams;
 }
 
-var children = [];
+var pool = [];
+var free = [];
 
 Minifier.maxThreads = os.cpus().length - 1;
 
 winston.verbose('[minifier] utilizing a maximum of ' + Minifier.maxThreads + ' additional threads');
 
 Minifier.killAll = function () {
-	children.forEach(function (child) {
+	pool.forEach(function (child) {
 		child.kill('SIGTERM');
 	});
 
-	children = [];
+	pool.length = 0;
 };
 
-function removeChild(proc) {
-	children = children.filter(function (child) {
-		return child !== proc;
-	});
-}
+function getChild() {
+	if (free.length) {
+		return free.shift();
+	}
 
-function forkAction(action, callback) {
 	var forkProcessParams = setupDebugging();
 	var proc = childProcess.fork(__filename, [], Object.assign({}, forkProcessParams, {
 		cwd: __dirname,
@@ -66,12 +65,26 @@ function forkAction(action, callback) {
 			minifier_child: true,
 		},
 	}));
+	pool.push(proc);
 
-	children.push(proc);
+	return proc;
+}
+
+function freeChild(proc) {
+	proc.removeAllListeners();
+	free.push(proc);
+}
+
+function removeChild(proc) {
+	var i = pool.indexOf(proc);
+	pool.splice(i, 1);
+}
+
+function forkAction(action, callback) {
+	var proc = getChild();
 
 	proc.on('message', function (message) {
-		proc.kill();
-		removeChild(proc);
+		freeChild(proc);
 
 		if (message.type === 'error') {
 			return callback(message.err);
@@ -102,7 +115,7 @@ if (process.env.minifier_child) {
 			if (typeof actions[action.act] !== 'function') {
 				process.send({
 					type: 'error',
-					message: 'Unknown action',
+					err: Error('Unknown action'),
 				});
 				return;
 			}
@@ -126,7 +139,7 @@ if (process.env.minifier_child) {
 }
 
 function executeAction(action, fork, callback) {
-	if (fork && children.length < Minifier.maxThreads) {
+	if (fork && (pool.length - free.length) < Minifier.maxThreads) {
 		forkAction(action, callback);
 	} else {
 		if (typeof actions[action.act] !== 'function') {
@@ -155,32 +168,38 @@ function concat(data, callback) {
 actions.concat = concat;
 
 function minifyJS(data, callback) {
-	var minified;
+	if (data.batch) {
+		async.eachLimit(data.files, 1000, function (ref, next) {
+			var srcPath = ref.srcPath;
+			var destPath = ref.destPath;
 
-	if (data.fromSource) {
-		var sources = data.source;
-		var multiple = Array.isArray(sources);
-		if (!multiple) {
-			sources = [sources];
-		}
+			fs.readFile(srcPath, function (err, buffer) {
+				if (err && err.code === 'ENOENT') {
+					return next(null, null);
+				}
+				if (err) {
+					return next(err);
+				}
 
-		try {
-			minified = sources.map(function (source) {
-				return uglifyjs.minify(source, {
-					// outSourceMap: data.filename + '.map',
-					compress: data.compress,
-					fromString: true,
-					output: {
-						// suppress uglify line length warnings
-						max_line_len: 400000,
-					},
-				});
+				try {
+					var minified = uglifyjs.minify(buffer.toString(), {
+						// outSourceMap: data.filename + '.map',
+						compress: data.compress,
+						fromString: true,
+						output: {
+							// suppress uglify line length warnings
+							max_line_len: 400000,
+						},
+					});
+
+					fs.writeFile(destPath, minified.code, next);
+				} catch (e) {
+					next(e);
+				}
 			});
-		} catch (e) {
-			return callback(e);
-		}
+		}, callback);
 
-		return callback(null, multiple ? minified : minified[0]);
+		return;
 	}
 
 	if (data.files && data.files.length) {
@@ -190,16 +209,16 @@ function minifyJS(data, callback) {
 			}
 
 			try {
-				minified = uglifyjs.minify(scripts, {
+				var minified = uglifyjs.minify(scripts, {
 					// outSourceMap: data.filename + '.map',
 					compress: data.compress,
 					fromString: false,
 				});
-			} catch (e) {
-				return callback(e);
-			}
 
-			callback(null, minified);
+				callback(null, minified);
+			} catch (e) {
+				callback(e);
+			}
 		});
 
 		return;
@@ -218,11 +237,11 @@ Minifier.js.bundle = function (scripts, minify, fork, callback) {
 	}, fork, callback);
 };
 
-Minifier.js.minify = function (source, fork, callback) {
+Minifier.js.minifyBatch = function (scripts, fork, callback) {
 	executeAction({
 		act: 'minifyJS',
-		fromSource: true,
-		source: source,
+		files: scripts,
+		batch: true,
 	}, fork, callback);
 };