From 6f120ff34ae2e6eeb241abc606cece94e30ad9e4 Mon Sep 17 00:00:00 2001
From: psychobunny <psycho.bunny@hotmail.com>
Date: Sun, 5 Jan 2014 23:19:52 -0500
Subject: [PATCH] closes #732, custom robots.txt

---
 public/templates/admin/settings.tpl |  8 ++++++++
 src/webserver.js                    | 11 ++++++++---
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/public/templates/admin/settings.tpl b/public/templates/admin/settings.tpl
index 6c22620426..fcb2114ce9 100644
--- a/public/templates/admin/settings.tpl
+++ b/public/templates/admin/settings.tpl
@@ -125,6 +125,14 @@
 	</div>
 </form>
 
+<form>
+	<h3>Web Crawler Settings</h3>
+	<div class="alert alert-warning">
+		<strong>Custom Robots.txt <small>Leave blank for default</small></strong><br /> 
+		<textarea class="form-control" data-field="robots.txt"></textarea>
+	</div>
+</form>
+
 <button class="btn btn-lg btn-primary" id="save">Save</button>
 
 <script>
diff --git a/src/webserver.js b/src/webserver.js
index 1436fc598b..f9761a2e40 100644
--- a/src/webserver.js
+++ b/src/webserver.js
@@ -754,9 +754,14 @@ if(nconf.get('ssl')) {
 
 		app.get('/robots.txt', function (req, res) {
 			res.set('Content-Type', 'text/plain');
-			res.send("User-agent: *\n" +
-				"Disallow: /admin/\n" +
-				"Sitemap: " + nconf.get('url') + "sitemap.xml");
+
+			if (meta.config["robots.txt"]) {
+				res.send(meta.config["robots.txt"])
+			} else {
+				res.send("User-agent: *\n" +
+					"Disallow: /admin/\n" +
+					"Sitemap: " + nconf.get('url') + "sitemap.xml");
+			}
 		});
 
 		app.get('/recent.rss', function(req, res) {