closes #732, custom robots.txt

public/templates/admin/settings.tpl

@@ -125,6 +125,14 @@
 	</div>
 </form>
 
+<form>
+	<h3>Web Crawler Settings</h3>
+	<div class="alert alert-warning">
+		<strong>Custom Robots.txt <small>Leave blank for default</small></strong><br /> 
+		<textarea class="form-control" data-field="robots.txt"></textarea>
+	</div>
+</form>
+
 <button class="btn btn-lg btn-primary" id="save">Save</button>
 
 <script>

src/webserver.js

@@ -754,9 +754,14 @@ if(nconf.get('ssl')) {
 
 		app.get('/robots.txt', function (req, res) {
 			res.set('Content-Type', 'text/plain');
-			res.send("User-agent: *\n" +
-				"Disallow: /admin/\n" +
-				"Sitemap: " + nconf.get('url') + "sitemap.xml");
+
+			if (meta.config["robots.txt"]) {
+				res.send(meta.config["robots.txt"])
+			} else {
+				res.send("User-agent: *\n" +
+					"Disallow: /admin/\n" +
+					"Sitemap: " + nconf.get('url') + "sitemap.xml");
+			}
 		});
 
 		app.get('/recent.rss', function(req, res) {