From 782858c7286992a79d12e43457d6faf64420da54 Mon Sep 17 00:00:00 2001
From: Julian Lam <julian.lam@gmail.com>
Date: Sat, 20 Jul 2013 11:19:08 -0400
Subject: [PATCH] added dynamically generated robots.txt

---
 src/webserver.js | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/webserver.js b/src/webserver.js
index 513d1f7453..ae641d95f4 100644
--- a/src/webserver.js
+++ b/src/webserver.js
@@ -237,6 +237,14 @@ var express = require('express'),
 			});
 		});
 
+		app.get('/robots.txt', function(req, res) {
+			res.set('Content-Type', 'text/plain');
+			res.send(	"User-agent: *\n" +
+						"Disallow: \n" +
+						"Disallow: /admin/\n" +
+						"Sitemap: " + global.nconf.get('url') + "sitemap.xml");
+		});
+
 		app.get('/api/:method', api_method);
 		app.get('/api/:method/:id', api_method);
 		// ok fine MUST ADD RECURSION style. I'll look for a better fix in future but unblocking baris for this: