closes #2304

install/package.json

@@ -95,6 +95,7 @@
         "socket.io-redis": "5.2.0",
         "socketio-wildcard": "2.0.0",
         "spdx-license-list": "^3.0.1",
+        "spider-detector": "1.0.18",
         "toobusy-js": "^0.5.1",
         "uglify-js": "^3.3.4",
         "validator": "9.2.0",

src/controllers/accounts/chats.js

@@ -97,7 +97,7 @@ chatsController.get = function (req, res, callback) {
 
 chatsController.redirectToChat = function (req, res, next) {
 	var roomid = parseInt(req.params.roomid, 10);
-	if (!req.uid) {
+	if (!req.loggedIn) {
 		return next();
 	}
 	async.waterfall([

src/controllers/admin/categories.js

@@ -13,7 +13,7 @@ categoriesController.get = function (req, res, callback) {
 	async.waterfall([
 		function (next) {
 			async.parallel({
-				category: async.apply(categories.getCategories, [req.params.category_id], req.user.uid),
+				category: async.apply(categories.getCategories, [req.params.category_id], req.uid),
 				allCategories: async.apply(categories.buildForSelect, req.uid, 'read'),
 			}, next);
 		},

src/controllers/admin/uploads.js

@@ -253,7 +253,7 @@ function uploadImage(filename, folder, uploadedFile, req, res, next) {
 	async.waterfall([
 		function (next) {
 			if (plugins.hasListeners('filter:uploadImage')) {
-				plugins.fireHook('filter:uploadImage', { image: uploadedFile, uid: req.user.uid }, next);
+				plugins.fireHook('filter:uploadImage', { image: uploadedFile, uid: req.uid }, next);
 			} else {
 				file.saveFileToLocal(filename, folder, uploadedFile.path, next);
 			}

src/controllers/admin/users.js

@@ -191,7 +191,7 @@ usersController.getCSV = function (req, res, next) {
 	}
 	events.log({
 		type: 'getUsersCSV',
-		uid: req.user.uid,
+		uid: req.uid,
 		ip: req.ip,
 	});
 	async.waterfall([

src/controllers/api.js

@@ -78,7 +78,7 @@ apiController.loadConfig = function (req, callback) {
 
 	async.waterfall([
 		function (next) {
-			if (!req.uid) {
+			if (!req.loggedIn) {
 				return next(null, config);
 			}
 			user.getSettings(req.uid, next);

src/controllers/authentication.js

@@ -417,7 +417,7 @@ authenticationController.localLogin = function (req, username, password, next) {
 };
 
 authenticationController.logout = function (req, res, next) {
-	if (!req.uid || !req.sessionID) {
+	if (!req.loggedIn || !req.sessionID) {
 		return res.status(200).send('not-logged-in');
 	}
 

src/controllers/helpers.js

@@ -24,7 +24,7 @@ helpers.noScriptErrors = function (req, res, error, httpStatus) {
 	middleware.buildHeader(req, res, function () {
 		res.status(httpStatus).render(httpStatusString, {
 			path: req.path,
-			loggedIn: true,
+			loggedIn: req.loggedIn,
 			error: error,
 			returnLink: true,
 			title: '[[global:' + httpStatusString + '.title]]',
@@ -67,11 +67,11 @@ helpers.notAllowed = function (req, res, error) {
 		if (err) {
 			return winston.error(err);
 		}
-		if (req.uid) {
+		if (req.loggedIn) {
 			if (res.locals.isAPI) {
 				res.status(403).json({
 					path: req.path.replace(/^\/api/, ''),
-					loggedIn: !!req.uid,
+					loggedIn: req.loggedIn,
 					error: error,
 					title: '[[global:403.title]]',
 				});
@@ -79,7 +79,7 @@ helpers.notAllowed = function (req, res, error) {
 				middleware.buildHeader(req, res, function () {
 					res.status(403).render('403', {
 						path: req.path,
-						loggedIn: !!req.uid,
+						loggedIn: req.loggedIn,
 						error: error,
 						title: '[[global:403.title]]',
 					});

src/controllers/index.js

@@ -113,7 +113,7 @@ Controllers.login = function (req, res, next) {
 		}
 		return res.redirect(nconf.get('relative_path') + data.authentication[0].url);
 	}
-	if (req.uid) {
+	if (req.loggedIn) {
 		user.getUserFields(req.uid, ['username', 'email'], function (err, user) {
 			if (err) {
 				return next(err);

src/controllers/popular.js

@@ -37,7 +37,7 @@ popularController.get = function (req, res, next) {
 		alltime: '[[global:header.popular]]',
 	};
 
-	if (!req.uid) {
+	if (!req.loggedIn) {
 		if (anonCache[term] && (Date.now() - lastUpdateTime) < 60 * 60 * 1000) {
 			return res.render('popular', anonCache[term]);
 		}
@@ -73,7 +73,7 @@ popularController.get = function (req, res, next) {
 				data.breadcrumbs = helpers.buildBreadcrumbs(breadcrumbs);
 			}
 
-			if (!req.uid) {
+			if (!req.loggedIn) {
 				anonCache[term] = data;
 				lastUpdateTime = Date.now();
 			}

src/controllers/recent.js

@@ -58,7 +58,7 @@ recentController.get = function (req, res, next) {
 			data.set = 'topics:recent';
 			data['feeds:disableRSS'] = parseInt(meta.config['feeds:disableRSS'], 10) === 1;
 			data.rssFeedUrl = nconf.get('relative_path') + '/recent.rss';
-			if (req.uid) {
+			if (req.loggedIn) {
 				data.rssFeedUrl += '?uid=' + req.uid + '&token=' + rssToken;
 			}
 			data.title = meta.config.homePageTitle || '[[pages:home]]';

src/controllers/search.js

@@ -11,15 +11,14 @@ var categories = require('../categories');
 var pagination = require('../pagination');
 var helpers = require('./helpers');
 
-
-var searchController = {};
+var searchController = module.exports;
 
 searchController.search = function (req, res, next) {
 	if (!plugins.hasListeners('filter:search.query')) {
 		return next();
 	}
 
-	if (!req.user && parseInt(meta.config.allowGuestSearching, 10) !== 1) {
+	if (!req.loggedIn && parseInt(meta.config.allowGuestSearching, 10) !== 1) {
 		return helpers.notAllowed(req, res);
 	}
 
@@ -78,5 +77,3 @@ searchController.search = function (req, res, next) {
 		res.render('search', searchData);
 	});
 };
-
-module.exports = searchController;

src/controllers/top.js

@@ -58,7 +58,7 @@ topController.get = function (req, res, next) {
 			data.set = 'topics:votes';
 			data['feeds:disableRSS'] = parseInt(meta.config['feeds:disableRSS'], 10) === 1;
 			data.rssFeedUrl = nconf.get('relative_path') + '/top.rss';
-			if (req.uid) {
+			if (req.loggedIn) {
 				data.rssFeedUrl += '?uid=' + req.uid + '&token=' + rssToken;
 			}
 			data.title = meta.config.homePageTitle || '[[pages:home]]';

src/controllers/topics.js

@@ -146,7 +146,7 @@ topicsController.get = function (req, res, callback) {
 			topicData.postDeleteDuration = parseInt(meta.config.postDeleteDuration, 10) || 0;
 			topicData.scrollToMyPost = settings.scrollToMyPost;
 			topicData.rssFeedUrl = nconf.get('relative_path') + '/topic/' + topicData.tid + '.rss';
-			if (req.uid) {
+			if (req.loggedIn) {
 				topicData.rssFeedUrl += '?uid=' + req.uid + '&token=' + rssToken;
 			}
 
@@ -165,7 +165,7 @@ topicsController.get = function (req, res, callback) {
 				req.session.tids_viewed[tid] = Date.now();
 			}
 
-			if (req.uid) {
+			if (req.loggedIn) {
 				topics.markAsRead([tid], req.uid, function (err, markedRead) {
 					if (err) {
 						return callback(err);

src/controllers/user.js

@@ -9,7 +9,7 @@ var accountHelpers = require('./accounts/helpers');
 var userController = module.exports;
 
 userController.getCurrentUser = function (req, res, next) {
-	if (!req.uid) {
+	if (!req.loggedIn) {
 		return res.status(401).json('not-authorized');
 	}
 	async.waterfall([

src/groups.js

@@ -22,7 +22,7 @@ require('./groups/posts')(Groups);
 require('./groups/user')(Groups);
 
 
-Groups.ephemeralGroups = ['guests'];
+Groups.ephemeralGroups = ['guests', 'spiders'];
 
 Groups.getEphemeralGroup = function (groupName) {
 	return {

src/middleware/header.js

@@ -93,7 +93,7 @@ module.exports = function (middleware) {
 							reputation: 0,
 							'email:confirmed': 0,
 						};
-						if (req.uid) {
+						if (req.loggedIn) {
 							user.getUserFields(req.uid, Object.keys(userData), next);
 						} else {
 							next(null, userData);

src/middleware/index.js

@@ -59,12 +59,12 @@ middleware.pageView = function (req, res, next) {
 
 	plugins.fireHook('action:middleware.pageView', { req: req });
 
-	if (req.user) {
-		user.updateLastOnlineTime(req.user.uid);
+	if (req.loggedIn) {
+		user.updateLastOnlineTime(req.uid);
 		if (req.path.startsWith('/api/users') || req.path.startsWith('/users')) {
-			user.updateOnlineUsers(req.user.uid, next);
+			user.updateOnlineUsers(req.uid, next);
 		} else {
-			user.updateOnlineUsers(req.user.uid);
+			user.updateOnlineUsers(req.uid);
 			next();
 		}
 	} else {
@@ -112,7 +112,7 @@ middleware.routeTouchIcon = function (req, res) {
 };
 
 middleware.privateTagListing = function (req, res, next) {
-	if (!req.user && parseInt(meta.config.privateTagListing, 10) === 1) {
+	if (!req.loggedIn && parseInt(meta.config.privateTagListing, 10) === 1) {
 		controllers.helpers.notAllowed(req, res);
 	} else {
 		next();
@@ -143,7 +143,7 @@ function expose(exposedField, method, field, req, res, next) {
 }
 
 middleware.privateUploads = function (req, res, next) {
-	if (req.user || parseInt(meta.config.privateUploads, 10) !== 1) {
+	if (req.loggedIn || parseInt(meta.config.privateUploads, 10) !== 1) {
 		return next();
 	}
 	if (req.path.startsWith(nconf.get('relative_path') + '/assets/uploads/files')) {

src/middleware/user.js

@@ -14,7 +14,7 @@ var controllers = {
 
 module.exports = function (middleware) {
 	middleware.authenticate = function (req, res, next) {
-		if (req.uid) {
+		if (req.loggedIn) {
 			return next();
 		}
 
@@ -44,7 +44,7 @@ module.exports = function (middleware) {
 		*/
 		async.waterfall([
 			function (next) {
-				if (!req.uid) {
+				if (!req.loggedIn) {
 					return setImmediate(next, null, false);
 				}
 
@@ -64,7 +64,7 @@ module.exports = function (middleware) {
 	}
 
 	middleware.checkGlobalPrivacySettings = function (req, res, next) {
-		if (!req.uid && !!parseInt(meta.config.privateUserInfo, 10)) {
+		if (!req.loggedIn && !!parseInt(meta.config.privateUserInfo, 10)) {
 			return middleware.authenticate(req, res, next);
 		}
 
@@ -202,7 +202,7 @@ module.exports = function (middleware) {
 	};
 
 	middleware.requireUser = function (req, res, next) {
-		if (req.uid) {
+		if (req.loggedIn) {
 			return next();
 		}
 

src/privileges/helpers.js

@@ -10,6 +10,11 @@ var plugins = require('../plugins');
 
 var helpers = module.exports;
 
+var uidToSystemGroup = {
+	0: 'guests',
+	'-1': 'spiders',
+};
+
 helpers.some = function (tasks, callback) {
 	async.some(tasks, function (task, next) {
 		task(next);
@@ -27,8 +32,8 @@ helpers.isUserAllowedTo = function (privilege, uid, cid, callback) {
 };
 
 function isUserAllowedToCids(privilege, uid, cids, callback) {
-	if (parseInt(uid, 10) === 0) {
-		return isGuestAllowedToCids(privilege, cids, callback);
+	if (parseInt(uid, 10) <= 0) {
+		return isSystemGroupAllowedToCids(privilege, uid, cids, callback);
 	}
 
 	var userKeys = [];
@@ -42,8 +47,8 @@ function isUserAllowedToCids(privilege, uid, cids, callback) {
 }
 
 function isUserAllowedToPrivileges(privileges, uid, cid, callback) {
-	if (parseInt(uid, 10) === 0) {
-		return isGuestAllowedToPrivileges(privileges, cid, callback);
+	if (parseInt(uid, 10) <= 0) {
+		return isSystemGroupAllowedToPrivileges(privileges, uid, cid, callback);
 	}
 
 	var userKeys = [];
@@ -100,20 +105,20 @@ helpers.isUsersAllowedTo = function (privilege, uids, cid, callback) {
 	], callback);
 };
 
-function isGuestAllowedToCids(privilege, cids, callback) {
+function isSystemGroupAllowedToCids(privilege, uid, cids, callback) {
 	var groupKeys = cids.map(function (cid) {
 		return 'cid:' + cid + ':privileges:groups:' + privilege;
 	});
 
-	groups.isMemberOfGroups('guests', groupKeys, callback);
+	groups.isMemberOfGroups(uidToSystemGroup[uid], groupKeys, callback);
 }
 
-function isGuestAllowedToPrivileges(privileges, cid, callback) {
+function isSystemGroupAllowedToPrivileges(privileges, uid, cid, callback) {
 	var groupKeys = privileges.map(function (privilege) {
 		return 'cid:' + cid + ':privileges:groups:' + privilege;
 	});
 
-	groups.isMemberOfGroups('guests', groupKeys, callback);
+	groups.isMemberOfGroups(uidToSystemGroup[uid], groupKeys, callback);
 }
 
 helpers.getUserPrivileges = function (cid, hookName, userPrivilegeList, callback) {

src/routes/authentication.js

@@ -20,7 +20,15 @@ Auth.initialize = function (app, middleware) {
 	app.use(passport.session());
 
 	app.use(function (req, res, next) {
-		req.uid = req.user ? parseInt(req.user.uid, 10) : 0;
+		var isSpider = req.isSpider();
+		req.loggedIn = !isSpider && !!req.user;
+		if (isSpider) {
+			req.uid = -1;
+		} else if (req.user) {
+			req.uid = parseInt(req.user.uid, 10);
+		} else {
+			req.uid = 0;
+		}
 		next();
 	});
 

src/upgrades/1.8.0/give_spiders_privileges.js

@@ -0,0 +1,49 @@
+'use strict';
+
+
+var async = require('async');
+var groups = require('../../groups');
+var privileges = require('../../privileges');
+var db = require('../../database');
+
+module.exports = {
+	name: 'Give category access privileges to spiders system group',
+	timestamp: Date.UTC(2018, 0, 31),
+	method: function (callback) {
+		db.getSortedSetRange('categories:cid', 0, -1, function (err, cids) {
+			if (err) {
+				return callback(err);
+			}
+			async.eachSeries(cids, function (cid, next) {
+				getGroupPrivileges(cid, function (err, groupPrivileges) {
+					if (err) {
+						return next(err);
+					}
+
+					var privs = [];
+					if (groupPrivileges['groups:find']) {
+						privs.push('find');
+					}
+					if (groupPrivileges['groups:read']) {
+						privs.push('read');
+					}
+					if (groupPrivileges['groups:topics:read']) {
+						privs.push('topics:read');
+					}
+
+					privileges.categories.give(privs, cid, 'spiders', next);
+				});
+			}, callback);
+		});
+	},
+};
+
+function getGroupPrivileges(cid, callback) {
+	var tasks = {};
+
+	['groups:find', 'groups:read', 'groups:topics:read'].forEach(function (privilege) {
+		tasks[privilege] = async.apply(groups.isMember, 'guests', 'cid:' + cid + ':privileges:' + privilege);
+	});
+
+	async.parallel(tasks, callback);
+}

src/webserver.js

@@ -16,6 +16,7 @@ var cookieParser = require('cookie-parser');
 var session = require('express-session');
 var useragent = require('express-useragent');
 var favicon = require('serve-favicon');
+var detector = require('spider-detector');
 
 var db = require('./database');
 var file = require('./file');
@@ -159,6 +160,7 @@ function setupExpressApp(app, callback) {
 	app.use(bodyParser.json());
 	app.use(cookieParser());
 	app.use(useragent.express());
+	app.use(detector.middleware());
 
 	app.use(session({
 		store: db.sessionStore,