"use strict"; var async = require('async'); var utils = require('../../../public/src/utils'); module.exports = function (db, module) { var helpers = module.helpers.mongo; module.sortedSetAdd = function (key, score, value, callback) { callback = callback || helpers.noop; if (!key) { return callback(); } if (Array.isArray(score) && Array.isArray(value)) { return sortedSetAddBulk(key, score, value, callback); } value = helpers.valueToString(value); db.collection('objects').update({_key: key, value: value}, {$set: {score: parseFloat(score)}}, {upsert:true, w: 1}, function (err) { if (err && err.message.startsWith('E11000 duplicate key error')) { return process.nextTick(module.sortedSetAdd, key, score, value, callback); } callback(err); }); }; function sortedSetAddBulk(key, scores, values, callback) { if (!scores.length || !values.length) { return callback(); } if (scores.length !== values.length) { return callback(new Error('[[error:invalid-data]]')); } values = values.map(helpers.valueToString); var bulk = db.collection('objects').initializeUnorderedBulkOp(); for(var i = 0; i < scores.length; ++i) { bulk.find({_key: key, value: values[i]}).upsert().updateOne({$set: {score: parseFloat(scores[i])}}); } bulk.execute(function (err) { callback(err); }); } module.sortedSetsAdd = function (keys, score, value, callback) { callback = callback || helpers.noop; if (!Array.isArray(keys) || !keys.length) { return callback(); } value = helpers.valueToString(value); var bulk = db.collection('objects').initializeUnorderedBulkOp(); for(var i = 0; i < keys.length; ++i) { bulk.find({_key: keys[i], value: value}).upsert().updateOne({$set: {score: parseFloat(score)}}); } bulk.execute(function (err) { callback(err); }); }; module.sortedSetRemove = function (key, value, callback) { function done(err) { callback(err); } callback = callback || helpers.noop; if (!key) { return callback(); } if (Array.isArray(value)) { value = value.map(helpers.valueToString); db.collection('objects').remove({_key: key, value: {$in: value}}, done); } else { value = helpers.valueToString(value); db.collection('objects').remove({_key: key, value: value}, done); } }; module.sortedSetsRemove = function (keys, value, callback) { callback = callback || helpers.noop; if (!Array.isArray(keys) || !keys.length) { return callback(); } value = helpers.valueToString(value); db.collection('objects').remove({_key: {$in: keys}, value: value}, function (err) { callback(err); }); }; module.sortedSetsRemoveRangeByScore = function (keys, min, max, callback) { callback = callback || helpers.noop; if (!Array.isArray(keys) || !keys.length) { return callback(); } var query = {_key: {$in: keys}}; if (min !== '-inf') { query.score = {$gte: min}; } if (max !== '+inf') { query.score = query.score || {}; query.score.$lte = max; } db.collection('objects').remove(query, function (err) { callback(err); }); }; module.getSortedSetRange = function (key, start, stop, callback) { getSortedSetRange(key, start, stop, 1, false, callback); }; module.getSortedSetRevRange = function (key, start, stop, callback) { getSortedSetRange(key, start, stop, -1, false, callback); }; module.getSortedSetRangeWithScores = function (key, start, stop, callback) { getSortedSetRange(key, start, stop, 1, true, callback); }; module.getSortedSetRevRangeWithScores = function (key, start, stop, callback) { getSortedSetRange(key, start, stop, -1, true, callback); }; function getSortedSetRange(key, start, stop, sort, withScores, callback) { if (!key) { return callback(); } var fields = {_id: 0, value: 1}; if (withScores) { fields.score = 1; } if (Array.isArray(key)) { key = {$in: key}; } var limit = stop - start + 1; if (limit <= 0) { limit = 0; } db.collection('objects').find({_key: key}, {fields: fields}) .limit(limit) .skip(start) .sort({score: sort}) .toArray(function (err, data) { if (err || !data) { return callback(err); } if (!withScores) { data = data.map(function (item) { return item.value; }); } callback(null, data); }); } module.getSortedSetRangeByScore = function (key, start, count, min, max, callback) { getSortedSetRangeByScore(key, start, count, min, max, 1, false, callback); }; module.getSortedSetRevRangeByScore = function (key, start, count, max, min, callback) { getSortedSetRangeByScore(key, start, count, min, max, -1, false, callback); }; module.getSortedSetRangeByScoreWithScores = function (key, start, count, min, max, callback) { getSortedSetRangeByScore(key, start, count, min, max, 1, true, callback); }; module.getSortedSetRevRangeByScoreWithScores = function (key, start, count, max, min, callback) { getSortedSetRangeByScore(key, start, count, min, max, -1, true, callback); }; function getSortedSetRangeByScore(key, start, count, min, max, sort, withScores, callback) { if (!key) { return callback(); } if(parseInt(count, 10) === -1) { count = 0; } var query = {_key: key}; if (min !== '-inf') { query.score = {$gte: min}; } if (max !== '+inf') { query.score = query.score || {}; query.score.$lte = max; } var fields = {_id: 0, value: 1}; if (withScores) { fields.score = 1; } db.collection('objects').find(query, {fields: fields}) .limit(count) .skip(start) .sort({score: sort}) .toArray(function (err, data) { if(err) { return callback(err); } if (!withScores) { data = data.map(function (item) { return item.value; }); } callback(err, data); }); } module.sortedSetCount = function (key, min, max, callback) { if (!key) { return callback(); } var query = {_key: key}; if (min !== '-inf') { query.score = {$gte: min}; } if (max !== '+inf') { query.score = query.score || {}; query.score.$lte = max; } db.collection('objects').count(query, function (err, count) { callback(err, count ? count : 0); }); }; module.sortedSetCard = function (key, callback) { if (!key) { return callback(null, 0); } db.collection('objects').count({_key: key}, function (err, count) { count = parseInt(count, 10); callback(err, count ? count : 0); }); }; module.sortedSetsCard = function (keys, callback) { if (!Array.isArray(keys) || !keys.length) { return callback(); } var pipeline = [ { $match : { _key : { $in: keys } } } , { $group: { _id: {_key: '$_key'}, count: { $sum: 1 } } }, { $project: { _id: 1, count: '$count' } } ]; db.collection('objects').aggregate(pipeline, function (err, results) { if (err) { return callback(err); } if (!Array.isArray(results)) { results = []; } var map = {}; results.forEach(function (item) { if (item && item._id._key) { map[item._id._key] = item.count; } }); results = keys.map(function (key) { return map[key] || 0; }); callback(null, results); }); }; module.sortedSetRank = function (key, value, callback) { getSortedSetRank(module.getSortedSetRange, key, value, callback); }; module.sortedSetRevRank = function (key, value, callback) { getSortedSetRank(module.getSortedSetRevRange, key, value, callback); }; function getSortedSetRank(method, key, value, callback) { if (!key) { return callback(); } value = helpers.valueToString(value); method(key, 0, -1, function (err, result) { if(err) { return callback(err); } var rank = result.indexOf(value); callback(null, rank !== -1 ? rank : null); }); } module.sortedSetsRanks = function (keys, values, callback) { if (!Array.isArray(keys) || !keys.length) { return callback(null, []); } var data = new Array(values.length); for (var i = 0; i < values.length; ++i) { data[i] = {key: keys[i], value: values[i]}; } async.map(data, function (item, next) { getSortedSetRank(module.getSortedSetRange, item.key, item.value, next); }, callback); }; module.sortedSetRanks = function (key, values, callback) { module.getSortedSetRange(key, 0, -1, function (err, sortedSet) { if (err) { return callback(err); } var result = values.map(function (value) { if (!value) { return null; } var index = sortedSet.indexOf(value.toString()); return index !== -1 ? index : null; }); callback(null, result); }); }; module.sortedSetScore = function (key, value, callback) { if (!key) { return callback(); } value = helpers.valueToString(value); db.collection('objects').findOne({_key: key, value: value}, {fields:{_id: 0, score: 1}}, function (err, result) { callback(err, result ? result.score : null); }); }; module.sortedSetsScore = function (keys, value, callback) { if (!Array.isArray(keys) || !keys.length) { return callback(); } value = helpers.valueToString(value); db.collection('objects').find({_key:{$in:keys}, value: value}, {_id:0, _key:1, score: 1}).toArray(function (err, result) { if (err) { return callback(err); } var map = helpers.toMap(result), returnData = [], item; for(var i = 0; i < keys.length; ++i) { item = map[keys[i]]; returnData.push(item ? item.score : null); } callback(null, returnData); }); }; module.sortedSetScores = function (key, values, callback) { if (!key) { return callback(); } values = values.map(helpers.valueToString); db.collection('objects').find({_key: key, value: {$in: values}}, {_id: 0, value: 1, score: 1}).toArray(function (err, result) { if (err) { return callback(err); } var map = {}; result.forEach(function (item) { map[item.value] = item.score; }); var returnData = new Array(values.length); var score; for(var i = 0; i < values.length; ++i) { score = map[values[i]]; returnData[i] = utils.isNumber(score) ? score : null; } callback(null, returnData); }); }; module.isSortedSetMember = function (key, value, callback) { if (!key) { return callback(); } value = helpers.valueToString(value); db.collection('objects').findOne({_key: key, value: value}, {_id: 0, value: 1}, function (err, result) { callback(err, !!result); }); }; module.isSortedSetMembers = function (key, values, callback) { if (!key) { return callback(); } values = values.map(helpers.valueToString); db.collection('objects').find({_key: key, value: {$in: values}}, {fields: {_id: 0, value: 1}}).toArray(function (err, results) { if (err) { return callback(err); } results = results.map(function (item) { return item.value; }); values = values.map(function (value) { return results.indexOf(value) !== -1; }); callback(null, values); }); }; module.isMemberOfSortedSets = function (keys, value, callback) { if (!Array.isArray(keys)) { return callback(); } value = helpers.valueToString(value); db.collection('objects').find({_key: {$in: keys}, value: value}, {fields: {_id: 0, _key: 1, value: 1}}).toArray(function (err, results) { if (err) { return callback(err); } results = results.map(function (item) { return item._key; }); results = keys.map(function (key) { return results.indexOf(key) !== -1; }); callback(null, results); }); }; module.getSortedSetsMembers = function (keys, callback) { if (!Array.isArray(keys) || !keys.length) { return callback(null, []); } db.collection('objects').find({_key: {$in: keys}}, {_id: 0, _key: 1, value: 1}).toArray(function (err, data) { if (err) { return callback(err); } var sets = {}; data.forEach(function (set) { sets[set._key] = sets[set._key] || []; sets[set._key].push(set.value); }); var returnData = new Array(keys.length); for(var i = 0; i < keys.length; ++i) { returnData[i] = sets[keys[i]] || []; } callback(null, returnData); }); }; module.sortedSetUnionCard = function (keys, callback) { if (!Array.isArray(keys) || !keys.length) { return callback(null, 0); } var pipeline = [ { $match: { _key: {$in: keys} } }, { $group: { _id: {value: '$value' } } }, { $group: { _id: null, count: { $sum: 1 } } } ]; var project = { _id: 0, count: '$count' }; pipeline.push({ $project: project }); db.collection('objects').aggregate(pipeline, function (err, data) { callback(err, Array.isArray(data) && data.length ? data[0].count : 0); }); }; module.getSortedSetUnion = function (params, callback) { params.sort = 1; getSortedSetUnion(params, callback); }; module.getSortedSetRevUnion = function (params, callback) { params.sort = -1; getSortedSetUnion(params, callback); }; function getSortedSetUnion(params, callback) { if (!Array.isArray(params.sets) || !params.sets.length) { return callback(); } var limit = params.stop - params.start + 1; if (limit <= 0) { limit = 0; } var aggregate = {}; if (params.aggregate) { aggregate['$' + params.aggregate.toLowerCase()] = '$score'; } else { aggregate.$sum = '$score'; } var pipeline = [ { $match: { _key: {$in: params.sets}} }, { $group: { _id: {value: '$value'}, totalScore: aggregate} }, { $sort: { totalScore: params.sort} } ]; if (params.start) { pipeline.push({ $skip: params.start }); } if (limit > 0) { pipeline.push({ $limit: limit }); } var project = { _id: 0, value: '$_id.value' }; if (params.withScores) { project.score = '$totalScore'; } pipeline.push({ $project: project }); db.collection('objects').aggregate(pipeline, function (err, data) { if (err || !data) { return callback(err); } if (!params.withScores) { data = data.map(function (item) { return item.value; }); } callback(null, data); }); } module.sortedSetIncrBy = function (key, increment, value, callback) { callback = callback || helpers.noop; if (!key) { return callback(); } var data = {}; value = helpers.valueToString(value); data.score = parseFloat(increment); db.collection('objects').findAndModify({_key: key, value: value}, {}, {$inc: data}, {new: true, upsert: true}, function (err, result) { // if there is duplicate key error retry the upsert // https://github.com/NodeBB/NodeBB/issues/4467 // https://jira.mongodb.org/browse/SERVER-14322 // https://docs.mongodb.org/manual/reference/command/findAndModify/#upsert-and-unique-index if (err && err.message.startsWith('E11000 duplicate key error')) { return process.nextTick(module.sortedSetIncrBy, key, increment, value, callback); } callback(err, result && result.value ? result.value.score : null); }); }; module.getSortedSetRangeByLex = function (key, min, max, start, count, callback) { sortedSetLex(key, min, max, 1, start, count, callback); }; module.getSortedSetRevRangeByLex = function (key, max, min, start, count, callback) { sortedSetLex(key, min, max, -1, start, count, callback); }; module.sortedSetLexCount = function (key, min, max, callback) { sortedSetLex(key, min, max, 1, 0, 0, function (err, data) { callback(err, data ? data.length : null); }); }; function sortedSetLex(key, min, max, sort, start, count, callback) { if (!callback) { callback = start; start = 0; count = 0; } var query = {_key: key}; if (min !== '-') { if (min.match(/^\(/)) { query.value = {$gt: min.slice(1)}; } else if (min.match(/^\[/)) { query.value = {$gte: min.slice(1)}; } else { query.value = {$gte: min}; } } if (max !== '+') { query.value = query.value || {}; if (max.match(/^\(/)) { query.value.$lt = max.slice(1); } else if (max.match(/^\[/)) { query.value.$lte = max.slice(1); } else { query.value.$lte = max; } } db.collection('objects').find(query, {_id: 0, value: 1}) .sort({value: sort}) .skip(start) .limit(count === -1 ? 0 : count) .toArray(function (err, data) { if (err) { return callback(err); } data = data.map(function (item) { return item && item.value; }); callback(err, data); }); } module.sortedSetRemoveRangeByLex = function (key, min, max, callback) { callback = callback || helpers.noop; var query = {_key: key}; if (min !== '-') { if (min.match(/^\(/)) { query.value = {$gt: min.slice(1)}; } else if (min.match(/^\[/)) { query.value = {$gte: min.slice(1)}; } else { query.value = {$gte: min}; } } if (max !== '+') { query.value = query.value || {}; if (max.match(/^\(/)) { query.value.$lt = max.slice(1); } else if (max.match(/^\[/)) { query.value.$lte = max.slice(1); } else { query.value.$lte = max; } } db.collection('objects').remove(query, function (err) { callback(err); }); }; module.processSortedSet = function (setKey, process, batch, callback) { var done = false; var ids = []; var cursor = db.collection('objects').find({_key: setKey}) .sort({score: 1}) .project({_id: 0, value: 1}) .batchSize(batch); async.whilst( function () { return !done; }, function (next) { cursor.next(function (err, item) { if (err) { return next(err); } if (item === null) { done = true; } else { ids.push(item.value); } if (ids.length < batch && (!done || ids.length === 0)) { return next(null); } process(ids, function (err) { ids = []; return next(err); }); }); }, callback ); }; module.sortedSetIntersectCard = function (keys, callback) { if (!Array.isArray(keys) || !keys.length) { return callback(null, 0); } var pipeline = [ { $match: { _key: {$in: keys}} }, { $group: { _id: {value: '$value'}, count: {$sum: 1}} }, { $match: { count: keys.length} }, { $group: { _id: null, count: { $sum: 1 } } } ]; db.collection('objects').aggregate(pipeline, function (err, data) { callback(err, Array.isArray(data) && data.length ? data[0].count : 0); }); }; module.getSortedSetIntersect = function (params, callback) { params.sort = 1; getSortedSetRevIntersect(params, callback); }; module.getSortedSetRevIntersect = function (params, callback) { params.sort = -1; getSortedSetRevIntersect(params, callback); }; function getSortedSetRevIntersect(params, callback) { var sets = params.sets; var start = params.hasOwnProperty('start') ? params.start : 0; var stop = params.hasOwnProperty('stop') ? params.stop : -1; var weights = params.weights || []; var aggregate = {}; if (params.aggregate) { aggregate['$' + params.aggregate.toLowerCase()] = '$score'; } else { aggregate.$sum = '$score'; } var limit = stop - start + 1; if (limit <= 0) { limit = 0; } var pipeline = [{ $match: { _key: {$in: sets}} }]; weights.forEach(function (weight, index) { if (weight !== 1) { pipeline.push({ $project: { value: 1, score: { $cond: { if: { $eq: [ "$_key", sets[index] ] }, then: { $multiply: [ '$score', weight ] }, else: '$score' } } } }); } }); pipeline.push({ $group: { _id: {value: '$value'}, totalScore: aggregate, count: {$sum: 1}} }); pipeline.push({ $match: { count: sets.length} }); pipeline.push({ $sort: { totalScore: params.sort} }); if (start) { pipeline.push({ $skip: start }); } if (limit > 0) { pipeline.push({ $limit: limit }); } var project = { _id: 0, value: '$_id.value'}; if (params.withScores) { project.score = '$totalScore'; } pipeline.push({ $project: project }); db.collection('objects').aggregate(pipeline, function (err, data) { if (err || !data) { return callback(err); } if (!params.withScores) { data = data.map(function (item) { return item.value; }); } callback(null, data); }); } };