From 746222d6c641b70f3eb4daabb515dc70df7b55a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bar=C4=B1=C5=9F=20Soner=20U=C5=9Fakl=C4=B1?=
 <barisusakli@gmail.com>
Date: Sun, 5 Jul 2020 20:49:18 -0400
Subject: [PATCH] fix: handle scan/zscan returning duplicate elements on redis

increase count on redis zscan
---
 src/database/redis/main.js   | 10 +++++++++-
 src/database/redis/sorted.js | 24 ++++++++++++++----------
 2 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/src/database/redis/main.js b/src/database/redis/main.js
index 7313f1c10e..5c5bee9428 100644
--- a/src/database/redis/main.js
+++ b/src/database/redis/main.js
@@ -26,11 +26,19 @@ module.exports = function (module) {
 	module.scan = async function (params) {
 		let cursor = '0';
 		let returnData = [];
+		const seen = {};
 		do {
 			/* eslint-disable no-await-in-loop */
 			const res = await module.client.async.scan(cursor, 'MATCH', params.match, 'COUNT', 10000);
 			cursor = res[0];
-			returnData = returnData.concat(res[1]);
+			const values = res[1].filter((value) => {
+				const isSeen = !!seen[value];
+				if (!isSeen) {
+					seen[value] = 1;
+				}
+				return !isSeen;
+			});
+			returnData = returnData.concat(values);
 		} while (cursor !== '0');
 		return returnData;
 	};
diff --git a/src/database/redis/sorted.js b/src/database/redis/sorted.js
index 8aa5de5c38..59d13dff28 100644
--- a/src/database/redis/sorted.js
+++ b/src/database/redis/sorted.js
@@ -282,24 +282,28 @@ module.exports = function (module) {
 
 		const returnData = [];
 		let done = false;
+		const seen = {};
 		do {
 			/* eslint-disable no-await-in-loop */
-			const res = await module.client.async.zscan(params.key, cursor, 'MATCH', params.match, 'COUNT', 100);
+			const res = await module.client.async.zscan(params.key, cursor, 'MATCH', params.match, 'COUNT', 5000);
 			cursor = res[0];
 			done = cursor === '0';
 			const data = res[1];
 
 			for (let i = 0; i < data.length; i += 2) {
 				const value = data[i];
-				const score = parseFloat(data[i + 1]);
-				if (params.withScores) {
-					returnData.push({ value: value, score: score });
-				} else {
-					returnData.push(value);
-				}
-				if (params.limit && returnData.length >= params.limit) {
-					done = true;
-					break;
+				if (!seen[value]) {
+					seen[value] = 1;
+
+					if (params.withScores) {
+						returnData.push({ value: value, score: parseFloat(data[i + 2]) });
+					} else {
+						returnData.push(value);
+					}
+					if (params.limit && returnData.length >= params.limit) {
+						done = true;
+						break;
+					}
 				}
 			}
 		} while (!done);