fix: some more upgrade script fixes

reduce parallel calls for psql
isekai-main
Barış Soner Uşaklı 1 year ago
parent c17ef3659c
commit f23b0b5bec

@ -1,3 +1,4 @@
/* eslint-disable no-await-in-loop */
'use strict'; 'use strict';
@ -13,7 +14,7 @@ module.exports = {
progress.total = await db.sortedSetCard(`chat:rooms`); progress.total = await db.sortedSetCard(`chat:rooms`);
await batch.processSortedSet(`chat:rooms`, async (roomIds) => { await batch.processSortedSet(`chat:rooms`, async (roomIds) => {
progress.incr(roomIds.length); progress.incr(roomIds.length);
await Promise.all(roomIds.map(async (roomId) => { for (const roomId of roomIds) {
await batch.processSortedSet(`chat:room:${roomId}:mids`, async (mids) => { await batch.processSortedSet(`chat:room:${roomId}:mids`, async (mids) => {
let messageData = await db.getObjects(mids.map(mid => `message:${mid}`)); let messageData = await db.getObjects(mids.map(mid => `message:${mid}`));
messageData.forEach((m, idx) => { messageData.forEach((m, idx) => {
@ -36,7 +37,7 @@ module.exports = {
}, { }, {
batch: 500, batch: 500,
}); });
})); }
}, { }, {
batch: 500, batch: 500,
}); });

@ -26,7 +26,7 @@ module.exports = {
}); });
await db.sortedSetAddBulk(bulkAdd); await db.sortedSetAddBulk(bulkAdd);
}, { }, {
batch: 500, batch: 100,
}); });
}, },
}; };

@ -1,10 +1,10 @@
'use strict'; /* eslint-disable no-await-in-loop */
'use strict';
const db = require('../../database'); const db = require('../../database');
const batch = require('../../batch'); const batch = require('../../batch');
module.exports = { module.exports = {
name: 'Update chat messages to add roomId field', name: 'Update chat messages to add roomId field',
timestamp: Date.UTC(2023, 6, 2), timestamp: Date.UTC(2023, 6, 2),
@ -31,55 +31,51 @@ module.exports = {
await batch.processArray(allRoomIds, async (roomIds) => { await batch.processArray(allRoomIds, async (roomIds) => {
const arrayOfRoomData = await db.getObjects(roomIds.map(roomId => `chat:room:${roomId}`)); const arrayOfRoomData = await db.getObjects(roomIds.map(roomId => `chat:room:${roomId}`));
for (const roomData of arrayOfRoomData) {
await Promise.all(roomIds.map(async (roomId, index) => { if (roomData) {
const roomData = arrayOfRoomData[index]; const midsSeen = Object.create(null);
if (!roomData) { const { roomId } = roomData;
return; await batch.processSortedSet(`chat:room:${roomId}:uids`, async (uids) => {
} for (const uid of uids) {
await batch.processSortedSet(`uid:${uid}:chat:room:${roomId}:mids`, async (mids) => {
const midsSeen = Object.create(null); const uniqMids = mids.filter(mid => !midsSeen.hasOwnProperty(mid));
await batch.processSortedSet(`chat:room:${roomId}:uids`, async (uids) => { if (!uniqMids.length) {
for (const uid of uids) { return;
// eslint-disable-next-line no-await-in-loop
await batch.processSortedSet(`uid:${uid}:chat:room:${roomId}:mids`, async (mids) => {
const uniqMids = mids.filter(mid => !midsSeen.hasOwnProperty(mid));
if (!uniqMids.length) {
return;
}
let messageData = await db.getObjects(uniqMids.map(mid => `message:${mid}`));
messageData.forEach((m, idx) => {
if (m) {
m.mid = parseInt(uniqMids[idx], 10);
} }
});
messageData = messageData.filter(Boolean);
const bulkSet = messageData.map( let messageData = await db.getObjects(uniqMids.map(mid => `message:${mid}`));
msg => [`message:${msg.mid}`, { roomId: roomId }] messageData.forEach((m, idx) => {
); if (m) {
m.mid = parseInt(uniqMids[idx], 10);
}
});
messageData = messageData.filter(Boolean);
const bulkSet = messageData.map(
msg => [`message:${msg.mid}`, { roomId: roomId }]
);
await db.setObjectBulk(bulkSet); await db.setObjectBulk(bulkSet);
await db.sortedSetAdd( await db.sortedSetAdd(
`chat:room:${roomId}:mids`, `chat:room:${roomId}:mids`,
messageData.map(m => m.timestamp), messageData.map(m => m.timestamp),
messageData.map(m => m.mid), messageData.map(m => m.mid),
); );
uniqMids.forEach((mid) => { uniqMids.forEach((mid) => {
midsSeen[mid] = 1; midsSeen[mid] = 1;
});
}, {
batch: 500,
}); });
}, { // eslint-disable-next-line no-await-in-loop
batch: 500, await db.deleteAll(`uid:${uid}:chat:room:${roomId}:mids`);
}); }
// eslint-disable-next-line no-await-in-loop progress.incr(uids.length);
await db.deleteAll(`uid:${uid}:chat:room:${roomId}:mids`); }, {
} batch: 500,
progress.incr(uids.length); });
}, { }
batch: 500, }
});
}));
}, { }, {
batch: 500, batch: 500,
}); });

Loading…
Cancel
Save