fix: some more upgrade script fixes

reduce parallel calls for psql
isekai-main
Barış Soner Uşaklı 1 year ago
parent c17ef3659c
commit f23b0b5bec

@ -1,3 +1,4 @@
/* eslint-disable no-await-in-loop */
'use strict';
@ -13,7 +14,7 @@ module.exports = {
progress.total = await db.sortedSetCard(`chat:rooms`);
await batch.processSortedSet(`chat:rooms`, async (roomIds) => {
progress.incr(roomIds.length);
await Promise.all(roomIds.map(async (roomId) => {
for (const roomId of roomIds) {
await batch.processSortedSet(`chat:room:${roomId}:mids`, async (mids) => {
let messageData = await db.getObjects(mids.map(mid => `message:${mid}`));
messageData.forEach((m, idx) => {
@ -36,7 +37,7 @@ module.exports = {
}, {
batch: 500,
});
}));
}
}, {
batch: 500,
});

@ -26,7 +26,7 @@ module.exports = {
});
await db.sortedSetAddBulk(bulkAdd);
}, {
batch: 500,
batch: 100,
});
},
};

@ -1,10 +1,10 @@
'use strict';
/* eslint-disable no-await-in-loop */
'use strict';
const db = require('../../database');
const batch = require('../../batch');
module.exports = {
name: 'Update chat messages to add roomId field',
timestamp: Date.UTC(2023, 6, 2),
@ -31,55 +31,51 @@ module.exports = {
await batch.processArray(allRoomIds, async (roomIds) => {
const arrayOfRoomData = await db.getObjects(roomIds.map(roomId => `chat:room:${roomId}`));
await Promise.all(roomIds.map(async (roomId, index) => {
const roomData = arrayOfRoomData[index];
if (!roomData) {
return;
}
const midsSeen = Object.create(null);
await batch.processSortedSet(`chat:room:${roomId}:uids`, async (uids) => {
for (const uid of uids) {
// eslint-disable-next-line no-await-in-loop
await batch.processSortedSet(`uid:${uid}:chat:room:${roomId}:mids`, async (mids) => {
const uniqMids = mids.filter(mid => !midsSeen.hasOwnProperty(mid));
if (!uniqMids.length) {
return;
}
let messageData = await db.getObjects(uniqMids.map(mid => `message:${mid}`));
messageData.forEach((m, idx) => {
if (m) {
m.mid = parseInt(uniqMids[idx], 10);
for (const roomData of arrayOfRoomData) {
if (roomData) {
const midsSeen = Object.create(null);
const { roomId } = roomData;
await batch.processSortedSet(`chat:room:${roomId}:uids`, async (uids) => {
for (const uid of uids) {
await batch.processSortedSet(`uid:${uid}:chat:room:${roomId}:mids`, async (mids) => {
const uniqMids = mids.filter(mid => !midsSeen.hasOwnProperty(mid));
if (!uniqMids.length) {
return;
}
});
messageData = messageData.filter(Boolean);
const bulkSet = messageData.map(
msg => [`message:${msg.mid}`, { roomId: roomId }]
);
let messageData = await db.getObjects(uniqMids.map(mid => `message:${mid}`));
messageData.forEach((m, idx) => {
if (m) {
m.mid = parseInt(uniqMids[idx], 10);
}
});
messageData = messageData.filter(Boolean);
const bulkSet = messageData.map(
msg => [`message:${msg.mid}`, { roomId: roomId }]
);
await db.setObjectBulk(bulkSet);
await db.sortedSetAdd(
`chat:room:${roomId}:mids`,
messageData.map(m => m.timestamp),
messageData.map(m => m.mid),
);
uniqMids.forEach((mid) => {
midsSeen[mid] = 1;
await db.setObjectBulk(bulkSet);
await db.sortedSetAdd(
`chat:room:${roomId}:mids`,
messageData.map(m => m.timestamp),
messageData.map(m => m.mid),
);
uniqMids.forEach((mid) => {
midsSeen[mid] = 1;
});
}, {
batch: 500,
});
}, {
batch: 500,
});
// eslint-disable-next-line no-await-in-loop
await db.deleteAll(`uid:${uid}:chat:room:${roomId}:mids`);
}
progress.incr(uids.length);
}, {
batch: 500,
});
}));
// eslint-disable-next-line no-await-in-loop
await db.deleteAll(`uid:${uid}:chat:room:${roomId}:mids`);
}
progress.incr(uids.length);
}, {
batch: 500,
});
}
}
}, {
batch: 500,
});

Loading…
Cancel
Save