diff --git a/src/api/util/handlers/Message.ts b/src/api/util/handlers/Message.ts index 7c9b6a0c6..cfa682374 100644 --- a/src/api/util/handlers/Message.ts +++ b/src/api/util/handlers/Message.ts @@ -19,6 +19,8 @@ import { fillMessageUrlEmbeds, randomString } from "@spacebar/api"; import { Application, + arrayDistributeSequentially, + arrayPartition, Attachment, Channel, CloudAttachment, @@ -33,6 +35,7 @@ import { Guild, handleFile, HERE_MENTION, + mathLogBase, Member, Message, MessageCreateEvent, @@ -42,6 +45,7 @@ import { Role, ROLE_MENTION, Session, + Snowflake, Sticker, Stopwatch, TraceNode, @@ -790,17 +794,29 @@ async function handleMessageMentionsAsync(message: Message) { return; } - const newReadStates = users - .values() - .map((user_id) => ({ user_id, channel_id: channel.id, read_state_type: ReadStateType.CHANNEL })) - .toArray(); - subTrace.calls.push("constructNewReadStates", { micros: subSw.getElapsedAndReset().totalMicroseconds }); + const newReadStateSeqs = arrayDistributeSequentially(users.values().toArray(), Math.max(1, mathLogBase(users.size, 2))).map((seq) => + seq.map((user_id) => ({ id: Snowflake.generate(), user_id, channel_id: channel.id, read_state_type: ReadStateType.CHANNEL })), + ); + subTrace.calls.push(`constructNewReadStatesChunked(${newReadStateSeqs.length})`, { micros: subSw.getElapsedAndReset().totalMicroseconds }); - await ReadState.insert(newReadStates).catch((e) => { - console.log("Failed to bulk insert", users.size, "new ReadStates, trying again (race condition?)...\nDetails:", e); - return fillInMissingIDs(users.values().toArray(), subTrace); - }); - subTrace.calls.push("insertNewReadStates", { micros: subSw.getElapsedAndReset().totalMicroseconds }); + await Promise.all( + newReadStateSeqs.map((seq) => + // just a safety thing... handle postgres hard limit at 65535 parameters, at 4 params per object... 16384 + seq.length > 15000 + ? fillInMissingIDs( + seq.map((rs) => rs.user_id), + subTrace, + ) + : ReadState.insert(seq).catch((e) => { + console.log("Failed to bulk insert", seq.length, "new ReadStates, trying again (race condition/too many params?)...\nDetails:", e); + return fillInMissingIDs( + seq.map((rs) => rs.user_id), + subTrace, + ); + }), + ), + ); + subTrace.calls.push("insertNewReadStatesChunked", { micros: subSw.getElapsedAndReset().totalMicroseconds }); } finally { trace?.calls.push(`fillInMissingIDs(${ids.length})`, { micros: fillMessageSw.getElapsedAndReset().totalMicroseconds, calls: subTrace.calls }); } diff --git a/src/util/util/extensions/Array.ts b/src/util/util/extensions/Array.ts index 61cf469b2..e3e6478c2 100644 --- a/src/util/util/extensions/Array.ts +++ b/src/util/util/extensions/Array.ts @@ -55,3 +55,22 @@ export function arrayGroupBy(array: T[], selector: (elem: T) => M): Map(array: T[], count: number): T[][] { + if (count <= 0) throw new Error("Count must be greater than 0"); + if (count == 1) return [array]; + + const list = array; + const groups: T[][] = []; + for (let i = 0; i < count; i++) groups.push([]); + + // [1,2,3],[4,5,6],[7,8,9]... + for (let i = 0; i < list.length; i++) { + const idx: number = Math.floor((i / list.length) * count); + // console.log(`${i}/${list.length} -> ${idx}:${groups[idx].length}/{count}`); + groups[idx].push(list[i]); + } + + return groups; +} diff --git a/src/util/util/extensions/Math.ts b/src/util/util/extensions/Math.ts new file mode 100644 index 000000000..c70af7f58 --- /dev/null +++ b/src/util/util/extensions/Math.ts @@ -0,0 +1,22 @@ +/* + Spacebar: A FOSS re-implementation and extension of the Discord.com backend. + Copyright (C) 2025 Spacebar and Spacebar Contributors + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/log +export function mathLogBase(num: number, base: number) { + return Math.log(num) / Math.log(base); +} diff --git a/src/util/util/extensions/index.ts b/src/util/util/extensions/index.ts index c9eb0d63d..53bd443c8 100644 --- a/src/util/util/extensions/index.ts +++ b/src/util/util/extensions/index.ts @@ -17,6 +17,7 @@ */ export * from "./Array"; +export * from "./Math"; // TODO: move to a separate file export async function sleep(ms: number) {