Break up bulk inserts for read states to avoid postgres parameter exhaustion

src/api/util/handlers/Message.ts

@@ -19,6 +19,8 @@
 import { fillMessageUrlEmbeds, randomString } from "@spacebar/api";
 import {
     Application,
+    arrayDistributeSequentially,
+    arrayPartition,
     Attachment,
     Channel,
     CloudAttachment,
@@ -33,6 +35,7 @@ import {
     Guild,
     handleFile,
     HERE_MENTION,
+    mathLogBase,
     Member,
     Message,
     MessageCreateEvent,
@@ -42,6 +45,7 @@ import {
     Role,
     ROLE_MENTION,
     Session,
+    Snowflake,
     Sticker,
     Stopwatch,
     TraceNode,
@@ -790,17 +794,29 @@ async function handleMessageMentionsAsync(message: Message) {
                 return;
             }
 
-            const newReadStates = users
-                .values()
-                .map((user_id) => ({ user_id, channel_id: channel.id, read_state_type: ReadStateType.CHANNEL }))
-                .toArray();
-            subTrace.calls.push("constructNewReadStates", { micros: subSw.getElapsedAndReset().totalMicroseconds });
+            const newReadStateSeqs = arrayDistributeSequentially(users.values().toArray(), Math.max(1, mathLogBase(users.size, 2))).map((seq) =>
+                seq.map((user_id) => ({ id: Snowflake.generate(), user_id, channel_id: channel.id, read_state_type: ReadStateType.CHANNEL })),
+            );
+            subTrace.calls.push(`constructNewReadStatesChunked(${newReadStateSeqs.length})`, { micros: subSw.getElapsedAndReset().totalMicroseconds });
 
-            await ReadState.insert(newReadStates).catch((e) => {
-                console.log("Failed to bulk insert", users.size, "new ReadStates, trying again (race condition?)...\nDetails:", e);
-                return fillInMissingIDs(users.values().toArray(), subTrace);
-            });
-            subTrace.calls.push("insertNewReadStates", { micros: subSw.getElapsedAndReset().totalMicroseconds });
+            await Promise.all(
+                newReadStateSeqs.map((seq) =>
+                    // just a safety thing... handle postgres hard limit at 65535 parameters, at 4 params per object... 16384
+                    seq.length > 15000
+                        ? fillInMissingIDs(
+                              seq.map((rs) => rs.user_id),
+                              subTrace,
+                          )
+                        : ReadState.insert(seq).catch((e) => {
+                              console.log("Failed to bulk insert", seq.length, "new ReadStates, trying again (race condition/too many params?)...\nDetails:", e);
+                              return fillInMissingIDs(
+                                  seq.map((rs) => rs.user_id),
+                                  subTrace,
+                              );
+                          }),
+                ),
+            );
+            subTrace.calls.push("insertNewReadStatesChunked", { micros: subSw.getElapsedAndReset().totalMicroseconds });
         } finally {
             trace?.calls.push(`fillInMissingIDs(${ids.length})`, { micros: fillMessageSw.getElapsedAndReset().totalMicroseconds, calls: subTrace.calls });
         }

src/util/util/extensions/Array.ts

@@ -55,3 +55,22 @@ export function arrayGroupBy<T, M>(array: T[], selector: (elem: T) => M): Map<M,
 
     return map;
 }
+
+// https://github.com/TheArcaneBrony/ArcaneLibs/blob/ca7ce2bb57a5dffd891b0b86ec5f358df02735c0/ArcaneLibs/Extensions/CollectionExtensions.cs#L103
+export function arrayDistributeSequentially<T>(array: T[], count: number): T[][] {
+    if (count <= 0) throw new Error("Count must be greater than 0");
+    if (count == 1) return [array];
+
+    const list = array;
+    const groups: T[][] = [];
+    for (let i = 0; i < count; i++) groups.push([]);
+
+    // [1,2,3],[4,5,6],[7,8,9]...
+    for (let i = 0; i < list.length; i++) {
+        const idx: number = Math.floor((i / list.length) * count);
+        // console.log(`${i}/${list.length} -> ${idx}:${groups[idx].length}/{count}`);
+        groups[idx].push(list[i]);
+    }
+
+    return groups;
+}

src/util/util/extensions/Math.ts

@@ -0,0 +1,22 @@
+/*
+	Spacebar: A FOSS re-implementation and extension of the Discord.com backend.
+	Copyright (C) 2025 Spacebar and Spacebar Contributors
+
+	This program is free software: you can redistribute it and/or modify
+	it under the terms of the GNU Affero General Public License as published
+	by the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU Affero General Public License for more details.
+
+	You should have received a copy of the GNU Affero General Public License
+	along with this program.  If not, see <https://www.gnu.org/licenses/>.
+*/
+
+// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/log
+export function mathLogBase(num: number, base: number) {
+    return Math.log(num) / Math.log(base);
+}

src/util/util/extensions/index.ts

@@ -17,6 +17,7 @@
 */
 
 export * from "./Array";
+export * from "./Math";
 
 // TODO: move to a separate file
 export async function sleep(ms: number) {