support bot, bots: paginate chat scan (#6935)

* bots: document APIGetChats command and CRApiChats response

* bots: regenerate API docs and TypeScript types

* simplex-chat-nodejs: add apiGetChats

* support bot: avoid OOM on large databases

apiListGroups / apiListContacts return every record in one response and
overflow V8's string allocation on large DBs. Replace list-then-find-by-id
patterns with apiGetChat(type, id, 0) lookups, and the one genuine scan
(refreshAllCards) with paginated apiGetChats, count=1000.

* support bot: update test assertions to match current message text

* bots: simplify PaginationByTime, expose only PTLast

* simplex-chat-nodejs: bump types and nodejs versions
This commit is contained in:
sh
2026-05-06 07:54:36 +00:00
committed by GitHub
parent db783d85d7
commit fefdea8ed0
22 changed files with 673 additions and 62 deletions
+39 -14
View File
@@ -1,9 +1,10 @@
import {describe, test, expect, beforeEach, vi} from "vitest"
import {core} from "simplex-chat"
import {SupportBot} from "./src/bot.js"
import {CardManager} from "./src/cards.js"
import {parseConfig} from "./src/config.js"
import {GrokApiClient} from "./src/grok.js"
import {welcomeMessage, queueMessage, grokActivatedMessage, teamLockedMessage} from "./src/messages.js"
import {welcomeMessage, queueMessage, grokActivatedMessage, teamLockedMessage, teamAlreadyInvitedMessage} from "./src/messages.js"
// Silence console output during tests
vi.spyOn(console, "log").mockImplementation(() => {})
@@ -83,15 +84,39 @@ class MockChatApi {
async apiListMembers(groupId: number) {
return this.members.get(groupId) || []
}
async apiGetChat(_chatType: string, chatId: number, _count: number) {
async apiGetChat(chatType: string, chatId: number, _count: number) {
if (chatType === ChatType.Direct) {
// Tests don't exercise direct lookups; throw the same shape production
// would so getContact() resolves to null instead of synthesizing a contact.
throw new core.ChatAPIError("contact not found", {
type: "errorStore",
storeError: {type: "contactNotFound", contactId: chatId},
} as any)
}
const baseGroupInfo = this.groups.get(chatId)
if (!baseGroupInfo) {
// Mirror production behavior: the real apiGetChat throws "groupNotFound"
// for an unknown id; getGroupInfo() catches and returns null.
throw new core.ChatAPIError("group not found", {
type: "errorStore",
storeError: {type: "groupNotFound", groupId: chatId},
} as any)
}
const items = this.chatItems.get(chatId) || []
const groupInfo = this.groups.get(chatId)
const groupInfo = {...baseGroupInfo, customData: this.customData.get(chatId)}
return {
chatInfo: {type: "group", groupInfo: groupInfo || makeGroupInfo(chatId)},
chatInfo: {type: "group", groupInfo},
chatItems: items,
chatStats: {unreadCount: 0, unreadMentions: 0, reportsCount: 0, minUnreadItemId: 0, unreadChat: false},
}
}
async apiGetChats(_userId: number, _pagination: any, _query?: any, _pcc?: boolean) {
return [...this.groups.values()].map(g => ({
chatInfo: {type: "group", groupInfo: {...g, customData: this.customData.get(g.groupId)}},
chatItems: [],
chatStats: {unreadCount: 0, unreadMentions: 0, reportsCount: 0, minUnreadItemId: 0, unreadChat: false},
}))
}
async apiListGroups(_userId: number) {
return [...this.groups.values()].map(g => ({...g, customData: this.customData.get(g.groupId)}))
}
@@ -638,7 +663,7 @@ describe("/grok Activation", () => {
await joinPromise
await bot.flush()
expectMemberAdded(CUSTOMER_GROUP_ID, GROK_CONTACT_ID)
expectSentToGroup(CUSTOMER_GROUP_ID, "You are chatting with Grok")
expectSentToGroup(CUSTOMER_GROUP_ID, grokActivatedMessage)
})
test("/grok as first message → WELCOME→GROK directly, no queue message", async () => {
@@ -646,7 +671,7 @@ describe("/grok Activation", () => {
await bot.onNewChatItems(customerMessage("/grok"))
await joinPromise
await bot.flush()
expectSentToGroup(CUSTOMER_GROUP_ID, "You are chatting with Grok")
expectSentToGroup(CUSTOMER_GROUP_ID, grokActivatedMessage)
expectNotSentToGroup(CUSTOMER_GROUP_ID, "The team will reply to your message")
expect(chat.sentTo(TEAM_GROUP_ID).length).toBeGreaterThan(0)
})
@@ -654,7 +679,7 @@ describe("/grok Activation", () => {
test("/grok in TEAM → rejected with teamLockedMessage", async () => {
await reachTeam()
await bot.onNewChatItems(customerMessage("/grok"))
expectSentToGroup(CUSTOMER_GROUP_ID, "team mode")
expectSentToGroup(CUSTOMER_GROUP_ID, teamLockedMessage)
})
test("/grok when grokContactId is null → grokUnavailableMessage", async () => {
@@ -864,7 +889,7 @@ describe("/team Activation", () => {
addBotMessage("We will reply within 24 hours.")
chat.members.set(CUSTOMER_GROUP_ID, [makeTeamMember(TEAM_MEMBER_1_ID, "Alice")])
await bot.onNewChatItems(customerMessage("/team"))
expectSentToGroup(CUSTOMER_GROUP_ID, "already been invited")
expectSentToGroup(CUSTOMER_GROUP_ID, teamAlreadyInvitedMessage)
})
test("/team with no team members → noTeamMembersMessage", async () => {
@@ -898,7 +923,7 @@ describe("One-Way Gate", () => {
test("/grok after gate → teamLockedMessage", async () => {
await reachTeam()
await bot.onNewChatItems(customerMessage("/grok"))
expectSentToGroup(CUSTOMER_GROUP_ID, "team mode")
expectSentToGroup(CUSTOMER_GROUP_ID, teamLockedMessage)
})
test("customer text in TEAM → card update scheduled, no bot reply", async () => {
@@ -1456,7 +1481,7 @@ describe("Error Handling", () => {
// Only the "Inviting Grok" message is sent — no activated/unavailable result
expect(chat.sent.length).toBe(sentBefore + 1)
expectSentToGroup(CUSTOMER_GROUP_ID, "Inviting Grok")
expectNotSentToGroup(CUSTOMER_GROUP_ID, "You are chatting with Grok")
expectNotSentToGroup(CUSTOMER_GROUP_ID, grokActivatedMessage)
expectNotSentToGroup(CUSTOMER_GROUP_ID, "temporarily unavailable")
})
@@ -1646,7 +1671,7 @@ describe("Grok Join Flow", () => {
await bot.flush()
expect(chat.joined).toContain(GROK_LOCAL_GROUP_ID)
expectSentToGroup(CUSTOMER_GROUP_ID, "You are chatting with Grok")
expectSentToGroup(CUSTOMER_GROUP_ID, grokActivatedMessage)
})
test("per-message responses suppressed during activateGrok initial response", async () => {
@@ -1794,7 +1819,7 @@ describe("End-to-End Flows", () => {
await joinPromise
await bot.flush()
expectSentToGroup(CUSTOMER_GROUP_ID, "You are chatting with Grok")
expectSentToGroup(CUSTOMER_GROUP_ID, grokActivatedMessage)
expectNotSentToGroup(CUSTOMER_GROUP_ID, "The team will reply to your message")
expect(chat.sentTo(TEAM_GROUP_ID).length).toBeGreaterThan(0)
})
@@ -1834,8 +1859,8 @@ describe("Message Templates", () => {
expect(grokActivatedMessage).toContain("chatting with Grok")
})
test("teamLockedMessage mentions team mode", () => {
expect(teamLockedMessage).toContain("team mode")
test("teamLockedMessage tells customer the team will handle the conversation", () => {
expect(teamLockedMessage).toContain("team")
})
test("queueMessage mentions hours", () => {
+2 -2
View File
@@ -8,10 +8,10 @@
"start": "node dist/index.js"
},
"dependencies": {
"@simplex-chat/types": "^0.5.0",
"@simplex-chat/types": "^0.6.0",
"async-mutex": "^0.5.0",
"commander": "^14.0.3",
"simplex-chat": "^6.5.0"
"simplex-chat": "^6.5.1"
},
"devDependencies": {
"@types/node": "^22.0.0",
+2 -5
View File
@@ -8,7 +8,7 @@ import {
teamAlreadyInvitedMessage, teamLockedMessage, noTeamMembersMessage,
grokUnavailableMessage, grokErrorMessage, grokNoHistoryMessage,
} from "./messages.js"
import {profileMutex, log, logError} from "./util.js"
import {profileMutex, log, logError, getGroupInfo} from "./util.js"
// True for any non-terminal status — invited but not yet accepted, through
// connected. Used to decide whether a contact is already in the group so we
@@ -795,10 +795,7 @@ export class SupportBot {
private async handleJoinCommand(targetGroupId: number, senderContactId: number): Promise<void> {
// Validate target is a business group
const groups = await this.withMainProfile(() =>
this.chat.apiListGroups(this.mainUserId)
)
const targetGroup = groups.find(g => g.groupId === targetGroupId)
const targetGroup = await this.withMainProfile(() => getGroupInfo(this.chat, targetGroupId))
if (!targetGroup?.businessChat) {
await this.sendToGroup(this.config.teamGroup.id, `Error: group ${targetGroupId} is not a business chat`)
return
+18 -12
View File
@@ -2,7 +2,7 @@ import {T} from "@simplex-chat/types"
import {api, util} from "simplex-chat"
import {Mutex} from "async-mutex"
import {Config} from "./config.js"
import {profileMutex, log, logError} from "./util.js"
import {profileMutex, log, logError, getGroupInfo} from "./util.js"
// State derivation types
export type ConversationState = "WELCOME" | "QUEUE" | "GROK" | "TEAM-PENDING" | "TEAM"
@@ -117,8 +117,7 @@ export class CardManager {
// Dispatches to create-path when cardItemId is absent so a failed createCard retries.
private async flushOne(groupId: number): Promise<void> {
const groups = await this.withMainProfile(() => this.chat.apiListGroups(this.mainUserId))
const groupInfo = groups.find(g => g.groupId === groupId)
const groupInfo = await this.withMainProfile(() => getGroupInfo(this.chat, groupId))
if (!groupInfo) return
const data = groupInfo.customData as Record<string, unknown> | undefined
if (typeof data?.cardItemId === "number") {
@@ -129,12 +128,22 @@ export class CardManager {
}
async refreshAllCards(): Promise<void> {
const groups = await this.withMainProfile(() => this.chat.apiListGroups(this.mainUserId))
// Scan the most recently active 1000 chats. Active cards live on
// recently-active customer chats by definition — a card stays open
// while the conversation is in flight. If the bot has been offline
// long enough that an active card has fallen outside this window, the
// card refreshes lazily on the next customer message (which moves the
// chat back into the recent window).
const chats = await this.withMainProfile(() =>
this.chat.apiGetChats(this.mainUserId, {type: "last", count: 1000})
)
const activeCards: {groupId: number; cardItemId: number}[] = []
for (const group of groups) {
const customData = group.customData as Record<string, unknown> | undefined
for (const c of chats) {
if (c.chatInfo.type !== "group") continue
const groupInfo = c.chatInfo.groupInfo
const customData = groupInfo.customData as Record<string, unknown> | undefined
if (customData && typeof customData.cardItemId === "number" && !customData.complete) {
activeCards.push({groupId: group.groupId, cardItemId: customData.cardItemId})
activeCards.push({groupId: groupInfo.groupId, cardItemId: customData.cardItemId})
}
}
if (activeCards.length === 0) return
@@ -210,8 +219,7 @@ export class CardManager {
// --- Custom data ---
async getRawCustomData(groupId: number): Promise<Partial<CardData> | null> {
const groups = await this.withMainProfile(() => this.chat.apiListGroups(this.mainUserId))
const group = groups.find(g => g.groupId === groupId)
const group = await this.withMainProfile(() => getGroupInfo(this.chat, groupId))
if (!group?.customData) return null
const data = group.customData as Record<string, unknown>
const result: Partial<CardData> = {}
@@ -247,9 +255,7 @@ export class CardManager {
// --- Internal ---
private async updateCard(groupId: number): Promise<void> {
// Read customData and groupInfo in one apiListGroups call
const groups = await this.withMainProfile(() => this.chat.apiListGroups(this.mainUserId))
const groupInfo = groups.find(g => g.groupId === groupId)
const groupInfo = await this.withMainProfile(() => getGroupInfo(this.chat, groupId))
if (!groupInfo) return
const customData = groupInfo.customData as Record<string, unknown> | undefined
+11 -14
View File
@@ -5,7 +5,7 @@ import {parseConfig} from "./config.js"
import {SupportBot} from "./bot.js"
import {GrokApiClient} from "./grok.js"
import {welcomeMessage} from "./messages.js"
import {profileMutex, log, logError} from "./util.js"
import {profileMutex, log, logError, getGroupInfo, getContact} from "./util.js"
interface BotState {
teamGroupId?: number
@@ -163,14 +163,12 @@ async function main(): Promise<void> {
await chat.apiSetAutoAcceptMemberContacts(mainUser.userId, true)
log("Auto-accept member contacts enabled")
// Step 5: List contacts, resolve Grok contact
const contacts = await chat.apiListContacts(mainUser.userId)
log(`Contacts connected: ${contacts.length || "(none)"}`)
// Step 5: Resolve Grok contact by ID. Avoid apiListContacts — it loads
// every contact in one response and OOMs the native binding on large DBs.
// Always restore grokContactId so the one-way gate can find and remove
// Grok members even when Grok API is disabled.
if (typeof state.grokContactId === "number") {
const found = contacts.find(c => c.contactId === state.grokContactId)
const found = await getContact(chat, state.grokContactId)
if (found) {
config.grokContactId = found.contactId
log(`Grok contact from state: ID=${config.grokContactId}`)
@@ -210,14 +208,13 @@ async function main(): Promise<void> {
}
}
// Step 6: Resolve team group
// Step 6: Resolve team group by ID. Avoid apiListGroups — it loads every
// group in one response and OOMs the native binding on large DBs.
log("Resolving team group...")
const groups = await chat.apiListGroups(mainUser.userId)
let existingGroup: T.GroupInfo | undefined
let existingGroup: T.GroupInfo | null = null
if (typeof state.teamGroupId === "number") {
existingGroup = groups.find(g => g.groupId === state.teamGroupId)
existingGroup = await getGroupInfo(chat, state.teamGroupId)
if (existingGroup) {
config.teamGroup.id = existingGroup.groupId
log(`Team group from state: ${config.teamGroup.id}:${existingGroup.groupProfile.displayName}`)
@@ -302,13 +299,13 @@ async function main(): Promise<void> {
inviteLinkTimer.unref()
}
// Step 9: Validate team members
// Step 9: Validate team members (lookup by ID, one round-trip per member)
if (config.teamMembers.length > 0) {
log("Validating team members...")
for (const member of config.teamMembers) {
const contact = contacts.find(c => c.contactId === member.id)
const contact = await getContact(chat, member.id)
if (!contact) {
console.error(`Team member not found: ID=${member.id}. Available: ${contacts.map(c => `${c.contactId}:${c.profile.displayName}`).join(", ") || "(none)"}`)
console.error(`Team member not found: ID=${member.id}`)
process.exit(1)
}
if (contact.profile.displayName !== member.name) {
+29
View File
@@ -1,7 +1,36 @@
import {Mutex} from "async-mutex"
import {api, core} from "simplex-chat"
import {T} from "@simplex-chat/types"
export const profileMutex = new Mutex()
export function isChatNotFound(err: unknown, kind: "group" | "contact"): boolean {
if (!(err instanceof core.ChatAPIError)) return false
if (err.chatError?.type !== "errorStore") return false
const seType = err.chatError.storeError.type
return kind === "group" ? seType === "groupNotFound" : seType === "contactNotFound"
}
export async function getGroupInfo(chat: api.ChatApi, groupId: number): Promise<T.GroupInfo | null> {
try {
const c = await chat.apiGetChat(T.ChatType.Group, groupId, 0)
return c.chatInfo.type === "group" ? c.chatInfo.groupInfo : null
} catch (err) {
if (isChatNotFound(err, "group")) return null
throw err
}
}
export async function getContact(chat: api.ChatApi, contactId: number): Promise<T.Contact | null> {
try {
const c = await chat.apiGetChat(T.ChatType.Direct, contactId, 0)
return c.chatInfo.type === "direct" ? c.chatInfo.contact : null
} catch (err) {
if (isChatNotFound(err, "contact")) return null
throw err
}
}
export function isWeekend(timezone: string): boolean {
const day = new Intl.DateTimeFormat("en-US", {timeZone: timezone, weekday: "short"}).format(new Date())
return day === "Sat" || day === "Sun"
@@ -19,8 +19,18 @@ function contactAddressStr(link) {
return link.connShortLink || link.connFullLink
}
// Mirrors core.ChatAPIError so isChatNotFound's instanceof check passes when
// MockChatApi throws. Tests should construct these directly.
class ChatAPIError extends Error {
constructor(message, chatError) {
super(message)
this.chatError = chatError
}
}
module.exports = {
api: {ChatApi: {}},
bot: {},
core: {ChatAPIError},
util: {ciContentText, ciBotCommand, contactAddressStr},
}
+41
View File
@@ -51,6 +51,7 @@ This file is generated automatically.
[Chat commands](#chat-commands)
- [APIListContacts](#apilistcontacts)
- [APIListGroups](#apilistgroups)
- [APIGetChats](#apigetchats)
- [APIDeleteChat](#apideletechat)
- [APISetGroupCustomData](#apisetgroupcustomdata)
- [APISetContactCustomData](#apisetcontactcustomdata)
@@ -1574,6 +1575,46 @@ ChatCmdError: Command error (only used in WebSockets API).
---
### APIGetChats
Get chat previews. Supports time-based pagination — use this instead of APIListContacts / APIListGroups when scanning at scale (those load every record into memory and fail on large databases).
*Network usage*: no.
**Parameters**:
- userId: int64
- pendingConnections: bool
- pagination: [PaginationByTime](./TYPES.md#paginationbytime)
- query: [ChatListQuery](./TYPES.md#chatlistquery)
**Syntax**:
```
/_get chats <userId>[ pcc=on] <str(pagination)> <json(query)>
```
```javascript
'/_get chats ' + userId + (pendingConnections ? ' pcc=on' : '') + ' ' + PaginationByTime.cmdString(pagination) + ' ' + JSON.stringify(query) // JavaScript
```
```python
'/_get chats ' + str(userId) + (' pcc=on' if pendingConnections else '') + ' ' + str(pagination) + ' ' + json.dumps(query) # Python
```
**Responses**:
ApiChats: Chat previews (paginated). Use this instead of CRContactsList / CRGroupsList when scanning at scale..
- type: "apiChats"
- user: [User](./TYPES.md#user)
- chats: [[AChat](./TYPES.md#achat)]
ChatCmdError: Command error (only used in WebSockets API).
- type: "chatCmdError"
- chatError: [ChatError](./TYPES.md#chaterror)
---
### APIDeleteChat
Delete chat.
+43
View File
@@ -41,6 +41,7 @@ This file is generated automatically.
- [ChatInfo](#chatinfo)
- [ChatItem](#chatitem)
- [ChatItemDeletion](#chatitemdeletion)
- [ChatListQuery](#chatlistquery)
- [ChatPeerType](#chatpeertype)
- [ChatRef](#chatref)
- [ChatSettings](#chatsettings)
@@ -136,6 +137,7 @@ This file is generated automatically.
- [NewUser](#newuser)
- [NoteFolder](#notefolder)
- [OwnerVerification](#ownerverification)
- [PaginationByTime](#paginationbytime)
- [PendingContactConnection](#pendingcontactconnection)
- [PrefEnabled](#prefenabled)
- [Preferences](#preferences)
@@ -1327,6 +1329,22 @@ Message deletion result.
- toChatItem: [AChatItem](#achatitem)?
---
## ChatListQuery
**Discriminated union type**:
Filters:
- type: "filters"
- favorite: bool
- unread: bool
Search:
- type: "search"
- search: string
---
## ChatPeerType
@@ -2893,6 +2911,31 @@ Failed:
- reason: string
---
## PaginationByTime
**Discriminated union type**:
Last:
- type: "last"
- count: int
**Syntax**:
```
count=<count>
```
```javascript
'count=' + count // JavaScript
```
```python
'count=' + str(count) # Python
```
---
## PendingContactConnection
+1 -1
View File
@@ -144,6 +144,7 @@ chatCommandsDocsData =
"Commands to list and delete conversations.",
[ ("APIListContacts", [], "Get contacts.", ["CRContactsList", "CRChatCmdError"], [], Nothing, "/_contacts " <> Param "userId"),
("APIListGroups", [], "Get groups.", ["CRGroupsList", "CRChatCmdError"], [], Nothing, "/_groups " <> Param "userId" <> Optional "" (" @" <> Param "$0") "contactId_" <> Optional "" (" " <> Param "$0") "search"),
("APIGetChats", [], "Get chat previews. Supports time-based pagination — use this instead of APIListContacts / APIListGroups when scanning at scale (those load every record into memory and fail on large databases).", ["CRApiChats", "CRChatCmdError"], [], Nothing, "/_get chats " <> Param "userId" <> OnOffParam "pcc" "pendingConnections" (Just False) <> " " <> Param "pagination" <> " " <> Json "query"),
("APIDeleteChat", [], "Delete chat.", ["CRContactDeleted", "CRContactConnectionDeleted", "CRGroupDeletedUser", "CRChatCmdError"], [], Just UNBackground, "/_delete " <> Param "chatRef" <> " " <> Param "chatDeleteMode"),
("APISetGroupCustomData", [], "Set group custom data.", ["CRCmdOk", "CRChatCmdError"], [], Nothing, "/_set custom #" <> Param "groupId" <> Optional "" (" " <> Json "$0") "customData"),
("APISetContactCustomData", [], "Set contact custom data.", ["CRCmdOk", "CRChatCmdError"], [], Nothing, "/_set custom @" <> Param "contactId" <> Optional "" (" " <> Json "$0") "customData"),
@@ -357,7 +358,6 @@ undocumentedCommands =
"APIGetChatItemInfo",
"APIGetChatItems",
"APIGetChatItemTTL",
"APIGetChats",
"APIGetChatTags",
"APIGetConnNtfMessages",
"APIGetContactCode",
+2 -3
View File
@@ -95,9 +95,9 @@ chatResponsesDocsData =
("CRUserDeletedMembers", "Members deleted"),
("CRUserProfileUpdated", "User profile updated"),
("CRUserProfileNoChange", "User profile was not changed"),
("CRUsersList", "Users")
("CRUsersList", "Users"),
("CRApiChats", "Chat previews (paginated). Use this instead of CRContactsList / CRGroupsList when scanning at scale.")
-- ("CRApiChat", "Chat and messages"),
-- ("CRApiChats", "Chats with the most recent messages"),
-- ("CRChatCleared", ""),
-- ("CRChatItemInfo", "Message information"),
-- ("CRChatItems", "The most recent messages"),
@@ -120,7 +120,6 @@ undocumentedResponses =
"CRAgentWorkersDetails",
"CRAgentWorkersSummary",
"CRApiChat",
"CRApiChats",
"CRAppSettings",
"CRArchiveExported",
"CRArchiveImported",
+8 -5
View File
@@ -374,11 +374,11 @@ chatTypesDocsData =
(sti @UserPwdHash, STRecord, "", [], "", ""),
(sti @XFTPErrorType, STUnion, "", [], "", ""),
(sti @XFTPRcvFile, STRecord, "", [], "", ""),
(sti @XFTPSndFile, STRecord, "", [], "", "")
(sti @XFTPSndFile, STRecord, "", [], "", ""),
-- (sti @DatabaseError, STUnion, "DB", [], "", ""),
-- (sti @ChatItemInfo, STRecord, "", [], "", ""),
-- (sti @ChatItemVersion, STRecord, "", [], "", ""),
-- (sti @ChatListQuery, STUnion, "CLQ", [], "", ""),
(sti @ChatListQuery, STUnion, "CLQ", [], "", ""),
-- (sti @ChatName, STRecord, "", [], "", ""),
-- (sti @ChatPagination, STRecord, "CP", [], "", ""),
-- (sti @ConnectionStats, STRecord, "", [], "", ""),
@@ -387,7 +387,10 @@ chatTypesDocsData =
-- (sti @MemberReaction, STRecord, "", [], "", ""),
-- (sti @MsgContentTag, (STEnum' $ dropPfxSfx "MC" '_'), "", ["MCUnknown_"], "", ""),
-- (sti @NavigationInfo, STRecord, "", [], "", ""),
-- (sti @PaginationByTime, STRecord, "", [], "", ""),
-- PTAfter / PTBefore are hidden — bots only need "tail last N chats".
-- The wire format is parsed by paginationByTimeP in
-- src/Simplex/Chat/Library/Commands.hs.
(sti @PaginationByTime, STUnion1, "PT", ["PTAfter", "PTBefore"], "count=" <> Param "count", "")
-- (sti @RcvQueueInfo, STRecord, "", [], "", ""),
-- (sti @RcvSwitchStatus, STEnum, "", [], "", ""), -- incorrect
-- (sti @SendRef, STRecord, "", [], "", ""),
@@ -589,7 +592,7 @@ deriving instance Generic XFTPSndFile
-- deriving instance Generic DatabaseError
-- deriving instance Generic ChatItemInfo
-- deriving instance Generic ChatItemVersion
-- deriving instance Generic ChatListQuery
deriving instance Generic ChatListQuery
-- deriving instance Generic ChatName
-- deriving instance Generic ChatPagination
-- deriving instance Generic ConnectionStats
@@ -599,7 +602,7 @@ deriving instance Generic XFTPSndFile
-- deriving instance Generic MemberReaction
-- deriving instance Generic MsgContentTag
-- deriving instance Generic NavigationInfo
-- deriving instance Generic PaginationByTime
deriving instance Generic PaginationByTime
-- deriving instance Generic RcvQueueInfo
-- deriving instance Generic RcvSwitchStatus
-- deriving instance Generic SendRef
@@ -1,6 +1,6 @@
{
"name": "@simplex-chat/types",
"version": "0.5.0",
"version": "0.6.0",
"description": "TypeScript types for SimpleX Chat bot libraries",
"main": "dist/index.js",
"types": "dist/index.d.ts",
@@ -575,6 +575,23 @@ export namespace APIListGroups {
}
}
// Get chat previews. Supports time-based pagination — use this instead of APIListContacts / APIListGroups when scanning at scale (those load every record into memory and fail on large databases).
// Network usage: no.
export interface APIGetChats {
userId: number // int64
pendingConnections: boolean
pagination: T.PaginationByTime
query: T.ChatListQuery
}
export namespace APIGetChats {
export type Response = CR.ApiChats | CR.ChatCmdError
export function cmdString(self: APIGetChats): string {
return '/_get chats ' + self.userId + (self.pendingConnections ? ' pcc=on' : '') + ' ' + T.PaginationByTime.cmdString(self.pagination) + ' ' + JSON.stringify(self.query)
}
}
// Delete chat.
// Network usage: background.
export interface APIDeleteChat {
@@ -55,6 +55,7 @@ export type ChatResponse =
| CR.UserProfileUpdated
| CR.UserProfileNoChange
| CR.UsersList
| CR.ApiChats
export namespace CR {
export type Tag =
@@ -109,6 +110,7 @@ export namespace CR {
| "userProfileUpdated"
| "userProfileNoChange"
| "usersList"
| "apiChats"
interface Interface {
type: Tag
@@ -443,4 +445,10 @@ export namespace CR {
type: "usersList"
users: T.UserInfo[]
}
export interface ApiChats extends Interface {
type: "apiChats"
user: T.User
chats: T.AChat[]
}
}
@@ -1577,6 +1577,27 @@ export interface ChatItemDeletion {
toChatItem?: AChatItem
}
export type ChatListQuery = ChatListQuery.Filters | ChatListQuery.Search
export namespace ChatListQuery {
export type Tag = "filters" | "search"
interface Interface {
type: Tag
}
export interface Filters extends Interface {
type: "filters"
favorite: boolean
unread: boolean
}
export interface Search extends Interface {
type: "search"
search: string
}
}
export enum ChatPeerType {
Human = "human",
Bot = "bot",
@@ -3190,6 +3211,25 @@ export namespace OwnerVerification {
}
}
export type PaginationByTime = PaginationByTime.Last
export namespace PaginationByTime {
export type Tag = "last"
interface Interface {
type: Tag
}
export interface Last extends Interface {
type: "last"
count: number // int
}
export function cmdString(self: PaginationByTime): string {
return 'count=' + self.count
}
}
export interface PendingContactConnection {
pccConnId: number // int64
pccAgentConnId: string
+1 -1
View File
@@ -14,7 +14,7 @@ Please share your use cases and implementations.
## Quick start: a simple bot
```
npm i simplex-chat@6.5.0-beta.10
npm i simplex-chat@6.5.1
```
Simple bot that replies with squares of numbers you send to it:
+2 -2
View File
@@ -1,6 +1,6 @@
{
"name": "simplex-chat",
"version": "6.5.0",
"version": "6.5.1",
"main": "dist/index.js",
"types": "dist/index.d.ts",
"files": [
@@ -24,7 +24,7 @@
"docs": "typedoc"
},
"dependencies": {
"@simplex-chat/types": "^0.5.0",
"@simplex-chat/types": "^0.6.0",
"extract-zip": "^2.0.1",
"fast-deep-equal": "^3.1.3",
"node-addon-api": "^8.5.0"
+19
View File
@@ -764,6 +764,25 @@ export class ChatApi {
throw new ChatCommandError("error listing groups", r)
}
/**
* Get chat previews (paginated).
* Network usage: no.
*
* Prefer this over apiListContacts / apiListGroups for any scan: those
* methods load every record into memory in a single response and will fail
* on large databases.
*/
async apiGetChats(
userId: number,
pagination: T.PaginationByTime,
query: T.ChatListQuery = {type: "filters", favorite: false, unread: false},
pendingConnections = false,
): Promise<T.AChat[]> {
const r = await this.sendChatCmd(CC.APIGetChats.cmdString({userId, pendingConnections, pagination, query}))
if (r.type === "apiChats") return r.chats
throw new ChatCommandError("error getting chats", r)
}
/**
* Delete chat.
* Network usage: background.
@@ -4,7 +4,7 @@ const path = require('path');
const extract = require('extract-zip');
const GITHUB_REPO = 'simplex-chat/simplex-chat-libs';
const RELEASE_TAG = 'v6.5.0';
const RELEASE_TAG = 'v6.5.1';
const BACKEND = (process.env.SIMPLEX_BACKEND || process.env.npm_config_simplex_backend || 'sqlite').toLowerCase();
if (BACKEND !== 'sqlite' && BACKEND !== 'postgres') {
@@ -0,0 +1,377 @@
# Plan: Fix support-bot crash on large databases — use pagination and direct lookup
## Context
The simplex-support-bot crashes during startup against large production
databases:
```
[2026-04-30T15:52:53.498Z] Grok contact from state: ID=142676
[2026-04-30T15:52:53.498Z] Resolving team group...
<anonymous_script>:0
[Error: Unknown failure]
```
The crash happens inside `chat.apiListGroups(mainUser.userId)` at
`apps/simplex-support-bot/src/index.ts:215`. The native binding marshals the
Haskell core's response to a JS string at
`packages/simplex-chat-nodejs/cpp/simplex.cc:255` (`chat_send_cmd`) →
`HandleCResult` (line 157) → `Napi::String::New` in `OnOK`. When the response
exceeds V8's max string length (~512 MB on 64-bit), N-API string allocation
fails. The literal string `"Unknown failure"` does **not** appear anywhere in
this repo — confirmed by full-tree search — so the message originates from V8
or N-API internals rather than the binding's own error path (which would say
`chat_send_cmd failed`). Hypothesis: oversized string allocation throws a JS
exception that propagates up unannotated.
Two distinct misuse patterns drive the payload size:
**A. List-then-find by ID** (most call sites). The bot pulls every contact /
every group with `apiListContacts` / `apiListGroups`, then calls `find(...)`
to locate one record by a known ID. This is gratuitous — there is already
`apiGetChat(chatType, chatId, count=0)` (`packages/simplex-chat-nodejs/src/api.ts:819`)
that returns one `AChat` whose `chatInfo` carries the full `GroupInfo` /
`Contact` (with `customData`) and zero items. The Haskell parser accepts
`count=0` (`src/Simplex/Chat/Library/Commands.hs:5210`), and
`getDirectChatLast_` / `getGroupChatLast_` return empty `chatItems` with full
`chatInfo`.
**B. Genuine multi-record scan** (one site).
`apps/simplex-support-bot/src/cards.ts:131` (`refreshAllCards`) enumerates
groups where `customData.cardItemId && !complete` to refresh in-flight cards
on restart. The Haskell side already supports paginated scans via
`APIGetChats` (`/_get chats {userId} pcc=on|off count=N`,
`src/Simplex/Chat/Library/Commands.hs:4868`). It is currently in
`undocumentedCommands` (`bots/src/API/Docs/Commands.hs:360`), so the codegen
does not emit it for the TypeScript bot library. Confirmed: the chat preview
returned by `getChatPreviews` carries `customData` on `GroupInfo`
(`src/Simplex/Chat/Store/Shared.hs:685`, `toGroupInfo`).
Active card state is already tracked on each group via `customData.cardItemId`
and `customData.complete` (written through `apiSetGroupCustomData` at
`apps/simplex-support-bot/src/cards.ts:103,231`). No `state.json` schema
change is needed — phase 3 reads exactly the same `customData` it already
writes, just via paginated `APIGetChats` instead of a full `apiListGroups`.
Per the constraint, `apiListContacts` / `apiListGroups` stay in the nodejs
library unchanged for other consumers. Audit confirmed no callers outside
support-bot use them today.
## Phase 1 — Plumb `APIGetChats` through the bot library
The codegen pipeline is test-driven: `tests/APIDocs.hs:4144` invokes
`testGenerate` against the functions in
`bots/src/API/Docs/Generate/TypeScript.hs`, which writes to:
- `packages/simplex-chat-client/types/typescript/src/commands.ts`
- `packages/simplex-chat-client/types/typescript/src/responses.ts`
- `packages/simplex-chat-client/types/typescript/src/types.ts`
Run via `cabal test`. The published `@simplex-chat/types` npm package is
built from this TypeScript source; the copy under
`packages/simplex-chat-nodejs/node_modules/@simplex-chat/types/dist/` is a
downstream build artifact and is **not** edited directly.
Currently missing from generated TS:
`T.PaginationByTime`, `T.ChatListQuery`, `CC.APIGetChats`, and the
`apiChats` response tag on `ChatResponse`.
### 1.1 `bots/src/API/Docs/Commands.hs`
- **Remove** `"APIGetChats",` from `undocumentedCommands` (line 360).
- **Add** an entry under "Chat commands" (next to `APIListContacts` /
`APIListGroups` at lines 145146). Match the Haskell parser at
`src/Simplex/Chat/Library/Commands.hs:4868`:
```haskell
( "APIGetChats",
[],
"Get chat previews. Supports time-based pagination — use this " <>
"instead of APIListContacts / APIListGroups when scanning at scale.",
["CRApiChats", "CRChatCmdError"],
[],
Nothing,
"/_get chats " <> Param "userId"
<> OnOffParam "pcc" "pendingConnections" (Just False)
<> Optional "" (" " <> Param "$0") "pagination"
<> Optional "" (" " <> Json "$0") "query"
)
```
Note: the `query` segment uses `" " <> Json "$0"` (no `"json "` prefix) —
the parser accepts `A.space *> jsonP` directly.
### 1.2 `bots/src/API/Docs/Types.hs`
The type universe already references `PaginationByTime` and `ChatListQuery`
in commented form (lines 381, 390 and 592, 602). Uncomment all four lines.
Confirm the constructor-prefix encoding (`STRecord`/`STUnion`, prefix
`""`/`"CLQ"`) matches the existing definitions in
`src/Simplex/Chat/Controller.hs:992,998` and the JSON deriving at line 1661
(`sumTypeJSON $ dropPrefix "CLQ"`).
### 1.3 `bots/src/API/Docs/Responses.hs`
- Uncomment `("CRApiChats", "...")` at line 100.
- Remove `"CRApiChats",` from `undocumentedResponses` at line 123.
### 1.4 Regenerate TypeScript types
Run `cabal test` (the `APIDocs` test suite drives generation). Inspect the
diffs in `packages/simplex-chat-client/types/typescript/src/{commands,responses,types}.ts`.
Verify:
- `T.PaginationByTime` (sum type with `PTLast`/`PTAfter`/`PTBefore`) exists
with a generated `cmdString`. Compare wire format against the Haskell
`paginationByTimeP` at `src/Simplex/Chat/Library/Commands.hs:5216`:
`count=N` | `after=TS count=N` | `before=TS count=N`.
- `T.ChatListQuery` exists with `CLQFilters` / `CLQSearch` JSON-encoded
variants.
- `CC.APIGetChats.cmdString({userId, pendingConnections, pagination, query})`
exists and emits the expected wire format.
- `r.type === "apiChats"` with `r.chats: T.AChat[]` exists in the response
union (drops `CR` prefix per `sumTypeJSON`,
`src/Simplex/Chat/Controller.hs:1743`).
Bump `@simplex-chat/types` version and re-link / re-build the
`simplex-chat-nodejs` package so the new symbols are available.
### 1.5 `packages/simplex-chat-nodejs/src/api.ts`
Add a single method next to `apiListGroups` (line 761):
```ts
/**
* Get chat previews (paginated).
* Network usage: no.
*
* Prefer this over apiListContacts / apiListGroups for any scan: those
* methods load the entire history into memory and will fail on large DBs.
*/
async apiGetChats(
userId: number,
pagination: T.PaginationByTime,
query: T.ChatListQuery = {type: "filters", favorite: false, unread: false},
pendingConnections = false,
): Promise<T.AChat[]> {
const r = await this.sendChatCmd(
CC.APIGetChats.cmdString({userId, pendingConnections, pagination, query})
)
if (r.type === "apiChats") return r.chats
throw new ChatCommandError("error getting chats", r)
}
```
(Exact `T.PaginationByTime` / `T.ChatListQuery` shapes come from the codegen
output of phase 1.4 — verify the discriminator field names before locking
this signature.)
## Phase 2 — Replace list-then-find with direct lookup
For every site below, replace `apiList…().find(…)` with
`apiGetChat(ChatType.X, id, 0)`. Treat "not found" — the chat was deleted —
as a clean missing-record case (log + skip). The wire format
`/_get chat #{id} count=0` is already supported.
### 2.1 Error matcher
The Haskell `SEContactNotFound` / `SEGroupNotFound` (in
`src/Simplex/Chat/Store/Shared.hs:863` and elsewhere) surface to TS as:
```ts
err.chatError?.type === "errorStore"
&& err.chatError.storeError.type === "groupNotFound" // or "contactNotFound"
```
Both discriminators are already present in the generated types
(`types.d.ts:2825` and `:2788`). Add a small helper in
`apps/simplex-support-bot/src/util.ts`:
```ts
export function isChatNotFound(err: unknown, kind: "group" | "contact"): boolean {
if (!(err instanceof core.ChatAPIError)) return false
if (err.chatError?.type !== "errorStore") return false
const seType = err.chatError.storeError.type
return kind === "group" ? seType === "groupNotFound" : seType === "contactNotFound"
}
```
(Strict — does not swallow other `errorStore` variants.)
### 2.2 Ergonomic wrappers
Add two thin helpers in `apps/simplex-support-bot/src/util.ts` (the constraint
forbids touching `apiListContacts` / `apiListGroups` in the nodejs library;
keeping these helpers in the support-bot util keeps the library surface
unchanged):
```ts
export async function getGroupInfo(chat: api.ChatApi, groupId: number): Promise<T.GroupInfo | null> {
try {
const c = await chat.apiGetChat(T.ChatType.Group, groupId, 0)
return c.chatInfo.type === "group" ? c.chatInfo.groupInfo : null
} catch (err) {
if (isChatNotFound(err, "group")) return null
throw err
}
}
export async function getContact(chat: api.ChatApi, contactId: number): Promise<T.Contact | null> {
try {
const c = await chat.apiGetChat(T.ChatType.Direct, contactId, 0)
return c.chatInfo.type === "direct" ? c.chatInfo.contact : null
} catch (err) {
if (isChatNotFound(err, "contact")) return null
throw err
}
}
```
### 2.3 Call-site changes
All sites must keep their existing `withMainProfile` / `profileMutex`
wrapping where present.
- **`apps/simplex-support-bot/src/index.ts:165180`** (Grok contact
resolution). Drop the `apiListContacts(mainUser.userId)` call entirely. If
`state.grokContactId` is set, call `getContact(chat, state.grokContactId)`
inside `profileMutex.runExclusive`. Preserve the existing log lines.
- **`apps/simplex-support-bot/src/index.ts:306320`** (team member
validation). Loop and `getContact(chat, member.id)` per member. Compare
`displayName` as before. Team rosters are small; N round-trips are fine.
- **`apps/simplex-support-bot/src/index.ts:213227`** (team group
resolution). Replace `apiListGroups` + `find` with
`getGroupInfo(chat, state.teamGroupId)`. Preserve the "create new group"
fallback when the lookup returns `null`.
- **`apps/simplex-support-bot/src/bot.ts:796805`** (`handleJoinCommand`).
Replace with `getGroupInfo(chat, targetGroupId)`; same `businessChat`
validation.
- **`apps/simplex-support-bot/src/cards.ts:120`** (`flushOne`). Direct
`getGroupInfo(chat, groupId)` (still inside `withMainProfile`).
- **`apps/simplex-support-bot/src/cards.ts:213`** (`getRawCustomData`).
Direct lookup. **Hot path** — called on every `mergeCustomData` /
`clearCustomData`. Largest single win.
- **`apps/simplex-support-bot/src/cards.ts:251`** (`updateCard`). Direct
lookup. The "Read customData and groupInfo in one apiListGroups call"
comment goes away.
After phase 2 the bot can boot and operate steadily on a large DB; phase 3
is purely about startup reconciliation.
## Phase 3 — Paginate `refreshAllCards`
`apps/simplex-support-bot/src/cards.ts:131` is the only legitimate
multi-record scan. Convert it to a single bounded `apiGetChats` call:
```ts
async refreshAllCards(): Promise<void> {
// Scan the most recently active 1000 chats. Active cards live on
// recently-active customer chats by definition — a card stays open while
// the conversation is in flight. If the bot has been offline long enough
// that an active card has fallen outside the recent-1000 window, that
// card refreshes lazily on the next customer message (which moves the
// chat back into the recent window).
const chats = await this.withMainProfile(() =>
this.chat.apiGetChats(
this.mainUserId,
{type: "last", count: 1000},
)
)
const activeCards: {groupId: number; cardItemId: number}[] = []
for (const c of chats) {
if (c.chatInfo.type !== "group") continue
const customData = c.chatInfo.groupInfo.customData as Record<string, unknown> | undefined
if (customData
&& typeof customData.cardItemId === "number"
&& !customData.complete) {
activeCards.push({
groupId: c.chatInfo.groupInfo.groupId,
cardItemId: customData.cardItemId,
})
}
}
// (sort and refresh loop unchanged)
}
```
`count = 1000` per the constraint. No `state.json` schema change. Card
status remains entirely on the group's `customData` (`cardItemId`,
`complete`), which is what the bot already reads and writes.
## Phase 4 — Verification
### 4.1 Stress test
Existing tests use `MockChatApi` (`apps/simplex-support-bot/bot.test.ts:24`),
which is in-memory and won't exercise the native binding. A meaningful
stress test needs a real `ChatApi.init` against Postgres.
Add a new test file (e.g.
`apps/simplex-support-bot/test/stress.test.ts`) that:
1. Starts an ephemeral Postgres or uses an existing test DB.
2. Calls `ChatApi.init` and seeds N synthetic groups + contacts via the
chat API. (No existing seeding helper — write one.) Reasonable N: 20k
each, large enough to expose the marshaling cliff but not so large that
the test takes minutes.
3. Boots the support-bot main flow against this DB and asserts: startup
completes within a wall-clock budget; resident memory stays bounded;
no native error.
This is new infrastructure — keep scope tight. If standing up Postgres in
CI is too heavy, run as a manual stress harness rather than a CI test.
### 4.2 Production replay
Replay against (a copy of) the affected production DB. Confirm the bot
starts and the team group / Grok contact / team members all resolve.
### 4.3 Smoke tests
Existing functional flows via `bot.test.ts` continue to pass after the
phase-2 changes. Manually exercise:
- Business-request acceptance.
- `/join` validation (the changed `bot.ts:799` path).
- Card create/update/complete cycle (`cards.ts` hot path).
- Restart-time card refresh (`refreshAllCards`).
## Risks and footguns
- **`/_get chats` parser default is `PTLast 5000`**
(`src/Simplex/Chat/Library/Commands.hs:4872`). Even 5000 previews can be
heavy. Support-bot now always passes an explicit `count=1000`, but the
default itself remains a footgun for other callers — flag for follow-up;
not changed here.
- **`apiListMembers` is per-group, not per-DB.** Used at `bot.ts:629,825`
and `cards.ts:165`. Bounded by group membership, not history size, so
out of scope for this fix. Flag if customer groups grow huge (>1000
members) — would warrant a paginated members API at that point.
- **Codegen output sanity.** Phase 1.4 must be inspected by hand — the
generated `cmdString` for `T.PaginationByTime` and the `r.type` /
`r.chats` shape on the response side are the integration points the rest
of the plan depends on. Do not skip eyeballing the diff.
- **`apiGetChat(..., 0)` semantics on a non-existent chatId.** Verified:
the error tag is `chatError.type === "errorStore"` with
`storeError.type === "groupNotFound"` or `"contactNotFound"`. Both
discriminators already exist in the generated types
(`types.d.ts:2825,2788`). `isChatNotFound` matches them precisely; do
not loosen it.
- **Native binding crash hypothesis is unverified.** The literal "Unknown
failure" string is not in this tree. Most likely V8/N-API surfacing a
string-allocation or JSON-parse failure. The fix in this plan addresses
the proximate cause (oversized response payload) regardless of the exact
surfacing path; if the same error reappears after the fix, dig into the
binding's `OnOK` handler to add explicit size-check / better diagnostics.
- **`@simplex-chat/types` package version bump.** Phase 1.4 produces
TypeScript changes in `packages/simplex-chat-client/types/typescript/`.
Bumping the version and re-publishing (or rebuilding locally) is required
before phase 1.5 lands. Coordinate the release sequence.
## Out of scope
- Deprecating or paginating `apiListContacts` / `apiListGroups` in the
nodejs library. They stay as-is; only support-bot stops calling them.
- Lowering the `/_get chats` parser default from `PTLast 5000`.
- Adding a paginated members API.
- Native binding diagnostics for oversized responses.
+1 -1
View File
@@ -993,7 +993,7 @@ data ChatPagination
deriving (Show)
data PaginationByTime
= PTLast Int
= PTLast {count :: Int}
| PTAfter UTCTime Int
| PTBefore UTCTime Int
deriving (Show)