core, libs: configurable queue size, library fixes (#7542)

* core: add chat_migrate_init_queue FFI export

* bots: fix BadgeServiceErrorCode API type

* nodejs: pass required command fields

* nodejs: fix migration error types

* nodejs: install libsimplex from SIMPLEX_LIBS_DIR

* nodejs: add queue size option

* nodejs: regenerate docs

* python: add queue size option

* bots: pass incognito in APIConnect

* nodejs: accept documented success responses

* python: accept documented success responses

* nodejs: dispatch each bot message once

* nodejs: fix startChat events loop lifecycle

* nodejs, python: parse multi-line bot commands

* nodejs: fix file buffer handling in addon

* nodejs: keep events loop when chat stop fails

* python: fix send_and_wait race, load lib off loop

* python: make queue size export optional

* nodejs: receive events on a dedicated thread

* nodejs: release haskell thread after receive

* python: receive on a dedicated thread per chat

* python: test receive shutdown order

* nodejs: one receive thread per chat controller

* nodejs: harden receiver shutdown and tests

* nodejs: stop chat before closing store

* python: stop chat before closing store

* nodejs, python: harden close regression and retry

* python: free results with ucrt on windows

* nodejs: enable c++ exceptions on mac and windows

packages/simplex-chat-nodejs/src/api.ts

@@ -106,13 +106,15 @@ export class ChatApi {
    * Initializes the ChatApi.
    * @param {DbConfig} db - Database configuration (sqlite or postgres).
    * @param {core.MigrationConfirmation} [confirm=core.MigrationConfirmation.YesUp] - Migration confirmation mode.
+   * @param {number} [queueSize] - Size of internal queues, the core default is used when omitted.
    */
   static async init(
     db: DbConfig,
-    confirm = core.MigrationConfirmation.YesUp
+    confirm = core.MigrationConfirmation.YesUp,
+    queueSize?: number
   ): Promise<ChatApi> {
     const [path, key] = dbConfigToMigrateArgs(db)
-    const ctrl = await core.chatMigrateInit(path, key, confirm)
+    const ctrl = await core.chatMigrateInit(path, key, confirm, queueSize)
     return new ChatApi(ctrl)
   }
 
@@ -120,39 +122,52 @@ export class ChatApi {
    * Start chat controller. Must be called with the existing user profile.
    */
   async startChat(): Promise<void> {
+    if (this.eventsLoop) throw new Error("chat already started")
+    const ctrl = this.ctrl
     this.receiveEvents = true
     this.eventsLoop = this.runEventsLoop()
-    const r = await this.sendChatCmd(CC.StartChat.cmdString({mainApp: true, enableSndFiles: true}))
+    let r: ChatResponse
+    try {
+      r = await core.chatSendCmd(ctrl, CC.StartChat.cmdString({mainApp: true, enableSndFiles: true, serviceRequests: false}))
+    } catch (e) {
+      await this.stopEventsLoop()
+      throw e
+    }
     if (r.type !== "chatStarted" && r.type !== "chatRunning") {
+      await this.stopEventsLoop()
       throw new ChatCommandError("error starting chat", r)
     }
   }
-  
+
   /**
    * Stop chat controller.
-   * Must be called before closing the database.
+   * `close` calls it before closing the database.
    * Usually doesn't need to be called in chat bots.
    */
   async stopChat(): Promise<void> {
     const r = await this.sendChatCmd("/_stop")
-    if (r.type !== "chatStopped") throw new ChatCommandError("error starting chat", r)
-    this.receiveEvents = false
-    if (this.eventsLoop) await this.eventsLoop
-    this.eventsLoop = undefined    
+    if (r.type !== "chatStopped") throw new ChatCommandError("error stopping chat", r)
+    await this.stopEventsLoop()
   }
 
   /**
-   * Close chat database.
+   * Stop chat controller and close chat database.
+   * The database is not closed if stopping fails.
    * Usually doesn't need to be called in chat bots.
    */
   async close(): Promise<void> {
-    this.receiveEvents = false
-    if (this.eventsLoop) await this.eventsLoop
-    this.eventsLoop = undefined    
+    // a running controller keeps using the database connections that closing frees
+    await this.stopChat()
     await core.chatCloseStore(this.ctrl)
     this.ctrl_ = undefined
   }
-  
+
+  private async stopEventsLoop(): Promise<void> {
+    this.receiveEvents = false
+    if (this.eventsLoop) await this.eventsLoop
+    this.eventsLoop = undefined
+  }
+
   private async runEventsLoop(): Promise<void> {
     while (this.receiveEvents) {
       try {
@@ -335,7 +350,7 @@ export class ChatApi {
     return await core.chatSendCmd(this.ctrl, cmd)
   }
   
-  async recvChatEvent(wait: number = 5_000_000): Promise<ChatEvent | undefined> {
+  async recvChatEvent(wait: number = 500_000): Promise<ChatEvent | undefined> {
     return await core.chatRecvMsgWait(this.ctrl, wait)
   }
   
@@ -386,8 +401,10 @@ export class ChatApi {
     switch (r.type) {
       case "userProfileUpdated":
         return r.updateSummary
+      case "userProfileNoChange":
+        return {updateSuccesses: 0, updateFailures: 0, changedContacts: []}
       default:
-        throw new ChatCommandError("error loading user address", r)
+        throw new ChatCommandError("error setting profile address", r)
     }
   }
 
@@ -460,7 +477,7 @@ export class ChatApi {
         updatedMessage: {msgContent, mentions: {}},
       })
     )
-    if (r.type === "chatItemUpdated") return r.chatItem.chatItem
+    if (r.type === "chatItemUpdated" || r.type === "chatItemNotChanged") return r.chatItem.chatItem
     throw new ChatCommandError("error updating chat item", r)
   }
 
@@ -499,10 +516,10 @@ export class ChatApi {
     chatItemId: number,
     add: boolean,
     reaction: T.MsgReaction
-  ) {
+  ): Promise<T.ACIReaction> {
     const r = await this.sendChatCmd(CC.APIChatItemReaction.cmdString({chatRef: {chatType, chatId}, chatItemId, add, reaction}))
-    if (r.type === "chatItemsDeleted") return r.chatItemDeletions
-    throw new ChatCommandError("error setting item reaction", r)  
+    if (r.type === "chatItemReaction") return r.reaction
+    throw new ChatCommandError("error setting item reaction", r)
   }
 
   /**
@@ -512,6 +529,7 @@ export class ChatApi {
   async apiReceiveFile(fileId: number): Promise<T.AChatItem> {
     const r = await this.sendChatCmd(CC.ReceiveFile.cmdString({fileId, userApprovedRelays: true}))
     if (r.type === "rcvFileAccepted") return r.chatItem
+    if (r.type === "rcvFileAcceptedSndCancelled") throw new ChatCommandError("file cancelled by sender", r)
     throw new ChatCommandError("error receiving file", r)
   }
 
@@ -698,7 +716,7 @@ export class ChatApi {
    * Connect via prepared SimpleX link. The link can be 1-time invitation link, contact address or group link
    * Network usage: interactive.
    */
-  async apiConnect(userId: number, incognito: boolean, preparedLink?: T.CreatedConnLink): Promise<ConnReqType> {
+  async apiConnect(userId: number, incognito: boolean, preparedLink: T.CreatedConnLink): Promise<ConnReqType> {
     const r = await this.sendChatCmd(CC.APIConnect.cmdString({userId, incognito, preparedLink_: preparedLink}))
     return this.handleConnectResult(r)
   }
@@ -740,7 +758,7 @@ export class ChatApi {
    * Network usage: no.
    */
   async apiRejectContactRequest(contactReqId: number): Promise<void> {
-    const r = await this.sendChatCmd(CC.APIRejectContact.cmdString({contactReqId}))
+    const r = await this.sendChatCmd(CC.APIRejectContact.cmdString({contactReqId, notify: false}))
     if (r.type === "contactRequestRejected") return
     throw new ChatCommandError("error rejecting contact request", r)
   }

packages/simplex-chat-nodejs/src/bot.ts

@@ -6,6 +6,7 @@ import equal = require("fast-deep-equal")
 
 export type BotDbOpts = api.DbConfig & {
   confirmMigrations?: core.MigrationConfirmation
+  queueSize?: number
 }
 
 export interface BotOptions {
@@ -45,10 +46,9 @@ export interface BotConfig {
 }
 
 export async function run({profile, dbOpts, options = defaultOpts, onMessage, onCommands = {}, events = {}}: BotConfig): Promise<[api.ChatApi, T.User, T.UserContactLink | undefined]> {
-  const bot = await api.ChatApi.init(dbOpts, dbOpts.confirmMigrations || core.MigrationConfirmation.YesUp)
+  const bot = await api.ChatApi.init(dbOpts, dbOpts.confirmMigrations || core.MigrationConfirmation.YesUp, dbOpts.queueSize)
   const opts = fullOptions(options)
-  if (onMessage) subscribeMessages(bot, onMessage)
-  if (Object.keys(onCommands).length > 0) subscribeCommands(bot, onCommands)
+  if (onMessage || Object.keys(onCommands).length > 0) subscribeChatItems(bot, onMessage, onCommands)
   if (Object.keys(events).length > 0) bot.on(events)
   subscribeLogEvents(bot, opts)
   const botProfile = mkBotProfile(profile, opts)
@@ -105,40 +105,27 @@ function mkBotProfile(profile: T.Profile, opts: Required<BotOptions>): T.Profile
   return profile 
 }
 
-function subscribeMessages(bot: api.ChatApi, onMessage: (chatItem: T.AChatItem, content: T.MsgContent) => void | Promise<void>) {
+export function subscribeChatItems(
+  bot: api.ChatApi,
+  onMessage: ((chatItem: T.AChatItem, content: T.MsgContent) => void | Promise<void>) | undefined,
+  commands: {[K in string]?: ((chatItem: T.AChatItem, command: util.BotCommand) => void | Promise<void>)}
+) {
   bot.on("newChatItems", async ({chatItems}) => {
     for (const ci of chatItems) {
-      if (ci.chatItem.content.type === "rcvMsgContent") {
-        try {
-          const p = onMessage(ci, ci.chatItem.content.msgContent)
-          if (p instanceof Promise) await p
-        } catch (e) {
-          console.log("message processing error", e)
-        }
+      const content = ci.chatItem.content
+      if (content.type !== "rcvMsgContent") continue
+      const cmd = util.ciBotCommand(ci.chatItem)
+      const cmdFunc = cmd && (commands[cmd.keyword] || commands[""])
+      try {
+        if (cmd && cmdFunc) await cmdFunc(ci, cmd)
+        else if (onMessage) await onMessage(ci, content.msgContent)
+      } catch (e) {
+        console.log(cmd && cmdFunc ? `${cmd.keyword} command processing error` : "message processing error", e)
       }
     }
   })
 }
 
-function subscribeCommands(bot: api.ChatApi, commands: {[K in string]?: ((chatItem: T.AChatItem, command: util.BotCommand) => void | Promise<void>)}) {
-  bot.on("newChatItems", async (evt) => {
-    for (const ci of evt.chatItems) {
-      const cmd = util.ciBotCommand(ci.chatItem)
-      if (cmd) {
-        const cmdFunc = commands[cmd.keyword] || commands[""]
-        if (cmdFunc) {
-          try {
-            const p = cmdFunc(ci, cmd)
-            if (p instanceof Promise) await p
-          } catch(e) {
-            console.log(`${cmd} command processing error`, e)
-          }
-        }
-      }
-    }
-  })    
-}
-
 function subscribeLogEvents(bot: api.ChatApi, opts: Required<BotOptions>) {
   if (opts.logContacts) {
     bot.on({

packages/simplex-chat-nodejs/src/core.ts

@@ -3,9 +3,12 @@ import * as simplex from "./simplex"
 
 /**
  * Initialize chat controller
+ * @param {number} [queueSize] - Size of internal queues, the core default is used when omitted.
  */
-export async function chatMigrateInit(dbPath: string, dbKey: string, confirm: MigrationConfirmation): Promise<bigint> {
-  const [ctrl, res] = await simplex.chat_migrate_init(dbPath, dbKey, confirm)
+export async function chatMigrateInit(dbPath: string, dbKey: string, confirm: MigrationConfirmation, queueSize?: number): Promise<bigint> {
+  const [ctrl, res] = queueSize === undefined
+    ? await simplex.chat_migrate_init(dbPath, dbKey, confirm)
+    : await simplex.chat_migrate_init_queue(dbPath, dbKey, confirm, queueSize)
   const json = JSON.parse(res)
   if (json.type === 'ok') return ctrl
   throw new ChatInitError("Database or migration error (see dbMigrationError property)", json as DBMigrationError)
@@ -47,7 +50,7 @@ export async function chatRecvMsgWait(ctrl: bigint, wait: number): Promise<ChatE
 /**
  * Write buffer to encrypted file
  */
-export async function chatWriteFile(ctrl: bigint, path: string, buffer: ArrayBuffer): Promise<CryptoArgs> {
+export async function chatWriteFile(ctrl: bigint, path: string, buffer: ArrayBuffer | Uint8Array): Promise<CryptoArgs> {
   const res = await simplex.chat_write_file(ctrl, path, buffer)
   return cryptoArgsResult(res)
 }
@@ -55,7 +58,7 @@ export async function chatWriteFile(ctrl: bigint, path: string, buffer: ArrayBuf
 /**
  * Read buffer from encrypted file
  */
-export async function chatReadFile(path: string, {fileKey, fileNonce}: CryptoArgs): Promise<ArrayBuffer> {
+export async function chatReadFile(path: string, {fileKey, fileNonce}: CryptoArgs): Promise<Buffer> {
   return await simplex.chat_read_file(path, fileKey, fileNonce)
 }
 
@@ -121,12 +124,13 @@ export class ChatInitError extends Error {
 
 export type DBMigrationError = 
   | DBMigrationError.InvalidConfirmation
+  | DBMigrationError.InvalidQueueSize
   | DBMigrationError.ErrorNotADatabase // invalid/corrupt database file or incorrect encryption key
   | DBMigrationError.ErrorMigration
   | DBMigrationError.ErrorSQL
 
 export namespace DBMigrationError {
-  export type Tag = "invalidConfirmation" | "errorNotADatabase" | "errorMigration" | "errorSQL"
+  export type Tag = "invalidConfirmation" | "invalidQueueSize" | "errorNotADatabase" | "errorMigration" | "errorSQL"
 
   interface Interface {
     type: Tag
@@ -136,6 +140,10 @@ export namespace DBMigrationError {
     type: "invalidConfirmation"
   }
 
+  export interface InvalidQueueSize extends Interface {
+    type: "invalidQueueSize"
+  }
+
   export interface ErrorNotADatabase extends Interface {
     type: "errorNotADatabase"
     dbFile: string
@@ -168,7 +176,7 @@ export namespace MigrationError {
 
   export interface MEUpgrade extends Interface {
     type: "upgrade"
-    upMigrations: UpMigration
+    upMigrations: UpMigration[]
   }
 
   export interface MEDowngrade extends Interface {
@@ -200,7 +208,7 @@ export namespace MTRError {
 
   export interface MTRENoDown extends Interface {
     type: "noDown"
-    upMigrations: UpMigration
+    dbMigrations: string[]
   }
 
   export interface MTREDifferent extends Interface {

packages/simplex-chat-nodejs/src/download-libs.js

@@ -6,6 +6,10 @@ const extract = require('extract-zip');
 const GITHUB_REPO = 'simplex-chat/simplex-chat-libs';
 const RELEASE_TAG = 'v7.1.0-beta.3';
 const BACKEND = (process.env.SIMPLEX_BACKEND || process.env.npm_config_simplex_backend || 'sqlite').toLowerCase();
+// A locally built libsimplex, copied into libs/ rather than loaded in place:
+// the addon's RUNPATH is $ORIGIN/../../libs.
+const LIBS_DIR_OVERRIDE = process.env.SIMPLEX_LIBS_DIR || process.env.npm_config_simplex_libs_dir;
+const LIB_NAMES = ['libsimplex.so', 'libsimplex.dylib', 'libsimplex.dll'];
 
 if (BACKEND !== 'sqlite' && BACKEND !== 'postgres') {
   console.error(`✗ Invalid SIMPLEX_BACKEND: "${BACKEND}". Must be "sqlite" or "postgres".`);
@@ -83,8 +87,31 @@ function isAlreadyInstalled() {
   }
 }
 
+// No version check: the files behind SIMPLEX_LIBS_DIR change on every rebuild.
+function installFromOverride() {
+  if (!fs.existsSync(LIBS_DIR_OVERRIDE)) {
+    throw new Error(`SIMPLEX_LIBS_DIR does not exist: ${LIBS_DIR_OVERRIDE}`);
+  }
+  const lib = LIB_NAMES.find((name) => fs.existsSync(path.join(LIBS_DIR_OVERRIDE, name)));
+  if (!lib) {
+    throw new Error(`No ${LIB_NAMES.join(' / ')} in SIMPLEX_LIBS_DIR: ${LIBS_DIR_OVERRIDE}`);
+  }
+  console.log(`Using libraries from SIMPLEX_LIBS_DIR: ${LIBS_DIR_OVERRIDE}`);
+  cleanLibsDirectory();
+  copyDirSync(LIBS_DIR_OVERRIDE, LIBS_DIR);
+  // Not a release tag: a later install without the variable sees a mismatch
+  // and replaces these files with the released ones.
+  fs.writeFileSync(INSTALLED_FILE, `${LIBS_DIR_OVERRIDE}:${BACKEND}`, 'utf-8');
+  console.log(`✓ Installed ${lib} and its runtime libraries from ${LIBS_DIR_OVERRIDE}`);
+}
+
 async function install() {
   try {
+    if (LIBS_DIR_OVERRIDE) {
+      installFromOverride();
+      return;
+    }
+
     // Check if already installed
     if (isAlreadyInstalled()) {
       return;

packages/simplex-chat-nodejs/src/simplex.d.ts

@@ -1,10 +1,11 @@
 // These functions are defined in CPP add-on ../cpp/simplex.cc
 
 export function chat_migrate_init(dbPath: string, dbKey: string, confirm: string): Promise<[bigint, string]>
+export function chat_migrate_init_queue(dbPath: string, dbKey: string, confirm: string, queueSize: number): Promise<[bigint, string]>
 export function chat_close_store(ctrl: bigint): Promise<string>
 export function chat_send_cmd(ctrl: bigint, cmd: string): Promise<string>
 export function chat_recv_msg_wait(ctrl: bigint, wait: number): Promise<string>
-export function chat_write_file(ctrl: bigint, path: string, buffer: ArrayBuffer): Promise<string>
-export function chat_read_file(path: string, key: string, nonce: string): Promise<ArrayBuffer>
+export function chat_write_file(ctrl: bigint, path: string, buffer: ArrayBuffer | Uint8Array): Promise<string>
+export function chat_read_file(path: string, key: string, nonce: string): Promise<Buffer>
 export function chat_encrypt_file(ctrl: bigint, fromPath: string, toPath: string): Promise<string>
 export function chat_decrypt_file(fromPath: string, key: string, nonce: string, toPath: string): Promise<string>

packages/simplex-chat-nodejs/src/util.ts

@@ -78,7 +78,7 @@ export interface BotCommand {
 export function ciBotCommand(chatItem: T.ChatItem): BotCommand | undefined {
   const msg = ciContentText(chatItem)?.trim()
   if (msg) {
-    const r = msg.match(/^\/([^\s]+)(.*)/)
+    const r = msg.match(/^\/([^\s]+)([\s\S]*)/)
     if (r && r.length >= 3) {
       return {keyword: r[1], params: r[2].trim()}
     }