core, libs: configurable queue size, library fixes (#7542)

* core: add chat_migrate_init_queue FFI export

* bots: fix BadgeServiceErrorCode API type

* nodejs: pass required command fields

* nodejs: fix migration error types

* nodejs: install libsimplex from SIMPLEX_LIBS_DIR

* nodejs: add queue size option

* nodejs: regenerate docs

* python: add queue size option

* bots: pass incognito in APIConnect

* nodejs: accept documented success responses

* python: accept documented success responses

* nodejs: dispatch each bot message once

* nodejs: fix startChat events loop lifecycle

* nodejs, python: parse multi-line bot commands

* nodejs: fix file buffer handling in addon

* nodejs: keep events loop when chat stop fails

* python: fix send_and_wait race, load lib off loop

* python: make queue size export optional

* nodejs: receive events on a dedicated thread

* nodejs: release haskell thread after receive

* python: receive on a dedicated thread per chat

* python: test receive shutdown order

* nodejs: one receive thread per chat controller

* nodejs: harden receiver shutdown and tests

* nodejs: stop chat before closing store

* python: stop chat before closing store

* nodejs, python: harden close regression and retry

* python: free results with ucrt on windows

* nodejs: enable c++ exceptions on mac and windows

src/Simplex/Chat/Mobile.hs

@@ -12,6 +12,7 @@ module Simplex.Chat.Mobile where
 
 import Control.Concurrent.STM
 import Control.Exception (SomeException, catch)
+import Control.Monad
 import Control.Monad.Except
 import Control.Monad.Reader
 import Data.Aeson (ToJSON (..))
@@ -35,6 +36,7 @@ import Foreign.Ptr
 import Foreign.StablePtr
 import Foreign.Storable (poke)
 import GHC.IO.Encoding (setFileSystemEncoding, setForeignEncoding, setLocaleEncoding)
+import Numeric.Natural (Natural)
 import Simplex.Chat
 import Simplex.Chat.Badges.Code (badgeCodeText, parseBadgeCode)
 import Simplex.Chat.Controller
@@ -73,6 +75,7 @@ import qualified Simplex.Messaging.Agent.Store.DB as DB
 data DBMigrationResult
   = DBMOk
   | DBMInvalidConfirmation
+  | DBMInvalidQueueSize
   | DBMErrorNotADatabase {dbFile :: String}
   | DBMErrorMigration {dbFile :: String, migrationError :: MigrationError}
   | DBMErrorSQL {dbFile :: String, migrationSQLError :: String}
@@ -113,6 +116,8 @@ foreign export ccall "chat_migrate_init" cChatMigrateInit :: CString -> CString
 
 foreign export ccall "chat_migrate_init_key" cChatMigrateInitKey :: CString -> CString -> CInt -> CString -> CInt -> Ptr (StablePtr ChatController) -> IO CJSONString
 
+foreign export ccall "chat_migrate_init_queue" cChatMigrateInitQueue :: CString -> CString -> CString -> CInt -> Ptr (StablePtr ChatController) -> IO CJSONString
+
 foreign export ccall "chat_close_store" cChatCloseStore :: StablePtr ChatController -> IO CString
 
 foreign export ccall "chat_reopen_store" cChatReopenStore :: StablePtr ChatController -> IO CString
@@ -166,7 +171,14 @@ cChatMigrateInit fp key conf = cChatMigrateInitKey fp key 0 conf 0
 
 -- For postgres first param is schema prefix, second param is database connection string.
 cChatMigrateInitKey :: CString -> CString -> CInt -> CString -> CInt -> Ptr (StablePtr ChatController) -> IO CJSONString
-cChatMigrateInitKey fp key keepKey conf background ctrl = do
+cChatMigrateInitKey fp key keepKey conf background = cChatMigrateInit_ fp key (keepKey /= 0) conf (background /= 0) mobileQueueSize
+
+-- | queueSize is the size of internal queues, same as terminal option --queue-size
+cChatMigrateInitQueue :: CString -> CString -> CString -> CInt -> Ptr (StablePtr ChatController) -> IO CJSONString
+cChatMigrateInitQueue fp key conf queueSize = cChatMigrateInit_ fp key False conf False (fromIntegral queueSize)
+
+cChatMigrateInit_ :: CString -> CString -> Bool -> CString -> Bool -> Int -> Ptr (StablePtr ChatController) -> IO CJSONString
+cChatMigrateInit_ fp key keepKey conf background queueSize ctrl = do
   -- ensure we are set to UTF-8; iOS does not have locale, and will default to
   -- US-ASCII all the time.
   setLocaleEncoding utf8
@@ -176,7 +188,7 @@ cChatMigrateInitKey fp key keepKey conf background ctrl = do
   chatDbOpts <- mobileDbOpts fp key
   confirm <- peekCAString conf
   r <-
-    chatMigrateInitKey chatDbOpts (keepKey /= 0) confirm (background /= 0) >>= \case
+    chatMigrateInitKey chatDbOpts keepKey confirm background queueSize >>= \case
       Right cc -> (newStablePtr cc >>= poke ctrl) $> DBMOk
       Left e -> pure e
   newCStringFromLazyBS $ J.encode r
@@ -254,8 +266,11 @@ cChatParseBadgeCode cCode = do
 cChatJsonLength :: CString -> IO CInt
 cChatJsonLength s = fromIntegral . subtract 2 . LB.length . J.encode . safeDecodeUtf8 <$> B.packCString s
 
-mobileChatOpts :: ChatDbOpts -> ChatOpts
-mobileChatOpts dbOptions =
+mobileQueueSize :: Int
+mobileQueueSize = 4096
+
+mobileChatOpts :: ChatDbOpts -> Natural -> ChatOpts
+mobileChatOpts dbOptions tbqSize =
   ChatOpts
     { coreOptions =
         CoreChatOpts
@@ -268,7 +283,7 @@ mobileChatOpts dbOptions =
             logServerHosts = True,
             logAgent = Nothing,
             logFile = Nothing,
-            tbqSize = 4096,
+            tbqSize,
             maxChats = 5000,
             deviceName = Nothing,
             chatRelay = False,
@@ -313,18 +328,19 @@ getActiveUser_ st = find activeUser <$> withTransaction st getUsers
 chatMigrateInit :: String -> ScrubbedBytes -> String -> IO (Either DBMigrationResult ChatController)
 chatMigrateInit dbFilePrefix dbKey confirm = do
   let chatDBOpts = ChatDbOpts {dbFilePrefix, dbKey, trackQueries = DB.TQSlow 5000, vacuumOnMigration = True}
-  chatMigrateInitKey chatDBOpts False confirm False
+  chatMigrateInitKey chatDBOpts False confirm False mobileQueueSize
 #endif
 
-chatMigrateInitKey :: ChatDbOpts -> Bool -> String -> Bool -> IO (Either DBMigrationResult ChatController)
-chatMigrateInitKey chatDbOpts keepKey confirm backgroundMode = runExceptT $ do
+chatMigrateInitKey :: ChatDbOpts -> Bool -> String -> Bool -> Int -> IO (Either DBMigrationResult ChatController)
+chatMigrateInitKey chatDbOpts keepKey confirm backgroundMode queueSize = runExceptT $ do
+  unless (queueSize > 0) $ throwError DBMInvalidQueueSize
   confirmMigrations <- liftEitherWith (const DBMInvalidConfirmation) $ strDecode $ B.pack confirm
   let migrationConfig = MigrationConfig confirmMigrations (Just "")
   chatStore <- migrate createChatStore (toDBOpts chatDbOpts chatSuffix keepKey chatDBFunctions) migrationConfig
   agentStore <- migrate createAgentStore (toDBOpts chatDbOpts agentSuffix keepKey []) migrationConfig
   ExceptT $ initialize chatStore ChatDatabase {chatStore, agentStore}
   where
-    opts = mobileChatOpts $ removeDbKey chatDbOpts
+    opts = mobileChatOpts (removeDbKey chatDbOpts) (fromIntegral queueSize)
     initialize st db = do
       user_ <- liftIO $ getActiveUser_ st
       first DBMAgentError <$> newChatController db user_ defaultMobileConfig opts backgroundMode