core: option to limit the max number of loaded chats (#7499)

* core: option to limit the max number of loaded chats

* update bot api

---------

Co-authored-by: Evgeny @ SimpleX Chat <259188159+evgeny-simplex@users.noreply.github.com>

bots/api/COMMANDS.md

@@ -1817,21 +1817,21 @@ Get chat previews. Supports time-based pagination — use this instead of APILis
 **Parameters**:
 - userId: int64
 - pendingConnections: bool
-- pagination: [PaginationByTime](./TYPES.md#paginationbytime)
+- pagination: [PaginationByTime](./TYPES.md#paginationbytime)?
 - query: [ChatListQuery](./TYPES.md#chatlistquery)
 
 **Syntax**:
 
 ```
-/_get chats <userId>[ pcc=on] <str(pagination)> <json(query)>
+/_get chats <userId>[ pcc=on][ <str(pagination)>] <json(query)>
 ```
 
 ```javascript
-'/_get chats ' + userId + (pendingConnections ? ' pcc=on' : '') + ' ' + PaginationByTime.cmdString(pagination) + ' ' + JSON.stringify(query) // JavaScript
+'/_get chats ' + userId + (pendingConnections ? ' pcc=on' : '') + (pagination ? ' ' + PaginationByTime.cmdString(pagination) : '') + ' ' + JSON.stringify(query) // JavaScript
 ```
 
 ```python
-'/_get chats ' + str(userId) + (' pcc=on' if pendingConnections else '') + ' ' + PaginationByTime_cmd_string(pagination) + ' ' + json.dumps(query) # Python
+'/_get chats ' + str(userId) + (' pcc=on' if pendingConnections else '') + ((' ' + PaginationByTime_cmd_string(pagination)) if pagination is not None else '') + ' ' + json.dumps(query) # Python
 ```
 
 **Responses**:

bots/src/API/Docs/Commands.hs

@@ -150,7 +150,7 @@ chatCommandsDocsData =
       "Commands to list and delete conversations.",
       [ ("APIListContacts", [], "Get contacts.", ["CRContactsList", "CRChatCmdError"], [], Nothing, "/_contacts " <> Param "userId"),
         ("APIListGroups", [], "Get groups.", ["CRGroupsList", "CRChatCmdError"], [], Nothing, "/_groups " <> Param "userId" <> Optional "" (" @" <> Param "$0") "contactId_" <> Optional "" (" " <> Param "$0") "search"),
-        ("APIGetChats", [], "Get chat previews. Supports time-based pagination — use this instead of APIListContacts / APIListGroups when scanning at scale (those load every record into memory and fail on large databases).", ["CRApiChats", "CRChatCmdError"], [], Nothing, "/_get chats " <> Param "userId" <> OnOffParam "pcc" "pendingConnections" (Just False) <> " " <> Param "pagination" <> " " <> Json "query"),
+        ("APIGetChats", [], "Get chat previews. Supports time-based pagination — use this instead of APIListContacts / APIListGroups when scanning at scale (those load every record into memory and fail on large databases).", ["CRApiChats", "CRChatCmdError"], [], Nothing, "/_get chats " <> Param "userId" <> OnOffParam "pcc" "pendingConnections" (Just False) <> Optional "" (" " <> Param "$0") "pagination" <> " " <> Json "query"),
         ("APIDeleteChat", [], "Delete chat.", ["CRContactDeleted", "CRContactConnectionDeleted", "CRGroupDeletedUser", "CRChatCmdError"], [], Just UNBackground, "/_delete " <> Param "chatRef" <> " " <> Param "chatDeleteMode"),
         ("APISetGroupCustomData", [], "Set group custom data.", ["CRCmdOk", "CRChatCmdError"], [], Nothing, "/_set custom #" <> Param "groupId" <> Optional "" (" " <> Json "$0") "customData"),
         ("APISetContactCustomData", [], "Set contact custom data.", ["CRCmdOk", "CRChatCmdError"], [], Nothing, "/_set custom @" <> Param "contactId" <> Optional "" (" " <> Json "$0") "customData"),

packages/simplex-chat-client/types/typescript/src/commands.ts

@@ -664,7 +664,7 @@ export namespace APIListGroups {
 export interface APIGetChats {
   userId: number // int64
   pendingConnections: boolean
-  pagination: T.PaginationByTime
+  pagination?: T.PaginationByTime
   query: T.ChatListQuery
 }
 
@@ -672,7 +672,7 @@ export namespace APIGetChats {
   export type Response = CR.ApiChats | CR.ChatCmdError
 
   export function cmdString(self: APIGetChats): string {
-    return '/_get chats ' + self.userId + (self.pendingConnections ? ' pcc=on' : '') + ' ' + T.PaginationByTime.cmdString(self.pagination) + ' ' + JSON.stringify(self.query)
+    return '/_get chats ' + self.userId + (self.pendingConnections ? ' pcc=on' : '') + (self.pagination ? ' ' + T.PaginationByTime.cmdString(self.pagination) : '') + ' ' + JSON.stringify(self.query)
   }
 }
 

packages/simplex-chat-python/src/simplex_chat/types/_commands.py

@@ -584,12 +584,12 @@ APIListGroups_Response = CR.GroupsList | CR.ChatCmdError
 class APIGetChats(TypedDict):
     userId: int  # int64
     pendingConnections: bool
-    pagination: "T.PaginationByTime"
+    pagination: NotRequired["T.PaginationByTime"]
     query: "T.ChatListQuery"
 
 
 def APIGetChats_cmd_string(self: APIGetChats) -> str:
-    return '/_get chats ' + str(self['userId']) + (' pcc=on' if self['pendingConnections'] else '') + ' ' + T.PaginationByTime_cmd_string(self['pagination']) + ' ' + json.dumps(self['query'])
+    return '/_get chats ' + str(self['userId']) + (' pcc=on' if self['pendingConnections'] else '') + ((' ' + T.PaginationByTime_cmd_string(self.get('pagination'))) if self.get('pagination') is not None else '') + ' ' + json.dumps(self['query'])
 
 APIGetChats_Response = CR.ApiChats | CR.ChatCmdError
 

src/Simplex/Chat.hs

@@ -102,6 +102,7 @@ defaultChatConfig =
       shortLinkPresetServers = allPresetServers,
       presetDomains = [".simplex.im", ".simplexonflux.com"],
       tbqSize = 1024,
+      maxChats = 5000,
       fileChunkSize = 15780, -- do not change
       xftpDescrPartSize = 14000,
       inlineFiles = defaultInlineFilesConfig,
@@ -147,11 +148,11 @@ newChatController
   ChatDatabase {chatStore, agentStore}
   user
   cfg@ChatConfig {agentConfig = aCfg, presetServers, inlineFiles, deviceNameForRemote, confirmMigrations}
-  ChatOpts {coreOptions = CoreChatOpts {smpServers, xftpServers, simpleNetCfg, logLevel, logConnections, logServerHosts, logFile, tbqSize, deviceName, webPreviewConfig, highlyAvailable, yesToUpMigrations}, optFilesFolder, optTempDirectory, showReactions, showFullLinks, allowInstantFiles, autoAcceptFileSize}
+  ChatOpts {coreOptions = CoreChatOpts {smpServers, xftpServers, simpleNetCfg, logLevel, logConnections, logServerHosts, logFile, tbqSize, maxChats, deviceName, webPreviewConfig, highlyAvailable, yesToUpMigrations}, optFilesFolder, optTempDirectory, showReactions, showFullLinks, allowInstantFiles, autoAcceptFileSize}
   backgroundMode = do
     let inlineFiles' = if allowInstantFiles || autoAcceptFileSize > 0 then inlineFiles else inlineFiles {sendChunks = 0, receiveInstant = False}
         confirmMigrations' = if confirmMigrations == MCConsole && yesToUpMigrations then MCYesUp else confirmMigrations
-        config = cfg {logLevel, showReactions, showFullLinks, tbqSize, subscriptionEvents = logConnections, hostEvents = logServerHosts, presetServers = presetServers', inlineFiles = inlineFiles', autoAcceptFileSize, webPreviewConfig, highlyAvailable, confirmMigrations = confirmMigrations'}
+        config = cfg {logLevel, showReactions, showFullLinks, tbqSize, maxChats, subscriptionEvents = logConnections, hostEvents = logServerHosts, presetServers = presetServers', inlineFiles = inlineFiles', autoAcceptFileSize, webPreviewConfig, highlyAvailable, confirmMigrations = confirmMigrations'}
     randomPresetServers <- chooseRandomServers presetServers'
     let rndSrvs = L.toList randomPresetServers
         operatorWithId (i, op) = (\o -> o {operatorId = DBEntityId i}) <$> pOperator op

src/Simplex/Chat/Controller.hs

@@ -148,6 +148,7 @@ data ChatConfig = ChatConfig
     shortLinkPresetServers :: NonEmpty SMPServer,
     presetDomains :: [HostName],
     tbqSize :: Natural,
+    maxChats :: Int,
     fileChunkSize :: Integer,
     xftpDescrPartSize :: Int,
     inlineFiles :: InlineFilesConfig,
@@ -381,7 +382,7 @@ data ChatCommand
   | APISaveAppSettings AppSettings
   | APIGetAppSettings (Maybe AppSettings)
   | APIGetChatTags UserId
-  | APIGetChats {userId :: UserId, pendingConnections :: Bool, pagination :: PaginationByTime, query :: ChatListQuery}
+  | APIGetChats {userId :: UserId, pendingConnections :: Bool, pagination :: Maybe PaginationByTime, query :: ChatListQuery}
   | APIGetChat {chatRef :: ChatRef, contentTag :: Maybe MsgContentTag, chatPagination :: ChatPagination, search :: Maybe Text}
   | APIGetChatContentTypes ChatRef
   | APIGetChatItems {chatPagination :: ChatPagination, search :: Maybe Text}

src/Simplex/Chat/Library/Commands.hs

@@ -652,7 +652,9 @@ processChatCommand cxt nm = \case
     tags <- withFastStore' (`getUserChatTags` user)
     pure $ CRChatTags user tags
   APIGetChats {userId, pendingConnections, pagination, query} -> withUserId' userId $ \user -> do
-    (errs, previews) <- partitionEithers <$> withFastStore' (\db -> getChatPreviews db cxt user pendingConnections pagination query)
+    ChatConfig {maxChats} <- asks config
+    let pagination' = fromMaybe (PTLast maxChats) pagination
+    (errs, previews) <- partitionEithers <$> withFastStore' (\db -> getChatPreviews db cxt user pendingConnections pagination' query)
     unless (null errs) $ toView $ CEvtChatErrors (map ChatErrorStore errs)
     pure $ CRApiChats user previews
   APIGetChat (ChatRef cType cId scope_) contentFilter pagination search -> withUser $ \user -> case cType of
@@ -3431,7 +3433,8 @@ processChatCommand cxt nm = \case
     folderId <- withFastStore (`getUserNoteFolderId` user)
     processChatCommand cxt nm $ APIClearChat (ChatRef CTLocal folderId Nothing)
   LastChats count_ -> withUser' $ \user -> do
-    let count = fromMaybe 5000 count_
+    ChatConfig {maxChats} <- asks config
+    let count = fromMaybe maxChats count_
     (errs, previews) <- partitionEithers <$> withFastStore' (\db -> getChatPreviews db cxt user False (PTLast count) clqNoFilters)
     unless (null errs) $ toView $ CEvtChatErrors (map ChatErrorStore errs)
     pure $ CRChats previews
@@ -5517,7 +5520,7 @@ chatCommandP =
         *> ( APIGetChats
               <$> A.decimal
               <*> (" pcc=on" $> True <|> " pcc=off" $> False <|> pure False)
-              <*> (A.space *> paginationByTimeP <|> pure (PTLast 5000))
+              <*> optional (A.space *> paginationByTimeP)
               <*> (A.space *> jsonP <|> pure clqNoFilters)
            ),
       "/_get chat " *> (APIGetChat <$> chatRefP <*> optional (" content=" *> strP) <* A.space <*> chatPaginationP <*> optional (" search=" *> textP)),

src/Simplex/Chat/Mobile.hs

@@ -259,6 +259,7 @@ mobileChatOpts dbOptions =
             logAgent = Nothing,
             logFile = Nothing,
             tbqSize = 4096,
+            maxChats = 5000,
             deviceName = Nothing,
             chatRelay = False,
             webPreviewConfig = Nothing,

src/Simplex/Chat/Options.hs

@@ -67,6 +67,7 @@ data CoreChatOpts = CoreChatOpts
     logAgent :: Maybe LogLevel,
     logFile :: Maybe FilePath,
     tbqSize :: Natural,
+    maxChats :: Int,
     deviceName :: Maybe Text,
     chatRelay :: Bool,
     webPreviewConfig :: Maybe WebPreviewConfig,
@@ -234,6 +235,15 @@ coreChatOptsP appDir defaultDbName = do
           <> value 1024
           <> showDefault
       )
+  maxChats <-
+    option
+      auto
+      ( long "max-chats"
+          <> metavar "COUNT"
+          <> help "Max number of chats loaded by chat list API"
+          <> value 5000
+          <> showDefault
+      )
   deviceName <-
     optional $
       strOption
@@ -340,6 +350,7 @@ coreChatOptsP appDir defaultDbName = do
         logAgent = if logAgent || logLevel == CLLDebug then Just $ agentLogLevel logLevel else Nothing,
         logFile,
         tbqSize,
+        maxChats,
         deviceName,
         chatRelay,
         webPreviewConfig,

tests/ChatClient.hs

@@ -157,6 +157,7 @@ testCoreOpts =
       logAgent = Nothing,
       logFile = Nothing,
       tbqSize = 16,
+      maxChats = 5000,
       deviceName = Nothing,
       chatRelay = False,
       webPreviewConfig = Nothing,

tests/ChatTests/ChatList.hs

@@ -5,11 +5,13 @@ import ChatTests.DBUtils
 import ChatTests.Utils
 import Data.Time.Clock (getCurrentTime)
 import Data.Time.Format.ISO8601 (iso8601Show)
+import Simplex.Chat.Options (ChatOpts (..), CoreChatOpts (..))
 import Test.Hspec hiding (it)
 
 chatListTests :: SpecWith TestParams
 chatListTests = do
   it "get last chats" testPaginationLast
+  it "get last chats with max chats option" testMaxChats
   it "get chats before/after timestamp" testPaginationTs
   it "filter by search query" testFilterSearch
   it "filter favorite" testFilterFavorite
@@ -33,6 +35,23 @@ testPaginationLast =
       alice <# "bob> hey"
       alice <# "@cath hey"
 
+testMaxChats :: HasCallStack => TestParams -> IO ()
+testMaxChats =
+  testChatOpts3 testOpts {coreOptions = testCoreOpts {maxChats = 1}} aliceProfile bobProfile cathProfile $
+    \alice bob cath -> do
+      connectUsers alice bob
+      alice <##> bob
+      connectUsers alice cath
+      cath <##> alice
+
+      alice ##> "/chats all"
+      alice <# "@cath hey"
+      alice ##> "/chats 2"
+      alice <# "bob> hey"
+      alice <# "@cath hey"
+      alice #$> ("/_get chats 1 pcc=on", chats, [("@cath", "hey")])
+      getChats_ alice "count=2" [("@cath", "hey"), ("@bob", "hey")]
+
 testPaginationTs :: HasCallStack => TestParams -> IO ()
 testPaginationTs =
   testChat3 aliceProfile bobProfile cathProfile $