diff --git a/bots/api/COMMANDS.md b/bots/api/COMMANDS.md index 42b3da2373..5f49b26653 100644 --- a/bots/api/COMMANDS.md +++ b/bots/api/COMMANDS.md @@ -1817,21 +1817,21 @@ Get chat previews. Supports time-based pagination — use this instead of APILis **Parameters**: - userId: int64 - pendingConnections: bool -- pagination: [PaginationByTime](./TYPES.md#paginationbytime) +- pagination: [PaginationByTime](./TYPES.md#paginationbytime)? - query: [ChatListQuery](./TYPES.md#chatlistquery) **Syntax**: ``` -/_get chats [ pcc=on] +/_get chats [ pcc=on][ ] ``` ```javascript -'/_get chats ' + userId + (pendingConnections ? ' pcc=on' : '') + ' ' + PaginationByTime.cmdString(pagination) + ' ' + JSON.stringify(query) // JavaScript +'/_get chats ' + userId + (pendingConnections ? ' pcc=on' : '') + (pagination ? ' ' + PaginationByTime.cmdString(pagination) : '') + ' ' + JSON.stringify(query) // JavaScript ``` ```python -'/_get chats ' + str(userId) + (' pcc=on' if pendingConnections else '') + ' ' + PaginationByTime_cmd_string(pagination) + ' ' + json.dumps(query) # Python +'/_get chats ' + str(userId) + (' pcc=on' if pendingConnections else '') + ((' ' + PaginationByTime_cmd_string(pagination)) if pagination is not None else '') + ' ' + json.dumps(query) # Python ``` **Responses**: diff --git a/bots/src/API/Docs/Commands.hs b/bots/src/API/Docs/Commands.hs index 782d0d21cd..93a44785ee 100644 --- a/bots/src/API/Docs/Commands.hs +++ b/bots/src/API/Docs/Commands.hs @@ -150,7 +150,7 @@ chatCommandsDocsData = "Commands to list and delete conversations.", [ ("APIListContacts", [], "Get contacts.", ["CRContactsList", "CRChatCmdError"], [], Nothing, "/_contacts " <> Param "userId"), ("APIListGroups", [], "Get groups.", ["CRGroupsList", "CRChatCmdError"], [], Nothing, "/_groups " <> Param "userId" <> Optional "" (" @" <> Param "$0") "contactId_" <> Optional "" (" " <> Param "$0") "search"), - ("APIGetChats", [], "Get chat previews. Supports time-based pagination — use this instead of APIListContacts / APIListGroups when scanning at scale (those load every record into memory and fail on large databases).", ["CRApiChats", "CRChatCmdError"], [], Nothing, "/_get chats " <> Param "userId" <> OnOffParam "pcc" "pendingConnections" (Just False) <> " " <> Param "pagination" <> " " <> Json "query"), + ("APIGetChats", [], "Get chat previews. Supports time-based pagination — use this instead of APIListContacts / APIListGroups when scanning at scale (those load every record into memory and fail on large databases).", ["CRApiChats", "CRChatCmdError"], [], Nothing, "/_get chats " <> Param "userId" <> OnOffParam "pcc" "pendingConnections" (Just False) <> Optional "" (" " <> Param "$0") "pagination" <> " " <> Json "query"), ("APIDeleteChat", [], "Delete chat.", ["CRContactDeleted", "CRContactConnectionDeleted", "CRGroupDeletedUser", "CRChatCmdError"], [], Just UNBackground, "/_delete " <> Param "chatRef" <> " " <> Param "chatDeleteMode"), ("APISetGroupCustomData", [], "Set group custom data.", ["CRCmdOk", "CRChatCmdError"], [], Nothing, "/_set custom #" <> Param "groupId" <> Optional "" (" " <> Json "$0") "customData"), ("APISetContactCustomData", [], "Set contact custom data.", ["CRCmdOk", "CRChatCmdError"], [], Nothing, "/_set custom @" <> Param "contactId" <> Optional "" (" " <> Json "$0") "customData"), diff --git a/packages/simplex-chat-client/types/typescript/src/commands.ts b/packages/simplex-chat-client/types/typescript/src/commands.ts index dcc9ec767d..fbde8ce9ac 100644 --- a/packages/simplex-chat-client/types/typescript/src/commands.ts +++ b/packages/simplex-chat-client/types/typescript/src/commands.ts @@ -664,7 +664,7 @@ export namespace APIListGroups { export interface APIGetChats { userId: number // int64 pendingConnections: boolean - pagination: T.PaginationByTime + pagination?: T.PaginationByTime query: T.ChatListQuery } @@ -672,7 +672,7 @@ export namespace APIGetChats { export type Response = CR.ApiChats | CR.ChatCmdError export function cmdString(self: APIGetChats): string { - return '/_get chats ' + self.userId + (self.pendingConnections ? ' pcc=on' : '') + ' ' + T.PaginationByTime.cmdString(self.pagination) + ' ' + JSON.stringify(self.query) + return '/_get chats ' + self.userId + (self.pendingConnections ? ' pcc=on' : '') + (self.pagination ? ' ' + T.PaginationByTime.cmdString(self.pagination) : '') + ' ' + JSON.stringify(self.query) } } diff --git a/packages/simplex-chat-python/src/simplex_chat/types/_commands.py b/packages/simplex-chat-python/src/simplex_chat/types/_commands.py index 53544324fc..fd4f561c3d 100644 --- a/packages/simplex-chat-python/src/simplex_chat/types/_commands.py +++ b/packages/simplex-chat-python/src/simplex_chat/types/_commands.py @@ -584,12 +584,12 @@ APIListGroups_Response = CR.GroupsList | CR.ChatCmdError class APIGetChats(TypedDict): userId: int # int64 pendingConnections: bool - pagination: "T.PaginationByTime" + pagination: NotRequired["T.PaginationByTime"] query: "T.ChatListQuery" def APIGetChats_cmd_string(self: APIGetChats) -> str: - return '/_get chats ' + str(self['userId']) + (' pcc=on' if self['pendingConnections'] else '') + ' ' + T.PaginationByTime_cmd_string(self['pagination']) + ' ' + json.dumps(self['query']) + return '/_get chats ' + str(self['userId']) + (' pcc=on' if self['pendingConnections'] else '') + ((' ' + T.PaginationByTime_cmd_string(self.get('pagination'))) if self.get('pagination') is not None else '') + ' ' + json.dumps(self['query']) APIGetChats_Response = CR.ApiChats | CR.ChatCmdError diff --git a/src/Simplex/Chat.hs b/src/Simplex/Chat.hs index 85df305f36..39342d59f2 100644 --- a/src/Simplex/Chat.hs +++ b/src/Simplex/Chat.hs @@ -102,6 +102,7 @@ defaultChatConfig = shortLinkPresetServers = allPresetServers, presetDomains = [".simplex.im", ".simplexonflux.com"], tbqSize = 1024, + maxChats = 5000, fileChunkSize = 15780, -- do not change xftpDescrPartSize = 14000, inlineFiles = defaultInlineFilesConfig, @@ -147,11 +148,11 @@ newChatController ChatDatabase {chatStore, agentStore} user cfg@ChatConfig {agentConfig = aCfg, presetServers, inlineFiles, deviceNameForRemote, confirmMigrations} - ChatOpts {coreOptions = CoreChatOpts {smpServers, xftpServers, simpleNetCfg, logLevel, logConnections, logServerHosts, logFile, tbqSize, deviceName, webPreviewConfig, highlyAvailable, yesToUpMigrations}, optFilesFolder, optTempDirectory, showReactions, showFullLinks, allowInstantFiles, autoAcceptFileSize} + ChatOpts {coreOptions = CoreChatOpts {smpServers, xftpServers, simpleNetCfg, logLevel, logConnections, logServerHosts, logFile, tbqSize, maxChats, deviceName, webPreviewConfig, highlyAvailable, yesToUpMigrations}, optFilesFolder, optTempDirectory, showReactions, showFullLinks, allowInstantFiles, autoAcceptFileSize} backgroundMode = do let inlineFiles' = if allowInstantFiles || autoAcceptFileSize > 0 then inlineFiles else inlineFiles {sendChunks = 0, receiveInstant = False} confirmMigrations' = if confirmMigrations == MCConsole && yesToUpMigrations then MCYesUp else confirmMigrations - config = cfg {logLevel, showReactions, showFullLinks, tbqSize, subscriptionEvents = logConnections, hostEvents = logServerHosts, presetServers = presetServers', inlineFiles = inlineFiles', autoAcceptFileSize, webPreviewConfig, highlyAvailable, confirmMigrations = confirmMigrations'} + config = cfg {logLevel, showReactions, showFullLinks, tbqSize, maxChats, subscriptionEvents = logConnections, hostEvents = logServerHosts, presetServers = presetServers', inlineFiles = inlineFiles', autoAcceptFileSize, webPreviewConfig, highlyAvailable, confirmMigrations = confirmMigrations'} randomPresetServers <- chooseRandomServers presetServers' let rndSrvs = L.toList randomPresetServers operatorWithId (i, op) = (\o -> o {operatorId = DBEntityId i}) <$> pOperator op diff --git a/src/Simplex/Chat/Controller.hs b/src/Simplex/Chat/Controller.hs index dc4f86472c..4be017bf9e 100644 --- a/src/Simplex/Chat/Controller.hs +++ b/src/Simplex/Chat/Controller.hs @@ -148,6 +148,7 @@ data ChatConfig = ChatConfig shortLinkPresetServers :: NonEmpty SMPServer, presetDomains :: [HostName], tbqSize :: Natural, + maxChats :: Int, fileChunkSize :: Integer, xftpDescrPartSize :: Int, inlineFiles :: InlineFilesConfig, @@ -381,7 +382,7 @@ data ChatCommand | APISaveAppSettings AppSettings | APIGetAppSettings (Maybe AppSettings) | APIGetChatTags UserId - | APIGetChats {userId :: UserId, pendingConnections :: Bool, pagination :: PaginationByTime, query :: ChatListQuery} + | APIGetChats {userId :: UserId, pendingConnections :: Bool, pagination :: Maybe PaginationByTime, query :: ChatListQuery} | APIGetChat {chatRef :: ChatRef, contentTag :: Maybe MsgContentTag, chatPagination :: ChatPagination, search :: Maybe Text} | APIGetChatContentTypes ChatRef | APIGetChatItems {chatPagination :: ChatPagination, search :: Maybe Text} diff --git a/src/Simplex/Chat/Library/Commands.hs b/src/Simplex/Chat/Library/Commands.hs index 2d65a1677a..01fc4b55ff 100644 --- a/src/Simplex/Chat/Library/Commands.hs +++ b/src/Simplex/Chat/Library/Commands.hs @@ -652,7 +652,9 @@ processChatCommand cxt nm = \case tags <- withFastStore' (`getUserChatTags` user) pure $ CRChatTags user tags APIGetChats {userId, pendingConnections, pagination, query} -> withUserId' userId $ \user -> do - (errs, previews) <- partitionEithers <$> withFastStore' (\db -> getChatPreviews db cxt user pendingConnections pagination query) + ChatConfig {maxChats} <- asks config + let pagination' = fromMaybe (PTLast maxChats) pagination + (errs, previews) <- partitionEithers <$> withFastStore' (\db -> getChatPreviews db cxt user pendingConnections pagination' query) unless (null errs) $ toView $ CEvtChatErrors (map ChatErrorStore errs) pure $ CRApiChats user previews APIGetChat (ChatRef cType cId scope_) contentFilter pagination search -> withUser $ \user -> case cType of @@ -3431,7 +3433,8 @@ processChatCommand cxt nm = \case folderId <- withFastStore (`getUserNoteFolderId` user) processChatCommand cxt nm $ APIClearChat (ChatRef CTLocal folderId Nothing) LastChats count_ -> withUser' $ \user -> do - let count = fromMaybe 5000 count_ + ChatConfig {maxChats} <- asks config + let count = fromMaybe maxChats count_ (errs, previews) <- partitionEithers <$> withFastStore' (\db -> getChatPreviews db cxt user False (PTLast count) clqNoFilters) unless (null errs) $ toView $ CEvtChatErrors (map ChatErrorStore errs) pure $ CRChats previews @@ -5517,7 +5520,7 @@ chatCommandP = *> ( APIGetChats <$> A.decimal <*> (" pcc=on" $> True <|> " pcc=off" $> False <|> pure False) - <*> (A.space *> paginationByTimeP <|> pure (PTLast 5000)) + <*> optional (A.space *> paginationByTimeP) <*> (A.space *> jsonP <|> pure clqNoFilters) ), "/_get chat " *> (APIGetChat <$> chatRefP <*> optional (" content=" *> strP) <* A.space <*> chatPaginationP <*> optional (" search=" *> textP)), diff --git a/src/Simplex/Chat/Mobile.hs b/src/Simplex/Chat/Mobile.hs index 09af069ef7..51e0409723 100644 --- a/src/Simplex/Chat/Mobile.hs +++ b/src/Simplex/Chat/Mobile.hs @@ -259,6 +259,7 @@ mobileChatOpts dbOptions = logAgent = Nothing, logFile = Nothing, tbqSize = 4096, + maxChats = 5000, deviceName = Nothing, chatRelay = False, webPreviewConfig = Nothing, diff --git a/src/Simplex/Chat/Options.hs b/src/Simplex/Chat/Options.hs index 4e47868fa0..d8bbb31606 100644 --- a/src/Simplex/Chat/Options.hs +++ b/src/Simplex/Chat/Options.hs @@ -67,6 +67,7 @@ data CoreChatOpts = CoreChatOpts logAgent :: Maybe LogLevel, logFile :: Maybe FilePath, tbqSize :: Natural, + maxChats :: Int, deviceName :: Maybe Text, chatRelay :: Bool, webPreviewConfig :: Maybe WebPreviewConfig, @@ -234,6 +235,15 @@ coreChatOptsP appDir defaultDbName = do <> value 1024 <> showDefault ) + maxChats <- + option + auto + ( long "max-chats" + <> metavar "COUNT" + <> help "Max number of chats loaded by chat list API" + <> value 5000 + <> showDefault + ) deviceName <- optional $ strOption @@ -340,6 +350,7 @@ coreChatOptsP appDir defaultDbName = do logAgent = if logAgent || logLevel == CLLDebug then Just $ agentLogLevel logLevel else Nothing, logFile, tbqSize, + maxChats, deviceName, chatRelay, webPreviewConfig, diff --git a/tests/ChatClient.hs b/tests/ChatClient.hs index 4f322c82d9..0a7914d366 100644 --- a/tests/ChatClient.hs +++ b/tests/ChatClient.hs @@ -157,6 +157,7 @@ testCoreOpts = logAgent = Nothing, logFile = Nothing, tbqSize = 16, + maxChats = 5000, deviceName = Nothing, chatRelay = False, webPreviewConfig = Nothing, diff --git a/tests/ChatTests/ChatList.hs b/tests/ChatTests/ChatList.hs index 889915a6e8..07b338c99f 100644 --- a/tests/ChatTests/ChatList.hs +++ b/tests/ChatTests/ChatList.hs @@ -5,11 +5,13 @@ import ChatTests.DBUtils import ChatTests.Utils import Data.Time.Clock (getCurrentTime) import Data.Time.Format.ISO8601 (iso8601Show) +import Simplex.Chat.Options (ChatOpts (..), CoreChatOpts (..)) import Test.Hspec hiding (it) chatListTests :: SpecWith TestParams chatListTests = do it "get last chats" testPaginationLast + it "get last chats with max chats option" testMaxChats it "get chats before/after timestamp" testPaginationTs it "filter by search query" testFilterSearch it "filter favorite" testFilterFavorite @@ -33,6 +35,23 @@ testPaginationLast = alice <# "bob> hey" alice <# "@cath hey" +testMaxChats :: HasCallStack => TestParams -> IO () +testMaxChats = + testChatOpts3 testOpts {coreOptions = testCoreOpts {maxChats = 1}} aliceProfile bobProfile cathProfile $ + \alice bob cath -> do + connectUsers alice bob + alice <##> bob + connectUsers alice cath + cath <##> alice + + alice ##> "/chats all" + alice <# "@cath hey" + alice ##> "/chats 2" + alice <# "bob> hey" + alice <# "@cath hey" + alice #$> ("/_get chats 1 pcc=on", chats, [("@cath", "hey")]) + getChats_ alice "count=2" [("@cath", "hey"), ("@bob", "hey")] + testPaginationTs :: HasCallStack => TestParams -> IO () testPaginationTs = testChat3 aliceProfile bobProfile cathProfile $