diff --git a/bots/src/API/Docs/Commands.hs b/bots/src/API/Docs/Commands.hs index 4cce44e588..41e8cdf019 100644 --- a/bots/src/API/Docs/Commands.hs +++ b/bots/src/API/Docs/Commands.hs @@ -340,6 +340,7 @@ undocumentedCommands = "APIGetAppSettings", "APIGetCallInvitations", "APIGetChat", + "APIGetChatContentTypes", "APIGetChatItemInfo", "APIGetChatItems", "APIGetChatItemTTL", diff --git a/bots/src/API/Docs/Responses.hs b/bots/src/API/Docs/Responses.hs index 154d44b6c2..c52b288603 100644 --- a/bots/src/API/Docs/Responses.hs +++ b/bots/src/API/Docs/Responses.hs @@ -121,6 +121,7 @@ undocumentedResponses = "CRBroadcastSent", "CRCallInvitations", "CRChatCleared", + "CRChatContentTypes", "CRChatHelp", "CRChatItemId", "CRChatItemInfo", diff --git a/simplex-chat.cabal b/simplex-chat.cabal index 95f386e9df..c021023e49 100644 --- a/simplex-chat.cabal +++ b/simplex-chat.cabal @@ -125,6 +125,7 @@ library Simplex.Chat.Store.Postgres.Migrations.M20251117_member_relations_vector Simplex.Chat.Store.Postgres.Migrations.M20251128_migrate_member_relations Simplex.Chat.Store.Postgres.Migrations.M20251230_strict_tables + Simplex.Chat.Store.Postgres.Migrations.M20260108_chat_indices else exposed-modules: Simplex.Chat.Archive @@ -273,6 +274,7 @@ library Simplex.Chat.Store.SQLite.Migrations.M20251117_member_relations_vector Simplex.Chat.Store.SQLite.Migrations.M20251128_migrate_member_relations Simplex.Chat.Store.SQLite.Migrations.M20251230_strict_tables + Simplex.Chat.Store.SQLite.Migrations.M20260108_chat_indices other-modules: Paths_simplex_chat hs-source-dirs: diff --git a/src/Simplex/Chat/Controller.hs b/src/Simplex/Chat/Controller.hs index 30df251fbb..9703226e1a 100644 --- a/src/Simplex/Chat/Controller.hs +++ b/src/Simplex/Chat/Controller.hs @@ -316,6 +316,7 @@ data ChatCommand | APIGetChatTags UserId | APIGetChats {userId :: UserId, pendingConnections :: Bool, pagination :: PaginationByTime, query :: ChatListQuery} | APIGetChat {chatRef :: ChatRef, contentTag :: Maybe MsgContentTag, chatPagination :: ChatPagination, search :: Maybe Text} + | APIGetChatContentTypes ChatRef | APIGetChatItems {chatPagination :: ChatPagination, search :: Maybe Text} | APIGetChatItemInfo {chatRef :: ChatRef, chatItemId :: ChatItemId} | APISendMessages {sendRef :: SendRef, liveMessage :: Bool, ttl :: Maybe Int, composedMessages :: NonEmpty ComposedMessage} @@ -637,6 +638,7 @@ data ChatResponse | CRApiChats {user :: User, chats :: [AChat]} | CRChats {chats :: [AChat]} | CRApiChat {user :: User, chat :: AChat, navInfo :: Maybe NavigationInfo} + | CRChatContentTypes {contentTypes :: [MsgContentTag]} | CRChatTags {user :: User, userTags :: [ChatTag]} | CRChatItems {user :: User, chatName_ :: Maybe ChatName, chatItems :: [AChatItem]} | CRChatItemInfo {user :: User, chatItem :: AChatItem, chatItemInfo :: ChatItemInfo} diff --git a/src/Simplex/Chat/Library/Commands.hs b/src/Simplex/Chat/Library/Commands.hs index 675ca03a8a..994b435297 100644 --- a/src/Simplex/Chat/Library/Commands.hs +++ b/src/Simplex/Chat/Library/Commands.hs @@ -535,16 +535,14 @@ processChatCommand vr nm = \case APIGetChat (ChatRef cType cId scope_) contentFilter pagination search -> withUser $ \user -> case cType of -- TODO optimize queries calculating ChatStats, currently they're disabled CTDirect -> do - when (isJust contentFilter) $ throwCmdError "content filter not supported" - (directChat, navInfo) <- withFastStore (\db -> getDirectChat db vr user cId pagination search) + (directChat, navInfo) <- withFastStore (\db -> getDirectChat db vr user cId contentFilter pagination search) pure $ CRApiChat user (AChat SCTDirect directChat) navInfo CTGroup -> do (groupChat, navInfo) <- withFastStore (\db -> getGroupChat db vr user cId scope_ contentFilter pagination search) groupChat' <- checkSupportChatAttention user groupChat pure $ CRApiChat user (AChat SCTGroup groupChat') navInfo CTLocal -> do - when (isJust contentFilter) $ throwCmdError "content filter not supported" - (localChat, navInfo) <- withFastStore (\db -> getLocalChat db user cId pagination search) + (localChat, navInfo) <- withFastStore (\db -> getLocalChat db user cId contentFilter pagination search) pure $ CRApiChat user (AChat SCTLocal localChat) navInfo CTContactRequest -> throwCmdError "not implemented" CTContactConnection -> throwCmdError "not supported" @@ -570,6 +568,8 @@ processChatCommand vr nm = \case newFromMember (CChatItem _ ChatItem {chatDir = CIGroupRcv m, meta = CIMeta {itemStatus = CISRcvNew}}) = groupMemberId' m == scopeGMId newFromMember _ = False + APIGetChatContentTypes chatRef -> withUser $ \user -> + CRChatContentTypes <$> withStore (\db -> getChatContentTypes db user chatRef) APIGetChatItems pagination search -> withUser $ \user -> do chatItems <- withFastStore $ \db -> getAllChatItems db vr user pagination search pure $ CRChatItems user Nothing chatItems @@ -4368,6 +4368,7 @@ chatCommandP = <*> (A.space *> jsonP <|> pure clqNoFilters) ), "/_get chat " *> (APIGetChat <$> chatRefP <*> optional (" content=" *> strP) <* A.space <*> chatPaginationP <*> optional (" search=" *> textP)), + "/_get content types " *> (APIGetChatContentTypes <$> chatRefP), "/_get items " *> (APIGetChatItems <$> chatPaginationP <*> optional (" search=" *> textP)), "/_get item info " *> (APIGetChatItemInfo <$> chatRefP <* A.space <*> A.decimal), "/_send " *> (APISendMessages <$> sendRefP <*> liveMessageP <*> sendMessageTTLP <*> (" json " *> jsonP <|> " text " *> composedMessagesTextP)), diff --git a/src/Simplex/Chat/Protocol.hs b/src/Simplex/Chat/Protocol.hs index 2aafad94a1..40b667365e 100644 --- a/src/Simplex/Chat/Protocol.hs +++ b/src/Simplex/Chat/Protocol.hs @@ -518,6 +518,8 @@ instance ToJSON MsgContentTag where toJSON = strToJSON toEncoding = strToJEncoding +instance FromField MsgContentTag where fromField = fromTextField_ $ eitherToMaybe . strDecode . encodeUtf8 + instance ToField MsgContentTag where toField = toField . safeDecodeUtf8 . strEncode data MsgContainer diff --git a/src/Simplex/Chat/Store/Groups.hs b/src/Simplex/Chat/Store/Groups.hs index fadc65960b..5721efb65e 100644 --- a/src/Simplex/Chat/Store/Groups.hs +++ b/src/Simplex/Chat/Store/Groups.hs @@ -184,7 +184,6 @@ import Simplex.Messaging.Util (eitherToMaybe, firstRow', safeDecodeUtf8, ($>>=), import Simplex.Messaging.Version import UnliftIO.STM #if defined(dbPostgres) -import qualified Data.Set as S import Database.PostgreSQL.Simple (In (..), Only (..), Query, (:.) (..)) import Database.PostgreSQL.Simple.SqlQQ (sql) #else diff --git a/src/Simplex/Chat/Store/Messages.hs b/src/Simplex/Chat/Store/Messages.hs index e3b6911b59..71e8a35386 100644 --- a/src/Simplex/Chat/Store/Messages.hs +++ b/src/Simplex/Chat/Store/Messages.hs @@ -45,6 +45,7 @@ module Simplex.Chat.Store.Messages createNewChatItem_, getChatPreviews, checkContactHasItems, + getChatContentTypes, getDirectChat, getGroupChat, getGroupChatScopeInfoForItem, @@ -1166,40 +1167,44 @@ checkContactHasItems db User {userId} Contact {contactId} = "SELECT EXISTS (SELECT 1 FROM chat_items WHERE user_id = ? AND contact_id = ?)" (userId, contactId) -getDirectChat :: DB.Connection -> VersionRangeChat -> User -> Int64 -> ChatPagination -> Maybe Text -> ExceptT StoreError IO (Chat 'CTDirect, Maybe NavigationInfo) -getDirectChat db vr user contactId pagination search_ = do +getChatContentTypes :: DB.Connection -> User -> ChatRef -> ExceptT StoreError IO [MsgContentTag] +getChatContentTypes db User {userId} (ChatRef cType chatId chatScope_) = case cType of + CTDirect -> getTypes " contact_id = ? " () + CTLocal -> getTypes " note_folder_id = ? " () + CTGroup -> case chatScope_ of + Nothing -> getTypes " group_id = ? AND group_scope_tag IS NULL AND group_scope_group_member_id IS NULL " () + Just (GCSMemberSupport mId_) -> getTypes " group_id = ? AND group_scope_tag = ? AND group_scope_group_member_id IS NOT DISTINCT FROM ? " (GCSTMemberSupport_, mId_) + _ -> throwError $ SEInternalError "unsupported chat type" + where + getTypes :: ToRow p => Query -> p -> ExceptT StoreError IO [MsgContentTag] + getTypes cond params = + liftIO $ map fromOnly + <$> DB.query + db + ("SELECT DISTINCT msg_content_tag FROM chat_items WHERE user_id = ? AND " <> cond <> " AND msg_content_tag IS NOT NULL ORDER BY msg_content_tag") + ((userId, chatId) :. params) + +getDirectChat :: DB.Connection -> VersionRangeChat -> User -> Int64 -> Maybe MsgContentTag -> ChatPagination -> Maybe Text -> ExceptT StoreError IO (Chat 'CTDirect, Maybe NavigationInfo) +getDirectChat db vr user contactId contentFilter pagination search_ = do let search = fromMaybe "" search_ ct <- getContact db vr user contactId case pagination of - CPLast count -> liftIO $ (,Nothing) <$> getDirectChatLast_ db user ct count search - CPAfter afterId count -> (,Nothing) <$> getDirectChatAfter_ db user ct afterId count search - CPBefore beforeId count -> (,Nothing) <$> getDirectChatBefore_ db user ct beforeId count search - CPAround aroundId count -> getDirectChatAround_ db user ct aroundId count search + CPLast count -> (,Nothing) <$> getDirectChatLast_ db user ct contentFilter count search + CPAfter afterId count -> (,Nothing) <$> getDirectChatAfter_ db user ct contentFilter afterId count search + CPBefore beforeId count -> (,Nothing) <$> getDirectChatBefore_ db user ct contentFilter beforeId count search + CPAround aroundId count -> getDirectChatAround_ db user ct contentFilter aroundId count search CPInitial count -> do unless (T.null search) $ throwError $ SEInternalError "initial chat pagination doesn't support search" - getDirectChatInitial_ db user ct count + getDirectChatInitial_ db user ct contentFilter count -- the last items in reverse order (the last item in the conversation is the first in the returned list) -getDirectChatLast_ :: DB.Connection -> User -> Contact -> Int -> Text -> IO (Chat 'CTDirect) -getDirectChatLast_ db user ct count search = do - ciIds <- getDirectChatItemIdsLast_ db user ct count search - ts <- getCurrentTime - cis <- mapM (safeGetDirectItem db user ct ts) ciIds - pure $ Chat (DirectChat ct) (reverse cis) emptyChatStats - -getDirectChatItemIdsLast_ :: DB.Connection -> User -> Contact -> Int -> Text -> IO [ChatItemId] -getDirectChatItemIdsLast_ db User {userId} Contact {contactId} count search = - map fromOnly - <$> DB.query - db - [sql| - SELECT chat_item_id - FROM chat_items - WHERE user_id = ? AND contact_id = ? AND LOWER(item_text) LIKE '%' || LOWER(?) || '%' - ORDER BY created_at DESC, chat_item_id DESC - LIMIT ? - |] - (userId, contactId, search, count) +getDirectChatLast_ :: DB.Connection -> User -> Contact -> Maybe MsgContentTag -> Int -> Text -> ExceptT StoreError IO (Chat 'CTDirect) +getDirectChatLast_ db user ct contentFilter count search = do + let cInfo = DirectChat ct + ciIds <- getChatItemIDs db user cInfo contentFilter CRLast count search + ts <- liftIO getCurrentTime + cis <- liftIO $ mapM (safeGetDirectItem db user ct ts) ciIds + pure $ Chat cInfo (reverse cis) emptyChatStats safeGetDirectItem :: DB.Connection -> User -> Contact -> UTCTime -> ChatItemId -> IO (CChatItem 'CTDirect) safeGetDirectItem db user ct currentTs itemId = @@ -1248,81 +1253,57 @@ getDirectChatItemLast db user@User {userId} contactId = do (userId, contactId) getDirectChatItem db user contactId chatItemId -getDirectChatAfter_ :: DB.Connection -> User -> Contact -> ChatItemId -> Int -> Text -> ExceptT StoreError IO (Chat 'CTDirect) -getDirectChatAfter_ db user ct@Contact {contactId} afterId count search = do +getDirectChatAfter_ :: DB.Connection -> User -> Contact -> Maybe MsgContentTag -> ChatItemId -> Int -> Text -> ExceptT StoreError IO (Chat 'CTDirect) +getDirectChatAfter_ db user ct@Contact {contactId} contentFilter afterId count search = do afterCI <- getDirectChatItem db user contactId afterId - ciIds <- liftIO $ getDirectCIsAfter_ db user ct afterCI count search + let cInfo = DirectChat ct + range = CRAfter (ciCreatedAt afterCI) (cChatItemId afterCI) + ciIds <- getChatItemIDs db user cInfo contentFilter range count search ts <- liftIO getCurrentTime cis <- liftIO $ mapM (safeGetDirectItem db user ct ts) ciIds - pure $ Chat (DirectChat ct) cis emptyChatStats + pure $ Chat cInfo cis emptyChatStats -getDirectCIsAfter_ :: DB.Connection -> User -> Contact -> CChatItem 'CTDirect -> Int -> Text -> IO [ChatItemId] -getDirectCIsAfter_ db User {userId} Contact {contactId} afterCI count search = - map fromOnly - <$> DB.query - db - [sql| - SELECT chat_item_id - FROM chat_items - WHERE user_id = ? AND contact_id = ? AND LOWER(item_text) LIKE '%' || LOWER(?) || '%' - AND (created_at > ? OR (created_at = ? AND chat_item_id > ?)) - ORDER BY created_at ASC, chat_item_id ASC - LIMIT ? - |] - (userId, contactId, search, ciCreatedAt afterCI, ciCreatedAt afterCI, cChatItemId afterCI, count) - -getDirectChatBefore_ :: DB.Connection -> User -> Contact -> ChatItemId -> Int -> Text -> ExceptT StoreError IO (Chat 'CTDirect) -getDirectChatBefore_ db user ct@Contact {contactId} beforeId count search = do +getDirectChatBefore_ :: DB.Connection -> User -> Contact -> Maybe MsgContentTag -> ChatItemId -> Int -> Text -> ExceptT StoreError IO (Chat 'CTDirect) +getDirectChatBefore_ db user ct@Contact {contactId} contentFilter beforeId count search = do beforeCI <- getDirectChatItem db user contactId beforeId - ciIds <- liftIO $ getDirectCIsBefore_ db user ct beforeCI count search + let cInfo = DirectChat ct + range = CRBefore (ciCreatedAt beforeCI) (cChatItemId beforeCI) + ciIds <- getChatItemIDs db user cInfo contentFilter range count search ts <- liftIO getCurrentTime cis <- liftIO $ mapM (safeGetDirectItem db user ct ts) ciIds - pure $ Chat (DirectChat ct) (reverse cis) emptyChatStats + pure $ Chat cInfo (reverse cis) emptyChatStats -getDirectCIsBefore_ :: DB.Connection -> User -> Contact -> CChatItem 'CTDirect -> Int -> Text -> IO [ChatItemId] -getDirectCIsBefore_ db User {userId} Contact {contactId} beforeCI count search = - map fromOnly - <$> DB.query - db - [sql| - SELECT chat_item_id - FROM chat_items - WHERE user_id = ? AND contact_id = ? AND LOWER(item_text) LIKE '%' || LOWER(?) || '%' - AND (created_at < ? OR (created_at = ? AND chat_item_id < ?)) - ORDER BY created_at DESC, chat_item_id DESC - LIMIT ? - |] - (userId, contactId, search, ciCreatedAt beforeCI, ciCreatedAt beforeCI, cChatItemId beforeCI, count) - -getDirectChatAround_ :: DB.Connection -> User -> Contact -> ChatItemId -> Int -> Text -> ExceptT StoreError IO (Chat 'CTDirect, Maybe NavigationInfo) -getDirectChatAround_ db user ct aroundId count search = do +getDirectChatAround_ :: DB.Connection -> User -> Contact -> Maybe MsgContentTag -> ChatItemId -> Int -> Text -> ExceptT StoreError IO (Chat 'CTDirect, Maybe NavigationInfo) +getDirectChatAround_ db user ct contentFilter aroundId count search = do stats <- liftIO $ getContactStats_ db user ct - getDirectChatAround' db user ct aroundId count search stats + getDirectChatAround' db user ct contentFilter aroundId count search stats -getDirectChatAround' :: DB.Connection -> User -> Contact -> ChatItemId -> Int -> Text -> ChatStats -> ExceptT StoreError IO (Chat 'CTDirect, Maybe NavigationInfo) -getDirectChatAround' db user ct@Contact {contactId} aroundId count search stats = do +getDirectChatAround' :: DB.Connection -> User -> Contact -> Maybe MsgContentTag -> ChatItemId -> Int -> Text -> ChatStats -> ExceptT StoreError IO (Chat 'CTDirect, Maybe NavigationInfo) +getDirectChatAround' db user ct@Contact {contactId} contentFilter aroundId count search stats = do aroundCI <- getDirectChatItem db user contactId aroundId - beforeIds <- liftIO $ getDirectCIsBefore_ db user ct aroundCI count search - afterIds <- liftIO $ getDirectCIsAfter_ db user ct aroundCI count search + let cInfo = DirectChat ct + range r = r (ciCreatedAt aroundCI) (cChatItemId aroundCI) + beforeIds <- getChatItemIDs db user cInfo contentFilter (range CRBefore) count search + afterIds <- getChatItemIDs db user cInfo contentFilter (range CRAfter) count search ts <- liftIO getCurrentTime beforeCIs <- liftIO $ mapM (safeGetDirectItem db user ct ts) beforeIds afterCIs <- liftIO $ mapM (safeGetDirectItem db user ct ts) afterIds let cis = reverse beforeCIs <> [aroundCI] <> afterCIs navInfo <- liftIO $ getNavInfo cis - pure (Chat (DirectChat ct) cis stats, Just navInfo) + pure (Chat cInfo cis stats, Just navInfo) where getNavInfo cis_ = case cis_ of [] -> pure $ NavigationInfo 0 0 cis -> getContactNavInfo_ db user ct (last cis) -getDirectChatInitial_ :: DB.Connection -> User -> Contact -> Int -> ExceptT StoreError IO (Chat 'CTDirect, Maybe NavigationInfo) -getDirectChatInitial_ db user ct count = do +getDirectChatInitial_ :: DB.Connection -> User -> Contact -> Maybe MsgContentTag -> Int -> ExceptT StoreError IO (Chat 'CTDirect, Maybe NavigationInfo) +getDirectChatInitial_ db user ct contentFilter count = do liftIO (getContactMinUnreadId_ db user ct) >>= \case Just minUnreadItemId -> do unreadCount <- liftIO $ getContactUnreadCount_ db user ct let stats = emptyChatStats {unreadCount, minUnreadItemId} - getDirectChatAround' db user ct minUnreadItemId count "" stats - Nothing -> liftIO $ (,Just $ NavigationInfo 0 0) <$> getDirectChatLast_ db user ct count "" + getDirectChatAround' db user ct contentFilter minUnreadItemId count "" stats + Nothing -> (,Just $ NavigationInfo 0 0) <$> getDirectChatLast_ db user ct contentFilter count "" getContactStats_ :: DB.Connection -> User -> Contact -> IO ChatStats getContactStats_ db user ct = do @@ -1471,64 +1452,81 @@ getGroupChatScopeForItem_ db itemId = getGroupChatLast_ :: DB.Connection -> User -> GroupInfo -> Maybe GroupChatScopeInfo -> Maybe MsgContentTag -> Int -> Text -> ChatStats -> ExceptT StoreError IO (Chat 'CTGroup) getGroupChatLast_ db user g scopeInfo_ contentFilter count search stats = do - ciIds <- getGroupChatItemIDs db user g scopeInfo_ contentFilter GRLast count search + let cInfo = GroupChat g scopeInfo_ + ciIds <- getChatItemIDs db user cInfo contentFilter CRLast count search ts <- liftIO getCurrentTime cis <- mapM (liftIO . safeGetGroupItem db user g ts) ciIds - pure $ Chat (GroupChat g scopeInfo_) (reverse cis) stats + pure $ Chat cInfo (reverse cis) stats -data GroupItemIDsRange = GRLast | GRAfter UTCTime ChatItemId | GRBefore UTCTime ChatItemId +data ChatItemIDsRange = CRLast | CRAfter UTCTime ChatItemId | CRBefore UTCTime ChatItemId -getGroupChatItemIDs :: DB.Connection -> User -> GroupInfo -> Maybe GroupChatScopeInfo -> Maybe MsgContentTag -> GroupItemIDsRange -> Int -> Text -> ExceptT StoreError IO [ChatItemId] -getGroupChatItemIDs db User {userId} GroupInfo {groupId} scopeInfo_ contentFilter range count search = case (scopeInfo_, contentFilter) of - (Nothing, Nothing) -> - liftIO $ - idsQuery - (baseCond <> " AND group_scope_tag IS NULL AND group_scope_group_member_id IS NULL ") - (userId, groupId) - (Nothing, Just mcTag) -> - liftIO $ - idsQuery - (baseCond <> " AND msg_content_tag = ? ") - (userId, groupId, mcTag) - (Just GCSIMemberSupport {groupMember_ = Just m}, Nothing) -> - liftIO $ - idsQuery - (baseCond <> " AND group_scope_tag = ? AND group_scope_group_member_id = ? ") - (userId, groupId, GCSTMemberSupport_, groupMemberId' m) - (Just GCSIMemberSupport {groupMember_ = Nothing}, Nothing) -> - liftIO $ - idsQuery - (baseCond <> " AND group_scope_tag = ? AND group_scope_group_member_id IS NULL ") - (userId, groupId, GCSTMemberSupport_) - (Just _scope, Just _mcTag) -> - throwError $ SEInternalError "group scope and content filter are not supported together" +getChatItemIDs :: DB.Connection -> User -> ChatInfo c -> Maybe MsgContentTag -> ChatItemIDsRange -> Int -> Text -> ExceptT StoreError IO [ChatItemId] +getChatItemIDs db User {userId} cInfo contentFilter range count search = case cInfo of + GroupChat GroupInfo {groupId} scopeInfo_ -> case (scopeInfo_, contentFilter) of + (Nothing, Nothing) -> + liftIO $ + idsQuery + (grCond <> " AND group_scope_tag IS NULL AND group_scope_group_member_id IS NULL ") + (userId, groupId) + "item_ts" + (Nothing, Just mcTag) -> + liftIO $ + idsQuery + (grCond <> " AND msg_content_tag = ? ") + (userId, groupId, mcTag) + "item_ts" + (Just GCSIMemberSupport {groupMember_ = m}, Nothing) -> + liftIO $ + idsQuery + (grCond <> " AND group_scope_tag = ? AND group_scope_group_member_id IS NOT DISTINCT FROM ? ") + (userId, groupId, GCSTMemberSupport_, groupMemberId' <$> m) + "item_ts" + (Just _scope, Just _mcTag) -> + throwError $ SEInternalError "group scope and content filter are not supported together" + where + grCond = " user_id = ? AND group_id = ? " + DirectChat Contact {contactId} -> liftIO $ case contentFilter of + Nothing -> idsQuery ctCond (userId, contactId) "created_at" + Just mcTag -> idsQuery (ctCond <> " AND msg_content_tag = ? ") (userId, contactId, mcTag) "created_at" + where + ctCond = " user_id = ? AND contact_id = ? " + LocalChat NoteFolder {noteFolderId} -> liftIO $ case contentFilter of + Nothing -> idsQuery nfCond (userId, noteFolderId) "created_at" + Just mcTag -> idsQuery (nfCond <> " AND msg_content_tag = ? ") (userId, noteFolderId, mcTag) "created_at" + where + nfCond = " user_id = ? AND note_folder_id = ? " + _ -> throwError $ SEInternalError "unsupported chat type" where baseQuery = " SELECT chat_item_id FROM chat_items WHERE " - baseCond = " user_id = ? AND group_id = ? " - idsQuery :: ToRow p => Query -> p -> IO [ChatItemId] - idsQuery c p = case range of - GRLast -> rangeQuery c p " ORDER BY item_ts DESC, chat_item_id DESC " - GRAfter ts itemId -> + -- parameterized by timestamp field `f` used to order chat items: + -- `item_ts` for groups, `created_at` for direct chats and notes. + idsQuery :: ToRow p => Query -> p -> Query -> IO [ChatItemId] + idsQuery c p f = case range of + CRLast -> rangeQuery c p (" ORDER BY " <> f <> " DESC, chat_item_id DESC ") + CRAfter ts itemId -> rangeQuery - (" item_ts > ? " `orCond` " item_ts = ? AND chat_item_id > ? ") + ((f <> " > ?") `orCond` (f <> " = ? AND chat_item_id > ?")) (orParams ts itemId) - " ORDER BY item_ts ASC, chat_item_id ASC " - GRBefore ts itemId -> + (" ORDER BY " <> f <> " ASC, chat_item_id ASC ") + CRBefore ts itemId -> rangeQuery - (" item_ts < ? " `orCond` " item_ts = ? AND chat_item_id < ? ") + ((f <> " < ?") `orCond` (f <> " = ? AND chat_item_id < ?")) (orParams ts itemId) - " ORDER BY item_ts DESC, chat_item_id DESC " + (" ORDER BY " <> f <> " DESC, chat_item_id DESC ") where + -- `orCond` creates this query: `(c AND c1) OR (c AND c2)`, + -- that is equivalent to `c AND (c1 OR c2)`. + -- OR has to be used on the top level for query planner to use indices + -- that include fields in c1 and c2. orCond c1 c2 = " ((" <> c <> " AND " <> c1 <> ") OR (" <> c <> " AND " <> c2 <> ")) " orParams ts itemId = (p :. (Only ts) :. p :. (ts, itemId)) rangeQuery :: ToRow p => Query -> p -> Query -> IO [ChatItemId] - rangeQuery c p ob - | T.null search = searchQuery "" () - | otherwise = searchQuery " AND LOWER(item_text) LIKE '%' || LOWER(?) || '%' " (Only search) - where - searchQuery :: ToRow p' => Query -> p' -> IO [ChatItemId] - searchQuery c' p' = - map fromOnly <$> DB.query db (baseQuery <> c <> c' <> ob <> " LIMIT ?") (p :. p' :. Only count) + rangeQuery c p ob = + map fromOnly + <$> if T.null search + then DB.query db (baseQuery <> c <> ob <> " LIMIT ?") (p :. Only count) + else DB.query db (baseQuery <> c <> searchCond <> ob <> " LIMIT ?") (p :. (search, count)) + searchCond = " AND LOWER(item_text) LIKE '%' || LOWER(?) || '%' " safeGetGroupItem :: DB.Connection -> User -> GroupInfo -> UTCTime -> ChatItemId -> IO (CChatItem 'CTGroup) safeGetGroupItem db user g currentTs itemId = @@ -1580,20 +1578,22 @@ getGroupMemberChatItemLast db user@User {userId} groupId groupMemberId = do getGroupChatAfter_ :: DB.Connection -> User -> GroupInfo -> Maybe GroupChatScopeInfo -> Maybe MsgContentTag -> ChatItemId -> Int -> Text -> ExceptT StoreError IO (Chat 'CTGroup) getGroupChatAfter_ db user g@GroupInfo {groupId} scopeInfo contentFilter afterId count search = do afterCI <- getGroupChatItem db user groupId afterId - let range = GRAfter (chatItemTs afterCI) (cChatItemId afterCI) - ciIds <- getGroupChatItemIDs db user g scopeInfo contentFilter range count search + let cInfo = GroupChat g scopeInfo + range = CRAfter (chatItemTs afterCI) (cChatItemId afterCI) + ciIds <- getChatItemIDs db user cInfo contentFilter range count search ts <- liftIO getCurrentTime cis <- liftIO $ mapM (safeGetGroupItem db user g ts) ciIds - pure $ Chat (GroupChat g scopeInfo) cis emptyChatStats + pure $ Chat cInfo cis emptyChatStats getGroupChatBefore_ :: DB.Connection -> User -> GroupInfo -> Maybe GroupChatScopeInfo -> Maybe MsgContentTag -> ChatItemId -> Int -> Text -> ExceptT StoreError IO (Chat 'CTGroup) getGroupChatBefore_ db user g@GroupInfo {groupId} scopeInfo contentFilter beforeId count search = do beforeCI <- getGroupChatItem db user groupId beforeId - let range = GRBefore (chatItemTs beforeCI) (cChatItemId beforeCI) - ciIds <- getGroupChatItemIDs db user g scopeInfo contentFilter range count search + let cInfo = GroupChat g scopeInfo + range = CRBefore (chatItemTs beforeCI) (cChatItemId beforeCI) + ciIds <- getChatItemIDs db user cInfo contentFilter range count search ts <- liftIO getCurrentTime cis <- liftIO $ mapM (safeGetGroupItem db user g ts) ciIds - pure $ Chat (GroupChat g scopeInfo) (reverse cis) emptyChatStats + pure $ Chat cInfo (reverse cis) emptyChatStats getGroupChatAround_ :: DB.Connection -> User -> GroupInfo -> Maybe GroupChatScopeInfo -> Maybe MsgContentTag -> ChatItemId -> Int -> Text -> ExceptT StoreError IO (Chat 'CTGroup, Maybe NavigationInfo) getGroupChatAround_ db user g scopeInfo contentFilter aroundId count search = do @@ -1603,16 +1603,16 @@ getGroupChatAround_ db user g scopeInfo contentFilter aroundId count search = do getGroupChatAround' :: DB.Connection -> User -> GroupInfo -> Maybe GroupChatScopeInfo -> Maybe MsgContentTag -> ChatItemId -> Int -> Text -> ChatStats -> ExceptT StoreError IO (Chat 'CTGroup, Maybe NavigationInfo) getGroupChatAround' db user g scopeInfo contentFilter aroundId count search stats = do aroundCI <- getGroupCIWithReactions db user g aroundId - let beforeRange = GRBefore (chatItemTs aroundCI) (cChatItemId aroundCI) - afterRange = GRAfter (chatItemTs aroundCI) (cChatItemId aroundCI) - beforeIds <- getGroupChatItemIDs db user g scopeInfo contentFilter beforeRange count search - afterIds <- getGroupChatItemIDs db user g scopeInfo contentFilter afterRange count search + let cInfo = GroupChat g scopeInfo + range r = r (chatItemTs aroundCI) (cChatItemId aroundCI) + beforeIds <- getChatItemIDs db user cInfo contentFilter (range CRBefore) count search + afterIds <- getChatItemIDs db user cInfo contentFilter (range CRAfter) count search ts <- liftIO getCurrentTime beforeCIs <- liftIO $ mapM (safeGetGroupItem db user g ts) beforeIds afterCIs <- liftIO $ mapM (safeGetGroupItem db user g ts) afterIds let cis = reverse beforeCIs <> [aroundCI] <> afterCIs navInfo <- liftIO $ getNavInfo cis - pure (Chat (GroupChat g scopeInfo) cis stats, Just navInfo) + pure (Chat cInfo cis stats, Just navInfo) where getNavInfo cis_ = case cis_ of [] -> pure $ NavigationInfo 0 0 @@ -1677,18 +1677,12 @@ queryUnreadGroupItems db User {userId} GroupInfo {groupId} scopeInfo_ contentFil db (baseQuery <> " AND msg_content_tag = ? AND item_status = ? " <> orderLimit) (userId, groupId, mcTag, CISRcvNew) - (Just GCSIMemberSupport {groupMember_ = Just m}, Nothing) -> + (Just GCSIMemberSupport {groupMember_ = m}, Nothing) -> liftIO $ DB.query db - (baseQuery <> " AND group_scope_tag = ? AND group_scope_group_member_id = ? AND item_status = ? " <> orderLimit) - (userId, groupId, GCSTMemberSupport_, groupMemberId' m, CISRcvNew) - (Just GCSIMemberSupport {groupMember_ = Nothing}, Nothing) -> - liftIO $ - DB.query - db - (baseQuery <> " AND group_scope_tag = ? AND group_scope_group_member_id IS NULL AND item_status = ? " <> orderLimit) - (userId, groupId, GCSTMemberSupport_, CISRcvNew) + (baseQuery <> " AND group_scope_tag = ? AND group_scope_group_member_id IS NOT DISTINCT FROM ? AND item_status = ? " <> orderLimit) + (userId, groupId, GCSTMemberSupport_, groupMemberId' <$> m, CISRcvNew) (Just _scope, Just _mcTag) -> throwError $ SEInternalError "group scope and content filter are not supported together" @@ -1743,39 +1737,26 @@ getGroupNavInfo_ db User {userId} GroupInfo {groupId} afterCI = do :. (userId, groupId, chatItemTs afterCI, cChatItemId afterCI) ) -getLocalChat :: DB.Connection -> User -> Int64 -> ChatPagination -> Maybe Text -> ExceptT StoreError IO (Chat 'CTLocal, Maybe NavigationInfo) -getLocalChat db user folderId pagination search_ = do +getLocalChat :: DB.Connection -> User -> Int64 -> Maybe MsgContentTag -> ChatPagination -> Maybe Text -> ExceptT StoreError IO (Chat 'CTLocal, Maybe NavigationInfo) +getLocalChat db user folderId contentFilter pagination search_ = do let search = fromMaybe "" search_ nf <- getNoteFolder db user folderId case pagination of - CPLast count -> liftIO $ (,Nothing) <$> getLocalChatLast_ db user nf count search - CPAfter afterId count -> (,Nothing) <$> getLocalChatAfter_ db user nf afterId count search - CPBefore beforeId count -> (,Nothing) <$> getLocalChatBefore_ db user nf beforeId count search - CPAround aroundId count -> getLocalChatAround_ db user nf aroundId count search + CPLast count -> (,Nothing) <$> getLocalChatLast_ db user nf contentFilter count search + CPAfter afterId count -> (,Nothing) <$> getLocalChatAfter_ db user nf contentFilter afterId count search + CPBefore beforeId count -> (,Nothing) <$> getLocalChatBefore_ db user nf contentFilter beforeId count search + CPAround aroundId count -> getLocalChatAround_ db user nf contentFilter aroundId count search CPInitial count -> do unless (T.null search) $ throwError $ SEInternalError "initial chat pagination doesn't support search" - getLocalChatInitial_ db user nf count + getLocalChatInitial_ db user nf contentFilter count -getLocalChatLast_ :: DB.Connection -> User -> NoteFolder -> Int -> Text -> IO (Chat 'CTLocal) -getLocalChatLast_ db user nf count search = do - ciIds <- getLocalChatItemIdsLast_ db user nf count search - ts <- getCurrentTime - cis <- mapM (safeGetLocalItem db user nf ts) ciIds - pure $ Chat (LocalChat nf) (reverse cis) emptyChatStats - -getLocalChatItemIdsLast_ :: DB.Connection -> User -> NoteFolder -> Int -> Text -> IO [ChatItemId] -getLocalChatItemIdsLast_ db User {userId} NoteFolder {noteFolderId} count search = - map fromOnly - <$> DB.query - db - [sql| - SELECT chat_item_id - FROM chat_items - WHERE user_id = ? AND note_folder_id = ? AND LOWER(item_text) LIKE '%' || LOWER(?) || '%' - ORDER BY created_at DESC, chat_item_id DESC - LIMIT ? - |] - (userId, noteFolderId, search, count) +getLocalChatLast_ :: DB.Connection -> User -> NoteFolder -> Maybe MsgContentTag -> Int -> Text -> ExceptT StoreError IO (Chat 'CTLocal) +getLocalChatLast_ db user nf contentFilter count search = do + let cInfo = LocalChat nf + ciIds <- getChatItemIDs db user cInfo contentFilter CRLast count search + ts <- liftIO getCurrentTime + cis <- liftIO $ mapM (safeGetLocalItem db user nf ts) ciIds + pure $ Chat cInfo (reverse cis) emptyChatStats safeGetLocalItem :: DB.Connection -> User -> NoteFolder -> UTCTime -> ChatItemId -> IO (CChatItem 'CTLocal) safeGetLocalItem db user NoteFolder {noteFolderId} currentTs itemId = @@ -1804,81 +1785,57 @@ safeToLocalItem currentTs itemId = \case file = Nothing } -getLocalChatAfter_ :: DB.Connection -> User -> NoteFolder -> ChatItemId -> Int -> Text -> ExceptT StoreError IO (Chat 'CTLocal) -getLocalChatAfter_ db user nf@NoteFolder {noteFolderId} afterId count search = do +getLocalChatAfter_ :: DB.Connection -> User -> NoteFolder -> Maybe MsgContentTag -> ChatItemId -> Int -> Text -> ExceptT StoreError IO (Chat 'CTLocal) +getLocalChatAfter_ db user nf@NoteFolder {noteFolderId} contentFilter afterId count search = do afterCI <- getLocalChatItem db user noteFolderId afterId - ciIds <- liftIO $ getLocalCIsAfter_ db user nf afterCI count search + let cInfo = LocalChat nf + range = CRAfter (ciCreatedAt afterCI) (cChatItemId afterCI) + ciIds <- getChatItemIDs db user cInfo contentFilter range count search ts <- liftIO getCurrentTime cis <- liftIO $ mapM (safeGetLocalItem db user nf ts) ciIds - pure $ Chat (LocalChat nf) cis emptyChatStats + pure $ Chat cInfo cis emptyChatStats -getLocalCIsAfter_ :: DB.Connection -> User -> NoteFolder -> CChatItem 'CTLocal -> Int -> Text -> IO [ChatItemId] -getLocalCIsAfter_ db User {userId} NoteFolder {noteFolderId} afterCI count search = - map fromOnly - <$> DB.query - db - [sql| - SELECT chat_item_id - FROM chat_items - WHERE user_id = ? AND note_folder_id = ? AND LOWER(item_text) LIKE '%' || LOWER(?) || '%' - AND (created_at > ? OR (created_at = ? AND chat_item_id > ?)) - ORDER BY created_at ASC, chat_item_id ASC - LIMIT ? - |] - (userId, noteFolderId, search, ciCreatedAt afterCI, ciCreatedAt afterCI, cChatItemId afterCI, count) - -getLocalChatBefore_ :: DB.Connection -> User -> NoteFolder -> ChatItemId -> Int -> Text -> ExceptT StoreError IO (Chat 'CTLocal) -getLocalChatBefore_ db user nf@NoteFolder {noteFolderId} beforeId count search = do +getLocalChatBefore_ :: DB.Connection -> User -> NoteFolder -> Maybe MsgContentTag -> ChatItemId -> Int -> Text -> ExceptT StoreError IO (Chat 'CTLocal) +getLocalChatBefore_ db user nf@NoteFolder {noteFolderId} contentFilter beforeId count search = do beforeCI <- getLocalChatItem db user noteFolderId beforeId - ciIds <- liftIO $ getLocalCIsBefore_ db user nf beforeCI count search + let cInfo = LocalChat nf + range = CRBefore (ciCreatedAt beforeCI) (cChatItemId beforeCI) + ciIds <- getChatItemIDs db user cInfo contentFilter range count search ts <- liftIO getCurrentTime cis <- liftIO $ mapM (safeGetLocalItem db user nf ts) ciIds - pure $ Chat (LocalChat nf) (reverse cis) emptyChatStats + pure $ Chat cInfo (reverse cis) emptyChatStats -getLocalCIsBefore_ :: DB.Connection -> User -> NoteFolder -> CChatItem 'CTLocal -> Int -> Text -> IO [ChatItemId] -getLocalCIsBefore_ db User {userId} NoteFolder {noteFolderId} beforeCI count search = - map fromOnly - <$> DB.query - db - [sql| - SELECT chat_item_id - FROM chat_items - WHERE user_id = ? AND note_folder_id = ? AND LOWER(item_text) LIKE '%' || LOWER(?) || '%' - AND (created_at < ? OR (created_at = ? AND chat_item_id < ?)) - ORDER BY created_at DESC, chat_item_id DESC - LIMIT ? - |] - (userId, noteFolderId, search, ciCreatedAt beforeCI, ciCreatedAt beforeCI, cChatItemId beforeCI, count) - -getLocalChatAround_ :: DB.Connection -> User -> NoteFolder -> ChatItemId -> Int -> Text -> ExceptT StoreError IO (Chat 'CTLocal, Maybe NavigationInfo) -getLocalChatAround_ db user nf aroundId count search = do +getLocalChatAround_ :: DB.Connection -> User -> NoteFolder -> Maybe MsgContentTag -> ChatItemId -> Int -> Text -> ExceptT StoreError IO (Chat 'CTLocal, Maybe NavigationInfo) +getLocalChatAround_ db user nf contentFilter aroundId count search = do stats <- liftIO $ getLocalStats_ db user nf - getLocalChatAround' db user nf aroundId count search stats + getLocalChatAround' db user nf contentFilter aroundId count search stats -getLocalChatAround' :: DB.Connection -> User -> NoteFolder -> ChatItemId -> Int -> Text -> ChatStats -> ExceptT StoreError IO (Chat 'CTLocal, Maybe NavigationInfo) -getLocalChatAround' db user nf@NoteFolder {noteFolderId} aroundId count search stats = do +getLocalChatAround' :: DB.Connection -> User -> NoteFolder -> Maybe MsgContentTag -> ChatItemId -> Int -> Text -> ChatStats -> ExceptT StoreError IO (Chat 'CTLocal, Maybe NavigationInfo) +getLocalChatAround' db user nf@NoteFolder {noteFolderId} contentFilter aroundId count search stats = do aroundCI <- getLocalChatItem db user noteFolderId aroundId - beforeIds <- liftIO $ getLocalCIsBefore_ db user nf aroundCI count search - afterIds <- liftIO $ getLocalCIsAfter_ db user nf aroundCI count search + let cInfo = LocalChat nf + range r = r (ciCreatedAt aroundCI) (cChatItemId aroundCI) + beforeIds <- getChatItemIDs db user cInfo contentFilter (range CRBefore) count search + afterIds <- getChatItemIDs db user cInfo contentFilter (range CRAfter) count search ts <- liftIO getCurrentTime beforeCIs <- liftIO $ mapM (safeGetLocalItem db user nf ts) beforeIds afterCIs <- liftIO $ mapM (safeGetLocalItem db user nf ts) afterIds let cis = reverse beforeCIs <> [aroundCI] <> afterCIs navInfo <- liftIO $ getNavInfo cis - pure (Chat (LocalChat nf) cis stats, Just navInfo) + pure (Chat cInfo cis stats, Just navInfo) where getNavInfo cis_ = case cis_ of [] -> pure $ NavigationInfo 0 0 cis -> getLocalNavInfo_ db user nf (last cis) -getLocalChatInitial_ :: DB.Connection -> User -> NoteFolder -> Int -> ExceptT StoreError IO (Chat 'CTLocal, Maybe NavigationInfo) -getLocalChatInitial_ db user nf count = do +getLocalChatInitial_ :: DB.Connection -> User -> NoteFolder -> Maybe MsgContentTag -> Int -> ExceptT StoreError IO (Chat 'CTLocal, Maybe NavigationInfo) +getLocalChatInitial_ db user nf contentFilter count = do liftIO (getLocalMinUnreadId_ db user nf) >>= \case Just minUnreadItemId -> do unreadCount <- liftIO $ getLocalUnreadCount_ db user nf let stats = emptyChatStats {unreadCount, minUnreadItemId} - getLocalChatAround' db user nf minUnreadItemId count "" stats - Nothing -> liftIO $ (,Just $ NavigationInfo 0 0) <$> getLocalChatLast_ db user nf count "" + getLocalChatAround' db user nf contentFilter minUnreadItemId count "" stats + Nothing -> (,Just $ NavigationInfo 0 0) <$> getLocalChatLast_ db user nf contentFilter count "" getLocalStats_ :: DB.Connection -> User -> NoteFolder -> IO ChatStats getLocalStats_ db user nf = do diff --git a/src/Simplex/Chat/Store/Postgres/Migrations.hs b/src/Simplex/Chat/Store/Postgres/Migrations.hs index ef6d96bd45..3f8cc5b64b 100644 --- a/src/Simplex/Chat/Store/Postgres/Migrations.hs +++ b/src/Simplex/Chat/Store/Postgres/Migrations.hs @@ -24,6 +24,7 @@ import Simplex.Chat.Store.Postgres.Migrations.M20251017_chat_tags_cascade import Simplex.Chat.Store.Postgres.Migrations.M20251117_member_relations_vector import Simplex.Chat.Store.Postgres.Migrations.M20251128_migrate_member_relations import Simplex.Chat.Store.Postgres.Migrations.M20251230_strict_tables +import Simplex.Chat.Store.Postgres.Migrations.M20260108_chat_indices import Simplex.Messaging.Agent.Store.Shared (Migration (..)) schemaMigrations :: [(String, Text, Maybe Text)] @@ -47,7 +48,8 @@ schemaMigrations = ("20251017_chat_tags_cascade", m20251017_chat_tags_cascade, Just down_m20251017_chat_tags_cascade), ("20251117_member_relations_vector", m20251117_member_relations_vector, Just down_m20251117_member_relations_vector), ("20251128_migrate_member_relations", m20251128_migrate_member_relations, Just down_m20251128_migrate_member_relations), - ("20251230_strict_tables", m20251230_strict_tables, Just down_m20251230_strict_tables) + ("20251230_strict_tables", m20251230_strict_tables, Just down_m20251230_strict_tables), + ("20260108_chat_indices", m20260108_chat_indices, Just down_m20260108_chat_indices) ] -- | The list of migrations in ascending order by date diff --git a/src/Simplex/Chat/Store/Postgres/Migrations/M20260108_chat_indices.hs b/src/Simplex/Chat/Store/Postgres/Migrations/M20260108_chat_indices.hs new file mode 100644 index 0000000000..e7104dfbe3 --- /dev/null +++ b/src/Simplex/Chat/Store/Postgres/Migrations/M20260108_chat_indices.hs @@ -0,0 +1,33 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE QuasiQuotes #-} + +module Simplex.Chat.Store.Postgres.Migrations.M20260108_chat_indices where + +import Data.Text (Text) +import Text.RawString.QQ (r) + +m20260108_chat_indices :: Text +m20260108_chat_indices = + [r| +CREATE INDEX idx_chat_items_contacts_msg_content_tag_created_at ON chat_items( + user_id, + contact_id, + msg_content_tag, + created_at +); + +CREATE INDEX idx_chat_items_note_folder_msg_content_tag_created_at ON chat_items( + user_id, + note_folder_id, + msg_content_tag, + created_at +); +|] + +down_m20260108_chat_indices :: Text +down_m20260108_chat_indices = + [r| +DROP INDEX idx_chat_items_contacts_msg_content_tag_created_at; + +DROP INDEX idx_chat_items_note_folder_msg_content_tag_created_at; +|] diff --git a/src/Simplex/Chat/Store/Postgres/Migrations/chat_schema.sql b/src/Simplex/Chat/Store/Postgres/Migrations/chat_schema.sql index 238a6bfdbb..c37d1360d6 100644 --- a/src/Simplex/Chat/Store/Postgres/Migrations/chat_schema.sql +++ b/src/Simplex/Chat/Store/Postgres/Migrations/chat_schema.sql @@ -1813,6 +1813,10 @@ CREATE INDEX idx_chat_items_contacts_created_at ON test_chat_schema.chat_items U +CREATE INDEX idx_chat_items_contacts_msg_content_tag_created_at ON test_chat_schema.chat_items USING btree (user_id, contact_id, msg_content_tag, created_at); + + + CREATE UNIQUE INDEX idx_chat_items_direct_shared_msg_id ON test_chat_schema.chat_items USING btree (user_id, contact_id, shared_msg_id); @@ -1897,6 +1901,10 @@ CREATE INDEX idx_chat_items_item_status ON test_chat_schema.chat_items USING btr +CREATE INDEX idx_chat_items_note_folder_msg_content_tag_created_at ON test_chat_schema.chat_items USING btree (user_id, note_folder_id, msg_content_tag, created_at); + + + CREATE INDEX idx_chat_items_notes ON test_chat_schema.chat_items USING btree (user_id, note_folder_id, item_status, created_at); diff --git a/src/Simplex/Chat/Store/SQLite/Migrations.hs b/src/Simplex/Chat/Store/SQLite/Migrations.hs index f740c5654f..fc0da3c04a 100644 --- a/src/Simplex/Chat/Store/SQLite/Migrations.hs +++ b/src/Simplex/Chat/Store/SQLite/Migrations.hs @@ -147,6 +147,7 @@ import Simplex.Chat.Store.SQLite.Migrations.M20251017_chat_tags_cascade import Simplex.Chat.Store.SQLite.Migrations.M20251117_member_relations_vector import Simplex.Chat.Store.SQLite.Migrations.M20251128_migrate_member_relations import Simplex.Chat.Store.SQLite.Migrations.M20251230_strict_tables +import Simplex.Chat.Store.SQLite.Migrations.M20260108_chat_indices import Simplex.Messaging.Agent.Store.Shared (Migration (..)) schemaMigrations :: [(String, Query, Maybe Query)] @@ -293,7 +294,8 @@ schemaMigrations = ("20251017_chat_tags_cascade", m20251017_chat_tags_cascade, Just down_m20251017_chat_tags_cascade), ("20251117_member_relations_vector", m20251117_member_relations_vector, Just down_m20251117_member_relations_vector), ("20251128_migrate_member_relations", m20251128_migrate_member_relations, Just down_m20251128_migrate_member_relations), - ("20251230_strict_tables", m20251230_strict_tables, Just down_m20251230_strict_tables) + ("20251230_strict_tables", m20251230_strict_tables, Just down_m20251230_strict_tables), + ("20260108_chat_indices", m20260108_chat_indices, Just down_m20260108_chat_indices) ] -- | The list of migrations in ascending order by date diff --git a/src/Simplex/Chat/Store/SQLite/Migrations/M20260108_chat_indices.hs b/src/Simplex/Chat/Store/SQLite/Migrations/M20260108_chat_indices.hs new file mode 100644 index 0000000000..b64bef72cf --- /dev/null +++ b/src/Simplex/Chat/Store/SQLite/Migrations/M20260108_chat_indices.hs @@ -0,0 +1,32 @@ +{-# LANGUAGE QuasiQuotes #-} + +module Simplex.Chat.Store.SQLite.Migrations.M20260108_chat_indices where + +import Database.SQLite.Simple (Query) +import Database.SQLite.Simple.QQ (sql) + +m20260108_chat_indices :: Query +m20260108_chat_indices = + [sql| +CREATE INDEX idx_chat_items_contacts_msg_content_tag_created_at ON chat_items( + user_id, + contact_id, + msg_content_tag, + created_at +); + +CREATE INDEX idx_chat_items_note_folder_msg_content_tag_created_at ON chat_items( + user_id, + note_folder_id, + msg_content_tag, + created_at +); +|] + +down_m20260108_chat_indices :: Query +down_m20260108_chat_indices = + [sql| +DROP INDEX idx_chat_items_contacts_msg_content_tag_created_at; + +DROP INDEX idx_chat_items_note_folder_msg_content_tag_created_at; +|] diff --git a/src/Simplex/Chat/Store/SQLite/Migrations/agent_query_plans.txt b/src/Simplex/Chat/Store/SQLite/Migrations/agent_query_plans.txt index 6d3f69d5fa..1b881bd446 100644 --- a/src/Simplex/Chat/Store/SQLite/Migrations/agent_query_plans.txt +++ b/src/Simplex/Chat/Store/SQLite/Migrations/agent_query_plans.txt @@ -59,6 +59,16 @@ Query: Plan: SEARCH commands USING INDEX idx_commands_server_commands (host=? AND port=?) +Query: + SELECT rcpt_status, snd_message_body_id FROM snd_messages + WHERE NOT EXISTS (SELECT 1 FROM snd_message_deliveries WHERE conn_id = ? AND internal_id = ? AND failed = 0) + AND conn_id = ? AND internal_id = ? + +Plan: +SEARCH snd_messages USING PRIMARY KEY (conn_id=?) +SCALAR SUBQUERY 1 +SEARCH snd_message_deliveries USING COVERING INDEX idx_snd_message_deliveries_expired (conn_id=?) + Query: SELECT rcv_file_chunk_id, chunk_no, chunk_size, digest, tmp_path FROM rcv_file_chunks @@ -69,6 +79,14 @@ Plan: SEARCH rcv_file_chunks USING INDEX idx_rcv_file_chunks_rcv_file_id (rcv_file_id=?) USE TEMP B-TREE FOR ORDER BY +Query: + SELECT rcv_file_entity_id, user_id, size, digest, key, nonce, chunk_size, prefix_path, tmp_path, save_path, save_file_key, save_file_nonce, status, deleted, redirect_id, redirect_entity_id, redirect_size, redirect_digest + FROM rcv_files + WHERE rcv_file_id = ? + +Plan: +SEARCH rcv_files USING INTEGER PRIMARY KEY (rowid=?) + Query: SELECT snd_file_chunk_id, chunk_no, chunk_offset, chunk_size, digest FROM snd_file_chunks @@ -77,6 +95,14 @@ Query: Plan: SEARCH snd_file_chunks USING INDEX idx_snd_file_chunks_snd_file_id (snd_file_id=?) +Query: + SELECT snd_file_entity_id, user_id, path, src_file_key, src_file_nonce, num_recipients, digest, prefix_path, key, nonce, status, deleted, redirect_size, redirect_digest + FROM snd_files + WHERE snd_file_id = ? + +Plan: +SEARCH snd_files USING INTEGER PRIMARY KEY (rowid=?) + Query: DELETE FROM snd_message_bodies WHERE NOT EXISTS (SELECT 1 FROM snd_messages WHERE snd_message_body_id = ?) @@ -201,24 +227,6 @@ SEARCH c USING INTEGER PRIMARY KEY (rowid=?) SEARCH f USING INTEGER PRIMARY KEY (rowid=?) USE TEMP B-TREE FOR ORDER BY -Query: - SELECT rcpt_status, snd_message_body_id FROM snd_messages - WHERE NOT EXISTS (SELECT 1 FROM snd_message_deliveries WHERE conn_id = ? AND internal_id = ? AND failed = 0) - AND conn_id = ? AND internal_id = ? - -Plan: -SEARCH snd_messages USING PRIMARY KEY (conn_id=?) -SCALAR SUBQUERY 1 -SEARCH snd_message_deliveries USING COVERING INDEX idx_snd_message_deliveries_expired (conn_id=?) - -Query: - SELECT rcv_file_entity_id, user_id, size, digest, key, nonce, chunk_size, prefix_path, tmp_path, save_path, save_file_key, save_file_nonce, status, deleted, redirect_id, redirect_entity_id, redirect_size, redirect_digest - FROM rcv_files - WHERE rcv_file_id = ? - -Plan: -SEARCH rcv_files USING INTEGER PRIMARY KEY (rowid=?) - Query: SELECT rcv_file_id FROM rcv_files @@ -230,14 +238,6 @@ Plan: SEARCH rcv_files USING INDEX idx_rcv_files_status_created_at (status=? AND created_at>?) USE TEMP B-TREE FOR ORDER BY -Query: - SELECT snd_file_entity_id, user_id, path, src_file_key, src_file_nonce, num_recipients, digest, prefix_path, key, nonce, status, deleted, redirect_size, redirect_digest - FROM snd_files - WHERE snd_file_id = ? - -Plan: -SEARCH snd_files USING INTEGER PRIMARY KEY (rowid=?) - Query: SELECT snd_file_id FROM snd_files @@ -257,6 +257,22 @@ Query: Plan: SEARCH messages USING PRIMARY KEY (conn_id=?) +Query: + SELECT last_internal_msg_id, last_internal_rcv_msg_id, last_external_snd_msg_id, last_rcv_msg_hash + FROM connections + WHERE conn_id = ? + +Plan: +SEARCH connections USING PRIMARY KEY (conn_id=?) + +Query: + SELECT last_internal_msg_id, last_internal_snd_msg_id, last_snd_msg_hash + FROM connections + WHERE conn_id = ? + +Plan: +SEARCH connections USING PRIMARY KEY (conn_id=?) + Query: SELECT user_id FROM users u WHERE u.user_id = ? @@ -268,6 +284,15 @@ SEARCH u USING INTEGER PRIMARY KEY (rowid=?) CORRELATED SCALAR SUBQUERY 1 SEARCH c USING COVERING INDEX idx_connections_user (user_id=?) +Query: + SELECT user_id, conn_id, conn_mode, smp_agent_version, enable_ntfs, + last_external_snd_msg_id, deleted, ratchet_sync_state, pq_support + FROM connections + WHERE conn_id = ? AND deleted = ? + +Plan: +SEARCH connections USING PRIMARY KEY (conn_id=?) + Query: INSERT INTO conn_confirmations (confirmation_id, conn_id, sender_key, e2e_snd_pub_key, ratchet_state, sender_conn_info, smp_reply_queues, smp_client_version, accepted) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0); @@ -287,6 +312,18 @@ Query: Plan: +Query: + INSERT INTO xftp_servers (xftp_host, xftp_port, xftp_key_hash) + VALUES (?, ?, ?) + ON CONFLICT (xftp_host, xftp_port, xftp_key_hash) + DO UPDATE SET xftp_host = EXCLUDED.xftp_host + RETURNING xftp_server_id + +Plan: +SEARCH deleted_snd_chunk_replicas USING COVERING INDEX idx_deleted_snd_chunk_replicas_xftp_server_id (xftp_server_id=?) +SEARCH snd_file_chunk_replicas USING COVERING INDEX idx_snd_file_chunk_replicas_xftp_server_id (xftp_server_id=?) +SEARCH rcv_file_chunk_replicas USING COVERING INDEX idx_rcv_file_chunk_replicas_xftp_server_id (xftp_server_id=?) + Query: SELECT r.internal_id, m.internal_ts, r.broker_id, r.broker_ts, r.external_snd_id, r.integrity, r.internal_hash, @@ -445,22 +482,6 @@ Query: Plan: SEARCH conn_invitations USING PRIMARY KEY (invitation_id=?) -Query: - SELECT last_internal_msg_id, last_internal_rcv_msg_id, last_external_snd_msg_id, last_rcv_msg_hash - FROM connections - WHERE conn_id = ? - -Plan: -SEARCH connections USING PRIMARY KEY (conn_id=?) - -Query: - SELECT last_internal_msg_id, last_internal_snd_msg_id, last_snd_msg_hash - FROM connections - WHERE conn_id = ? - -Plan: -SEARCH connections USING PRIMARY KEY (conn_id=?) - Query: SELECT link_id, snd_private_key FROM inv_short_links @@ -497,15 +518,6 @@ Plan: SEARCH s USING PRIMARY KEY (conn_id=? AND internal_snd_id=?) SEARCH m USING PRIMARY KEY (conn_id=? AND internal_id=?) -Query: - SELECT user_id, conn_id, conn_mode, smp_agent_version, enable_ntfs, - last_external_snd_msg_id, deleted, ratchet_sync_state, pq_support - FROM connections - WHERE conn_id = ? AND deleted = ? - -Plan: -SEARCH connections USING PRIMARY KEY (conn_id=?) - Query: DELETE FROM conn_confirmations WHERE conn_id = ? @@ -1027,8 +1039,13 @@ Plan: Query: INSERT INTO rcv_files (rcv_file_entity_id, user_id, size, digest, key, nonce, chunk_size, prefix_path, tmp_path, save_path, save_file_key, save_file_nonce, status, redirect_id, redirect_entity_id, redirect_digest, redirect_size, approved_relays) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) Plan: -Query: INSERT INTO servers (host, port, key_hash) VALUES (?,?,?) +Query: INSERT INTO servers (host, port, key_hash) VALUES (?,?,?) ON CONFLICT (host, port) DO NOTHING RETURNING 1 Plan: +SEARCH inv_short_links USING COVERING INDEX idx_inv_short_links_link_id (host=? AND port=?) +SEARCH commands USING COVERING INDEX idx_commands_server_commands (host=? AND port=?) +SEARCH ntf_subscriptions USING COVERING INDEX idx_ntf_subscriptions_smp_host_smp_port (smp_host=? AND smp_port=?) +SEARCH snd_queues USING COVERING INDEX idx_snd_queues_host_port (host=? AND port=?) +SEARCH rcv_queues USING COVERING INDEX idx_rcv_queues_link_id (host=? AND port=?) Query: INSERT INTO skipped_messages (conn_id, header_key, msg_n, msg_key) VALUES (?, ?, ?, ?) Plan: @@ -1052,9 +1069,6 @@ Plan: Query: INSERT INTO users DEFAULT VALUES Plan: -Query: INSERT INTO xftp_servers (xftp_host, xftp_port, xftp_key_hash) VALUES (?,?,?) -Plan: - Query: SELECT 1 FROM encrypted_rcv_message_hashes WHERE conn_id = ? AND hash = ? LIMIT 1 Plan: SEARCH encrypted_rcv_message_hashes USING COVERING INDEX idx_encrypted_rcv_message_hashes_hash (conn_id=? AND hash=?) @@ -1159,10 +1173,6 @@ Query: SELECT x3dh_priv_key_1, x3dh_priv_key_2, pq_priv_kem FROM ratchets WHERE Plan: SEARCH ratchets USING PRIMARY KEY (conn_id=?) -Query: SELECT xftp_server_id FROM xftp_servers WHERE xftp_host = ? AND xftp_port = ? AND xftp_key_hash = ? -Plan: -SEARCH xftp_servers USING COVERING INDEX sqlite_autoindex_xftp_servers_1 (xftp_host=? AND xftp_port=? AND xftp_key_hash=?) - Query: UPDATE connections SET deleted = ? WHERE conn_id = ? Plan: SEARCH connections USING PRIMARY KEY (conn_id=?) diff --git a/src/Simplex/Chat/Store/SQLite/Migrations/chat_query_plans.txt b/src/Simplex/Chat/Store/SQLite/Migrations/chat_query_plans.txt index 69fa665911..30751ee535 100644 --- a/src/Simplex/Chat/Store/SQLite/Migrations/chat_query_plans.txt +++ b/src/Simplex/Chat/Store/SQLite/Migrations/chat_query_plans.txt @@ -736,6 +736,26 @@ Query: Plan: SEARCH messages USING INDEX idx_messages_group_id_shared_msg_id (group_id=? AND shared_msg_id=?) +Query: + SELECT chat_item_id + FROM chat_items + WHERE user_id = ? AND contact_id = ? + ORDER BY created_at DESC, chat_item_id DESC + LIMIT 1 + +Plan: +SEARCH chat_items USING COVERING INDEX idx_chat_items_contacts_created_at (user_id=? AND contact_id=?) + +Query: + SELECT chat_item_id + FROM chat_items + WHERE user_id = ? AND group_id = ? AND group_member_id = ? + ORDER BY item_ts DESC, chat_item_id DESC + LIMIT 1 + +Plan: +SEARCH chat_items USING INDEX idx_chat_items_groups_item_ts (user_id=? AND group_id=?) + Query: SELECT chat_item_id, contact_id, group_id, group_scope_tag, group_scope_group_member_id, note_folder_id FROM chat_items @@ -745,7 +765,7 @@ Query: LIMIT ? Plan: -SEARCH chat_items USING INDEX idx_chat_items_group_scope_item_ts (user_id=?) +SEARCH chat_items USING INDEX idx_chat_items_note_folder_msg_content_tag_created_at (user_id=?) USE TEMP B-TREE FOR ORDER BY Query: @@ -756,7 +776,7 @@ Query: LIMIT ? Plan: -SEARCH chat_items USING INDEX idx_chat_items_group_scope_item_ts (user_id=?) +SEARCH chat_items USING INDEX idx_chat_items_note_folder_msg_content_tag_created_at (user_id=?) USE TEMP B-TREE FOR ORDER BY Query: @@ -1225,26 +1245,6 @@ Plan: SEARCH c USING INDEX idx_connections_contact_id (contact_id=?) SEARCH ct USING INTEGER PRIMARY KEY (rowid=?) -Query: - SELECT chat_item_id - FROM chat_items - WHERE user_id = ? AND contact_id = ? - ORDER BY created_at DESC, chat_item_id DESC - LIMIT 1 - -Plan: -SEARCH chat_items USING COVERING INDEX idx_chat_items_contacts_created_at (user_id=? AND contact_id=?) - -Query: - SELECT chat_item_id - FROM chat_items - WHERE user_id = ? AND group_id = ? AND group_member_id = ? - ORDER BY item_ts DESC, chat_item_id DESC - LIMIT 1 - -Plan: -SEARCH chat_items USING INDEX idx_chat_items_groups_item_ts (user_id=? AND group_id=?) - Query: SELECT chat_item_id FROM chat_items @@ -1292,7 +1292,7 @@ Query: LIMIT ? Plan: -SEARCH chat_items USING INDEX idx_chat_items_group_scope_item_ts (user_id=?) +SEARCH chat_items USING INDEX idx_chat_items_note_folder_msg_content_tag_created_at (user_id=?) USE TEMP B-TREE FOR ORDER BY Query: @@ -3131,38 +3131,6 @@ SEARCH chat_item_messages USING INDEX sqlite_autoindex_chat_item_messages_1 (mes LIST SUBQUERY 1 SEARCH msg_deliveries USING INDEX idx_msg_deliveries_agent_msg_id (connection_id=? AND agent_msg_id=?) -Query: - SELECT chat_item_id - FROM chat_items - WHERE user_id = ? AND contact_id = ? AND LOWER(item_text) LIKE '%' || LOWER(?) || '%' - AND (created_at < ? OR (created_at = ? AND chat_item_id < ?)) - ORDER BY created_at DESC, chat_item_id DESC - LIMIT ? - -Plan: -SEARCH chat_items USING INDEX idx_chat_items_contacts_created_at (user_id=? AND contact_id=?) - -Query: - SELECT chat_item_id - FROM chat_items - WHERE user_id = ? AND contact_id = ? AND LOWER(item_text) LIKE '%' || LOWER(?) || '%' - AND (created_at > ? OR (created_at = ? AND chat_item_id > ?)) - ORDER BY created_at ASC, chat_item_id ASC - LIMIT ? - -Plan: -SEARCH chat_items USING INDEX idx_chat_items_contacts_created_at (user_id=? AND contact_id=?) - -Query: - SELECT chat_item_id - FROM chat_items - WHERE user_id = ? AND contact_id = ? AND LOWER(item_text) LIKE '%' || LOWER(?) || '%' - ORDER BY created_at DESC, chat_item_id DESC - LIMIT ? - -Plan: -SEARCH chat_items USING INDEX idx_chat_items_contacts_created_at (user_id=? AND contact_id=?) - Query: SELECT chat_item_id FROM chat_items @@ -3213,38 +3181,6 @@ Query: Plan: SEARCH chat_items USING INDEX idx_chat_items_group_id (group_id=?) -Query: - SELECT chat_item_id - FROM chat_items - WHERE user_id = ? AND note_folder_id = ? AND LOWER(item_text) LIKE '%' || LOWER(?) || '%' - AND (created_at < ? OR (created_at = ? AND chat_item_id < ?)) - ORDER BY created_at DESC, chat_item_id DESC - LIMIT ? - -Plan: -SEARCH chat_items USING INDEX idx_chat_items_notes_created_at (user_id=? AND note_folder_id=?) - -Query: - SELECT chat_item_id - FROM chat_items - WHERE user_id = ? AND note_folder_id = ? AND LOWER(item_text) LIKE '%' || LOWER(?) || '%' - AND (created_at > ? OR (created_at = ? AND chat_item_id > ?)) - ORDER BY created_at ASC, chat_item_id ASC - LIMIT ? - -Plan: -SEARCH chat_items USING INDEX idx_chat_items_notes_created_at (user_id=? AND note_folder_id=?) - -Query: - SELECT chat_item_id - FROM chat_items - WHERE user_id = ? AND note_folder_id = ? AND LOWER(item_text) LIKE '%' || LOWER(?) || '%' - ORDER BY created_at DESC, chat_item_id DESC - LIMIT ? - -Plan: -SEARCH chat_items USING INDEX idx_chat_items_notes_created_at (user_id=? AND note_folder_id=?) - Query: SELECT chat_item_id FROM chat_items @@ -5205,7 +5141,7 @@ Query: JOIN files f ON f.chat_item_id = i.chat_item_id WHERE i.user_id = ? Plan: -SEARCH i USING COVERING INDEX idx_chat_items_group_scope_item_ts (user_id=?) +SEARCH i USING COVERING INDEX idx_chat_items_note_folder_msg_content_tag_created_at (user_id=?) SEARCH f USING INDEX idx_files_chat_item_id (chat_item_id=?) Query: @@ -5475,7 +5411,25 @@ Query: Plan: SEARCH user_contact_links USING INDEX sqlite_autoindex_user_contact_links_1 (user_id=?) -Query: SELECT chat_item_id FROM chat_items WHERE (( user_id = ? AND group_id = ? AND group_scope_tag IS NULL AND group_scope_group_member_id IS NULL AND item_ts < ? ) OR ( user_id = ? AND group_id = ? AND group_scope_tag IS NULL AND group_scope_group_member_id IS NULL AND item_ts = ? AND chat_item_id < ? )) ORDER BY item_ts DESC, chat_item_id DESC LIMIT ? +Query: SELECT chat_item_id FROM chat_items WHERE (( user_id = ? AND contact_id = ? AND created_at < ?) OR ( user_id = ? AND contact_id = ? AND created_at = ? AND chat_item_id < ?)) ORDER BY created_at DESC, chat_item_id DESC LIMIT ? +Plan: +MULTI-INDEX OR +INDEX 1 +SEARCH chat_items USING COVERING INDEX idx_chat_items_contacts_created_at (user_id=? AND contact_id=? AND created_at ?) OR ( user_id = ? AND contact_id = ? AND created_at = ? AND chat_item_id > ?)) ORDER BY created_at ASC, chat_item_id ASC LIMIT ? +Plan: +MULTI-INDEX OR +INDEX 1 +SEARCH chat_items USING COVERING INDEX idx_chat_items_contacts_created_at (user_id=? AND contact_id=? AND created_at>?) +INDEX 2 +SEARCH chat_items USING COVERING INDEX idx_chat_items_contacts_created_at (user_id=? AND contact_id=? AND created_at=? AND rowid>?) +USE TEMP B-TREE FOR ORDER BY + +Query: SELECT chat_item_id FROM chat_items WHERE (( user_id = ? AND group_id = ? AND group_scope_tag IS NULL AND group_scope_group_member_id IS NULL AND item_ts < ?) OR ( user_id = ? AND group_id = ? AND group_scope_tag IS NULL AND group_scope_group_member_id IS NULL AND item_ts = ? AND chat_item_id < ?)) ORDER BY item_ts DESC, chat_item_id DESC LIMIT ? Plan: MULTI-INDEX OR INDEX 1 @@ -5484,7 +5438,7 @@ INDEX 2 SEARCH chat_items USING COVERING INDEX idx_chat_items_group_scope_item_ts (user_id=? AND group_id=? AND group_scope_tag=? AND group_scope_group_member_id=? AND item_ts=? AND rowid ? ) OR ( user_id = ? AND group_id = ? AND group_scope_tag IS NULL AND group_scope_group_member_id IS NULL AND item_ts = ? AND chat_item_id > ? )) ORDER BY item_ts ASC, chat_item_id ASC LIMIT ? +Query: SELECT chat_item_id FROM chat_items WHERE (( user_id = ? AND group_id = ? AND group_scope_tag IS NULL AND group_scope_group_member_id IS NULL AND item_ts > ?) OR ( user_id = ? AND group_id = ? AND group_scope_tag IS NULL AND group_scope_group_member_id IS NULL AND item_ts = ? AND chat_item_id > ?)) ORDER BY item_ts ASC, chat_item_id ASC LIMIT ? Plan: MULTI-INDEX OR INDEX 1 @@ -5493,11 +5447,37 @@ INDEX 2 SEARCH chat_items USING COVERING INDEX idx_chat_items_group_scope_item_ts (user_id=? AND group_id=? AND group_scope_tag=? AND group_scope_group_member_id=? AND item_ts=? AND rowid>?) USE TEMP B-TREE FOR ORDER BY -Query: SELECT chat_item_id FROM chat_items WHERE user_id = ? AND group_id = ? AND group_scope_tag = ? AND group_scope_group_member_id = ? ORDER BY item_ts DESC, chat_item_id DESC LIMIT ? +Query: SELECT chat_item_id FROM chat_items WHERE (( user_id = ? AND note_folder_id = ? AND created_at < ?) OR ( user_id = ? AND note_folder_id = ? AND created_at = ? AND chat_item_id < ?)) ORDER BY created_at DESC, chat_item_id DESC LIMIT ? Plan: -SEARCH chat_items USING COVERING INDEX idx_chat_items_group_scope_item_ts (user_id=? AND group_id=? AND group_scope_tag=? AND group_scope_group_member_id=?) +MULTI-INDEX OR +INDEX 1 +SEARCH chat_items USING COVERING INDEX idx_chat_items_notes_created_at (user_id=? AND note_folder_id=? AND created_at ?) OR ( user_id = ? AND note_folder_id = ? AND created_at = ? AND chat_item_id > ?)) ORDER BY created_at ASC, chat_item_id ASC LIMIT ? +Plan: +MULTI-INDEX OR +INDEX 1 +SEARCH chat_items USING COVERING INDEX idx_chat_items_notes_created_at (user_id=? AND note_folder_id=? AND created_at>?) +INDEX 2 +SEARCH chat_items USING COVERING INDEX idx_chat_items_notes_created_at (user_id=? AND note_folder_id=? AND created_at=? AND rowid>?) +USE TEMP B-TREE FOR ORDER BY + +Query: SELECT chat_item_id FROM chat_items WHERE user_id = ? AND contact_id = ? AND LOWER(item_text) LIKE '%' || LOWER(?) || '%' ORDER BY created_at DESC, chat_item_id DESC LIMIT ? +Plan: +SEARCH chat_items USING INDEX idx_chat_items_contacts_created_at (user_id=? AND contact_id=?) + +Query: SELECT chat_item_id FROM chat_items WHERE user_id = ? AND contact_id = ? AND msg_content_tag = ? ORDER BY created_at DESC, chat_item_id DESC LIMIT ? +Plan: +SEARCH chat_items USING COVERING INDEX idx_chat_items_contacts_msg_content_tag_created_at (user_id=? AND contact_id=? AND msg_content_tag=?) + +Query: SELECT chat_item_id FROM chat_items WHERE user_id = ? AND contact_id = ? ORDER BY created_at DESC, chat_item_id DESC LIMIT ? +Plan: +SEARCH chat_items USING COVERING INDEX idx_chat_items_contacts_created_at (user_id=? AND contact_id=?) + +Query: SELECT chat_item_id FROM chat_items WHERE user_id = ? AND group_id = ? AND group_scope_tag = ? AND group_scope_group_member_id IS NOT DISTINCT FROM ? ORDER BY item_ts DESC, chat_item_id DESC LIMIT ? Plan: SEARCH chat_items USING COVERING INDEX idx_chat_items_group_scope_item_ts (user_id=? AND group_id=? AND group_scope_tag=? AND group_scope_group_member_id=?) @@ -5513,6 +5493,18 @@ Query: SELECT chat_item_id FROM chat_items WHERE user_id = ? AND group_id = ? Plan: SEARCH chat_items USING COVERING INDEX idx_chat_items_groups_msg_content_tag_item_ts (user_id=? AND group_id=? AND msg_content_tag=?) +Query: SELECT chat_item_id FROM chat_items WHERE user_id = ? AND note_folder_id = ? AND LOWER(item_text) LIKE '%' || LOWER(?) || '%' ORDER BY created_at DESC, chat_item_id DESC LIMIT ? +Plan: +SEARCH chat_items USING INDEX idx_chat_items_notes_created_at (user_id=? AND note_folder_id=?) + +Query: SELECT chat_item_id FROM chat_items WHERE user_id = ? AND note_folder_id = ? AND msg_content_tag = ? ORDER BY created_at DESC, chat_item_id DESC LIMIT ? +Plan: +SEARCH chat_items USING COVERING INDEX idx_chat_items_note_folder_msg_content_tag_created_at (user_id=? AND note_folder_id=? AND msg_content_tag=?) + +Query: SELECT chat_item_id FROM chat_items WHERE user_id = ? AND note_folder_id = ? ORDER BY created_at DESC, chat_item_id DESC LIMIT ? +Plan: +SEARCH chat_items USING COVERING INDEX idx_chat_items_notes_created_at (user_id=? AND note_folder_id=?) + Query: CREATE TABLE temp_delete_members (contact_profile_id INTEGER, member_profile_id INTEGER, local_display_name TEXT) Error: SQLite3 returned ErrorError while attempting to perform prepare "explain query plan CREATE TABLE temp_delete_members (contact_profile_id INTEGER, member_profile_id INTEGER, local_display_name TEXT)": table temp_delete_members already exists @@ -5911,7 +5903,7 @@ SEARCH protocol_servers USING COVERING INDEX idx_smp_servers_user_id (user_id=?) SEARCH settings USING COVERING INDEX idx_settings_user_id (user_id=?) SEARCH commands USING COVERING INDEX idx_commands_user_id (user_id=?) SEARCH calls USING COVERING INDEX idx_calls_user_id (user_id=?) -SEARCH chat_items USING COVERING INDEX idx_chat_items_group_scope_item_ts (user_id=?) +SEARCH chat_items USING COVERING INDEX idx_chat_items_note_folder_msg_content_tag_created_at (user_id=?) SEARCH contact_requests USING COVERING INDEX sqlite_autoindex_contact_requests_2 (user_id=?) SEARCH user_contact_links USING COVERING INDEX sqlite_autoindex_user_contact_links_1 (user_id=?) SEARCH connections USING COVERING INDEX idx_connections_to_subscribe (user_id=?) @@ -6059,6 +6051,18 @@ Query: SELECT COUNT(1), COALESCE(SUM(user_mention), 0) FROM chat_items WHERE use Plan: SEARCH chat_items USING COVERING INDEX idx_chat_items_group_scope_stats_all (user_id=? AND group_id=? AND group_scope_tag=? AND group_scope_group_member_id=? AND item_status=?) +Query: SELECT DISTINCT msg_content_tag FROM chat_items WHERE user_id = ? AND contact_id = ? AND msg_content_tag IS NOT NULL ORDER BY msg_content_tag +Plan: +SEARCH chat_items USING COVERING INDEX idx_chat_items_contacts_msg_content_tag_created_at (user_id=? AND contact_id=? AND msg_content_tag>?) + +Query: SELECT DISTINCT msg_content_tag FROM chat_items WHERE user_id = ? AND group_id = ? AND group_scope_tag IS NULL AND group_scope_group_member_id IS NULL AND msg_content_tag IS NOT NULL ORDER BY msg_content_tag +Plan: +SEARCH chat_items USING INDEX idx_chat_items_groups_msg_content_tag_deleted (user_id=? AND group_id=? AND msg_content_tag>?) + +Query: SELECT DISTINCT msg_content_tag FROM chat_items WHERE user_id = ? AND note_folder_id = ? AND msg_content_tag IS NOT NULL ORDER BY msg_content_tag +Plan: +SEARCH chat_items USING COVERING INDEX idx_chat_items_note_folder_msg_content_tag_created_at (user_id=? AND note_folder_id=? AND msg_content_tag>?) + Query: SELECT EXISTS (SELECT 1 FROM chat_items WHERE user_id = ? AND contact_id = ?) Plan: SCAN CONSTANT ROW diff --git a/src/Simplex/Chat/Store/SQLite/Migrations/chat_schema.sql b/src/Simplex/Chat/Store/SQLite/Migrations/chat_schema.sql index fd4fa914d0..83b4c7dbe6 100644 --- a/src/Simplex/Chat/Store/SQLite/Migrations/chat_schema.sql +++ b/src/Simplex/Chat/Store/SQLite/Migrations/chat_schema.sql @@ -1187,6 +1187,18 @@ CREATE UNIQUE INDEX idx_group_members_group_id_index_in_group ON group_members( group_id, index_in_group ); +CREATE INDEX idx_chat_items_contacts_msg_content_tag_created_at ON chat_items( + user_id, + contact_id, + msg_content_tag, + created_at +); +CREATE INDEX idx_chat_items_note_folder_msg_content_tag_created_at ON chat_items( + user_id, + note_folder_id, + msg_content_tag, + created_at +); CREATE TRIGGER on_group_members_insert_update_summary AFTER INSERT ON group_members FOR EACH ROW diff --git a/src/Simplex/Chat/View.hs b/src/Simplex/Chat/View.hs index b515123928..1926bfc3d6 100644 --- a/src/Simplex/Chat/View.hs +++ b/src/Simplex/Chat/View.hs @@ -122,6 +122,7 @@ chatResponseToView hu cfg@ChatConfig {logLevel, showReactions, testView} liveIte CRApiChats u chats -> ttyUser u $ if testView then testViewChats chats else [viewJSON chats] CRChats chats -> viewChats ts tz chats CRApiChat u chat _ -> ttyUser u $ if testView then testViewChat chat else [viewJSON chat] + CRChatContentTypes cts -> [plain $ "Chat content types: " <> T.intercalate ", " (map (safeDecodeUtf8 . strEncode) cts)] CRChatTags u tags -> ttyUser u $ [viewJSON tags] CRServerTestResult u srv testFailure -> ttyUser u $ viewServerTestResult srv testFailure CRServerOperatorConditions (ServerOperatorConditions ops _ ca) -> viewServerOperators ops ca diff --git a/tests/ChatTests/Files.hs b/tests/ChatTests/Files.hs index a71e7ae173..530b85fa91 100644 --- a/tests/ChatTests/Files.hs +++ b/tests/ChatTests/Files.hs @@ -68,7 +68,7 @@ runTestMessageWithFile :: HasCallStack => TestParams -> IO () runTestMessageWithFile = testChat2 aliceProfile bobProfile $ \alice bob -> withXFTPServer $ do connectUsers alice bob - alice ##> "/_send @2 json [{\"filePath\": \"./tests/fixtures/test.jpg\", \"msgContent\": {\"type\": \"text\", \"text\": \"hi, sending a file\"}}]" + alice ##> "/_send @2 json [{\"filePath\": \"./tests/fixtures/test.jpg\", \"msgContent\": {\"type\": \"file\", \"text\": \"hi, sending a file\"}}]" alice <# "@bob hi, sending a file" alice <# "/f @bob ./tests/fixtures/test.jpg" alice <## "use /fc 1 to cancel sending" @@ -83,12 +83,22 @@ runTestMessageWithFile = testChat2 aliceProfile bobProfile $ \alice bob -> withX "started receiving file 1 (test.jpg) from alice" ] bob <## "completed receiving file 1 (test.jpg) from alice" + bob #> "@alice received" + alice <# "bob> received" src <- B.readFile "./tests/fixtures/test.jpg" dest <- B.readFile "./tests/tmp/test.jpg" dest `shouldBe` src - alice #$> ("/_get chat @2 count=100", chatF, chatFeaturesF <> [((1, "hi, sending a file"), Just "./tests/fixtures/test.jpg")]) - bob #$> ("/_get chat @2 count=100", chatF, chatFeaturesF <> [((0, "hi, sending a file"), Just "./tests/tmp/test.jpg")]) + + alice #$> ("/_get chat @2 count=100", chatF, chatFeaturesF <> [((1, "hi, sending a file"), Just "./tests/fixtures/test.jpg"), ((0, "received"), Nothing)]) + alice ##> "/_get content types @2" + alice <## "Chat content types: file, text" + alice #$> ("/_get chat @2 content=file count=100", chatF, [((1, "hi, sending a file"), Just "./tests/fixtures/test.jpg")]) + + bob #$> ("/_get chat @2 count=100", chatF, chatFeaturesF <> [((0, "hi, sending a file"), Just "./tests/tmp/test.jpg"), ((1, "received"), Nothing)]) + bob ##> "/_get content types @2" + bob <## "Chat content types: file, text" + bob #$> ("/_get chat @2 content=file count=100", chatF, [((0, "hi, sending a file"), Just "./tests/tmp/test.jpg")]) testSendImage :: HasCallStack => TestParams -> IO () testSendImage = @@ -343,15 +353,33 @@ testGroupSendImage = "started receiving file 1 (test.jpg) from alice" ] cath <## "completed receiving file 1 (test.jpg) from alice" + threadDelay 1000000 + bob #> "#team received" + [alice, cath] *<# "#team bob> received" + threadDelay 1000000 + cath #> "#team received too" + [alice, bob] *<# "#team cath> received too" src <- B.readFile "./tests/fixtures/test.jpg" dest <- B.readFile "./tests/tmp/test.jpg" dest `shouldBe` src dest2 <- B.readFile "./tests/tmp/test_1.jpg" dest2 `shouldBe` src - alice #$> ("/_get chat #1 count=1", chatF, [((1, ""), Just "./tests/fixtures/test.jpg")]) - bob #$> ("/_get chat #1 count=1", chatF, [((0, ""), Just "./tests/tmp/test.jpg")]) - cath #$> ("/_get chat #1 count=1", chatF, [((0, ""), Just "./tests/tmp/test_1.jpg")]) + + alice #$> ("/_get chat #1 count=3", chatF, [((1, ""), Just "./tests/fixtures/test.jpg"), ((0, "received"), Nothing), ((0, "received too"), Nothing)]) + alice ##> "/_get content types #1" + alice <## "Chat content types: image, text" + alice #$> ("/_get chat #1 content=image count=100", chatF, [((1, ""), Just "./tests/fixtures/test.jpg")]) + + bob #$> ("/_get chat #1 count=3", chatF, [((0, ""), Just "./tests/tmp/test.jpg"), ((1, "received"), Nothing), ((0, "received too"), Nothing)]) + bob ##> "/_get content types #1" + bob <## "Chat content types: image, text" + bob #$> ("/_get chat #1 content=image count=100", chatF, [((0, ""), Just "./tests/tmp/test.jpg")]) + + cath #$> ("/_get chat #1 count=3", chatF, [((0, ""), Just "./tests/tmp/test_1.jpg"), ((0, "received"), Nothing), ((1, "received too"), Nothing)]) + cath ##> "/_get content types #1" + cath <## "Chat content types: image, text" + cath #$> ("/_get chat #1 content=image count=100", chatF, [((0, ""), Just "./tests/tmp/test_1.jpg")]) testGroupSendImageWithTextAndQuote :: HasCallStack => TestParams -> IO () testGroupSendImageWithTextAndQuote = diff --git a/tests/ChatTests/Local.hs b/tests/ChatTests/Local.hs index 985586816f..e4a4da5166 100644 --- a/tests/ChatTests/Local.hs +++ b/tests/ChatTests/Local.hs @@ -134,10 +134,13 @@ testFiles ps = withNewTestChat ps "alice" aliceProfile $ \alice -> do alice ##> "/tail" alice <# "* hi myself" alice <# "* file 1 (test.jpg)" + alice `send` "/* text note" + alice <# "* text note" - alice ##> "/_get chat *1 count=100" - r <- chatF <$> getTermLine alice - r `shouldBe` [((1, "hi myself"), Just "test.jpg")] + alice #$> ("/_get chat *1 count=100", chatF, [((1, "hi myself"), Just "test.jpg"), ((1, "text note"), Nothing)]) + alice ##> "/_get content types *1" + alice <## "Chat content types: image, text" + alice #$> ("/_get chat *1 content=image count=100", chatF, [((1, "hi myself"), Just "test.jpg")]) alice ##> "/fs 1" alice <## "bad chat command: not supported for local files" @@ -151,7 +154,7 @@ testFiles ps = withNewTestChat ps "alice" aliceProfile $ \alice -> do alice ##> "/_create *1 json [{\"filePath\": \"another_test.jpg\", \"msgContent\": {\"text\":\"\",\"type\":\"image\",\"image\":\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII=\"}}]" alice <# "* file 2 (another_test.jpg)" - alice ##> "/_delete item *1 2 internal" + alice ##> "/_delete item *1 3 internal" alice <## "message deleted" doesFileExist stored2 `shouldReturn` False doesFileExist stored `shouldReturn` True