dandri/simplex-chat
1
0
Fork 0
mirror of https://github.com/simplex-chat/simplex-chat.git synced 2026-05-25 02:05:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

core: improve chat list pagination performance, simplify logic by always reading chat stats and last item id for previews (#3541)

Browse Source 
* core: improve chat list pagination performance

* fix query

* core: improve chat list pagination performance, simplify logic by always reading chat stats (#3543)

* microseconds

* fix

* update simplexmq

* simplify queries

---------

Co-authored-by: Evgeny Poberezkin <2769109+epoberezkin@users.noreply.github.com>
This commit is contained in:
spaced4ndy
2023-12-13 15:32:23 +04:00
committed by GitHub
parent 7f9f9a674c
commit b6b041490f
3 changed files with 121 additions and 218 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Simplex/Chat/Migrations/M20231207_chat_list_pagination.hs
-6
View File
@@ -26,9 +26,6 @@ CREATE INDEX idx_contacts_chat_ts ON contacts(user_id, chat_ts);
CREATE INDEX idx_groups_chat_ts ON groups(user_id, chat_ts);
CREATE INDEX idx_contact_requests_updated_at ON contact_requests(user_id, updated_at);
CREATE INDEX idx_connections_updated_at ON connections(user_id, updated_at);
CREATE INDEX idx_chat_items_contact_id_item_status ON chat_items(contact_id, item_status);
CREATE INDEX idx_chat_items_group_id_item_status ON chat_items(group_id, item_status);
|]
down_m20231207_chat_list_pagination :: Query
@@ -38,7 +35,4 @@ DROP INDEX idx_contacts_chat_ts;
DROP INDEX idx_groups_chat_ts;
DROP INDEX idx_contact_requests_updated_at;
DROP INDEX idx_connections_updated_at;
DROP INDEX idx_chat_items_contact_id_item_status;
DROP INDEX idx_chat_items_group_id_item_status;
|]
src/Simplex/Chat/Migrations/chat_schema.sql
-8
View File
@@ -817,11 +817,3 @@ CREATE INDEX idx_contact_requests_updated_at ON contact_requests(
updated_at
);
CREATE INDEX idx_connections_updated_at ON connections(user_id, updated_at);
CREATE INDEX idx_chat_items_contact_id_item_status ON chat_items(
contact_id,
item_status
);
CREATE INDEX idx_chat_items_group_id_item_status ON chat_items(
group_id,
item_status
);
src/Simplex/Chat/Store/Messages.hs
+121 -204
View File
@@ -499,8 +499,8 @@ getChatPreviews db user withPCC pagination query = do
where
ts :: AChatPreviewData -> UTCTime
ts (ACPD _ cpd) = case cpd of
(DirectChatPD t _ _) -> t
(GroupChatPD t _ _) -> t
(DirectChatPD t _ _ _) -> t
(GroupChatPD t _ _ _) -> t
(ContactRequestPD t _) -> t
(ContactConnectionPD t _) -> t
sortTake = case pagination of
@@ -515,8 +515,8 @@ getChatPreviews db user withPCC pagination query = do
SCTContactConnection -> let (ContactConnectionPD _ chat) = cpd in pure chat
data ChatPreviewData (c :: ChatType) where
DirectChatPD :: UTCTime -> ContactId -> Maybe ChatStats -> ChatPreviewData 'CTDirect
GroupChatPD :: UTCTime -> GroupId -> Maybe ChatStats -> ChatPreviewData 'CTGroup
DirectChatPD :: UTCTime -> ContactId -> Maybe ChatItemId -> ChatStats -> ChatPreviewData 'CTDirect
GroupChatPD :: UTCTime -> GroupId -> Maybe ChatItemId -> ChatStats -> ChatPreviewData 'CTGroup
ContactRequestPD :: UTCTime -> AChat -> ChatPreviewData 'CTContactRequest
ContactConnectionPD :: UTCTime -> AChat -> ChatPreviewData 'CTContactConnection
@@ -528,283 +528,200 @@ paginationByTimeFilter = \case
PTAfter ts count -> ("\nAND ts > :ts ORDER BY ts ASC LIMIT :count", [":ts" := ts, ":count" := count])
PTBefore ts count -> ("\nAND ts < :ts ORDER BY ts DESC LIMIT :count", [":ts" := ts, ":count" := count])
type MaybeChatStatsRow = (Maybe Int, Maybe ChatItemId, Maybe Bool)
type ChatStatsRow = (Int, ChatItemId, Bool)
toMaybeChatStats :: MaybeChatStatsRow -> Maybe ChatStats
toMaybeChatStats (Just unreadCount, Just minUnreadItemId, Just unreadChat) = Just ChatStats {unreadCount, minUnreadItemId, unreadChat}
toMaybeChatStats _ = Nothing
toChatStats :: ChatStatsRow -> ChatStats
toChatStats (unreadCount, minUnreadItemId, unreadChat) = ChatStats {unreadCount, minUnreadItemId, unreadChat}
findDirectChatPreviews_ :: DB.Connection -> User -> PaginationByTime -> ChatListQuery -> IO [AChatPreviewData]
findDirectChatPreviews_ db User {userId} pagination clq =
map toPreview <$> getPreviews
where
toPreview :: (ContactId, UTCTime) :. MaybeChatStatsRow -> AChatPreviewData
toPreview ((contactId, ts) :. statsRow_) =
ACPD SCTDirect $ DirectChatPD ts contactId (toMaybeChatStats statsRow_)
toPreview :: (ContactId, UTCTime, Maybe ChatItemId) :. ChatStatsRow -> AChatPreviewData
toPreview ((contactId, ts, lastItemId_) :. statsRow) =
ACPD SCTDirect $ DirectChatPD ts contactId lastItemId_ (toChatStats statsRow)
baseQuery =
[sql|
SELECT ct.contact_id, ct.chat_ts as ts, LastItems.chat_item_id, COALESCE(ChatStats.UnreadCount, 0), COALESCE(ChatStats.MinUnread, 0), ct.unread_chat
FROM contacts ct
LEFT JOIN (
SELECT contact_id, chat_item_id, MAX(created_at)
FROM chat_items
GROUP BY contact_id
) LastItems ON LastItems.contact_id = ct.contact_id
LEFT JOIN (
SELECT contact_id, COUNT(1) AS UnreadCount, MIN(chat_item_id) AS MinUnread
FROM chat_items
WHERE item_status = :rcv_new
GROUP BY contact_id
) ChatStats ON ChatStats.contact_id = ct.contact_id
|]
(pagQuery, pagParams) = paginationByTimeFilter pagination
getPreviews = case clq of
CLQFilters {favorite = False, unread = False} ->
DB.queryNamed
db
( [sql|
SELECT ct.contact_id, ct.chat_ts as ts, NULL, NULL, NULL
FROM contacts ct
WHERE ct.user_id = :user_id AND ct.is_user = 0 AND ct.deleted = 0 AND ct.contact_used
|]
( baseQuery
<> [sql|
WHERE ct.user_id = :user_id AND ct.is_user = 0 AND ct.deleted = 0 AND ct.contact_used
|]
<> pagQuery
)
([":user_id" := userId] <> pagParams)
([":user_id" := userId, ":rcv_new" := CISRcvNew] <> pagParams)
CLQFilters {favorite = True, unread = False} ->
DB.queryNamed
db
( [sql|
SELECT ct.contact_id, ct.chat_ts as ts, NULL, NULL, NULL
FROM contacts ct
WHERE ct.user_id = :user_id AND ct.is_user = 0 AND ct.deleted = 0 AND ct.contact_used
AND ct.favorite = 1
|]
( baseQuery
<> [sql|
WHERE ct.user_id = :user_id AND ct.is_user = 0 AND ct.deleted = 0 AND ct.contact_used
AND ct.favorite = 1
|]
<> pagQuery
)
([":user_id" := userId] <> pagParams)
([":user_id" := userId, ":rcv_new" := CISRcvNew] <> pagParams)
CLQFilters {favorite = False, unread = True} ->
DB.queryNamed
db
( [sql|
SELECT ct.contact_id, ct.chat_ts as ts, COALESCE(ChatStats.UnreadCount, 0), COALESCE(ChatStats.MinUnread, 0), ct.unread_chat
FROM contacts ct
LEFT JOIN (
SELECT contact_id, COUNT(1) AS UnreadCount, MIN(chat_item_id) AS MinUnread
FROM chat_items
WHERE item_status = :rcv_new
GROUP BY contact_id
) ChatStats ON ChatStats.contact_id = ct.contact_id
WHERE ct.user_id = :user_id AND ct.is_user = 0 AND ct.deleted = 0 AND ct.contact_used
AND (ct.unread_chat = 1 OR ChatStats.UnreadCount > 0)
|]
( baseQuery
<> [sql|
WHERE ct.user_id = :user_id AND ct.is_user = 0 AND ct.deleted = 0 AND ct.contact_used
AND (ct.unread_chat = 1 OR ChatStats.UnreadCount > 0)
|]
<> pagQuery
)
([":user_id" := userId, ":rcv_new" := CISRcvNew] <> pagParams)
CLQFilters {favorite = True, unread = True} ->
DB.queryNamed
db
( [sql|
SELECT ct.contact_id, ct.chat_ts as ts, COALESCE(ChatStats.UnreadCount, 0), COALESCE(ChatStats.MinUnread, 0), ct.unread_chat
FROM contacts ct
LEFT JOIN (
SELECT contact_id, COUNT(1) AS UnreadCount, MIN(chat_item_id) AS MinUnread
FROM chat_items
WHERE item_status = :rcv_new
GROUP BY contact_id
) ChatStats ON ChatStats.contact_id = ct.contact_id
WHERE ct.user_id = :user_id AND ct.is_user = 0 AND ct.deleted = 0 AND ct.contact_used
AND (ct.favorite = 1
OR ct.unread_chat = 1 OR ChatStats.UnreadCount > 0)
|]
( baseQuery
<> [sql|
WHERE ct.user_id = :user_id AND ct.is_user = 0 AND ct.deleted = 0 AND ct.contact_used
AND (ct.favorite = 1
OR ct.unread_chat = 1 OR ChatStats.UnreadCount > 0)
|]
<> pagQuery
)
([":user_id" := userId, ":rcv_new" := CISRcvNew] <> pagParams)
CLQSearch {search} ->
DB.queryNamed
db
( [sql|
SELECT ct.contact_id, ct.chat_ts as ts, NULL, NULL, NULL
FROM contacts ct
JOIN contact_profiles cp ON ct.contact_profile_id = cp.contact_profile_id
WHERE ct.user_id = :user_id AND ct.is_user = 0 AND ct.deleted = 0 AND ct.contact_used
AND (
ct.local_display_name LIKE '%' || :search || '%'
OR cp.display_name LIKE '%' || :search || '%'
OR cp.full_name LIKE '%' || :search || '%'
OR cp.local_alias LIKE '%' || :search || '%'
)
|]
( baseQuery
<> [sql|
JOIN contact_profiles cp ON ct.contact_profile_id = cp.contact_profile_id
WHERE ct.user_id = :user_id AND ct.is_user = 0 AND ct.deleted = 0 AND ct.contact_used
AND (
ct.local_display_name LIKE '%' || :search || '%'
OR cp.display_name LIKE '%' || :search || '%'
OR cp.full_name LIKE '%' || :search || '%'
OR cp.local_alias LIKE '%' || :search || '%'
)
|]
<> pagQuery
)
([":user_id" := userId, ":search" := search] <> pagParams)
([":user_id" := userId, ":rcv_new" := CISRcvNew, ":search" := search] <> pagParams)
getDirectChatPreview_ :: DB.Connection -> User -> ChatPreviewData 'CTDirect -> ExceptT StoreError IO AChat
getDirectChatPreview_ db user (DirectChatPD _ contactId stats_) = do
getDirectChatPreview_ db user (DirectChatPD _ contactId lastItemId_ stats) = do
contact <- getContact db user contactId
lastItem <- getLastItem
stats <- maybe getChatStats pure stats_
lastItem <- case lastItemId_ of
Just lastItemId -> (: []) <$> getDirectChatItem db user contactId lastItemId
Nothing -> pure []
pure $ AChat SCTDirect (Chat (DirectChat contact) lastItem stats)
where
getLastItem :: ExceptT StoreError IO [CChatItem 'CTDirect]
getLastItem =
liftIO getLastItemId >>= \case
Nothing -> pure []
Just lastItemId -> (: []) <$> getDirectChatItem db user contactId lastItemId
getLastItemId :: IO (Maybe ChatItemId)
getLastItemId =
maybeFirstRow fromOnly $
DB.query
db
[sql|
SELECT chat_item_id FROM (
SELECT contact_id, chat_item_id, MAX(created_at)
FROM chat_items
WHERE contact_id = ?
GROUP BY contact_id
)
|]
(Only contactId)
getChatStats :: ExceptT StoreError IO ChatStats
getChatStats = do
r_ <- liftIO getUnreadStats
let (unreadCount, minUnreadItemId) = maybe (0, 0) (\(_, unreadCnt, minId) -> (unreadCnt, minId)) r_
-- unread_chat could be read into contact to not search twice
unreadChat <-
ExceptT . firstRow fromOnly (SEInternalError $ "unread_chat not found for contact " <> show contactId) $
DB.query db "SELECT unread_chat FROM contacts WHERE contact_id = ?" (Only contactId)
pure ChatStats {unreadCount, minUnreadItemId, unreadChat}
getUnreadStats :: IO (Maybe (ContactId, Int, ChatItemId))
getUnreadStats =
maybeFirstRow id $
DB.query
db
[sql|
SELECT contact_id, COUNT(1) AS UnreadCount, MIN(chat_item_id) AS MinUnread
FROM chat_items
WHERE contact_id = ? AND item_status = ?
GROUP BY contact_id
|]
(contactId, CISRcvNew)
findGroupChatPreviews_ :: DB.Connection -> User -> PaginationByTime -> ChatListQuery -> IO [AChatPreviewData]
findGroupChatPreviews_ db User {userId} pagination clq =
map toPreview <$> getPreviews
where
toPreview :: (GroupId, UTCTime) :. MaybeChatStatsRow -> AChatPreviewData
toPreview ((groupId, ts) :. statsRow_) =
ACPD SCTGroup $ GroupChatPD ts groupId (toMaybeChatStats statsRow_)
toPreview :: (GroupId, UTCTime, Maybe ChatItemId) :. ChatStatsRow -> AChatPreviewData
toPreview ((groupId, ts, lastItemId_) :. statsRow) =
ACPD SCTGroup $ GroupChatPD ts groupId lastItemId_ (toChatStats statsRow)
baseQuery =
[sql|
SELECT g.group_id, g.chat_ts as ts, LastItems.chat_item_id, COALESCE(ChatStats.UnreadCount, 0), COALESCE(ChatStats.MinUnread, 0), g.unread_chat
FROM groups g
LEFT JOIN (
SELECT group_id, chat_item_id, MAX(item_ts)
FROM chat_items
GROUP BY group_id
) LastItems ON LastItems.group_id = g.group_id
LEFT JOIN (
SELECT group_id, COUNT(1) AS UnreadCount, MIN(chat_item_id) AS MinUnread
FROM chat_items
WHERE item_status = :rcv_new
GROUP BY group_id
) ChatStats ON ChatStats.group_id = g.group_id
|]
(pagQuery, pagParams) = paginationByTimeFilter pagination
getPreviews = case clq of
CLQFilters {favorite = False, unread = False} ->
DB.queryNamed
db
( [sql|
SELECT g.group_id, g.chat_ts as ts, NULL, NULL, NULL
FROM groups g
WHERE g.user_id = :user_id
|]
( baseQuery
<> [sql|
WHERE g.user_id = :user_id
|]
<> pagQuery
)
([":user_id" := userId] <> pagParams)
([":user_id" := userId, ":rcv_new" := CISRcvNew] <> pagParams)
CLQFilters {favorite = True, unread = False} ->
DB.queryNamed
db
( [sql|
SELECT g.group_id, g.chat_ts as ts, NULL, NULL, NULL
FROM groups g
WHERE g.user_id = :user_id
AND g.favorite = 1
|]
( baseQuery
<> [sql|
WHERE g.user_id = :user_id
AND g.favorite = 1
|]
<> pagQuery
)
([":user_id" := userId] <> pagParams)
([":user_id" := userId, ":rcv_new" := CISRcvNew] <> pagParams)
CLQFilters {favorite = False, unread = True} ->
DB.queryNamed
db
( [sql|
SELECT g.group_id, g.chat_ts as ts, COALESCE(ChatStats.UnreadCount, 0), COALESCE(ChatStats.MinUnread, 0), g.unread_chat
FROM groups g
LEFT JOIN (
SELECT group_id, COUNT(1) AS UnreadCount, MIN(chat_item_id) AS MinUnread
FROM chat_items
WHERE item_status = :rcv_new
GROUP BY group_id
) ChatStats ON ChatStats.group_id = g.group_id
WHERE g.user_id = :user_id
AND (g.unread_chat = 1 OR ChatStats.UnreadCount > 0)
|]
( baseQuery
<> [sql|
WHERE g.user_id = :user_id
AND (g.unread_chat = 1 OR ChatStats.UnreadCount > 0)
|]
<> pagQuery
)
([":user_id" := userId, ":rcv_new" := CISRcvNew] <> pagParams)
CLQFilters {favorite = True, unread = True} ->
DB.queryNamed
db
( [sql|
SELECT g.group_id, g.chat_ts as ts, COALESCE(ChatStats.UnreadCount, 0), COALESCE(ChatStats.MinUnread, 0), g.unread_chat
FROM groups g
LEFT JOIN (
SELECT group_id, COUNT(1) AS UnreadCount, MIN(chat_item_id) AS MinUnread
FROM chat_items
WHERE item_status = :rcv_new
GROUP BY group_id
) ChatStats ON ChatStats.group_id = g.group_id
WHERE g.user_id = :user_id
AND (g.favorite = 1
OR g.unread_chat = 1 OR ChatStats.UnreadCount > 0)
|]
( baseQuery
<> [sql|
WHERE g.user_id = :user_id
AND (g.favorite = 1
OR g.unread_chat = 1 OR ChatStats.UnreadCount > 0)
|]
<> pagQuery
)
([":user_id" := userId, ":rcv_new" := CISRcvNew] <> pagParams)
CLQSearch {search} ->
DB.queryNamed
db
( [sql|
SELECT g.group_id, g.chat_ts as ts, NULL, NULL, NULL
FROM groups g
JOIN group_profiles gp ON gp.group_profile_id = g.group_profile_id
WHERE g.user_id = :user_id
AND (
g.local_display_name LIKE '%' || :search || '%'
OR gp.display_name LIKE '%' || :search || '%'
OR gp.full_name LIKE '%' || :search || '%'
OR gp.description LIKE '%' || :search || '%'
)
|]
( baseQuery
<> [sql|
JOIN group_profiles gp ON gp.group_profile_id = g.group_profile_id
WHERE g.user_id = :user_id
AND (
g.local_display_name LIKE '%' || :search || '%'
OR gp.display_name LIKE '%' || :search || '%'
OR gp.full_name LIKE '%' || :search || '%'
OR gp.description LIKE '%' || :search || '%'
)
|]
<> pagQuery
)
([":user_id" := userId, ":search" := search] <> pagParams)
([":user_id" := userId, ":rcv_new" := CISRcvNew, ":search" := search] <> pagParams)
getGroupChatPreview_ :: DB.Connection -> User -> ChatPreviewData 'CTGroup -> ExceptT StoreError IO AChat
getGroupChatPreview_ db user (GroupChatPD _ groupId stats_) = do
getGroupChatPreview_ db user (GroupChatPD _ groupId lastItemId_ stats) = do
groupInfo <- getGroupInfo db user groupId
lastItem <- getLastItem
stats <- maybe getChatStats pure stats_
lastItem <- case lastItemId_ of
Just lastItemId -> (: []) <$> getGroupChatItem db user groupId lastItemId
Nothing -> pure []
pure $ AChat SCTGroup (Chat (GroupChat groupInfo) lastItem stats)
where
getLastItem :: ExceptT StoreError IO [CChatItem 'CTGroup]
getLastItem =
liftIO getLastItemId >>= \case
Nothing -> pure []
Just lastItemId -> (: []) <$> getGroupChatItem db user groupId lastItemId
getLastItemId :: IO (Maybe ChatItemId)
getLastItemId =
maybeFirstRow fromOnly $
DB.query
db
[sql|
SELECT chat_item_id FROM (
SELECT group_id, chat_item_id, MAX(item_ts)
FROM chat_items
WHERE group_id = ?
GROUP BY group_id
)
|]
(Only groupId)
getChatStats :: ExceptT StoreError IO ChatStats
getChatStats = do
r_ <- liftIO getUnreadStats
let (unreadCount, minUnreadItemId) = maybe (0, 0) (\(_, unreadCnt, minId) -> (unreadCnt, minId)) r_
-- unread_chat could be read into group to not search twice
unreadChat <-
ExceptT . firstRow fromOnly (SEInternalError $ "unread_chat not found for group " <> show groupId) $
DB.query db "SELECT unread_chat FROM groups WHERE group_id = ?" (Only groupId)
pure ChatStats {unreadCount, minUnreadItemId, unreadChat}
getUnreadStats :: IO (Maybe (GroupId, Int, ChatItemId))
getUnreadStats =
maybeFirstRow id $
DB.query
db
[sql|
SELECT group_id, COUNT(1) AS UnreadCount, MIN(chat_item_id) AS MinUnread
FROM chat_items
WHERE group_id = ? AND item_status = ?
GROUP BY group_id
|]
(groupId, CISRcvNew)
getContactRequestChatPreviews_ :: DB.Connection -> User -> PaginationByTime -> ChatListQuery -> IO [AChatPreviewData]
getContactRequestChatPreviews_ db User {userId} pagination clq = case clq of