core: get group history faster (#5562)

* core: get group history faster

* revert join, add index (fix test)

* fix postgres compilation

* fix postgres schema

simplex-chat.cabal

@@ -220,6 +220,7 @@ library
           Simplex.Chat.Store.SQLite.Migrations.M20241230_reports
           Simplex.Chat.Store.SQLite.Migrations.M20250105_indexes
           Simplex.Chat.Store.SQLite.Migrations.M20250115_chat_ttl
+          Simplex.Chat.Store.SQLite.Migrations.M20250122_chat_items_include_in_history
   other-modules:
       Paths_simplex_chat
   hs-source-dirs:

src/Simplex/Chat/Store/Groups.hs

@@ -165,9 +165,13 @@ import Simplex.Messaging.Protocol (SubscriptionMode (..))
 import Simplex.Messaging.Util (eitherToMaybe, ($>>=), (<$$>))
 import Simplex.Messaging.Version
 import UnliftIO.STM
-
+#if defined(dbPostgres)
+import Database.PostgreSQL.Simple (Only (..), Query, (:.) (..))
+import Database.PostgreSQL.Simple.SqlQQ (sql)
+#else
 import Database.SQLite.Simple (Only (..), Query, (:.) (..))
 import Database.SQLite.Simple.QQ (sql)
+#endif
 
 type MaybeGroupMemberRow = ((Maybe Int64, Maybe Int64, Maybe MemberId, Maybe VersionChat, Maybe VersionChat, Maybe GroupMemberRole, Maybe GroupMemberCategory, Maybe GroupMemberStatus, Maybe BoolInt, Maybe MemberRestrictionStatus) :. (Maybe Int64, Maybe GroupMemberId, Maybe ContactName, Maybe ContactId, Maybe ProfileId, Maybe ProfileId, Maybe ContactName, Maybe Text, Maybe ImageData, Maybe ConnReqContact, Maybe LocalAlias, Maybe Preferences))
 

src/Simplex/Chat/Store/Messages.hs

@@ -415,20 +415,20 @@ createNewChatItem_ db User {userId} chatDirection msgId_ sharedMsgId ciContent q
         user_id, created_by_msg_id, contact_id, group_id, group_member_id, note_folder_id,
         -- meta
         item_sent, item_ts, item_content, item_content_tag, item_text, item_status, msg_content_tag, shared_msg_id,
-        forwarded_by_group_member_id, created_at, updated_at, item_live, timed_ttl, timed_delete_at,
+        forwarded_by_group_member_id, include_in_history, created_at, updated_at, item_live, timed_ttl, timed_delete_at,
         -- quote
         quoted_shared_msg_id, quoted_sent_at, quoted_content, quoted_sent, quoted_member_id,
         -- forwarded from
         fwd_from_tag, fwd_from_chat_name, fwd_from_msg_dir, fwd_from_contact_id, fwd_from_group_id, fwd_from_chat_item_id
-      ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
+      ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
     |]
     ((userId, msgId_) :. idsRow :. itemRow :. quoteRow' :. forwardedFromRow)
   ciId <- insertedRowId db
   forM_ msgId_ $ \msgId -> insertChatItemMessage_ db ciId msgId createdAt
   pure ciId
   where
-    itemRow :: (SMsgDirection d, UTCTime, CIContent d, Text, Text, CIStatus d, Maybe MsgContentTag, Maybe SharedMsgId, Maybe GroupMemberId) :. (UTCTime, UTCTime, Maybe BoolInt) :. (Maybe Int, Maybe UTCTime)
-    itemRow = (msgDirection @d, itemTs, ciContent, toCIContentTag ciContent, ciContentToText ciContent, ciCreateStatus ciContent, msgContentTag <$> ciMsgContent ciContent, sharedMsgId, forwardedByMember) :. (createdAt, createdAt, BI <$> (justTrue live)) :. ciTimedRow timed
+    itemRow :: (SMsgDirection d, UTCTime, CIContent d, Text, Text, CIStatus d, Maybe MsgContentTag, Maybe SharedMsgId, Maybe GroupMemberId, BoolInt) :. (UTCTime, UTCTime, Maybe BoolInt) :. (Maybe Int, Maybe UTCTime)
+    itemRow = (msgDirection @d, itemTs, ciContent, toCIContentTag ciContent, ciContentToText ciContent, ciCreateStatus ciContent, msgContentTag <$> ciMsgContent ciContent, sharedMsgId, forwardedByMember, BI includeInHistory) :. (createdAt, createdAt, BI <$> (justTrue live)) :. ciTimedRow timed
     quoteRow' = let (a, b, c, d, e) = quoteRow in (a, b, c, BI <$> d, e)
     idsRow :: (Maybe Int64, Maybe Int64, Maybe Int64, Maybe Int64)
     idsRow = case chatDirection of
@@ -438,6 +438,10 @@ createNewChatItem_ db User {userId} chatDirection msgId_ sharedMsgId ciContent q
       CDGroupSnd GroupInfo {groupId} -> (Nothing, Just groupId, Nothing, Nothing)
       CDLocalRcv NoteFolder {noteFolderId} -> (Nothing, Nothing, Nothing, Just noteFolderId)
       CDLocalSnd NoteFolder {noteFolderId} -> (Nothing, Nothing, Nothing, Just noteFolderId)
+    includeInHistory :: Bool
+    includeInHistory =
+      let (_, groupId_, _, _) = idsRow
+       in isJust groupId_ && isJust (ciMsgContent ciContent) && ((msgContentTag <$> ciMsgContent ciContent) /= Just MCReport_)
     forwardedFromRow :: (Maybe CIForwardedFromTag, Maybe Text, Maybe MsgDirection, Maybe Int64, Maybe Int64, Maybe Int64)
     forwardedFromRow = case itemForwarded of
       Nothing ->
@@ -3070,12 +3074,11 @@ getGroupHistoryItems db user@User {userId} GroupInfo {groupId} m count = do
             SELECT i.chat_item_id
             FROM chat_items i
             LEFT JOIN group_snd_item_statuses s ON s.chat_item_id = i.chat_item_id AND s.group_member_id = ?
-            WHERE i.user_id = ? AND i.group_id = ?
-              AND i.item_content_tag IN (?,?)
-              AND i.msg_content_tag NOT IN (?)
+            WHERE s.group_snd_item_status_id IS NULL
+              AND i.user_id = ? AND i.group_id = ?
+              AND i.include_in_history = 1
               AND i.item_deleted = 0
-              AND s.group_snd_item_status_id IS NULL
             ORDER BY i.item_ts DESC, i.chat_item_id DESC
             LIMIT ?
           |]
-          (groupMemberId' m, userId, groupId, rcvMsgContentTag, sndMsgContentTag, MCReport_, count)
+          (groupMemberId' m, userId, groupId, count)

src/Simplex/Chat/Store/Postgres/Migrations/M20241220_initial.hs

@@ -425,7 +425,8 @@ CREATE TABLE chat_items(
   fwd_from_group_id BIGINT REFERENCES groups ON DELETE SET NULL,
   fwd_from_chat_item_id BIGINT REFERENCES chat_items ON DELETE SET NULL,
   via_proxy SMALLINT,
-  msg_content_tag TEXT
+  msg_content_tag TEXT,
+  include_in_history SMALLINT NOT NULL DEFAULT 0
 );
 ALTER TABLE groups
 ADD CONSTRAINT fk_groups_chat_items
@@ -1012,4 +1013,16 @@ CREATE INDEX idx_chat_items_groups_msg_content_tag_deleted ON chat_items(
   item_deleted,
   item_sent
 );
+CREATE INDEX idx_chat_items_groups_history ON chat_items(
+  user_id,
+  group_id,
+  include_in_history,
+  item_deleted,
+  item_ts,
+  chat_item_id
+);
+CREATE INDEX idx_group_snd_item_statuses_chat_item_id_group_member_id ON group_snd_item_statuses(
+  chat_item_id,
+  group_member_id
+);
 |]

src/Simplex/Chat/Store/SQLite/Migrations.hs

@@ -124,6 +124,7 @@ import Simplex.Chat.Store.SQLite.Migrations.M20241223_chat_tags
 import Simplex.Chat.Store.SQLite.Migrations.M20241230_reports
 import Simplex.Chat.Store.SQLite.Migrations.M20250105_indexes
 import Simplex.Chat.Store.SQLite.Migrations.M20250115_chat_ttl
+import Simplex.Chat.Store.SQLite.Migrations.M20250122_chat_items_include_in_history
 import Simplex.Messaging.Agent.Store.Shared (Migration (..))
 
 schemaMigrations :: [(String, Query, Maybe Query)]
@@ -247,7 +248,8 @@ schemaMigrations =
     ("20241223_chat_tags", m20241223_chat_tags, Just down_m20241223_chat_tags),
     ("20241230_reports", m20241230_reports, Just down_m20241230_reports),
     ("20250105_indexes", m20250105_indexes, Just down_m20250105_indexes),
-    ("20250115_chat_ttl", m20250115_chat_ttl, Just down_m20250115_chat_ttl)
+    ("20250115_chat_ttl", m20250115_chat_ttl, Just down_m20250115_chat_ttl),
+    ("20250122_chat_items_include_in_history", m20250122_chat_items_include_in_history, Just down_m20250122_chat_items_include_in_history)
   ]
 
 -- | The list of migrations in ascending order by date

src/Simplex/Chat/Store/SQLite/Migrations/M20250122_chat_items_include_in_history.hs

@@ -0,0 +1,39 @@
+{-# LANGUAGE QuasiQuotes #-}
+
+module Simplex.Chat.Store.SQLite.Migrations.M20250122_chat_items_include_in_history where
+
+import Database.SQLite.Simple (Query)
+import Database.SQLite.Simple.QQ (sql)
+
+m20250122_chat_items_include_in_history :: Query
+m20250122_chat_items_include_in_history =
+  [sql|
+ALTER TABLE chat_items ADD COLUMN include_in_history INTEGER NOT NULL DEFAULT 0;
+
+CREATE INDEX idx_chat_items_groups_history ON chat_items(
+  user_id,
+  group_id,
+  include_in_history,
+  item_deleted,
+  item_ts,
+  chat_item_id
+);
+
+UPDATE chat_items
+SET include_in_history = 1
+WHERE group_id IS NOT NULL
+  AND item_content_tag IN ('rcvMsgContent', 'sndMsgContent')
+  AND msg_content_tag NOT IN ('report');
+
+CREATE INDEX idx_group_snd_item_statuses_chat_item_id_group_member_id ON group_snd_item_statuses(chat_item_id, group_member_id);
+|]
+
+down_m20250122_chat_items_include_in_history :: Query
+down_m20250122_chat_items_include_in_history =
+  [sql|
+DROP INDEX idx_group_snd_item_statuses_chat_item_id_group_member_id;
+
+DROP INDEX idx_chat_items_groups_history;
+
+ALTER TABLE chat_items DROP COLUMN include_in_history;
+|]

src/Simplex/Chat/Store/SQLite/Migrations/chat_schema.sql

@@ -406,7 +406,8 @@ CREATE TABLE chat_items(
   fwd_from_group_id INTEGER REFERENCES groups ON DELETE SET NULL,
   fwd_from_chat_item_id INTEGER REFERENCES chat_items ON DELETE SET NULL,
   via_proxy INTEGER,
-  msg_content_tag TEXT
+  msg_content_tag TEXT,
+  include_in_history INTEGER NOT NULL DEFAULT 0
 );
 CREATE TABLE sqlite_sequence(name,seq);
 CREATE TABLE chat_item_messages(
@@ -978,3 +979,15 @@ CREATE INDEX idx_chat_items_groups_msg_content_tag_deleted ON chat_items(
   item_deleted,
   item_sent
 );
+CREATE INDEX idx_chat_items_groups_history ON chat_items(
+  user_id,
+  group_id,
+  include_in_history,
+  item_deleted,
+  item_ts,
+  chat_item_id
+);
+CREATE INDEX idx_group_snd_item_statuses_chat_item_id_group_member_id ON group_snd_item_statuses(
+  chat_item_id,
+  group_member_id
+);