From 9ccea0dc500e9981bfad919e6a8e8f08dea48d60 Mon Sep 17 00:00:00 2001
From: spaced4ndy <8711996+spaced4ndy@users.noreply.github.com>
Date: Wed, 22 Jan 2025 23:33:54 +0400
Subject: [PATCH] core: get group history faster (#5562)

* core: get group history faster

* revert join, add index (fix test)

* fix postgres compilation

* fix postgres schema
---
 simplex-chat.cabal                            |  1 +
 src/Simplex/Chat/Store/Groups.hs              |  6 ++-
 src/Simplex/Chat/Store/Messages.hs            | 21 +++++-----
 .../Postgres/Migrations/M20241220_initial.hs  | 15 ++++++-
 src/Simplex/Chat/Store/SQLite/Migrations.hs   |  4 +-
 ...M20250122_chat_items_include_in_history.hs | 39 +++++++++++++++++++
 .../Store/SQLite/Migrations/chat_schema.sql   | 15 ++++++-
 7 files changed, 88 insertions(+), 13 deletions(-)
 create mode 100644 src/Simplex/Chat/Store/SQLite/Migrations/M20250122_chat_items_include_in_history.hs

diff --git a/simplex-chat.cabal b/simplex-chat.cabal
index 6b0b8bdd82..e481c91687 100644
--- a/simplex-chat.cabal
+++ b/simplex-chat.cabal
@@ -220,6 +220,7 @@ library
           Simplex.Chat.Store.SQLite.Migrations.M20241230_reports
           Simplex.Chat.Store.SQLite.Migrations.M20250105_indexes
           Simplex.Chat.Store.SQLite.Migrations.M20250115_chat_ttl
+          Simplex.Chat.Store.SQLite.Migrations.M20250122_chat_items_include_in_history
   other-modules:
       Paths_simplex_chat
   hs-source-dirs:
diff --git a/src/Simplex/Chat/Store/Groups.hs b/src/Simplex/Chat/Store/Groups.hs
index 1d1a715b78..eb820eb66d 100644
--- a/src/Simplex/Chat/Store/Groups.hs
+++ b/src/Simplex/Chat/Store/Groups.hs
@@ -165,9 +165,13 @@ import Simplex.Messaging.Protocol (SubscriptionMode (..))
 import Simplex.Messaging.Util (eitherToMaybe, ($>>=), (<$$>))
 import Simplex.Messaging.Version
 import UnliftIO.STM
-
+#if defined(dbPostgres)
+import Database.PostgreSQL.Simple (Only (..), Query, (:.) (..))
+import Database.PostgreSQL.Simple.SqlQQ (sql)
+#else
 import Database.SQLite.Simple (Only (..), Query, (:.) (..))
 import Database.SQLite.Simple.QQ (sql)
+#endif
 
 type MaybeGroupMemberRow = ((Maybe Int64, Maybe Int64, Maybe MemberId, Maybe VersionChat, Maybe VersionChat, Maybe GroupMemberRole, Maybe GroupMemberCategory, Maybe GroupMemberStatus, Maybe BoolInt, Maybe MemberRestrictionStatus) :. (Maybe Int64, Maybe GroupMemberId, Maybe ContactName, Maybe ContactId, Maybe ProfileId, Maybe ProfileId, Maybe ContactName, Maybe Text, Maybe ImageData, Maybe ConnReqContact, Maybe LocalAlias, Maybe Preferences))
 
diff --git a/src/Simplex/Chat/Store/Messages.hs b/src/Simplex/Chat/Store/Messages.hs
index a828a30925..58877210dc 100644
--- a/src/Simplex/Chat/Store/Messages.hs
+++ b/src/Simplex/Chat/Store/Messages.hs
@@ -415,20 +415,20 @@ createNewChatItem_ db User {userId} chatDirection msgId_ sharedMsgId ciContent q
         user_id, created_by_msg_id, contact_id, group_id, group_member_id, note_folder_id,
         -- meta
         item_sent, item_ts, item_content, item_content_tag, item_text, item_status, msg_content_tag, shared_msg_id,
-        forwarded_by_group_member_id, created_at, updated_at, item_live, timed_ttl, timed_delete_at,
+        forwarded_by_group_member_id, include_in_history, created_at, updated_at, item_live, timed_ttl, timed_delete_at,
         -- quote
         quoted_shared_msg_id, quoted_sent_at, quoted_content, quoted_sent, quoted_member_id,
         -- forwarded from
         fwd_from_tag, fwd_from_chat_name, fwd_from_msg_dir, fwd_from_contact_id, fwd_from_group_id, fwd_from_chat_item_id
-      ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
+      ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
     |]
     ((userId, msgId_) :. idsRow :. itemRow :. quoteRow' :. forwardedFromRow)
   ciId <- insertedRowId db
   forM_ msgId_ $ \msgId -> insertChatItemMessage_ db ciId msgId createdAt
   pure ciId
   where
-    itemRow :: (SMsgDirection d, UTCTime, CIContent d, Text, Text, CIStatus d, Maybe MsgContentTag, Maybe SharedMsgId, Maybe GroupMemberId) :. (UTCTime, UTCTime, Maybe BoolInt) :. (Maybe Int, Maybe UTCTime)
-    itemRow = (msgDirection @d, itemTs, ciContent, toCIContentTag ciContent, ciContentToText ciContent, ciCreateStatus ciContent, msgContentTag <$> ciMsgContent ciContent, sharedMsgId, forwardedByMember) :. (createdAt, createdAt, BI <$> (justTrue live)) :. ciTimedRow timed
+    itemRow :: (SMsgDirection d, UTCTime, CIContent d, Text, Text, CIStatus d, Maybe MsgContentTag, Maybe SharedMsgId, Maybe GroupMemberId, BoolInt) :. (UTCTime, UTCTime, Maybe BoolInt) :. (Maybe Int, Maybe UTCTime)
+    itemRow = (msgDirection @d, itemTs, ciContent, toCIContentTag ciContent, ciContentToText ciContent, ciCreateStatus ciContent, msgContentTag <$> ciMsgContent ciContent, sharedMsgId, forwardedByMember, BI includeInHistory) :. (createdAt, createdAt, BI <$> (justTrue live)) :. ciTimedRow timed
     quoteRow' = let (a, b, c, d, e) = quoteRow in (a, b, c, BI <$> d, e)
     idsRow :: (Maybe Int64, Maybe Int64, Maybe Int64, Maybe Int64)
     idsRow = case chatDirection of
@@ -438,6 +438,10 @@ createNewChatItem_ db User {userId} chatDirection msgId_ sharedMsgId ciContent q
       CDGroupSnd GroupInfo {groupId} -> (Nothing, Just groupId, Nothing, Nothing)
       CDLocalRcv NoteFolder {noteFolderId} -> (Nothing, Nothing, Nothing, Just noteFolderId)
       CDLocalSnd NoteFolder {noteFolderId} -> (Nothing, Nothing, Nothing, Just noteFolderId)
+    includeInHistory :: Bool
+    includeInHistory =
+      let (_, groupId_, _, _) = idsRow
+       in isJust groupId_ && isJust (ciMsgContent ciContent) && ((msgContentTag <$> ciMsgContent ciContent) /= Just MCReport_)
     forwardedFromRow :: (Maybe CIForwardedFromTag, Maybe Text, Maybe MsgDirection, Maybe Int64, Maybe Int64, Maybe Int64)
     forwardedFromRow = case itemForwarded of
       Nothing ->
@@ -3070,12 +3074,11 @@ getGroupHistoryItems db user@User {userId} GroupInfo {groupId} m count = do
             SELECT i.chat_item_id
             FROM chat_items i
             LEFT JOIN group_snd_item_statuses s ON s.chat_item_id = i.chat_item_id AND s.group_member_id = ?
-            WHERE i.user_id = ? AND i.group_id = ?
-              AND i.item_content_tag IN (?,?)
-              AND i.msg_content_tag NOT IN (?)
+            WHERE s.group_snd_item_status_id IS NULL
+              AND i.user_id = ? AND i.group_id = ?
+              AND i.include_in_history = 1
               AND i.item_deleted = 0
-              AND s.group_snd_item_status_id IS NULL
             ORDER BY i.item_ts DESC, i.chat_item_id DESC
             LIMIT ?
           |]
-          (groupMemberId' m, userId, groupId, rcvMsgContentTag, sndMsgContentTag, MCReport_, count)
+          (groupMemberId' m, userId, groupId, count)
diff --git a/src/Simplex/Chat/Store/Postgres/Migrations/M20241220_initial.hs b/src/Simplex/Chat/Store/Postgres/Migrations/M20241220_initial.hs
index e8fd77aa0d..60a2ace8f3 100644
--- a/src/Simplex/Chat/Store/Postgres/Migrations/M20241220_initial.hs
+++ b/src/Simplex/Chat/Store/Postgres/Migrations/M20241220_initial.hs
@@ -425,7 +425,8 @@ CREATE TABLE chat_items(
   fwd_from_group_id BIGINT REFERENCES groups ON DELETE SET NULL,
   fwd_from_chat_item_id BIGINT REFERENCES chat_items ON DELETE SET NULL,
   via_proxy SMALLINT,
-  msg_content_tag TEXT
+  msg_content_tag TEXT,
+  include_in_history SMALLINT NOT NULL DEFAULT 0
 );
 ALTER TABLE groups
 ADD CONSTRAINT fk_groups_chat_items
@@ -1012,4 +1013,16 @@ CREATE INDEX idx_chat_items_groups_msg_content_tag_deleted ON chat_items(
   item_deleted,
   item_sent
 );
+CREATE INDEX idx_chat_items_groups_history ON chat_items(
+  user_id,
+  group_id,
+  include_in_history,
+  item_deleted,
+  item_ts,
+  chat_item_id
+);
+CREATE INDEX idx_group_snd_item_statuses_chat_item_id_group_member_id ON group_snd_item_statuses(
+  chat_item_id,
+  group_member_id
+);
 |]
diff --git a/src/Simplex/Chat/Store/SQLite/Migrations.hs b/src/Simplex/Chat/Store/SQLite/Migrations.hs
index f8bdc0d788..c4d4237f98 100644
--- a/src/Simplex/Chat/Store/SQLite/Migrations.hs
+++ b/src/Simplex/Chat/Store/SQLite/Migrations.hs
@@ -124,6 +124,7 @@ import Simplex.Chat.Store.SQLite.Migrations.M20241223_chat_tags
 import Simplex.Chat.Store.SQLite.Migrations.M20241230_reports
 import Simplex.Chat.Store.SQLite.Migrations.M20250105_indexes
 import Simplex.Chat.Store.SQLite.Migrations.M20250115_chat_ttl
+import Simplex.Chat.Store.SQLite.Migrations.M20250122_chat_items_include_in_history
 import Simplex.Messaging.Agent.Store.Shared (Migration (..))
 
 schemaMigrations :: [(String, Query, Maybe Query)]
@@ -247,7 +248,8 @@ schemaMigrations =
     ("20241223_chat_tags", m20241223_chat_tags, Just down_m20241223_chat_tags),
     ("20241230_reports", m20241230_reports, Just down_m20241230_reports),
     ("20250105_indexes", m20250105_indexes, Just down_m20250105_indexes),
-    ("20250115_chat_ttl", m20250115_chat_ttl, Just down_m20250115_chat_ttl)
+    ("20250115_chat_ttl", m20250115_chat_ttl, Just down_m20250115_chat_ttl),
+    ("20250122_chat_items_include_in_history", m20250122_chat_items_include_in_history, Just down_m20250122_chat_items_include_in_history)
   ]
 
 -- | The list of migrations in ascending order by date
diff --git a/src/Simplex/Chat/Store/SQLite/Migrations/M20250122_chat_items_include_in_history.hs b/src/Simplex/Chat/Store/SQLite/Migrations/M20250122_chat_items_include_in_history.hs
new file mode 100644
index 0000000000..afc8544b4a
--- /dev/null
+++ b/src/Simplex/Chat/Store/SQLite/Migrations/M20250122_chat_items_include_in_history.hs
@@ -0,0 +1,39 @@
+{-# LANGUAGE QuasiQuotes #-}
+
+module Simplex.Chat.Store.SQLite.Migrations.M20250122_chat_items_include_in_history where
+
+import Database.SQLite.Simple (Query)
+import Database.SQLite.Simple.QQ (sql)
+
+m20250122_chat_items_include_in_history :: Query
+m20250122_chat_items_include_in_history =
+  [sql|
+ALTER TABLE chat_items ADD COLUMN include_in_history INTEGER NOT NULL DEFAULT 0;
+
+CREATE INDEX idx_chat_items_groups_history ON chat_items(
+  user_id,
+  group_id,
+  include_in_history,
+  item_deleted,
+  item_ts,
+  chat_item_id
+);
+
+UPDATE chat_items
+SET include_in_history = 1
+WHERE group_id IS NOT NULL
+  AND item_content_tag IN ('rcvMsgContent', 'sndMsgContent')
+  AND msg_content_tag NOT IN ('report');
+
+CREATE INDEX idx_group_snd_item_statuses_chat_item_id_group_member_id ON group_snd_item_statuses(chat_item_id, group_member_id);
+|]
+
+down_m20250122_chat_items_include_in_history :: Query
+down_m20250122_chat_items_include_in_history =
+  [sql|
+DROP INDEX idx_group_snd_item_statuses_chat_item_id_group_member_id;
+
+DROP INDEX idx_chat_items_groups_history;
+
+ALTER TABLE chat_items DROP COLUMN include_in_history;
+|]
diff --git a/src/Simplex/Chat/Store/SQLite/Migrations/chat_schema.sql b/src/Simplex/Chat/Store/SQLite/Migrations/chat_schema.sql
index 923928ad5c..0fef1b43d2 100644
--- a/src/Simplex/Chat/Store/SQLite/Migrations/chat_schema.sql
+++ b/src/Simplex/Chat/Store/SQLite/Migrations/chat_schema.sql
@@ -406,7 +406,8 @@ CREATE TABLE chat_items(
   fwd_from_group_id INTEGER REFERENCES groups ON DELETE SET NULL,
   fwd_from_chat_item_id INTEGER REFERENCES chat_items ON DELETE SET NULL,
   via_proxy INTEGER,
-  msg_content_tag TEXT
+  msg_content_tag TEXT,
+  include_in_history INTEGER NOT NULL DEFAULT 0
 );
 CREATE TABLE sqlite_sequence(name,seq);
 CREATE TABLE chat_item_messages(
@@ -978,3 +979,15 @@ CREATE INDEX idx_chat_items_groups_msg_content_tag_deleted ON chat_items(
   item_deleted,
   item_sent
 );
+CREATE INDEX idx_chat_items_groups_history ON chat_items(
+  user_id,
+  group_id,
+  include_in_history,
+  item_deleted,
+  item_ts,
+  chat_item_id
+);
+CREATE INDEX idx_group_snd_item_statuses_chat_item_id_group_member_id ON group_snd_item_statuses(
+  chat_item_id,
+  group_member_id
+);