core: fix report count when loading chat (#5505)

* core: fix report count when loading chat

* remove "deleted" parameter from api

src/Simplex/Chat/Controller.hs

@@ -301,7 +301,7 @@ data ChatCommand
   | APIGetAppSettings (Maybe AppSettings)
   | APIGetChatTags UserId
   | APIGetChats {userId :: UserId, pendingConnections :: Bool, pagination :: PaginationByTime, query :: ChatListQuery}
-  | APIGetChat ChatRef (Maybe ContentFilter) ChatPagination (Maybe String)
+  | APIGetChat ChatRef (Maybe MsgContentTag) ChatPagination (Maybe String)
   | APIGetChatItems ChatPagination (Maybe String)
   | APIGetChatItemInfo ChatRef ChatItemId
   | APISendMessages {chatRef :: ChatRef, liveMessage :: Bool, ttl :: Maybe Int, composedMessages :: NonEmpty ComposedMessage}
@@ -878,12 +878,6 @@ logResponseToFile = \case
   CRMessageError {} -> True
   _ -> False
 
-data ContentFilter = ContentFilter
-  { mcTag :: MsgContentTag,
-    deleted :: Maybe Bool
-  }
-  deriving (Show)
-
 data ChatPagination
   = CPLast Int
   | CPAfter ChatItemId Int

src/Simplex/Chat/Library/Commands.hs

@@ -3593,7 +3593,7 @@ chatCommandP =
               <*> (A.space *> paginationByTimeP <|> pure (PTLast 5000))
               <*> (A.space *> jsonP <|> pure clqNoFilters)
            ),
-      "/_get chat " *> (APIGetChat <$> chatRefP <* A.space <*> optional (contentFilterP <* A.space) <*> chatPaginationP <*> optional (" search=" *> stringP)),
+      "/_get chat " *> (APIGetChat <$> chatRefP <*> optional (" content=" *> strP) <* A.space <*> chatPaginationP <*> optional (" search=" *> stringP)),
       "/_get items " *> (APIGetChatItems <$> chatPaginationP <*> optional (" search=" *> stringP)),
       "/_get item info " *> (APIGetChatItemInfo <$> chatRefP <* A.space <*> A.decimal),
       "/_send " *> (APISendMessages <$> chatRefP <*> liveMessageP <*> sendMessageTTLP <*> (" json " *> jsonP <|> " text " *> composedMessagesTextP)),
@@ -3968,7 +3968,6 @@ chatCommandP =
         ct -> ChatName ct <$> displayName
     chatNameP' = ChatName <$> (chatTypeP <|> pure CTDirect) <*> displayName
     chatRefP = ChatRef <$> chatTypeP <*> A.decimal
-    contentFilterP = ContentFilter <$> ("content=" *> strP) <*> optional (" deleted=" *> onOffP)
     msgCountP = A.space *> A.decimal <|> pure 10
     ciTTLDecimal = ("none" $> Nothing) <|> (Just <$> A.decimal)
     ciTTL =

src/Simplex/Chat/Messages.hs

@@ -318,14 +318,13 @@ deriving instance Show AChat
 data ChatStats = ChatStats
   { unreadCount :: Int, -- returned both in /_get chat initial API and in /_get chats API
     reportsCount :: Int, -- returned both in /_get chat initial API and in /_get chats API
-    archivedReportsCount :: Int, -- only returned in /_get chat initial API
     minUnreadItemId :: ChatItemId,
     unreadChat :: Bool
   }
   deriving (Show)
 
 emptyChatStats :: ChatStats
-emptyChatStats = ChatStats 0 0 0 0 False
+emptyChatStats = ChatStats 0 0 0 False
 
 data NavigationInfo = NavigationInfo
   { afterUnread :: Int,

src/Simplex/Chat/Protocol.hs

@@ -54,7 +54,7 @@ import qualified Simplex.Messaging.Agent.Store.DB as DB
 import Simplex.Messaging.Compression (Compressed, compress1, decompress1)
 import Simplex.Messaging.Encoding
 import Simplex.Messaging.Encoding.String
-import Simplex.Messaging.Parsers (blobFieldDecoder, defaultJSON, dropPrefix, fromTextField_, fstToLower, parseAll, sumTypeJSON, taggedObjectJSON)
+import Simplex.Messaging.Parsers (defaultJSON, dropPrefix, fromTextField_, fstToLower, parseAll, sumTypeJSON, taggedObjectJSON)
 import Simplex.Messaging.Protocol (MsgBody)
 import Simplex.Messaging.Util (decodeJSON, eitherToMaybe, encodeJSON, safeDecodeUtf8, (<$?>))
 import Simplex.Messaging.Version hiding (version)

src/Simplex/Chat/Store/Messages.hs

@@ -141,7 +141,7 @@ import Data.Text (Text)
 import qualified Data.Text as T
 import Data.Time (addUTCTime)
 import Data.Time.Clock (UTCTime (..), getCurrentTime)
-import Simplex.Chat.Controller (ChatListQuery (..), ChatPagination (..), ContentFilter (..), PaginationByTime (..))
+import Simplex.Chat.Controller (ChatListQuery (..), ChatPagination (..), PaginationByTime (..))
 import Simplex.Chat.Markdown
 import Simplex.Chat.Messages
 import Simplex.Chat.Messages.CIContent
@@ -560,7 +560,7 @@ data AChatPreviewData = forall c. ChatTypeI c => ACPD (SChatType c) (ChatPreview
 type ChatStatsRow = (Int, Int, ChatItemId, BoolInt)
 
 toChatStats :: ChatStatsRow -> ChatStats
-toChatStats (unreadCount, reportsCount, minUnreadItemId, BI unreadChat) = ChatStats {unreadCount, reportsCount, archivedReportsCount = 0, minUnreadItemId, unreadChat}
+toChatStats (unreadCount, reportsCount, minUnreadItemId, BI unreadChat) = ChatStats {unreadCount, reportsCount, minUnreadItemId, unreadChat}
 
 findDirectChatPreviews_ :: DB.Connection -> User -> PaginationByTime -> ChatListQuery -> IO [AChatPreviewData]
 findDirectChatPreviews_ db User {userId} pagination clq =
@@ -1197,12 +1197,12 @@ getContactNavInfo_ db User {userId} Contact {contactId} afterCI = do
               :. (userId, contactId, ciCreatedAt afterCI, cChatItemId afterCI)
           )
 
-getGroupChat :: DB.Connection -> VersionRangeChat -> User -> Int64 -> Maybe ContentFilter -> ChatPagination -> Maybe String -> ExceptT StoreError IO (Chat 'CTGroup, Maybe NavigationInfo)
+getGroupChat :: DB.Connection -> VersionRangeChat -> User -> Int64 -> Maybe MsgContentTag -> ChatPagination -> Maybe String -> ExceptT StoreError IO (Chat 'CTGroup, Maybe NavigationInfo)
 getGroupChat db vr user groupId contentFilter pagination search_ = do
   let search = fromMaybe "" search_
   g <- getGroupInfo db vr user groupId
   case pagination of
-    CPLast count -> liftIO $ (,Nothing) <$> getGroupChatLast_ db user g contentFilter count search
+    CPLast count -> liftIO $ (,Nothing) <$> getGroupChatLast_ db user g contentFilter count search emptyChatStats
     CPAfter afterId count -> (,Nothing) <$> getGroupChatAfter_ db user g contentFilter afterId count search
     CPBefore beforeId count -> (,Nothing) <$> getGroupChatBefore_ db user g contentFilter beforeId count search
     CPAround aroundId count -> getGroupChatAround_ db user g contentFilter aroundId count search
@@ -1210,20 +1210,18 @@ getGroupChat db vr user groupId contentFilter pagination search_ = do
       unless (null search) $ throwError $ SEInternalError "initial chat pagination doesn't support search"
       getGroupChatInitial_ db user g contentFilter count
 
-getGroupChatLast_ :: DB.Connection -> User -> GroupInfo -> Maybe ContentFilter -> Int -> String -> IO (Chat 'CTGroup)
-getGroupChatLast_ db user g contentFilter count search = do
+getGroupChatLast_ :: DB.Connection -> User -> GroupInfo -> Maybe MsgContentTag -> Int -> String -> ChatStats -> IO (Chat 'CTGroup)
+getGroupChatLast_ db user g contentFilter count search stats = do
   ciIds <- getGroupChatItemIDs db user g contentFilter GRLast count search
   ts <- getCurrentTime
   cis <- mapM (safeGetGroupItem db user g ts) ciIds
-  pure $ Chat (GroupChat g) (reverse cis) emptyChatStats
+  pure $ Chat (GroupChat g) (reverse cis) stats
 
 data GroupItemIDsRange = GRLast | GRAfter UTCTime ChatItemId | GRBefore UTCTime ChatItemId
 
-getGroupChatItemIDs :: DB.Connection -> User -> GroupInfo -> Maybe ContentFilter -> GroupItemIDsRange -> Int -> String -> IO [ChatItemId]
+getGroupChatItemIDs :: DB.Connection -> User -> GroupInfo -> Maybe MsgContentTag -> GroupItemIDsRange -> Int -> String -> IO [ChatItemId]
 getGroupChatItemIDs db User {userId} GroupInfo {groupId} contentFilter range count search = case contentFilter of
-  Just ContentFilter {mcTag, deleted} -> case deleted of
-    Just deleted' -> idsQuery (baseCond <> " AND msg_content_tag = ? AND item_deleted = ? ") (userId, groupId, mcTag, BI deleted')
-    Nothing -> idsQuery (baseCond <> " AND msg_content_tag = ? ") (userId, groupId, mcTag)
+  Just mcTag -> idsQuery (baseCond <> " AND msg_content_tag = ? ") (userId, groupId, mcTag)
   Nothing -> idsQuery baseCond (userId, groupId)
   where
     baseQuery = " SELECT chat_item_id FROM chat_items WHERE "
@@ -1295,7 +1293,7 @@ getGroupMemberChatItemLast db user@User {userId} groupId groupMemberId = do
         (userId, groupId, groupMemberId)
   getGroupChatItem db user groupId chatItemId
 
-getGroupChatAfter_ :: DB.Connection -> User -> GroupInfo -> Maybe ContentFilter -> ChatItemId -> Int -> String -> ExceptT StoreError IO (Chat 'CTGroup)
+getGroupChatAfter_ :: DB.Connection -> User -> GroupInfo -> Maybe MsgContentTag -> ChatItemId -> Int -> String -> ExceptT StoreError IO (Chat 'CTGroup)
 getGroupChatAfter_ db user g@GroupInfo {groupId} contentFilter afterId count search = do
   afterCI <- getGroupChatItem db user groupId afterId
   let range = GRAfter (chatItemTs afterCI) (cChatItemId afterCI)
@@ -1304,7 +1302,7 @@ getGroupChatAfter_ db user g@GroupInfo {groupId} contentFilter afterId count sea
   cis <- liftIO $ mapM (safeGetGroupItem db user g ts) ciIds
   pure $ Chat (GroupChat g) cis emptyChatStats
 
-getGroupChatBefore_ :: DB.Connection -> User -> GroupInfo -> Maybe ContentFilter -> ChatItemId -> Int -> String -> ExceptT StoreError IO (Chat 'CTGroup)
+getGroupChatBefore_ :: DB.Connection -> User -> GroupInfo -> Maybe MsgContentTag -> ChatItemId -> Int -> String -> ExceptT StoreError IO (Chat 'CTGroup)
 getGroupChatBefore_ db user g@GroupInfo {groupId} contentFilter beforeId count search = do
   beforeCI <- getGroupChatItem db user groupId beforeId
   let range = GRBefore (chatItemTs beforeCI) (cChatItemId beforeCI)
@@ -1313,12 +1311,12 @@ getGroupChatBefore_ db user g@GroupInfo {groupId} contentFilter beforeId count s
   cis <- liftIO $ mapM (safeGetGroupItem db user g ts) ciIds
   pure $ Chat (GroupChat g) (reverse cis) emptyChatStats
 
-getGroupChatAround_ :: DB.Connection -> User -> GroupInfo -> Maybe ContentFilter -> ChatItemId -> Int -> String -> ExceptT StoreError IO (Chat 'CTGroup, Maybe NavigationInfo)
+getGroupChatAround_ :: DB.Connection -> User -> GroupInfo -> Maybe MsgContentTag -> ChatItemId -> Int -> String -> ExceptT StoreError IO (Chat 'CTGroup, Maybe NavigationInfo)
 getGroupChatAround_ db user g contentFilter aroundId count search = do
   stats <- liftIO $ getGroupStats_ db user g
   getGroupChatAround' db user g contentFilter aroundId count search stats
 
-getGroupChatAround' :: DB.Connection -> User -> GroupInfo -> Maybe ContentFilter -> ChatItemId -> Int -> String -> ChatStats -> ExceptT StoreError IO (Chat 'CTGroup, Maybe NavigationInfo)
+getGroupChatAround' :: DB.Connection -> User -> GroupInfo -> Maybe MsgContentTag -> ChatItemId -> Int -> String -> ChatStats -> ExceptT StoreError IO (Chat 'CTGroup, Maybe NavigationInfo)
 getGroupChatAround' db user g@GroupInfo {groupId} contentFilter aroundId count search stats = do
   aroundCI <- getGroupChatItem db user groupId aroundId
   let beforeRange = GRBefore (chatItemTs aroundCI) (cChatItemId aroundCI)
@@ -1336,26 +1334,28 @@ getGroupChatAround' db user g@GroupInfo {groupId} contentFilter aroundId count s
       [] -> pure $ NavigationInfo 0 0
       cis -> getGroupNavInfo_ db user g (last cis)
 
-getGroupChatInitial_ :: DB.Connection -> User -> GroupInfo -> Maybe ContentFilter -> Int -> ExceptT StoreError IO (Chat 'CTGroup, Maybe NavigationInfo)
-getGroupChatInitial_ db user g contentFilter count =
+getGroupChatInitial_ :: DB.Connection -> User -> GroupInfo -> Maybe MsgContentTag -> Int -> ExceptT StoreError IO (Chat 'CTGroup, Maybe NavigationInfo)
+getGroupChatInitial_ db user g contentFilter count = do
   liftIO (getGroupMinUnreadId_ db user g contentFilter) >>= \case
     Just minUnreadItemId -> do
-      unreadCount <- liftIO $ getGroupUnreadCount_ db user g Nothing
-      reportsCount <- liftIO $ getGroupReportsCount_ db user g False
-      archivedReportsCount <- liftIO $ getGroupReportsCount_ db user g True
-      let stats = ChatStats {unreadCount, reportsCount, archivedReportsCount, minUnreadItemId, unreadChat = False}
+      stats <- liftIO $ getStats minUnreadItemId =<< getGroupUnreadCount_ db user g Nothing
       getGroupChatAround' db user g contentFilter minUnreadItemId count "" stats
-    Nothing -> liftIO $ (,Just $ NavigationInfo 0 0) <$> getGroupChatLast_ db user g contentFilter count ""
+    Nothing -> liftIO $ do
+      stats <- getStats 0 0
+      (,Just $ NavigationInfo 0 0) <$> getGroupChatLast_ db user g contentFilter count "" stats
+  where
+    getStats minUnreadItemId unreadCount = do
+      reportsCount <- getGroupReportsCount_ db user g False
+      pure ChatStats {unreadCount, reportsCount, minUnreadItemId, unreadChat = False}
 
 getGroupStats_ :: DB.Connection -> User -> GroupInfo -> IO ChatStats
 getGroupStats_ db user g = do
   minUnreadItemId <- fromMaybe 0 <$> getGroupMinUnreadId_ db user g Nothing
   unreadCount <- getGroupUnreadCount_ db user g Nothing
   reportsCount <- getGroupReportsCount_ db user g False
-  archivedReportsCount <- getGroupReportsCount_ db user g True
-  pure ChatStats {unreadCount, reportsCount, archivedReportsCount, minUnreadItemId, unreadChat = False}
+  pure ChatStats {unreadCount, reportsCount, minUnreadItemId, unreadChat = False}
 
-getGroupMinUnreadId_ :: DB.Connection -> User -> GroupInfo -> Maybe ContentFilter -> IO (Maybe ChatItemId)
+getGroupMinUnreadId_ :: DB.Connection -> User -> GroupInfo -> Maybe MsgContentTag -> IO (Maybe ChatItemId)
 getGroupMinUnreadId_ db user g contentFilter =
   fmap join . maybeFirstRow fromOnly $
     queryUnreadGroupItems db user g contentFilter baseQuery orderLimit
@@ -1363,7 +1363,7 @@ getGroupMinUnreadId_ db user g contentFilter =
     baseQuery = "SELECT chat_item_id FROM chat_items WHERE user_id = ? AND group_id = ? "
     orderLimit = " ORDER BY item_ts ASC, chat_item_id ASC LIMIT 1"
 
-getGroupUnreadCount_ :: DB.Connection -> User -> GroupInfo -> Maybe ContentFilter -> IO Int
+getGroupUnreadCount_ :: DB.Connection -> User -> GroupInfo -> Maybe MsgContentTag -> IO Int
 getGroupUnreadCount_ db user g contentFilter =
   fromOnly . head <$> queryUnreadGroupItems db user g contentFilter baseQuery ""
   where
@@ -1377,20 +1377,14 @@ getGroupReportsCount_ db User {userId} GroupInfo {groupId} archived =
       "SELECT COUNT(1) FROM chat_items WHERE user_id = ? AND group_id = ? AND msg_content_tag = ? AND item_deleted = ? AND item_sent = 0"
       (userId, groupId, MCReport_, BI archived)
 
-queryUnreadGroupItems :: FromRow r => DB.Connection -> User -> GroupInfo -> Maybe ContentFilter -> Query -> Query -> IO [r]
+queryUnreadGroupItems :: FromRow r => DB.Connection -> User -> GroupInfo -> Maybe MsgContentTag -> Query -> Query -> IO [r]
 queryUnreadGroupItems db User {userId} GroupInfo {groupId} contentFilter baseQuery orderLimit =
   case contentFilter of
-    Just ContentFilter {mcTag, deleted} -> case deleted of
-      Just deleted' ->
-        DB.query
-          db
-          (baseQuery <> " AND msg_content_tag = ? AND item_deleted = ? AND item_status = ? " <> orderLimit)
-          (userId, groupId, mcTag, BI deleted', CISRcvNew)
-      Nothing ->
-        DB.query
-          db
-          (baseQuery <> " AND msg_content_tag = ? AND item_status = ? " <> orderLimit)
-          (userId, groupId, mcTag, CISRcvNew)
+    Just mcTag ->
+      DB.query
+        db
+        (baseQuery <> " AND msg_content_tag = ? AND item_status = ? " <> orderLimit)
+        (userId, groupId, mcTag, CISRcvNew)
     Nothing ->
       DB.query
         db

src/Simplex/Chat/Store/Postgres/Migrations/M20241220_initial.hs

@@ -1002,11 +1002,4 @@ CREATE INDEX idx_chat_items_groups_msg_content_tag_item_ts ON chat_items(
   msg_content_tag,
   item_ts
 );
-CREATE INDEX idx_chat_items_groups_msg_content_tag_item_deleted_item_ts ON chat_items(
-  user_id,
-  group_id,
-  msg_content_tag,
-  item_deleted,
-  item_ts
-);
 |]

src/Simplex/Chat/Store/SQLite/Migrations/M20250105_indexes.hs

@@ -9,12 +9,10 @@ m20250105_indexes :: Query
 m20250105_indexes =
   [sql|
 CREATE INDEX idx_chat_items_groups_msg_content_tag_item_ts ON chat_items(user_id, group_id, msg_content_tag, item_ts);
-CREATE INDEX idx_chat_items_groups_msg_content_tag_item_deleted_item_ts ON chat_items(user_id, group_id, msg_content_tag, item_deleted, item_ts);
 |]
 
 down_m20250105_indexes :: Query
 down_m20250105_indexes =
   [sql|
 DROP INDEX idx_chat_items_groups_msg_content_tag_item_ts;
-DROP INDEX idx_chat_items_groups_msg_content_tag_item_deleted_item_ts;
 |]

src/Simplex/Chat/Store/SQLite/Migrations/chat_schema.sql

@@ -968,10 +968,3 @@ CREATE INDEX idx_chat_items_groups_msg_content_tag_item_ts ON chat_items(
   msg_content_tag,
   item_ts
 );
-CREATE INDEX idx_chat_items_groups_msg_content_tag_item_deleted_item_ts ON chat_items(
-  user_id,
-  group_id,
-  msg_content_tag,
-  item_deleted,
-  item_ts
-);

tests/ChatTests/Groups.hs

@@ -6611,14 +6611,8 @@ testGroupMemberReports =
           (cath </)
         ]
       alice #$> ("/_get chat #1 content=report count=100", chat, [(0, "report content")])
-      alice #$> ("/_get chat #1 content=report deleted=off count=100", chat, [(0, "report content")])
-      alice #$> ("/_get chat #1 content=report deleted=on count=100", chat, [])
       bob #$> ("/_get chat #1 content=report count=100", chat, [(0, "report content")])
-      bob #$> ("/_get chat #1 content=report deleted=off count=100", chat, [(0, "report content")])
-      bob #$> ("/_get chat #1 content=report deleted=on count=100", chat, [])
       dan #$> ("/_get chat #1 content=report count=100", chat, [(1, "report content")])
-      dan #$> ("/_get chat #1 content=report deleted=off count=100", chat, [(1, "report content")])
-      dan #$> ("/_get chat #1 content=report deleted=on count=100", chat, [])
       alice ##> "\\\\ #jokes cath inappropriate joke"
       concurrentlyN_
         [ do
@@ -6633,11 +6627,5 @@ testGroupMemberReports =
             dan <## "#jokes: 1 messages deleted by member alice"
         ]
       alice #$> ("/_get chat #1 content=report count=100", chat, [(0, "report content [marked deleted by you]")])
-      alice #$> ("/_get chat #1 content=report deleted=off count=100", chat, [])
-      alice #$> ("/_get chat #1 content=report deleted=on count=100", chat, [(0, "report content [marked deleted by you]")])
       bob #$> ("/_get chat #1 content=report count=100", chat, [(0, "report content [marked deleted by alice]")])
-      bob #$> ("/_get chat #1 content=report deleted=off count=100", chat, [])
-      bob #$> ("/_get chat #1 content=report deleted=on count=100", chat, [(0, "report content [marked deleted by alice]")])
       dan #$> ("/_get chat #1 content=report count=100", chat, [(1, "report content [marked deleted by alice]")])
-      dan #$> ("/_get chat #1 content=report deleted=off count=100", chat, [])
-      dan #$> ("/_get chat #1 content=report deleted=on count=100", chat, [(1, "report content [marked deleted by alice]")])