core: fix opening chats on new unread items (after sent or viewed items) (#6747)

* core: fix opening chats on new unread items (after sent or viewed items)

* fix test

* sqlite schema and query plan change

* fix postgresql, update schema

* stabilize tests

---------

Co-authored-by: Evgeny @ SimpleX Chat <259188159+evgeny-simplex@users.noreply.github.com>

simplex-chat.cabal

@@ -129,6 +129,7 @@ library
           Simplex.Chat.Store.Postgres.Migrations.M20260108_chat_indices
           Simplex.Chat.Store.Postgres.Migrations.M20260122_has_link
           Simplex.Chat.Store.Postgres.Migrations.M20260222_chat_relays
+          Simplex.Chat.Store.Postgres.Migrations.M20260403_item_viewed
   else
       exposed-modules:
           Simplex.Chat.Archive
@@ -280,6 +281,7 @@ library
           Simplex.Chat.Store.SQLite.Migrations.M20260108_chat_indices
           Simplex.Chat.Store.SQLite.Migrations.M20260122_has_link
           Simplex.Chat.Store.SQLite.Migrations.M20260222_chat_relays
+          Simplex.Chat.Store.SQLite.Migrations.M20260403_item_viewed
   other-modules:
       Paths_simplex_chat
   hs-source-dirs:

src/Simplex/Chat/Store/Direct.hs

@@ -635,7 +635,7 @@ setUserChatsRead db User {userId} = do
   DB.execute db "UPDATE contacts SET unread_chat = ?, updated_at = ? WHERE user_id = ? AND unread_chat = ?" (BI False, updatedAt, userId, BI True)
   DB.execute db "UPDATE groups SET unread_chat = ?, updated_at = ? WHERE user_id = ? AND unread_chat = ?" (BI False, updatedAt, userId, BI True)
   DB.execute db "UPDATE note_folders SET unread_chat = ?, updated_at = ? WHERE user_id = ? AND unread_chat = ?" (BI False, updatedAt, userId, BI True)
-  DB.execute db "UPDATE chat_items SET item_status = ?, updated_at = ? WHERE user_id = ? AND item_status = ?" (CISRcvRead, updatedAt, userId, CISRcvNew)
+  DB.execute db "UPDATE chat_items SET item_status = ?, item_viewed = 1, updated_at = ? WHERE user_id = ? AND item_status = ?" (CISRcvRead, updatedAt, userId, CISRcvNew)
 
 updateContactStatus :: DB.Connection -> User -> Contact -> ContactStatus -> IO Contact
 updateContactStatus db User {userId} ct@Contact {contactId} contactStatus = do

src/Simplex/Chat/Store/Messages.hs

@@ -185,9 +185,11 @@ import UnliftIO.STM
 #if defined(dbPostgres)
 import Database.PostgreSQL.Simple (FromRow, In (..), Only (..), Query, ToRow, (:.) (..))
 import Database.PostgreSQL.Simple.SqlQQ (sql)
+import Database.PostgreSQL.Simple.ToField (ToField)
 #else
 import Database.SQLite.Simple (FromRow, Only (..), Query, ToRow, (:.) (..))
 import Database.SQLite.Simple.QQ (sql)
+import Database.SQLite.Simple.ToField (ToField)
 #endif
 
 deleteContactCIs :: DB.Connection -> User -> Contact -> IO ()
@@ -586,20 +588,20 @@ createNewChatItem_ db User {userId} chatDirection showGroupAsSender msgId_ share
         user_id, created_by_msg_id, contact_id, group_id, group_member_id, note_folder_id, group_scope_tag, group_scope_group_member_id,
         -- meta
         item_sent, item_ts, item_content, item_content_tag, item_text, item_status, msg_content_tag, shared_msg_id,
-        forwarded_by_group_member_id, include_in_history, created_at, updated_at, item_live, user_mention, has_link, show_group_as_sender, msg_signed, timed_ttl, timed_delete_at,
+        forwarded_by_group_member_id, include_in_history, created_at, updated_at, item_live, user_mention, has_link, item_viewed, show_group_as_sender, msg_signed, timed_ttl, timed_delete_at,
         -- quote
         quoted_shared_msg_id, quoted_sent_at, quoted_content, quoted_sent, quoted_member_id,
         -- forwarded from
         fwd_from_tag, fwd_from_chat_name, fwd_from_msg_dir, fwd_from_contact_id, fwd_from_group_id, fwd_from_chat_item_id
-      ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
+      ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
     |]
     ((userId, msgId_) :. idsRow :. groupScopeRow :. itemRow :. quoteRow' :. forwardedFromRow)
   ciId <- insertedRowId db
   forM_ msgId_ $ \msgId -> insertChatItemMessage_ db ciId msgId createdAt
   pure ciId
   where
-    itemRow :: (SMsgDirection d, UTCTime, CIContent d, Text, Text, CIStatus d, Maybe MsgContentTag, Maybe SharedMsgId, Maybe GroupMemberId, BoolInt) :. (UTCTime, UTCTime, Maybe BoolInt, BoolInt, BoolInt, BoolInt, Maybe MsgSigStatus) :. (Maybe Int, Maybe UTCTime)
-    itemRow = (msgDirection @d, itemTs, ciContent, toCIContentTag ciContent, ciContentToText ciContent, ciCreateStatus ciContent, msgContentTag <$> ciMsgContent ciContent, sharedMsgId, forwardedByMember, BI includeInHistory) :. (createdAt, createdAt, BI <$> justTrue live, BI userMention, BI hasLink, BI showGroupAsSender, msgSigned) :. ciTimedRow timed
+    itemRow :: (SMsgDirection d, UTCTime, CIContent d, Text, Text, CIStatus d, Maybe MsgContentTag, Maybe SharedMsgId, Maybe GroupMemberId, BoolInt) :. (UTCTime, UTCTime, Maybe BoolInt, BoolInt, BoolInt, BoolInt, BoolInt, Maybe MsgSigStatus) :. (Maybe Int, Maybe UTCTime)
+    itemRow = (msgDirection @d, itemTs, ciContent, toCIContentTag ciContent, ciContentToText ciContent, ciCreateStatus ciContent, mcTag_, sharedMsgId, forwardedByMember, BI includeInHistory) :. (createdAt, createdAt, BI <$> justTrue live, BI userMention, BI hasLink, BI itemViewed, BI showGroupAsSender, msgSigned) :. ciTimedRow timed
     quoteRow' = let (a, b, c, d, e) = quoteRow in (a, b, c, BI <$> d, e)
     idsRow :: (Maybe ContactId, Maybe GroupId, Maybe GroupMemberId, Maybe NoteFolderId)
     idsRow = case chatDirection of
@@ -622,8 +624,13 @@ createNewChatItem_ db User {userId} chatDirection showGroupAsSender msgId_ share
       _ -> (Nothing, Nothing)
     includeInHistory :: Bool
     includeInHistory = case groupScope of
-      Just Nothing -> isJust (ciMsgContent ciContent) && ((msgContentTag <$> ciMsgContent ciContent) /= Just MCReport_)
+      Just Nothing -> isJust mcTag_ && mcTag_ /= Just MCReport_
       _ -> False
+    itemViewed :: Bool
+    itemViewed = case msgDirection @d of
+      SMDSnd -> isJust mcTag_
+      SMDRcv -> False
+    mcTag_ = msgContentTag <$> ciMsgContent ciContent
     forwardedFromRow :: (Maybe CIForwardedFromTag, Maybe Text, Maybe MsgDirection, Maybe Int64, Maybe Int64, Maybe Int64)
     forwardedFromRow = case itemForwarded of
       Nothing ->
@@ -1346,7 +1353,7 @@ getContactMinUnreadId_ db User {userId} Contact {contactId} =
       |]
       (userId, contactId, CISRcvNew)
 
--- max viewed item: received read or sent (any item_status != CISRcvNew)
+-- max viewed item: sent content or received read (excludes born-read events)
 getContactMaxViewedItemId_ :: DB.Connection -> User -> Contact -> IO (Maybe ChatItemId)
 getContactMaxViewedItemId_ db User {userId} Contact {contactId} =
   fmap join . maybeFirstRow fromOnly $
@@ -1355,11 +1362,11 @@ getContactMaxViewedItemId_ db User {userId} Contact {contactId} =
       [sql|
         SELECT chat_item_id
         FROM chat_items
-        WHERE user_id = ? AND contact_id = ? AND item_status != ?
+        WHERE user_id = ? AND contact_id = ? AND item_viewed = 1
         ORDER BY created_at DESC, chat_item_id DESC
         LIMIT 1
       |]
-      (userId, contactId, CISRcvNew)
+      (userId, contactId)
 
 getContactUnreadCount_ :: DB.Connection -> User -> Contact -> IO Int
 getContactUnreadCount_ db User {userId} Contact {contactId} =
@@ -1688,23 +1695,23 @@ getGroupStats_ db user g scopeInfo_ = do
 getGroupMinUnreadId_ :: DB.Connection -> User -> GroupInfo -> Maybe GroupChatScopeInfo -> Maybe MsgContentTag -> ExceptT StoreError IO (Maybe ChatItemId)
 getGroupMinUnreadId_ db user g scopeInfo_ contentFilter =
   fmap join . maybeFirstRow fromOnly $
-    queryUnreadGroupItems db user g scopeInfo_ contentFilter " item_status = ? " baseQuery orderLimit
+    queryUnreadGroupItems db user g scopeInfo_ contentFilter " item_status = ? " CISRcvNew baseQuery orderLimit
   where
     baseQuery = "SELECT chat_item_id FROM chat_items WHERE user_id = ? AND group_id = ? "
     orderLimit = " ORDER BY item_ts ASC, chat_item_id ASC LIMIT 1"
 
--- max viewed item: received read or sent (any item_status != CISRcvNew)
+-- max viewed item: sent content or received read (excludes born-read events)
 getGroupMaxViewedItemId_ :: DB.Connection -> User -> GroupInfo -> Maybe GroupChatScopeInfo -> Maybe MsgContentTag -> ExceptT StoreError IO (Maybe ChatItemId)
 getGroupMaxViewedItemId_ db user g scopeInfo_ contentFilter =
   fmap join . maybeFirstRow fromOnly $
-    queryUnreadGroupItems db user g scopeInfo_ contentFilter " item_status != ? " baseQuery orderLimit
+    queryUnreadGroupItems db user g scopeInfo_ contentFilter " item_viewed = ? " (BI True) baseQuery orderLimit
   where
     baseQuery = "SELECT chat_item_id FROM chat_items WHERE user_id = ? AND group_id = ? "
     orderLimit = " ORDER BY item_ts DESC, chat_item_id DESC LIMIT 1"
 
 getGroupUnreadCount_ :: DB.Connection -> User -> GroupInfo -> Maybe GroupChatScopeInfo -> Maybe MsgContentTag -> ExceptT StoreError IO (Int, Int)
 getGroupUnreadCount_ db user g scopeInfo_ contentFilter =
-  head <$> queryUnreadGroupItems db user g scopeInfo_ contentFilter " item_status = ? " baseQuery ""
+  head <$> queryUnreadGroupItems db user g scopeInfo_ contentFilter " item_status = ? " CISRcvNew baseQuery ""
   where
     baseQuery = "SELECT COUNT(1), COALESCE(SUM(user_mention), 0) FROM chat_items WHERE user_id = ? AND group_id = ? "
 
@@ -1716,27 +1723,27 @@ getGroupReportsCount_ db User {userId} GroupInfo {groupId} archived =
       "SELECT COUNT(1) FROM chat_items WHERE user_id = ? AND group_id = ? AND msg_content_tag = ? AND item_deleted = ? AND item_sent = 0"
       (userId, groupId, MCReport_, BI archived)
 
-queryUnreadGroupItems :: FromRow r => DB.Connection -> User -> GroupInfo -> Maybe GroupChatScopeInfo -> Maybe MsgContentTag -> Query -> Query -> Query -> ExceptT StoreError IO [r]
-queryUnreadGroupItems db User {userId} GroupInfo {groupId} scopeInfo_ contentFilter statusCond baseQuery orderLimit =
+queryUnreadGroupItems :: (ToField p, FromRow r) => DB.Connection -> User -> GroupInfo -> Maybe GroupChatScopeInfo -> Maybe MsgContentTag -> Query -> p -> Query -> Query -> ExceptT StoreError IO [r]
+queryUnreadGroupItems db User {userId} GroupInfo {groupId} scopeInfo_ contentFilter statusCond statusParam baseQuery orderLimit =
   case (scopeInfo_, contentFilter) of
     (Nothing, Nothing) ->
       liftIO $
         DB.query
           db
           (baseQuery <> " AND group_scope_tag IS NULL AND group_scope_group_member_id IS NULL AND " <> statusCond <> orderLimit)
-          (userId, groupId, CISRcvNew)
+          (userId, groupId, statusParam)
     (Nothing, Just mcTag) ->
       liftIO $
         DB.query
           db
           (baseQuery <> " AND msg_content_tag = ? AND " <> statusCond <> orderLimit)
-          (userId, groupId, mcTag, CISRcvNew)
+          (userId, groupId, mcTag, statusParam)
     (Just GCSIMemberSupport {groupMember_ = m}, Nothing) ->
       liftIO $
         DB.query
           db
           (baseQuery <> " AND group_scope_tag = ? AND group_scope_group_member_id IS NOT DISTINCT FROM ? AND " <> statusCond <> orderLimit)
-          (userId, groupId, GCSTMemberSupport_, groupMemberId' <$> m, CISRcvNew)
+          (userId, groupId, GCSTMemberSupport_, groupMemberId' <$> m, statusParam)
     (Just _scope, Just _mcTag) ->
       throwError $ SEInternalError "group scope and content filter are not supported together"
 
@@ -1997,7 +2004,7 @@ updateDirectChatItemsRead db User {userId} contactId = do
   DB.execute
     db
     [sql|
-      UPDATE chat_items SET item_status = ?, updated_at = ?
+      UPDATE chat_items SET item_status = ?, item_viewed = 1, updated_at = ?
       WHERE user_id = ? AND contact_id = ? AND item_status = ?
     |]
     (CISRcvRead, currentTs, userId, contactId, CISRcvNew)
@@ -2042,7 +2049,7 @@ setDirectChatItemRead_ db User {userId} contactId itemId currentTs =
   DB.execute
     db
     [sql|
-      UPDATE chat_items SET item_status = ?, updated_at = ?
+      UPDATE chat_items SET item_status = ?, item_viewed = 1, updated_at = ?
       WHERE user_id = ? AND contact_id = ? AND item_status = ? AND chat_item_id = ?
     |]
     (CISRcvRead, currentTs, userId, contactId, CISRcvNew, itemId)
@@ -2062,7 +2069,7 @@ updateGroupChatItemsRead db User {userId} GroupInfo {groupId} = do
   DB.execute
     db
     [sql|
-      UPDATE chat_items SET item_status = ?, updated_at = ?
+      UPDATE chat_items SET item_status = ?, item_viewed = 1, updated_at = ?
       WHERE user_id = ? AND group_id = ?
         AND item_status = ?
     |]
@@ -2076,7 +2083,7 @@ updateSupportChatItemsRead db vr user@User {userId} g@GroupInfo {groupId, member
       DB.execute
         db
         [sql|
-          UPDATE chat_items SET item_status = ?, updated_at = ?
+          UPDATE chat_items SET item_status = ?, item_viewed = 1, updated_at = ?
           WHERE user_id = ? AND group_id = ?
             AND group_scope_tag = ? AND group_scope_group_member_id IS NOT DISTINCT FROM ?
             AND item_status = ?
@@ -2154,7 +2161,7 @@ updateGroupChatItemsReadList db vr user@User {userId} g@GroupInfo {groupId} scop
         DB.query
           db
           [sql|
-            UPDATE chat_items SET item_status = ?, updated_at = ?
+            UPDATE chat_items SET item_status = ?, item_viewed = 1, updated_at = ?
             WHERE user_id = ? AND group_id = ? AND item_status = ? AND chat_item_id = ?
             RETURNING chat_item_id, timed_ttl, timed_delete_at, group_member_id, user_mention
           |]
@@ -2237,7 +2244,7 @@ updateLocalChatItemsRead db User {userId} noteFolderId = do
   DB.execute
     db
     [sql|
-      UPDATE chat_items SET item_status = ?, updated_at = ?
+      UPDATE chat_items SET item_status = ?, item_viewed = 1, updated_at = ?
       WHERE user_id = ? AND note_folder_id = ? AND item_status = ?
     |]
     (CISRcvRead, currentTs, userId, noteFolderId, CISRcvNew)

src/Simplex/Chat/Store/Postgres/Migrations.hs

@@ -27,6 +27,7 @@ import Simplex.Chat.Store.Postgres.Migrations.M20251230_strict_tables
 import Simplex.Chat.Store.Postgres.Migrations.M20260108_chat_indices
 import Simplex.Chat.Store.Postgres.Migrations.M20260122_has_link
 import Simplex.Chat.Store.Postgres.Migrations.M20260222_chat_relays
+import Simplex.Chat.Store.Postgres.Migrations.M20260403_item_viewed
 import Simplex.Messaging.Agent.Store.Shared (Migration (..))
 
 schemaMigrations :: [(String, Text, Maybe Text)]
@@ -53,7 +54,8 @@ schemaMigrations =
     ("20251230_strict_tables", m20251230_strict_tables, Just down_m20251230_strict_tables),
     ("20260108_chat_indices", m20260108_chat_indices, Just down_m20260108_chat_indices),
     ("20260122_has_link", m20260122_has_link, Just down_m20260122_has_link),
-    ("20260222_chat_relays", m20260222_chat_relays, Just down_m20260222_chat_relays)
+    ("20260222_chat_relays", m20260222_chat_relays, Just down_m20260222_chat_relays),
+    ("20260403_item_viewed", m20260403_item_viewed, Just down_m20260403_item_viewed)
   ]
 
 -- | The list of migrations in ascending order by date

src/Simplex/Chat/Store/Postgres/Migrations/M20260222_chat_relays.hs

@@ -1,15 +1,14 @@
+{-# LANGUAGE OverloadedStrings #-}
 {-# LANGUAGE QuasiQuotes #-}
 
 module Simplex.Chat.Store.Postgres.Migrations.M20260222_chat_relays where
 
 import Data.Text (Text)
-import qualified Data.Text as T
 import Text.RawString.QQ (r)
 
 m20260222_chat_relays :: Text
 m20260222_chat_relays =
-  T.pack
-    [r|
+  [r|
 CREATE TABLE chat_relays(
   chat_relay_id BIGINT PRIMARY KEY GENERATED ALWAYS AS IDENTITY,
   address BYTEA NOT NULL,
@@ -80,8 +79,7 @@ ALTER TABLE connections ADD COLUMN relay_test SMALLINT NOT NULL DEFAULT 0;
 
 down_m20260222_chat_relays :: Text
 down_m20260222_chat_relays =
-  T.pack
-    [r|
+  [r|
 UPDATE group_members SET member_role = 'observer' WHERE member_role = 'relay';
 
 ALTER TABLE users DROP COLUMN is_user_chat_relay;

src/Simplex/Chat/Store/Postgres/Migrations/M20260403_item_viewed.hs

@@ -0,0 +1,23 @@
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE QuasiQuotes #-}
+
+module Simplex.Chat.Store.Postgres.Migrations.M20260403_item_viewed where
+
+import Data.Text (Text)
+import Text.RawString.QQ (r)
+
+m20260403_item_viewed :: Text
+m20260403_item_viewed =
+  [r|
+ALTER TABLE chat_items ADD COLUMN item_viewed SMALLINT NOT NULL DEFAULT 0;
+CREATE INDEX idx_chat_items_contacts_item_viewed ON chat_items(user_id, contact_id, item_viewed, created_at);
+CREATE INDEX idx_chat_items_groups_item_viewed ON chat_items(user_id, group_id, item_viewed, item_ts);
+|]
+
+down_m20260403_item_viewed :: Text
+down_m20260403_item_viewed =
+  [r|
+DROP INDEX idx_chat_items_contacts_item_viewed;
+DROP INDEX idx_chat_items_groups_item_viewed;
+ALTER TABLE chat_items DROP COLUMN item_viewed;
+|]

src/Simplex/Chat/Store/Postgres/Migrations/chat_schema.sql

@@ -344,7 +344,8 @@ CREATE TABLE test_chat_schema.chat_items (
     group_scope_group_member_id bigint,
     show_group_as_sender smallint DEFAULT 0 NOT NULL,
     has_link smallint DEFAULT 0 NOT NULL,
-    msg_signed text
+    msg_signed text,
+    item_viewed smallint DEFAULT 0 NOT NULL
 );
 
 
@@ -1906,6 +1907,10 @@ CREATE INDEX idx_chat_items_contacts_has_link_created_at ON test_chat_schema.cha
 
 
 
+CREATE INDEX idx_chat_items_contacts_item_viewed ON test_chat_schema.chat_items USING btree (user_id, contact_id, item_viewed, created_at);
+
+
+
 CREATE INDEX idx_chat_items_contacts_msg_content_tag_created_at ON test_chat_schema.chat_items USING btree (user_id, contact_id, msg_content_tag, created_at);
 
 
@@ -1978,6 +1983,10 @@ CREATE INDEX idx_chat_items_groups_item_ts ON test_chat_schema.chat_items USING
 
 
 
+CREATE INDEX idx_chat_items_groups_item_viewed ON test_chat_schema.chat_items USING btree (user_id, group_id, item_viewed, item_ts);
+
+
+
 CREATE INDEX idx_chat_items_groups_msg_content_tag_deleted ON test_chat_schema.chat_items USING btree (user_id, group_id, msg_content_tag, item_deleted, item_sent);
 
 

src/Simplex/Chat/Store/SQLite/Migrations.hs

@@ -150,6 +150,7 @@ import Simplex.Chat.Store.SQLite.Migrations.M20251230_strict_tables
 import Simplex.Chat.Store.SQLite.Migrations.M20260108_chat_indices
 import Simplex.Chat.Store.SQLite.Migrations.M20260122_has_link
 import Simplex.Chat.Store.SQLite.Migrations.M20260222_chat_relays
+import Simplex.Chat.Store.SQLite.Migrations.M20260403_item_viewed
 import Simplex.Messaging.Agent.Store.Shared (Migration (..))
 
 schemaMigrations :: [(String, Query, Maybe Query)]
@@ -299,7 +300,8 @@ schemaMigrations =
     ("20251230_strict_tables", m20251230_strict_tables, Just down_m20251230_strict_tables),
     ("20260108_chat_indices", m20260108_chat_indices, Just down_m20260108_chat_indices),
     ("20260122_has_link", m20260122_has_link, Just down_m20260122_has_link),
-    ("20260222_chat_relays", m20260222_chat_relays, Just down_m20260222_chat_relays)
+    ("20260222_chat_relays", m20260222_chat_relays, Just down_m20260222_chat_relays),
+    ("20260403_item_viewed", m20260403_item_viewed, Just down_m20260403_item_viewed)
   ]
 
 -- | The list of migrations in ascending order by date

src/Simplex/Chat/Store/SQLite/Migrations/M20260403_item_viewed.hs

@@ -0,0 +1,22 @@
+{-# LANGUAGE QuasiQuotes #-}
+
+module Simplex.Chat.Store.SQLite.Migrations.M20260403_item_viewed where
+
+import Database.SQLite.Simple (Query)
+import Database.SQLite.Simple.QQ (sql)
+
+m20260403_item_viewed :: Query
+m20260403_item_viewed =
+  [sql|
+ALTER TABLE chat_items ADD COLUMN item_viewed INTEGER NOT NULL DEFAULT 0;
+CREATE INDEX idx_chat_items_contacts_item_viewed ON chat_items(user_id, contact_id, item_viewed, created_at);
+CREATE INDEX idx_chat_items_groups_item_viewed ON chat_items(user_id, group_id, item_viewed, item_ts);
+|]
+
+down_m20260403_item_viewed :: Query
+down_m20260403_item_viewed =
+  [sql|
+DROP INDEX idx_chat_items_contacts_item_viewed;
+DROP INDEX idx_chat_items_groups_item_viewed;
+ALTER TABLE chat_items DROP COLUMN item_viewed;
+|]

src/Simplex/Chat/Store/SQLite/Migrations/chat_query_plans.txt

@@ -868,7 +868,7 @@ Query:
             LIMIT ?
           
 Plan:
-SEARCH chat_items USING INDEX idx_chat_items_note_folder_has_link_created_at (user_id=?)
+SEARCH chat_items USING INDEX idx_chat_items_groups_item_viewed (user_id=?)
 USE TEMP B-TREE FOR ORDER BY
 
 Query: 
@@ -879,7 +879,7 @@ Query:
             LIMIT ?
           
 Plan:
-SEARCH chat_items USING INDEX idx_chat_items_note_folder_has_link_created_at (user_id=?)
+SEARCH chat_items USING INDEX idx_chat_items_groups_item_viewed (user_id=?)
 USE TEMP B-TREE FOR ORDER BY
 
 Query: 
@@ -1064,7 +1064,7 @@ SEARCH g USING INTEGER PRIMARY KEY (rowid=?) LEFT-JOIN
 SEARCH h USING INDEX idx_sent_probe_hashes_sent_probe_id (sent_probe_id=?)
 
 Query: 
-            UPDATE chat_items SET item_status = ?, updated_at = ?
+            UPDATE chat_items SET item_status = ?, item_viewed = 1, updated_at = ?
             WHERE user_id = ? AND group_id = ? AND item_status = ? AND chat_item_id = ?
             RETURNING chat_item_id, timed_ttl, timed_delete_at, group_member_id, user_mention
           
@@ -1469,7 +1469,7 @@ Query:
           LIMIT ?
         
 Plan:
-SEARCH chat_items USING INDEX idx_chat_items_note_folder_has_link_created_at (user_id=?)
+SEARCH chat_items USING INDEX idx_chat_items_groups_item_viewed (user_id=?)
 USE TEMP B-TREE FOR ORDER BY
 
 Query: 
@@ -1651,7 +1651,7 @@ SEARCH m USING INTEGER PRIMARY KEY (rowid=?) LEFT-JOIN
 SEARCH g USING INTEGER PRIMARY KEY (rowid=?) LEFT-JOIN
 
 Query: 
-          UPDATE chat_items SET item_status = ?, updated_at = ?
+          UPDATE chat_items SET item_status = ?, item_viewed = 1, updated_at = ?
           WHERE user_id = ? AND group_id = ?
             AND group_scope_tag = ? AND group_scope_group_member_id IS NOT DISTINCT FROM ?
             AND item_status = ?
@@ -3382,16 +3382,6 @@ Query:
 Plan:
 SEARCH chat_items USING INDEX idx_chat_items_contact_id (contact_id=?)
 
-Query: 
-        SELECT chat_item_id
-        FROM chat_items
-        WHERE user_id = ? AND contact_id = ? AND item_status != ?
-        ORDER BY created_at DESC, chat_item_id DESC
-        LIMIT 1
-      
-Plan:
-SEARCH chat_items USING INDEX idx_chat_items_contacts_created_at (user_id=? AND contact_id=?)
-
 Query: 
         SELECT chat_item_id
         FROM chat_items
@@ -3412,6 +3402,16 @@ Query:
 Plan:
 SEARCH chat_items USING INDEX idx_chat_items_contact_id (contact_id=?)
 
+Query: 
+        SELECT chat_item_id
+        FROM chat_items
+        WHERE user_id = ? AND contact_id = ? AND item_viewed = 1
+        ORDER BY created_at DESC, chat_item_id DESC
+        LIMIT 1
+      
+Plan:
+SEARCH chat_items USING COVERING INDEX idx_chat_items_contacts_item_viewed (user_id=? AND contact_id=? AND item_viewed=?)
+
 Query: 
         SELECT chat_item_id
         FROM chat_items
@@ -4540,12 +4540,12 @@ Query:
         user_id, created_by_msg_id, contact_id, group_id, group_member_id, note_folder_id, group_scope_tag, group_scope_group_member_id,
         -- meta
         item_sent, item_ts, item_content, item_content_tag, item_text, item_status, msg_content_tag, shared_msg_id,
-        forwarded_by_group_member_id, include_in_history, created_at, updated_at, item_live, user_mention, has_link, show_group_as_sender, msg_signed, timed_ttl, timed_delete_at,
+        forwarded_by_group_member_id, include_in_history, created_at, updated_at, item_live, user_mention, has_link, item_viewed, show_group_as_sender, msg_signed, timed_ttl, timed_delete_at,
         -- quote
         quoted_shared_msg_id, quoted_sent_at, quoted_content, quoted_sent, quoted_member_id,
         -- forwarded from
         fwd_from_tag, fwd_from_chat_name, fwd_from_msg_dir, fwd_from_contact_id, fwd_from_group_id, fwd_from_chat_item_id
-      ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
+      ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
     
 Plan:
 
@@ -4763,21 +4763,21 @@ Plan:
 SEARCH chat_items USING INTEGER PRIMARY KEY (rowid=?)
 
 Query: 
-      UPDATE chat_items SET item_status = ?, updated_at = ?
+      UPDATE chat_items SET item_status = ?, item_viewed = 1, updated_at = ?
       WHERE user_id = ? AND contact_id = ? AND item_status = ?
     
 Plan:
 SEARCH chat_items USING INDEX idx_chat_items_contacts (user_id=? AND contact_id=? AND item_status=?)
 
 Query: 
-      UPDATE chat_items SET item_status = ?, updated_at = ?
+      UPDATE chat_items SET item_status = ?, item_viewed = 1, updated_at = ?
       WHERE user_id = ? AND contact_id = ? AND item_status = ? AND chat_item_id = ?
     
 Plan:
 SEARCH chat_items USING INTEGER PRIMARY KEY (rowid=?)
 
 Query: 
-      UPDATE chat_items SET item_status = ?, updated_at = ?
+      UPDATE chat_items SET item_status = ?, item_viewed = 1, updated_at = ?
       WHERE user_id = ? AND group_id = ?
         AND item_status = ?
     
@@ -4785,7 +4785,7 @@ Plan:
 SEARCH chat_items USING INDEX idx_chat_items_groups_user_mention (user_id=? AND group_id=? AND item_status=?)
 
 Query: 
-      UPDATE chat_items SET item_status = ?, updated_at = ?
+      UPDATE chat_items SET item_status = ?, item_viewed = 1, updated_at = ?
       WHERE user_id = ? AND note_folder_id = ? AND item_status = ?
     
 Plan:
@@ -5545,7 +5545,7 @@ Query:
     JOIN files f ON f.chat_item_id = i.chat_item_id
    WHERE i.user_id = ?
 Plan:
-SEARCH i USING COVERING INDEX idx_chat_items_note_folder_has_link_created_at (user_id=?)
+SEARCH i USING COVERING INDEX idx_chat_items_groups_item_viewed (user_id=?)
 SEARCH f USING INDEX idx_files_chat_item_id (chat_item_id=?)
 
 Query: 
@@ -5554,7 +5554,7 @@ Query:
     JOIN files f ON f.chat_item_id = i.chat_item_id
    WHERE i.user_id = ? AND i.contact_id = ?
 Plan:
-SEARCH i USING COVERING INDEX idx_chat_items_contacts_has_link_created_at (user_id=? AND contact_id=?)
+SEARCH i USING COVERING INDEX idx_chat_items_contacts_item_viewed (user_id=? AND contact_id=?)
 SEARCH f USING INDEX idx_files_chat_item_id (chat_item_id=?)
 
 Query: 
@@ -5572,7 +5572,7 @@ Query:
     JOIN files f ON f.chat_item_id = i.chat_item_id
    WHERE i.user_id = ? AND i.group_id = ?
 Plan:
-SEARCH i USING COVERING INDEX idx_chat_items_groups_has_link_item_ts (user_id=? AND group_id=?)
+SEARCH i USING COVERING INDEX idx_chat_items_groups_item_viewed (user_id=? AND group_id=?)
 SEARCH f USING INDEX idx_files_chat_item_id (chat_item_id=?)
 
 Query: 
@@ -6027,7 +6027,7 @@ SEARCH chat_item_versions USING COVERING INDEX idx_chat_item_versions_chat_item_
 
 Query: DELETE FROM chat_items WHERE user_id = ? AND contact_id = ?
 Plan:
-SEARCH chat_items USING COVERING INDEX idx_chat_items_contacts_has_link_created_at (user_id=? AND contact_id=?)
+SEARCH chat_items USING COVERING INDEX idx_chat_items_contacts_item_viewed (user_id=? AND contact_id=?)
 SEARCH chat_item_mentions USING COVERING INDEX idx_chat_item_mentions_chat_item_id (chat_item_id=?)
 SEARCH group_snd_item_statuses USING COVERING INDEX idx_group_snd_item_statuses_chat_item_id (chat_item_id=?)
 SEARCH chat_item_versions USING COVERING INDEX idx_chat_item_versions_chat_item_id (chat_item_id=?)
@@ -6051,7 +6051,7 @@ SEARCH groups USING COVERING INDEX idx_groups_chat_item_id (chat_item_id=?)
 
 Query: DELETE FROM chat_items WHERE user_id = ? AND contact_id = ? AND item_content_tag != 'chatBanner'
 Plan:
-SEARCH chat_items USING INDEX idx_chat_items_contacts_has_link_created_at (user_id=? AND contact_id=?)
+SEARCH chat_items USING INDEX idx_chat_items_contacts_item_viewed (user_id=? AND contact_id=?)
 SEARCH chat_item_mentions USING COVERING INDEX idx_chat_item_mentions_chat_item_id (chat_item_id=?)
 SEARCH group_snd_item_statuses USING COVERING INDEX idx_group_snd_item_statuses_chat_item_id (chat_item_id=?)
 SEARCH chat_item_versions USING COVERING INDEX idx_chat_item_versions_chat_item_id (chat_item_id=?)
@@ -6063,7 +6063,7 @@ SEARCH groups USING COVERING INDEX idx_groups_chat_item_id (chat_item_id=?)
 
 Query: DELETE FROM chat_items WHERE user_id = ? AND group_id = ?
 Plan:
-SEARCH chat_items USING COVERING INDEX idx_chat_items_groups_has_link_item_ts (user_id=? AND group_id=?)
+SEARCH chat_items USING COVERING INDEX idx_chat_items_groups_item_viewed (user_id=? AND group_id=?)
 SEARCH chat_item_mentions USING COVERING INDEX idx_chat_item_mentions_chat_item_id (chat_item_id=?)
 SEARCH group_snd_item_statuses USING COVERING INDEX idx_group_snd_item_statuses_chat_item_id (chat_item_id=?)
 SEARCH chat_item_versions USING COVERING INDEX idx_chat_item_versions_chat_item_id (chat_item_id=?)
@@ -6087,7 +6087,7 @@ SEARCH groups USING COVERING INDEX idx_groups_chat_item_id (chat_item_id=?)
 
 Query: DELETE FROM chat_items WHERE user_id = ? AND group_id = ? AND item_content_tag != 'chatBanner'
 Plan:
-SEARCH chat_items USING INDEX idx_chat_items_groups_has_link_item_ts (user_id=? AND group_id=?)
+SEARCH chat_items USING INDEX idx_chat_items_groups_item_viewed (user_id=? AND group_id=?)
 SEARCH chat_item_mentions USING COVERING INDEX idx_chat_item_mentions_chat_item_id (chat_item_id=?)
 SEARCH group_snd_item_statuses USING COVERING INDEX idx_group_snd_item_statuses_chat_item_id (chat_item_id=?)
 SEARCH chat_item_versions USING COVERING INDEX idx_chat_item_versions_chat_item_id (chat_item_id=?)
@@ -6390,7 +6390,7 @@ SEARCH protocol_servers USING COVERING INDEX idx_smp_servers_user_id (user_id=?)
 SEARCH settings USING COVERING INDEX idx_settings_user_id (user_id=?)
 SEARCH commands USING COVERING INDEX idx_commands_user_id (user_id=?)
 SEARCH calls USING COVERING INDEX idx_calls_user_id (user_id=?)
-SEARCH chat_items USING COVERING INDEX idx_chat_items_note_folder_has_link_created_at (user_id=?)
+SEARCH chat_items USING COVERING INDEX idx_chat_items_groups_item_viewed (user_id=?)
 SEARCH contact_requests USING COVERING INDEX sqlite_autoindex_contact_requests_2 (user_id=?)
 SEARCH user_contact_links USING COVERING INDEX sqlite_autoindex_user_contact_links_1 (user_id=?)
 SEARCH connections USING COVERING INDEX idx_connections_to_subscribe (user_id=?)
@@ -6554,7 +6554,7 @@ Query: SELECT EXISTS (SELECT 1 FROM chat_items WHERE user_id = ? AND contact_id
 Plan:
 SCAN CONSTANT ROW
 SCALAR SUBQUERY 1
-SEARCH chat_items USING COVERING INDEX idx_chat_items_contacts_has_link_created_at (user_id=? AND contact_id=?)
+SEARCH chat_items USING COVERING INDEX idx_chat_items_contacts_item_viewed (user_id=? AND contact_id=?)
 
 Query: SELECT EXISTS (SELECT 1 FROM group_members WHERE group_id = ? AND member_id = ?)
 Plan:
@@ -6588,14 +6588,14 @@ Query: SELECT chat_item_id FROM chat_items WHERE user_id = ? AND contact_id = ?
 Plan:
 SEARCH chat_items USING INDEX idx_chat_items_direct_shared_msg_id (user_id=? AND contact_id=? AND shared_msg_id=?)
 
-Query: SELECT chat_item_id FROM chat_items WHERE user_id = ? AND group_id = ?  AND group_scope_tag IS NULL AND group_scope_group_member_id IS NULL AND  item_status != ?  ORDER BY item_ts DESC, chat_item_id DESC LIMIT 1
-Plan:
-SEARCH chat_items USING INDEX idx_chat_items_group_scope_item_ts (user_id=? AND group_id=? AND group_scope_tag=? AND group_scope_group_member_id=?)
-
 Query: SELECT chat_item_id FROM chat_items WHERE user_id = ? AND group_id = ?  AND group_scope_tag IS NULL AND group_scope_group_member_id IS NULL AND  item_status = ?  ORDER BY item_ts ASC, chat_item_id ASC LIMIT 1
 Plan:
 SEARCH chat_items USING COVERING INDEX idx_chat_items_group_scope_item_status (user_id=? AND group_id=? AND group_scope_tag=? AND group_scope_group_member_id=? AND item_status=?)
 
+Query: SELECT chat_item_id FROM chat_items WHERE user_id = ? AND group_id = ?  AND group_scope_tag IS NULL AND group_scope_group_member_id IS NULL AND  item_viewed = ?  ORDER BY item_ts DESC, chat_item_id DESC LIMIT 1
+Plan:
+SEARCH chat_items USING INDEX idx_chat_items_groups_item_viewed (user_id=? AND group_id=? AND item_viewed=?)
+
 Query: SELECT chat_item_id FROM chat_items WHERE user_id = ? AND group_id = ? AND group_member_id = ? LIMIT 1
 Plan:
 SEARCH chat_items USING COVERING INDEX idx_chat_items_group_shared_msg_id (user_id=? AND group_id=? AND group_member_id=?)
@@ -6820,6 +6820,10 @@ Query: SELECT xgrplinkmem_received FROM group_members WHERE group_member_id = ?
 Plan:
 SEARCH group_members USING INTEGER PRIMARY KEY (rowid=?)
 
+Query: UPDATE chat_items SET item_status = ?, item_viewed = 1, updated_at = ? WHERE user_id = ? AND item_status = ?
+Plan:
+SEARCH chat_items USING INDEX idx_chat_items_user_id_item_status (user_id=? AND item_status=?)
+
 Query: UPDATE chat_items SET item_status = ?, updated_at = ? WHERE user_id = ? AND contact_id = ? AND chat_item_id = ?
 Plan:
 SEARCH chat_items USING INTEGER PRIMARY KEY (rowid=?)
@@ -6828,10 +6832,6 @@ Query: UPDATE chat_items SET item_status = ?, updated_at = ? WHERE user_id = ? A
 Plan:
 SEARCH chat_items USING INTEGER PRIMARY KEY (rowid=?)
 
-Query: UPDATE chat_items SET item_status = ?, updated_at = ? WHERE user_id = ? AND item_status = ?
-Plan:
-SEARCH chat_items USING INDEX idx_chat_items_user_id_item_status (user_id=? AND item_status=?)
-
 Query: UPDATE chat_items SET timed_delete_at = ? WHERE user_id = ? AND contact_id = ? AND chat_item_id = ?
 Plan:
 SEARCH chat_items USING INTEGER PRIMARY KEY (rowid=?)

src/Simplex/Chat/Store/SQLite/Migrations/chat_schema.sql

@@ -464,7 +464,8 @@ CREATE TABLE chat_items(
   group_scope_group_member_id INTEGER REFERENCES group_members(group_member_id) ON DELETE CASCADE,
   show_group_as_sender INTEGER NOT NULL DEFAULT 0,
   has_link INTEGER NOT NULL DEFAULT 0,
-  msg_signed TEXT
+  msg_signed TEXT,
+  item_viewed INTEGER NOT NULL DEFAULT 0
 ) STRICT;
 CREATE TABLE sqlite_sequence(name,seq);
 CREATE TABLE chat_item_messages(
@@ -1278,6 +1279,18 @@ CREATE UNIQUE INDEX idx_group_relays_group_member_id ON group_relays(
   group_member_id
 );
 CREATE INDEX idx_group_relays_chat_relay_id ON group_relays(chat_relay_id);
+CREATE INDEX idx_chat_items_contacts_item_viewed ON chat_items(
+  user_id,
+  contact_id,
+  item_viewed,
+  created_at
+);
+CREATE INDEX idx_chat_items_groups_item_viewed ON chat_items(
+  user_id,
+  group_id,
+  item_viewed,
+  item_ts
+);
 CREATE TRIGGER on_group_members_insert_update_summary
 AFTER INSERT ON group_members
 FOR EACH ROW

tests/ChatTests/Direct.hs

@@ -389,7 +389,7 @@ testChatPaginationInitial = testChatOpts2 opts aliceProfile bobProfile $ \alice
   -- Read next 2 items
   let itemIds = intercalate "," $ map itemId [1 .. 2]
   bob #$> ("/_read chat items @2 " <> itemIds, id, "items read for chat")
-  bob #$> ("/_get chat @2 initial=2", chat, [(0, "1"), (0, "2"), (0, "3"), (0, "4"), (0, "5")])
+  bob #$> ("/_get chat @2 initial=2", chat, [(0, "Audio/video calls: enabled"), (0, "1"), (0, "2"), (0, "3"), (0, "4")])
 
   -- Read all items
   bob #$> ("/_read chat @2", id, "ok")

tests/ChatTests/Files.hs

@@ -50,7 +50,7 @@ chatFileTests = do
     it "delete uploaded file in group" testXFTPDeleteUploadedFileGroup
     it "with relative paths: send and receive file" testXFTPWithRelativePaths
     xit' "continue receiving file after restart" testXFTPContinueRcv
-    xit' "receive file marked to receive on chat start" testXFTPMarkToReceive
+    it "receive file marked to receive on chat start" testXFTPMarkToReceive
     it "error receiving file" testXFTPRcvError
     it "cancel receiving file, repeat receive" testXFTPCancelRcvRepeat
     it "should accept file automatically with CLI option" testAutoAcceptFile
@@ -913,10 +913,10 @@ testXFTPMarkToReceive = do
       threadDelay 100000
 
       bob ##> "/_start"
-      bob <## "chat started"
 
       bob
-        <### [ "subscribed 1 connections on server localhost",
+        <### [ "chat started",
+               "subscribed 1 connections on server localhost",
                "started receiving file 1 (test.pdf) from alice",
                "saving file 1 from alice to test.pdf"
              ]

tests/ChatTests/Groups.hs

@@ -456,7 +456,7 @@ testChatPaginationInitial = testChatOpts2 opts aliceProfile bobProfile $ \alice
   -- Read next 2 items
   let itemIds = intercalate "," $ map groupItemId [1 .. 2]
   bob #$> ("/_read chat items #1 " <> itemIds, id, "items read for chat")
-  bob #$> ("/_get chat #1 initial=2", chat, [(0, "1"), (0, "2"), (0, "3"), (0, "4"), (0, "5")])
+  bob #$> ("/_get chat #1 initial=2", chat, [(0, "connected"), (0, "1"), (0, "2"), (0, "3"), (0, "4")])
 
   -- Read all items
   bob #$> ("/_read chat #1", id, "ok")

tests/ChatTests/Profiles.hs

@@ -4111,6 +4111,7 @@ testShortLinkGroupChangeProfileReceived = testChat3 aliceProfile bobProfile cath
       cath <## "changed to #club"
       alice <## "cath updated group #team:"
       alice <## "changed to #club"
+      threadDelay 250000
 
       bob ##> ("/_connect plan 1 " <> shortLink)
       bob <## "group link: ok to connect directly"