core: request roster (#7121)

This commit is contained in:
spaced4ndy authored and GitHub committed 2026-07-03 19:32:29 +00:00
1 parent 8495b6fbe1
commit 6bb1da9e8d
15 files changed
+562 -84

No files matched your search

+55 -10
View File
@@ -92,7 +92,12 @@ module Simplex.Chat.Store.Groups
getPublishableGroupRelays,
setGroupRosterVersion,
getGroupRosterVersion,
getGroupRoster,
getStoredRosterVersion,
setMemberRosterServedVersion,
getMemberRosterServedVersion,
setCompleteRosterVersion,
getCompleteRosterVersion,
getStoredGroupRoster,
RcvRosterTransfer (..),
createRosterTransfer,
getRosterTransferVersion,
@@ -1499,21 +1504,57 @@ getGroupRosterVersion db GroupInfo {groupId} =
fmap join . maybeFirstRow fromOnly $
DB.query db "SELECT roster_version FROM groups WHERE group_id = ?" (Only groupId)
-- The live roster header a relay re-serves to joiners, with the completed blob served alongside it
-- (both are written together at completion, so the blob is present whenever the header is).
getGroupRoster :: DB.Connection -> GroupInfo -> IO (Maybe (GroupMemberId, UTCTime, SignedMsg, Maybe ByteString))
getGroupRoster db GroupInfo {groupId} =
-- The version of the roster blob actually stored (written with the blob in setGroupLiveRoster), as opposed to
-- roster_version (the acceptance gate). The owner sends the blob before the delta, so these normally match; a
-- failed blob send leaves the gate (advanced by the delta) ahead of the stored blob until a later roster completes.
getStoredRosterVersion :: DB.Connection -> GroupInfo -> IO (Maybe VersionRoster)
getStoredRosterVersion db GroupInfo {groupId} =
fmap join . maybeFirstRow fromOnly $
DB.query db "SELECT stored_roster_version FROM groups WHERE group_id = ?" (Only groupId)
-- The newest roster version a relay re-served to this member on its catch-up request: bounds reflected
-- amplification, so a member can't re-trigger a full serve at a version it was already served.
setMemberRosterServedVersion :: DB.Connection -> GroupMember -> VersionRoster -> IO ()
setMemberRosterServedVersion db GroupMember {groupMemberId} v = do
currentTs <- getCurrentTime
DB.execute db "UPDATE group_members SET roster_served_version = ?, updated_at = ? WHERE group_member_id = ?" (v, currentTs, groupMemberId)
getMemberRosterServedVersion :: DB.Connection -> GroupMember -> IO (Maybe VersionRoster)
getMemberRosterServedVersion db GroupMember {groupMemberId} =
fmap join . maybeFirstRow fromOnly $
DB.query db "SELECT roster_served_version FROM group_members WHERE group_member_id = ?" (Only groupMemberId)
-- The highest version up to which the subscriber holds a complete, contiguous picture: advances by 1 on a
-- contiguous delta and to the roster's version on a roster apply, but stays put on a gapped delta (so a stuck
-- value re-triggers the catch-up request on every following delta until a roster fills the gap). This is the
-- subscriber's "what I have" for both gap detection and the request - as opposed to roster_version (highest seen,
-- the revert gate) and stored_roster_version (the blob a relay holds).
setCompleteRosterVersion :: DB.Connection -> GroupInfo -> VersionRoster -> IO ()
setCompleteRosterVersion db GroupInfo {groupId} v = do
currentTs <- getCurrentTime
DB.execute db "UPDATE groups SET applied_complete_roster_version = ?, updated_at = ? WHERE group_id = ?" (v, currentTs, groupId)
getCompleteRosterVersion :: DB.Connection -> GroupInfo -> IO (Maybe VersionRoster)
getCompleteRosterVersion db GroupInfo {groupId} =
fmap join . maybeFirstRow fromOnly $
DB.query db "SELECT applied_complete_roster_version FROM groups WHERE group_id = ?" (Only groupId)
-- The live roster header a relay re-serves to joiners, with the completed blob and its stored version
-- (all written together at completion, so the blob and version are present whenever the header is).
-- Returns the stored version, not roster_version (the gate), so callers serve/record exactly what they hold.
getStoredGroupRoster :: DB.Connection -> GroupInfo -> IO (Maybe (GroupMemberId, UTCTime, SignedMsg, Maybe ByteString, Maybe VersionRoster))
getStoredGroupRoster db GroupInfo {groupId} =
(>>= toRoster)
<$> maybeFirstRow
id
( DB.query
db
"SELECT roster_sending_owner_gm_id, roster_broker_ts, roster_msg_chat_binding, roster_msg_signatures, roster_msg_body, roster_blob FROM groups WHERE group_id = ?"
"SELECT roster_sending_owner_gm_id, roster_broker_ts, roster_msg_chat_binding, roster_msg_signatures, roster_msg_body, roster_blob, stored_roster_version FROM groups WHERE group_id = ?"
(Only groupId)
)
where
toRoster (Just ownerGMId, Just brokerTs, Just cb, Just (Binary sigsBs), Just (Binary body), blob_) =
(\sigs -> (ownerGMId, brokerTs, SignedMsg cb sigs body, (\(Binary b) -> b) <$> blob_)) <$> eitherToMaybe (smpDecode sigsBs)
toRoster (Just ownerGMId, Just brokerTs, Just cb, Just (Binary sigsBs), Just (Binary body), blob_, storedVer_) =
(\sigs -> (ownerGMId, brokerTs, SignedMsg cb sigs body, (\(Binary b) -> b) <$> blob_, storedVer_)) <$> eitherToMaybe (smpDecode sigsBs)
toRoster _ = Nothing
-- A per-source in-flight roster transfer, keyed (group_id, from_member_id): replaces the single
@@ -1598,6 +1639,10 @@ getRosterTransfer db fileId =
-- Write the single live roster on groups from a completed transfer's values (header NULL on a member,
-- so its live roster_msg_* stay NULL and it never re-serves; only relays re-serve).
-- Sets all three versions to the completed blob's version: the gate (roster_version - refuse anything older),
-- the stored version (stored_roster_version - the blob actually held and re-served), and the complete frontier
-- (applied_complete_roster_version - a snapshot makes the picture complete up to its version). Deltas advance the
-- gate always and the complete frontier only when contiguous, so complete <= stored <= roster_version normally.
setGroupLiveRoster :: DB.Connection -> GroupInfo -> VersionRoster -> GroupMemberId -> UTCTime -> Maybe SignedMsg -> ByteString -> IO ()
setGroupLiveRoster db GroupInfo {groupId} v ownerGMId brokerTs sm_ blob = do
currentTs <- getCurrentTime
@@ -1605,13 +1650,13 @@ setGroupLiveRoster db GroupInfo {groupId} v ownerGMId brokerTs sm_ blob = do
db
[sql|
UPDATE groups SET
roster_version = ?, roster_blob = ?,
roster_version = ?, stored_roster_version = ?, applied_complete_roster_version = ?, roster_blob = ?,
roster_sending_owner_gm_id = ?, roster_broker_ts = ?,
roster_msg_chat_binding = ?, roster_msg_signatures = ?, roster_msg_body = ?,
updated_at = ?
WHERE group_id = ?
|]
( (v, Binary blob, ownerGMId, brokerTs)
( (v, v, v, Binary blob, ownerGMId, brokerTs)
:. ((\SignedMsg {chatBinding} -> chatBinding) <$> sm_, (\SignedMsg {signatures} -> Binary (smpEncode signatures)) <$> sm_, (\SignedMsg {signedBody} -> Binary signedBody) <$> sm_, currentTs, groupId)
)
@@ -39,6 +39,7 @@ import Simplex.Chat.Store.Postgres.Migrations.M20260531_member_removed_at
import Simplex.Chat.Store.Postgres.Migrations.M20260601_relay_sent_web_domain
import Simplex.Chat.Store.Postgres.Migrations.M20260602_group_roster
import Simplex.Chat.Store.Postgres.Migrations.M20260603_simplex_name
import Simplex.Chat.Store.Postgres.Migrations.M20260629_roster_catchup
import Simplex.Messaging.Agent.Store.Shared (Migration (..))
schemaMigrations :: [(String, Text, Maybe Text)]
@@ -77,7 +78,8 @@ schemaMigrations =
("20260531_member_removed_at", m20260531_member_removed_at, Just down_m20260531_member_removed_at),
("20260601_relay_sent_web_domain", m20260601_relay_sent_web_domain, Just down_m20260601_relay_sent_web_domain),
("20260602_group_roster", m20260602_group_roster, Just down_m20260602_group_roster),
("20260603_simplex_name", m20260603_simplex_name, Just down_m20260603_simplex_name)
("20260603_simplex_name", m20260603_simplex_name, Just down_m20260603_simplex_name),
("20260629_roster_catchup", m20260629_roster_catchup, Just down_m20260629_roster_catchup)
]
-- | The list of migrations in ascending order by date
@@ -0,0 +1,41 @@
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE QuasiQuotes #-}
module Simplex.Chat.Store.Postgres.Migrations.M20260629_roster_catchup where
import Data.Text (Text)
import Text.RawString.QQ (r)
-- Roster catch-up bookkeeping. Three monotonic per-group roster versions with distinct roles - normally equal,
-- diverging across gaps and failed transfers (applied_complete <= stored <= roster_version):
-- roster_version (added in M20260602) - the GATE: highest version seen. Advanced by every accepted owner delta
-- and by a roster apply. Revert protection: a completing roster older than this is rejected, since its
-- snapshot would undo a newer applied delta.
-- stored_roster_version - the blob HELD: the version of the roster blob actually stored (written with the blob).
-- What a relay can re-serve; a failed blob receive leaves it behind the gate (which the delta still advances)
-- until a later roster completes. Relay-side; on a member it is set but the blob is unused.
-- applied_complete_roster_version - the COMPLETE frontier: highest version up to which the picture is contiguous.
-- Advances by 1 on a contiguous delta and to the roster's version on apply, but stays put on a gapped delta.
-- The subscriber's "what I have" for gap detection and the catch-up request: a value below the gate means
-- missed versions, so each following delta re-asks the forwarding relay until a roster fills the frontier.
-- Also adds group_members.roster_served_version - the newest version a relay re-served a given member, bounding
-- reflected amplification (a member can't re-trigger a full serve at a version it was already served).
-- Backfill an existing roster's stored and complete versions from roster_version: pre-upgrade the picture is
-- contiguous up to roster_version (no gap detection existed), so a fresh NULL frontier would read every group's
-- next delta as a gap and make every subscriber request a re-serve at once.
m20260629_roster_catchup :: Text
m20260629_roster_catchup =
[r|
ALTER TABLE group_members ADD COLUMN roster_served_version BIGINT;
ALTER TABLE groups ADD COLUMN stored_roster_version BIGINT;
ALTER TABLE groups ADD COLUMN applied_complete_roster_version BIGINT;
UPDATE groups SET stored_roster_version = roster_version, applied_complete_roster_version = roster_version WHERE roster_version IS NOT NULL;
|]
down_m20260629_roster_catchup :: Text
down_m20260629_roster_catchup =
[r|
ALTER TABLE group_members DROP COLUMN roster_served_version;
ALTER TABLE groups DROP COLUMN stored_roster_version;
ALTER TABLE groups DROP COLUMN applied_complete_roster_version;
|]
@@ -834,7 +834,8 @@ CREATE TABLE test_chat_schema.group_members (
member_relations_vector bytea,
relay_link bytea,
member_pub_key bytea,
removed_at timestamp with time zone
removed_at timestamp with time zone,
roster_served_version bigint
);
@@ -994,7 +995,9 @@ CREATE TABLE test_chat_schema.groups (
roster_sending_owner_gm_id bigint,
roster_broker_ts timestamp with time zone,
roster_blob bytea,
group_domain_verified smallint
group_domain_verified smallint,
stored_roster_version bigint,
applied_complete_roster_version bigint
);
+3 -1
View File
@@ -162,6 +162,7 @@ import Simplex.Chat.Store.SQLite.Migrations.M20260531_member_removed_at
import Simplex.Chat.Store.SQLite.Migrations.M20260601_relay_sent_web_domain
import Simplex.Chat.Store.SQLite.Migrations.M20260602_group_roster
import Simplex.Chat.Store.SQLite.Migrations.M20260603_simplex_name
import Simplex.Chat.Store.SQLite.Migrations.M20260629_roster_catchup
import Simplex.Messaging.Agent.Store.Shared (Migration (..))
schemaMigrations :: [(String, Query, Maybe Query)]
@@ -323,7 +324,8 @@ schemaMigrations =
("20260531_member_removed_at", m20260531_member_removed_at, Just down_m20260531_member_removed_at),
("20260601_relay_sent_web_domain", m20260601_relay_sent_web_domain, Just down_m20260601_relay_sent_web_domain),
("20260602_group_roster", m20260602_group_roster, Just down_m20260602_group_roster),
("20260603_simplex_name", m20260603_simplex_name, Just down_m20260603_simplex_name)
("20260603_simplex_name", m20260603_simplex_name, Just down_m20260603_simplex_name),
("20260629_roster_catchup", m20260629_roster_catchup, Just down_m20260629_roster_catchup)
]
-- | The list of migrations in ascending order by date
@@ -0,0 +1,40 @@
{-# LANGUAGE QuasiQuotes #-}
module Simplex.Chat.Store.SQLite.Migrations.M20260629_roster_catchup where
import Database.SQLite.Simple (Query)
import Database.SQLite.Simple.QQ (sql)
-- Roster catch-up bookkeeping. Three monotonic per-group roster versions with distinct roles - normally equal,
-- diverging across gaps and failed transfers (applied_complete <= stored <= roster_version):
-- roster_version (added in M20260602) - the GATE: highest version seen. Advanced by every accepted owner delta
-- and by a roster apply. Revert protection: a completing roster older than this is rejected, since its
-- snapshot would undo a newer applied delta.
-- stored_roster_version - the blob HELD: the version of the roster blob actually stored (written with the blob).
-- What a relay can re-serve; a failed blob receive leaves it behind the gate (which the delta still advances)
-- until a later roster completes. Relay-side; on a member it is set but the blob is unused.
-- applied_complete_roster_version - the COMPLETE frontier: highest version up to which the picture is contiguous.
-- Advances by 1 on a contiguous delta and to the roster's version on apply, but stays put on a gapped delta.
-- The subscriber's "what I have" for gap detection and the catch-up request: a value below the gate means
-- missed versions, so each following delta re-asks the forwarding relay until a roster fills the frontier.
-- Also adds group_members.roster_served_version - the newest version a relay re-served a given member, bounding
-- reflected amplification (a member can't re-trigger a full serve at a version it was already served).
-- Backfill an existing roster's stored and complete versions from roster_version: pre-upgrade the picture is
-- contiguous up to roster_version (no gap detection existed), so a fresh NULL frontier would read every group's
-- next delta as a gap and make every subscriber request a re-serve at once.
m20260629_roster_catchup :: Query
m20260629_roster_catchup =
[sql|
ALTER TABLE group_members ADD COLUMN roster_served_version INTEGER;
ALTER TABLE groups ADD COLUMN stored_roster_version INTEGER;
ALTER TABLE groups ADD COLUMN applied_complete_roster_version INTEGER;
UPDATE groups SET stored_roster_version = roster_version, applied_complete_roster_version = roster_version WHERE roster_version IS NOT NULL;
|]
down_m20260629_roster_catchup :: Query
down_m20260629_roster_catchup =
[sql|
ALTER TABLE group_members DROP COLUMN roster_served_version;
ALTER TABLE groups DROP COLUMN stored_roster_version;
ALTER TABLE groups DROP COLUMN applied_complete_roster_version;
|]
@@ -5370,7 +5370,7 @@ SEARCH groups USING INTEGER PRIMARY KEY (rowid=?)
Query:
UPDATE groups SET
roster_version = ?, roster_blob = ?,
roster_version = ?, stored_roster_version = ?, applied_complete_roster_version = ?, roster_blob = ?,
roster_sending_owner_gm_id = ?, roster_broker_ts = ?,
roster_msg_chat_binding = ?, roster_msg_signatures = ?, roster_msg_body = ?,
updated_at = ?
@@ -6665,6 +6665,36 @@ SEARCH rcv_files USING INTEGER PRIMARY KEY (rowid=?)
SEARCH snd_files USING COVERING INDEX idx_snd_files_file_id (file_id=?)
SEARCH files USING COVERING INDEX idx_files_redirect_file_id (redirect_file_id=?)
Query: DELETE FROM group_members WHERE member_id = ?
Plan:
SCAN group_members
SEARCH rcv_roster_transfers USING COVERING INDEX idx_rcv_roster_transfers_from_member_id (from_member_id=?)
SEARCH group_relays USING COVERING INDEX idx_group_relays_group_member_id (group_member_id=?)
SEARCH delivery_jobs USING COVERING INDEX idx_delivery_jobs_job_scope_support_gm_id (job_scope_support_gm_id=?)
SEARCH delivery_tasks USING COVERING INDEX idx_delivery_tasks_sender_group_member_id (sender_group_member_id=?)
SEARCH delivery_tasks USING COVERING INDEX idx_delivery_tasks_job_scope_support_gm_id (job_scope_support_gm_id=?)
SEARCH received_probes USING COVERING INDEX idx_received_probes_group_member_id (group_member_id=?)
SEARCH sent_probe_hashes USING COVERING INDEX idx_sent_probe_hashes_group_member_id (group_member_id=?)
SEARCH sent_probes USING COVERING INDEX idx_sent_probes_group_member_id (group_member_id=?)
SEARCH group_snd_item_statuses USING COVERING INDEX idx_group_snd_item_statuses_group_member_id (group_member_id=?)
SEARCH chat_item_moderations USING COVERING INDEX idx_chat_item_moderations_moderator_member_id (moderator_member_id=?)
SEARCH chat_item_reactions USING COVERING INDEX idx_chat_item_reactions_group_member_id (group_member_id=?)
SEARCH chat_items USING COVERING INDEX idx_chat_items_group_scope_group_member_id (group_scope_group_member_id=?)
SEARCH chat_items USING COVERING INDEX idx_chat_items_forwarded_by_group_member_id (forwarded_by_group_member_id=?)
SEARCH chat_items USING COVERING INDEX idx_chat_items_item_deleted_by_group_member_id (item_deleted_by_group_member_id=?)
SEARCH chat_items USING COVERING INDEX idx_chat_items_group_member_id (group_member_id=?)
SEARCH pending_group_messages USING COVERING INDEX idx_pending_group_messages_group_member_id (group_member_id=?)
SEARCH messages USING COVERING INDEX idx_messages_forwarded_by_group_member_id (forwarded_by_group_member_id=?)
SEARCH messages USING COVERING INDEX idx_messages_author_group_member_id (author_group_member_id=?)
SEARCH connections USING COVERING INDEX idx_connections_group_member_id (group_member_id=?)
SEARCH rcv_files USING COVERING INDEX idx_rcv_files_group_member_id (group_member_id=?)
SEARCH snd_files USING COVERING INDEX idx_snd_files_group_member_id (group_member_id=?)
SEARCH group_member_intros USING COVERING INDEX idx_group_member_intros_to_group_member_id (to_group_member_id=?)
SEARCH group_member_intros USING COVERING INDEX idx_group_member_intros_re_group_member_id (re_group_member_id=?)
SEARCH group_members USING COVERING INDEX idx_group_members_invited_by_group_member_id (invited_by_group_member_id=?)
SEARCH contacts USING COVERING INDEX idx_contacts_grp_direct_inv_from_group_member_id (grp_direct_inv_from_group_member_id=?)
SEARCH contacts USING COVERING INDEX idx_contacts_contact_group_member_id (contact_group_member_id=?)
Query: DELETE FROM group_members WHERE user_id = ? AND group_id = ?
Plan:
SEARCH group_members USING COVERING INDEX idx_group_members_group_id (user_id=? AND group_id=?)
@@ -7076,6 +7106,10 @@ Query: SELECT app_settings FROM app_settings
Plan:
SCAN app_settings
Query: SELECT applied_complete_roster_version FROM groups WHERE group_id = ?
Plan:
SEARCH groups USING INTEGER PRIMARY KEY (rowid=?)
Query: SELECT auth_err_counter FROM connections WHERE user_id = ? AND connection_id = ?
Plan:
SEARCH connections USING INTEGER PRIMARY KEY (rowid=?)
@@ -7285,6 +7319,10 @@ Query: SELECT max(active_order) FROM users
Plan:
SEARCH users
Query: SELECT member_id FROM group_members WHERE local_display_name = ?
Plan:
SCAN group_members
Query: SELECT member_id FROM group_members WHERE member_role = ? LIMIT 1
Plan:
SCAN group_members
@@ -7313,6 +7351,10 @@ Query: SELECT member_role, member_pub_key FROM group_members WHERE local_display
Plan:
SCAN group_members
Query: SELECT member_role, member_pub_key FROM group_members WHERE member_id = ?
Plan:
SCAN group_members
Query: SELECT member_status FROM group_members WHERE local_display_name = ?
Plan:
SCAN group_members
@@ -7361,10 +7403,14 @@ Query: SELECT roster_blob FROM groups WHERE roster_blob IS NOT NULL
Plan:
SCAN groups
Query: SELECT roster_sending_owner_gm_id, roster_broker_ts, roster_msg_chat_binding, roster_msg_signatures, roster_msg_body, roster_blob FROM groups WHERE group_id = ?
Query: SELECT roster_sending_owner_gm_id, roster_broker_ts, roster_msg_chat_binding, roster_msg_signatures, roster_msg_body, roster_blob, stored_roster_version FROM groups WHERE group_id = ?
Plan:
SEARCH groups USING INTEGER PRIMARY KEY (rowid=?)
Query: SELECT roster_served_version FROM group_members WHERE group_member_id = ?
Plan:
SEARCH group_members USING INTEGER PRIMARY KEY (rowid=?)
Query: SELECT roster_transfer_id FROM rcv_roster_transfers WHERE group_id = ? AND from_member_id = ?
Plan:
SEARCH rcv_roster_transfers USING COVERING INDEX idx_rcv_roster_transfers_group_id_from_member_id (group_id=? AND from_member_id=?)
@@ -7389,6 +7435,10 @@ Query: SELECT should_sync FROM connections_sync WHERE connections_sync_id = 1
Plan:
SEARCH connections_sync USING INTEGER PRIMARY KEY (rowid=?)
Query: SELECT stored_roster_version FROM groups WHERE group_id = ?
Plan:
SEARCH groups USING INTEGER PRIMARY KEY (rowid=?)
Query: SELECT summary_current_members_count FROM groups WHERE group_id = ?
Plan:
SEARCH groups USING INTEGER PRIMARY KEY (rowid=?)
@@ -7633,6 +7683,10 @@ Query: UPDATE group_members SET member_role = ?, member_pub_key = NULL WHERE loc
Plan:
SCAN group_members
Query: UPDATE group_members SET roster_served_version = ?, updated_at = ? WHERE group_member_id = ?
Plan:
SEARCH group_members USING INTEGER PRIMARY KEY (rowid=?)
Query: UPDATE group_members SET support_chat_items_member_attention = ?, updated_at = ? WHERE group_member_id = ?
Plan:
SEARCH group_members USING INTEGER PRIMARY KEY (rowid=?)
@@ -7653,6 +7707,14 @@ Query: UPDATE group_relays SET relay_status = ?, updated_at = ? WHERE group_rela
Plan:
SEARCH group_relays USING INTEGER PRIMARY KEY (rowid=?)
Query: UPDATE groups SET applied_complete_roster_version = ? WHERE group_id = ?
Plan:
SEARCH groups USING INTEGER PRIMARY KEY (rowid=?)
Query: UPDATE groups SET applied_complete_roster_version = ?, updated_at = ? WHERE group_id = ?
Plan:
SEARCH groups USING INTEGER PRIMARY KEY (rowid=?)
Query: UPDATE groups SET business_member_id = ?, customer_member_id = ? WHERE group_id = ?
Plan:
SEARCH groups USING INTEGER PRIMARY KEY (rowid=?)
@@ -204,7 +204,9 @@ CREATE TABLE groups(
roster_sending_owner_gm_id INTEGER,
roster_broker_ts TEXT,
roster_blob BLOB,
group_domain_verified INTEGER, -- received
group_domain_verified INTEGER,
stored_roster_version INTEGER,
applied_complete_roster_version INTEGER, -- received
FOREIGN KEY(user_id, local_display_name)
REFERENCES display_names(user_id, local_display_name)
ON DELETE CASCADE
@@ -249,6 +251,7 @@ CREATE TABLE group_members(
relay_link BLOB,
member_pub_key BLOB,
removed_at TEXT,
roster_served_version INTEGER,
FOREIGN KEY(user_id, local_display_name)
REFERENCES display_names(user_id, local_display_name)
ON DELETE CASCADE