dandri/simplex-chat
1
0
Fork 0
mirror of https://github.com/simplex-chat/simplex-chat.git synced 2026-07-18 04:56:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

core: rework synchronous group forwarding in receive loop into asynchronous delivery tasks (#6178)

Browse Source
This commit is contained in:
spaced4ndy
2025-09-12 13:22:34 +00:00
committed by GitHub
parent f2061a7c88
commit 382241fe3e
30 changed files with 2151 additions and 328 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Simplex/Chat/Store/Delivery.hs
+362
View File
@@ -0,0 +1,362 @@
{-# LANGUAGE CPP #-}
{-# LANGUAGE DataKinds #-}
{-# LANGUAGE DuplicateRecordFields #-}
{-# LANGUAGE LambdaCase #-}
{-# LANGUAGE NamedFieldPuns #-}
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE QuasiQuotes #-}
{-# LANGUAGE TypeOperators #-}
module Simplex.Chat.Store.Delivery
( createMsgDeliveryTask,
deleteGroupDeliveryTasks,
deleteGroupDeliveryJobs,
getPendingDeliveryTaskScopes,
getNextDeliveryTask,
getNextDeliveryTasks,
updateDeliveryTaskStatus,
setDeliveryTaskErrStatus,
deleteDoneDeliveryTasks,
createMsgDeliveryJob,
getPendingDeliveryJobScopes,
getNextDeliveryJob,
updateDeliveryJobStatus,
setDeliveryJobErrStatus,
getGroupMembersByCursor,
updateDeliveryJobCursor,
deleteDoneDeliveryJobs,
)
where
import Control.Monad.Except
import Data.ByteString.Char8 (ByteString)
import Data.Either (rights)
import Data.Int (Int64)
import Data.Text (Text)
import Data.Time.Clock (UTCTime, getCurrentTime)
import Simplex.Chat.Delivery
import Simplex.Chat.Protocol hiding (Binary)
import Simplex.Chat.Store.Groups (getGroupMemberById)
import Simplex.Chat.Store.Shared
import Simplex.Chat.Types
import Simplex.Messaging.Agent.Store.AgentStore (getWorkItem, getWorkItems, maybeFirstRow)
import Simplex.Messaging.Agent.Store.DB (Binary (..), BoolInt (..))
import qualified Simplex.Messaging.Agent.Store.DB as DB
import Simplex.Messaging.Util (firstRow')
#if defined(dbPostgres)
import Database.PostgreSQL.Simple (Only (..), (:.) (..))
import Database.PostgreSQL.Simple.SqlQQ (sql)
#else
import Database.SQLite.Simple (Only (..), (:.) (..))
import Database.SQLite.Simple.QQ (sql)
#endif
type DeliveryJobScopeRow = (DeliveryWorkerScope, Maybe DeliveryJobSpecTag, Maybe BoolInt, Maybe GroupMemberId)
jobScopeRow_ :: DeliveryJobScope -> DeliveryJobScopeRow
jobScopeRow_ = \case
DJSGroup {jobSpec} -> case jobSpec of
DJDeliveryJob {includePending} -> (DWSGroup, Just DJSTDeliveryJob, Just (BI includePending), Nothing)
DJRelayRemoved -> (DWSGroup, Just DJSTRelayRemoved, Nothing, Nothing)
DJSMemberSupport {supportGMId} -> (DWSMemberSupport, Nothing, Nothing, Just supportGMId)
toJobScope_ :: DeliveryJobScopeRow -> Maybe DeliveryJobScope
toJobScope_ = \case
(DWSGroup, Just DJSTDeliveryJob, Just (BI includePending), Nothing) -> Just $ DJSGroup {jobSpec = DJDeliveryJob {includePending}}
(DWSGroup, Just DJSTRelayRemoved, Nothing, Nothing) -> Just $ DJSGroup {jobSpec = DJRelayRemoved}
(DWSMemberSupport, Nothing, Nothing, Just supportGMId) -> Just $ DJSMemberSupport {supportGMId}
_ -> Nothing
createMsgDeliveryTask :: DB.Connection -> GroupInfo -> GroupMember -> NewMessageDeliveryTask -> IO ()
createMsgDeliveryTask db gInfo sender newTask = do
currentTs <- getCurrentTime
DB.execute
db
[sql|
INSERT INTO delivery_tasks (
group_id,
worker_scope, job_scope_spec_tag, job_scope_include_pending, job_scope_support_gm_id,
sender_group_member_id, message_id, message_from_channel, task_status,
created_at, updated_at
) VALUES (?,?,?,?,?,?,?,?,?,?,?)
|]
((Only groupId) :. jobScopeRow_ jobScope :. (groupMemberId' sender, messageId, BI messageFromChannel, DTSNew, currentTs, currentTs))
where
GroupInfo {groupId} = gInfo
NewMessageDeliveryTask {messageId, jobScope, messageFromChannel} = newTask
deleteGroupDeliveryTasks :: DB.Connection -> GroupInfo -> IO ()
deleteGroupDeliveryTasks db GroupInfo {groupId} =
DB.execute db "DELETE FROM delivery_tasks WHERE group_id = ?" (Only groupId)
deleteGroupDeliveryJobs :: DB.Connection -> GroupInfo -> IO ()
deleteGroupDeliveryJobs db GroupInfo {groupId} =
DB.execute db "DELETE FROM delivery_jobs WHERE group_id = ?" (Only groupId)
getPendingDeliveryTaskScopes :: DB.Connection -> IO [DeliveryWorkerKey]
getPendingDeliveryTaskScopes db =
DB.query
db
[sql|
SELECT DISTINCT group_id, worker_scope
FROM delivery_tasks
WHERE failed = 0 AND task_status = ?
|]
(Only DTSNew)
getNextDeliveryTask :: DB.Connection -> DeliveryWorkerKey -> IO (Either StoreError (Maybe MessageDeliveryTask))
getNextDeliveryTask db deliveryKey = do
getWorkItem "delivery task" getTaskId (getMsgDeliveryTask_ db) (markDeliveryTaskFailed_ db)
where
(groupId, workerScope) = deliveryKey
getTaskId :: IO (Maybe Int64)
getTaskId =
maybeFirstRow fromOnly $
DB.query
db
[sql|
SELECT delivery_task_id
FROM delivery_tasks
WHERE group_id = ? AND worker_scope = ?
AND failed = 0 AND task_status = ?
ORDER BY delivery_task_id ASC
LIMIT 1
|]
(groupId, workerScope, DTSNew)
type MessageDeliveryTaskRow = (Only Int64) :. DeliveryJobScopeRow :. (GroupMemberId, MemberId, ContactName, UTCTime, ChatMessage 'Json, BoolInt)
getMsgDeliveryTask_ :: DB.Connection -> Int64 -> IO (Either StoreError MessageDeliveryTask)
getMsgDeliveryTask_ db taskId =
firstRow' toTask (SEDeliveryTaskNotFound taskId) $
DB.query
db
[sql|
SELECT
t.delivery_task_id,
t.worker_scope, t.job_scope_spec_tag, t.job_scope_include_pending, t.job_scope_support_gm_id,
m.group_member_id, m.member_id, p.display_name, msg.broker_ts, msg.msg_body, t.message_from_channel
FROM delivery_tasks t
JOIN messages msg ON msg.message_id = t.message_id
JOIN group_members m ON m.group_member_id = t.sender_group_member_id
JOIN contact_profiles p ON p.contact_profile_id = COALESCE(m.member_profile_id, m.contact_profile_id)
WHERE t.delivery_task_id = ?
|]
(Only taskId)
where
toTask :: MessageDeliveryTaskRow -> Either StoreError MessageDeliveryTask
toTask ((Only taskId') :. jobScopeRow :. (senderGMId, senderMemberId, senderMemberName, brokerTs, chatMessage, BI messageFromChannel)) =
case toJobScope_ jobScopeRow of
Just jobScope -> Right $ MessageDeliveryTask {taskId = taskId', jobScope, senderGMId, senderMemberId, senderMemberName, brokerTs, chatMessage, messageFromChannel}
Nothing -> Left $ SEInvalidDeliveryTask taskId'
markDeliveryTaskFailed_ :: DB.Connection -> Int64 -> IO ()
markDeliveryTaskFailed_ db taskId =
DB.execute db "UPDATE delivery_tasks SET failed = 1 where delivery_task_id = ?" (Only taskId)
-- TODO [channels fwd] possible optimization is to read and add tasks to batch iteratively to avoid reading too many tasks
-- passed MessageDeliveryTask defines the jobScope to search for
getNextDeliveryTasks :: DB.Connection -> GroupInfo -> MessageDeliveryTask -> IO (Either StoreError [Either StoreError MessageDeliveryTask])
getNextDeliveryTasks db gInfo task =
getWorkItems "message delivery task" getTaskIds (getMsgDeliveryTask_ db) (markDeliveryTaskFailed_ db)
where
GroupInfo {groupId, useRelays} = gInfo
MessageDeliveryTask {jobScope, senderGMId} = task
getTaskIds :: IO [Int64]
getTaskIds
| useRelays =
map fromOnly
<$> DB.query
db
[sql|
SELECT delivery_task_id
FROM delivery_tasks
WHERE group_id = ?
AND worker_scope = ?
AND job_scope_spec_tag IS NOT DISTINCT FROM ?
AND job_scope_include_pending IS NOT DISTINCT FROM ?
AND job_scope_support_gm_id IS NOT DISTINCT FROM ?
AND failed = 0
AND task_status = ?
ORDER BY delivery_task_id ASC
|]
((Only groupId) :. jobScopeRow_ jobScope :. (Only DTSNew))
| otherwise =
-- For fully connected groups we guarantee a singleSenderGMId for a delivery job by additionally filtering
-- on sender_group_member_id here, so that the job can then retrieve less members as recipients,
-- optimizing for this single sender (see processDeliveryJob -> getForwardIntroducedMembers, etc.).
-- We do this optimization in the job to decrease load on admins using mobile devices for clients.
map fromOnly
<$> DB.query
db
[sql|
SELECT delivery_task_id
FROM delivery_tasks
WHERE group_id = ?
AND worker_scope = ?
AND job_scope_spec_tag IS NOT DISTINCT FROM ?
AND job_scope_include_pending IS NOT DISTINCT FROM ?
AND job_scope_support_gm_id IS NOT DISTINCT FROM ?
AND sender_group_member_id = ?
AND failed = 0
AND task_status = ?
ORDER BY delivery_task_id ASC
|]
((Only groupId) :. jobScopeRow_ jobScope :. (senderGMId, DTSNew))
updateDeliveryTaskStatus :: DB.Connection -> Int64 -> DeliveryTaskStatus -> IO ()
updateDeliveryTaskStatus db taskId status = updateDeliveryTaskStatus_ db taskId status Nothing
setDeliveryTaskErrStatus :: DB.Connection -> Int64 -> Text -> IO ()
setDeliveryTaskErrStatus db taskId errReason = updateDeliveryTaskStatus_ db taskId DTSError (Just errReason)
updateDeliveryTaskStatus_ :: DB.Connection -> Int64 -> DeliveryTaskStatus -> Maybe Text -> IO ()
updateDeliveryTaskStatus_ db taskId status errReason_ = do
currentTs <- getCurrentTime
DB.execute
db
"UPDATE delivery_tasks SET task_status = ?, task_err_reason = ?, updated_at = ? WHERE delivery_task_id = ?"
(status, errReason_, currentTs, taskId)
deleteDoneDeliveryTasks :: DB.Connection -> UTCTime -> IO ()
deleteDoneDeliveryTasks db createdAtCutoff = do
DB.execute
db
[sql|
DELETE FROM delivery_tasks
WHERE created_at <= ?
AND (task_status IN (?,?) OR failed = 1)
|]
(createdAtCutoff, DTSProcessed, DTSError)
createMsgDeliveryJob :: DB.Connection -> GroupInfo -> DeliveryJobScope -> Maybe GroupMemberId -> ByteString -> IO ()
createMsgDeliveryJob db gInfo jobScope singleSenderGMId_ body = do
currentTs <- getCurrentTime
DB.execute
db
[sql|
INSERT INTO delivery_jobs (
group_id,
worker_scope, job_scope_spec_tag, job_scope_include_pending, job_scope_support_gm_id,
single_sender_group_member_id, body, job_status, created_at, updated_at
) VALUES (?,?,?,?,?,?,?,?,?,?)
|]
((Only groupId) :. jobScopeRow_ jobScope :. (singleSenderGMId_, Binary body, DJSPending, currentTs, currentTs))
where
GroupInfo {groupId} = gInfo
getPendingDeliveryJobScopes :: DB.Connection -> IO [DeliveryWorkerKey]
getPendingDeliveryJobScopes db =
DB.query
db
[sql|
SELECT DISTINCT group_id, worker_scope
FROM delivery_jobs
WHERE failed = 0 AND job_status = ?
|]
(Only DJSPending)
type MessageDeliveryJobRow = (Only Int64) :. DeliveryJobScopeRow :. (Maybe GroupMemberId, Binary ByteString, Maybe GroupMemberId)
getNextDeliveryJob :: DB.Connection -> DeliveryWorkerKey -> IO (Either StoreError (Maybe MessageDeliveryJob))
getNextDeliveryJob db deliveryKey = do
getWorkItem "delivery job" getJobId getJob markJobFailed
where
(groupId, workerScope) = deliveryKey
getJobId :: IO (Maybe Int64)
getJobId =
maybeFirstRow fromOnly $
DB.query
db
[sql|
SELECT delivery_job_id
FROM delivery_jobs
WHERE group_id = ? AND worker_scope = ?
AND failed = 0 AND job_status = ?
ORDER BY delivery_job_id ASC
LIMIT 1
|]
(groupId, workerScope, DJSPending)
getJob :: Int64 -> IO (Either StoreError MessageDeliveryJob)
getJob jobId =
firstRow' toDeliveryJob (SEDeliveryJobNotFound jobId) $
DB.query
db
[sql|
SELECT
delivery_job_id,
worker_scope, job_scope_spec_tag, job_scope_include_pending, job_scope_support_gm_id,
single_sender_group_member_id, body, cursor_group_member_id
FROM delivery_jobs
WHERE delivery_job_id = ?
|]
(Only jobId)
where
toDeliveryJob :: MessageDeliveryJobRow -> Either StoreError MessageDeliveryJob
toDeliveryJob ((Only jobId') :. jobScopeRow :. (singleSenderGMId_, Binary body, cursorGMId_)) =
case toJobScope_ jobScopeRow of
Just jobScope -> Right $ MessageDeliveryJob {jobId = jobId', jobScope, singleSenderGMId_, body, cursorGMId_}
Nothing -> Left $ SEInvalidDeliveryJob jobId'
markJobFailed :: Int64 -> IO ()
markJobFailed jobId =
DB.execute db "UPDATE delivery_jobs SET failed = 1 where delivery_job_id = ?" (Only jobId)
updateDeliveryJobStatus :: DB.Connection -> Int64 -> DeliveryJobStatus -> IO ()
updateDeliveryJobStatus db jobId status = updateDeliveryJobStatus_ db jobId status Nothing
setDeliveryJobErrStatus :: DB.Connection -> Int64 -> Text -> IO ()
setDeliveryJobErrStatus db jobId errReason = updateDeliveryJobStatus_ db jobId DJSError (Just errReason)
updateDeliveryJobStatus_ :: DB.Connection -> Int64 -> DeliveryJobStatus -> Maybe Text -> IO ()
updateDeliveryJobStatus_ db jobId status errReason_ = do
currentTs <- getCurrentTime
DB.execute
db
"UPDATE delivery_jobs SET job_status = ?, job_err_reason = ?, updated_at = ? WHERE delivery_job_id = ?"
(status, errReason_, currentTs, jobId)
-- TODO [channels fwd] possible improvement is to prioritize owners and "active" members
getGroupMembersByCursor :: DB.Connection -> VersionRangeChat -> User -> GroupInfo -> Maybe GroupMemberId -> Maybe GroupMemberId -> Int -> IO [GroupMember]
getGroupMembersByCursor db vr user GroupInfo {groupId} cursorGMId_ singleSenderGMId_ count = do
memberIds <-
map fromOnly <$> case cursorGMId_ of
Nothing ->
DB.query
db
(query <> orderLimit)
(groupId, singleSenderGMId_, GSMemIntroduced, GSMemIntroInvited, GSMemAccepted, GSMemAnnounced, GSMemConnected, GSMemComplete, count)
Just cursorGMId ->
DB.query
db
(query <> " AND group_member_id > ?" <> orderLimit)
(groupId, singleSenderGMId_, GSMemIntroduced, GSMemIntroInvited, GSMemAccepted, GSMemAnnounced, GSMemConnected, GSMemComplete, cursorGMId, count)
rights <$> mapM (runExceptT . getGroupMemberById db vr user) memberIds
where
query =
[sql|
SELECT group_member_id
FROM group_members
WHERE group_id = ?
AND group_member_id IS DISTINCT FROM ?
AND member_status IN (?,?,?,?,?,?)
|]
orderLimit = " ORDER BY group_member_id ASC LIMIT ?"
updateDeliveryJobCursor :: DB.Connection -> Int64 -> GroupMemberId -> IO ()
updateDeliveryJobCursor db jobId cursorGMId = do
currentTs <- getCurrentTime
DB.execute
db
"UPDATE delivery_jobs SET cursor_group_member_id = ?, updated_at = ? WHERE delivery_job_id = ?"
(cursorGMId, currentTs, jobId)
deleteDoneDeliveryJobs :: DB.Connection -> UTCTime -> IO ()
deleteDoneDeliveryJobs db createdAtCutoff = do
DB.execute
db
[sql|
DELETE FROM delivery_jobs
WHERE created_at <= ?
AND (job_status IN (?,?) OR failed = 1)
|]
(createdAtCutoff, DJSComplete, DJSError)
src/Simplex/Chat/Store/Groups.hs
+3 -1
View File
@@ -175,7 +175,7 @@ import Data.Text (Text)
import Data.Time.Clock (UTCTime (..), getCurrentTime)
import Data.Text.Encoding (encodeUtf8)
import Simplex.Chat.Messages
import Simplex.Chat.Protocol (MsgMention (..), groupForwardVersion)
import Simplex.Chat.Protocol hiding (Binary)
import Simplex.Chat.Store.Direct
import Simplex.Chat.Store.Shared
import Simplex.Chat.Types
@@ -359,6 +359,7 @@ createNewGroup db vr gVar user@User {userId} groupProfile incognitoProfile = Exc
pure
GroupInfo
{ groupId,
useRelays = False,
localDisplayName = ldn,
groupProfile,
localAlias = "",
@@ -432,6 +433,7 @@ createGroupInvitation db vr user@User {userId} contact@Contact {contactId, activ
pure
( GroupInfo
{ groupId,
useRelays = False,
localDisplayName,
groupProfile,
localAlias = "",
src/Simplex/Chat/Store/Messages.hs
+13 -10
View File
@@ -286,7 +286,7 @@ getLastRcvMsgInfo db connId =
RcvMsgInfo {msgId, msgDeliveryId, msgDeliveryStatus, agentMsgId, agentMsgMeta}
createNewRcvMessage :: forall e. MsgEncodingI e => DB.Connection -> ConnOrGroupId -> NewRcvMessage e -> Maybe SharedMsgId -> Maybe GroupMemberId -> Maybe GroupMemberId -> ExceptT StoreError IO RcvMessage
createNewRcvMessage db connOrGroupId NewRcvMessage {chatMsgEvent, msgBody} sharedMsgId_ authorMember forwardedByMember =
createNewRcvMessage db connOrGroupId NewRcvMessage {chatMsgEvent, msgBody, brokerTs} sharedMsgId_ authorMember forwardedByMember =
case connOrGroupId of
ConnectionId connId -> liftIO $ insertRcvMsg (Just connId) Nothing
GroupId groupId -> case sharedMsgId_ of
@@ -314,10 +314,12 @@ createNewRcvMessage db connOrGroupId NewRcvMessage {chatMsgEvent, msgBody} share
db
[sql|
INSERT INTO messages
(msg_sent, chat_msg_event, msg_body, created_at, updated_at, connection_id, group_id, shared_msg_id, author_group_member_id, forwarded_by_group_member_id)
VALUES (?,?,?,?,?,?,?,?,?,?)
(msg_sent, chat_msg_event, msg_body, broker_ts, created_at, updated_at, connection_id, group_id,
shared_msg_id, author_group_member_id, forwarded_by_group_member_id)
VALUES (?,?,?,?,?,?,?,?,?,?,?)
|]
(MDRcv, toCMEventTag chatMsgEvent, DB.Binary msgBody, currentTs, currentTs, connId_, groupId_, sharedMsgId_, authorMember, forwardedByMember)
((MDRcv, toCMEventTag chatMsgEvent, DB.Binary msgBody, brokerTs, currentTs, currentTs, connId_, groupId_)
:. (sharedMsgId_, authorMember, forwardedByMember))
msgId <- insertedRowId db
pure RcvMessage {msgId, chatMsgEvent = ACME (encoding @e) chatMsgEvent, sharedMsgId_, msgBody, authorMember, forwardedByMember}
@@ -509,17 +511,17 @@ setSupportChatTs db groupMemberId chatTs =
setSupportChatMemberAttention :: DB.Connection -> VersionRangeChat -> User -> GroupInfo -> GroupMember -> Int64 -> IO (GroupInfo, GroupMember)
setSupportChatMemberAttention db vr user g m memberAttention = do
m' <- updateGMAttention m
m' <- updateGMAttention
g' <- updateGroupMembersRequireAttention db user g m m'
pure (g', m')
where
updateGMAttention m@GroupMember {groupMemberId} = do
updateGMAttention = do
currentTs <- getCurrentTime
DB.execute
db
"UPDATE group_members SET support_chat_items_member_attention = ?, updated_at = ? WHERE group_member_id = ?"
(memberAttention, currentTs, groupMemberId' m)
m_ <- runExceptT $ getGroupMemberById db vr user groupMemberId
m_ <- runExceptT $ getGroupMemberById db vr user (groupMemberId' m)
pure $ either (const m) id m_ -- Left shouldn't happen, but types require it
createNewSndChatItem :: DB.Connection -> User -> ChatDirection c 'MDSnd -> SndMessage -> CIContent 'MDSnd -> Maybe (CIQuote c) -> Maybe CIForwardedFrom -> Maybe CITimed -> Bool -> UTCTime -> IO ChatItemId
@@ -2037,7 +2039,7 @@ setDirectChatItemsDeleteAt db User {userId} contactId itemIds currentTs = forM i
pure (chatItemId, deleteAt)
updateGroupChatItemsRead :: DB.Connection -> User -> GroupInfo -> IO ()
updateGroupChatItemsRead db User {userId} GroupInfo {groupId, membership} = do
updateGroupChatItemsRead db User {userId} GroupInfo {groupId} = do
currentTs <- getCurrentTime
DB.execute
db
@@ -2082,10 +2084,11 @@ updateSupportChatItemsRead db vr user@User {userId} g@GroupInfo {groupId, member
UPDATE group_members
SET support_chat_items_unread = 0,
support_chat_items_member_attention = 0,
support_chat_items_mentions = 0
support_chat_items_mentions = 0,
updated_at = ?
WHERE group_member_id = ?
|]
(Only groupMemberId)
(currentTs, groupMemberId)
m_ <- runExceptT $ getGroupMemberById db vr user groupMemberId
pure $ either (const m) id m_ -- Left shouldn't happen, but types require it
src/Simplex/Chat/Store/Postgres/Migrations.hs
+3 -1
View File
@@ -16,6 +16,7 @@ import Simplex.Chat.Store.Postgres.Migrations.M20250721_indexes
import Simplex.Chat.Store.Postgres.Migrations.M20250729_member_contact_requests
import Simplex.Chat.Store.Postgres.Migrations.M20250801_via_group_link_uri
import Simplex.Chat.Store.Postgres.Migrations.M20250802_chat_peer_type
import Simplex.Chat.Store.Postgres.Migrations.M20250813_delivery_tasks
import Simplex.Messaging.Agent.Store.Shared (Migration (..))
schemaMigrations :: [(String, Text, Maybe Text)]
@@ -31,7 +32,8 @@ schemaMigrations =
("20250721_indexes", m20250721_indexes, Just down_m20250721_indexes),
("20250729_member_contact_requests", m20250729_member_contact_requests, Just down_m20250729_member_contact_requests),
("20250801_via_group_link_uri", m20250801_via_group_link_uri, Just down_m20250801_via_group_link_uri),
("20250802_chat_peer_type", m20250802_chat_peer_type, Just down_m20250802_chat_peer_type)
("20250802_chat_peer_type", m20250802_chat_peer_type, Just down_m20250802_chat_peer_type),
("20250813_delivery_tasks", m20250813_delivery_tasks, Just down_m20250813_delivery_tasks)
]
-- | The list of migrations in ascending order by date
src/Simplex/Chat/Store/Postgres/Migrations/M20250813_delivery_tasks.hs
+128
View File
@@ -0,0 +1,128 @@
{-# LANGUAGE QuasiQuotes #-}
module Simplex.Chat.Store.Postgres.Migrations.M20250813_delivery_tasks where
import Data.Text (Text)
import qualified Data.Text as T
import Text.RawString.QQ (r)
m20250813_delivery_tasks :: Text
m20250813_delivery_tasks =
T.pack
[r|
CREATE TABLE delivery_tasks (
delivery_task_id BIGINT PRIMARY KEY GENERATED ALWAYS AS IDENTITY,
group_id BIGINT NOT NULL REFERENCES groups ON DELETE CASCADE,
worker_scope TEXT NOT NULL,
job_scope_spec_tag TEXT,
job_scope_include_pending SMALLINT,
job_scope_support_gm_id BIGINT REFERENCES group_members(group_member_id) ON DELETE CASCADE,
sender_group_member_id BIGINT NOT NULL REFERENCES group_members(group_member_id) ON DELETE CASCADE,
message_id BIGINT REFERENCES messages ON DELETE CASCADE,
message_from_channel SMALLINT NOT NULL DEFAULT 0,
task_status TEXT NOT NULL,
task_err_reason TEXT,
failed SMALLINT DEFAULT 0,
created_at TIMESTAMPTZ NOT NULL DEFAULT (now()),
updated_at TIMESTAMPTZ NOT NULL DEFAULT (now())
);
CREATE INDEX idx_delivery_tasks_group_id ON delivery_tasks(group_id);
CREATE INDEX idx_delivery_tasks_job_scope_support_gm_id ON delivery_tasks(job_scope_support_gm_id);
CREATE INDEX idx_delivery_tasks_sender_group_member_id ON delivery_tasks(sender_group_member_id);
CREATE INDEX idx_delivery_tasks_message_id ON delivery_tasks(message_id);
CREATE INDEX idx_delivery_tasks_next ON delivery_tasks(
group_id,
worker_scope,
failed,
task_status
);
CREATE INDEX idx_delivery_tasks_next_for_job_scope ON delivery_tasks(
group_id,
worker_scope,
job_scope_spec_tag,
job_scope_include_pending,
job_scope_support_gm_id,
failed,
task_status
);
CREATE INDEX idx_delivery_tasks_next_for_job_scope_sender ON delivery_tasks(
group_id,
worker_scope,
job_scope_spec_tag,
job_scope_include_pending,
job_scope_support_gm_id,
sender_group_member_id,
failed,
task_status
);
CREATE INDEX idx_delivery_tasks_created_at ON delivery_tasks(created_at);
CREATE TABLE delivery_jobs (
delivery_job_id BIGINT PRIMARY KEY GENERATED ALWAYS AS IDENTITY,
group_id BIGINT NOT NULL REFERENCES groups ON DELETE CASCADE,
worker_scope TEXT NOT NULL,
job_scope_spec_tag TEXT,
job_scope_include_pending SMALLINT,
job_scope_support_gm_id BIGINT REFERENCES group_members(group_member_id) ON DELETE CASCADE,
single_sender_group_member_id BIGINT REFERENCES group_members(group_member_id) ON DELETE CASCADE,
body BYTEA,
cursor_group_member_id BIGINT,
job_status TEXT NOT NULL,
job_err_reason TEXT,
failed SMALLINT DEFAULT 0,
created_at TIMESTAMPTZ NOT NULL DEFAULT (now()),
updated_at TIMESTAMPTZ NOT NULL DEFAULT (now())
);
CREATE INDEX idx_delivery_jobs_group_id ON delivery_jobs(group_id);
CREATE INDEX idx_delivery_jobs_job_scope_support_gm_id ON delivery_jobs(job_scope_support_gm_id);
CREATE INDEX idx_delivery_jobs_single_sender_group_member_id ON delivery_jobs(single_sender_group_member_id);
CREATE INDEX idx_delivery_jobs_next ON delivery_jobs(
group_id,
worker_scope,
failed,
job_status
);
CREATE INDEX idx_delivery_jobs_created_at ON delivery_jobs(created_at);
ALTER TABLE messages ADD COLUMN broker_ts TIMESTAMPTZ;
|]
down_m20250813_delivery_tasks :: Text
down_m20250813_delivery_tasks =
T.pack
[r|
ALTER TABLE messages DROP COLUMN broker_ts;
DROP INDEX idx_delivery_jobs_group_id;
DROP INDEX idx_delivery_jobs_job_scope_support_gm_id;
DROP INDEX idx_delivery_jobs_single_sender_group_member_id;
DROP INDEX idx_delivery_jobs_next;
DROP INDEX idx_delivery_jobs_created_at;
DROP TABLE delivery_jobs;
DROP INDEX idx_delivery_tasks_group_id;
DROP INDEX idx_delivery_tasks_job_scope_support_gm_id;
DROP INDEX idx_delivery_tasks_sender_group_member_id;
DROP INDEX idx_delivery_tasks_message_id;
DROP INDEX idx_delivery_tasks_next;
DROP INDEX idx_delivery_tasks_next_for_job_scope;
DROP INDEX idx_delivery_tasks_next_for_job_scope_sender;
DROP INDEX idx_delivery_tasks_created_at;
DROP TABLE delivery_tasks;
|]
src/Simplex/Chat/Store/Postgres/Migrations/chat_schema.sql
+159 -1
View File
@@ -432,6 +432,66 @@ ALTER TABLE test_chat_schema.contacts ALTER COLUMN contact_id ADD GENERATED ALWA
CREATE TABLE test_chat_schema.delivery_jobs (
delivery_job_id bigint NOT NULL,
group_id bigint NOT NULL,
worker_scope text NOT NULL,
job_scope_spec_tag text,
job_scope_include_pending smallint,
job_scope_support_gm_id bigint,
single_sender_group_member_id bigint,
body bytea,
cursor_group_member_id bigint,
job_status text NOT NULL,
job_err_reason text,
failed smallint DEFAULT 0,
created_at timestamp with time zone DEFAULT now() NOT NULL,
updated_at timestamp with time zone DEFAULT now() NOT NULL
);
ALTER TABLE test_chat_schema.delivery_jobs ALTER COLUMN delivery_job_id ADD GENERATED ALWAYS AS IDENTITY (
SEQUENCE NAME test_chat_schema.delivery_jobs_delivery_job_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1
);
CREATE TABLE test_chat_schema.delivery_tasks (
delivery_task_id bigint NOT NULL,
group_id bigint NOT NULL,
worker_scope text NOT NULL,
job_scope_spec_tag text,
job_scope_include_pending smallint,
job_scope_support_gm_id bigint,
sender_group_member_id bigint NOT NULL,
message_id bigint,
message_from_channel smallint DEFAULT 0 NOT NULL,
task_status text NOT NULL,
task_err_reason text,
failed smallint DEFAULT 0,
created_at timestamp with time zone DEFAULT now() NOT NULL,
updated_at timestamp with time zone DEFAULT now() NOT NULL
);
ALTER TABLE test_chat_schema.delivery_tasks ALTER COLUMN delivery_task_id ADD GENERATED ALWAYS AS IDENTITY (
SEQUENCE NAME test_chat_schema.delivery_tasks_delivery_task_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1
);
CREATE TABLE test_chat_schema.display_names (
user_id bigint NOT NULL,
local_display_name text NOT NULL,
@@ -707,7 +767,8 @@ CREATE TABLE test_chat_schema.messages (
shared_msg_id bytea,
shared_msg_id_user smallint,
author_group_member_id bigint,
forwarded_by_group_member_id bigint
forwarded_by_group_member_id bigint,
broker_ts timestamp with time zone
);
@@ -1274,6 +1335,16 @@ ALTER TABLE ONLY test_chat_schema.contacts
ALTER TABLE ONLY test_chat_schema.delivery_jobs
ADD CONSTRAINT delivery_jobs_pkey PRIMARY KEY (delivery_job_id);
ALTER TABLE ONLY test_chat_schema.delivery_tasks
ADD CONSTRAINT delivery_tasks_pkey PRIMARY KEY (delivery_task_id);
ALTER TABLE ONLY test_chat_schema.display_names
ADD CONSTRAINT display_names_pkey PRIMARY KEY (user_id, local_display_name);
@@ -1837,6 +1908,58 @@ CREATE INDEX idx_contacts_xcontact_id ON test_chat_schema.contacts USING btree (
CREATE INDEX idx_delivery_jobs_created_at ON test_chat_schema.delivery_jobs USING btree (created_at);
CREATE INDEX idx_delivery_jobs_group_id ON test_chat_schema.delivery_jobs USING btree (group_id);
CREATE INDEX idx_delivery_jobs_job_scope_support_gm_id ON test_chat_schema.delivery_jobs USING btree (job_scope_support_gm_id);
CREATE INDEX idx_delivery_jobs_next ON test_chat_schema.delivery_jobs USING btree (group_id, worker_scope, failed, job_status);
CREATE INDEX idx_delivery_jobs_single_sender_group_member_id ON test_chat_schema.delivery_jobs USING btree (single_sender_group_member_id);
CREATE INDEX idx_delivery_tasks_created_at ON test_chat_schema.delivery_tasks USING btree (created_at);
CREATE INDEX idx_delivery_tasks_group_id ON test_chat_schema.delivery_tasks USING btree (group_id);
CREATE INDEX idx_delivery_tasks_job_scope_support_gm_id ON test_chat_schema.delivery_tasks USING btree (job_scope_support_gm_id);
CREATE INDEX idx_delivery_tasks_message_id ON test_chat_schema.delivery_tasks USING btree (message_id);
CREATE INDEX idx_delivery_tasks_next ON test_chat_schema.delivery_tasks USING btree (group_id, worker_scope, failed, task_status);
CREATE INDEX idx_delivery_tasks_next_for_job_scope ON test_chat_schema.delivery_tasks USING btree (group_id, worker_scope, job_scope_spec_tag, job_scope_include_pending, job_scope_support_gm_id, failed, task_status);
CREATE INDEX idx_delivery_tasks_next_for_job_scope_sender ON test_chat_schema.delivery_tasks USING btree (group_id, worker_scope, job_scope_spec_tag, job_scope_include_pending, job_scope_support_gm_id, sender_group_member_id, failed, task_status);
CREATE INDEX idx_delivery_tasks_sender_group_member_id ON test_chat_schema.delivery_tasks USING btree (sender_group_member_id);
CREATE INDEX idx_extra_xftp_file_descriptions_file_id ON test_chat_schema.extra_xftp_file_descriptions USING btree (file_id);
@@ -2390,6 +2513,41 @@ ALTER TABLE ONLY test_chat_schema.contacts
ALTER TABLE ONLY test_chat_schema.delivery_jobs
ADD CONSTRAINT delivery_jobs_group_id_fkey FOREIGN KEY (group_id) REFERENCES test_chat_schema.groups(group_id) ON DELETE CASCADE;
ALTER TABLE ONLY test_chat_schema.delivery_jobs
ADD CONSTRAINT delivery_jobs_job_scope_support_gm_id_fkey FOREIGN KEY (job_scope_support_gm_id) REFERENCES test_chat_schema.group_members(group_member_id) ON DELETE CASCADE;
ALTER TABLE ONLY test_chat_schema.delivery_jobs
ADD CONSTRAINT delivery_jobs_single_sender_group_member_id_fkey FOREIGN KEY (single_sender_group_member_id) REFERENCES test_chat_schema.group_members(group_member_id) ON DELETE CASCADE;
ALTER TABLE ONLY test_chat_schema.delivery_tasks
ADD CONSTRAINT delivery_tasks_group_id_fkey FOREIGN KEY (group_id) REFERENCES test_chat_schema.groups(group_id) ON DELETE CASCADE;
ALTER TABLE ONLY test_chat_schema.delivery_tasks
ADD CONSTRAINT delivery_tasks_job_scope_support_gm_id_fkey FOREIGN KEY (job_scope_support_gm_id) REFERENCES test_chat_schema.group_members(group_member_id) ON DELETE CASCADE;
ALTER TABLE ONLY test_chat_schema.delivery_tasks
ADD CONSTRAINT delivery_tasks_message_id_fkey FOREIGN KEY (message_id) REFERENCES test_chat_schema.messages(message_id) ON DELETE CASCADE;
ALTER TABLE ONLY test_chat_schema.delivery_tasks
ADD CONSTRAINT delivery_tasks_sender_group_member_id_fkey FOREIGN KEY (sender_group_member_id) REFERENCES test_chat_schema.group_members(group_member_id) ON DELETE CASCADE;
ALTER TABLE ONLY test_chat_schema.display_names
ADD CONSTRAINT display_names_user_id_fkey FOREIGN KEY (user_id) REFERENCES test_chat_schema.users(user_id) ON DELETE CASCADE;
src/Simplex/Chat/Store/SQLite/Migrations.hs
+3 -1
View File
@@ -139,6 +139,7 @@ import Simplex.Chat.Store.SQLite.Migrations.M20250721_indexes
import Simplex.Chat.Store.SQLite.Migrations.M20250729_member_contact_requests
import Simplex.Chat.Store.SQLite.Migrations.M20250801_via_group_link_uri
import Simplex.Chat.Store.SQLite.Migrations.M20250802_chat_peer_type
import Simplex.Chat.Store.SQLite.Migrations.M20250813_delivery_tasks
import Simplex.Messaging.Agent.Store.Shared (Migration (..))
schemaMigrations :: [(String, Query, Maybe Query)]
@@ -277,7 +278,8 @@ schemaMigrations =
("20250721_indexes", m20250721_indexes, Just down_m20250721_indexes),
("20250729_member_contact_requests", m20250729_member_contact_requests, Just down_m20250729_member_contact_requests),
("20250801_via_group_link_uri", m20250801_via_group_link_uri, Just down_m20250801_via_group_link_uri),
("20250802_chat_peer_type", m20250802_chat_peer_type, Just down_m20250802_chat_peer_type)
("20250802_chat_peer_type", m20250802_chat_peer_type, Just down_m20250802_chat_peer_type),
("20250813_delivery_tasks", m20250813_delivery_tasks, Just down_m20250813_delivery_tasks)
]
-- | The list of migrations in ascending order by date
src/Simplex/Chat/Store/SQLite/Migrations/M20250813_delivery_tasks.hs
+172
View File
@@ -0,0 +1,172 @@
{-# LANGUAGE QuasiQuotes #-}
module Simplex.Chat.Store.SQLite.Migrations.M20250813_delivery_tasks where
import Database.SQLite.Simple (Query)
import Database.SQLite.Simple.QQ (sql)
-- TODO [channels fwd] add later in new migration for MemberProfileUpdate delivery jobs:
-- TODO - ALTER TABLE group_members ADD COLUMN last_profile_delivery_ts TEXT;
-- TODO - ALTER TABLE group_members ADD COLUMN join_ts TEXT;
-- How columns correspond to types:
-- both tables:
-- - <group_id, worker_scope> <-> DeliveryWorkerKey = (GroupId, DeliveryWorkerScope),
-- - job_scope_spec_tag <-> Maybe DeliveryJobSpecTag (for DJSGroup job scope),
-- - job_scope_include_pending <-> Maybe Bool (for DJDeliveryJob job spec),
-- - job_scope_support_gm_id <-> Maybe GroupMemberId (for DJSMemberSupport job scope),
-- - failed <-> Bool (for internal worker use, to mark failed work items).
-- delivery_tasks table:
-- - sender_group_member_id <-> GroupMemberId (sender of the original message that created task),
-- - message_id <-> MessageId (reference to the original message that created task),
-- - message_from_channel <-> Maybe MessageFromChannel (for MessageDeliveryTask),
-- - task_status <-> DeliveryTaskStatus,
-- - task_err_reason <-> Maybe Text (set when task status is DTSError, not encoded in status to allow filtering by DTSError in queries).
-- delivery_jobs table:
-- - single_sender_group_member_id <-> Maybe GroupMemberId (set when all messages in job's delivery body are from the same sender),
-- - body <-> ByteString (JSON encoded batch of messages or single message packed with forwarding metadata (XGrpMsgForward)),
-- - cursor_group_member_id <-> Maybe GroupMemberId (for tracking progress of job processing buckets of recipient members),
-- - job_status <-> DeliveryJobStatus,
-- - job_err_reason <-> Maybe Text (set when job status is DJSError, not encoded in status to allow filtering by DJSError in queries).
--
-- Pair of columns <group_id, worker_scope> defines the scope of work for a worker.
--
-- Both tasks and jobs are defined by the same range of worker scopes, with a delivery task worker
-- converting tasks into jobs for a delivery job worker of the same scope.
-- Each group can have up to 1 task worker and 1 job worker for each worker scope.
-- See DeliveryWorkerScope.
--
-- Columns job_scope_spec_tag, job_scope_include_pending, job_scope_support_gm_id narrow down delivery scope for a job.
-- See DeliveryJobScope.
--
-- In some cases multiple tasks of the same job scope may be converted into a single job.
-- For example, messages referenced in DJDeliveryJob tasks of the same scope/spec can be batched for a single delivery.
-- Some tasks, for example of DJRelayRemoved specialization, are converted one-to-one into jobs.
--
-- Delivery scopes can be expanded to create more specialized jobs.
-- For example DWSMemberProfileUpdate workers are planned to deliver senders' profiles to
-- all members satisfying criteria: sender last_profile_delivery_ts < recipient join_ts.
-- Tasks for these jobs are planned to be created based on sender and group state, rather than per message,
-- so member profile updates will be delivered separately from message deliveries.
-- See rfc doc for more details.
m20250813_delivery_tasks :: Query
m20250813_delivery_tasks =
[sql|
CREATE TABLE delivery_tasks (
delivery_task_id INTEGER PRIMARY KEY,
group_id INTEGER NOT NULL REFERENCES groups ON DELETE CASCADE,
worker_scope TEXT NOT NULL,
job_scope_spec_tag TEXT,
job_scope_include_pending INTEGER,
job_scope_support_gm_id INTEGER REFERENCES group_members(group_member_id) ON DELETE CASCADE,
sender_group_member_id INTEGER NOT NULL REFERENCES group_members(group_member_id) ON DELETE CASCADE,
message_id INTEGER REFERENCES messages ON DELETE CASCADE,
message_from_channel INTEGER NOT NULL DEFAULT 0,
task_status TEXT NOT NULL,
task_err_reason TEXT,
failed INTEGER DEFAULT 0,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
);
CREATE INDEX idx_delivery_tasks_group_id ON delivery_tasks(group_id);
CREATE INDEX idx_delivery_tasks_job_scope_support_gm_id ON delivery_tasks(job_scope_support_gm_id);
CREATE INDEX idx_delivery_tasks_sender_group_member_id ON delivery_tasks(sender_group_member_id);
CREATE INDEX idx_delivery_tasks_message_id ON delivery_tasks(message_id);
CREATE INDEX idx_delivery_tasks_next ON delivery_tasks(
group_id,
worker_scope,
failed,
task_status
);
CREATE INDEX idx_delivery_tasks_next_for_job_scope ON delivery_tasks(
group_id,
worker_scope,
job_scope_spec_tag,
job_scope_include_pending,
job_scope_support_gm_id,
failed,
task_status
);
CREATE INDEX idx_delivery_tasks_next_for_job_scope_sender ON delivery_tasks(
group_id,
worker_scope,
job_scope_spec_tag,
job_scope_include_pending,
job_scope_support_gm_id,
sender_group_member_id,
failed,
task_status
);
CREATE INDEX idx_delivery_tasks_created_at ON delivery_tasks(created_at);
CREATE TABLE delivery_jobs (
delivery_job_id INTEGER PRIMARY KEY,
group_id INTEGER NOT NULL REFERENCES groups ON DELETE CASCADE,
worker_scope TEXT NOT NULL,
job_scope_spec_tag TEXT,
job_scope_include_pending INTEGER,
job_scope_support_gm_id INTEGER REFERENCES group_members(group_member_id) ON DELETE CASCADE,
single_sender_group_member_id INTEGER REFERENCES group_members(group_member_id) ON DELETE CASCADE,
body BLOB,
cursor_group_member_id INTEGER,
job_status TEXT NOT NULL,
job_err_reason TEXT,
failed INTEGER DEFAULT 0,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
);
CREATE INDEX idx_delivery_jobs_group_id ON delivery_jobs(group_id);
CREATE INDEX idx_delivery_jobs_job_scope_support_gm_id ON delivery_jobs(job_scope_support_gm_id);
CREATE INDEX idx_delivery_jobs_single_sender_group_member_id ON delivery_jobs(single_sender_group_member_id);
CREATE INDEX idx_delivery_jobs_next ON delivery_jobs(
group_id,
worker_scope,
failed,
job_status
);
CREATE INDEX idx_delivery_jobs_created_at ON delivery_jobs(created_at);
ALTER TABLE messages ADD COLUMN broker_ts TEXT;
|]
down_m20250813_delivery_tasks :: Query
down_m20250813_delivery_tasks =
[sql|
ALTER TABLE messages DROP COLUMN broker_ts;
DROP INDEX idx_delivery_jobs_group_id;
DROP INDEX idx_delivery_jobs_job_scope_support_gm_id;
DROP INDEX idx_delivery_jobs_single_sender_group_member_id;
DROP INDEX idx_delivery_jobs_next;
DROP INDEX idx_delivery_jobs_created_at;
DROP TABLE delivery_jobs;
DROP INDEX idx_delivery_tasks_group_id;
DROP INDEX idx_delivery_tasks_job_scope_support_gm_id;
DROP INDEX idx_delivery_tasks_sender_group_member_id;
DROP INDEX idx_delivery_tasks_message_id;
DROP INDEX idx_delivery_tasks_next;
DROP INDEX idx_delivery_tasks_next_for_job_scope;
DROP INDEX idx_delivery_tasks_next_for_job_scope_sender;
DROP INDEX idx_delivery_tasks_created_at;
DROP TABLE delivery_tasks;
|]
src/Simplex/Chat/Store/SQLite/Migrations/chat_query_plans.txt
+153 -3
View File
@@ -95,6 +95,22 @@ SEARCH gp USING INTEGER PRIMARY KEY (rowid=?)
SEARCH mu USING INDEX idx_group_members_contact_id (contact_id=?)
SEARCH pu USING INTEGER PRIMARY KEY (rowid=?)
Query:
SELECT delivery_task_id
FROM delivery_tasks
WHERE group_id = ?
AND worker_scope = ?
AND job_scope_spec_tag IS NOT DISTINCT FROM ?
AND job_scope_include_pending IS NOT DISTINCT FROM ?
AND job_scope_support_gm_id IS NOT DISTINCT FROM ?
AND sender_group_member_id = ?
AND failed = 0
AND task_status = ?
ORDER BY delivery_task_id ASC
Plan:
SEARCH delivery_tasks USING COVERING INDEX idx_delivery_tasks_next_for_job_scope_sender (group_id=? AND worker_scope=? AND job_scope_spec_tag=? AND job_scope_include_pending=? AND job_scope_support_gm_id=? AND sender_group_member_id=? AND failed=? AND task_status=?)
Query:
UPDATE contact_profiles
SET user_id = ?, updated_at = ?
@@ -417,6 +433,17 @@ Plan:
SEARCH cr USING INDEX idx_contact_requests_xcontact_id (user_id=? AND xcontact_id=?)
SEARCH p USING INTEGER PRIMARY KEY (rowid=?)
Query:
SELECT
delivery_job_id,
worker_scope, job_scope_spec_tag, job_scope_include_pending, job_scope_support_gm_id,
single_sender_group_member_id, body, cursor_group_member_id
FROM delivery_jobs
WHERE delivery_job_id = ?
Plan:
SEARCH delivery_jobs USING INTEGER PRIMARY KEY (rowid=?)
Query:
SELECT COUNT(1)
FROM (
@@ -648,6 +675,28 @@ Plan:
SEARCH m USING INTEGER PRIMARY KEY (rowid=?)
SEARCH ct USING INTEGER PRIMARY KEY (rowid=?)
Query:
SELECT delivery_job_id
FROM delivery_jobs
WHERE group_id = ? AND worker_scope = ?
AND failed = 0 AND job_status = ?
ORDER BY delivery_job_id ASC
LIMIT 1
Plan:
SEARCH delivery_jobs USING COVERING INDEX idx_delivery_jobs_next (group_id=? AND worker_scope=? AND failed=? AND job_status=?)
Query:
SELECT delivery_task_id
FROM delivery_tasks
WHERE group_id = ? AND worker_scope = ?
AND failed = 0 AND task_status = ?
ORDER BY delivery_task_id ASC
LIMIT 1
Plan:
SEARCH delivery_tasks USING COVERING INDEX idx_delivery_tasks_next (group_id=? AND worker_scope=? AND failed=? AND task_status=?)
Query:
SELECT gp.display_name, gp.full_name, gp.short_descr, gp.description, gp.image, gp.preferences, gp.member_admission
FROM group_profiles gp
@@ -822,8 +871,9 @@ Plan:
Query:
INSERT INTO messages
(msg_sent, chat_msg_event, msg_body, created_at, updated_at, connection_id, group_id, shared_msg_id, author_group_member_id, forwarded_by_group_member_id)
VALUES (?,?,?,?,?,?,?,?,?,?)
(msg_sent, chat_msg_event, msg_body, broker_ts, created_at, updated_at, connection_id, group_id,
shared_msg_id, author_group_member_id, forwarded_by_group_member_id)
VALUES (?,?,?,?,?,?,?,?,?,?,?)
Plan:
@@ -1502,7 +1552,8 @@ Query:
UPDATE group_members
SET support_chat_items_unread = 0,
support_chat_items_member_attention = 0,
support_chat_items_mentions = 0
support_chat_items_mentions = 0,
updated_at = ?
WHERE group_member_id = ?
Plan:
@@ -2996,6 +3047,23 @@ CORRELATED SCALAR SUBQUERY 1
SEARCH ci USING COVERING INDEX idx_chat_items_notes_created_at (user_id=? AND note_folder_id=?)
USE TEMP B-TREE FOR ORDER BY
Query:
SELECT
t.delivery_task_id,
t.worker_scope, t.job_scope_spec_tag, t.job_scope_include_pending, t.job_scope_support_gm_id,
m.group_member_id, m.member_id, p.display_name, msg.broker_ts, msg.msg_body, t.message_from_channel
FROM delivery_tasks t
JOIN messages msg ON msg.message_id = t.message_id
JOIN group_members m ON m.group_member_id = t.sender_group_member_id
JOIN contact_profiles p ON p.contact_profile_id = COALESCE(m.member_profile_id, m.contact_profile_id)
WHERE t.delivery_task_id = ?
Plan:
SEARCH t USING INTEGER PRIMARY KEY (rowid=?)
SEARCH msg USING INTEGER PRIMARY KEY (rowid=?)
SEARCH m USING INTEGER PRIMARY KEY (rowid=?)
SEARCH p USING INTEGER PRIMARY KEY (rowid=?)
Query:
SELECT COUNT(1)
FROM chat_items
@@ -3990,6 +4058,22 @@ SEARCH group_members USING COVERING INDEX idx_group_members_member_profile_id (m
SEARCH group_members USING COVERING INDEX idx_group_members_contact_profile_id (contact_profile_id=?)
SEARCH contacts USING COVERING INDEX idx_contacts_contact_profile_id (contact_profile_id=?)
Query:
DELETE FROM delivery_jobs
WHERE created_at <= ?
AND (job_status IN (?,?) OR failed = 1)
Plan:
SEARCH delivery_jobs USING INDEX idx_delivery_jobs_created_at (created_at<?)
Query:
DELETE FROM delivery_tasks
WHERE created_at <= ?
AND (task_status IN (?,?) OR failed = 1)
Plan:
SEARCH delivery_tasks USING INDEX idx_delivery_tasks_created_at (created_at<?)
Query:
DELETE FROM display_names
WHERE
@@ -4241,6 +4325,25 @@ Query:
Plan:
SEARCH contact_requests USING COVERING INDEX idx_contact_requests_contact_profile_id (contact_profile_id=?)
Query:
INSERT INTO delivery_jobs (
group_id,
worker_scope, job_scope_spec_tag, job_scope_include_pending, job_scope_support_gm_id,
single_sender_group_member_id, body, job_status, created_at, updated_at
) VALUES (?,?,?,?,?,?,?,?,?,?)
Plan:
Query:
INSERT INTO delivery_tasks (
group_id,
worker_scope, job_scope_spec_tag, job_scope_include_pending, job_scope_support_gm_id,
sender_group_member_id, message_id, message_from_channel, task_status,
created_at, updated_at
) VALUES (?,?,?,?,?,?,?,?,?,?,?)
Plan:
Query:
INSERT INTO files
( user_id, note_folder_id,
@@ -4287,6 +4390,22 @@ Query:
Plan:
SEARCH group_members USING INDEX sqlite_autoindex_group_members_1 (group_id=?)
Query:
SELECT DISTINCT group_id, worker_scope
FROM delivery_jobs
WHERE failed = 0 AND job_status = ?
Plan:
SCAN delivery_jobs USING COVERING INDEX idx_delivery_jobs_next
Query:
SELECT DISTINCT group_id, worker_scope
FROM delivery_tasks
WHERE failed = 0 AND task_status = ?
Plan:
SCAN delivery_tasks USING COVERING INDEX idx_delivery_tasks_next
Query:
SELECT chat_item_id, timed_ttl
FROM chat_items
@@ -4764,6 +4883,7 @@ Plan:
SEARCH messages USING INTEGER PRIMARY KEY (rowid=?)
LIST SUBQUERY 1
SEARCH chat_item_messages USING COVERING INDEX sqlite_autoindex_chat_item_messages_2 (chat_item_id=?)
SEARCH delivery_tasks USING COVERING INDEX idx_delivery_tasks_message_id (message_id=?)
SEARCH msg_deliveries USING COVERING INDEX idx_msg_deliveries_message_id (message_id=?)
SEARCH chat_item_moderations USING COVERING INDEX idx_chat_item_moderations_created_by_msg_id (created_by_msg_id=?)
SEARCH chat_item_reactions USING COVERING INDEX idx_chat_item_reactions_created_by_msg_id (created_by_msg_id=?)
@@ -5534,6 +5654,14 @@ SEARCH group_members USING COVERING INDEX idx_group_members_contact_id (contact_
SEARCH group_members USING COVERING INDEX idx_group_members_invited_by (invited_by=?)
SEARCH users USING COVERING INDEX sqlite_autoindex_users_1 (contact_id=?)
Query: DELETE FROM delivery_jobs WHERE group_id = ?
Plan:
SEARCH delivery_jobs USING COVERING INDEX idx_delivery_jobs_group_id (group_id=?)
Query: DELETE FROM delivery_tasks WHERE group_id = ?
Plan:
SEARCH delivery_tasks USING COVERING INDEX idx_delivery_tasks_group_id (group_id=?)
Query: DELETE FROM display_names WHERE user_id = ? AND local_display_name = ?
Plan:
SEARCH display_names USING PRIMARY KEY (user_id=? AND local_display_name=?)
@@ -5554,6 +5682,10 @@ SEARCH files USING COVERING INDEX idx_files_redirect_file_id (redirect_file_id=?
Query: DELETE FROM group_members WHERE user_id = ? AND group_id = ?
Plan:
SEARCH group_members USING COVERING INDEX idx_group_members_group_id (user_id=? AND group_id=?)
SEARCH delivery_jobs USING COVERING INDEX idx_delivery_jobs_single_sender_group_member_id (single_sender_group_member_id=?)
SEARCH delivery_jobs USING COVERING INDEX idx_delivery_jobs_job_scope_support_gm_id (job_scope_support_gm_id=?)
SEARCH delivery_tasks USING COVERING INDEX idx_delivery_tasks_sender_group_member_id (sender_group_member_id=?)
SEARCH delivery_tasks USING COVERING INDEX idx_delivery_tasks_job_scope_support_gm_id (job_scope_support_gm_id=?)
SEARCH received_probes USING COVERING INDEX idx_received_probes_group_member_id (group_member_id=?)
SEARCH sent_probe_hashes USING COVERING INDEX idx_sent_probe_hashes_group_member_id (group_member_id=?)
SEARCH sent_probes USING COVERING INDEX idx_sent_probes_group_member_id (group_member_id=?)
@@ -5579,6 +5711,10 @@ SEARCH contacts USING COVERING INDEX idx_contacts_contact_group_member_id (conta
Query: DELETE FROM group_members WHERE user_id = ? AND group_member_id = ?
Plan:
SEARCH group_members USING INTEGER PRIMARY KEY (rowid=?)
SEARCH delivery_jobs USING COVERING INDEX idx_delivery_jobs_single_sender_group_member_id (single_sender_group_member_id=?)
SEARCH delivery_jobs USING COVERING INDEX idx_delivery_jobs_job_scope_support_gm_id (job_scope_support_gm_id=?)
SEARCH delivery_tasks USING COVERING INDEX idx_delivery_tasks_sender_group_member_id (sender_group_member_id=?)
SEARCH delivery_tasks USING COVERING INDEX idx_delivery_tasks_job_scope_support_gm_id (job_scope_support_gm_id=?)
SEARCH received_probes USING COVERING INDEX idx_received_probes_group_member_id (group_member_id=?)
SEARCH sent_probe_hashes USING COVERING INDEX idx_sent_probe_hashes_group_member_id (group_member_id=?)
SEARCH sent_probes USING COVERING INDEX idx_sent_probes_group_member_id (group_member_id=?)
@@ -5604,6 +5740,8 @@ SEARCH contacts USING COVERING INDEX idx_contacts_contact_group_member_id (conta
Query: DELETE FROM groups WHERE user_id = ? AND group_id = ?
Plan:
SEARCH groups USING INTEGER PRIMARY KEY (rowid=?)
SEARCH delivery_jobs USING COVERING INDEX idx_delivery_jobs_group_id (group_id=?)
SEARCH delivery_tasks USING COVERING INDEX idx_delivery_tasks_group_id (group_id=?)
SEARCH chat_item_mentions USING COVERING INDEX idx_chat_item_mentions_group_id (group_id=?)
SEARCH chat_tags_chats USING COVERING INDEX idx_chat_tags_chats_chat_tag_id_group_id (group_id=?)
SEARCH chat_item_moderations USING COVERING INDEX idx_chat_item_moderations_group_id (group_id=?)
@@ -5621,6 +5759,7 @@ SEARCH contacts USING COVERING INDEX idx_contacts_via_group (via_group=?)
Query: DELETE FROM messages WHERE connection_id = ?
Plan:
SEARCH messages USING COVERING INDEX idx_messages_connection_id (connection_id=?)
SEARCH delivery_tasks USING COVERING INDEX idx_delivery_tasks_message_id (message_id=?)
SEARCH msg_deliveries USING COVERING INDEX idx_msg_deliveries_message_id (message_id=?)
SEARCH chat_item_moderations USING COVERING INDEX idx_chat_item_moderations_created_by_msg_id (created_by_msg_id=?)
SEARCH chat_item_reactions USING COVERING INDEX idx_chat_item_reactions_created_by_msg_id (created_by_msg_id=?)
@@ -5631,6 +5770,7 @@ SEARCH pending_group_messages USING COVERING INDEX idx_pending_group_messages_me
Query: DELETE FROM messages WHERE connection_id = ? AND created_at <= ?
Plan:
SEARCH messages USING INDEX idx_messages_connection_id (connection_id=?)
SEARCH delivery_tasks USING COVERING INDEX idx_delivery_tasks_message_id (message_id=?)
SEARCH msg_deliveries USING COVERING INDEX idx_msg_deliveries_message_id (message_id=?)
SEARCH chat_item_moderations USING COVERING INDEX idx_chat_item_moderations_created_by_msg_id (created_by_msg_id=?)
SEARCH chat_item_reactions USING COVERING INDEX idx_chat_item_reactions_created_by_msg_id (created_by_msg_id=?)
@@ -5641,6 +5781,7 @@ SEARCH pending_group_messages USING COVERING INDEX idx_pending_group_messages_me
Query: DELETE FROM messages WHERE created_at <= ?
Plan:
SEARCH messages USING COVERING INDEX idx_messages_created_at (created_at<?)
SEARCH delivery_tasks USING COVERING INDEX idx_delivery_tasks_message_id (message_id=?)
SEARCH msg_deliveries USING COVERING INDEX idx_msg_deliveries_message_id (message_id=?)
SEARCH chat_item_moderations USING COVERING INDEX idx_chat_item_moderations_created_by_msg_id (created_by_msg_id=?)
SEARCH chat_item_reactions USING COVERING INDEX idx_chat_item_reactions_created_by_msg_id (created_by_msg_id=?)
@@ -5651,6 +5792,7 @@ SEARCH pending_group_messages USING COVERING INDEX idx_pending_group_messages_me
Query: DELETE FROM messages WHERE group_id = ?
Plan:
SEARCH messages USING COVERING INDEX idx_messages_group_id (group_id=?)
SEARCH delivery_tasks USING COVERING INDEX idx_delivery_tasks_message_id (message_id=?)
SEARCH msg_deliveries USING COVERING INDEX idx_msg_deliveries_message_id (message_id=?)
SEARCH chat_item_moderations USING COVERING INDEX idx_chat_item_moderations_created_by_msg_id (created_by_msg_id=?)
SEARCH chat_item_reactions USING COVERING INDEX idx_chat_item_reactions_created_by_msg_id (created_by_msg_id=?)
@@ -6234,6 +6376,14 @@ Query: UPDATE contacts SET xcontact_id = ? WHERE contact_id = ?
Plan:
SEARCH contacts USING INTEGER PRIMARY KEY (rowid=?)
Query: UPDATE delivery_jobs SET job_status = ?, job_err_reason = ?, updated_at = ? WHERE delivery_job_id = ?
Plan:
SEARCH delivery_jobs USING INTEGER PRIMARY KEY (rowid=?)
Query: UPDATE delivery_tasks SET task_status = ?, task_err_reason = ?, updated_at = ? WHERE delivery_task_id = ?
Plan:
SEARCH delivery_tasks USING INTEGER PRIMARY KEY (rowid=?)
Query: UPDATE files SET agent_snd_file_deleted = 1, updated_at = ? WHERE user_id = ? AND file_id = ?
Plan:
SEARCH files USING INTEGER PRIMARY KEY (rowid=?)
src/Simplex/Chat/Store/SQLite/Migrations/chat_schema.sql
+82 -1
View File
@@ -395,7 +395,8 @@ CREATE TABLE messages(
shared_msg_id BLOB,
shared_msg_id_user INTEGER,
author_group_member_id INTEGER REFERENCES group_members ON DELETE SET NULL,
forwarded_by_group_member_id INTEGER REFERENCES group_members ON DELETE SET NULL
forwarded_by_group_member_id INTEGER REFERENCES group_members ON DELETE SET NULL,
broker_ts TEXT
);
CREATE TABLE pending_group_messages(
pending_group_message_id INTEGER PRIMARY KEY,
@@ -691,6 +692,38 @@ CREATE TABLE chat_item_mentions(
chat_item_id INTEGER NOT NULL REFERENCES chat_items ON DELETE CASCADE,
display_name TEXT NOT NULL
);
CREATE TABLE delivery_tasks(
delivery_task_id INTEGER PRIMARY KEY,
group_id INTEGER NOT NULL REFERENCES groups ON DELETE CASCADE,
worker_scope TEXT NOT NULL,
job_scope_spec_tag TEXT,
job_scope_include_pending INTEGER,
job_scope_support_gm_id INTEGER REFERENCES group_members(group_member_id) ON DELETE CASCADE,
sender_group_member_id INTEGER NOT NULL REFERENCES group_members(group_member_id) ON DELETE CASCADE,
message_id INTEGER REFERENCES messages ON DELETE CASCADE,
message_from_channel INTEGER NOT NULL DEFAULT 0,
task_status TEXT NOT NULL,
task_err_reason TEXT,
failed INTEGER DEFAULT 0,
created_at TEXT NOT NULL DEFAULT(datetime('now')),
updated_at TEXT NOT NULL DEFAULT(datetime('now'))
);
CREATE TABLE delivery_jobs(
delivery_job_id INTEGER PRIMARY KEY,
group_id INTEGER NOT NULL REFERENCES groups ON DELETE CASCADE,
worker_scope TEXT NOT NULL,
job_scope_spec_tag TEXT,
job_scope_include_pending INTEGER,
job_scope_support_gm_id INTEGER REFERENCES group_members(group_member_id) ON DELETE CASCADE,
single_sender_group_member_id INTEGER REFERENCES group_members(group_member_id) ON DELETE CASCADE,
body BLOB,
cursor_group_member_id INTEGER,
job_status TEXT NOT NULL,
job_err_reason TEXT,
failed INTEGER DEFAULT 0,
created_at TEXT NOT NULL DEFAULT(datetime('now')),
updated_at TEXT NOT NULL DEFAULT(datetime('now'))
);
CREATE INDEX contact_profiles_index ON contact_profiles(
display_name,
full_name
@@ -1103,3 +1136,51 @@ CREATE INDEX idx_contacts_grp_direct_inv_from_group_member_id ON contacts(
CREATE INDEX idx_contacts_grp_direct_inv_from_member_conn_id ON contacts(
grp_direct_inv_from_member_conn_id
);
CREATE INDEX idx_delivery_tasks_group_id ON delivery_tasks(group_id);
CREATE INDEX idx_delivery_tasks_job_scope_support_gm_id ON delivery_tasks(
job_scope_support_gm_id
);
CREATE INDEX idx_delivery_tasks_sender_group_member_id ON delivery_tasks(
sender_group_member_id
);
CREATE INDEX idx_delivery_tasks_message_id ON delivery_tasks(message_id);
CREATE INDEX idx_delivery_tasks_next ON delivery_tasks(
group_id,
worker_scope,
failed,
task_status
);
CREATE INDEX idx_delivery_tasks_next_for_job_scope ON delivery_tasks(
group_id,
worker_scope,
job_scope_spec_tag,
job_scope_include_pending,
job_scope_support_gm_id,
failed,
task_status
);
CREATE INDEX idx_delivery_tasks_next_for_job_scope_sender ON delivery_tasks(
group_id,
worker_scope,
job_scope_spec_tag,
job_scope_include_pending,
job_scope_support_gm_id,
sender_group_member_id,
failed,
task_status
);
CREATE INDEX idx_delivery_tasks_created_at ON delivery_tasks(created_at);
CREATE INDEX idx_delivery_jobs_group_id ON delivery_jobs(group_id);
CREATE INDEX idx_delivery_jobs_job_scope_support_gm_id ON delivery_jobs(
job_scope_support_gm_id
);
CREATE INDEX idx_delivery_jobs_single_sender_group_member_id ON delivery_jobs(
single_sender_group_member_id
);
CREATE INDEX idx_delivery_jobs_next ON delivery_jobs(
group_id,
worker_scope,
failed,
job_status
);
CREATE INDEX idx_delivery_jobs_created_at ON delivery_jobs(created_at);
src/Simplex/Chat/Store/Shared.hs
+14 -1
View File
@@ -38,6 +38,7 @@ import Simplex.Chat.Types.Preferences
import Simplex.Chat.Types.Shared
import Simplex.Chat.Types.UITheme
import Simplex.Messaging.Agent.Protocol (AConnectionRequestUri (..), AConnShortLink (..), ACreatedConnLink (..), ConnId, ConnShortLink, ConnectionRequestUri, CreatedConnLink (..), UserId, connMode)
import Simplex.Messaging.Agent.Store (AnyStoreError (..))
import Simplex.Messaging.Agent.Store.AgentStore (firstRow, maybeFirstRow)
import Simplex.Messaging.Agent.Store.DB (BoolInt (..))
import qualified Simplex.Messaging.Agent.Store.DB as DB
@@ -148,12 +149,24 @@ data StoreError
| SEUsageConditionsNotFound
| SEInvalidQuote
| SEInvalidMention
| SEInvalidDeliveryTask {taskId :: Int64}
| SEDeliveryTaskNotFound {taskId :: Int64}
| SEInvalidDeliveryJob {jobId :: Int64}
| SEDeliveryJobNotFound {jobId :: Int64}
| -- | Error when reading work item that suspends worker - do not use!
SEWorkItemError {errContext :: String}
deriving (Show, Exception)
instance AnyError StoreError where
fromSomeException = SEInternalError . show
{-# INLINE fromSomeException #-}
instance AnyStoreError StoreError where
isWorkItemError = \case
SEWorkItemError {} -> True
_ -> False
mkWorkItemError errContext = SEWorkItemError {errContext}
$(J.deriveJSON (sumTypeJSON $ dropPrefix "SE") ''StoreError)
insertedRowId :: DB.Connection -> IO Int64
@@ -657,7 +670,7 @@ toGroupInfo vr userContactId chatTags ((groupId, localDisplayName, displayName,
groupProfile = GroupProfile {displayName, fullName, shortDescr, description, image, groupPreferences, memberAdmission}
businessChat = toBusinessChatInfo businessRow
preparedGroup = toPreparedGroup preparedGroupRow
in GroupInfo {groupId, localDisplayName, groupProfile, localAlias, businessChat, fullGroupPreferences, membership, chatSettings, createdAt, updatedAt, chatTs, userMemberProfileSentAt, preparedGroup, chatTags, chatItemTTL, uiThemes, customData, membersRequireAttention, viaGroupLinkUri}
in GroupInfo {groupId, useRelays = False, localDisplayName, groupProfile, localAlias, businessChat, fullGroupPreferences, membership, chatSettings, createdAt, updatedAt, chatTs, userMemberProfileSentAt, preparedGroup, chatTags, chatItemTTL, uiThemes, customData, membersRequireAttention, viaGroupLinkUri}
toPreparedGroup :: PreparedGroupRow -> Maybe PreparedGroup
toPreparedGroup = \case