core: update simplexmq, show skipped messages, read work items in one snapshot (#8190)

* core: lock delivery and relay request work items on PostgreSQL

* core: retry store transactions aborted by PostgreSQL deadlocks

* core: show skipped messages followed by agent messages

* core: read delivery and relay request work items in one snapshot

* core: use the conflict retry of the store transaction for work item reads

* core: show integrity errors of agent messages from MSG without body

* core: show integrity errors from MSGERR, log MSGERR without an error

* core: update simplexmq, set server forward workers in tests

* core: update simplexmq
This commit is contained in:
Evgeny @ SimpleX Chat authored and GitHub committed 2026-11-14 10:28:07 +00:00
1 parent 01151021bd
commit 36e7dc3f9e
5 files changed
+21 -8

No files matched your search

+1 -1
View File
@@ -21,7 +21,7 @@ constraints: zip +disable-bzip2 +disable-zstd
source-repository-package
type: git
location: https://github.com/simplex-chat/simplexmq.git
tag: 71179c4d4e96893fbddc25770d279a0f52fc41f7
tag: 73f734927fdc97514f320054b42b9eb2fa61fee3
source-repository-package
type: git
+1 -1
View File
@@ -1,5 +1,5 @@
{
"https://github.com/simplex-chat/simplexmq.git"."71179c4d4e96893fbddc25770d279a0f52fc41f7" = "0m4363xb93xfxmxknymk5j56xya84121qyqcxrpzwgbahnm1x9yn";
"https://github.com/simplex-chat/simplexmq.git"."73f734927fdc97514f320054b42b9eb2fa61fee3" = "1xk2vg6clz2s35kjxcbn2agk7yvzgqirs827kyxymw6hz9fw8fvv";
"https://github.com/simplex-chat/hs-socks.git"."a30cc7a79a08d8108316094f8f2f82a0c5e1ac51" = "0yasvnr7g91k76mjkamvzab2kvlb1g5pspjyjn2fr6v83swjhj38";
"https://github.com/simplex-chat/direct-sqlcipher.git"."2330df3dc8c4b660674d3c02d76bb86f66c2abee" = "11pfj1vqsjdl26nplkr2l6z1xj7cn153qijj5gknmlzq3krn7x1i";
"https://github.com/simplex-chat/sqlcipher-simple.git"."a46bd361a19376c5211f1058908fc0ae6bf42446" = "1z0r78d8f0812kxbgsm735qf6xx8lvaz27k1a0b4a2m0sshpd5gl";
+6 -1
View File
@@ -77,7 +77,7 @@ import Simplex.Messaging.Agent.Env.SQLite (AgentConfig, NetworkConfig, ServerCfg
import Simplex.Messaging.Agent.Lock
import Simplex.Messaging.Agent.RetryInterval (RetryInterval (..))
import Simplex.Messaging.Agent.Protocol
import Simplex.Messaging.Agent.Store.Common (DBStore, withTransaction, withTransactionPriority)
import Simplex.Messaging.Agent.Store.Common (DBStore, IsolationLevel, withTransaction, withTransactionIsolation, withTransactionPriority)
import Simplex.Messaging.Agent.Store.Shared (MigrationConfirmation, UpMigration)
import Simplex.Messaging.Agent.Store.DB (SQLError)
import qualified Simplex.Messaging.Agent.Store.DB as DB
@@ -1825,6 +1825,11 @@ withStorePriority priority action = do
ChatController {chatStore} <- ask
liftIOEither $ withTransactionPriority chatStore priority (runExceptT . withExceptT ChatErrorStore . action) `catchOwn'` handleDBErrors
withStoreIsolation' :: IsolationLevel -> (DB.Connection -> IO a) -> CM a
withStoreIsolation' isolation action = do
ChatController {chatStore} <- ask
liftIOEither $ withTransactionIsolation chatStore isolation (fmap Right . action) `catchOwn'` handleDBErrors
withStoreBatch :: Traversable t => (DB.Connection -> t (IO (Either ChatError a))) -> CM' (t (Either ChatError a))
withStoreBatch actions = do
ChatController {chatStore} <- ask
+11 -4
View File
@@ -86,6 +86,7 @@ import Simplex.Messaging.Agent.Env.SQLite (Worker (..))
import Simplex.Messaging.Agent.Protocol
import qualified Simplex.Messaging.Agent.Protocol as AP (AgentErrorType (..))
import Simplex.Messaging.Agent.RetryInterval (RetryInterval (..), nextRetryDelay)
import Simplex.Messaging.Agent.Store.Common (IsolationLevel (..))
import qualified Simplex.Messaging.Agent.Store.DB as DB
import Simplex.Messaging.Client (NetworkRequestMode (..), ProxyClientError (..))
import qualified Simplex.Messaging.Crypto as C
@@ -616,6 +617,8 @@ processAgentMessageConn cxt user@User {userId} entity gks_ corrId agentConnId ag
RCVD msgMeta msgRcpt ->
withAckMessage' "contact rcvd" agentConnId msgMeta $
directMsgReceived ct conn msgMeta msgRcpt
MSGERR MsgMeta {integrity = MsgOk} -> logError "MSGERR with MsgOk integrity"
MSGERR msgMeta -> checkIntegrityCreateItem (CDDirectRcv ct) msgMeta `catchAllErrors` \_ -> pure ()
CONF confId pqSupport _ connInfo -> do
conn' <- processCONFpqSupport conn pqSupport
ChatMessage {chatVRange, chatMsgEvent} <- parseChatMessage conn' connInfo
@@ -1184,6 +1187,10 @@ processAgentMessageConn cxt user@User {userId} entity gks_ corrId agentConnId ag
RCVD msgMeta msgRcpt ->
withAckMessage' "group rcvd" agentConnId msgMeta $
groupMsgReceived gInfo m conn msgMeta msgRcpt
MSGERR MsgMeta {integrity = MsgOk} -> logError "MSGERR with MsgOk integrity"
MSGERR msgMeta -> do
(gInfo', m', scopeInfo) <- mkGroupChatScope gInfo m
checkIntegrityCreateItem (CDGroupRcv gInfo' scopeInfo m') msgMeta `catchAllErrors` \_ -> pure ()
SENT msgId proxy -> do
continued <- continueSending connEntity conn
(fileEvent_, acis) <- withStore $ \db -> do
@@ -4214,7 +4221,7 @@ runDeliveryTaskWorker a deliveryKey Worker {doWork} = do
(groupId, workerScope) = deliveryKey
runDeliveryTaskOperation :: StoreCxt -> GroupInfo -> CM ()
runDeliveryTaskOperation cxt gInfo = do
withWork_ a doWork (withStore' $ \db -> getNextDeliveryTask db deliveryKey) $ \task ->
withWork_ a doWork (withStoreIsolation' RepeatableRead $ \db -> getNextDeliveryTask db deliveryKey) $ \task ->
processDeliveryTask task
`catchAllErrors` \e -> do
withStore' $ \db -> setDeliveryTaskErrStatus db (deliveryTaskId task) (tshow e)
@@ -4228,7 +4235,7 @@ runDeliveryTaskWorker a deliveryKey Worker {doWork} = do
logWarn "delivery task worker: relay inactive"
withStore' $ \db -> setDeliveryTaskErrStatus db (deliveryTaskId task) "relay inactive"
| otherwise ->
withWorkItems a doWork (withStore' $ \db -> getNextDeliveryTasks db gInfo task) $ \nextTasks -> do
withWorkItems a doWork (withStoreIsolation' RepeatableRead $ \db -> getNextDeliveryTasks db gInfo task) $ \nextTasks -> do
let (body_, acceptedTasks, largeTasks) = batchDeliveryTasks1 (vr cxt) maxForwardBatchLength nextTasks
senderGMIds = S.toList . S.fromList $ map (\MessageDeliveryTask {senderGMId} -> senderGMId) acceptedTasks
notification = any (\MessageDeliveryTask {verifiedMsg} -> msgHasNotification verifiedMsg) acceptedTasks
@@ -4304,7 +4311,7 @@ runDeliveryJobWorker a deliveryKey Worker {doWork} = do
(groupId, workerScope) = deliveryKey
runDeliveryJobOperation :: StoreCxt -> User -> GroupInfo -> CM ()
runDeliveryJobOperation cxt user gInfo = do
withWork_ a doWork (withStore' $ \db -> getNextDeliveryJob db deliveryKey) $ \job ->
withWork_ a doWork (withStoreIsolation' RepeatableRead $ \db -> getNextDeliveryJob db deliveryKey) $ \job ->
processDeliveryJob job
`catchAllErrors` \e -> do
withStore' $ \db -> setDeliveryJobErrStatus db (deliveryJobId job) (tshow e)
@@ -4530,7 +4537,7 @@ runRelayRequestWorker a Worker {doWork} = do
where
getReadyRelayRequest :: CM (Either StoreError (Maybe (GroupId, RelayRequestData)))
getReadyRelayRequest =
withStore' getNextPendingRelayRequest >>= \case
withStoreIsolation' RepeatableRead getNextPendingRelayRequest >>= \case
Right (Just (groupId, rrd@RelayRequestData {reqExecuteAt})) -> do
currentTs <- liftIO getCurrentTime
let delay = diffUTCTime reqExecuteAt currentTs
+2 -1
View File
@@ -60,7 +60,7 @@ import Simplex.Messaging.Client (ProtocolClientConfig (..))
import Simplex.Messaging.Client.Agent (defaultSMPClientAgentConfig)
import Simplex.Messaging.Protocol (ProtoServerWithAuth (..), ProtocolServer (..), ProtocolType (..))
import Simplex.Messaging.Server (runSMPServerBlocking)
import Simplex.Messaging.Server.Env.STM (ServerConfig (..), ServerStoreCfg (..), StartOptions (..), StorePaths (..), defaultMaxClientCmdWait, defaultMessageExpiration, defaultNtfExpiration, defaultInactiveClientExpiration)
import Simplex.Messaging.Server.Env.STM (ServerConfig (..), ServerStoreCfg (..), StartOptions (..), StorePaths (..), defaultForwardWorkers, defaultMaxClientCmdWait, defaultMessageExpiration, defaultNtfExpiration, defaultInactiveClientExpiration)
import NameResolver (NameRegistry, resolverNamesConfig, withNameResolver)
import Simplex.Messaging.Server.MsgStore.STM (STMMsgStore)
import Simplex.Messaging.Transport
@@ -678,6 +678,7 @@ smpServerCfg p =
allowSMPProxy = True,
serverClientConcurrency = 16,
serverResolverConcurrency = 1000,
serverForwardWorkers = defaultForwardWorkers,
maxClientCmdWait = defaultMaxClientCmdWait,
namesConfig = Nothing,
information = Nothing,