smp server: deliver messages sent while the recipient subscribes (#2008)

* tests: reproduce SUB and SEND race in one block

* smp server: move subscriber registration out of serverThread

* smp server: deliver messages sent while the recipient subscribes

* smp server: keep subscriber registration in smpServer, pass it to client

* tests: receive SUB and SEND responses in order

* smp server: register subscription of a new queue directly

* smp server: name server subscribers srvSubs in runClient

* tests: sign SUB with authTransmission, keep test helpers unchanged

* smp server: remove do and mask_ from subscribeNewQueue

* smp server: send the first message once when SEND delivers it during SUB

* smp server: plan with SUB and SEND sequences before and after the change

* smp server: one sequence diagram with added and removed steps in the plan

* smp server: process all subscription changes in the client thread, remove subQ and serverThread

* smp server: send the first message once after SUBS, keep service counts exact

* smp server: name the reader monad in the type of setService

* plans: correct the diagrams and decisions of subscription processing

* plans: diagrams of a duplicate SUB and of SUBS, corrected decisions

* plans: duplicate SUB with SEND, service counts before and after the change

* tests: switch subscription with SUB and SEND in one block

* docs: remove the outdated SMP server design diagram

* smp server: move processClientSub back to the place of serverThread, passed to client threads as a parameter

* smp server: pass the pair of subscription processors to client, applied to the client where called

* smp server: show subscribeQueue and sharedSubscribeQueue as comments at their master places, temporary [skip ci]

* smp server: remove the temporary comments, the tree equals 966701dd [skip ci]

* tests: cover the changed subscription paths with concurrent, blocked, service and notifier scenarios

* tests: cover SMP server subscription changes

* smp server: remove the subscription branch for a disconnected client
This commit is contained in:
Evgeny @ SimpleX Chat authored and GitHub committed 2026-11-09 12:41:55 +00:00
1 parent 402548eb55
commit 71bb0d34a4
7 files changed
+503 -512

No files matched your search

+168 -167
View File
@@ -61,7 +61,7 @@ import qualified Data.ByteString.Char8 as B
import qualified Data.ByteString.Lazy.Char8 as LB
import Data.Constraint (Dict (..))
import Data.Dynamic (toDyn)
import Data.Either (fromRight, partitionEithers)
import Data.Either (fromRight, isRight, partitionEithers)
import Data.Foldable (foldrM)
import Data.Functor (($>), (<&>))
import Data.IORef
@@ -163,10 +163,10 @@ runSMPServerBlocking started cfg attachHTTP_ = newEnv cfg >>= runReaderT (smpSer
type M s a = ReaderT (Env s) IO a
type AttachHTTP = Socket -> TLS.Context -> IO ()
-- actions used in serverThread to reduce STM transaction scope
-- actions used in processClientSub to reduce STM transaction scope
data ClientSubAction
= CSAEndSub QueueId -- end single direct queue subscription
| CSAEndServiceSub QueueId -- end service subscription to one queue
| CSAEndServiceSub ServiceId QueueId -- end service subscription to one queue
| CSADecreaseSubs (Int64, IdsHash) -- reduce service subscriptions when cancelling. Fixed number is used to correctly handle race conditions when service resubscribes
type PrevClientSub s = (Client s, ClientSubAction, (EntityId, BrokerMsg))
@@ -185,9 +185,7 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOpt
stopServer s
liftIO $ exitSuccess
raceAny_
( serverThread "server subscribers" s subscribers subscriptions serviceSubsCount (Just cancelSub)
: serverThread "server ntfSubscribers" s ntfSubscribers ntfSubscriptions ntfServiceSubsCount Nothing
: deliverNtfsThread s
( deliverNtfsThread s
: sendPendingEvtsThread s
: receiveFromProxyAgent pa
: expireNtfsThread cfg
@@ -248,70 +246,49 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOpt
asks (smpAgent . proxyAgent) >>= liftIO . closeSMPClientAgent
asks namesEnv >>= liftIO . mapM_ closeNamesEnv
serverThread ::
forall sub. String ->
processClientSub ::
forall sub.
Server s ->
(Server s -> ServerSubscribers s) ->
(Client s -> TMap QueueId sub) ->
(Client s -> TVar (Int64, IdsHash)) ->
Maybe (sub -> IO ()) ->
M s ()
serverThread label srv srvSubscribers clientSubs clientServiceSubs unsub_ = do
labelMyThread label
liftIO . forever $ do
-- Reading clients outside of `updateSubscribers` transaction to avoid transaction re-evaluation on each new connected client.
-- In case client disconnects during the transaction (its `connected` property is read),
-- the transaction will still be re-evaluated, and the client won't be stored as subscribed.
sub@(_, clntId) <- atomically $ readTQueue subQ
c_ <- getServerClient clntId srv
atomically (updateSubscribers c_ sub)
Client s -> ClientSub -> IO ()
processClientSub srv srvSubscribers clientSubs clientServiceSubs unsub_ clnt sub =
uninterruptibleMask_ $
atomically (updateSubscribers clnt sub)
>>= endPreviousSubscriptions
where
ServerSubscribers {subQ, queueSubscribers, serviceSubscribers, totalServiceSubs, subClients, pendingEvents} = srvSubscribers srv
updateSubscribers :: Maybe (Client s) -> (ClientSub, ClientId) -> STM [PrevClientSub s]
updateSubscribers c_ (clntSub, clntId) = case c_ of
Just c@Client {connected} -> ifM (readTVar connected) (updateSubConnected c) updateSubDisconnected
Nothing -> updateSubDisconnected
ServerSubscribers {queueSubscribers, serviceSubscribers, totalServiceSubs, subClients, pendingEvents} = srvSubscribers srv
updateSubscribers :: Client s -> ClientSub -> STM [PrevClientSub s]
updateSubscribers c@Client {clientId = clntId} clntSub = case clntSub of
CSClient qId prevServiceId serviceId_ -> do
modifyTVar' subClients $ IS.insert clntId -- add ID to server's subscribed cients
as'' <- if prevServiceId == serviceId_ then pure [] else endServiceSub prevServiceId qId END
case serviceId_ of
Just serviceId -> do
modifyTVar' totalServiceSubs $ addServiceSubs (1, queueIdHash qId) -- server count and IDs hash for all services
as <- endQueueSub qId END
as' <- cancelServiceSubs serviceId =<< upsertSubscribedClient serviceId c serviceSubscribers
pure $ as ++ as' ++ as''
Nothing -> do
as <- prevSub qId END (CSAEndSub qId) =<< upsertSubscribedClient qId c queueSubscribers
pure $ as ++ as''
CSDeleted qId serviceId -> do
removeWhenNoSubs c
as <- endQueueSub qId DELD
as' <- endServiceSub serviceId qId DELD
pure $ as ++ as'
CSService serviceId changedSubs -> do
modifyTVar' subClients $ IS.insert clntId -- add ID to server's subscribed cients
modifyTVar' totalServiceSubs $ addServiceSubs changedSubs -- server count and IDs hash for all services
cancelServiceSubs serviceId =<< upsertSubscribedClient serviceId c serviceSubscribers
where
updateSubConnected c = case clntSub of
CSClient qId prevServiceId serviceId_ -> do
modifyTVar' subClients $ IS.insert clntId -- add ID to server's subscribed cients
as'' <- if prevServiceId == serviceId_ then pure [] else endServiceSub prevServiceId qId END
case serviceId_ of
Just serviceId -> do
modifyTVar' totalServiceSubs $ addServiceSubs (1, queueIdHash qId) -- server count and IDs hash for all services
as <- endQueueSub qId END
as' <- cancelServiceSubs serviceId =<< upsertSubscribedClient serviceId c serviceSubscribers
pure $ as ++ as' ++ as''
Nothing -> do
as <- prevSub qId END (CSAEndSub qId) =<< upsertSubscribedClient qId c queueSubscribers
pure $ as ++ as''
CSDeleted qId serviceId -> do
removeWhenNoSubs c
as <- endQueueSub qId DELD
as' <- endServiceSub serviceId qId DELD
pure $ as ++ as'
CSService serviceId changedSubs -> do
modifyTVar' subClients $ IS.insert clntId -- add ID to server's subscribed cients
modifyTVar' totalServiceSubs $ addServiceSubs changedSubs -- server count and IDs hash for all services
cancelServiceSubs serviceId =<< upsertSubscribedClient serviceId c serviceSubscribers
updateSubDisconnected = case clntSub of
-- do not insert client if it is already disconnected, but send END/DELD to any other client subscribed to this queue or service
CSClient qId prevServiceId serviceId -> do
as <- endQueueSub qId END
as' <- endServiceSub serviceId qId END
as'' <- if prevServiceId == serviceId then pure [] else endServiceSub prevServiceId qId END
pure $ as ++ as' ++ as''
CSDeleted qId serviceId -> do
as <- endQueueSub qId DELD
as' <- endServiceSub serviceId qId DELD
pure $ as ++ as'
CSService serviceId _ -> cancelServiceSubs serviceId =<< lookupSubscribedClient serviceId serviceSubscribers
endQueueSub :: QueueId -> BrokerMsg -> STM [PrevClientSub s]
endQueueSub qId msg = prevSub qId msg (CSAEndSub qId) =<< lookupDeleteSubscribedClient qId queueSubscribers
endServiceSub :: Maybe ServiceId -> QueueId -> BrokerMsg -> STM [PrevClientSub s]
endServiceSub Nothing _ _ = pure []
endServiceSub (Just serviceId) qId msg = prevSub qId msg (CSAEndServiceSub qId) =<< lookupSubscribedClient serviceId serviceSubscribers
endServiceSub (Just serviceId) qId msg = prevSub qId msg (CSAEndServiceSub serviceId qId) =<< lookupSubscribedClient serviceId serviceSubscribers
prevSub :: QueueId -> BrokerMsg -> ClientSubAction -> Maybe (Client s) -> STM [PrevClientSub s]
prevSub qId msg action =
checkAnotherClient $ \c -> pure [(c, action, (qId, msg))]
@@ -333,7 +310,7 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOpt
CSAEndSub qId -> atomically (endSub c qId) >>= unsubPrev
-- like endSub, also removes the delivery subscription from the service client's subscriptions map,
-- otherwise the map retains entries for queues unassociated/deleted while the service stays connected.
CSAEndServiceSub qId -> atomically (endServiceQueueSub c qId) >>= unsubPrev
CSAEndServiceSub serviceId qId -> atomically (endServiceQueueSub c serviceId qId) >>= unsubPrev
CSADecreaseSubs changedSubs -> do
atomically $ modifyTVar' totalServiceSubs $ subtractServiceSubs changedSubs
forM_ unsub_ $ \unsub -> atomically (swapTVar (clientSubs c) M.empty) >>= mapM_ unsub
@@ -342,10 +319,11 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOpt
unsubPrev s_ = sequence_ (unsub_ <*> s_)
endSub :: Client s -> QueueId -> STM (Maybe sub)
endSub c qId = TM.lookupDelete qId (clientSubs c) >>= (removeWhenNoSubs c $>)
endServiceQueueSub :: Client s -> QueueId -> STM (Maybe sub)
endServiceQueueSub c qId = do
modifyTVar' (clientServiceSubs c) decrease
modifyTVar' totalServiceSubs decrease
endServiceQueueSub :: Client s -> ServiceId -> QueueId -> STM (Maybe sub)
endServiceQueueSub c serviceId qId = do
whenM (maybe False (sameClientId c) <$> lookupSubscribedClient serviceId serviceSubscribers) $ do
modifyTVar' (clientServiceSubs c) decrease
modifyTVar' totalServiceSubs decrease
endSub c qId
where
decrease = subtractServiceSubs (1, queueIdHash qId)
@@ -733,6 +711,7 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOpt
runClient :: Transport c => X.CertificateChain -> C.APrivateSignKey -> TProxy c 'TServer -> c 'TServer -> M s ()
runClient srvCert srvSignKey tp h = do
ms <- asks msgStore
srv <- asks server
g <- asks random
idSize <- asks $ queueIdBytes . config
kh <- asks serverIdentity
@@ -741,8 +720,10 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOpt
let serverInfo = LB.toStrict . J.encode <$> information
labelMyThread $ "smp handshake for " <> transportName tp
liftIO (timeout smpHandshakeTimeout . runExceptT $ smpServerHandshake srvCert srvSignKey h ks kh smpServerVRange serverInfo $ getClientService ms g idSize) >>= \case
Just (Right th) -> runClientTransport th
Just (Right th) -> runClientTransport (processSubs srv) th
_ -> pure ()
where
processSubs srv = (processClientSub srv subscribers subscriptions serviceSubsCount (Just cancelSub), processClientSub srv ntfSubscribers ntfSubscriptions ntfServiceSubsCount Nothing)
getClientService :: s -> TVar ChaChaDRG -> Int -> SMPServiceRole -> X.CertificateChain -> XV.Fingerprint -> ExceptT TransportError IO ServiceId
getClientService ms g idSize role cert fp = do
@@ -1059,8 +1040,8 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOpt
logError "Unauthorized control port command"
hPutStrLn h "AUTH"
runClientTransport :: forall c s. (Transport c, MsgStoreClass s) => THandleSMP c 'TServer -> M s ()
runClientTransport h@THandle {params = thParams@THandleParams {sessionId}} = do
runClientTransport :: forall c s. (Transport c, MsgStoreClass s) => (Client s -> ClientSub -> IO (), Client s -> ClientSub -> IO ()) -> THandleSMP c 'TServer -> M s ()
runClientTransport processSubs h@THandle {params = thParams@THandleParams {sessionId}} = do
q <- asks $ tbqSize . config
ts <- liftIO getSystemTime
nextClientId <- asks clientSeq
@@ -1076,7 +1057,7 @@ runClientTransport h@THandle {params = thParams@THandleParams {sessionId}} = do
expCfg <- asks $ inactiveClientExpiration . config
th <- newMVar h -- put TH under a fair lock to interleave messages and command responses
labelMyThread . B.unpack $ "client $" <> encode sessionId
raceAny_ $ [liftIO $ send th c, liftIO $ sendMsg th c, client s ms c, receive h ms c] <> disconnectThread_ c s expCfg
raceAny_ $ [liftIO $ send th c, liftIO $ sendMsg th c, client processSubs s ms c, receive h ms c] <> disconnectThread_ c s expCfg
disconnectThread_ :: Client s -> Server s -> Maybe ExpirationConfig -> [M s ()]
disconnectThread_ c s (Just expCfg) = [liftIO $ disconnectTransport h (rcvActiveAt c) (sndActiveAt c) expCfg (noSubscriptions c s)]
disconnectThread_ _ _ _ = []
@@ -1405,11 +1386,12 @@ forkClient Client {endThreads, endThreadSeq} label action = do
action `finally` atomically (modifyTVar' endThreads $ IM.delete tId)
mkWeakThreadId t >>= atomically . modifyTVar' endThreads . IM.insert tId
client :: forall s. MsgStoreClass s => Server s -> s -> Client s -> M s ()
client :: forall s. MsgStoreClass s => (Client s -> ClientSub -> IO (), Client s -> ClientSub -> IO ()) -> Server s -> s -> Client s -> M s ()
client
Server {subscribers, ntfSubscribers}
(processSub, processNtfSub)
Server {subscribers}
ms
clnt@Client {clientId, rcvQ, sndQ, msgQ, clientTHParams = thParams'@THandleParams {sessionId}, procThreads} = do
clnt@Client {rcvQ, sndQ, msgQ, clientTHParams = thParams'@THandleParams {sessionId}, procThreads} = do
labelMyThread . B.unpack $ "client $" <> encode sessionId <> " commands"
stats <- asks serverStats
authDelay <- asks $ authErrorDelay . config
@@ -1427,25 +1409,100 @@ client
prepareBatchSubs ::
Maybe ServiceId ->
NonEmpty (VerifiedTransmission s) ->
M s (Either ErrorType (Map RecipientId Message, Map RecipientId (Either ErrorType ()), Map RecipientId (Either ErrorType ())))
M s (Either ErrorType (Map RecipientId Message, Map RecipientId (Either ErrorType ()), Map RecipientId (Either ErrorType ()), Map RecipientId (Either ErrorType (Bool, Maybe Sub))))
prepareBatchSubs clntServiceId_ batch = do
let (subMsgQs, rcvAssocQs, ntfAssocQs) = foldr partitionSubs ([], [], []) batch
partitionSubs t (msgQs, rcvQs, ntfQs) = case t of
(Just (q, qr), (_, _, Cmd SRecipient SUB))
| clntServiceId_ /= rcvServiceId qr -> (q : msgQs, q : rcvQs, ntfQs)
| otherwise -> (q : msgQs, rcvQs, ntfQs)
| clntServiceId_ /= rcvServiceId qr -> ((q, qr) : msgQs, q : rcvQs, ntfQs)
| otherwise -> ((q, qr) : msgQs, rcvQs, ntfQs)
(Just (q, qr), (_, _, Cmd SNotifier NSUB))
| clntServiceId_ /= (notifier qr >>= ntfServiceId) -> (msgQs, rcvQs, q : ntfQs)
_ -> (msgQs, rcvQs, ntfQs)
liftIO $ runExceptT $ do
runExceptT $ do
rcvAssocs <- ifNotNull rcvAssocQs $ setService SRecipientService clntServiceId_
ntfAssocs <- ifNotNull ntfAssocQs $ setService SNotifierService clntServiceId_
msgs <- ifNotNull subMsgQs $ tryPeekMsgs ms
pure (msgs, rcvAssocs, ntfAssocs)
subs <- lift $ M.fromList <$> mapM (\(q, qr) -> (recipientId q,) <$> subscribeQueue clntServiceId_ q qr) (filter (subscribable rcvAssocs) subMsgQs)
msgs <- ifNotNull (map fst subMsgQs) $ mapExceptT liftIO . tryPeekMsgs ms
pure (msgs, rcvAssocs, ntfAssocs, subs)
where
ifNotNull qs f = if null qs then pure M.empty else f qs
setService :: (PartyI p, ServiceParty p) => SParty p -> Maybe ServiceId -> [StoreQueue s] -> ExceptT ErrorType IO (Map RecipientId (Either ErrorType ()))
setService party sId = ExceptT . setQueueServices (queueStore ms) party sId
setService :: (PartyI p, ServiceParty p) => SParty p -> Maybe ServiceId -> [StoreQueue s] -> ExceptT ErrorType (ReaderT (Env s) IO) (Map RecipientId (Either ErrorType ()))
setService party sId = ExceptT . liftIO . setQueueServices (queueStore ms) party sId
subscribable rcvAssocs (q, QueueRec {status}) = case status of
EntityBlocked _ -> False
_ -> all isRight $ M.lookup (recipientId q) rcvAssocs
subscribeQueue :: Maybe ServiceId -> StoreQueue s -> QueueRec -> M s (Either ErrorType (Bool, Maybe Sub))
subscribeQueue clntServiceId q QueueRec {rcvServiceId} =
liftIO (TM.lookupIO rId $ subscriptions clnt) >>= \case
Nothing -> Right <$> sharedSubscribeQueue clntServiceId rId q rcvServiceId processSub subscriptions serviceSubsCount (newSubscription NoSub) rcvServices
Just s@Sub {subThread} -> do
stats <- asks serverStats
case subThread of
ProhibitSub -> do
-- cannot use SUB in the same connection where GET was used
incStat $ qSubProhibited stats
pure $ Left $ CMD PROHIBITED
_ -> do
incStat $ qSubDuplicate stats
Right (True, Just s) <$ atomically (writeTVar (delivered s) Nothing)
where
rId = recipientId q
sharedSubscribeQueue ::
Maybe ServiceId ->
QueueId ->
StoreQueue s ->
Maybe ServiceId ->
(Client s -> ClientSub -> IO ()) ->
(Client s -> TMap QueueId sub) ->
(Client s -> TVar (Int64, IdsHash)) ->
STM sub ->
(ServerStats -> ServiceStats) ->
M s (Bool, Maybe sub)
sharedSubscribeQueue clntServiceId entId q queueServiceId processSub' clientSubs clientServiceSubs mkSub servicesSel = do
stats <- asks serverStats
let incSrvStat sel = incStat $ sel $ servicesSel stats
writeSub = processSub' clnt $ CSClient entId queueServiceId clntServiceId
liftIO $ case clntServiceId of
Just serviceId
| queueServiceId == Just serviceId -> do
-- duplicate queue-service association - can only happen in case of response error/timeout
hasSub <- atomically $ ifM hasServiceSub (pure True) (False <$ incServiceQueueSubs)
unless hasSub $ do
writeSub
incSrvStat srvSubCount
incSrvStat srvSubQueues
incSrvStat srvAssocDuplicate
pure (hasSub, Nothing)
| otherwise -> do
-- association already done in prepareBatchSubs
hasSub <- atomically $ (<$ incServiceQueueSubs) =<< hasServiceSub
writeSub
unless hasSub $ incSrvStat srvSubCount
incSrvStat srvSubQueues
incSrvStat $ maybe srvAssocNew (const srvAssocUpdated) queueServiceId
pure (hasSub, Nothing)
where
hasServiceSub = ((0 /=) . fst) <$> readTVar (clientServiceSubs clnt)
-- This function is used when queue association with the service is created.
incServiceQueueSubs = modifyTVar' (clientServiceSubs clnt) $ addServiceSubs (1, queueIdHash (recipientId q)) -- service count and IDS hash
Nothing -> case queueServiceId of
Just _ -> do
-- unassociation already done in prepareBatchSubs
incSrvStat srvAssocRemoved
-- getSubscription may find a subscription for receiving service, where clientSubs also hold active deliveries for service subscriptions.
-- For notification service it can only find one if storage and session states diverge.
r <- getSubscription
writeSub
pure r
Nothing -> do
r@(hasSub, _) <- getSubscription
unless hasSub writeSub
pure r
where
getSubscription = either ((False,) . Just) ((True,) . Just) <$> TM.lookupOrInsertIO entId mkSub (clientSubs clnt)
processProxiedCmd :: Transmission (Command 'ProxiedClient) -> M s (Maybe ResponseAndMessage)
processProxiedCmd (corrId, EntityId sessId, command) = (\t -> ((corrId, EntityId sessId, t), Nothing)) <$$> case command of
@@ -1573,7 +1630,7 @@ client
mkIncProxyStats ps psOwn own sel = do
incStat $ sel ps
when own $ incStat $ sel psOwn
processCommand :: Maybe ServiceId -> Either ErrorType (Map RecipientId Message, Map RecipientId (Either ErrorType ()), Map RecipientId (Either ErrorType ())) -> VerifiedTransmission s -> M s (Maybe ResponseAndMessage)
processCommand :: Maybe ServiceId -> Either ErrorType (Map RecipientId Message, Map RecipientId (Either ErrorType ()), Map RecipientId (Either ErrorType ()), Map RecipientId (Either ErrorType (Bool, Maybe Sub))) -> VerifiedTransmission s -> M s (Maybe ResponseAndMessage)
processCommand clntServiceId batchSubs (q_, (corrId, entId, cmd)) = case cmd of
Cmd SProxiedClient command -> processProxiedCmd (corrId, entId, command)
Cmd SSender command -> case command of
@@ -1590,7 +1647,7 @@ client
Cmd SNotifier NSUB -> response . (corrId,entId,) <$> case q_ of
Just (q, QueueRec {notifier = Just ntfCreds}) ->
either (pure . ERR) (\_ -> subscribeNotifications q ntfCreds)
$ batchSubs >>= \(_, _, ntfAssocs) -> sequence (M.lookup (recipientId q) ntfAssocs)
$ batchSubs >>= \(_, _, ntfAssocs, _) -> sequence (M.lookup (recipientId q) ntfAssocs)
_ -> pure $ ERR INTERNAL
Cmd SNotifierService (NSUBS n idsHash) -> response . (corrId,entId,) <$> case clntServiceId of
Just serviceId -> subscribeServiceNotifications serviceId (n, idsHash)
@@ -1603,9 +1660,9 @@ client
pure $ allowNewQueues && maybe True ((== auth_) . Just) newQueueBasicAuth
Cmd SRecipient command ->
case command of
SUB -> case batchSubs >>= \(msgs, rcvAssocs, _) -> sequence (M.lookup entId rcvAssocs) $> msgs of
SUB -> case batchSubs >>= \(msgs, rcvAssocs, _, subs) -> sequence (M.lookup entId rcvAssocs) $> (msgs, subs) of
Left e -> pure $ Just (err e, Nothing)
Right msgs -> withQueue' $ subscribeQueueAndDeliver $ M.lookup entId msgs
Right (msgs, subs) -> withQueue' $ deliverSubMsg (M.lookup entId msgs) (M.lookup entId subs)
GET -> withQueue getMessage
ACK msgId -> withQueue $ acknowledgeMsg msgId
KEY sKey -> withQueue $ \q _ -> either err (corrId,entId,) <$> secureQueue_ q sKey
@@ -1726,7 +1783,7 @@ client
Right nc_ -> do
incStat . ntfCreated =<< asks serverStats
forM_ nc_ $ \NtfCreds {notifierId = nId, ntfServiceId} ->
atomically $ writeTQueue (subQ ntfSubscribers) (CSDeleted nId ntfServiceId, clientId)
liftIO $ processNtfSub clnt $ CSDeleted nId ntfServiceId
pure $ NID notifierId rcvPublicDhKey
deleteQueueNotifier_ :: StoreQueue s -> M s (Transmission BrokerMsg)
@@ -1737,7 +1794,7 @@ client
stats <- asks serverStats
deleted <- asks ntfStore >>= liftIO . (`deleteNtfs` nId)
when (deleted > 0) $ liftIO $ atomicModifyIORef'_ (ntfCount stats) (subtract deleted)
atomically $ writeTQueue (subQ ntfSubscribers) (CSDeleted nId ntfServiceId, clientId)
liftIO $ processNtfSub clnt $ CSDeleted nId ntfServiceId
incStat $ ntfDeleted stats
pure ok
Right Nothing -> pure ok
@@ -1746,32 +1803,24 @@ client
suspendQueue_ :: (StoreQueue s, QueueRec) -> M s (Transmission BrokerMsg)
suspendQueue_ (q, _) = liftIO $ either err (const ok) <$> suspendQueue (queueStore ms) q
subscribeQueueAndDeliver :: Maybe Message -> StoreQueue s -> QueueRec -> M s ResponseAndMessage
subscribeQueueAndDeliver msg_ q qr@QueueRec {rcvServiceId} =
liftIO (TM.lookupIO entId $ subscriptions clnt) >>= \case
Nothing ->
deliver =<< sharedSubscribeQueue q rcvServiceId subscribers subscriptions serviceSubsCount (newSubscription NoSub) rcvServices
Just s@Sub {subThread} -> do
stats <- asks serverStats
case subThread of
ProhibitSub -> do
-- cannot use SUB in the same connection where GET was used
incStat $ qSubProhibited stats
pure (err (CMD PROHIBITED), Nothing)
_ -> do
incStat $ qSubDuplicate stats
atomically (writeTVar (delivered s) Nothing) >> deliver (True, Just s)
deliverSubMsg :: Maybe Message -> Maybe (Either ErrorType (Bool, Maybe Sub)) -> StoreQueue s -> QueueRec -> M s ResponseAndMessage
deliverSubMsg msg_ r_ _ qr = case r_ of
Just (Right r) -> deliver r
Just (Left e) -> pure (err e, Nothing)
Nothing -> pure (err INTERNAL, Nothing)
where
deliver :: (Bool, Maybe Sub) -> M s ResponseAndMessage
deliver (hasSub, sub_) = do
stats <- asks serverStats
fmap (either ((,Nothing) . err) id) $ liftIO $ runExceptT $ do
msg' <- forM msg_ $ \msg -> liftIO $ do
msg' <- fmap join $ forM msg_ $ \msg -> liftIO $ do
ts <- getSystemSeconds
sub <- maybe getSub pure sub_
atomically $ setDelivered sub msg ts
unless hasSub $ incStat $ qSub stats
pure (NoCorrId, entId, MSG (encryptMsg qr msg))
ifM
(atomically $ trySetDelivered sub msg ts)
(pure $ Just (NoCorrId, entId, MSG (encryptMsg qr msg)))
(pure Nothing)
pure ((corrId, entId, SOK clntServiceId), msg')
getSub :: IO Sub
@@ -1784,7 +1833,7 @@ client
Nothing -> do
sub <- atomically $ newSubscription NoSub
atomically $ TM.insert rId sub $ subscriptions clnt
atomically $ writeTQueue (subQ subscribers) (CSClient rId rcvServiceId rcvServiceId, clientId)
liftIO $ processSub clnt $ CSClient rId rcvServiceId rcvServiceId
-- clients that use GET are not added to server subscribers
getMessage :: StoreQueue s -> QueueRec -> M s (Transmission BrokerMsg)
@@ -1840,66 +1889,14 @@ client
subscribeNotifications :: StoreQueue s -> NtfCreds -> M s BrokerMsg
subscribeNotifications q NtfCreds {ntfServiceId} = do
(hasSub, _) <- sharedSubscribeQueue q ntfServiceId ntfSubscribers ntfSubscriptions ntfServiceSubsCount (pure ()) ntfServices
(hasSub, _) <- sharedSubscribeQueue clntServiceId entId q ntfServiceId processNtfSub ntfSubscriptions ntfServiceSubsCount (pure ()) ntfServices
when (isNothing clntServiceId) $
asks serverStats >>= incStat . (if hasSub then ntfSubDuplicate else ntfSub)
pure $ SOK clntServiceId
sharedSubscribeQueue ::
StoreQueue s ->
Maybe ServiceId ->
ServerSubscribers s ->
(Client s -> TMap QueueId sub) ->
(Client s -> TVar (Int64, IdsHash)) ->
STM sub ->
(ServerStats -> ServiceStats) ->
M s (Bool, Maybe sub)
sharedSubscribeQueue q queueServiceId srvSubscribers clientSubs clientServiceSubs mkSub servicesSel = do
stats <- asks serverStats
let incSrvStat sel = incStat $ sel $ servicesSel stats
writeSub = writeTQueue (subQ srvSubscribers) (CSClient entId queueServiceId clntServiceId, clientId)
liftIO $ case clntServiceId of
Just serviceId
| queueServiceId == Just serviceId -> do
-- duplicate queue-service association - can only happen in case of response error/timeout
hasSub <- atomically $ ifM hasServiceSub (pure True) (False <$ incServiceQueueSubs)
unless hasSub $ do
atomically writeSub
incSrvStat srvSubCount
incSrvStat srvSubQueues
incSrvStat srvAssocDuplicate
pure (hasSub, Nothing)
| otherwise -> do
-- association already done in prepareBatchSubs
hasSub <- atomically $ (<$ incServiceQueueSubs) =<< hasServiceSub
atomically writeSub
unless hasSub $ incSrvStat srvSubCount
incSrvStat srvSubQueues
incSrvStat $ maybe srvAssocNew (const srvAssocUpdated) queueServiceId
pure (hasSub, Nothing)
where
hasServiceSub = ((0 /=) . fst) <$> readTVar (clientServiceSubs clnt)
-- This function is used when queue association with the service is created.
incServiceQueueSubs = modifyTVar' (clientServiceSubs clnt) $ addServiceSubs (1, queueIdHash (recipientId q)) -- service count and IDS hash
Nothing -> case queueServiceId of
Just _ -> do
-- unassociation already done in prepareBatchSubs
incSrvStat srvAssocRemoved
-- getSubscription may find a subscription for receiving service, where clientSubs also hold active deliveries for service subscriptions.
-- For notification service it can only find one if storage and session states diverge.
r <- getSubscription
atomically writeSub
pure r
Nothing -> do
r@(hasSub, _) <- getSubscription
unless hasSub $ atomically writeSub
pure r
where
getSubscription = either ((False,) . Just) ((True,) . Just) <$> TM.lookupOrInsertIO entId mkSub (clientSubs clnt)
subscribeServiceMessages :: ServiceId -> (Int64, IdsHash) -> M s BrokerMsg
subscribeServiceMessages serviceId expected =
sharedSubscribeService SRecipientService serviceId expected subscribers serviceSubscribed serviceSubsCount rcvServices >>= \case
sharedSubscribeService SRecipientService serviceId expected processSub serviceSubscribed serviceSubsCount rcvServices >>= \case
Left e -> pure $ ERR e
Right (hasSub, (count, idsHash)) -> do
stats <- asks serverStats
@@ -1930,17 +1927,18 @@ client
Right _sub -> pure (qCnt + 1, msgCnt, dupCnt + 1, evts)
Left sub -> do
ts <- getSystemSeconds
atomically $ setDelivered sub msg ts
atomically $ writeTBQueue msgQ [(NoCorrId, rId, MSG (encryptMsg qr msg))]
pure (qCnt + 1, msgCnt + 1, dupCnt, evts)
ifM
(atomically $ trySetDelivered sub msg ts)
((qCnt + 1, msgCnt + 1, dupCnt, evts) <$ atomically (writeTBQueue msgQ [(NoCorrId, rId, MSG (encryptMsg qr msg))]))
(pure (qCnt + 1, msgCnt, dupCnt + 1, evts))
getSubscription rId = TM.lookupOrInsertIO rId (newSubscription NoSub) (subscriptions clnt)
subscribeServiceNotifications :: ServiceId -> (Int64, IdsHash) -> M s BrokerMsg
subscribeServiceNotifications serviceId expected =
either ERR (uncurry SOKS . snd) <$> sharedSubscribeService SNotifierService serviceId expected ntfSubscribers ntfServiceSubscribed ntfServiceSubsCount ntfServices
either ERR (uncurry SOKS . snd) <$> sharedSubscribeService SNotifierService serviceId expected processNtfSub ntfServiceSubscribed ntfServiceSubsCount ntfServices
sharedSubscribeService :: (PartyI p, ServiceParty p) => SParty p -> ServiceId -> (Int64, IdsHash) -> ServerSubscribers s -> (Client s -> TVar Bool) -> (Client s -> TVar (Int64, IdsHash)) -> (ServerStats -> ServiceStats) -> M s (Either ErrorType (Bool, (Int64, IdsHash)))
sharedSubscribeService party serviceId (count, idsHash) srvSubscribers clientServiceSubscribed clientServiceSubs servicesSel = do
sharedSubscribeService :: (PartyI p, ServiceParty p) => SParty p -> ServiceId -> (Int64, IdsHash) -> (Client s -> ClientSub -> IO ()) -> (Client s -> TVar Bool) -> (Client s -> TVar (Int64, IdsHash)) -> (ServerStats -> ServiceStats) -> M s (Either ErrorType (Bool, (Int64, IdsHash)))
sharedSubscribeService party serviceId (count, idsHash) processSub' clientServiceSubscribed clientServiceSubs servicesSel = do
subscribed <- readTVarIO $ clientServiceSubscribed clnt
stats <- asks serverStats
liftIO $ runExceptT $
@@ -1960,7 +1958,7 @@ client
| diff > 0 -> incSrvStat srvSubMore 1 >> incSrvStat srvSubMoreTotal diff
| diff < 0 -> incSrvStat srvSubFewer 1 >> incSrvStat srvSubFewerTotal (- diff)
| otherwise -> incSrvStat srvSubDiff 1
atomically $ writeTQueue (subQ srvSubscribers) (CSService serviceId subsChange, clientId)
liftIO $ processSub' clnt $ CSService serviceId subsChange
pure (count', idsHash')
acknowledgeMsg :: MsgId -> StoreQueue s -> QueueRec -> M s (Transmission BrokerMsg)
@@ -2204,7 +2202,7 @@ client
resolveNameMsg (thVersion clntTHParams) nenv d >>= sendResolved
-- INTERNAL because processCommand never returns Nothing for sender commands;
-- `fst` drops the empty message only returned for SUB.
_ -> Just . maybe (corrId', entId', ERR INTERNAL) fst <$> lift (processCommand Nothing (Right (M.empty, M.empty, M.empty)) t'')
_ -> Just . maybe (corrId', entId', ERR INTERNAL) fst <$> lift (processCommand Nothing (Right (M.empty, M.empty, M.empty, M.empty)) t'')
stats <- asks serverStats
incStat $ pMsgFwdsRecv stats
traverse encodeResp r_ >>= \case
@@ -2246,6 +2244,9 @@ client
let !msgId = messageId msg
writeTVar delivered $ Just (msgId, ts)
trySetDelivered :: Sub -> Message -> SystemSeconds -> STM Bool
trySetDelivered sub msg ts = readTVar (delivered sub) >>= maybe (True <$ setDelivered sub msg ts) (const $ pure False)
delQueueAndMsgs :: (StoreQueue s, QueueRec) -> M s (Transmission BrokerMsg)
delQueueAndMsgs (q, QueueRec {rcvServiceId}) = do
liftIO (deleteQueue ms q) >>= \case
@@ -2256,14 +2257,14 @@ client
sub <- atomically $ TM.lookupDelete entId $ subscriptions clnt
liftIO $ mapM_ cancelSub sub
when (isJust rcvServiceId) $ atomically $ modifyTVar' (serviceSubsCount clnt) $ subtractServiceSubs (1, queueIdHash (recipientId q))
atomically $ writeTQueue (subQ subscribers) (CSDeleted entId rcvServiceId, clientId)
liftIO $ processSub clnt $ CSDeleted entId rcvServiceId
forM_ (notifier qr) $ \NtfCreds {notifierId = nId, ntfServiceId} -> do
-- queue is deleted by a different client from the one subscribed to notifications,
-- so we don't need to remove subscription from the current client.
stats <- asks serverStats
deleted <- asks ntfStore >>= liftIO . (`deleteNtfs` nId)
when (deleted > 0) $ liftIO $ atomicModifyIORef'_ (ntfCount stats) (subtract deleted)
atomically $ writeTQueue (subQ ntfSubscribers) (CSDeleted nId ntfServiceId, clientId)
liftIO $ processNtfSub clnt $ CSDeleted nId ntfServiceId
updateDeletedStats qr
pure ok
Left e -> pure $ err e
+2 -4
View File
@@ -347,8 +347,7 @@ data Server s = Server
newtype ServerClients s = ServerClients {serverClients :: TVar (IntMap (Client s))}
data ServerSubscribers s = ServerSubscribers
{ subQ :: TQueue (ClientSub, ClientId),
queueSubscribers :: SubscribedClients s,
{ queueSubscribers :: SubscribedClients s,
serviceSubscribers :: SubscribedClients s, -- service clients with long-term certificates that have subscriptions
totalServiceSubs :: TVar (Int64, IdsHash),
subClients :: TVar IntSet, -- clients with individual or service subscriptions
@@ -486,13 +485,12 @@ deleteServerClient cId Server {clients} = atomically $ modifyTVar' (serverClient
newServerSubscribers :: IO (ServerSubscribers s)
newServerSubscribers = do
subQ <- newTQueueIO
queueSubscribers <- SubscribedClients <$> TM.emptyIO
serviceSubscribers <- SubscribedClients <$> TM.emptyIO
totalServiceSubs <- newTVarIO (0, mempty)
subClients <- newTVarIO IS.empty
pendingEvents <- newTVarIO IM.empty
pure ServerSubscribers {subQ, queueSubscribers, serviceSubscribers, totalServiceSubs, subClients, pendingEvents}
pure ServerSubscribers {queueSubscribers, serviceSubscribers, totalServiceSubs, subClients, pendingEvents}
newClient :: ClientId -> Natural -> THandleParams SMPVersion 'TServer -> SystemTime -> IO (Client s)
newClient clientId qSize clientTHParams createdAt = do