From 3ed0af5c4d2fcab1f49a2265bbe2c8dece3d9cab Mon Sep 17 00:00:00 2001 From: "Evgeny @ SimpleX Chat" <259188159+evgeny-simplex@users.noreply.github.com> Date: Tue, 3 Nov 2026 20:54:53 +0000 Subject: [PATCH] smp server: process all subscription changes in the client thread, remove subQ and serverThread --- plans/2026-10-31-sub-register-first.md | 33 +- src/Simplex/Messaging/Server.hs | 449 +++++++++++------------- src/Simplex/Messaging/Server/Env/STM.hs | 6 +- tests/ServerTests.hs | 26 ++ 4 files changed, 257 insertions(+), 257 deletions(-) diff --git a/plans/2026-10-31-sub-register-first.md b/plans/2026-10-31-sub-register-first.md index 52f8ec50e..b56a7be8a 100644 --- a/plans/2026-10-31-sub-register-first.md +++ b/plans/2026-10-31-sub-register-first.md @@ -1,12 +1,13 @@ -# Subscriber registration before the first-message read +# Subscription changes processed in the client thread ## Decisions -- SUB from a client without a service certificate to an active queue not associated with a service: the subscriber is registered in the client thread, before the first message of the queue is read. -- NEW with subscription from such a client: the subscriber is registered in the client thread. -- The registration is `processClientSub`, the code of `serverThread` for one subscription change, called under `mask_`. +- Each subscription change is processed by `processClientSub` under `mask_`, in the client thread of the command. +- `serverThread` and `subQ` are removed. +- SUB, from clients with and without a service certificate: the subscriber is registered in `prepareBatchSubs`, after the queue associations and before the first-message read. +- SUB of a blocked queue, or of a queue with a failed association: the subscriber is registered only by a later successful SUB. - The SUB response includes the first message only when no message of the subscription is marked as delivered. -- Other subscription changes are registered by `serverThread` from `subQ`, as before. +- SUBS, NSUB, NSUBS, NEW with subscription, NKEY, NDEL, DEL: the change is processed while the command is processed. ## Sequence @@ -17,11 +18,11 @@ sequenceDiagram participant R as Recipient agent participant C as Client thread of R participant Q as Message store - participant T as serverThread + participant T as serverThread, removed participant S as Client thread of sender R->>C: SUB q, signature verified in receive rect rgba(0, 160, 0, 0.18) - C->>C: added - prepareBatchSubs: Sub of q inserted + C->>C: added - prepareBatchSubs, subscribeQueue: Sub of q inserted C->>C: added - processClientSub under mask_: R registered in queueSubscribers, END to the previous subscriber end C->>Q: prepareBatchSubs: tryPeekMsgs q @@ -50,10 +51,16 @@ sequenceDiagram ## Mechanics - `Server.hs`: - - `processClientSub`: the per-change code of `serverThread`, beside it in `smpServer`; `serverThread` calls it for each `subQ` entry. - - `runClient`, `runClientTransport`, `client`: `processClientSub` for message subscribers passed to the client thread. - - `prepareBatchSubs`: selected SUBs inserted with `TM.lookupOrInsertIO`; `processClientSub` for each inserted subscription, before `tryPeekMsgs`; the inserted subscriptions returned as the fourth map of the batch result. - - `subscribeQueueAndDeliver`: an inserted subscription passed to `deliver` as new. - - `subscribeNewQueue`: `processClientSub` for a client without a service certificate. + - `processClientSub`: top level; the code of `serverThread` for one change of the current client. + - `smpServer`: `serverThread` removed. + - `client`: `processSub` and `processNtfSub` are `processClientSub` for message and notification subscribers. + - `prepareBatchSubs`: queue associations; `subscribeQueue` for each SUB of a queue that is not blocked and has no failed association; `tryPeekMsgs`; the `subscribeQueue` results returned as the fourth map of the batch result. + - `subscribeQueue`: the first part of the former `subscribeQueueAndDeliver`. + - `deliverSubMsg`: the second part of the former `subscribeQueueAndDeliver`; the `subscribeQueue` result passed to `deliver`. + - `sharedSubscribeQueue`: moved from `processCommand` to `client`; `processSub` or `processNtfSub` passed in place of the server subscribers. + - `sharedSubscribeService`, `subscribeNewQueue`, `addQueueNotifier_`, `deleteQueueNotifier_`, `delQueueAndMsgs`: `processSub` or `processNtfSub` in place of the `subQ` write. - `deliver`, `trySetDelivered`: the first message marked as delivered and included only when no message is marked. -- `tests/ServerTests.hs`: `SUB and SEND in one block`. +- `Env/STM.hs`: `subQ` removed from `ServerSubscribers`. +- `tests/ServerTests.hs`: + - `SUB and SEND in one block` + - `should deliver message sent in the same block as service SUB` diff --git a/src/Simplex/Messaging/Server.hs b/src/Simplex/Messaging/Server.hs index 44cb0e94d..9cc40a18b 100644 --- a/src/Simplex/Messaging/Server.hs +++ b/src/Simplex/Messaging/Server.hs @@ -61,7 +61,7 @@ import qualified Data.ByteString.Char8 as B import qualified Data.ByteString.Lazy.Char8 as LB import Data.Constraint (Dict (..)) import Data.Dynamic (toDyn) -import Data.Either (fromRight, partitionEithers) +import Data.Either (fromRight, isRight, partitionEithers) import Data.Foldable (foldrM) import Data.Functor (($>), (<&>)) import Data.IORef @@ -73,7 +73,7 @@ import Data.List.NonEmpty (NonEmpty (..), (<|)) import qualified Data.List.NonEmpty as L import Data.Map.Strict (Map) import qualified Data.Map.Strict as M -import Data.Maybe (catMaybes, fromMaybe, isJust, isNothing) +import Data.Maybe (fromMaybe, isJust, isNothing) import Data.Semigroup (Sum (..)) import qualified Data.Set as S import Data.Text (Text) @@ -163,7 +163,7 @@ runSMPServerBlocking started cfg attachHTTP_ = newEnv cfg >>= runReaderT (smpSer type M s a = ReaderT (Env s) IO a type AttachHTTP = Socket -> TLS.Context -> IO () --- actions used in serverThread to reduce STM transaction scope +-- actions used in processClientSub to reduce STM transaction scope data ClientSubAction = CSAEndSub QueueId -- end single direct queue subscription | CSAEndServiceSub QueueId -- end service subscription to one queue @@ -171,6 +171,91 @@ data ClientSubAction type PrevClientSub s = (Client s, ClientSubAction, (EntityId, BrokerMsg)) +processClientSub :: + forall s sub. + ServerSubscribers s -> + (Client s -> TMap QueueId sub) -> + (Client s -> TVar (Int64, IdsHash)) -> + Maybe (sub -> IO ()) -> + Client s -> + ClientSub -> + IO () +processClientSub ServerSubscribers {queueSubscribers, serviceSubscribers, totalServiceSubs, subClients, pendingEvents} clientSubs clientServiceSubs unsub_ clnt@Client {clientId = clntId} clntSub = + mask_ $ + atomically updateSubscribers + >>= endPreviousSubscriptions + where + updateSubscribers :: STM [PrevClientSub s] + updateSubscribers = case clntSub of + CSClient qId prevServiceId serviceId_ -> do + modifyTVar' subClients $ IS.insert clntId -- add ID to server's subscribed cients + as'' <- if prevServiceId == serviceId_ then pure [] else endServiceSub prevServiceId qId END + case serviceId_ of + Just serviceId -> do + modifyTVar' totalServiceSubs $ addServiceSubs (1, queueIdHash qId) -- server count and IDs hash for all services + as <- endQueueSub qId END + as' <- cancelServiceSubs serviceId =<< upsertSubscribedClient serviceId clnt serviceSubscribers + pure $ as ++ as' ++ as'' + Nothing -> do + as <- prevSub qId END (CSAEndSub qId) =<< upsertSubscribedClient qId clnt queueSubscribers + pure $ as ++ as'' + CSDeleted qId serviceId -> do + removeWhenNoSubs clnt + as <- endQueueSub qId DELD + as' <- endServiceSub serviceId qId DELD + pure $ as ++ as' + CSService serviceId changedSubs -> do + modifyTVar' subClients $ IS.insert clntId -- add ID to server's subscribed cients + modifyTVar' totalServiceSubs $ addServiceSubs changedSubs -- server count and IDs hash for all services + cancelServiceSubs serviceId =<< upsertSubscribedClient serviceId clnt serviceSubscribers + endQueueSub :: QueueId -> BrokerMsg -> STM [PrevClientSub s] + endQueueSub qId msg = prevSub qId msg (CSAEndSub qId) =<< lookupDeleteSubscribedClient qId queueSubscribers + endServiceSub :: Maybe ServiceId -> QueueId -> BrokerMsg -> STM [PrevClientSub s] + endServiceSub Nothing _ _ = pure [] + endServiceSub (Just serviceId) qId msg = prevSub qId msg (CSAEndServiceSub qId) =<< lookupSubscribedClient serviceId serviceSubscribers + prevSub :: QueueId -> BrokerMsg -> ClientSubAction -> Maybe (Client s) -> STM [PrevClientSub s] + prevSub qId msg action = + checkAnotherClient $ \c -> pure [(c, action, (qId, msg))] + cancelServiceSubs :: ServiceId -> Maybe (Client s) -> STM [PrevClientSub s] + cancelServiceSubs serviceId = + checkAnotherClient $ \c -> do + changedSubs@(n, idsHash) <- swapTVar (clientServiceSubs c) (0, mempty) + pure [(c, CSADecreaseSubs changedSubs, (serviceId, ENDS n idsHash))] + checkAnotherClient :: (Client s -> STM [PrevClientSub s]) -> Maybe (Client s) -> STM [PrevClientSub s] + checkAnotherClient mkSub = \case + Just c@Client {clientId, connected} | clntId /= clientId -> + ifM (readTVar connected) (mkSub c) (pure []) + _ -> pure [] + + endPreviousSubscriptions :: [PrevClientSub s] -> IO () + endPreviousSubscriptions = mapM_ $ \(c, subAction, evt) -> do + atomically $ modifyTVar' pendingEvents $ IM.alter (Just . maybe [evt] (evt <|)) (clientId c) + case subAction of + CSAEndSub qId -> atomically (endSub c qId) >>= unsubPrev + -- like endSub, also removes the delivery subscription from the service client's subscriptions map, + -- otherwise the map retains entries for queues unassociated/deleted while the service stays connected. + CSAEndServiceSub qId -> atomically (endServiceQueueSub c qId) >>= unsubPrev + CSADecreaseSubs changedSubs -> do + atomically $ modifyTVar' totalServiceSubs $ subtractServiceSubs changedSubs + forM_ unsub_ $ \unsub -> atomically (swapTVar (clientSubs c) M.empty) >>= mapM_ unsub + where + unsubPrev :: Maybe sub -> IO () + unsubPrev s_ = sequence_ (unsub_ <*> s_) + endSub :: Client s -> QueueId -> STM (Maybe sub) + endSub c qId = TM.lookupDelete qId (clientSubs c) >>= (removeWhenNoSubs c $>) + endServiceQueueSub :: Client s -> QueueId -> STM (Maybe sub) + endServiceQueueSub c qId = do + modifyTVar' (clientServiceSubs c) decrease + modifyTVar' totalServiceSubs decrease + endSub c qId + where + decrease = subtractServiceSubs (1, queueIdHash qId) + -- remove client from server's subscribed cients + removeWhenNoSubs c = do + noClientSubs <- null <$> readTVar (clientSubs c) + noServiceSubs <- ((0 ==) . fst) <$> readTVar (clientServiceSubs c) + when (noClientSubs && noServiceSubs) $ modifyTVar' subClients $ IS.delete (clientId c) + smpServer :: forall s. MsgStoreClass s => TMVar Bool -> ServerConfig s -> Maybe AttachHTTP -> M s () smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOptions} attachHTTP_ = do s <- asks server @@ -185,9 +270,7 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOpt stopServer s liftIO $ exitSuccess raceAny_ - ( serverThread "server subscribers" s subscribers subscriptions serviceSubsCount (Just cancelSub) - : serverThread "server ntfSubscribers" s ntfSubscribers ntfSubscriptions ntfServiceSubsCount Nothing - : deliverNtfsThread s + ( deliverNtfsThread s : sendPendingEvtsThread s : receiveFromProxyAgent pa : expireNtfsThread cfg @@ -248,126 +331,6 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOpt asks (smpAgent . proxyAgent) >>= liftIO . closeSMPClientAgent asks namesEnv >>= liftIO . mapM_ closeNamesEnv - serverThread :: - forall sub. String -> - Server s -> - (Server s -> ServerSubscribers s) -> - (Client s -> TMap QueueId sub) -> - (Client s -> TVar (Int64, IdsHash)) -> - Maybe (sub -> IO ()) -> - M s () - serverThread label srv srvSubscribers clientSubs clientServiceSubs unsub_ = do - labelMyThread label - liftIO . forever $ do - -- Reading clients outside of `updateSubscribers` transaction to avoid transaction re-evaluation on each new connected client. - -- In case client disconnects during the transaction (its `connected` property is read), - -- the transaction will still be re-evaluated, and the client won't be stored as subscribed. - sub@(_, clntId) <- atomically $ readTQueue subQ - c_ <- getServerClient clntId srv - processClientSub srvSubs clientSubs clientServiceSubs unsub_ c_ sub - where - srvSubs@ServerSubscribers {subQ} = srvSubscribers srv - - processClientSub :: - forall sub. - ServerSubscribers s -> - (Client s -> TMap QueueId sub) -> - (Client s -> TVar (Int64, IdsHash)) -> - Maybe (sub -> IO ()) -> - Maybe (Client s) -> - (ClientSub, ClientId) -> - IO () - processClientSub ServerSubscribers {queueSubscribers, serviceSubscribers, totalServiceSubs, subClients, pendingEvents} clientSubs clientServiceSubs unsub_ c_ (clntSub, clntId) = - atomically updateSubscribers - >>= endPreviousSubscriptions - where - updateSubscribers :: STM [PrevClientSub s] - updateSubscribers = case c_ of - Just c@Client {connected} -> ifM (readTVar connected) (updateSubConnected c) updateSubDisconnected - Nothing -> updateSubDisconnected - where - updateSubConnected c = case clntSub of - CSClient qId prevServiceId serviceId_ -> do - modifyTVar' subClients $ IS.insert clntId -- add ID to server's subscribed cients - as'' <- if prevServiceId == serviceId_ then pure [] else endServiceSub prevServiceId qId END - case serviceId_ of - Just serviceId -> do - modifyTVar' totalServiceSubs $ addServiceSubs (1, queueIdHash qId) -- server count and IDs hash for all services - as <- endQueueSub qId END - as' <- cancelServiceSubs serviceId =<< upsertSubscribedClient serviceId c serviceSubscribers - pure $ as ++ as' ++ as'' - Nothing -> do - as <- prevSub qId END (CSAEndSub qId) =<< upsertSubscribedClient qId c queueSubscribers - pure $ as ++ as'' - CSDeleted qId serviceId -> do - removeWhenNoSubs c - as <- endQueueSub qId DELD - as' <- endServiceSub serviceId qId DELD - pure $ as ++ as' - CSService serviceId changedSubs -> do - modifyTVar' subClients $ IS.insert clntId -- add ID to server's subscribed cients - modifyTVar' totalServiceSubs $ addServiceSubs changedSubs -- server count and IDs hash for all services - cancelServiceSubs serviceId =<< upsertSubscribedClient serviceId c serviceSubscribers - updateSubDisconnected = case clntSub of - -- do not insert client if it is already disconnected, but send END/DELD to any other client subscribed to this queue or service - CSClient qId prevServiceId serviceId -> do - as <- endQueueSub qId END - as' <- endServiceSub serviceId qId END - as'' <- if prevServiceId == serviceId then pure [] else endServiceSub prevServiceId qId END - pure $ as ++ as' ++ as'' - CSDeleted qId serviceId -> do - as <- endQueueSub qId DELD - as' <- endServiceSub serviceId qId DELD - pure $ as ++ as' - CSService serviceId _ -> cancelServiceSubs serviceId =<< lookupSubscribedClient serviceId serviceSubscribers - endQueueSub :: QueueId -> BrokerMsg -> STM [PrevClientSub s] - endQueueSub qId msg = prevSub qId msg (CSAEndSub qId) =<< lookupDeleteSubscribedClient qId queueSubscribers - endServiceSub :: Maybe ServiceId -> QueueId -> BrokerMsg -> STM [PrevClientSub s] - endServiceSub Nothing _ _ = pure [] - endServiceSub (Just serviceId) qId msg = prevSub qId msg (CSAEndServiceSub qId) =<< lookupSubscribedClient serviceId serviceSubscribers - prevSub :: QueueId -> BrokerMsg -> ClientSubAction -> Maybe (Client s) -> STM [PrevClientSub s] - prevSub qId msg action = - checkAnotherClient $ \c -> pure [(c, action, (qId, msg))] - cancelServiceSubs :: ServiceId -> Maybe (Client s) -> STM [PrevClientSub s] - cancelServiceSubs serviceId = - checkAnotherClient $ \c -> do - changedSubs@(n, idsHash) <- swapTVar (clientServiceSubs c) (0, mempty) - pure [(c, CSADecreaseSubs changedSubs, (serviceId, ENDS n idsHash))] - checkAnotherClient :: (Client s -> STM [PrevClientSub s]) -> Maybe (Client s) -> STM [PrevClientSub s] - checkAnotherClient mkSub = \case - Just c@Client {clientId, connected} | clntId /= clientId -> - ifM (readTVar connected) (mkSub c) (pure []) - _ -> pure [] - - endPreviousSubscriptions :: [PrevClientSub s] -> IO () - endPreviousSubscriptions = mapM_ $ \(c, subAction, evt) -> do - atomically $ modifyTVar' pendingEvents $ IM.alter (Just . maybe [evt] (evt <|)) (clientId c) - case subAction of - CSAEndSub qId -> atomically (endSub c qId) >>= unsubPrev - -- like endSub, also removes the delivery subscription from the service client's subscriptions map, - -- otherwise the map retains entries for queues unassociated/deleted while the service stays connected. - CSAEndServiceSub qId -> atomically (endServiceQueueSub c qId) >>= unsubPrev - CSADecreaseSubs changedSubs -> do - atomically $ modifyTVar' totalServiceSubs $ subtractServiceSubs changedSubs - forM_ unsub_ $ \unsub -> atomically (swapTVar (clientSubs c) M.empty) >>= mapM_ unsub - where - unsubPrev :: Maybe sub -> IO () - unsubPrev s_ = sequence_ (unsub_ <*> s_) - endSub :: Client s -> QueueId -> STM (Maybe sub) - endSub c qId = TM.lookupDelete qId (clientSubs c) >>= (removeWhenNoSubs c $>) - endServiceQueueSub :: Client s -> QueueId -> STM (Maybe sub) - endServiceQueueSub c qId = do - modifyTVar' (clientServiceSubs c) decrease - modifyTVar' totalServiceSubs decrease - endSub c qId - where - decrease = subtractServiceSubs (1, queueIdHash qId) - -- remove client from server's subscribed cients - removeWhenNoSubs c = do - noClientSubs <- null <$> readTVar (clientSubs c) - noServiceSubs <- ((0 ==) . fst) <$> readTVar (clientServiceSubs c) - when (noClientSubs && noServiceSubs) $ modifyTVar' subClients $ IS.delete (clientId c) - deliverNtfsThread :: Server s -> M s () deliverNtfsThread srv@Server {ntfSubscribers = ServerSubscribers {subClients, serviceSubscribers}} = do ntfInt <- asks $ ntfDeliveryInterval . config @@ -746,7 +709,6 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOpt runClient :: Transport c => X.CertificateChain -> C.APrivateSignKey -> TProxy c 'TServer -> c 'TServer -> M s () runClient srvCert srvSignKey tp h = do ms <- asks msgStore - srvSubs <- asks $ subscribers . server g <- asks random idSize <- asks $ queueIdBytes . config kh <- asks serverIdentity @@ -755,7 +717,7 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOpt let serverInfo = LB.toStrict . J.encode <$> information labelMyThread $ "smp handshake for " <> transportName tp liftIO (timeout smpHandshakeTimeout . runExceptT $ smpServerHandshake srvCert srvSignKey h ks kh smpServerVRange serverInfo $ getClientService ms g idSize) >>= \case - Just (Right th) -> runClientTransport (processClientSub srvSubs subscriptions serviceSubsCount (Just cancelSub)) th + Just (Right th) -> runClientTransport th _ -> pure () getClientService :: s -> TVar ChaChaDRG -> Int -> SMPServiceRole -> X.CertificateChain -> XV.Fingerprint -> ExceptT TransportError IO ServiceId @@ -1073,8 +1035,8 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOpt logError "Unauthorized control port command" hPutStrLn h "AUTH" -runClientTransport :: forall c s. (Transport c, MsgStoreClass s) => (Maybe (Client s) -> (ClientSub, ClientId) -> IO ()) -> THandleSMP c 'TServer -> M s () -runClientTransport processSub h@THandle {params = thParams@THandleParams {sessionId}} = do +runClientTransport :: forall c s. (Transport c, MsgStoreClass s) => THandleSMP c 'TServer -> M s () +runClientTransport h@THandle {params = thParams@THandleParams {sessionId}} = do q <- asks $ tbqSize . config ts <- liftIO getSystemTime nextClientId <- asks clientSeq @@ -1090,7 +1052,7 @@ runClientTransport processSub h@THandle {params = thParams@THandleParams {sessio expCfg <- asks $ inactiveClientExpiration . config th <- newMVar h -- put TH under a fair lock to interleave messages and command responses labelMyThread . B.unpack $ "client $" <> encode sessionId - raceAny_ $ [liftIO $ send th c, liftIO $ sendMsg th c, client processSub s ms c, receive h ms c] <> disconnectThread_ c s expCfg + raceAny_ $ [liftIO $ send th c, liftIO $ sendMsg th c, client s ms c, receive h ms c] <> disconnectThread_ c s expCfg disconnectThread_ :: Client s -> Server s -> Maybe ExpirationConfig -> [M s ()] disconnectThread_ c s (Just expCfg) = [liftIO $ disconnectTransport h (rcvActiveAt c) (sndActiveAt c) expCfg (noSubscriptions c s)] disconnectThread_ _ _ _ = [] @@ -1419,12 +1381,11 @@ forkClient Client {endThreads, endThreadSeq} label action = do action `finally` atomically (modifyTVar' endThreads $ IM.delete tId) mkWeakThreadId t >>= atomically . modifyTVar' endThreads . IM.insert tId -client :: forall s. MsgStoreClass s => (Maybe (Client s) -> (ClientSub, ClientId) -> IO ()) -> Server s -> s -> Client s -> M s () +client :: forall s. MsgStoreClass s => Server s -> s -> Client s -> M s () client - processSub Server {subscribers, ntfSubscribers} ms - clnt@Client {clientId, rcvQ, sndQ, msgQ, clientTHParams = thParams'@THandleParams {sessionId}, procThreads} = do + clnt@Client {rcvQ, sndQ, msgQ, clientTHParams = thParams'@THandleParams {sessionId}, procThreads} = do labelMyThread . B.unpack $ "client $" <> encode sessionId <> " commands" stats <- asks serverStats authDelay <- asks $ authErrorDelay . config @@ -1442,34 +1403,107 @@ client prepareBatchSubs :: Maybe ServiceId -> NonEmpty (VerifiedTransmission s) -> - M s (Either ErrorType (Map RecipientId Message, Map RecipientId (Either ErrorType ()), Map RecipientId (Either ErrorType ()), Map RecipientId Sub)) + M s (Either ErrorType (Map RecipientId Message, Map RecipientId (Either ErrorType ()), Map RecipientId (Either ErrorType ()), Map RecipientId (Either ErrorType (Bool, Maybe Sub)))) prepareBatchSubs clntServiceId_ batch = do - let (subMsgQs, rcvAssocQs, ntfAssocQs, newSubQs) = foldr partitionSubs ([], [], [], []) batch - partitionSubs t (msgQs, rcvQs, ntfQs, newQs) = case t of + let (subQueues, rcvAssocQs, ntfAssocQs) = foldr partitionSubs ([], [], []) batch + partitionSubs t (msgQs, rcvQs, ntfQs) = case t of (Just (q, qr), (_, _, Cmd SRecipient SUB)) - | clntServiceId_ /= rcvServiceId qr -> (q : msgQs, q : rcvQs, ntfQs, newQs) - | isNothing clntServiceId_ && status qr == EntityActive -> (q : msgQs, rcvQs, ntfQs, q : newQs) - | otherwise -> (q : msgQs, rcvQs, ntfQs, newQs) + | clntServiceId_ /= rcvServiceId qr -> ((q, qr) : msgQs, q : rcvQs, ntfQs) + | otherwise -> ((q, qr) : msgQs, rcvQs, ntfQs) (Just (q, qr), (_, _, Cmd SNotifier NSUB)) - | clntServiceId_ /= (notifier qr >>= ntfServiceId) -> (msgQs, rcvQs, q : ntfQs, newQs) - _ -> (msgQs, rcvQs, ntfQs, newQs) - liftIO $ runExceptT $ do - rcvAssocs <- ifNotNull rcvAssocQs $ setService SRecipientService clntServiceId_ - ntfAssocs <- ifNotNull ntfAssocQs $ setService SNotifierService clntServiceId_ - newSubs <- liftIO $ M.fromList . catMaybes <$> mapM subscribeQueue newSubQs - msgs <- ifNotNull subMsgQs $ tryPeekMsgs ms - pure (msgs, rcvAssocs, ntfAssocs, newSubs) + | clntServiceId_ /= (notifier qr >>= ntfServiceId) -> (msgQs, rcvQs, q : ntfQs) + _ -> (msgQs, rcvQs, ntfQs) + liftIO (runExceptT $ (,) <$> ifNotNull rcvAssocQs (setService SRecipientService clntServiceId_) <*> ifNotNull ntfAssocQs (setService SNotifierService clntServiceId_)) >>= \case + Left e -> pure $ Left e + Right (rcvAssocs, ntfAssocs) -> do + subs <- M.fromList <$> mapM (\(q, qr) -> (recipientId q,) <$> subscribeQueue clntServiceId_ q qr) (filter (subscribable rcvAssocs) subQueues) + liftIO $ runExceptT $ do + msgs <- ifNotNull (map fst subQueues) $ tryPeekMsgs ms + pure (msgs, rcvAssocs, ntfAssocs, subs) where ifNotNull qs f = if null qs then pure M.empty else f qs setService :: (PartyI p, ServiceParty p) => SParty p -> Maybe ServiceId -> [StoreQueue s] -> ExceptT ErrorType IO (Map RecipientId (Either ErrorType ())) setService party sId = ExceptT . setQueueServices (queueStore ms) party sId - subscribeQueue :: StoreQueue s -> IO (Maybe (RecipientId, Sub)) - subscribeQueue q = - TM.lookupOrInsertIO rId (newSubscription NoSub) (subscriptions clnt) >>= \case - Left sub -> Just (rId, sub) <$ mask_ (processSub (Just clnt) (CSClient rId Nothing Nothing, clientId)) - Right _ -> pure Nothing + subscribable rcvAssocs (q, QueueRec {status}) = case status of + EntityBlocked _ -> False + _ -> all isRight $ M.lookup (recipientId q) rcvAssocs + + processSub :: ClientSub -> IO () + processSub = processClientSub subscribers subscriptions serviceSubsCount (Just cancelSub) clnt + + processNtfSub :: ClientSub -> IO () + processNtfSub = processClientSub ntfSubscribers ntfSubscriptions ntfServiceSubsCount Nothing clnt + + subscribeQueue :: Maybe ServiceId -> StoreQueue s -> QueueRec -> M s (Either ErrorType (Bool, Maybe Sub)) + subscribeQueue clntServiceId q QueueRec {rcvServiceId} = + liftIO (TM.lookupIO rId $ subscriptions clnt) >>= \case + Nothing -> Right <$> sharedSubscribeQueue clntServiceId rId q rcvServiceId processSub subscriptions serviceSubsCount (newSubscription NoSub) rcvServices + Just s@Sub {subThread} -> do + stats <- asks serverStats + case subThread of + ProhibitSub -> do + -- cannot use SUB in the same connection where GET was used + incStat $ qSubProhibited stats + pure $ Left $ CMD PROHIBITED + _ -> do + incStat $ qSubDuplicate stats + Right (True, Just s) <$ atomically (writeTVar (delivered s) Nothing) + where + rId = recipientId q + + sharedSubscribeQueue :: + Maybe ServiceId -> + QueueId -> + StoreQueue s -> + Maybe ServiceId -> + (ClientSub -> IO ()) -> + (Client s -> TMap QueueId sub) -> + (Client s -> TVar (Int64, IdsHash)) -> + STM sub -> + (ServerStats -> ServiceStats) -> + M s (Bool, Maybe sub) + sharedSubscribeQueue clntServiceId entId q queueServiceId processSub' clientSubs clientServiceSubs mkSub servicesSel = do + stats <- asks serverStats + let incSrvStat sel = incStat $ sel $ servicesSel stats + writeSub = processSub' $ CSClient entId queueServiceId clntServiceId + liftIO $ case clntServiceId of + Just serviceId + | queueServiceId == Just serviceId -> do + -- duplicate queue-service association - can only happen in case of response error/timeout + hasSub <- atomically $ ifM hasServiceSub (pure True) (False <$ incServiceQueueSubs) + unless hasSub $ do + writeSub + incSrvStat srvSubCount + incSrvStat srvSubQueues + incSrvStat srvAssocDuplicate + pure (hasSub, Nothing) + | otherwise -> do + -- association already done in prepareBatchSubs + hasSub <- atomically $ (<$ incServiceQueueSubs) =<< hasServiceSub + writeSub + unless hasSub $ incSrvStat srvSubCount + incSrvStat srvSubQueues + incSrvStat $ maybe srvAssocNew (const srvAssocUpdated) queueServiceId + pure (hasSub, Nothing) where - rId = recipientId q + hasServiceSub = ((0 /=) . fst) <$> readTVar (clientServiceSubs clnt) + -- This function is used when queue association with the service is created. + incServiceQueueSubs = modifyTVar' (clientServiceSubs clnt) $ addServiceSubs (1, queueIdHash (recipientId q)) -- service count and IDS hash + Nothing -> case queueServiceId of + Just _ -> do + -- unassociation already done in prepareBatchSubs + incSrvStat srvAssocRemoved + -- getSubscription may find a subscription for receiving service, where clientSubs also hold active deliveries for service subscriptions. + -- For notification service it can only find one if storage and session states diverge. + r <- getSubscription + writeSub + pure r + Nothing -> do + r@(hasSub, _) <- getSubscription + unless hasSub writeSub + pure r + where + getSubscription = either ((False,) . Just) ((True,) . Just) <$> TM.lookupOrInsertIO entId mkSub (clientSubs clnt) processProxiedCmd :: Transmission (Command 'ProxiedClient) -> M s (Maybe ResponseAndMessage) processProxiedCmd (corrId, EntityId sessId, command) = (\t -> ((corrId, EntityId sessId, t), Nothing)) <$$> case command of @@ -1597,7 +1631,7 @@ client mkIncProxyStats ps psOwn own sel = do incStat $ sel ps when own $ incStat $ sel psOwn - processCommand :: Maybe ServiceId -> Either ErrorType (Map RecipientId Message, Map RecipientId (Either ErrorType ()), Map RecipientId (Either ErrorType ()), Map RecipientId Sub) -> VerifiedTransmission s -> M s (Maybe ResponseAndMessage) + processCommand :: Maybe ServiceId -> Either ErrorType (Map RecipientId Message, Map RecipientId (Either ErrorType ()), Map RecipientId (Either ErrorType ()), Map RecipientId (Either ErrorType (Bool, Maybe Sub))) -> VerifiedTransmission s -> M s (Maybe ResponseAndMessage) processCommand clntServiceId batchSubs (q_, (corrId, entId, cmd)) = case cmd of Cmd SProxiedClient command -> processProxiedCmd (corrId, entId, command) Cmd SSender command -> case command of @@ -1627,9 +1661,9 @@ client pure $ allowNewQueues && maybe True ((== auth_) . Just) newQueueBasicAuth Cmd SRecipient command -> case command of - SUB -> case batchSubs >>= \(msgs, rcvAssocs, _, newSubs) -> sequence (M.lookup entId rcvAssocs) $> (msgs, newSubs) of + SUB -> case batchSubs >>= \(msgs, rcvAssocs, _, subs) -> sequence (M.lookup entId rcvAssocs) $> (msgs, subs) of Left e -> pure $ Just (err e, Nothing) - Right (msgs, newSubs) -> withQueue' $ subscribeQueueAndDeliver (M.lookup entId msgs) (M.lookup entId newSubs) + Right (msgs, subs) -> withQueue' $ deliverSubMsg (M.lookup entId msgs) (M.lookup entId subs) GET -> withQueue getMessage ACK msgId -> withQueue $ acknowledgeMsg msgId KEY sKey -> withQueue $ \q _ -> either err (corrId,entId,) <$> secureQueue_ q sKey @@ -1750,7 +1784,7 @@ client Right nc_ -> do incStat . ntfCreated =<< asks serverStats forM_ nc_ $ \NtfCreds {notifierId = nId, ntfServiceId} -> - atomically $ writeTQueue (subQ ntfSubscribers) (CSDeleted nId ntfServiceId, clientId) + liftIO $ processNtfSub $ CSDeleted nId ntfServiceId pure $ NID notifierId rcvPublicDhKey deleteQueueNotifier_ :: StoreQueue s -> M s (Transmission BrokerMsg) @@ -1761,7 +1795,7 @@ client stats <- asks serverStats deleted <- asks ntfStore >>= liftIO . (`deleteNtfs` nId) when (deleted > 0) $ liftIO $ atomicModifyIORef'_ (ntfCount stats) (subtract deleted) - atomically $ writeTQueue (subQ ntfSubscribers) (CSDeleted nId ntfServiceId, clientId) + liftIO $ processNtfSub $ CSDeleted nId ntfServiceId incStat $ ntfDeleted stats pure ok Right Nothing -> pure ok @@ -1770,22 +1804,11 @@ client suspendQueue_ :: (StoreQueue s, QueueRec) -> M s (Transmission BrokerMsg) suspendQueue_ (q, _) = liftIO $ either err (const ok) <$> suspendQueue (queueStore ms) q - subscribeQueueAndDeliver :: Maybe Message -> Maybe Sub -> StoreQueue s -> QueueRec -> M s ResponseAndMessage - subscribeQueueAndDeliver msg_ newSub_ q qr@QueueRec {rcvServiceId} = case newSub_ of - Just s -> deliver (False, Just s) - Nothing -> liftIO (TM.lookupIO entId $ subscriptions clnt) >>= \case - Nothing -> - deliver =<< sharedSubscribeQueue q rcvServiceId subscribers subscriptions serviceSubsCount (newSubscription NoSub) rcvServices - Just s@Sub {subThread} -> do - stats <- asks serverStats - case subThread of - ProhibitSub -> do - -- cannot use SUB in the same connection where GET was used - incStat $ qSubProhibited stats - pure (err (CMD PROHIBITED), Nothing) - _ -> do - incStat $ qSubDuplicate stats - atomically (writeTVar (delivered s) Nothing) >> deliver (True, Just s) + deliverSubMsg :: Maybe Message -> Maybe (Either ErrorType (Bool, Maybe Sub)) -> StoreQueue s -> QueueRec -> M s ResponseAndMessage + deliverSubMsg msg_ r_ _ qr = case r_ of + Just (Right r) -> deliver r + Just (Left e) -> pure (err e, Nothing) + Nothing -> pure (err INTERNAL, Nothing) where deliver :: (Bool, Maybe Sub) -> M s ResponseAndMessage deliver (hasSub, sub_) = do @@ -1804,15 +1827,13 @@ client getSub = either id id <$> TM.lookupOrInsertIO entId (newSubscription NoSub) (subscriptions clnt) subscribeNewQueue :: RecipientId -> QueueRec -> M s () - subscribeNewQueue rId QueueRec {rcvServiceId} = + subscribeNewQueue rId QueueRec {rcvServiceId} = do case rcvServiceId of - Just _ -> do - atomically $ modifyTVar' (serviceSubsCount clnt) $ addServiceSubs (1, queueIdHash rId) - atomically $ writeTQueue (subQ subscribers) (CSClient rId rcvServiceId rcvServiceId, clientId) + Just _ -> atomically $ modifyTVar' (serviceSubsCount clnt) $ addServiceSubs (1, queueIdHash rId) Nothing -> do sub <- atomically $ newSubscription NoSub atomically $ TM.insert rId sub $ subscriptions clnt - liftIO $ processSub (Just clnt) (CSClient rId Nothing Nothing, clientId) + liftIO $ processSub $ CSClient rId rcvServiceId rcvServiceId -- clients that use GET are not added to server subscribers getMessage :: StoreQueue s -> QueueRec -> M s (Transmission BrokerMsg) @@ -1868,66 +1889,14 @@ client subscribeNotifications :: StoreQueue s -> NtfCreds -> M s BrokerMsg subscribeNotifications q NtfCreds {ntfServiceId} = do - (hasSub, _) <- sharedSubscribeQueue q ntfServiceId ntfSubscribers ntfSubscriptions ntfServiceSubsCount (pure ()) ntfServices + (hasSub, _) <- sharedSubscribeQueue clntServiceId entId q ntfServiceId processNtfSub ntfSubscriptions ntfServiceSubsCount (pure ()) ntfServices when (isNothing clntServiceId) $ asks serverStats >>= incStat . (if hasSub then ntfSubDuplicate else ntfSub) pure $ SOK clntServiceId - sharedSubscribeQueue :: - StoreQueue s -> - Maybe ServiceId -> - ServerSubscribers s -> - (Client s -> TMap QueueId sub) -> - (Client s -> TVar (Int64, IdsHash)) -> - STM sub -> - (ServerStats -> ServiceStats) -> - M s (Bool, Maybe sub) - sharedSubscribeQueue q queueServiceId srvSubscribers clientSubs clientServiceSubs mkSub servicesSel = do - stats <- asks serverStats - let incSrvStat sel = incStat $ sel $ servicesSel stats - writeSub = writeTQueue (subQ srvSubscribers) (CSClient entId queueServiceId clntServiceId, clientId) - liftIO $ case clntServiceId of - Just serviceId - | queueServiceId == Just serviceId -> do - -- duplicate queue-service association - can only happen in case of response error/timeout - hasSub <- atomically $ ifM hasServiceSub (pure True) (False <$ incServiceQueueSubs) - unless hasSub $ do - atomically writeSub - incSrvStat srvSubCount - incSrvStat srvSubQueues - incSrvStat srvAssocDuplicate - pure (hasSub, Nothing) - | otherwise -> do - -- association already done in prepareBatchSubs - hasSub <- atomically $ (<$ incServiceQueueSubs) =<< hasServiceSub - atomically writeSub - unless hasSub $ incSrvStat srvSubCount - incSrvStat srvSubQueues - incSrvStat $ maybe srvAssocNew (const srvAssocUpdated) queueServiceId - pure (hasSub, Nothing) - where - hasServiceSub = ((0 /=) . fst) <$> readTVar (clientServiceSubs clnt) - -- This function is used when queue association with the service is created. - incServiceQueueSubs = modifyTVar' (clientServiceSubs clnt) $ addServiceSubs (1, queueIdHash (recipientId q)) -- service count and IDS hash - Nothing -> case queueServiceId of - Just _ -> do - -- unassociation already done in prepareBatchSubs - incSrvStat srvAssocRemoved - -- getSubscription may find a subscription for receiving service, where clientSubs also hold active deliveries for service subscriptions. - -- For notification service it can only find one if storage and session states diverge. - r <- getSubscription - atomically writeSub - pure r - Nothing -> do - r@(hasSub, _) <- getSubscription - unless hasSub $ atomically writeSub - pure r - where - getSubscription = either ((False,) . Just) ((True,) . Just) <$> TM.lookupOrInsertIO entId mkSub (clientSubs clnt) - subscribeServiceMessages :: ServiceId -> (Int64, IdsHash) -> M s BrokerMsg subscribeServiceMessages serviceId expected = - sharedSubscribeService SRecipientService serviceId expected subscribers serviceSubscribed serviceSubsCount rcvServices >>= \case + sharedSubscribeService SRecipientService serviceId expected processSub serviceSubscribed serviceSubsCount rcvServices >>= \case Left e -> pure $ ERR e Right (hasSub, (count, idsHash)) -> do stats <- asks serverStats @@ -1965,10 +1934,10 @@ client subscribeServiceNotifications :: ServiceId -> (Int64, IdsHash) -> M s BrokerMsg subscribeServiceNotifications serviceId expected = - either ERR (uncurry SOKS . snd) <$> sharedSubscribeService SNotifierService serviceId expected ntfSubscribers ntfServiceSubscribed ntfServiceSubsCount ntfServices + either ERR (uncurry SOKS . snd) <$> sharedSubscribeService SNotifierService serviceId expected processNtfSub ntfServiceSubscribed ntfServiceSubsCount ntfServices - sharedSubscribeService :: (PartyI p, ServiceParty p) => SParty p -> ServiceId -> (Int64, IdsHash) -> ServerSubscribers s -> (Client s -> TVar Bool) -> (Client s -> TVar (Int64, IdsHash)) -> (ServerStats -> ServiceStats) -> M s (Either ErrorType (Bool, (Int64, IdsHash))) - sharedSubscribeService party serviceId (count, idsHash) srvSubscribers clientServiceSubscribed clientServiceSubs servicesSel = do + sharedSubscribeService :: (PartyI p, ServiceParty p) => SParty p -> ServiceId -> (Int64, IdsHash) -> (ClientSub -> IO ()) -> (Client s -> TVar Bool) -> (Client s -> TVar (Int64, IdsHash)) -> (ServerStats -> ServiceStats) -> M s (Either ErrorType (Bool, (Int64, IdsHash))) + sharedSubscribeService party serviceId (count, idsHash) processSub' clientServiceSubscribed clientServiceSubs servicesSel = do subscribed <- readTVarIO $ clientServiceSubscribed clnt stats <- asks serverStats liftIO $ runExceptT $ @@ -1988,7 +1957,7 @@ client | diff > 0 -> incSrvStat srvSubMore 1 >> incSrvStat srvSubMoreTotal diff | diff < 0 -> incSrvStat srvSubFewer 1 >> incSrvStat srvSubFewerTotal (- diff) | otherwise -> incSrvStat srvSubDiff 1 - atomically $ writeTQueue (subQ srvSubscribers) (CSService serviceId subsChange, clientId) + liftIO $ processSub' $ CSService serviceId subsChange pure (count', idsHash') acknowledgeMsg :: MsgId -> StoreQueue s -> QueueRec -> M s (Transmission BrokerMsg) @@ -2287,14 +2256,14 @@ client sub <- atomically $ TM.lookupDelete entId $ subscriptions clnt liftIO $ mapM_ cancelSub sub when (isJust rcvServiceId) $ atomically $ modifyTVar' (serviceSubsCount clnt) $ subtractServiceSubs (1, queueIdHash (recipientId q)) - atomically $ writeTQueue (subQ subscribers) (CSDeleted entId rcvServiceId, clientId) + liftIO $ processSub $ CSDeleted entId rcvServiceId forM_ (notifier qr) $ \NtfCreds {notifierId = nId, ntfServiceId} -> do -- queue is deleted by a different client from the one subscribed to notifications, -- so we don't need to remove subscription from the current client. stats <- asks serverStats deleted <- asks ntfStore >>= liftIO . (`deleteNtfs` nId) when (deleted > 0) $ liftIO $ atomicModifyIORef'_ (ntfCount stats) (subtract deleted) - atomically $ writeTQueue (subQ ntfSubscribers) (CSDeleted nId ntfServiceId, clientId) + liftIO $ processNtfSub $ CSDeleted nId ntfServiceId updateDeletedStats qr pure ok Left e -> pure $ err e diff --git a/src/Simplex/Messaging/Server/Env/STM.hs b/src/Simplex/Messaging/Server/Env/STM.hs index 4c0b1d566..481b2a15a 100644 --- a/src/Simplex/Messaging/Server/Env/STM.hs +++ b/src/Simplex/Messaging/Server/Env/STM.hs @@ -347,8 +347,7 @@ data Server s = Server newtype ServerClients s = ServerClients {serverClients :: TVar (IntMap (Client s))} data ServerSubscribers s = ServerSubscribers - { subQ :: TQueue (ClientSub, ClientId), - queueSubscribers :: SubscribedClients s, + { queueSubscribers :: SubscribedClients s, serviceSubscribers :: SubscribedClients s, -- service clients with long-term certificates that have subscriptions totalServiceSubs :: TVar (Int64, IdsHash), subClients :: TVar IntSet, -- clients with individual or service subscriptions @@ -486,13 +485,12 @@ deleteServerClient cId Server {clients} = atomically $ modifyTVar' (serverClient newServerSubscribers :: IO (ServerSubscribers s) newServerSubscribers = do - subQ <- newTQueueIO queueSubscribers <- SubscribedClients <$> TM.emptyIO serviceSubscribers <- SubscribedClients <$> TM.emptyIO totalServiceSubs <- newTVarIO (0, mempty) subClients <- newTVarIO IS.empty pendingEvents <- newTVarIO IM.empty - pure ServerSubscribers {subQ, queueSubscribers, serviceSubscribers, totalServiceSubs, subClients, pendingEvents} + pure ServerSubscribers {queueSubscribers, serviceSubscribers, totalServiceSubs, subClients, pendingEvents} newClient :: ClientId -> Natural -> THandleParams SMPVersion 'TServer -> SystemTime -> IO (Client s) newClient clientId qSize clientTHParams createdAt = do diff --git a/tests/ServerTests.hs b/tests/ServerTests.hs index 4feb619df..52a63ea3f 100644 --- a/tests/ServerTests.hs +++ b/tests/ServerTests.hs @@ -95,6 +95,7 @@ serverTests = do testServiceDeliverSubscribe testServiceUpgradeAndDowngrade testServiceSubsTotalCount + testServiceSubSendBlock describe "Store log" testWithStoreLog describe "Restore messages" testRestoreMessages describe "Restore messages (old / v2)" testRestoreExpireMessages @@ -1019,6 +1020,31 @@ testServiceSubsTotalCount = threadDelay 1500000 B.readFile testPrometheusMetricsFile >>= \m -> readServiceSubsMetric (B.unpack m) `shouldBe` Just 3 +testServiceSubSendBlock :: SpecWith (ASrvTransport, AStoreType) +testServiceSubSendBlock = + it "should deliver message sent in the same block as service SUB" $ \(at@(ATransport t), msType) -> do + g <- C.newRandom + creds <- genCredentials g Nothing (0, 2400) "localhost" + let (_fp, tlsCred) = tlsCredentials [creds] + serviceKeys@(_, servicePK) <- atomically $ C.generateKeyPair g + withSmpServerConfigOn at (cfgMS msType) testPort $ \_ -> do + (rPub, rKey) <- atomically $ C.generateAuthKeyPair C.SEd25519 g + (dhPub, dhPriv :: C.PrivateKeyX25519) <- atomically $ C.generateKeyPair g + (rId, sId, dec, serviceId) <- runSMPServiceClient t (tlsCred, serviceKeys) $ \sh -> do + Resp "1" NoEntity (Ids_ rId sId srvDh serviceId) <- serviceSignSendRecv sh rKey servicePK ("1", NoEntity, New rPub dhPub) + pure (rId, sId, decryptMsgV3 $ C.dh' srvDh dhPriv, serviceId) + runSMPServiceClient t (tlsCred, serviceKeys) $ \sh@THandle {params} -> do + let TransmissionForAuth {tForAuth, tToSend = subT} = encodeTransmissionForAuth params (CorrId "2", rId, SUB) + TransmissionForAuth {tToSend = sendT} = encodeTransmissionForAuth params (CorrId "3", sId, _SEND "hello") + Right subAuth <- pure $ authTransmission (thAuth params) True (Just rKey) Nothing tForAuth + [Right (), Right ()] <- tPut sh [Right (subAuth, subT), Right (Nothing, sendT)] + Resp "" _ (Msg mId msg) <- tGet1 sh + dec mId msg `shouldBe` Right "hello" + [Resp "2" _ (SOK (Just serviceId')), Resp "3" _ OK] <- tGetClient sh + serviceId' `shouldBe` serviceId + Resp "4" _ OK <- signSendRecv sh rKey ("4", rId, ACK mId) + pure () + readServiceSubsMetric :: String -> Maybe Int readServiceSubsMetric content = case filter ("simplex_smp_subscribtion_service_subs_total " `isPrefixOf`) (lines content) of