mirror of
https://github.com/simplex-chat/simplexmq.git
synced 2026-11-22 19:58:39 +00:00
smp server: move processClientSub back to the place of serverThread, passed to client threads as a parameter
This commit is contained in:
1 parent
2b24e739e6
commit
8e739dbece
2 files changed
+116
-103
No files matched your search
@@ -4,7 +4,7 @@
|
||||
|
||||
- Each subscription change is processed by `processClientSub` under `uninterruptibleMask_`, in the client thread of the connection.
|
||||
- `serverThread` and `subQ` are removed.
|
||||
- The branch of `serverThread` for a disconnected client is omitted from `processClientSub`.
|
||||
- `processClientSub` takes the place of `serverThread` in `smpServer`; its branch for a disconnected client is kept.
|
||||
- SUB, from clients with and without a service certificate: the subscriber is registered in `prepareBatchSubs`, after the queue associations and before `tryPeekMsgs`, except for a service client when the queue is associated with its service and the connection has a service count, as on master.
|
||||
- Duplicate SUB: `delivered` of the subscription is reset in `prepareBatchSubs`, before `tryPeekMsgs`.
|
||||
- SUB of a blocked queue, or of a queue with a failed association: the subscriber is registered only by a later successful SUB.
|
||||
@@ -45,7 +45,7 @@ flowchart LR
|
||||
S[sendPendingEvtsThread]
|
||||
C -- "removed - change written" --> Q
|
||||
Q -- "removed - change read" --> T
|
||||
T -- "removed - client read by getServerClient<br>connected checked" --> X
|
||||
T -- "removed - client read by getServerClient" --> X
|
||||
T -- "removed - run" --> U
|
||||
T -- "removed - run" --> V
|
||||
C -- "added - processClientSub called" --> P
|
||||
@@ -207,11 +207,11 @@ sequenceDiagram
|
||||
## Mechanics
|
||||
|
||||
- `Server.hs`:
|
||||
- `processClientSub`: top level; the registration code of `serverThread`, for the client of the calling thread, under `uninterruptibleMask_`.
|
||||
- `processClientSub`: `serverThread` in its place, for one change of the client of the calling thread, under `uninterruptibleMask_`.
|
||||
- `CSAEndServiceSub`: the service ID added.
|
||||
- `endServiceQueueSub`: the counts decreased only while the previous client is registered for the service.
|
||||
- `smpServer`: `serverThread` removed.
|
||||
- `client`: `processSub` and `processNtfSub` are `processClientSub` for message and notification subscribers.
|
||||
- `smpServer`: the two `serverThread` threads removed.
|
||||
- `runClient`: `processClientSub` for message and notification subscribers passed through `runClientTransport` to `client`, as `processSub` and `processNtfSub`.
|
||||
- `prepareBatchSubs`: one `runExceptT` block:
|
||||
- queue associations
|
||||
- `subscribeQueue` for each SUB of a queue that is not blocked, when its association is unchanged or changed successfully
|
||||
|
||||
+111
-98
@@ -171,92 +171,6 @@ data ClientSubAction
|
||||
|
||||
type PrevClientSub s = (Client s, ClientSubAction, (EntityId, BrokerMsg))
|
||||
|
||||
processClientSub ::
|
||||
forall s sub.
|
||||
ServerSubscribers s ->
|
||||
(Client s -> TMap QueueId sub) ->
|
||||
(Client s -> TVar (Int64, IdsHash)) ->
|
||||
Maybe (sub -> IO ()) ->
|
||||
Client s ->
|
||||
ClientSub ->
|
||||
IO ()
|
||||
processClientSub ServerSubscribers {queueSubscribers, serviceSubscribers, totalServiceSubs, subClients, pendingEvents} clientSubs clientServiceSubs unsub_ clnt@Client {clientId = clntId} clntSub =
|
||||
uninterruptibleMask_ $
|
||||
atomically updateSubscribers
|
||||
>>= endPreviousSubscriptions
|
||||
where
|
||||
updateSubscribers :: STM [PrevClientSub s]
|
||||
updateSubscribers = case clntSub of
|
||||
CSClient qId prevServiceId serviceId_ -> do
|
||||
modifyTVar' subClients $ IS.insert clntId -- add ID to server's subscribed cients
|
||||
as'' <- if prevServiceId == serviceId_ then pure [] else endServiceSub prevServiceId qId END
|
||||
case serviceId_ of
|
||||
Just serviceId -> do
|
||||
modifyTVar' totalServiceSubs $ addServiceSubs (1, queueIdHash qId) -- server count and IDs hash for all services
|
||||
as <- endQueueSub qId END
|
||||
as' <- cancelServiceSubs serviceId =<< upsertSubscribedClient serviceId clnt serviceSubscribers
|
||||
pure $ as ++ as' ++ as''
|
||||
Nothing -> do
|
||||
as <- prevSub qId END (CSAEndSub qId) =<< upsertSubscribedClient qId clnt queueSubscribers
|
||||
pure $ as ++ as''
|
||||
CSDeleted qId serviceId -> do
|
||||
removeWhenNoSubs clnt
|
||||
as <- endQueueSub qId DELD
|
||||
as' <- endServiceSub serviceId qId DELD
|
||||
pure $ as ++ as'
|
||||
CSService serviceId changedSubs -> do
|
||||
modifyTVar' subClients $ IS.insert clntId -- add ID to server's subscribed cients
|
||||
modifyTVar' totalServiceSubs $ addServiceSubs changedSubs -- server count and IDs hash for all services
|
||||
cancelServiceSubs serviceId =<< upsertSubscribedClient serviceId clnt serviceSubscribers
|
||||
endQueueSub :: QueueId -> BrokerMsg -> STM [PrevClientSub s]
|
||||
endQueueSub qId msg = prevSub qId msg (CSAEndSub qId) =<< lookupDeleteSubscribedClient qId queueSubscribers
|
||||
endServiceSub :: Maybe ServiceId -> QueueId -> BrokerMsg -> STM [PrevClientSub s]
|
||||
endServiceSub Nothing _ _ = pure []
|
||||
endServiceSub (Just serviceId) qId msg = prevSub qId msg (CSAEndServiceSub serviceId qId) =<< lookupSubscribedClient serviceId serviceSubscribers
|
||||
prevSub :: QueueId -> BrokerMsg -> ClientSubAction -> Maybe (Client s) -> STM [PrevClientSub s]
|
||||
prevSub qId msg action =
|
||||
checkAnotherClient $ \c -> pure [(c, action, (qId, msg))]
|
||||
cancelServiceSubs :: ServiceId -> Maybe (Client s) -> STM [PrevClientSub s]
|
||||
cancelServiceSubs serviceId =
|
||||
checkAnotherClient $ \c -> do
|
||||
changedSubs@(n, idsHash) <- swapTVar (clientServiceSubs c) (0, mempty)
|
||||
pure [(c, CSADecreaseSubs changedSubs, (serviceId, ENDS n idsHash))]
|
||||
checkAnotherClient :: (Client s -> STM [PrevClientSub s]) -> Maybe (Client s) -> STM [PrevClientSub s]
|
||||
checkAnotherClient mkSub = \case
|
||||
Just c@Client {clientId, connected} | clntId /= clientId ->
|
||||
ifM (readTVar connected) (mkSub c) (pure [])
|
||||
_ -> pure []
|
||||
|
||||
endPreviousSubscriptions :: [PrevClientSub s] -> IO ()
|
||||
endPreviousSubscriptions = mapM_ $ \(c, subAction, evt) -> do
|
||||
atomically $ modifyTVar' pendingEvents $ IM.alter (Just . maybe [evt] (evt <|)) (clientId c)
|
||||
case subAction of
|
||||
CSAEndSub qId -> atomically (endSub c qId) >>= unsubPrev
|
||||
-- like endSub, also removes the delivery subscription from the service client's subscriptions map,
|
||||
-- otherwise the map retains entries for queues unassociated/deleted while the service stays connected.
|
||||
CSAEndServiceSub serviceId qId -> atomically (endServiceQueueSub c serviceId qId) >>= unsubPrev
|
||||
CSADecreaseSubs changedSubs -> do
|
||||
atomically $ modifyTVar' totalServiceSubs $ subtractServiceSubs changedSubs
|
||||
forM_ unsub_ $ \unsub -> atomically (swapTVar (clientSubs c) M.empty) >>= mapM_ unsub
|
||||
where
|
||||
unsubPrev :: Maybe sub -> IO ()
|
||||
unsubPrev s_ = sequence_ (unsub_ <*> s_)
|
||||
endSub :: Client s -> QueueId -> STM (Maybe sub)
|
||||
endSub c qId = TM.lookupDelete qId (clientSubs c) >>= (removeWhenNoSubs c $>)
|
||||
endServiceQueueSub :: Client s -> ServiceId -> QueueId -> STM (Maybe sub)
|
||||
endServiceQueueSub c serviceId qId = do
|
||||
whenM (maybe False (sameClientId c) <$> lookupSubscribedClient serviceId serviceSubscribers) $ do
|
||||
modifyTVar' (clientServiceSubs c) decrease
|
||||
modifyTVar' totalServiceSubs decrease
|
||||
endSub c qId
|
||||
where
|
||||
decrease = subtractServiceSubs (1, queueIdHash qId)
|
||||
-- remove client from server's subscribed cients
|
||||
removeWhenNoSubs c = do
|
||||
noClientSubs <- null <$> readTVar (clientSubs c)
|
||||
noServiceSubs <- ((0 ==) . fst) <$> readTVar (clientServiceSubs c)
|
||||
when (noClientSubs && noServiceSubs) $ modifyTVar' subClients $ IS.delete (clientId c)
|
||||
|
||||
smpServer :: forall s. MsgStoreClass s => TMVar Bool -> ServerConfig s -> Maybe AttachHTTP -> M s ()
|
||||
smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOptions} attachHTTP_ = do
|
||||
s <- asks server
|
||||
@@ -332,6 +246,108 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOpt
|
||||
asks (smpAgent . proxyAgent) >>= liftIO . closeSMPClientAgent
|
||||
asks namesEnv >>= liftIO . mapM_ closeNamesEnv
|
||||
|
||||
processClientSub ::
|
||||
forall sub.
|
||||
Server s ->
|
||||
(Server s -> ServerSubscribers s) ->
|
||||
(Client s -> TMap QueueId sub) ->
|
||||
(Client s -> TVar (Int64, IdsHash)) ->
|
||||
Maybe (sub -> IO ()) ->
|
||||
Client s -> ClientSub -> IO ()
|
||||
processClientSub srv srvSubscribers clientSubs clientServiceSubs unsub_ clnt sub =
|
||||
uninterruptibleMask_ $
|
||||
atomically (updateSubscribers (Just clnt) (sub, clientId clnt))
|
||||
>>= endPreviousSubscriptions
|
||||
where
|
||||
ServerSubscribers {queueSubscribers, serviceSubscribers, totalServiceSubs, subClients, pendingEvents} = srvSubscribers srv
|
||||
updateSubscribers :: Maybe (Client s) -> (ClientSub, ClientId) -> STM [PrevClientSub s]
|
||||
updateSubscribers c_ (clntSub, clntId) = case c_ of
|
||||
Just c@Client {connected} -> ifM (readTVar connected) (updateSubConnected c) updateSubDisconnected
|
||||
Nothing -> updateSubDisconnected
|
||||
where
|
||||
updateSubConnected c = case clntSub of
|
||||
CSClient qId prevServiceId serviceId_ -> do
|
||||
modifyTVar' subClients $ IS.insert clntId -- add ID to server's subscribed cients
|
||||
as'' <- if prevServiceId == serviceId_ then pure [] else endServiceSub prevServiceId qId END
|
||||
case serviceId_ of
|
||||
Just serviceId -> do
|
||||
modifyTVar' totalServiceSubs $ addServiceSubs (1, queueIdHash qId) -- server count and IDs hash for all services
|
||||
as <- endQueueSub qId END
|
||||
as' <- cancelServiceSubs serviceId =<< upsertSubscribedClient serviceId c serviceSubscribers
|
||||
pure $ as ++ as' ++ as''
|
||||
Nothing -> do
|
||||
as <- prevSub qId END (CSAEndSub qId) =<< upsertSubscribedClient qId c queueSubscribers
|
||||
pure $ as ++ as''
|
||||
CSDeleted qId serviceId -> do
|
||||
removeWhenNoSubs c
|
||||
as <- endQueueSub qId DELD
|
||||
as' <- endServiceSub serviceId qId DELD
|
||||
pure $ as ++ as'
|
||||
CSService serviceId changedSubs -> do
|
||||
modifyTVar' subClients $ IS.insert clntId -- add ID to server's subscribed cients
|
||||
modifyTVar' totalServiceSubs $ addServiceSubs changedSubs -- server count and IDs hash for all services
|
||||
cancelServiceSubs serviceId =<< upsertSubscribedClient serviceId c serviceSubscribers
|
||||
updateSubDisconnected = case clntSub of
|
||||
-- do not insert client if it is already disconnected, but send END/DELD to any other client subscribed to this queue or service
|
||||
CSClient qId prevServiceId serviceId -> do
|
||||
as <- endQueueSub qId END
|
||||
as' <- endServiceSub serviceId qId END
|
||||
as'' <- if prevServiceId == serviceId then pure [] else endServiceSub prevServiceId qId END
|
||||
pure $ as ++ as' ++ as''
|
||||
CSDeleted qId serviceId -> do
|
||||
as <- endQueueSub qId DELD
|
||||
as' <- endServiceSub serviceId qId DELD
|
||||
pure $ as ++ as'
|
||||
CSService serviceId _ -> cancelServiceSubs serviceId =<< lookupSubscribedClient serviceId serviceSubscribers
|
||||
endQueueSub :: QueueId -> BrokerMsg -> STM [PrevClientSub s]
|
||||
endQueueSub qId msg = prevSub qId msg (CSAEndSub qId) =<< lookupDeleteSubscribedClient qId queueSubscribers
|
||||
endServiceSub :: Maybe ServiceId -> QueueId -> BrokerMsg -> STM [PrevClientSub s]
|
||||
endServiceSub Nothing _ _ = pure []
|
||||
endServiceSub (Just serviceId) qId msg = prevSub qId msg (CSAEndServiceSub serviceId qId) =<< lookupSubscribedClient serviceId serviceSubscribers
|
||||
prevSub :: QueueId -> BrokerMsg -> ClientSubAction -> Maybe (Client s) -> STM [PrevClientSub s]
|
||||
prevSub qId msg action =
|
||||
checkAnotherClient $ \c -> pure [(c, action, (qId, msg))]
|
||||
cancelServiceSubs :: ServiceId -> Maybe (Client s) -> STM [PrevClientSub s]
|
||||
cancelServiceSubs serviceId =
|
||||
checkAnotherClient $ \c -> do
|
||||
changedSubs@(n, idsHash) <- swapTVar (clientServiceSubs c) (0, mempty)
|
||||
pure [(c, CSADecreaseSubs changedSubs, (serviceId, ENDS n idsHash))]
|
||||
checkAnotherClient :: (Client s -> STM [PrevClientSub s]) -> Maybe (Client s) -> STM [PrevClientSub s]
|
||||
checkAnotherClient mkSub = \case
|
||||
Just c@Client {clientId, connected} | clntId /= clientId ->
|
||||
ifM (readTVar connected) (mkSub c) (pure [])
|
||||
_ -> pure []
|
||||
|
||||
endPreviousSubscriptions :: [PrevClientSub s] -> IO ()
|
||||
endPreviousSubscriptions = mapM_ $ \(c, subAction, evt) -> do
|
||||
atomically $ modifyTVar' pendingEvents $ IM.alter (Just . maybe [evt] (evt <|)) (clientId c)
|
||||
case subAction of
|
||||
CSAEndSub qId -> atomically (endSub c qId) >>= unsubPrev
|
||||
-- like endSub, also removes the delivery subscription from the service client's subscriptions map,
|
||||
-- otherwise the map retains entries for queues unassociated/deleted while the service stays connected.
|
||||
CSAEndServiceSub serviceId qId -> atomically (endServiceQueueSub c serviceId qId) >>= unsubPrev
|
||||
CSADecreaseSubs changedSubs -> do
|
||||
atomically $ modifyTVar' totalServiceSubs $ subtractServiceSubs changedSubs
|
||||
forM_ unsub_ $ \unsub -> atomically (swapTVar (clientSubs c) M.empty) >>= mapM_ unsub
|
||||
where
|
||||
unsubPrev :: Maybe sub -> IO ()
|
||||
unsubPrev s_ = sequence_ (unsub_ <*> s_)
|
||||
endSub :: Client s -> QueueId -> STM (Maybe sub)
|
||||
endSub c qId = TM.lookupDelete qId (clientSubs c) >>= (removeWhenNoSubs c $>)
|
||||
endServiceQueueSub :: Client s -> ServiceId -> QueueId -> STM (Maybe sub)
|
||||
endServiceQueueSub c serviceId qId = do
|
||||
whenM (maybe False (sameClientId c) <$> lookupSubscribedClient serviceId serviceSubscribers) $ do
|
||||
modifyTVar' (clientServiceSubs c) decrease
|
||||
modifyTVar' totalServiceSubs decrease
|
||||
endSub c qId
|
||||
where
|
||||
decrease = subtractServiceSubs (1, queueIdHash qId)
|
||||
-- remove client from server's subscribed cients
|
||||
removeWhenNoSubs c = do
|
||||
noClientSubs <- null <$> readTVar (clientSubs c)
|
||||
noServiceSubs <- ((0 ==) . fst) <$> readTVar (clientServiceSubs c)
|
||||
when (noClientSubs && noServiceSubs) $ modifyTVar' subClients $ IS.delete (clientId c)
|
||||
|
||||
deliverNtfsThread :: Server s -> M s ()
|
||||
deliverNtfsThread srv@Server {ntfSubscribers = ServerSubscribers {subClients, serviceSubscribers}} = do
|
||||
ntfInt <- asks $ ntfDeliveryInterval . config
|
||||
@@ -710,6 +726,7 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOpt
|
||||
runClient :: Transport c => X.CertificateChain -> C.APrivateSignKey -> TProxy c 'TServer -> c 'TServer -> M s ()
|
||||
runClient srvCert srvSignKey tp h = do
|
||||
ms <- asks msgStore
|
||||
srv <- asks server
|
||||
g <- asks random
|
||||
idSize <- asks $ queueIdBytes . config
|
||||
kh <- asks serverIdentity
|
||||
@@ -718,7 +735,7 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOpt
|
||||
let serverInfo = LB.toStrict . J.encode <$> information
|
||||
labelMyThread $ "smp handshake for " <> transportName tp
|
||||
liftIO (timeout smpHandshakeTimeout . runExceptT $ smpServerHandshake srvCert srvSignKey h ks kh smpServerVRange serverInfo $ getClientService ms g idSize) >>= \case
|
||||
Just (Right th) -> runClientTransport th
|
||||
Just (Right th) -> runClientTransport (processClientSub srv subscribers subscriptions serviceSubsCount (Just cancelSub)) (processClientSub srv ntfSubscribers ntfSubscriptions ntfServiceSubsCount Nothing) th
|
||||
_ -> pure ()
|
||||
|
||||
getClientService :: s -> TVar ChaChaDRG -> Int -> SMPServiceRole -> X.CertificateChain -> XV.Fingerprint -> ExceptT TransportError IO ServiceId
|
||||
@@ -1036,8 +1053,8 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOpt
|
||||
logError "Unauthorized control port command"
|
||||
hPutStrLn h "AUTH"
|
||||
|
||||
runClientTransport :: forall c s. (Transport c, MsgStoreClass s) => THandleSMP c 'TServer -> M s ()
|
||||
runClientTransport h@THandle {params = thParams@THandleParams {sessionId}} = do
|
||||
runClientTransport :: forall c s. (Transport c, MsgStoreClass s) => (Client s -> ClientSub -> IO ()) -> (Client s -> ClientSub -> IO ()) -> THandleSMP c 'TServer -> M s ()
|
||||
runClientTransport processSub processNtfSub h@THandle {params = thParams@THandleParams {sessionId}} = do
|
||||
q <- asks $ tbqSize . config
|
||||
ts <- liftIO getSystemTime
|
||||
nextClientId <- asks clientSeq
|
||||
@@ -1053,7 +1070,7 @@ runClientTransport h@THandle {params = thParams@THandleParams {sessionId}} = do
|
||||
expCfg <- asks $ inactiveClientExpiration . config
|
||||
th <- newMVar h -- put TH under a fair lock to interleave messages and command responses
|
||||
labelMyThread . B.unpack $ "client $" <> encode sessionId
|
||||
raceAny_ $ [liftIO $ send th c, liftIO $ sendMsg th c, client s ms c, receive h ms c] <> disconnectThread_ c s expCfg
|
||||
raceAny_ $ [liftIO $ send th c, liftIO $ sendMsg th c, client (processSub c) (processNtfSub c) s ms c, receive h ms c] <> disconnectThread_ c s expCfg
|
||||
disconnectThread_ :: Client s -> Server s -> Maybe ExpirationConfig -> [M s ()]
|
||||
disconnectThread_ c s (Just expCfg) = [liftIO $ disconnectTransport h (rcvActiveAt c) (sndActiveAt c) expCfg (noSubscriptions c s)]
|
||||
disconnectThread_ _ _ _ = []
|
||||
@@ -1382,9 +1399,11 @@ forkClient Client {endThreads, endThreadSeq} label action = do
|
||||
action `finally` atomically (modifyTVar' endThreads $ IM.delete tId)
|
||||
mkWeakThreadId t >>= atomically . modifyTVar' endThreads . IM.insert tId
|
||||
|
||||
client :: forall s. MsgStoreClass s => Server s -> s -> Client s -> M s ()
|
||||
client :: forall s. MsgStoreClass s => (ClientSub -> IO ()) -> (ClientSub -> IO ()) -> Server s -> s -> Client s -> M s ()
|
||||
client
|
||||
Server {subscribers, ntfSubscribers}
|
||||
processSub
|
||||
processNtfSub
|
||||
Server {subscribers}
|
||||
ms
|
||||
clnt@Client {rcvQ, sndQ, msgQ, clientTHParams = thParams'@THandleParams {sessionId}, procThreads} = do
|
||||
labelMyThread . B.unpack $ "client $" <> encode sessionId <> " commands"
|
||||
@@ -1428,12 +1447,6 @@ client
|
||||
EntityBlocked _ -> False
|
||||
_ -> all isRight $ M.lookup (recipientId q) rcvAssocs
|
||||
|
||||
processSub :: ClientSub -> IO ()
|
||||
processSub = processClientSub subscribers subscriptions serviceSubsCount (Just cancelSub) clnt
|
||||
|
||||
processNtfSub :: ClientSub -> IO ()
|
||||
processNtfSub = processClientSub ntfSubscribers ntfSubscriptions ntfServiceSubsCount Nothing clnt
|
||||
|
||||
subscribeQueue :: Maybe ServiceId -> StoreQueue s -> QueueRec -> M s (Either ErrorType (Bool, Maybe Sub))
|
||||
subscribeQueue clntServiceId q QueueRec {rcvServiceId} =
|
||||
liftIO (TM.lookupIO rId $ subscriptions clnt) >>= \case
|
||||
|
||||
Reference in new issue
Block a user