diff --git a/README.md b/README.md index 66dd75fc0..3ece2a4c2 100644 --- a/README.md +++ b/README.md @@ -308,10 +308,6 @@ smp-server delete smp-server init [-l] -n ``` -## SMP server design - -![SMP server design](./design/server.svg) - ## SMP agent design ![SMP agent design](./design/agent2.svg) diff --git a/design/server.gv b/design/server.gv deleted file mode 100644 index e17836311..000000000 --- a/design/server.gv +++ /dev/null @@ -1,57 +0,0 @@ -digraph SMPServer { - graph [fontname=arial] - node [fontname=arial fontsize=11 shape=box] - edge [fontname=arial fontsize=10 arrowhead=open arrowtail=open] - - subgraph clusterPersistence { - label="persistence (STM)" - msgQueues [shape=cylinder label="Message\nqueues"] - queueStore [shape=cylinder label="SMP queue\nrecords"] - } - - subgraph clusterServer { - label="server threads" - main [shape=hexagon color=orange label="main\nthread"] - ss [label="server TCP socket" color=blue] - subgraph clusterThreads { - label="" - node [shape=hexagon color=orange] - runClient [label="runClient\nthread"] - server [label="server\nthread"] - } - subscribedQ [shape="larrow" label="subscribed\nTBQueue"] - main -> {server runClient} [style=dashed label=race color=orange fontcolor=orange] - ss -> runClient [color=blue] - subscribedQ -> server - } - - subgraph clusterConnection { - label="1 group per client connection" - cs [label="client connection TCP socket" color=blue] - - subgraph clusterThreads { - node [shape=hexagon, color=orange] - label="" - receive [label="receive\nthread"] - client [label="client\nthread"] - send [label="send\nthread"] - } - - runClient -> cs [style=dashed label="connect" color=blue fontcolor=blue] - runClient -> {client receive send} [style=dashed label=race color=orange fontcolor=orange] - server -> inq [label="END"] - - subscriber [shape=hexagon color=orange label="subscriber\nthread"] - - inq [shape=rarrow label="receive\nTBQueue"] - outq [shape=larrow label="send\nTBQueue"] - cs -> receive -> inq -> client [color=blue] - msgQueues -> subscriber [label="Message" color=green fontcolor=green] - subscriber -> outq [label="MSG" color=blue fontcolor=blue constraint=false] - client -> queueStore [dir=both] - client -> subscriber [style=dashed label="1 fork per\nSMP queue" color=orange] - client -> msgQueues [dir="both" label="SEND,\nSUB,\nACK" color=green] - client -> outq -> send -> cs [color=blue] - client -> subscribedQ [label="(rId, Client)"] - } -} diff --git a/design/server.svg b/design/server.svg deleted file mode 100644 index ba90fff56..000000000 --- a/design/server.svg +++ /dev/null @@ -1,277 +0,0 @@ - - - - - - -SMPServer - - -clusterPersistence - -persistence (STM) - - -clusterServer - -server threads - - -clusterThreads - - - -clusterConnection - -1 group per client connection - - -clusterThreads - - - - -msgQueues - - -Message -queues - - - -subscriber - -subscriber -thread - - - -msgQueues->subscriber - - -Message - - - -queueStore - - -SMP queue -records - - - -main - -main -thread - - - -runClient - -runClient -thread - - - -main->runClient - - -race - - - -server - -server -thread - - - -main->server - - -race - - - -ss - -server TCP socket - - - -ss->runClient - - - - - -cs - -client connection TCP socket - - - -runClient->cs - - -connect - - - -receive - -receive -thread - - - -runClient->receive - - -race - - - -client - -client -thread - - - -runClient->client - - -race - - - -send - -send -thread - - - -runClient->send - - -race - - - -inq - -receive -TBQueue - - - -server->inq - - -END - - - -subscribedQ - -subscribed -TBQueue - - - -subscribedQ->server - - - - - -cs->receive - - - - - -receive->inq - - - - - -client->msgQueues - - - -SEND, -SUB, -ACK - - - -client->queueStore - - - - - - -client->subscribedQ - - -(rId, Client) - - - -client->subscriber - - -1 fork per -SMP queue - - - -outq - -send -TBQueue - - - -client->outq - - - - - -send->cs - - - - - -inq->client - - - - - -subscriber->outq - - -MSG - - - -outq->send - - - - - diff --git a/plans/2026-10-31-sub-register-first.md b/plans/2026-10-31-sub-register-first.md new file mode 100644 index 000000000..3103050b9 --- /dev/null +++ b/plans/2026-10-31-sub-register-first.md @@ -0,0 +1,240 @@ +# Subscription changes processed in the client thread + +## Decisions + +- Each subscription change is processed by `processClientSub` under `uninterruptibleMask_`, in the client thread of the connection. +- `serverThread` and `subQ` are removed. +- `processClientSub` takes the place of `serverThread` in `smpServer`; its branch for a disconnected client is removed. +- SUB, from clients with and without a service certificate: the subscriber is registered in `prepareBatchSubs`, after the queue associations and before `tryPeekMsgs`, except for a service client when the queue is associated with its service and the connection has a service count, as on master. +- Duplicate SUB: `delivered` of the subscription is reset in `prepareBatchSubs`, before `tryPeekMsgs`. +- SUB of a blocked queue, or of a queue with a failed association: the subscriber is registered only by a later successful SUB. +- For a new subscription, the first message is marked as delivered once, and it is sent only where it is marked, in one of these: + - the MSG pushed after SEND + - the SUB response + - the MSG sent by `deliverServiceMessages` after SUBS +- After a duplicate SUB, a message pushed before `delivered` is reset is marked and sent again, as on master. +- `qSub` is counted for a new subscription when a message is returned by `tryPeekMsgs`, with these exceptions: + - two SUBs of one queue in one block + - an `updateQueueTime` error after the registration +- The service counts of a previous service subscriber are decreased for one queue only while it is registered for the service. +- The subscription changes of each of these commands are processed during that command, in the client thread: + - SUBS + - NSUB + - NSUBS + - NEW with subscription + - NKEY + - NDEL + - DEL +- The SMP server design diagram and its README section are removed. + +## Composition + +Processing of one subscription change. Green: added by this pull request. Red: removed by it. + +```mermaid +flowchart LR + C[Client thread of a connection] + Q[subQ] + T[serverThread] + X[Server clients] + P[processClientSub
under uninterruptibleMask_] + U[updateSubscribers
one STM transaction] + V[endPreviousSubscriptions
later STM transactions] + R[queueSubscribers
serviceSubscribers
totalServiceSubs
subClients
maps and counts of clients] + E[pendingEvents] + S[sendPendingEvtsThread] + C -- "removed - change written" --> Q + Q -- "removed - change read" --> T + T -- "removed - client read by getServerClient" --> X + T -- "removed - run" --> U + T -- "removed - run" --> V + C -- "added - processClientSub called" --> P + P -- "added - run" --> U + P -- "added - run" --> V + U -- "client registered
client added to subClients
totalServiceSubs increased
entries of the previous subscribers removed or replaced
count of the previous service connection swapped to 0
client removed from subClients after a deletion without other subscriptions" --> R + V -- "previous subscriptions removed from client maps
previous client removed from subClients without other subscriptions
counts decreased
delivery threads stopped" --> R + V -- "END
ENDS
DELD" --> E + E -- "events read" --> S + classDef removed fill:#f4cccc,stroke:#cc0000 + classDef added fill:#d9ead3,stroke:#008800 + class Q,T removed + class P added + linkStyle 0,1,2,3,4 stroke:#cc0000 + linkStyle 5,6,7 stroke:#008800 +``` + +## SUB and SEND + +SUB of an empty queue and a SEND to it right after `tryPeekMsgs`. Green: added by this pull request. Red: removed by it. + +```mermaid +sequenceDiagram + participant R as Recipient agent + participant C as Client thread of R + participant Q as Message store + participant T as serverThread, removed + participant S as Client thread of sender + R->>C: SUB q, signature verified in receive + rect rgba(0, 160, 0, 0.18) + C->>C: added - prepareBatchSubs, subscribeQueue: Sub of q inserted + C->>C: added - updateSubscribers under uninterruptibleMask_: R registered in queueSubscribers + C->>C: added - endPreviousSubscriptions: END queued for the previous subscriber + end + C->>Q: prepareBatchSubs: tryPeekMsgs q + Q-->>C: empty queue + S->>Q: SEND m: writeMsg, the queue was empty + rect rgba(220, 0, 0, 0.18) + S->>S: removed - tryDeliverMessage: R not found, m not pushed to R + end + rect rgba(0, 160, 0, 0.18) + S-->>R: added - tryDeliverMessage: R found, MSG m pushed and marked as delivered + end + rect rgba(220, 0, 0, 0.18) + C->>C: removed - subscribeQueueAndDeliver: Sub of q inserted + C->>T: removed - CSClient q written to subQ + T->>T: removed - updateSubscribers: R registered in queueSubscribers + T->>T: removed - endPreviousSubscriptions: END queued for the previous subscriber + end + C-->>R: SOK + rect rgba(220, 0, 0, 0.18) + Note over R,Q: removed - m returned to R only in the response to the next SUB + end + rect rgba(0, 160, 0, 0.18) + Note over R,Q: added - when m is written after the registration and before tryPeekMsgs, m is returned by tryPeekMsgs, marked as delivered once, after SEND or for the SUB response, and sent only where it is marked + end +``` + +## Duplicate SUB + +SUB of a queue that is subscribed in this connection, with message m delivered and not acknowledged. Green: added by this pull request. Red: removed by it. + +```mermaid +sequenceDiagram + participant R as Recipient agent + participant C as Client thread of R + participant Q as Message store + R->>C: SUB q + rect rgba(0, 160, 0, 0.18) + C->>C: added - prepareBatchSubs, subscribeQueue: delivered of q reset + end + C->>Q: prepareBatchSubs: tryPeekMsgs q + Q-->>C: m + rect rgba(220, 0, 0, 0.18) + C->>C: removed - subscribeQueueAndDeliver: delivered of q reset + C->>C: removed - deliver: m marked by setDelivered and included in the SUB response + end + rect rgba(0, 160, 0, 0.18) + C->>C: added - deliver: m included in the SUB response only when it is marked by trySetDelivered + end + C-->>R: SOK, MSG m +``` + +## Duplicate SUB and SEND + +Duplicate SUB of an empty queue, with nothing marked as delivered, and a SEND of m to it before `tryPeekMsgs`. Green: added by this pull request. Red: removed by it. + +```mermaid +sequenceDiagram + participant R as Recipient agent + participant C as Client thread of R + participant Q as Message store + participant S as Client thread of sender + R->>C: SUB q + rect rgba(0, 160, 0, 0.18) + C->>C: added - prepareBatchSubs, subscribeQueue: delivered of q reset + end + S->>Q: SEND m: writeMsg, the queue was empty + S-->>R: tryDeliverMessage: MSG m pushed and marked as delivered + C->>Q: prepareBatchSubs: tryPeekMsgs q + Q-->>C: m + rect rgba(220, 0, 0, 0.18) + C->>C: removed - subscribeQueueAndDeliver: delivered of q reset + C-->>R: removed - SOK, MSG m again + end + rect rgba(0, 160, 0, 0.18) + C-->>R: added - SOK only, as m is marked already + end +``` + +## SUBS + +SUBS of a service connection, and a SEND of m to an empty queue q of the service while its messages are read. Green: added by this pull request. Red: removed by it. + +```mermaid +sequenceDiagram + participant C as Client thread of the service connection + participant T as serverThread, removed + participant F as deliverServiceMessages + participant S as Client thread of sender + rect rgba(220, 0, 0, 0.18) + C->>T: removed - CSService written to subQ, service registered by serverThread + end + rect rgba(0, 160, 0, 0.18) + C->>C: added - processClientSub: service registered + end + C->>F: deliverServiceMessages forked + S->>S: SEND m: writeMsg, the queue was empty + F->>F: foldRcvServiceMessages: m of q read, Sub of q inserted + S-->>C: tryDeliverMessage: Sub of q found, MSG m pushed and marked as delivered + rect rgba(220, 0, 0, 0.18) + F-->>C: removed - m marked by setDelivered, MSG m sent again + end + rect rgba(0, 160, 0, 0.18) + F->>F: added - trySetDelivered: m marked already, MSG omitted, counted in rcvServicesSubDuplicate + end +``` + +## Service counts + +A SUB by which the association of queue q with service S is removed, and SUBS from a new connection N of S. O is the previous connection of S. Green: added by this pull request. Red: removed by it. + +```mermaid +sequenceDiagram + participant D as Client thread of the SUB + participant N as Client thread of N + participant G as Registries + D->>G: updateSubscribers: O recorded for CSAEndServiceSub S q + N->>G: updateSubscribers: N registered for S, the count of O swapped to 0 + N->>G: endPreviousSubscriptions: the swapped count subtracted from totalServiceSubs + rect rgba(220, 0, 0, 0.18) + D->>G: removed - endServiceQueueSub: the count of O and totalServiceSubs decreased for q again + end + rect rgba(0, 160, 0, 0.18) + D->>G: added - endServiceQueueSub: O not registered for S, counts unchanged + end +``` + +## Mechanics + +- `Server.hs`: + - `processClientSub`: `serverThread` in its place, for one change of the client of the calling thread, under `uninterruptibleMask_`. + - `CSAEndServiceSub`: the service ID added. + - `endServiceQueueSub`: the counts decreased only while the previous client is registered for the service. + - `smpServer`: the two `serverThread` threads removed. + - `runClient`: `processSubs`, the pair of `processClientSub` for message and notification subscribers, passed through `runClientTransport` to `client`. + - `client`: `processSub` and `processNtfSub` from the pair. + - `prepareBatchSubs`: one `runExceptT` block: + - queue associations + - `subscribeQueue` for each SUB of a queue that is not blocked, when its association is unchanged or changed successfully + - `tryPeekMsgs` + - the `subscribeQueue` results returned as the fourth map of the batch result + - `subscribeQueue`: the first part of the former `subscribeQueueAndDeliver`. + - `deliverSubMsg`: the second part of the former `subscribeQueueAndDeliver`; the `subscribeQueue` result passed to `deliver`. + - `sharedSubscribeQueue`: moved from `processCommand` to `client`; the service ID of the client and the queue ID added as parameters; `processSub` or `processNtfSub` passed in place of the server subscribers. + - `processSub` or `processNtfSub` called with the client and the subscription change where the change was written to `subQ`: + - `sharedSubscribeService` + - `subscribeNewQueue` + - `addQueueNotifier_` + - `deleteQueueNotifier_` + - `delQueueAndMsgs` + - `deliver`: `qSub` counted before `trySetDelivered`; the first message included only when it is marked by `trySetDelivered`. + - `deliverServiceMessages`: MSG sent only when the message is marked by `trySetDelivered`. +- `Env/STM.hs`: `subQ` removed from `ServerSubscribers`. +- `README.md`: the section "SMP server design" removed. +- `design/server.gv`, `design/server.svg`: removed. +- `tests/ServerTests.hs`: + - `should deliver message sent in the same block as SUB` + - `should deliver message sent in the same block as SUB to the new connection` + - `should deliver message once for two SUBs in the same block` + - `should deliver message sent in the same block as service SUB` + - `should deliver message after SUB while delivery waits for full send queue`: the message recorded before or after SOK. diff --git a/src/Simplex/Messaging/Server.hs b/src/Simplex/Messaging/Server.hs index ec656a685..414dc1f59 100644 --- a/src/Simplex/Messaging/Server.hs +++ b/src/Simplex/Messaging/Server.hs @@ -61,7 +61,7 @@ import qualified Data.ByteString.Char8 as B import qualified Data.ByteString.Lazy.Char8 as LB import Data.Constraint (Dict (..)) import Data.Dynamic (toDyn) -import Data.Either (fromRight, partitionEithers) +import Data.Either (fromRight, isRight, partitionEithers) import Data.Foldable (foldrM) import Data.Functor (($>), (<&>)) import Data.IORef @@ -163,10 +163,10 @@ runSMPServerBlocking started cfg attachHTTP_ = newEnv cfg >>= runReaderT (smpSer type M s a = ReaderT (Env s) IO a type AttachHTTP = Socket -> TLS.Context -> IO () --- actions used in serverThread to reduce STM transaction scope +-- actions used in processClientSub to reduce STM transaction scope data ClientSubAction = CSAEndSub QueueId -- end single direct queue subscription - | CSAEndServiceSub QueueId -- end service subscription to one queue + | CSAEndServiceSub ServiceId QueueId -- end service subscription to one queue | CSADecreaseSubs (Int64, IdsHash) -- reduce service subscriptions when cancelling. Fixed number is used to correctly handle race conditions when service resubscribes type PrevClientSub s = (Client s, ClientSubAction, (EntityId, BrokerMsg)) @@ -185,9 +185,7 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOpt stopServer s liftIO $ exitSuccess raceAny_ - ( serverThread "server subscribers" s subscribers subscriptions serviceSubsCount (Just cancelSub) - : serverThread "server ntfSubscribers" s ntfSubscribers ntfSubscriptions ntfServiceSubsCount Nothing - : deliverNtfsThread s + ( deliverNtfsThread s : sendPendingEvtsThread s : receiveFromProxyAgent pa : expireNtfsThread cfg @@ -248,70 +246,49 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOpt asks (smpAgent . proxyAgent) >>= liftIO . closeSMPClientAgent asks namesEnv >>= liftIO . mapM_ closeNamesEnv - serverThread :: - forall sub. String -> + processClientSub :: + forall sub. Server s -> (Server s -> ServerSubscribers s) -> (Client s -> TMap QueueId sub) -> (Client s -> TVar (Int64, IdsHash)) -> Maybe (sub -> IO ()) -> - M s () - serverThread label srv srvSubscribers clientSubs clientServiceSubs unsub_ = do - labelMyThread label - liftIO . forever $ do - -- Reading clients outside of `updateSubscribers` transaction to avoid transaction re-evaluation on each new connected client. - -- In case client disconnects during the transaction (its `connected` property is read), - -- the transaction will still be re-evaluated, and the client won't be stored as subscribed. - sub@(_, clntId) <- atomically $ readTQueue subQ - c_ <- getServerClient clntId srv - atomically (updateSubscribers c_ sub) + Client s -> ClientSub -> IO () + processClientSub srv srvSubscribers clientSubs clientServiceSubs unsub_ clnt sub = + uninterruptibleMask_ $ + atomically (updateSubscribers clnt sub) >>= endPreviousSubscriptions where - ServerSubscribers {subQ, queueSubscribers, serviceSubscribers, totalServiceSubs, subClients, pendingEvents} = srvSubscribers srv - updateSubscribers :: Maybe (Client s) -> (ClientSub, ClientId) -> STM [PrevClientSub s] - updateSubscribers c_ (clntSub, clntId) = case c_ of - Just c@Client {connected} -> ifM (readTVar connected) (updateSubConnected c) updateSubDisconnected - Nothing -> updateSubDisconnected + ServerSubscribers {queueSubscribers, serviceSubscribers, totalServiceSubs, subClients, pendingEvents} = srvSubscribers srv + updateSubscribers :: Client s -> ClientSub -> STM [PrevClientSub s] + updateSubscribers c@Client {clientId = clntId} clntSub = case clntSub of + CSClient qId prevServiceId serviceId_ -> do + modifyTVar' subClients $ IS.insert clntId -- add ID to server's subscribed cients + as'' <- if prevServiceId == serviceId_ then pure [] else endServiceSub prevServiceId qId END + case serviceId_ of + Just serviceId -> do + modifyTVar' totalServiceSubs $ addServiceSubs (1, queueIdHash qId) -- server count and IDs hash for all services + as <- endQueueSub qId END + as' <- cancelServiceSubs serviceId =<< upsertSubscribedClient serviceId c serviceSubscribers + pure $ as ++ as' ++ as'' + Nothing -> do + as <- prevSub qId END (CSAEndSub qId) =<< upsertSubscribedClient qId c queueSubscribers + pure $ as ++ as'' + CSDeleted qId serviceId -> do + removeWhenNoSubs c + as <- endQueueSub qId DELD + as' <- endServiceSub serviceId qId DELD + pure $ as ++ as' + CSService serviceId changedSubs -> do + modifyTVar' subClients $ IS.insert clntId -- add ID to server's subscribed cients + modifyTVar' totalServiceSubs $ addServiceSubs changedSubs -- server count and IDs hash for all services + cancelServiceSubs serviceId =<< upsertSubscribedClient serviceId c serviceSubscribers where - updateSubConnected c = case clntSub of - CSClient qId prevServiceId serviceId_ -> do - modifyTVar' subClients $ IS.insert clntId -- add ID to server's subscribed cients - as'' <- if prevServiceId == serviceId_ then pure [] else endServiceSub prevServiceId qId END - case serviceId_ of - Just serviceId -> do - modifyTVar' totalServiceSubs $ addServiceSubs (1, queueIdHash qId) -- server count and IDs hash for all services - as <- endQueueSub qId END - as' <- cancelServiceSubs serviceId =<< upsertSubscribedClient serviceId c serviceSubscribers - pure $ as ++ as' ++ as'' - Nothing -> do - as <- prevSub qId END (CSAEndSub qId) =<< upsertSubscribedClient qId c queueSubscribers - pure $ as ++ as'' - CSDeleted qId serviceId -> do - removeWhenNoSubs c - as <- endQueueSub qId DELD - as' <- endServiceSub serviceId qId DELD - pure $ as ++ as' - CSService serviceId changedSubs -> do - modifyTVar' subClients $ IS.insert clntId -- add ID to server's subscribed cients - modifyTVar' totalServiceSubs $ addServiceSubs changedSubs -- server count and IDs hash for all services - cancelServiceSubs serviceId =<< upsertSubscribedClient serviceId c serviceSubscribers - updateSubDisconnected = case clntSub of - -- do not insert client if it is already disconnected, but send END/DELD to any other client subscribed to this queue or service - CSClient qId prevServiceId serviceId -> do - as <- endQueueSub qId END - as' <- endServiceSub serviceId qId END - as'' <- if prevServiceId == serviceId then pure [] else endServiceSub prevServiceId qId END - pure $ as ++ as' ++ as'' - CSDeleted qId serviceId -> do - as <- endQueueSub qId DELD - as' <- endServiceSub serviceId qId DELD - pure $ as ++ as' - CSService serviceId _ -> cancelServiceSubs serviceId =<< lookupSubscribedClient serviceId serviceSubscribers endQueueSub :: QueueId -> BrokerMsg -> STM [PrevClientSub s] endQueueSub qId msg = prevSub qId msg (CSAEndSub qId) =<< lookupDeleteSubscribedClient qId queueSubscribers endServiceSub :: Maybe ServiceId -> QueueId -> BrokerMsg -> STM [PrevClientSub s] endServiceSub Nothing _ _ = pure [] - endServiceSub (Just serviceId) qId msg = prevSub qId msg (CSAEndServiceSub qId) =<< lookupSubscribedClient serviceId serviceSubscribers + endServiceSub (Just serviceId) qId msg = prevSub qId msg (CSAEndServiceSub serviceId qId) =<< lookupSubscribedClient serviceId serviceSubscribers prevSub :: QueueId -> BrokerMsg -> ClientSubAction -> Maybe (Client s) -> STM [PrevClientSub s] prevSub qId msg action = checkAnotherClient $ \c -> pure [(c, action, (qId, msg))] @@ -333,7 +310,7 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOpt CSAEndSub qId -> atomically (endSub c qId) >>= unsubPrev -- like endSub, also removes the delivery subscription from the service client's subscriptions map, -- otherwise the map retains entries for queues unassociated/deleted while the service stays connected. - CSAEndServiceSub qId -> atomically (endServiceQueueSub c qId) >>= unsubPrev + CSAEndServiceSub serviceId qId -> atomically (endServiceQueueSub c serviceId qId) >>= unsubPrev CSADecreaseSubs changedSubs -> do atomically $ modifyTVar' totalServiceSubs $ subtractServiceSubs changedSubs forM_ unsub_ $ \unsub -> atomically (swapTVar (clientSubs c) M.empty) >>= mapM_ unsub @@ -342,10 +319,11 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOpt unsubPrev s_ = sequence_ (unsub_ <*> s_) endSub :: Client s -> QueueId -> STM (Maybe sub) endSub c qId = TM.lookupDelete qId (clientSubs c) >>= (removeWhenNoSubs c $>) - endServiceQueueSub :: Client s -> QueueId -> STM (Maybe sub) - endServiceQueueSub c qId = do - modifyTVar' (clientServiceSubs c) decrease - modifyTVar' totalServiceSubs decrease + endServiceQueueSub :: Client s -> ServiceId -> QueueId -> STM (Maybe sub) + endServiceQueueSub c serviceId qId = do + whenM (maybe False (sameClientId c) <$> lookupSubscribedClient serviceId serviceSubscribers) $ do + modifyTVar' (clientServiceSubs c) decrease + modifyTVar' totalServiceSubs decrease endSub c qId where decrease = subtractServiceSubs (1, queueIdHash qId) @@ -733,6 +711,7 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOpt runClient :: Transport c => X.CertificateChain -> C.APrivateSignKey -> TProxy c 'TServer -> c 'TServer -> M s () runClient srvCert srvSignKey tp h = do ms <- asks msgStore + srv <- asks server g <- asks random idSize <- asks $ queueIdBytes . config kh <- asks serverIdentity @@ -741,8 +720,10 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOpt let serverInfo = LB.toStrict . J.encode <$> information labelMyThread $ "smp handshake for " <> transportName tp liftIO (timeout smpHandshakeTimeout . runExceptT $ smpServerHandshake srvCert srvSignKey h ks kh smpServerVRange serverInfo $ getClientService ms g idSize) >>= \case - Just (Right th) -> runClientTransport th + Just (Right th) -> runClientTransport (processSubs srv) th _ -> pure () + where + processSubs srv = (processClientSub srv subscribers subscriptions serviceSubsCount (Just cancelSub), processClientSub srv ntfSubscribers ntfSubscriptions ntfServiceSubsCount Nothing) getClientService :: s -> TVar ChaChaDRG -> Int -> SMPServiceRole -> X.CertificateChain -> XV.Fingerprint -> ExceptT TransportError IO ServiceId getClientService ms g idSize role cert fp = do @@ -1059,8 +1040,8 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOpt logError "Unauthorized control port command" hPutStrLn h "AUTH" -runClientTransport :: forall c s. (Transport c, MsgStoreClass s) => THandleSMP c 'TServer -> M s () -runClientTransport h@THandle {params = thParams@THandleParams {sessionId}} = do +runClientTransport :: forall c s. (Transport c, MsgStoreClass s) => (Client s -> ClientSub -> IO (), Client s -> ClientSub -> IO ()) -> THandleSMP c 'TServer -> M s () +runClientTransport processSubs h@THandle {params = thParams@THandleParams {sessionId}} = do q <- asks $ tbqSize . config ts <- liftIO getSystemTime nextClientId <- asks clientSeq @@ -1076,7 +1057,7 @@ runClientTransport h@THandle {params = thParams@THandleParams {sessionId}} = do expCfg <- asks $ inactiveClientExpiration . config th <- newMVar h -- put TH under a fair lock to interleave messages and command responses labelMyThread . B.unpack $ "client $" <> encode sessionId - raceAny_ $ [liftIO $ send th c, liftIO $ sendMsg th c, client s ms c, receive h ms c] <> disconnectThread_ c s expCfg + raceAny_ $ [liftIO $ send th c, liftIO $ sendMsg th c, client processSubs s ms c, receive h ms c] <> disconnectThread_ c s expCfg disconnectThread_ :: Client s -> Server s -> Maybe ExpirationConfig -> [M s ()] disconnectThread_ c s (Just expCfg) = [liftIO $ disconnectTransport h (rcvActiveAt c) (sndActiveAt c) expCfg (noSubscriptions c s)] disconnectThread_ _ _ _ = [] @@ -1405,11 +1386,12 @@ forkClient Client {endThreads, endThreadSeq} label action = do action `finally` atomically (modifyTVar' endThreads $ IM.delete tId) mkWeakThreadId t >>= atomically . modifyTVar' endThreads . IM.insert tId -client :: forall s. MsgStoreClass s => Server s -> s -> Client s -> M s () +client :: forall s. MsgStoreClass s => (Client s -> ClientSub -> IO (), Client s -> ClientSub -> IO ()) -> Server s -> s -> Client s -> M s () client - Server {subscribers, ntfSubscribers} + (processSub, processNtfSub) + Server {subscribers} ms - clnt@Client {clientId, rcvQ, sndQ, msgQ, clientTHParams = thParams'@THandleParams {sessionId}, procThreads} = do + clnt@Client {rcvQ, sndQ, msgQ, clientTHParams = thParams'@THandleParams {sessionId}, procThreads} = do labelMyThread . B.unpack $ "client $" <> encode sessionId <> " commands" stats <- asks serverStats authDelay <- asks $ authErrorDelay . config @@ -1427,25 +1409,100 @@ client prepareBatchSubs :: Maybe ServiceId -> NonEmpty (VerifiedTransmission s) -> - M s (Either ErrorType (Map RecipientId Message, Map RecipientId (Either ErrorType ()), Map RecipientId (Either ErrorType ()))) + M s (Either ErrorType (Map RecipientId Message, Map RecipientId (Either ErrorType ()), Map RecipientId (Either ErrorType ()), Map RecipientId (Either ErrorType (Bool, Maybe Sub)))) prepareBatchSubs clntServiceId_ batch = do let (subMsgQs, rcvAssocQs, ntfAssocQs) = foldr partitionSubs ([], [], []) batch partitionSubs t (msgQs, rcvQs, ntfQs) = case t of (Just (q, qr), (_, _, Cmd SRecipient SUB)) - | clntServiceId_ /= rcvServiceId qr -> (q : msgQs, q : rcvQs, ntfQs) - | otherwise -> (q : msgQs, rcvQs, ntfQs) + | clntServiceId_ /= rcvServiceId qr -> ((q, qr) : msgQs, q : rcvQs, ntfQs) + | otherwise -> ((q, qr) : msgQs, rcvQs, ntfQs) (Just (q, qr), (_, _, Cmd SNotifier NSUB)) | clntServiceId_ /= (notifier qr >>= ntfServiceId) -> (msgQs, rcvQs, q : ntfQs) _ -> (msgQs, rcvQs, ntfQs) - liftIO $ runExceptT $ do + runExceptT $ do rcvAssocs <- ifNotNull rcvAssocQs $ setService SRecipientService clntServiceId_ ntfAssocs <- ifNotNull ntfAssocQs $ setService SNotifierService clntServiceId_ - msgs <- ifNotNull subMsgQs $ tryPeekMsgs ms - pure (msgs, rcvAssocs, ntfAssocs) + subs <- lift $ M.fromList <$> mapM (\(q, qr) -> (recipientId q,) <$> subscribeQueue clntServiceId_ q qr) (filter (subscribable rcvAssocs) subMsgQs) + msgs <- ifNotNull (map fst subMsgQs) $ mapExceptT liftIO . tryPeekMsgs ms + pure (msgs, rcvAssocs, ntfAssocs, subs) where ifNotNull qs f = if null qs then pure M.empty else f qs - setService :: (PartyI p, ServiceParty p) => SParty p -> Maybe ServiceId -> [StoreQueue s] -> ExceptT ErrorType IO (Map RecipientId (Either ErrorType ())) - setService party sId = ExceptT . setQueueServices (queueStore ms) party sId + setService :: (PartyI p, ServiceParty p) => SParty p -> Maybe ServiceId -> [StoreQueue s] -> ExceptT ErrorType (ReaderT (Env s) IO) (Map RecipientId (Either ErrorType ())) + setService party sId = ExceptT . liftIO . setQueueServices (queueStore ms) party sId + subscribable rcvAssocs (q, QueueRec {status}) = case status of + EntityBlocked _ -> False + _ -> all isRight $ M.lookup (recipientId q) rcvAssocs + + subscribeQueue :: Maybe ServiceId -> StoreQueue s -> QueueRec -> M s (Either ErrorType (Bool, Maybe Sub)) + subscribeQueue clntServiceId q QueueRec {rcvServiceId} = + liftIO (TM.lookupIO rId $ subscriptions clnt) >>= \case + Nothing -> Right <$> sharedSubscribeQueue clntServiceId rId q rcvServiceId processSub subscriptions serviceSubsCount (newSubscription NoSub) rcvServices + Just s@Sub {subThread} -> do + stats <- asks serverStats + case subThread of + ProhibitSub -> do + -- cannot use SUB in the same connection where GET was used + incStat $ qSubProhibited stats + pure $ Left $ CMD PROHIBITED + _ -> do + incStat $ qSubDuplicate stats + Right (True, Just s) <$ atomically (writeTVar (delivered s) Nothing) + where + rId = recipientId q + + sharedSubscribeQueue :: + Maybe ServiceId -> + QueueId -> + StoreQueue s -> + Maybe ServiceId -> + (Client s -> ClientSub -> IO ()) -> + (Client s -> TMap QueueId sub) -> + (Client s -> TVar (Int64, IdsHash)) -> + STM sub -> + (ServerStats -> ServiceStats) -> + M s (Bool, Maybe sub) + sharedSubscribeQueue clntServiceId entId q queueServiceId processSub' clientSubs clientServiceSubs mkSub servicesSel = do + stats <- asks serverStats + let incSrvStat sel = incStat $ sel $ servicesSel stats + writeSub = processSub' clnt $ CSClient entId queueServiceId clntServiceId + liftIO $ case clntServiceId of + Just serviceId + | queueServiceId == Just serviceId -> do + -- duplicate queue-service association - can only happen in case of response error/timeout + hasSub <- atomically $ ifM hasServiceSub (pure True) (False <$ incServiceQueueSubs) + unless hasSub $ do + writeSub + incSrvStat srvSubCount + incSrvStat srvSubQueues + incSrvStat srvAssocDuplicate + pure (hasSub, Nothing) + | otherwise -> do + -- association already done in prepareBatchSubs + hasSub <- atomically $ (<$ incServiceQueueSubs) =<< hasServiceSub + writeSub + unless hasSub $ incSrvStat srvSubCount + incSrvStat srvSubQueues + incSrvStat $ maybe srvAssocNew (const srvAssocUpdated) queueServiceId + pure (hasSub, Nothing) + where + hasServiceSub = ((0 /=) . fst) <$> readTVar (clientServiceSubs clnt) + -- This function is used when queue association with the service is created. + incServiceQueueSubs = modifyTVar' (clientServiceSubs clnt) $ addServiceSubs (1, queueIdHash (recipientId q)) -- service count and IDS hash + Nothing -> case queueServiceId of + Just _ -> do + -- unassociation already done in prepareBatchSubs + incSrvStat srvAssocRemoved + -- getSubscription may find a subscription for receiving service, where clientSubs also hold active deliveries for service subscriptions. + -- For notification service it can only find one if storage and session states diverge. + r <- getSubscription + writeSub + pure r + Nothing -> do + r@(hasSub, _) <- getSubscription + unless hasSub writeSub + pure r + where + getSubscription = either ((False,) . Just) ((True,) . Just) <$> TM.lookupOrInsertIO entId mkSub (clientSubs clnt) processProxiedCmd :: Transmission (Command 'ProxiedClient) -> M s (Maybe ResponseAndMessage) processProxiedCmd (corrId, EntityId sessId, command) = (\t -> ((corrId, EntityId sessId, t), Nothing)) <$$> case command of @@ -1573,7 +1630,7 @@ client mkIncProxyStats ps psOwn own sel = do incStat $ sel ps when own $ incStat $ sel psOwn - processCommand :: Maybe ServiceId -> Either ErrorType (Map RecipientId Message, Map RecipientId (Either ErrorType ()), Map RecipientId (Either ErrorType ())) -> VerifiedTransmission s -> M s (Maybe ResponseAndMessage) + processCommand :: Maybe ServiceId -> Either ErrorType (Map RecipientId Message, Map RecipientId (Either ErrorType ()), Map RecipientId (Either ErrorType ()), Map RecipientId (Either ErrorType (Bool, Maybe Sub))) -> VerifiedTransmission s -> M s (Maybe ResponseAndMessage) processCommand clntServiceId batchSubs (q_, (corrId, entId, cmd)) = case cmd of Cmd SProxiedClient command -> processProxiedCmd (corrId, entId, command) Cmd SSender command -> case command of @@ -1590,7 +1647,7 @@ client Cmd SNotifier NSUB -> response . (corrId,entId,) <$> case q_ of Just (q, QueueRec {notifier = Just ntfCreds}) -> either (pure . ERR) (\_ -> subscribeNotifications q ntfCreds) - $ batchSubs >>= \(_, _, ntfAssocs) -> sequence (M.lookup (recipientId q) ntfAssocs) + $ batchSubs >>= \(_, _, ntfAssocs, _) -> sequence (M.lookup (recipientId q) ntfAssocs) _ -> pure $ ERR INTERNAL Cmd SNotifierService (NSUBS n idsHash) -> response . (corrId,entId,) <$> case clntServiceId of Just serviceId -> subscribeServiceNotifications serviceId (n, idsHash) @@ -1603,9 +1660,9 @@ client pure $ allowNewQueues && maybe True ((== auth_) . Just) newQueueBasicAuth Cmd SRecipient command -> case command of - SUB -> case batchSubs >>= \(msgs, rcvAssocs, _) -> sequence (M.lookup entId rcvAssocs) $> msgs of + SUB -> case batchSubs >>= \(msgs, rcvAssocs, _, subs) -> sequence (M.lookup entId rcvAssocs) $> (msgs, subs) of Left e -> pure $ Just (err e, Nothing) - Right msgs -> withQueue' $ subscribeQueueAndDeliver $ M.lookup entId msgs + Right (msgs, subs) -> withQueue' $ deliverSubMsg (M.lookup entId msgs) (M.lookup entId subs) GET -> withQueue getMessage ACK msgId -> withQueue $ acknowledgeMsg msgId KEY sKey -> withQueue $ \q _ -> either err (corrId,entId,) <$> secureQueue_ q sKey @@ -1726,7 +1783,7 @@ client Right nc_ -> do incStat . ntfCreated =<< asks serverStats forM_ nc_ $ \NtfCreds {notifierId = nId, ntfServiceId} -> - atomically $ writeTQueue (subQ ntfSubscribers) (CSDeleted nId ntfServiceId, clientId) + liftIO $ processNtfSub clnt $ CSDeleted nId ntfServiceId pure $ NID notifierId rcvPublicDhKey deleteQueueNotifier_ :: StoreQueue s -> M s (Transmission BrokerMsg) @@ -1737,7 +1794,7 @@ client stats <- asks serverStats deleted <- asks ntfStore >>= liftIO . (`deleteNtfs` nId) when (deleted > 0) $ liftIO $ atomicModifyIORef'_ (ntfCount stats) (subtract deleted) - atomically $ writeTQueue (subQ ntfSubscribers) (CSDeleted nId ntfServiceId, clientId) + liftIO $ processNtfSub clnt $ CSDeleted nId ntfServiceId incStat $ ntfDeleted stats pure ok Right Nothing -> pure ok @@ -1746,32 +1803,24 @@ client suspendQueue_ :: (StoreQueue s, QueueRec) -> M s (Transmission BrokerMsg) suspendQueue_ (q, _) = liftIO $ either err (const ok) <$> suspendQueue (queueStore ms) q - subscribeQueueAndDeliver :: Maybe Message -> StoreQueue s -> QueueRec -> M s ResponseAndMessage - subscribeQueueAndDeliver msg_ q qr@QueueRec {rcvServiceId} = - liftIO (TM.lookupIO entId $ subscriptions clnt) >>= \case - Nothing -> - deliver =<< sharedSubscribeQueue q rcvServiceId subscribers subscriptions serviceSubsCount (newSubscription NoSub) rcvServices - Just s@Sub {subThread} -> do - stats <- asks serverStats - case subThread of - ProhibitSub -> do - -- cannot use SUB in the same connection where GET was used - incStat $ qSubProhibited stats - pure (err (CMD PROHIBITED), Nothing) - _ -> do - incStat $ qSubDuplicate stats - atomically (writeTVar (delivered s) Nothing) >> deliver (True, Just s) + deliverSubMsg :: Maybe Message -> Maybe (Either ErrorType (Bool, Maybe Sub)) -> StoreQueue s -> QueueRec -> M s ResponseAndMessage + deliverSubMsg msg_ r_ _ qr = case r_ of + Just (Right r) -> deliver r + Just (Left e) -> pure (err e, Nothing) + Nothing -> pure (err INTERNAL, Nothing) where deliver :: (Bool, Maybe Sub) -> M s ResponseAndMessage deliver (hasSub, sub_) = do stats <- asks serverStats fmap (either ((,Nothing) . err) id) $ liftIO $ runExceptT $ do - msg' <- forM msg_ $ \msg -> liftIO $ do + msg' <- fmap join $ forM msg_ $ \msg -> liftIO $ do ts <- getSystemSeconds sub <- maybe getSub pure sub_ - atomically $ setDelivered sub msg ts unless hasSub $ incStat $ qSub stats - pure (NoCorrId, entId, MSG (encryptMsg qr msg)) + ifM + (atomically $ trySetDelivered sub msg ts) + (pure $ Just (NoCorrId, entId, MSG (encryptMsg qr msg))) + (pure Nothing) pure ((corrId, entId, SOK clntServiceId), msg') getSub :: IO Sub @@ -1784,7 +1833,7 @@ client Nothing -> do sub <- atomically $ newSubscription NoSub atomically $ TM.insert rId sub $ subscriptions clnt - atomically $ writeTQueue (subQ subscribers) (CSClient rId rcvServiceId rcvServiceId, clientId) + liftIO $ processSub clnt $ CSClient rId rcvServiceId rcvServiceId -- clients that use GET are not added to server subscribers getMessage :: StoreQueue s -> QueueRec -> M s (Transmission BrokerMsg) @@ -1840,66 +1889,14 @@ client subscribeNotifications :: StoreQueue s -> NtfCreds -> M s BrokerMsg subscribeNotifications q NtfCreds {ntfServiceId} = do - (hasSub, _) <- sharedSubscribeQueue q ntfServiceId ntfSubscribers ntfSubscriptions ntfServiceSubsCount (pure ()) ntfServices + (hasSub, _) <- sharedSubscribeQueue clntServiceId entId q ntfServiceId processNtfSub ntfSubscriptions ntfServiceSubsCount (pure ()) ntfServices when (isNothing clntServiceId) $ asks serverStats >>= incStat . (if hasSub then ntfSubDuplicate else ntfSub) pure $ SOK clntServiceId - sharedSubscribeQueue :: - StoreQueue s -> - Maybe ServiceId -> - ServerSubscribers s -> - (Client s -> TMap QueueId sub) -> - (Client s -> TVar (Int64, IdsHash)) -> - STM sub -> - (ServerStats -> ServiceStats) -> - M s (Bool, Maybe sub) - sharedSubscribeQueue q queueServiceId srvSubscribers clientSubs clientServiceSubs mkSub servicesSel = do - stats <- asks serverStats - let incSrvStat sel = incStat $ sel $ servicesSel stats - writeSub = writeTQueue (subQ srvSubscribers) (CSClient entId queueServiceId clntServiceId, clientId) - liftIO $ case clntServiceId of - Just serviceId - | queueServiceId == Just serviceId -> do - -- duplicate queue-service association - can only happen in case of response error/timeout - hasSub <- atomically $ ifM hasServiceSub (pure True) (False <$ incServiceQueueSubs) - unless hasSub $ do - atomically writeSub - incSrvStat srvSubCount - incSrvStat srvSubQueues - incSrvStat srvAssocDuplicate - pure (hasSub, Nothing) - | otherwise -> do - -- association already done in prepareBatchSubs - hasSub <- atomically $ (<$ incServiceQueueSubs) =<< hasServiceSub - atomically writeSub - unless hasSub $ incSrvStat srvSubCount - incSrvStat srvSubQueues - incSrvStat $ maybe srvAssocNew (const srvAssocUpdated) queueServiceId - pure (hasSub, Nothing) - where - hasServiceSub = ((0 /=) . fst) <$> readTVar (clientServiceSubs clnt) - -- This function is used when queue association with the service is created. - incServiceQueueSubs = modifyTVar' (clientServiceSubs clnt) $ addServiceSubs (1, queueIdHash (recipientId q)) -- service count and IDS hash - Nothing -> case queueServiceId of - Just _ -> do - -- unassociation already done in prepareBatchSubs - incSrvStat srvAssocRemoved - -- getSubscription may find a subscription for receiving service, where clientSubs also hold active deliveries for service subscriptions. - -- For notification service it can only find one if storage and session states diverge. - r <- getSubscription - atomically writeSub - pure r - Nothing -> do - r@(hasSub, _) <- getSubscription - unless hasSub $ atomically writeSub - pure r - where - getSubscription = either ((False,) . Just) ((True,) . Just) <$> TM.lookupOrInsertIO entId mkSub (clientSubs clnt) - subscribeServiceMessages :: ServiceId -> (Int64, IdsHash) -> M s BrokerMsg subscribeServiceMessages serviceId expected = - sharedSubscribeService SRecipientService serviceId expected subscribers serviceSubscribed serviceSubsCount rcvServices >>= \case + sharedSubscribeService SRecipientService serviceId expected processSub serviceSubscribed serviceSubsCount rcvServices >>= \case Left e -> pure $ ERR e Right (hasSub, (count, idsHash)) -> do stats <- asks serverStats @@ -1930,17 +1927,18 @@ client Right _sub -> pure (qCnt + 1, msgCnt, dupCnt + 1, evts) Left sub -> do ts <- getSystemSeconds - atomically $ setDelivered sub msg ts - atomically $ writeTBQueue msgQ [(NoCorrId, rId, MSG (encryptMsg qr msg))] - pure (qCnt + 1, msgCnt + 1, dupCnt, evts) + ifM + (atomically $ trySetDelivered sub msg ts) + ((qCnt + 1, msgCnt + 1, dupCnt, evts) <$ atomically (writeTBQueue msgQ [(NoCorrId, rId, MSG (encryptMsg qr msg))])) + (pure (qCnt + 1, msgCnt, dupCnt + 1, evts)) getSubscription rId = TM.lookupOrInsertIO rId (newSubscription NoSub) (subscriptions clnt) subscribeServiceNotifications :: ServiceId -> (Int64, IdsHash) -> M s BrokerMsg subscribeServiceNotifications serviceId expected = - either ERR (uncurry SOKS . snd) <$> sharedSubscribeService SNotifierService serviceId expected ntfSubscribers ntfServiceSubscribed ntfServiceSubsCount ntfServices + either ERR (uncurry SOKS . snd) <$> sharedSubscribeService SNotifierService serviceId expected processNtfSub ntfServiceSubscribed ntfServiceSubsCount ntfServices - sharedSubscribeService :: (PartyI p, ServiceParty p) => SParty p -> ServiceId -> (Int64, IdsHash) -> ServerSubscribers s -> (Client s -> TVar Bool) -> (Client s -> TVar (Int64, IdsHash)) -> (ServerStats -> ServiceStats) -> M s (Either ErrorType (Bool, (Int64, IdsHash))) - sharedSubscribeService party serviceId (count, idsHash) srvSubscribers clientServiceSubscribed clientServiceSubs servicesSel = do + sharedSubscribeService :: (PartyI p, ServiceParty p) => SParty p -> ServiceId -> (Int64, IdsHash) -> (Client s -> ClientSub -> IO ()) -> (Client s -> TVar Bool) -> (Client s -> TVar (Int64, IdsHash)) -> (ServerStats -> ServiceStats) -> M s (Either ErrorType (Bool, (Int64, IdsHash))) + sharedSubscribeService party serviceId (count, idsHash) processSub' clientServiceSubscribed clientServiceSubs servicesSel = do subscribed <- readTVarIO $ clientServiceSubscribed clnt stats <- asks serverStats liftIO $ runExceptT $ @@ -1960,7 +1958,7 @@ client | diff > 0 -> incSrvStat srvSubMore 1 >> incSrvStat srvSubMoreTotal diff | diff < 0 -> incSrvStat srvSubFewer 1 >> incSrvStat srvSubFewerTotal (- diff) | otherwise -> incSrvStat srvSubDiff 1 - atomically $ writeTQueue (subQ srvSubscribers) (CSService serviceId subsChange, clientId) + liftIO $ processSub' clnt $ CSService serviceId subsChange pure (count', idsHash') acknowledgeMsg :: MsgId -> StoreQueue s -> QueueRec -> M s (Transmission BrokerMsg) @@ -2204,7 +2202,7 @@ client resolveNameMsg (thVersion clntTHParams) nenv d >>= sendResolved -- INTERNAL because processCommand never returns Nothing for sender commands; -- `fst` drops the empty message only returned for SUB. - _ -> Just . maybe (corrId', entId', ERR INTERNAL) fst <$> lift (processCommand Nothing (Right (M.empty, M.empty, M.empty)) t'') + _ -> Just . maybe (corrId', entId', ERR INTERNAL) fst <$> lift (processCommand Nothing (Right (M.empty, M.empty, M.empty, M.empty)) t'') stats <- asks serverStats incStat $ pMsgFwdsRecv stats traverse encodeResp r_ >>= \case @@ -2246,6 +2244,9 @@ client let !msgId = messageId msg writeTVar delivered $ Just (msgId, ts) + trySetDelivered :: Sub -> Message -> SystemSeconds -> STM Bool + trySetDelivered sub msg ts = readTVar (delivered sub) >>= maybe (True <$ setDelivered sub msg ts) (const $ pure False) + delQueueAndMsgs :: (StoreQueue s, QueueRec) -> M s (Transmission BrokerMsg) delQueueAndMsgs (q, QueueRec {rcvServiceId}) = do liftIO (deleteQueue ms q) >>= \case @@ -2256,14 +2257,14 @@ client sub <- atomically $ TM.lookupDelete entId $ subscriptions clnt liftIO $ mapM_ cancelSub sub when (isJust rcvServiceId) $ atomically $ modifyTVar' (serviceSubsCount clnt) $ subtractServiceSubs (1, queueIdHash (recipientId q)) - atomically $ writeTQueue (subQ subscribers) (CSDeleted entId rcvServiceId, clientId) + liftIO $ processSub clnt $ CSDeleted entId rcvServiceId forM_ (notifier qr) $ \NtfCreds {notifierId = nId, ntfServiceId} -> do -- queue is deleted by a different client from the one subscribed to notifications, -- so we don't need to remove subscription from the current client. stats <- asks serverStats deleted <- asks ntfStore >>= liftIO . (`deleteNtfs` nId) when (deleted > 0) $ liftIO $ atomicModifyIORef'_ (ntfCount stats) (subtract deleted) - atomically $ writeTQueue (subQ ntfSubscribers) (CSDeleted nId ntfServiceId, clientId) + liftIO $ processNtfSub clnt $ CSDeleted nId ntfServiceId updateDeletedStats qr pure ok Left e -> pure $ err e diff --git a/src/Simplex/Messaging/Server/Env/STM.hs b/src/Simplex/Messaging/Server/Env/STM.hs index 4c0b1d566..481b2a15a 100644 --- a/src/Simplex/Messaging/Server/Env/STM.hs +++ b/src/Simplex/Messaging/Server/Env/STM.hs @@ -347,8 +347,7 @@ data Server s = Server newtype ServerClients s = ServerClients {serverClients :: TVar (IntMap (Client s))} data ServerSubscribers s = ServerSubscribers - { subQ :: TQueue (ClientSub, ClientId), - queueSubscribers :: SubscribedClients s, + { queueSubscribers :: SubscribedClients s, serviceSubscribers :: SubscribedClients s, -- service clients with long-term certificates that have subscriptions totalServiceSubs :: TVar (Int64, IdsHash), subClients :: TVar IntSet, -- clients with individual or service subscriptions @@ -486,13 +485,12 @@ deleteServerClient cId Server {clients} = atomically $ modifyTVar' (serverClient newServerSubscribers :: IO (ServerSubscribers s) newServerSubscribers = do - subQ <- newTQueueIO queueSubscribers <- SubscribedClients <$> TM.emptyIO serviceSubscribers <- SubscribedClients <$> TM.emptyIO totalServiceSubs <- newTVarIO (0, mempty) subClients <- newTVarIO IS.empty pendingEvents <- newTVarIO IM.empty - pure ServerSubscribers {subQ, queueSubscribers, serviceSubscribers, totalServiceSubs, subClients, pendingEvents} + pure ServerSubscribers {queueSubscribers, serviceSubscribers, totalServiceSubs, subClients, pendingEvents} newClient :: ClientId -> Natural -> THandleParams SMPVersion 'TServer -> SystemTime -> IO (Client s) newClient clientId qSize clientTHParams createdAt = do diff --git a/tests/ServerTests.hs b/tests/ServerTests.hs index b1d42d13d..f97c8dd85 100644 --- a/tests/ServerTests.hs +++ b/tests/ServerTests.hs @@ -92,11 +92,15 @@ serverTests = do describe "Concurrent sending and delivery" testConcurrentSendDelivery describe "SUB while delivery waits" testSubPendingDelivery describe "SUB to a queue blocked by the operator" testSubBlockedQueue + describe "SUB and SEND in one block" testSubSendBlock + describe "switch subscription with SUB and SEND in one block" testSwitchSubSendBlock + describe "two SUBs in one block" testSubTwiceBlock describe "subscribe the same queues from two connections at once" testSubSameQueuesTwoConnections describe "Service message subscriptions" $ do testServiceDeliverSubscribe testServiceUpgradeAndDowngrade testServiceSubsTotalCount + testServiceSubSendBlock testServiceSubAssociatedQueue testServiceDowngradeConnected testServiceSubsReplacesConnection @@ -721,8 +725,8 @@ testSubPendingDelivery = Resp "1" _ OK <- sendRecv sh ("", "1", sId, _SEND "hello") signSend_ rh rKey Nothing ("2", rId, SUB) threadDelay 500000 - let drain st@(n, _, mId_) = case mId_ of - Just mId | n == length fillers -> pure mId + let drain st@(n, subscribed, mId_) = case mId_ of + Just mId | n == length fillers && subscribed -> pure mId _ -> 5000000 `timeout` tGetClient rh >>= maybe (error "drain timeout") (foldM received st) >>= drain received st@(n, subscribed, mId_) = \case Resp "" _ (LNK _ _) -> pure st @@ -731,7 +735,7 @@ testSubPendingDelivery = Resp "" rId' (Msg mId msg) | rId' == rId -> do dec mId msg `shouldBe` Right "hello" - pure (n, subscribed, if subscribed then Just mId else mId_) + pure (n, subscribed, Just mId) | rId' `elem` map fst fillers -> pure (n + 1, subscribed, mId_) r -> error $ "unexpected event: " <> show r mId <- drain (0 :: Int, False, Nothing) @@ -753,6 +757,67 @@ testSubPendingDelivery = _ -> pure Nothing ] +testSubSendBlock :: SpecWith (ASrvTransport, AStoreType) +testSubSendBlock = + it "should deliver message sent in the same block as SUB" $ \(ATransport t, msType) -> + smpTest t msType $ \h@THandle {params} -> do + g <- C.newRandom + (rPub, rKey) <- atomically $ C.generateAuthKeyPair C.SEd25519 g + (dhPub, dhPriv :: C.PrivateKeyX25519) <- atomically $ C.generateKeyPair g + Resp "1" NoEntity (Ids rId sId srvDh) <- signSendRecv h rKey ("1", NoEntity, NEW (NewQueueReq rPub dhPub Nothing SMOnlyCreate (Just (QRMessaging Nothing)) Nothing)) + let dec = decryptMsgV3 $ C.dh' srvDh dhPriv + TransmissionForAuth {tForAuth, tToSend = subT} = encodeTransmissionForAuth params (CorrId "2", rId, SUB) + TransmissionForAuth {tToSend = sendT} = encodeTransmissionForAuth params (CorrId "3", sId, _SEND "hello") + Right subAuth <- pure $ authTransmission (thAuth params) False (Just rKey) Nothing tForAuth + [Right (), Right ()] <- tPut h [Right (subAuth, subT), Right (Nothing, sendT)] + Resp "" _ (Msg mId msg) <- tGet1 h + dec mId msg `shouldBe` Right "hello" + [Resp "2" _ (SOK Nothing), Resp "3" _ OK] <- tGetClient h + Resp "4" _ OK <- signSendRecv h rKey ("4", rId, ACK mId) + pure () + +testSwitchSubSendBlock :: SpecWith (ASrvTransport, AStoreType) +testSwitchSubSendBlock = + it "should deliver message sent in the same block as SUB to the new connection" $ \(ATransport t, msType) -> + smpTest2 t msType $ \rh1 rh2@THandle {params} -> do + g <- C.newRandom + (rPub, rKey) <- atomically $ C.generateAuthKeyPair C.SEd25519 g + (dhPub, dhPriv :: C.PrivateKeyX25519) <- atomically $ C.generateKeyPair g + Resp "1" NoEntity (Ids rId sId srvDh) <- signSendRecv rh1 rKey ("1", NoEntity, New rPub dhPub) + let dec = decryptMsgV3 $ C.dh' srvDh dhPriv + TransmissionForAuth {tForAuth, tToSend = subT} = encodeTransmissionForAuth params (CorrId "2", rId, SUB) + TransmissionForAuth {tToSend = sendT} = encodeTransmissionForAuth params (CorrId "3", sId, _SEND "hello") + Right subAuth <- pure $ authTransmission (thAuth params) False (Just rKey) Nothing tForAuth + [Right (), Right ()] <- tPut rh2 [Right (subAuth, subT), Right (Nothing, sendT)] + Resp "" _ end <- tGet1 rh1 + end `shouldBe` END + Resp "" _ (Msg mId msg) <- tGet1 rh2 + dec mId msg `shouldBe` Right "hello" + [Resp "2" _ (SOK Nothing), Resp "3" _ OK] <- tGetClient rh2 + Resp "4" _ OK <- signSendRecv rh2 rKey ("4", rId, ACK mId) + pure () + +testSubTwiceBlock :: SpecWith (ASrvTransport, AStoreType) +testSubTwiceBlock = + it "should deliver message once for two SUBs in the same block" $ \(ATransport t, msType) -> + smpTest t msType $ \h@THandle {params} -> do + g <- C.newRandom + (rPub, rKey) <- atomically $ C.generateAuthKeyPair C.SEd25519 g + (dhPub, dhPriv :: C.PrivateKeyX25519) <- atomically $ C.generateKeyPair g + Resp "1" NoEntity (Ids rId sId srvDh) <- signSendRecv h rKey ("1", NoEntity, NEW (NewQueueReq rPub dhPub Nothing SMOnlyCreate (Just (QRMessaging Nothing)) Nothing)) + let dec = decryptMsgV3 $ C.dh' srvDh dhPriv + Resp "2" _ OK <- sendRecv h ("", "2", sId, _SEND "hello") + let TransmissionForAuth {tForAuth = forAuth1, tToSend = sub1} = encodeTransmissionForAuth params (CorrId "3", rId, SUB) + TransmissionForAuth {tForAuth = forAuth2, tToSend = sub2} = encodeTransmissionForAuth params (CorrId "4", rId, SUB) + Right auth1 <- pure $ authTransmission (thAuth params) False (Just rKey) Nothing forAuth1 + Right auth2 <- pure $ authTransmission (thAuth params) False (Just rKey) Nothing forAuth2 + [Right (), Right ()] <- tPut h [Right (auth1, sub1), Right (auth2, sub2)] + [Resp "3" _ (SOK Nothing), Resp "4" _ (SOK Nothing), Resp "" _ (Msg mId msg)] <- tGetClient h + dec mId msg `shouldBe` Right "hello" + Nothing <- 100000 `timeout` tGetClient @SMPVersion @ErrorType @BrokerMsg h + Resp "5" _ OK <- signSendRecv h rKey ("5", rId, ACK mId) + pure () + testSubBlockedQueue :: SpecWith (ASrvTransport, AStoreType) testSubBlockedQueue = it "should return BLOCKED to SUB after the operator blocks the queue" $ \(ATransport t, msType) -> do @@ -1084,6 +1149,31 @@ testServiceSubsTotalCount = threadDelay 1500000 B.readFile testPrometheusMetricsFile >>= \m -> readServiceSubsMetric (B.unpack m) `shouldBe` Just 3 +testServiceSubSendBlock :: SpecWith (ASrvTransport, AStoreType) +testServiceSubSendBlock = + it "should deliver message sent in the same block as service SUB" $ \(at@(ATransport t), msType) -> do + g <- C.newRandom + creds <- genCredentials g Nothing (0, 2400) "localhost" + let (_fp, tlsCred) = tlsCredentials [creds] + serviceKeys@(_, servicePK) <- atomically $ C.generateKeyPair g + withSmpServerConfigOn at (cfgMS msType) testPort $ \_ -> do + (rPub, rKey) <- atomically $ C.generateAuthKeyPair C.SEd25519 g + (dhPub, dhPriv :: C.PrivateKeyX25519) <- atomically $ C.generateKeyPair g + (rId, sId, dec, serviceId) <- runSMPServiceClient t (tlsCred, serviceKeys) $ \sh -> do + Resp "1" NoEntity (Ids_ rId sId srvDh serviceId) <- serviceSignSendRecv sh rKey servicePK ("1", NoEntity, New rPub dhPub) + pure (rId, sId, decryptMsgV3 $ C.dh' srvDh dhPriv, serviceId) + runSMPServiceClient t (tlsCred, serviceKeys) $ \sh@THandle {params} -> do + let TransmissionForAuth {tForAuth, tToSend = subT} = encodeTransmissionForAuth params (CorrId "2", rId, SUB) + TransmissionForAuth {tToSend = sendT} = encodeTransmissionForAuth params (CorrId "3", sId, _SEND "hello") + Right subAuth <- pure $ authTransmission (thAuth params) True (Just rKey) Nothing tForAuth + [Right (), Right ()] <- tPut sh [Right (subAuth, subT), Right (Nothing, sendT)] + Resp "" _ (Msg mId msg) <- tGet1 sh + dec mId msg `shouldBe` Right "hello" + [Resp "2" _ (SOK (Just serviceId')), Resp "3" _ OK] <- tGetClient sh + serviceId' `shouldBe` serviceId + Resp "4" _ OK <- signSendRecv sh rKey ("4", rId, ACK mId) + pure () + testServiceSubAssociatedQueue :: SpecWith (ASrvTransport, AStoreType) testServiceSubAssociatedQueue = it "should deliver the waiting message when the service subscribes its queue with SUB after reconnecting" $ \(at@(ATransport t), msType) -> do