smp-server: process forwarded messages to different queues in parallel (#2047)

* smp-server: process forwarded messages to different queues in parallel

* smp-server: name forward workers without "forwarded"

* smp server: process messages of all clients in a shared worker pool

* smp server: revert the shared worker pool

* smp server: run forwarded commands in workers without join

* smp server: queue server actions to forward workers

* smp server: choose the forward worker by queue ID outside the command loop

* smp server: define the server monad with the environment

* smp server: queue forwarded sender commands to workers after verification

* smp server: verify forwarded sender commands in their worker

* smp server: reject or verify forwarded commands in their worker

* smp server: dispatch forwarded commands in one case

* smp: restrict forwarded commands to forward parties at type level

* smp server: return no response for commands queued to workers

* smp server: queue forwarded sender commands in their dispatch branch

* smp server: define the forward worker where it is used
This commit is contained in:
Evgeny @ SimpleX Chat authored and GitHub committed 2026-11-11 23:41:44 +00:00
1 parent 71179c4d4e
commit e0afca78d3
9 files changed
+143 -43

No files matched your search

+1 -1
View File
@@ -1217,7 +1217,7 @@ proxyOKSMPCommand c nm proxiedRelay spKey sId command =
proxySMPCommand ::
forall p.
PartyI p =>
(PartyI p, ForwardParty p) =>
SMPClient ->
NetworkRequestMode ->
-- proxy session from PKEY
+17
View File
@@ -71,6 +71,7 @@ module Simplex.Messaging.Protocol
QueueParty,
BatchParty,
ServiceParty,
ForwardParty,
ASubscriberParty (..),
BrokerMsg (..),
SParty (..),
@@ -183,6 +184,7 @@ module Simplex.Messaging.Protocol
queueParty,
batchParty,
serviceParty,
forwardParty,
partyClientRole,
partyServiceRole,
userProtocol,
@@ -459,6 +461,21 @@ serviceParty = \case
_ -> Nothing
{-# INLINE serviceParty #-}
type family ForwardParty (p :: Party) :: Constraint where
ForwardParty Sender = ()
ForwardParty LinkClient = ()
ForwardParty Resolver = ()
ForwardParty p =
(Int ~ Bool, TypeError (Type.Text "Party " :<>: ShowType p :<>: Type.Text " is not ForwardParty"))
forwardParty :: SParty p -> Maybe (Dict (PartyI p, ForwardParty p))
forwardParty = \case
SSender -> Just Dict
SSenderLink -> Just Dict
SResolver -> Just Dict
_ -> Nothing
{-# INLINE forwardParty #-}
data ASubscriberParty = forall p. (PartyI p, ServiceParty p) => ASP (SParty p)
deriving instance Show ASubscriberParty
+47 -40
View File
@@ -64,6 +64,7 @@ import Data.Dynamic (toDyn)
import Data.Either (fromRight, isRight, partitionEithers)
import Data.Foldable (foldrM)
import Data.Functor (($>), (<&>))
import Data.Hashable (hash)
import Data.IORef
import Data.Int (Int64)
import qualified Data.IntMap.Strict as IM
@@ -160,7 +161,6 @@ runSMPServer cfg attachHTTP_ = do
runSMPServerBlocking :: MsgStoreClass s => TMVar Bool -> ServerConfig s -> Maybe AttachHTTP -> IO ()
runSMPServerBlocking started cfg attachHTTP_ = newEnv cfg >>= runReaderT (smpServer started cfg attachHTTP_)
type M s a = ReaderT (Env s) IO a
type AttachHTTP = Socket -> TLS.Context -> IO ()
-- actions used in processClientSub to reduce STM transaction scope
@@ -1399,7 +1399,7 @@ client
(processSub, processNtfSub)
Server {subscribers}
ms
clnt@Client {rcvQ, sndQ, msgQ, clientTHParams = thParams'@THandleParams {sessionId}, procThreads} = do
clnt@Client {rcvQ, sndQ, msgQ, clientTHParams = thParams'@THandleParams {sessionId}, procThreads, fwdWorkers} = do
labelMyThread . B.unpack $ "client $" <> encode sessionId <> " commands"
stats <- asks serverStats
authDelay <- asks $ authErrorDelay . config
@@ -2195,47 +2195,54 @@ client
r2 <- liftEitherWith (const BLOCK) $ EncResponse <$> C.cbEncrypt clientSecret (fromMaybe (C.reverseNonce clientNonce) nonce_) r' paddedProxiedTLength
let fr = FwdResponse {fwdCorrId, fwdResponse = r2}
pure $ RRES nonce_ $ EncFwdResponse $ C.cbEncryptNoPad sessSecret (C.reverseNonce proxyNonce) (smpEncode fr)
-- the inner response, or Nothing if forked (RSLV).
r_ <- lift (rejectOrVerify clntThAuth t') >>= \case
-- rejectOrVerify filters allowed commands, no need to repeat it here.
Left r -> pure $ Just r
Right t''@(_, (corrId', entId', cmd')) -> case cmd' of
Cmd SResolver (RSLV d) -> lift $ rslvNamesEnv >>= \case
Nothing -> pure $ Just (corrId', entId', ERR (NAME NO_RESOLVER))
Just nenv -> do
let sendResolved msg = do
rres <- either ERR id <$> runExceptT (encodeResp (corrId', entId', msg))
atomically $ writeTBQueue sndQ ([(corrId, NoEntity, rres)], [])
forkCmd_ serverResolverConcurrency forwardedCmdWait (sendResolved rslvTooManyRequests) $
resolveNameMsg (thVersion clntTHParams) nenv d >>= sendResolved
-- INTERNAL because processCommand never returns Nothing for sender commands;
-- `fst` drops the empty message only returned for SUB.
_ -> Just . maybe (corrId', entId', ERR INTERNAL) fst <$> lift (processCommand Nothing (Right (M.empty, M.empty, M.empty, M.empty)) t'')
-- the inner response, or Nothing if forked (RSLV) or queued to a worker.
r_ <- case t' of
Left (corrId', entId', e) -> pure $ Just (corrId', entId', ERR e)
Right t@(_, _, t''@(corrId', entId', Cmd p cmd)) -> case forwardParty p of
Just Dict -> case cmd of
RSLV d -> lift $ rslvNamesEnv >>= \case
Nothing -> pure $ Just (corrId', entId', ERR (NAME NO_RESOLVER))
Just nenv -> do
let sendResolved msg = do
rres <- either ERR id <$> runExceptT (encodeResp (corrId', entId', msg))
atomically $ writeTBQueue sndQ ([(corrId, NoEntity, rres)], [])
forkCmd_ serverResolverConcurrency forwardedCmdWait (sendResolved rslvTooManyRequests) $
resolveNameMsg (thVersion clntTHParams) nenv d >>= sendResolved
_ -> lift $ do
q <- forwardWorker entId'
atomically $ writeTBQueue q $ do
r <- liftIO (verifyTransmission ms Nothing clntThAuth t) >>= \case
-- INTERNAL because processCommand never returns Nothing for sender commands;
-- `fst` drops the empty message only returned for SUB.
VRVerified q_ -> maybe (corrId', entId', ERR INTERNAL) fst <$> processCommand Nothing (Right (M.empty, M.empty, M.empty, M.empty)) (q_, t'')
VRFailed e -> pure (corrId', entId', ERR e)
rres <- either ERR id <$> runExceptT (encodeResp r)
stats <- asks serverStats
authDelay <- asks $ authErrorDelay . config
liftIO $
if isAuthError r
then sendAfterDelay clnt stats authDelay startedAt ([(corrId, NoEntity, rres)], [])
else atomically $ writeTBQueue sndQ ([(corrId, NoEntity, rres)], [])
pure Nothing
Nothing -> pure $ Just (corrId', entId', ERR $ CMD PROHIBITED)
stats <- asks serverStats
incStat $ pMsgFwdsRecv stats
traverse encodeResp r_ >>= \case
Just rres | any isAuthError r_ -> do
authDelay <- asks $ authErrorDelay . config
Nothing <$ liftIO (sendAfterDelay clnt stats authDelay startedAt ([(corrId, NoEntity, rres)], []))
rres_ -> pure rres_
traverse encodeResp r_
where
rejectOrVerify :: Maybe (THandleAuth 'TServer) -> SignedTransmissionOrError ErrorType Cmd -> M s (VerifiedTransmissionOrError s)
rejectOrVerify clntThAuth = \case
Left (corrId', entId', e) -> pure $ Left (corrId', entId', ERR e)
Right t'@(_, _, t''@(corrId', entId', cmd'))
| allowed -> liftIO $ verified <$> verifyTransmission ms Nothing clntThAuth t'
| otherwise -> pure $ Left (corrId', entId', ERR $ CMD PROHIBITED)
where
allowed = case cmd' of
Cmd SSender SEND {} -> True
Cmd SSender (SKEY _) -> True
Cmd SSenderLink (LKEY _) -> True
Cmd SSenderLink LGET -> True
Cmd SResolver (RSLV _) -> True
_ -> False
verified = \case
VRVerified q -> Right (q, t'')
VRFailed e -> Left (corrId', entId', ERR e)
forwardWorker :: EntityId -> M s (TBQueue (M s ()))
forwardWorker (EntityId eId) = do
n <- asks $ serverForwardWorkers . config
let i = hash eId `mod` n
IM.lookup i <$> readTVarIO fwdWorkers >>= \case
Just q -> pure q
Nothing -> do
q <- newTBQueueIO =<< asks (tbqSize . config)
atomically $ modifyTVar' fwdWorkers $ IM.insert i q
forkClient clnt (B.unpack ("client $" <> encode sessionId <> " forward ") <> show i) $
forever $ do
cmd <- atomically $ readTBQueue q
cmd `catchAny` \e -> logError $ "forwarded command error: " <> tshow e
pure q
encryptMsg :: QueueRec -> Message -> RcvMessage
encryptMsg qr msg = encrypt . encodeRcvMsgBody $ case msg of
+12
View File
@@ -26,6 +26,7 @@ module Simplex.Messaging.Server.Env.STM
StorePaths (..),
StartOptions (..),
Env (..),
M,
Server (..),
ServerSubscribers (..),
SubscribedClients,
@@ -67,6 +68,7 @@ module Simplex.Messaging.Server.Env.STM
defaultInactiveClientExpiration,
defaultProxyClientConcurrency,
defaultNameResolverConcurrency,
defaultForwardWorkers,
defaultMaxClientCmdWait,
defaultAuthErrorDelayMs,
readWriteQueueStore,
@@ -80,6 +82,7 @@ where
import Control.Concurrent (ThreadId)
import Control.Logger.Simple
import Control.Monad
import Control.Monad.Reader (ReaderT)
import qualified Crypto.PubKey.RSA as RSA
import Crypto.Random
import Data.Int (Int64)
@@ -198,6 +201,7 @@ data ServerConfig s = ServerConfig
-- aggregate over a single proxy->relay connection (only servers send proxied
-- requests), so bounding them by the per-client limit would throttle unduly.
serverResolverConcurrency :: Int,
serverForwardWorkers :: Int,
maxClientCmdWait :: Int,
-- | public-namespace resolver config; Nothing disables the names role
namesConfig :: Maybe NamesConfig,
@@ -247,6 +251,9 @@ defaultProxyClientConcurrency = 500
defaultNameResolverConcurrency :: Int
defaultNameResolverConcurrency = 5000
defaultForwardWorkers :: Int
defaultForwardWorkers = 8
defaultMaxClientCmdWait :: Int
defaultMaxClientCmdWait = 60000000 -- microseconds
@@ -274,6 +281,8 @@ data Env s = Env
namesEnv :: Maybe NamesEnv -- public-namespace resolver, present when [NAMES] enable: on
}
type M s a = ReaderT (Env s) IO a
msgStore :: Env s -> s
msgStore = fromMsgStore . msgStore_
{-# INLINE msgStore #-}
@@ -433,6 +442,7 @@ data Client s = Client
sndQ :: TBQueue (NonEmpty (Transmission BrokerMsg), [Transmission BrokerMsg]),
msgQ :: TBQueue (NonEmpty (Transmission BrokerMsg)),
procThreads :: TVar Int,
fwdWorkers :: TVar (IntMap (TBQueue (M s ()))),
endThreads :: TVar (IntMap (Weak ThreadId)),
endThreadSeq :: TVar Int,
clientTHParams :: THandleParams SMPVersion 'TServer,
@@ -505,6 +515,7 @@ newClient clientId qSize clientTHParams createdAt = do
sndQ <- newTBQueueIO qSize
msgQ <- newTBQueueIO qSize
procThreads <- newTVarIO 0
fwdWorkers <- newTVarIO IM.empty
endThreads <- newTVarIO IM.empty
endThreadSeq <- newTVarIO 0
connected <- newTVarIO True
@@ -523,6 +534,7 @@ newClient clientId qSize clientTHParams createdAt = do
sndQ,
msgQ,
procThreads,
fwdWorkers,
endThreads,
endThreadSeq,
clientTHParams,
+1
View File
@@ -531,6 +531,7 @@ smpServerCLI_ generateSite serveStaticFiles attachStaticFiles cfgPath logPath =
allowSMPProxy = True,
serverClientConcurrency = readIniDefault defaultProxyClientConcurrency "PROXY" "client_concurrency" ini,
serverResolverConcurrency = readIniDefault defaultNameResolverConcurrency "NAMES" "resolver_concurrency" ini,
serverForwardWorkers = readIniDefault defaultForwardWorkers "PROXY" "forward_workers" ini,
maxClientCmdWait = defaultMaxClientCmdWait,
namesConfig = readNamesConfig ini,
information = serverPublicInfo ini,
@@ -156,6 +156,10 @@ iniFileContent cfgPath logPath opts host basicAuth controlPortPwds =
\# socks_mode = onion\n\n\
\# Limit number of threads a client can spawn to process proxy commands and name resolutions in parallel.\n"
<> ("# client_concurrency = " <> tshow defaultProxyClientConcurrency)
<> "\n\n\
\# Number of workers processing messages that one proxy server forwards to this server.\n\
\# Messages to one queue are processed by one worker, in the order they are received.\n"
<> ("# forward_workers = " <> tshow defaultForwardWorkers)
<> "\n\n\
\[NAMES]\n\
\# Public-namespace resolution via the snrc-resolve.py REST resolver.\n\