mirror of
https://github.com/simplex-chat/simplexmq.git
synced 2026-11-22 19:58:39 +00:00
smp-server: process forwarded messages to different queues in parallel (#2047)
* smp-server: process forwarded messages to different queues in parallel * smp-server: name forward workers without "forwarded" * smp server: process messages of all clients in a shared worker pool * smp server: revert the shared worker pool * smp server: run forwarded commands in workers without join * smp server: queue server actions to forward workers * smp server: choose the forward worker by queue ID outside the command loop * smp server: define the server monad with the environment * smp server: queue forwarded sender commands to workers after verification * smp server: verify forwarded sender commands in their worker * smp server: reject or verify forwarded commands in their worker * smp server: dispatch forwarded commands in one case * smp: restrict forwarded commands to forward parties at type level * smp server: return no response for commands queued to workers * smp server: queue forwarded sender commands in their dispatch branch * smp server: define the forward worker where it is used
This commit is contained in:
1 parent
71179c4d4e
commit
e0afca78d3
9 files changed
+143
-43
No files matched your search
@@ -1217,7 +1217,7 @@ proxyOKSMPCommand c nm proxiedRelay spKey sId command =
|
||||
|
||||
proxySMPCommand ::
|
||||
forall p.
|
||||
PartyI p =>
|
||||
(PartyI p, ForwardParty p) =>
|
||||
SMPClient ->
|
||||
NetworkRequestMode ->
|
||||
-- proxy session from PKEY
|
||||
|
||||
@@ -71,6 +71,7 @@ module Simplex.Messaging.Protocol
|
||||
QueueParty,
|
||||
BatchParty,
|
||||
ServiceParty,
|
||||
ForwardParty,
|
||||
ASubscriberParty (..),
|
||||
BrokerMsg (..),
|
||||
SParty (..),
|
||||
@@ -183,6 +184,7 @@ module Simplex.Messaging.Protocol
|
||||
queueParty,
|
||||
batchParty,
|
||||
serviceParty,
|
||||
forwardParty,
|
||||
partyClientRole,
|
||||
partyServiceRole,
|
||||
userProtocol,
|
||||
@@ -459,6 +461,21 @@ serviceParty = \case
|
||||
_ -> Nothing
|
||||
{-# INLINE serviceParty #-}
|
||||
|
||||
type family ForwardParty (p :: Party) :: Constraint where
|
||||
ForwardParty Sender = ()
|
||||
ForwardParty LinkClient = ()
|
||||
ForwardParty Resolver = ()
|
||||
ForwardParty p =
|
||||
(Int ~ Bool, TypeError (Type.Text "Party " :<>: ShowType p :<>: Type.Text " is not ForwardParty"))
|
||||
|
||||
forwardParty :: SParty p -> Maybe (Dict (PartyI p, ForwardParty p))
|
||||
forwardParty = \case
|
||||
SSender -> Just Dict
|
||||
SSenderLink -> Just Dict
|
||||
SResolver -> Just Dict
|
||||
_ -> Nothing
|
||||
{-# INLINE forwardParty #-}
|
||||
|
||||
data ASubscriberParty = forall p. (PartyI p, ServiceParty p) => ASP (SParty p)
|
||||
|
||||
deriving instance Show ASubscriberParty
|
||||
|
||||
@@ -64,6 +64,7 @@ import Data.Dynamic (toDyn)
|
||||
import Data.Either (fromRight, isRight, partitionEithers)
|
||||
import Data.Foldable (foldrM)
|
||||
import Data.Functor (($>), (<&>))
|
||||
import Data.Hashable (hash)
|
||||
import Data.IORef
|
||||
import Data.Int (Int64)
|
||||
import qualified Data.IntMap.Strict as IM
|
||||
@@ -160,7 +161,6 @@ runSMPServer cfg attachHTTP_ = do
|
||||
runSMPServerBlocking :: MsgStoreClass s => TMVar Bool -> ServerConfig s -> Maybe AttachHTTP -> IO ()
|
||||
runSMPServerBlocking started cfg attachHTTP_ = newEnv cfg >>= runReaderT (smpServer started cfg attachHTTP_)
|
||||
|
||||
type M s a = ReaderT (Env s) IO a
|
||||
type AttachHTTP = Socket -> TLS.Context -> IO ()
|
||||
|
||||
-- actions used in processClientSub to reduce STM transaction scope
|
||||
@@ -1399,7 +1399,7 @@ client
|
||||
(processSub, processNtfSub)
|
||||
Server {subscribers}
|
||||
ms
|
||||
clnt@Client {rcvQ, sndQ, msgQ, clientTHParams = thParams'@THandleParams {sessionId}, procThreads} = do
|
||||
clnt@Client {rcvQ, sndQ, msgQ, clientTHParams = thParams'@THandleParams {sessionId}, procThreads, fwdWorkers} = do
|
||||
labelMyThread . B.unpack $ "client $" <> encode sessionId <> " commands"
|
||||
stats <- asks serverStats
|
||||
authDelay <- asks $ authErrorDelay . config
|
||||
@@ -2195,47 +2195,54 @@ client
|
||||
r2 <- liftEitherWith (const BLOCK) $ EncResponse <$> C.cbEncrypt clientSecret (fromMaybe (C.reverseNonce clientNonce) nonce_) r' paddedProxiedTLength
|
||||
let fr = FwdResponse {fwdCorrId, fwdResponse = r2}
|
||||
pure $ RRES nonce_ $ EncFwdResponse $ C.cbEncryptNoPad sessSecret (C.reverseNonce proxyNonce) (smpEncode fr)
|
||||
-- the inner response, or Nothing if forked (RSLV).
|
||||
r_ <- lift (rejectOrVerify clntThAuth t') >>= \case
|
||||
-- rejectOrVerify filters allowed commands, no need to repeat it here.
|
||||
Left r -> pure $ Just r
|
||||
Right t''@(_, (corrId', entId', cmd')) -> case cmd' of
|
||||
Cmd SResolver (RSLV d) -> lift $ rslvNamesEnv >>= \case
|
||||
Nothing -> pure $ Just (corrId', entId', ERR (NAME NO_RESOLVER))
|
||||
Just nenv -> do
|
||||
let sendResolved msg = do
|
||||
rres <- either ERR id <$> runExceptT (encodeResp (corrId', entId', msg))
|
||||
atomically $ writeTBQueue sndQ ([(corrId, NoEntity, rres)], [])
|
||||
forkCmd_ serverResolverConcurrency forwardedCmdWait (sendResolved rslvTooManyRequests) $
|
||||
resolveNameMsg (thVersion clntTHParams) nenv d >>= sendResolved
|
||||
-- INTERNAL because processCommand never returns Nothing for sender commands;
|
||||
-- `fst` drops the empty message only returned for SUB.
|
||||
_ -> Just . maybe (corrId', entId', ERR INTERNAL) fst <$> lift (processCommand Nothing (Right (M.empty, M.empty, M.empty, M.empty)) t'')
|
||||
-- the inner response, or Nothing if forked (RSLV) or queued to a worker.
|
||||
r_ <- case t' of
|
||||
Left (corrId', entId', e) -> pure $ Just (corrId', entId', ERR e)
|
||||
Right t@(_, _, t''@(corrId', entId', Cmd p cmd)) -> case forwardParty p of
|
||||
Just Dict -> case cmd of
|
||||
RSLV d -> lift $ rslvNamesEnv >>= \case
|
||||
Nothing -> pure $ Just (corrId', entId', ERR (NAME NO_RESOLVER))
|
||||
Just nenv -> do
|
||||
let sendResolved msg = do
|
||||
rres <- either ERR id <$> runExceptT (encodeResp (corrId', entId', msg))
|
||||
atomically $ writeTBQueue sndQ ([(corrId, NoEntity, rres)], [])
|
||||
forkCmd_ serverResolverConcurrency forwardedCmdWait (sendResolved rslvTooManyRequests) $
|
||||
resolveNameMsg (thVersion clntTHParams) nenv d >>= sendResolved
|
||||
_ -> lift $ do
|
||||
q <- forwardWorker entId'
|
||||
atomically $ writeTBQueue q $ do
|
||||
r <- liftIO (verifyTransmission ms Nothing clntThAuth t) >>= \case
|
||||
-- INTERNAL because processCommand never returns Nothing for sender commands;
|
||||
-- `fst` drops the empty message only returned for SUB.
|
||||
VRVerified q_ -> maybe (corrId', entId', ERR INTERNAL) fst <$> processCommand Nothing (Right (M.empty, M.empty, M.empty, M.empty)) (q_, t'')
|
||||
VRFailed e -> pure (corrId', entId', ERR e)
|
||||
rres <- either ERR id <$> runExceptT (encodeResp r)
|
||||
stats <- asks serverStats
|
||||
authDelay <- asks $ authErrorDelay . config
|
||||
liftIO $
|
||||
if isAuthError r
|
||||
then sendAfterDelay clnt stats authDelay startedAt ([(corrId, NoEntity, rres)], [])
|
||||
else atomically $ writeTBQueue sndQ ([(corrId, NoEntity, rres)], [])
|
||||
pure Nothing
|
||||
Nothing -> pure $ Just (corrId', entId', ERR $ CMD PROHIBITED)
|
||||
stats <- asks serverStats
|
||||
incStat $ pMsgFwdsRecv stats
|
||||
traverse encodeResp r_ >>= \case
|
||||
Just rres | any isAuthError r_ -> do
|
||||
authDelay <- asks $ authErrorDelay . config
|
||||
Nothing <$ liftIO (sendAfterDelay clnt stats authDelay startedAt ([(corrId, NoEntity, rres)], []))
|
||||
rres_ -> pure rres_
|
||||
traverse encodeResp r_
|
||||
where
|
||||
rejectOrVerify :: Maybe (THandleAuth 'TServer) -> SignedTransmissionOrError ErrorType Cmd -> M s (VerifiedTransmissionOrError s)
|
||||
rejectOrVerify clntThAuth = \case
|
||||
Left (corrId', entId', e) -> pure $ Left (corrId', entId', ERR e)
|
||||
Right t'@(_, _, t''@(corrId', entId', cmd'))
|
||||
| allowed -> liftIO $ verified <$> verifyTransmission ms Nothing clntThAuth t'
|
||||
| otherwise -> pure $ Left (corrId', entId', ERR $ CMD PROHIBITED)
|
||||
where
|
||||
allowed = case cmd' of
|
||||
Cmd SSender SEND {} -> True
|
||||
Cmd SSender (SKEY _) -> True
|
||||
Cmd SSenderLink (LKEY _) -> True
|
||||
Cmd SSenderLink LGET -> True
|
||||
Cmd SResolver (RSLV _) -> True
|
||||
_ -> False
|
||||
verified = \case
|
||||
VRVerified q -> Right (q, t'')
|
||||
VRFailed e -> Left (corrId', entId', ERR e)
|
||||
forwardWorker :: EntityId -> M s (TBQueue (M s ()))
|
||||
forwardWorker (EntityId eId) = do
|
||||
n <- asks $ serverForwardWorkers . config
|
||||
let i = hash eId `mod` n
|
||||
IM.lookup i <$> readTVarIO fwdWorkers >>= \case
|
||||
Just q -> pure q
|
||||
Nothing -> do
|
||||
q <- newTBQueueIO =<< asks (tbqSize . config)
|
||||
atomically $ modifyTVar' fwdWorkers $ IM.insert i q
|
||||
forkClient clnt (B.unpack ("client $" <> encode sessionId <> " forward ") <> show i) $
|
||||
forever $ do
|
||||
cmd <- atomically $ readTBQueue q
|
||||
cmd `catchAny` \e -> logError $ "forwarded command error: " <> tshow e
|
||||
pure q
|
||||
|
||||
encryptMsg :: QueueRec -> Message -> RcvMessage
|
||||
encryptMsg qr msg = encrypt . encodeRcvMsgBody $ case msg of
|
||||
|
||||
@@ -26,6 +26,7 @@ module Simplex.Messaging.Server.Env.STM
|
||||
StorePaths (..),
|
||||
StartOptions (..),
|
||||
Env (..),
|
||||
M,
|
||||
Server (..),
|
||||
ServerSubscribers (..),
|
||||
SubscribedClients,
|
||||
@@ -67,6 +68,7 @@ module Simplex.Messaging.Server.Env.STM
|
||||
defaultInactiveClientExpiration,
|
||||
defaultProxyClientConcurrency,
|
||||
defaultNameResolverConcurrency,
|
||||
defaultForwardWorkers,
|
||||
defaultMaxClientCmdWait,
|
||||
defaultAuthErrorDelayMs,
|
||||
readWriteQueueStore,
|
||||
@@ -80,6 +82,7 @@ where
|
||||
import Control.Concurrent (ThreadId)
|
||||
import Control.Logger.Simple
|
||||
import Control.Monad
|
||||
import Control.Monad.Reader (ReaderT)
|
||||
import qualified Crypto.PubKey.RSA as RSA
|
||||
import Crypto.Random
|
||||
import Data.Int (Int64)
|
||||
@@ -198,6 +201,7 @@ data ServerConfig s = ServerConfig
|
||||
-- aggregate over a single proxy->relay connection (only servers send proxied
|
||||
-- requests), so bounding them by the per-client limit would throttle unduly.
|
||||
serverResolverConcurrency :: Int,
|
||||
serverForwardWorkers :: Int,
|
||||
maxClientCmdWait :: Int,
|
||||
-- | public-namespace resolver config; Nothing disables the names role
|
||||
namesConfig :: Maybe NamesConfig,
|
||||
@@ -247,6 +251,9 @@ defaultProxyClientConcurrency = 500
|
||||
defaultNameResolverConcurrency :: Int
|
||||
defaultNameResolverConcurrency = 5000
|
||||
|
||||
defaultForwardWorkers :: Int
|
||||
defaultForwardWorkers = 8
|
||||
|
||||
defaultMaxClientCmdWait :: Int
|
||||
defaultMaxClientCmdWait = 60000000 -- microseconds
|
||||
|
||||
@@ -274,6 +281,8 @@ data Env s = Env
|
||||
namesEnv :: Maybe NamesEnv -- public-namespace resolver, present when [NAMES] enable: on
|
||||
}
|
||||
|
||||
type M s a = ReaderT (Env s) IO a
|
||||
|
||||
msgStore :: Env s -> s
|
||||
msgStore = fromMsgStore . msgStore_
|
||||
{-# INLINE msgStore #-}
|
||||
@@ -433,6 +442,7 @@ data Client s = Client
|
||||
sndQ :: TBQueue (NonEmpty (Transmission BrokerMsg), [Transmission BrokerMsg]),
|
||||
msgQ :: TBQueue (NonEmpty (Transmission BrokerMsg)),
|
||||
procThreads :: TVar Int,
|
||||
fwdWorkers :: TVar (IntMap (TBQueue (M s ()))),
|
||||
endThreads :: TVar (IntMap (Weak ThreadId)),
|
||||
endThreadSeq :: TVar Int,
|
||||
clientTHParams :: THandleParams SMPVersion 'TServer,
|
||||
@@ -505,6 +515,7 @@ newClient clientId qSize clientTHParams createdAt = do
|
||||
sndQ <- newTBQueueIO qSize
|
||||
msgQ <- newTBQueueIO qSize
|
||||
procThreads <- newTVarIO 0
|
||||
fwdWorkers <- newTVarIO IM.empty
|
||||
endThreads <- newTVarIO IM.empty
|
||||
endThreadSeq <- newTVarIO 0
|
||||
connected <- newTVarIO True
|
||||
@@ -523,6 +534,7 @@ newClient clientId qSize clientTHParams createdAt = do
|
||||
sndQ,
|
||||
msgQ,
|
||||
procThreads,
|
||||
fwdWorkers,
|
||||
endThreads,
|
||||
endThreadSeq,
|
||||
clientTHParams,
|
||||
|
||||
@@ -531,6 +531,7 @@ smpServerCLI_ generateSite serveStaticFiles attachStaticFiles cfgPath logPath =
|
||||
allowSMPProxy = True,
|
||||
serverClientConcurrency = readIniDefault defaultProxyClientConcurrency "PROXY" "client_concurrency" ini,
|
||||
serverResolverConcurrency = readIniDefault defaultNameResolverConcurrency "NAMES" "resolver_concurrency" ini,
|
||||
serverForwardWorkers = readIniDefault defaultForwardWorkers "PROXY" "forward_workers" ini,
|
||||
maxClientCmdWait = defaultMaxClientCmdWait,
|
||||
namesConfig = readNamesConfig ini,
|
||||
information = serverPublicInfo ini,
|
||||
|
||||
@@ -156,6 +156,10 @@ iniFileContent cfgPath logPath opts host basicAuth controlPortPwds =
|
||||
\# socks_mode = onion\n\n\
|
||||
\# Limit number of threads a client can spawn to process proxy commands and name resolutions in parallel.\n"
|
||||
<> ("# client_concurrency = " <> tshow defaultProxyClientConcurrency)
|
||||
<> "\n\n\
|
||||
\# Number of workers processing messages that one proxy server forwards to this server.\n\
|
||||
\# Messages to one queue are processed by one worker, in the order they are received.\n"
|
||||
<> ("# forward_workers = " <> tshow defaultForwardWorkers)
|
||||
<> "\n\n\
|
||||
\[NAMES]\n\
|
||||
\# Public-namespace resolution via the snrc-resolve.py REST resolver.\n\
|
||||
|
||||
Reference in new issue
Block a user