smp-server: process forwarded messages to different queues in parallel (#2047)

* smp-server: process forwarded messages to different queues in parallel

* smp-server: name forward workers without "forwarded"

* smp server: process messages of all clients in a shared worker pool

* smp server: revert the shared worker pool

* smp server: run forwarded commands in workers without join

* smp server: queue server actions to forward workers

* smp server: choose the forward worker by queue ID outside the command loop

* smp server: define the server monad with the environment

* smp server: queue forwarded sender commands to workers after verification

* smp server: verify forwarded sender commands in their worker

* smp server: reject or verify forwarded commands in their worker

* smp server: dispatch forwarded commands in one case

* smp: restrict forwarded commands to forward parties at type level

* smp server: return no response for commands queued to workers

* smp server: queue forwarded sender commands in their dispatch branch

* smp server: define the forward worker where it is used
This commit is contained in:
Evgeny @ SimpleX Chat authored and GitHub committed 2026-11-11 23:41:44 +00:00
1 parent 71179c4d4e
commit e0afca78d3
9 files changed
+143 -43

No files matched your search

+1 -1
View File
@@ -1217,7 +1217,7 @@ proxyOKSMPCommand c nm proxiedRelay spKey sId command =
proxySMPCommand ::
forall p.
PartyI p =>
(PartyI p, ForwardParty p) =>
SMPClient ->
NetworkRequestMode ->
-- proxy session from PKEY
+17
View File
@@ -71,6 +71,7 @@ module Simplex.Messaging.Protocol
QueueParty,
BatchParty,
ServiceParty,
ForwardParty,
ASubscriberParty (..),
BrokerMsg (..),
SParty (..),
@@ -183,6 +184,7 @@ module Simplex.Messaging.Protocol
queueParty,
batchParty,
serviceParty,
forwardParty,
partyClientRole,
partyServiceRole,
userProtocol,
@@ -459,6 +461,21 @@ serviceParty = \case
_ -> Nothing
{-# INLINE serviceParty #-}
type family ForwardParty (p :: Party) :: Constraint where
ForwardParty Sender = ()
ForwardParty LinkClient = ()
ForwardParty Resolver = ()
ForwardParty p =
(Int ~ Bool, TypeError (Type.Text "Party " :<>: ShowType p :<>: Type.Text " is not ForwardParty"))
forwardParty :: SParty p -> Maybe (Dict (PartyI p, ForwardParty p))
forwardParty = \case
SSender -> Just Dict
SSenderLink -> Just Dict
SResolver -> Just Dict
_ -> Nothing
{-# INLINE forwardParty #-}
data ASubscriberParty = forall p. (PartyI p, ServiceParty p) => ASP (SParty p)
deriving instance Show ASubscriberParty
+47 -40
View File
@@ -64,6 +64,7 @@ import Data.Dynamic (toDyn)
import Data.Either (fromRight, isRight, partitionEithers)
import Data.Foldable (foldrM)
import Data.Functor (($>), (<&>))
import Data.Hashable (hash)
import Data.IORef
import Data.Int (Int64)
import qualified Data.IntMap.Strict as IM
@@ -160,7 +161,6 @@ runSMPServer cfg attachHTTP_ = do
runSMPServerBlocking :: MsgStoreClass s => TMVar Bool -> ServerConfig s -> Maybe AttachHTTP -> IO ()
runSMPServerBlocking started cfg attachHTTP_ = newEnv cfg >>= runReaderT (smpServer started cfg attachHTTP_)
type M s a = ReaderT (Env s) IO a
type AttachHTTP = Socket -> TLS.Context -> IO ()
-- actions used in processClientSub to reduce STM transaction scope
@@ -1399,7 +1399,7 @@ client
(processSub, processNtfSub)
Server {subscribers}
ms
clnt@Client {rcvQ, sndQ, msgQ, clientTHParams = thParams'@THandleParams {sessionId}, procThreads} = do
clnt@Client {rcvQ, sndQ, msgQ, clientTHParams = thParams'@THandleParams {sessionId}, procThreads, fwdWorkers} = do
labelMyThread . B.unpack $ "client $" <> encode sessionId <> " commands"
stats <- asks serverStats
authDelay <- asks $ authErrorDelay . config
@@ -2195,47 +2195,54 @@ client
r2 <- liftEitherWith (const BLOCK) $ EncResponse <$> C.cbEncrypt clientSecret (fromMaybe (C.reverseNonce clientNonce) nonce_) r' paddedProxiedTLength
let fr = FwdResponse {fwdCorrId, fwdResponse = r2}
pure $ RRES nonce_ $ EncFwdResponse $ C.cbEncryptNoPad sessSecret (C.reverseNonce proxyNonce) (smpEncode fr)
-- the inner response, or Nothing if forked (RSLV).
r_ <- lift (rejectOrVerify clntThAuth t') >>= \case
-- rejectOrVerify filters allowed commands, no need to repeat it here.
Left r -> pure $ Just r
Right t''@(_, (corrId', entId', cmd')) -> case cmd' of
Cmd SResolver (RSLV d) -> lift $ rslvNamesEnv >>= \case
Nothing -> pure $ Just (corrId', entId', ERR (NAME NO_RESOLVER))
Just nenv -> do
let sendResolved msg = do
rres <- either ERR id <$> runExceptT (encodeResp (corrId', entId', msg))
atomically $ writeTBQueue sndQ ([(corrId, NoEntity, rres)], [])
forkCmd_ serverResolverConcurrency forwardedCmdWait (sendResolved rslvTooManyRequests) $
resolveNameMsg (thVersion clntTHParams) nenv d >>= sendResolved
-- INTERNAL because processCommand never returns Nothing for sender commands;
-- `fst` drops the empty message only returned for SUB.
_ -> Just . maybe (corrId', entId', ERR INTERNAL) fst <$> lift (processCommand Nothing (Right (M.empty, M.empty, M.empty, M.empty)) t'')
-- the inner response, or Nothing if forked (RSLV) or queued to a worker.
r_ <- case t' of
Left (corrId', entId', e) -> pure $ Just (corrId', entId', ERR e)
Right t@(_, _, t''@(corrId', entId', Cmd p cmd)) -> case forwardParty p of
Just Dict -> case cmd of
RSLV d -> lift $ rslvNamesEnv >>= \case
Nothing -> pure $ Just (corrId', entId', ERR (NAME NO_RESOLVER))
Just nenv -> do
let sendResolved msg = do
rres <- either ERR id <$> runExceptT (encodeResp (corrId', entId', msg))
atomically $ writeTBQueue sndQ ([(corrId, NoEntity, rres)], [])
forkCmd_ serverResolverConcurrency forwardedCmdWait (sendResolved rslvTooManyRequests) $
resolveNameMsg (thVersion clntTHParams) nenv d >>= sendResolved
_ -> lift $ do
q <- forwardWorker entId'
atomically $ writeTBQueue q $ do
r <- liftIO (verifyTransmission ms Nothing clntThAuth t) >>= \case
-- INTERNAL because processCommand never returns Nothing for sender commands;
-- `fst` drops the empty message only returned for SUB.
VRVerified q_ -> maybe (corrId', entId', ERR INTERNAL) fst <$> processCommand Nothing (Right (M.empty, M.empty, M.empty, M.empty)) (q_, t'')
VRFailed e -> pure (corrId', entId', ERR e)
rres <- either ERR id <$> runExceptT (encodeResp r)
stats <- asks serverStats
authDelay <- asks $ authErrorDelay . config
liftIO $
if isAuthError r
then sendAfterDelay clnt stats authDelay startedAt ([(corrId, NoEntity, rres)], [])
else atomically $ writeTBQueue sndQ ([(corrId, NoEntity, rres)], [])
pure Nothing
Nothing -> pure $ Just (corrId', entId', ERR $ CMD PROHIBITED)
stats <- asks serverStats
incStat $ pMsgFwdsRecv stats
traverse encodeResp r_ >>= \case
Just rres | any isAuthError r_ -> do
authDelay <- asks $ authErrorDelay . config
Nothing <$ liftIO (sendAfterDelay clnt stats authDelay startedAt ([(corrId, NoEntity, rres)], []))
rres_ -> pure rres_
traverse encodeResp r_
where
rejectOrVerify :: Maybe (THandleAuth 'TServer) -> SignedTransmissionOrError ErrorType Cmd -> M s (VerifiedTransmissionOrError s)
rejectOrVerify clntThAuth = \case
Left (corrId', entId', e) -> pure $ Left (corrId', entId', ERR e)
Right t'@(_, _, t''@(corrId', entId', cmd'))
| allowed -> liftIO $ verified <$> verifyTransmission ms Nothing clntThAuth t'
| otherwise -> pure $ Left (corrId', entId', ERR $ CMD PROHIBITED)
where
allowed = case cmd' of
Cmd SSender SEND {} -> True
Cmd SSender (SKEY _) -> True
Cmd SSenderLink (LKEY _) -> True
Cmd SSenderLink LGET -> True
Cmd SResolver (RSLV _) -> True
_ -> False
verified = \case
VRVerified q -> Right (q, t'')
VRFailed e -> Left (corrId', entId', ERR e)
forwardWorker :: EntityId -> M s (TBQueue (M s ()))
forwardWorker (EntityId eId) = do
n <- asks $ serverForwardWorkers . config
let i = hash eId `mod` n
IM.lookup i <$> readTVarIO fwdWorkers >>= \case
Just q -> pure q
Nothing -> do
q <- newTBQueueIO =<< asks (tbqSize . config)
atomically $ modifyTVar' fwdWorkers $ IM.insert i q
forkClient clnt (B.unpack ("client $" <> encode sessionId <> " forward ") <> show i) $
forever $ do
cmd <- atomically $ readTBQueue q
cmd `catchAny` \e -> logError $ "forwarded command error: " <> tshow e
pure q
encryptMsg :: QueueRec -> Message -> RcvMessage
encryptMsg qr msg = encrypt . encodeRcvMsgBody $ case msg of
+12
View File
@@ -26,6 +26,7 @@ module Simplex.Messaging.Server.Env.STM
StorePaths (..),
StartOptions (..),
Env (..),
M,
Server (..),
ServerSubscribers (..),
SubscribedClients,
@@ -67,6 +68,7 @@ module Simplex.Messaging.Server.Env.STM
defaultInactiveClientExpiration,
defaultProxyClientConcurrency,
defaultNameResolverConcurrency,
defaultForwardWorkers,
defaultMaxClientCmdWait,
defaultAuthErrorDelayMs,
readWriteQueueStore,
@@ -80,6 +82,7 @@ where
import Control.Concurrent (ThreadId)
import Control.Logger.Simple
import Control.Monad
import Control.Monad.Reader (ReaderT)
import qualified Crypto.PubKey.RSA as RSA
import Crypto.Random
import Data.Int (Int64)
@@ -198,6 +201,7 @@ data ServerConfig s = ServerConfig
-- aggregate over a single proxy->relay connection (only servers send proxied
-- requests), so bounding them by the per-client limit would throttle unduly.
serverResolverConcurrency :: Int,
serverForwardWorkers :: Int,
maxClientCmdWait :: Int,
-- | public-namespace resolver config; Nothing disables the names role
namesConfig :: Maybe NamesConfig,
@@ -247,6 +251,9 @@ defaultProxyClientConcurrency = 500
defaultNameResolverConcurrency :: Int
defaultNameResolverConcurrency = 5000
defaultForwardWorkers :: Int
defaultForwardWorkers = 8
defaultMaxClientCmdWait :: Int
defaultMaxClientCmdWait = 60000000 -- microseconds
@@ -274,6 +281,8 @@ data Env s = Env
namesEnv :: Maybe NamesEnv -- public-namespace resolver, present when [NAMES] enable: on
}
type M s a = ReaderT (Env s) IO a
msgStore :: Env s -> s
msgStore = fromMsgStore . msgStore_
{-# INLINE msgStore #-}
@@ -433,6 +442,7 @@ data Client s = Client
sndQ :: TBQueue (NonEmpty (Transmission BrokerMsg), [Transmission BrokerMsg]),
msgQ :: TBQueue (NonEmpty (Transmission BrokerMsg)),
procThreads :: TVar Int,
fwdWorkers :: TVar (IntMap (TBQueue (M s ()))),
endThreads :: TVar (IntMap (Weak ThreadId)),
endThreadSeq :: TVar Int,
clientTHParams :: THandleParams SMPVersion 'TServer,
@@ -505,6 +515,7 @@ newClient clientId qSize clientTHParams createdAt = do
sndQ <- newTBQueueIO qSize
msgQ <- newTBQueueIO qSize
procThreads <- newTVarIO 0
fwdWorkers <- newTVarIO IM.empty
endThreads <- newTVarIO IM.empty
endThreadSeq <- newTVarIO 0
connected <- newTVarIO True
@@ -523,6 +534,7 @@ newClient clientId qSize clientTHParams createdAt = do
sndQ,
msgQ,
procThreads,
fwdWorkers,
endThreads,
endThreadSeq,
clientTHParams,
+1
View File
@@ -531,6 +531,7 @@ smpServerCLI_ generateSite serveStaticFiles attachStaticFiles cfgPath logPath =
allowSMPProxy = True,
serverClientConcurrency = readIniDefault defaultProxyClientConcurrency "PROXY" "client_concurrency" ini,
serverResolverConcurrency = readIniDefault defaultNameResolverConcurrency "NAMES" "resolver_concurrency" ini,
serverForwardWorkers = readIniDefault defaultForwardWorkers "PROXY" "forward_workers" ini,
maxClientCmdWait = defaultMaxClientCmdWait,
namesConfig = readNamesConfig ini,
information = serverPublicInfo ini,
@@ -156,6 +156,10 @@ iniFileContent cfgPath logPath opts host basicAuth controlPortPwds =
\# socks_mode = onion\n\n\
\# Limit number of threads a client can spawn to process proxy commands and name resolutions in parallel.\n"
<> ("# client_concurrency = " <> tshow defaultProxyClientConcurrency)
<> "\n\n\
\# Number of workers processing messages that one proxy server forwards to this server.\n\
\# Messages to one queue are processed by one worker, in the order they are received.\n"
<> ("# forward_workers = " <> tshow defaultForwardWorkers)
<> "\n\n\
\[NAMES]\n\
\# Public-namespace resolution via the snrc-resolve.py REST resolver.\n\
+1
View File
@@ -286,6 +286,7 @@ cfgMS msType = withStoreCfg (testServerStoreConfig msType) $ \serverStoreCfg ->
allowSMPProxy = False,
serverClientConcurrency = 2,
serverResolverConcurrency = defaultNameResolverConcurrency,
serverForwardWorkers = defaultForwardWorkers,
maxClientCmdWait = defaultMaxClientCmdWait,
namesConfig = Nothing,
information = Nothing,
+57 -1
View File
@@ -49,6 +49,16 @@ import Test.Hspec hiding (fit, it, xit)
import UnliftIO
import Util
#if defined(dbServerPostgres)
import Data.Hashable (hash)
import Data.Int (Int64)
import Database.PostgreSQL.Simple (Binary (..), Only (..))
import qualified Database.PostgreSQL.Simple as PSQL
import Database.PostgreSQL.Simple.Types (Query (..))
import Simplex.Messaging.Agent.Store.Postgres.Options (DBOpts (..))
import Simplex.Messaging.Server.Env.STM (defaultForwardWorkers)
#endif
smpProxyTests :: SpecWith AStoreType
smpProxyTests = do
describe "server configuration" $ do
@@ -111,6 +121,7 @@ smpProxyTests = do
it "1x1000" $ twoServersFirstProxy $ deliver 1000
it "5x200" $ twoServersFirstProxy $ 5 `inParrallel` deliver 200
it "10x100" $ twoServersFirstProxy $ 10 `inParrallel` deliver 100
it "500x20" $ twoServersFirstProxy $ 500 `inParrallel` deliver 20
describe "stress test - no host" $ do
it "1x1000, no delay" $ oneServer $ proxyConnectDeadRelay 1000 0 srv1
xit "1x1000, 100ms" $ oneServer $ proxyConnectDeadRelay 1000 100000 srv1
@@ -125,7 +136,6 @@ smpProxyTests = do
it "100x100 N4 C2" $ twoServersFirstProxy $ withNumCapabilities 4 $ 100 `inParrallel` deliver 100
it "100x100 N4 C16" $ twoServersMoreConc $ withNumCapabilities 4 $ 100 `inParrallel` deliver 100
it "100x100 N" $ twoServersFirstProxy $ withNCPUCapabilities $ 100 `inParrallel` deliver 100
it "500x20" $ twoServersFirstProxy $ 500 `inParrallel` deliver 20
describe "agent API" $ do
describe "one server" $ do
it "always via proxy" $ oneServer $
@@ -512,6 +522,52 @@ testForwardedAuthErrorDelay =
sendTime `shouldSatisfy` (>= 0.5)
getLinkTime `shouldSatisfy` (>= 0.5)
#if defined(dbServerPostgres)
smpProxyPostgresTests :: Spec
smpProxyPostgresTests =
it "relay processes forwarded messages to other queues while one queue is locked" testForwardedQueueLocked
testForwardedQueueLocked :: HasTestEnv => IO ()
testForwardedQueueLocked =
withSmpServerConfigOn (transport @TLS) proxyCfg testPort $ \_ ->
withSmpServerConfigOn (transport @TLS) (cfgS2QS SQSPostgres) testPort2 $ \_ -> do
g <- C.newRandom
ts <- getCurrentTime
rc <- either (fail . show) pure =<< getProtocolClient g NRMInteractive (2, testSMPServer2, Nothing) defaultSMPClientConfig [] Nothing ts (\_ -> pure ())
let newQueue = do
(rPub, rPriv) <- atomically $ C.generateAuthKeyPair C.SEd448 g
(rdhPub, _ :: C.PrivateKeyX25519) <- atomically $ C.generateKeyPair g
SMP.QIK {rcvId, sndId} <- runExceptT' $ createSMPQueue rc NRMInteractive Nothing (rPub, rPriv) rdhPub Nothing SMOnlyCreate (QRMessaging Nothing) Nothing
pure (rcvId, sndId)
queueWorker (SMP.EntityId sId) = hash sId `mod` defaultForwardWorkers
(SMP.EntityId rIdA, sIdA) <- newQueue
let newQueueB = newQueue >>= \(_, sId) -> if queueWorker sId == queueWorker sIdA then newQueueB else pure sId
sIdB <- newQueueB
pc <- either (fail . show) pure =<< getProtocolClient g NRMInteractive (1, testSMPServer, Nothing) defaultSMPClientConfig [] Nothing ts (\_ -> pure ())
sess <- runExceptT' $ connectSMPProxiedRelay pc NRMInteractive testSMPServer2 Nothing
let proxySend sId = runExceptT' $ proxySMPMessage pc NRMInteractive sess Nothing sId noMsgFlags "hello"
withPostgres $ \locker -> withPostgres $ \db -> do
PSQL.begin locker
[Only (1 :: Int)] <- PSQL.query locker "SELECT 1 FROM msg_queues WHERE recipient_id = ? FOR UPDATE" (Only $ Binary rIdA)
[Only (lockerPid :: Int)] <- PSQL.query_ locker "SELECT pg_backend_pid()"
withAsync (proxySend sIdA) $ \sendA -> do
let waitBlocked =
PSQL.query db "SELECT count(*) FROM pg_locks WHERE NOT granted AND ? = ANY(pg_blocking_pids(pid))" (Only lockerPid) >>= \case
[Only (n :: Int64)] | n > 0 -> pure ()
_ -> threadDelay 10000 >> waitBlocked
timeout 10000000 waitBlocked `shouldReturn` Just ()
timeout 10000000 (proxySend sIdB) `shouldReturn` Just (Right ())
Nothing <- poll sendA
PSQL.rollback locker
wait sendA `shouldReturn` Right ()
where
withPostgres = bracket connectPostgres PSQL.close
connectPostgres = do
db <- PSQL.connectPostgreSQL testServerDBConnstr
_ <- PSQL.execute_ db $ Query $ "SET search_path TO " <> schema testStoreDBOpts2
pure db
#endif
-- Shared "phase 2" of the reconnection tests: start a healthy relay, confirm it is reachable
-- directly (PING, not via the proxy) so a proxy failure can only mean the proxy didn't reconnect,
-- let any stored connection error expire, then require the proxy to establish the session (PKEY).
+3 -1
View File
@@ -54,6 +54,7 @@ import CoreTests.XFTPStoreTests (xftpStoreTests, xftpMigrationTests)
import NtfServerTests (ntfServerTests)
import NtfClient (ntfTestServerDBConnectInfo, ntfTestServerDBOpts)
import SMPClient (testServerDBConnectInfo, testServerDBOpts)
import SMPProxyTests (smpProxyPostgresTests)
import Simplex.Messaging.Notifications.Server.Store.Migrations (ntfServerMigrations)
import Simplex.Messaging.Server.QueueStore.Postgres.Migrations (serverMigrations)
import XFTPClient (testXFTPDBConnectInfo)
@@ -117,8 +118,9 @@ main = do
describe "Notifications server (SMP server: postgres-only store)" $
ntfServerTests (transport @TLS, ASType SQSPostgres SMSPostgres)
describe "SMP client agent, server postgres-only message store" $ agentTests (transport @TLS, ASType SQSPostgres SMSPostgres)
describe "SMP proxy, postgres-only message store" $
describe "SMP proxy, postgres-only message store" $ do
before (pure $ ASType SQSPostgres SMSPostgres) smpProxyTests
smpProxyPostgresTests
#endif
describe "SMP client agent, server memory message store" $ agentTests (transport @TLS, ASType SQSMemory SMSMemory)
describe "SMP proxy, memory message store" $