From e0afca78d3a3259e15a3d413b8ce3fb670144bfc Mon Sep 17 00:00:00 2001 From: "Evgeny @ SimpleX Chat" <259188159+evgeny-simplex@users.noreply.github.com> Date: Wed, 11 Nov 2026 23:41:44 +0000 Subject: [PATCH] smp-server: process forwarded messages to different queues in parallel (#2047) * smp-server: process forwarded messages to different queues in parallel * smp-server: name forward workers without "forwarded" * smp server: process messages of all clients in a shared worker pool * smp server: revert the shared worker pool * smp server: run forwarded commands in workers without join * smp server: queue server actions to forward workers * smp server: choose the forward worker by queue ID outside the command loop * smp server: define the server monad with the environment * smp server: queue forwarded sender commands to workers after verification * smp server: verify forwarded sender commands in their worker * smp server: reject or verify forwarded commands in their worker * smp server: dispatch forwarded commands in one case * smp: restrict forwarded commands to forward parties at type level * smp server: return no response for commands queued to workers * smp server: queue forwarded sender commands in their dispatch branch * smp server: define the forward worker where it is used --- src/Simplex/Messaging/Client.hs | 2 +- src/Simplex/Messaging/Protocol.hs | 17 +++++ src/Simplex/Messaging/Server.hs | 87 ++++++++++++----------- src/Simplex/Messaging/Server/Env/STM.hs | 12 ++++ src/Simplex/Messaging/Server/Main.hs | 1 + src/Simplex/Messaging/Server/Main/Init.hs | 4 ++ tests/SMPClient.hs | 1 + tests/SMPProxyTests.hs | 58 ++++++++++++++- tests/Test.hs | 4 +- 9 files changed, 143 insertions(+), 43 deletions(-) diff --git a/src/Simplex/Messaging/Client.hs b/src/Simplex/Messaging/Client.hs index ba4bceec5..884e783d0 100644 --- a/src/Simplex/Messaging/Client.hs +++ b/src/Simplex/Messaging/Client.hs @@ -1217,7 +1217,7 @@ proxyOKSMPCommand c nm proxiedRelay spKey sId command = proxySMPCommand :: forall p. - PartyI p => + (PartyI p, ForwardParty p) => SMPClient -> NetworkRequestMode -> -- proxy session from PKEY diff --git a/src/Simplex/Messaging/Protocol.hs b/src/Simplex/Messaging/Protocol.hs index 8ff1bfcc0..05efa40c5 100644 --- a/src/Simplex/Messaging/Protocol.hs +++ b/src/Simplex/Messaging/Protocol.hs @@ -71,6 +71,7 @@ module Simplex.Messaging.Protocol QueueParty, BatchParty, ServiceParty, + ForwardParty, ASubscriberParty (..), BrokerMsg (..), SParty (..), @@ -183,6 +184,7 @@ module Simplex.Messaging.Protocol queueParty, batchParty, serviceParty, + forwardParty, partyClientRole, partyServiceRole, userProtocol, @@ -459,6 +461,21 @@ serviceParty = \case _ -> Nothing {-# INLINE serviceParty #-} +type family ForwardParty (p :: Party) :: Constraint where + ForwardParty Sender = () + ForwardParty LinkClient = () + ForwardParty Resolver = () + ForwardParty p = + (Int ~ Bool, TypeError (Type.Text "Party " :<>: ShowType p :<>: Type.Text " is not ForwardParty")) + +forwardParty :: SParty p -> Maybe (Dict (PartyI p, ForwardParty p)) +forwardParty = \case + SSender -> Just Dict + SSenderLink -> Just Dict + SResolver -> Just Dict + _ -> Nothing +{-# INLINE forwardParty #-} + data ASubscriberParty = forall p. (PartyI p, ServiceParty p) => ASP (SParty p) deriving instance Show ASubscriberParty diff --git a/src/Simplex/Messaging/Server.hs b/src/Simplex/Messaging/Server.hs index fd410754a..9c9eb9f11 100644 --- a/src/Simplex/Messaging/Server.hs +++ b/src/Simplex/Messaging/Server.hs @@ -64,6 +64,7 @@ import Data.Dynamic (toDyn) import Data.Either (fromRight, isRight, partitionEithers) import Data.Foldable (foldrM) import Data.Functor (($>), (<&>)) +import Data.Hashable (hash) import Data.IORef import Data.Int (Int64) import qualified Data.IntMap.Strict as IM @@ -160,7 +161,6 @@ runSMPServer cfg attachHTTP_ = do runSMPServerBlocking :: MsgStoreClass s => TMVar Bool -> ServerConfig s -> Maybe AttachHTTP -> IO () runSMPServerBlocking started cfg attachHTTP_ = newEnv cfg >>= runReaderT (smpServer started cfg attachHTTP_) -type M s a = ReaderT (Env s) IO a type AttachHTTP = Socket -> TLS.Context -> IO () -- actions used in processClientSub to reduce STM transaction scope @@ -1399,7 +1399,7 @@ client (processSub, processNtfSub) Server {subscribers} ms - clnt@Client {rcvQ, sndQ, msgQ, clientTHParams = thParams'@THandleParams {sessionId}, procThreads} = do + clnt@Client {rcvQ, sndQ, msgQ, clientTHParams = thParams'@THandleParams {sessionId}, procThreads, fwdWorkers} = do labelMyThread . B.unpack $ "client $" <> encode sessionId <> " commands" stats <- asks serverStats authDelay <- asks $ authErrorDelay . config @@ -2195,47 +2195,54 @@ client r2 <- liftEitherWith (const BLOCK) $ EncResponse <$> C.cbEncrypt clientSecret (fromMaybe (C.reverseNonce clientNonce) nonce_) r' paddedProxiedTLength let fr = FwdResponse {fwdCorrId, fwdResponse = r2} pure $ RRES nonce_ $ EncFwdResponse $ C.cbEncryptNoPad sessSecret (C.reverseNonce proxyNonce) (smpEncode fr) - -- the inner response, or Nothing if forked (RSLV). - r_ <- lift (rejectOrVerify clntThAuth t') >>= \case - -- rejectOrVerify filters allowed commands, no need to repeat it here. - Left r -> pure $ Just r - Right t''@(_, (corrId', entId', cmd')) -> case cmd' of - Cmd SResolver (RSLV d) -> lift $ rslvNamesEnv >>= \case - Nothing -> pure $ Just (corrId', entId', ERR (NAME NO_RESOLVER)) - Just nenv -> do - let sendResolved msg = do - rres <- either ERR id <$> runExceptT (encodeResp (corrId', entId', msg)) - atomically $ writeTBQueue sndQ ([(corrId, NoEntity, rres)], []) - forkCmd_ serverResolverConcurrency forwardedCmdWait (sendResolved rslvTooManyRequests) $ - resolveNameMsg (thVersion clntTHParams) nenv d >>= sendResolved - -- INTERNAL because processCommand never returns Nothing for sender commands; - -- `fst` drops the empty message only returned for SUB. - _ -> Just . maybe (corrId', entId', ERR INTERNAL) fst <$> lift (processCommand Nothing (Right (M.empty, M.empty, M.empty, M.empty)) t'') + -- the inner response, or Nothing if forked (RSLV) or queued to a worker. + r_ <- case t' of + Left (corrId', entId', e) -> pure $ Just (corrId', entId', ERR e) + Right t@(_, _, t''@(corrId', entId', Cmd p cmd)) -> case forwardParty p of + Just Dict -> case cmd of + RSLV d -> lift $ rslvNamesEnv >>= \case + Nothing -> pure $ Just (corrId', entId', ERR (NAME NO_RESOLVER)) + Just nenv -> do + let sendResolved msg = do + rres <- either ERR id <$> runExceptT (encodeResp (corrId', entId', msg)) + atomically $ writeTBQueue sndQ ([(corrId, NoEntity, rres)], []) + forkCmd_ serverResolverConcurrency forwardedCmdWait (sendResolved rslvTooManyRequests) $ + resolveNameMsg (thVersion clntTHParams) nenv d >>= sendResolved + _ -> lift $ do + q <- forwardWorker entId' + atomically $ writeTBQueue q $ do + r <- liftIO (verifyTransmission ms Nothing clntThAuth t) >>= \case + -- INTERNAL because processCommand never returns Nothing for sender commands; + -- `fst` drops the empty message only returned for SUB. + VRVerified q_ -> maybe (corrId', entId', ERR INTERNAL) fst <$> processCommand Nothing (Right (M.empty, M.empty, M.empty, M.empty)) (q_, t'') + VRFailed e -> pure (corrId', entId', ERR e) + rres <- either ERR id <$> runExceptT (encodeResp r) + stats <- asks serverStats + authDelay <- asks $ authErrorDelay . config + liftIO $ + if isAuthError r + then sendAfterDelay clnt stats authDelay startedAt ([(corrId, NoEntity, rres)], []) + else atomically $ writeTBQueue sndQ ([(corrId, NoEntity, rres)], []) + pure Nothing + Nothing -> pure $ Just (corrId', entId', ERR $ CMD PROHIBITED) stats <- asks serverStats incStat $ pMsgFwdsRecv stats - traverse encodeResp r_ >>= \case - Just rres | any isAuthError r_ -> do - authDelay <- asks $ authErrorDelay . config - Nothing <$ liftIO (sendAfterDelay clnt stats authDelay startedAt ([(corrId, NoEntity, rres)], [])) - rres_ -> pure rres_ + traverse encodeResp r_ where - rejectOrVerify :: Maybe (THandleAuth 'TServer) -> SignedTransmissionOrError ErrorType Cmd -> M s (VerifiedTransmissionOrError s) - rejectOrVerify clntThAuth = \case - Left (corrId', entId', e) -> pure $ Left (corrId', entId', ERR e) - Right t'@(_, _, t''@(corrId', entId', cmd')) - | allowed -> liftIO $ verified <$> verifyTransmission ms Nothing clntThAuth t' - | otherwise -> pure $ Left (corrId', entId', ERR $ CMD PROHIBITED) - where - allowed = case cmd' of - Cmd SSender SEND {} -> True - Cmd SSender (SKEY _) -> True - Cmd SSenderLink (LKEY _) -> True - Cmd SSenderLink LGET -> True - Cmd SResolver (RSLV _) -> True - _ -> False - verified = \case - VRVerified q -> Right (q, t'') - VRFailed e -> Left (corrId', entId', ERR e) + forwardWorker :: EntityId -> M s (TBQueue (M s ())) + forwardWorker (EntityId eId) = do + n <- asks $ serverForwardWorkers . config + let i = hash eId `mod` n + IM.lookup i <$> readTVarIO fwdWorkers >>= \case + Just q -> pure q + Nothing -> do + q <- newTBQueueIO =<< asks (tbqSize . config) + atomically $ modifyTVar' fwdWorkers $ IM.insert i q + forkClient clnt (B.unpack ("client $" <> encode sessionId <> " forward ") <> show i) $ + forever $ do + cmd <- atomically $ readTBQueue q + cmd `catchAny` \e -> logError $ "forwarded command error: " <> tshow e + pure q encryptMsg :: QueueRec -> Message -> RcvMessage encryptMsg qr msg = encrypt . encodeRcvMsgBody $ case msg of diff --git a/src/Simplex/Messaging/Server/Env/STM.hs b/src/Simplex/Messaging/Server/Env/STM.hs index a0c803334..e97120751 100644 --- a/src/Simplex/Messaging/Server/Env/STM.hs +++ b/src/Simplex/Messaging/Server/Env/STM.hs @@ -26,6 +26,7 @@ module Simplex.Messaging.Server.Env.STM StorePaths (..), StartOptions (..), Env (..), + M, Server (..), ServerSubscribers (..), SubscribedClients, @@ -67,6 +68,7 @@ module Simplex.Messaging.Server.Env.STM defaultInactiveClientExpiration, defaultProxyClientConcurrency, defaultNameResolverConcurrency, + defaultForwardWorkers, defaultMaxClientCmdWait, defaultAuthErrorDelayMs, readWriteQueueStore, @@ -80,6 +82,7 @@ where import Control.Concurrent (ThreadId) import Control.Logger.Simple import Control.Monad +import Control.Monad.Reader (ReaderT) import qualified Crypto.PubKey.RSA as RSA import Crypto.Random import Data.Int (Int64) @@ -198,6 +201,7 @@ data ServerConfig s = ServerConfig -- aggregate over a single proxy->relay connection (only servers send proxied -- requests), so bounding them by the per-client limit would throttle unduly. serverResolverConcurrency :: Int, + serverForwardWorkers :: Int, maxClientCmdWait :: Int, -- | public-namespace resolver config; Nothing disables the names role namesConfig :: Maybe NamesConfig, @@ -247,6 +251,9 @@ defaultProxyClientConcurrency = 500 defaultNameResolverConcurrency :: Int defaultNameResolverConcurrency = 5000 +defaultForwardWorkers :: Int +defaultForwardWorkers = 8 + defaultMaxClientCmdWait :: Int defaultMaxClientCmdWait = 60000000 -- microseconds @@ -274,6 +281,8 @@ data Env s = Env namesEnv :: Maybe NamesEnv -- public-namespace resolver, present when [NAMES] enable: on } +type M s a = ReaderT (Env s) IO a + msgStore :: Env s -> s msgStore = fromMsgStore . msgStore_ {-# INLINE msgStore #-} @@ -433,6 +442,7 @@ data Client s = Client sndQ :: TBQueue (NonEmpty (Transmission BrokerMsg), [Transmission BrokerMsg]), msgQ :: TBQueue (NonEmpty (Transmission BrokerMsg)), procThreads :: TVar Int, + fwdWorkers :: TVar (IntMap (TBQueue (M s ()))), endThreads :: TVar (IntMap (Weak ThreadId)), endThreadSeq :: TVar Int, clientTHParams :: THandleParams SMPVersion 'TServer, @@ -505,6 +515,7 @@ newClient clientId qSize clientTHParams createdAt = do sndQ <- newTBQueueIO qSize msgQ <- newTBQueueIO qSize procThreads <- newTVarIO 0 + fwdWorkers <- newTVarIO IM.empty endThreads <- newTVarIO IM.empty endThreadSeq <- newTVarIO 0 connected <- newTVarIO True @@ -523,6 +534,7 @@ newClient clientId qSize clientTHParams createdAt = do sndQ, msgQ, procThreads, + fwdWorkers, endThreads, endThreadSeq, clientTHParams, diff --git a/src/Simplex/Messaging/Server/Main.hs b/src/Simplex/Messaging/Server/Main.hs index 7189585f5..65a546838 100644 --- a/src/Simplex/Messaging/Server/Main.hs +++ b/src/Simplex/Messaging/Server/Main.hs @@ -531,6 +531,7 @@ smpServerCLI_ generateSite serveStaticFiles attachStaticFiles cfgPath logPath = allowSMPProxy = True, serverClientConcurrency = readIniDefault defaultProxyClientConcurrency "PROXY" "client_concurrency" ini, serverResolverConcurrency = readIniDefault defaultNameResolverConcurrency "NAMES" "resolver_concurrency" ini, + serverForwardWorkers = readIniDefault defaultForwardWorkers "PROXY" "forward_workers" ini, maxClientCmdWait = defaultMaxClientCmdWait, namesConfig = readNamesConfig ini, information = serverPublicInfo ini, diff --git a/src/Simplex/Messaging/Server/Main/Init.hs b/src/Simplex/Messaging/Server/Main/Init.hs index a0015b0cf..7762ec609 100644 --- a/src/Simplex/Messaging/Server/Main/Init.hs +++ b/src/Simplex/Messaging/Server/Main/Init.hs @@ -156,6 +156,10 @@ iniFileContent cfgPath logPath opts host basicAuth controlPortPwds = \# socks_mode = onion\n\n\ \# Limit number of threads a client can spawn to process proxy commands and name resolutions in parallel.\n" <> ("# client_concurrency = " <> tshow defaultProxyClientConcurrency) + <> "\n\n\ + \# Number of workers processing messages that one proxy server forwards to this server.\n\ + \# Messages to one queue are processed by one worker, in the order they are received.\n" + <> ("# forward_workers = " <> tshow defaultForwardWorkers) <> "\n\n\ \[NAMES]\n\ \# Public-namespace resolution via the snrc-resolve.py REST resolver.\n\ diff --git a/tests/SMPClient.hs b/tests/SMPClient.hs index a8bcbb55c..9ca5f764c 100644 --- a/tests/SMPClient.hs +++ b/tests/SMPClient.hs @@ -286,6 +286,7 @@ cfgMS msType = withStoreCfg (testServerStoreConfig msType) $ \serverStoreCfg -> allowSMPProxy = False, serverClientConcurrency = 2, serverResolverConcurrency = defaultNameResolverConcurrency, + serverForwardWorkers = defaultForwardWorkers, maxClientCmdWait = defaultMaxClientCmdWait, namesConfig = Nothing, information = Nothing, diff --git a/tests/SMPProxyTests.hs b/tests/SMPProxyTests.hs index b2459e443..6ef67ad0b 100644 --- a/tests/SMPProxyTests.hs +++ b/tests/SMPProxyTests.hs @@ -49,6 +49,16 @@ import Test.Hspec hiding (fit, it, xit) import UnliftIO import Util +#if defined(dbServerPostgres) +import Data.Hashable (hash) +import Data.Int (Int64) +import Database.PostgreSQL.Simple (Binary (..), Only (..)) +import qualified Database.PostgreSQL.Simple as PSQL +import Database.PostgreSQL.Simple.Types (Query (..)) +import Simplex.Messaging.Agent.Store.Postgres.Options (DBOpts (..)) +import Simplex.Messaging.Server.Env.STM (defaultForwardWorkers) +#endif + smpProxyTests :: SpecWith AStoreType smpProxyTests = do describe "server configuration" $ do @@ -111,6 +121,7 @@ smpProxyTests = do it "1x1000" $ twoServersFirstProxy $ deliver 1000 it "5x200" $ twoServersFirstProxy $ 5 `inParrallel` deliver 200 it "10x100" $ twoServersFirstProxy $ 10 `inParrallel` deliver 100 + it "500x20" $ twoServersFirstProxy $ 500 `inParrallel` deliver 20 describe "stress test - no host" $ do it "1x1000, no delay" $ oneServer $ proxyConnectDeadRelay 1000 0 srv1 xit "1x1000, 100ms" $ oneServer $ proxyConnectDeadRelay 1000 100000 srv1 @@ -125,7 +136,6 @@ smpProxyTests = do it "100x100 N4 C2" $ twoServersFirstProxy $ withNumCapabilities 4 $ 100 `inParrallel` deliver 100 it "100x100 N4 C16" $ twoServersMoreConc $ withNumCapabilities 4 $ 100 `inParrallel` deliver 100 it "100x100 N" $ twoServersFirstProxy $ withNCPUCapabilities $ 100 `inParrallel` deliver 100 - it "500x20" $ twoServersFirstProxy $ 500 `inParrallel` deliver 20 describe "agent API" $ do describe "one server" $ do it "always via proxy" $ oneServer $ @@ -512,6 +522,52 @@ testForwardedAuthErrorDelay = sendTime `shouldSatisfy` (>= 0.5) getLinkTime `shouldSatisfy` (>= 0.5) +#if defined(dbServerPostgres) +smpProxyPostgresTests :: Spec +smpProxyPostgresTests = + it "relay processes forwarded messages to other queues while one queue is locked" testForwardedQueueLocked + +testForwardedQueueLocked :: HasTestEnv => IO () +testForwardedQueueLocked = + withSmpServerConfigOn (transport @TLS) proxyCfg testPort $ \_ -> + withSmpServerConfigOn (transport @TLS) (cfgS2QS SQSPostgres) testPort2 $ \_ -> do + g <- C.newRandom + ts <- getCurrentTime + rc <- either (fail . show) pure =<< getProtocolClient g NRMInteractive (2, testSMPServer2, Nothing) defaultSMPClientConfig [] Nothing ts (\_ -> pure ()) + let newQueue = do + (rPub, rPriv) <- atomically $ C.generateAuthKeyPair C.SEd448 g + (rdhPub, _ :: C.PrivateKeyX25519) <- atomically $ C.generateKeyPair g + SMP.QIK {rcvId, sndId} <- runExceptT' $ createSMPQueue rc NRMInteractive Nothing (rPub, rPriv) rdhPub Nothing SMOnlyCreate (QRMessaging Nothing) Nothing + pure (rcvId, sndId) + queueWorker (SMP.EntityId sId) = hash sId `mod` defaultForwardWorkers + (SMP.EntityId rIdA, sIdA) <- newQueue + let newQueueB = newQueue >>= \(_, sId) -> if queueWorker sId == queueWorker sIdA then newQueueB else pure sId + sIdB <- newQueueB + pc <- either (fail . show) pure =<< getProtocolClient g NRMInteractive (1, testSMPServer, Nothing) defaultSMPClientConfig [] Nothing ts (\_ -> pure ()) + sess <- runExceptT' $ connectSMPProxiedRelay pc NRMInteractive testSMPServer2 Nothing + let proxySend sId = runExceptT' $ proxySMPMessage pc NRMInteractive sess Nothing sId noMsgFlags "hello" + withPostgres $ \locker -> withPostgres $ \db -> do + PSQL.begin locker + [Only (1 :: Int)] <- PSQL.query locker "SELECT 1 FROM msg_queues WHERE recipient_id = ? FOR UPDATE" (Only $ Binary rIdA) + [Only (lockerPid :: Int)] <- PSQL.query_ locker "SELECT pg_backend_pid()" + withAsync (proxySend sIdA) $ \sendA -> do + let waitBlocked = + PSQL.query db "SELECT count(*) FROM pg_locks WHERE NOT granted AND ? = ANY(pg_blocking_pids(pid))" (Only lockerPid) >>= \case + [Only (n :: Int64)] | n > 0 -> pure () + _ -> threadDelay 10000 >> waitBlocked + timeout 10000000 waitBlocked `shouldReturn` Just () + timeout 10000000 (proxySend sIdB) `shouldReturn` Just (Right ()) + Nothing <- poll sendA + PSQL.rollback locker + wait sendA `shouldReturn` Right () + where + withPostgres = bracket connectPostgres PSQL.close + connectPostgres = do + db <- PSQL.connectPostgreSQL testServerDBConnstr + _ <- PSQL.execute_ db $ Query $ "SET search_path TO " <> schema testStoreDBOpts2 + pure db +#endif + -- Shared "phase 2" of the reconnection tests: start a healthy relay, confirm it is reachable -- directly (PING, not via the proxy) so a proxy failure can only mean the proxy didn't reconnect, -- let any stored connection error expire, then require the proxy to establish the session (PKEY). diff --git a/tests/Test.hs b/tests/Test.hs index 34bcafff8..57d4d8796 100644 --- a/tests/Test.hs +++ b/tests/Test.hs @@ -54,6 +54,7 @@ import CoreTests.XFTPStoreTests (xftpStoreTests, xftpMigrationTests) import NtfServerTests (ntfServerTests) import NtfClient (ntfTestServerDBConnectInfo, ntfTestServerDBOpts) import SMPClient (testServerDBConnectInfo, testServerDBOpts) +import SMPProxyTests (smpProxyPostgresTests) import Simplex.Messaging.Notifications.Server.Store.Migrations (ntfServerMigrations) import Simplex.Messaging.Server.QueueStore.Postgres.Migrations (serverMigrations) import XFTPClient (testXFTPDBConnectInfo) @@ -117,8 +118,9 @@ main = do describe "Notifications server (SMP server: postgres-only store)" $ ntfServerTests (transport @TLS, ASType SQSPostgres SMSPostgres) describe "SMP client agent, server postgres-only message store" $ agentTests (transport @TLS, ASType SQSPostgres SMSPostgres) - describe "SMP proxy, postgres-only message store" $ + describe "SMP proxy, postgres-only message store" $ do before (pure $ ASType SQSPostgres SMSPostgres) smpProxyTests + smpProxyPostgresTests #endif describe "SMP client agent, server memory message store" $ agentTests (transport @TLS, ASType SQSMemory SMSMemory) describe "SMP proxy, memory message store" $