diff --git a/src/Simplex/Messaging/Server.hs b/src/Simplex/Messaging/Server.hs index 9c9eb9f11..fe0e405fa 100644 --- a/src/Simplex/Messaging/Server.hs +++ b/src/Simplex/Messaging/Server.hs @@ -695,8 +695,10 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOpt smpSubs <- getSubscribersMetrics subscribers ntfSubs <- getSubscribersMetrics ntfSubscribers loadedCounts <- loadedQueueCounts $ fromMsgStore ms - pure RealTimeMetrics {socketStats, threadsCount, clientsCount, deliveredSubs, deliveredTimes, smpSubs, ntfSubs, loadedCounts} + fwdWorkerQLength <- fmap sum . mapM fwdQLength =<< getServerClients srv + pure RealTimeMetrics {socketStats, threadsCount, clientsCount, deliveredSubs, deliveredTimes, smpSubs, ntfSubs, loadedCounts, fwdWorkerQLength} where + fwdQLength Client {fwdWorkers} = atomically $ readTVar fwdWorkers >>= fmap sum . mapM lengthTBQueue getSubscribersMetrics ServerSubscribers {queueSubscribers, serviceSubscribers, totalServiceSubs, subClients} = do subsCount <- M.size <$> getSubscribedClients queueSubscribers subClientsCount <- IS.size <$> readTVarIO subClients @@ -1208,6 +1210,13 @@ updateSlotWaitStats ServerStats {cmdSlotWaits, cmdSlotWaitMs, cmdSlotWaitMaxMs, atomicModifyIORef'_ cmdSlotWaitMaxMs (max waitMs) unless taken $ incStat cmdSlotWaitTimeouts +updateFwdWorkerStats :: ServerStats -> SystemTime -> IO () +updateFwdWorkerStats ServerStats {fwdWorkerCmds, fwdWorkerCmdMs, fwdWorkerCmdMaxMs} since = do + cmdMs <- fromIntegral . (`div` 1000) . subtract (microseconds since) . microseconds <$> getSystemTime + incStat fwdWorkerCmds + atomicModifyIORef'_ fwdWorkerCmdMs (+ cmdMs) + atomicModifyIORef'_ fwdWorkerCmdMaxMs (max cmdMs) + rslvTooManyRequests :: BrokerMsg rslvTooManyRequests = ERR $ NAME $ RESOLVER "too many requests" @@ -2219,10 +2228,11 @@ client rres <- either ERR id <$> runExceptT (encodeResp r) stats <- asks serverStats authDelay <- asks $ authErrorDelay . config - liftIO $ + liftIO $ do if isAuthError r then sendAfterDelay clnt stats authDelay startedAt ([(corrId, NoEntity, rres)], []) else atomically $ writeTBQueue sndQ ([(corrId, NoEntity, rres)], []) + updateFwdWorkerStats stats startedAt pure Nothing Nothing -> pure $ Just (corrId', entId', ERR $ CMD PROHIBITED) stats <- asks serverStats diff --git a/src/Simplex/Messaging/Server/Prometheus.hs b/src/Simplex/Messaging/Server/Prometheus.hs index 41ad5517b..d0c8f4480 100644 --- a/src/Simplex/Messaging/Server/Prometheus.hs +++ b/src/Simplex/Messaging/Server/Prometheus.hs @@ -20,6 +20,7 @@ import Data.Time.Clock (UTCTime (..), diffUTCTime) import Data.Time.Clock.System (systemEpochDay) import Data.Time.Format.ISO8601 (iso8601Show) import Network.Socket (ServiceName) +import Numeric.Natural (Natural) import Simplex.Messaging.Server.MsgStore.Types (LoadedQueueCounts (..)) import Simplex.Messaging.Server.QueueStore.Types (EntityCounts (..)) import Simplex.Messaging.Server.Stats @@ -46,7 +47,8 @@ data RealTimeMetrics = RealTimeMetrics deliveredTimes :: TimeBuckets, smpSubs :: RTSubscriberMetrics, ntfSubs :: RTSubscriberMetrics, - loadedCounts :: LoadedQueueCounts + loadedCounts :: LoadedQueueCounts, + fwdWorkerQLength :: Natural } data RTSubscriberMetrics = RTSubscriberMetrics @@ -70,7 +72,8 @@ prometheusMetrics sm rtm ts = deliveredTimes, smpSubs, ntfSubs, - loadedCounts + loadedCounts, + fwdWorkerQLength } = rtm ServerStatsData { _fromTime, @@ -131,6 +134,9 @@ prometheusMetrics sm rtm ts = _cmdSlotWaitMs, _cmdSlotWaitMaxMs, _cmdSlotWaitTimeouts, + _fwdWorkerCmds, + _fwdWorkerCmdMs, + _fwdWorkerCmdMaxMs, _qCount, _msgCount, _ntfCount, @@ -395,6 +401,19 @@ prometheusMetrics sm rtm ts = \# HELP simplex_smp_relay_messages_received Relay messages statistics.\n\ \# TYPE simplex_smp_relay_messages_received counter\n\ \simplex_smp_relay_messages_received " <> mshow _pMsgFwdsRecv <> "\n# pMsgFwdsRecv\n\ + \\n\ + \# HELP simplex_smp_forward_worker_cmd_ms Time from the start of processing a forwarded command to queueing its response, for commands queued to forward workers, including the wait in the worker queue and excluding AUTH error delay, in ms.\n\ + \# TYPE simplex_smp_forward_worker_cmd_ms summary\n\ + \simplex_smp_forward_worker_cmd_ms_sum " <> mshow _fwdWorkerCmdMs <> "\n# fwdWorkerCmdMs\n\ + \simplex_smp_forward_worker_cmd_ms_count " <> mshow _fwdWorkerCmds <> "\n# fwdWorkerCmds\n\ + \\n\ + \# HELP simplex_smp_forward_worker_cmd_ms_max The longest time of simplex_smp_forward_worker_cmd_ms since the server start, in ms.\n\ + \# TYPE simplex_smp_forward_worker_cmd_ms_max gauge\n\ + \simplex_smp_forward_worker_cmd_ms_max " <> mshow _fwdWorkerCmdMaxMs <> "\n# fwdWorkerCmdMaxMs\n\ + \\n\ + \# HELP simplex_smp_forward_worker_queue_length Forwarded commands waiting in forward worker queues.\n\ + \# TYPE simplex_smp_forward_worker_queue_length gauge\n\ + \simplex_smp_forward_worker_queue_length " <> mshow fwdWorkerQLength <> "\n# fwdWorkerQLength\n\ \\n" services = "# Services\n\ diff --git a/src/Simplex/Messaging/Server/Stats.hs b/src/Simplex/Messaging/Server/Stats.hs index a1c9dfa29..70a450652 100644 --- a/src/Simplex/Messaging/Server/Stats.hs +++ b/src/Simplex/Messaging/Server/Stats.hs @@ -134,6 +134,9 @@ data ServerStats = ServerStats cmdSlotWaitMs :: IORef Int, cmdSlotWaitMaxMs :: IORef Int, cmdSlotWaitTimeouts :: IORef Int, + fwdWorkerCmds :: IORef Int, + fwdWorkerCmdMs :: IORef Int, + fwdWorkerCmdMaxMs :: IORef Int, qCount :: IORef Int, msgCount :: IORef Int, ntfCount :: IORef Int, @@ -201,6 +204,9 @@ data ServerStatsData = ServerStatsData _cmdSlotWaitMs :: Int, _cmdSlotWaitMaxMs :: Int, _cmdSlotWaitTimeouts :: Int, + _fwdWorkerCmds :: Int, + _fwdWorkerCmdMs :: Int, + _fwdWorkerCmdMaxMs :: Int, _qCount :: Int, _msgCount :: Int, _ntfCount :: Int, @@ -270,6 +276,9 @@ newServerStats ts = do cmdSlotWaitMs <- newIORef 0 cmdSlotWaitMaxMs <- newIORef 0 cmdSlotWaitTimeouts <- newIORef 0 + fwdWorkerCmds <- newIORef 0 + fwdWorkerCmdMs <- newIORef 0 + fwdWorkerCmdMaxMs <- newIORef 0 qCount <- newIORef 0 msgCount <- newIORef 0 ntfCount <- newIORef 0 @@ -336,6 +345,9 @@ newServerStats ts = do cmdSlotWaitMs, cmdSlotWaitMaxMs, cmdSlotWaitTimeouts, + fwdWorkerCmds, + fwdWorkerCmdMs, + fwdWorkerCmdMaxMs, qCount, msgCount, ntfCount, @@ -404,6 +416,9 @@ getServerStatsData s = do _cmdSlotWaitMs <- readIORef $ cmdSlotWaitMs s _cmdSlotWaitMaxMs <- readIORef $ cmdSlotWaitMaxMs s _cmdSlotWaitTimeouts <- readIORef $ cmdSlotWaitTimeouts s + _fwdWorkerCmds <- readIORef $ fwdWorkerCmds s + _fwdWorkerCmdMs <- readIORef $ fwdWorkerCmdMs s + _fwdWorkerCmdMaxMs <- readIORef $ fwdWorkerCmdMaxMs s _qCount <- readIORef $ qCount s _msgCount <- readIORef $ msgCount s _ntfCount <- readIORef $ ntfCount s @@ -470,6 +485,9 @@ getServerStatsData s = do _cmdSlotWaitMs, _cmdSlotWaitMaxMs, _cmdSlotWaitTimeouts, + _fwdWorkerCmds, + _fwdWorkerCmdMs, + _fwdWorkerCmdMaxMs, _qCount, _msgCount, _ntfCount, @@ -539,6 +557,9 @@ setServerStats s d = do writeIORef (cmdSlotWaitMs s) $! _cmdSlotWaitMs d writeIORef (cmdSlotWaitMaxMs s) $! _cmdSlotWaitMaxMs d writeIORef (cmdSlotWaitTimeouts s) $! _cmdSlotWaitTimeouts d + writeIORef (fwdWorkerCmds s) $! _fwdWorkerCmds d + writeIORef (fwdWorkerCmdMs s) $! _fwdWorkerCmdMs d + writeIORef (fwdWorkerCmdMaxMs s) $! _fwdWorkerCmdMaxMs d writeIORef (qCount s) $! _qCount d writeIORef (msgCount s) $! _msgCount d writeIORef (ntfCount s) $! _ntfCount d @@ -746,6 +767,9 @@ instance StrEncoding ServerStatsData where _cmdSlotWaitMs = 0, _cmdSlotWaitMaxMs = 0, _cmdSlotWaitTimeouts = 0, + _fwdWorkerCmds = 0, + _fwdWorkerCmdMs = 0, + _fwdWorkerCmdMaxMs = 0, _qCount, _msgCount = 0, _ntfCount = 0, diff --git a/tests/SMPProxyTests.hs b/tests/SMPProxyTests.hs index 6ef67ad0b..e60631b20 100644 --- a/tests/SMPProxyTests.hs +++ b/tests/SMPProxyTests.hs @@ -50,6 +50,7 @@ import UnliftIO import Util #if defined(dbServerPostgres) +import qualified Data.ByteString.Char8 as B import Data.Hashable (hash) import Data.Int (Int64) import Database.PostgreSQL.Simple (Binary (..), Only (..)) @@ -530,7 +531,7 @@ smpProxyPostgresTests = testForwardedQueueLocked :: HasTestEnv => IO () testForwardedQueueLocked = withSmpServerConfigOn (transport @TLS) proxyCfg testPort $ \_ -> - withSmpServerConfigOn (transport @TLS) (cfgS2QS SQSPostgres) testPort2 $ \_ -> do + withSmpServerConfigOn (transport @TLS) (updateCfg (cfgS2QS SQSPostgres) $ \cfg_ -> cfg_ {prometheusInterval = Just 1}) testPort2 $ \_ -> do g <- C.newRandom ts <- getCurrentTime rc <- either (fail . show) pure =<< getProtocolClient g NRMInteractive (2, testSMPServer2, Nothing) defaultSMPClientConfig [] Nothing ts (\_ -> pure ()) @@ -557,10 +558,24 @@ testForwardedQueueLocked = _ -> threadDelay 10000 >> waitBlocked timeout 10000000 waitBlocked `shouldReturn` Just () timeout 10000000 (proxySend sIdB) `shouldReturn` Just (Right ()) - Nothing <- poll sendA - PSQL.rollback locker - wait sendA `shouldReturn` Right () + withAsync (proxySend sIdA) $ \sendA' -> do + threadDelay 1500000 + metric "simplex_smp_forward_worker_queue_length" `shouldReturn` 1 + Nothing <- poll sendA + PSQL.rollback locker + wait sendA `shouldReturn` Right () + wait sendA' `shouldReturn` Right () + threadDelay 1500000 + metric "simplex_smp_forward_worker_cmd_ms_count" `shouldReturn` 3 + maxMs <- metric "simplex_smp_forward_worker_cmd_ms_max" + maxMs `shouldSatisfy` (>= 1500) + metric "simplex_smp_forward_worker_cmd_ms_sum" >>= (`shouldSatisfy` (>= maxMs + 1000)) where + metric name = do + ms <- map B.words . B.lines <$> B.readFile testPrometheusMetricsFile + case [v | n : v : _ <- ms, n == name] of + [v] -> pure (read $ B.unpack v :: Int) + _ -> fail $ "no metric " <> B.unpack name withPostgres = bracket connectPostgres PSQL.close connectPostgres = do db <- PSQL.connectPostgreSQL testServerDBConnstr