smp-server: forward worker queue length and command time metrics (#2048)

* smp-server: forward worker queue length and command time metrics

* smp-server: describe forward worker time metric start, test queue wait

* smp-server: state worker queue wait in forward worker time metric

* smp-server: state counted commands and max period in forward worker metrics
This commit is contained in:
Evgeny @ SimpleX Chat authored and GitHub committed 2026-11-12 11:09:47 +00:00
1 parent e0afca78d3
commit ca43321733
4 files changed
+76 -8

No files matched your search

+12 -2
View File
@@ -695,8 +695,10 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOpt
smpSubs <- getSubscribersMetrics subscribers
ntfSubs <- getSubscribersMetrics ntfSubscribers
loadedCounts <- loadedQueueCounts $ fromMsgStore ms
pure RealTimeMetrics {socketStats, threadsCount, clientsCount, deliveredSubs, deliveredTimes, smpSubs, ntfSubs, loadedCounts}
fwdWorkerQLength <- fmap sum . mapM fwdQLength =<< getServerClients srv
pure RealTimeMetrics {socketStats, threadsCount, clientsCount, deliveredSubs, deliveredTimes, smpSubs, ntfSubs, loadedCounts, fwdWorkerQLength}
where
fwdQLength Client {fwdWorkers} = atomically $ readTVar fwdWorkers >>= fmap sum . mapM lengthTBQueue
getSubscribersMetrics ServerSubscribers {queueSubscribers, serviceSubscribers, totalServiceSubs, subClients} = do
subsCount <- M.size <$> getSubscribedClients queueSubscribers
subClientsCount <- IS.size <$> readTVarIO subClients
@@ -1208,6 +1210,13 @@ updateSlotWaitStats ServerStats {cmdSlotWaits, cmdSlotWaitMs, cmdSlotWaitMaxMs,
atomicModifyIORef'_ cmdSlotWaitMaxMs (max waitMs)
unless taken $ incStat cmdSlotWaitTimeouts
updateFwdWorkerStats :: ServerStats -> SystemTime -> IO ()
updateFwdWorkerStats ServerStats {fwdWorkerCmds, fwdWorkerCmdMs, fwdWorkerCmdMaxMs} since = do
cmdMs <- fromIntegral . (`div` 1000) . subtract (microseconds since) . microseconds <$> getSystemTime
incStat fwdWorkerCmds
atomicModifyIORef'_ fwdWorkerCmdMs (+ cmdMs)
atomicModifyIORef'_ fwdWorkerCmdMaxMs (max cmdMs)
rslvTooManyRequests :: BrokerMsg
rslvTooManyRequests = ERR $ NAME $ RESOLVER "too many requests"
@@ -2219,10 +2228,11 @@ client
rres <- either ERR id <$> runExceptT (encodeResp r)
stats <- asks serverStats
authDelay <- asks $ authErrorDelay . config
liftIO $
liftIO $ do
if isAuthError r
then sendAfterDelay clnt stats authDelay startedAt ([(corrId, NoEntity, rres)], [])
else atomically $ writeTBQueue sndQ ([(corrId, NoEntity, rres)], [])
updateFwdWorkerStats stats startedAt
pure Nothing
Nothing -> pure $ Just (corrId', entId', ERR $ CMD PROHIBITED)
stats <- asks serverStats
+21 -2
View File
@@ -20,6 +20,7 @@ import Data.Time.Clock (UTCTime (..), diffUTCTime)
import Data.Time.Clock.System (systemEpochDay)
import Data.Time.Format.ISO8601 (iso8601Show)
import Network.Socket (ServiceName)
import Numeric.Natural (Natural)
import Simplex.Messaging.Server.MsgStore.Types (LoadedQueueCounts (..))
import Simplex.Messaging.Server.QueueStore.Types (EntityCounts (..))
import Simplex.Messaging.Server.Stats
@@ -46,7 +47,8 @@ data RealTimeMetrics = RealTimeMetrics
deliveredTimes :: TimeBuckets,
smpSubs :: RTSubscriberMetrics,
ntfSubs :: RTSubscriberMetrics,
loadedCounts :: LoadedQueueCounts
loadedCounts :: LoadedQueueCounts,
fwdWorkerQLength :: Natural
}
data RTSubscriberMetrics = RTSubscriberMetrics
@@ -70,7 +72,8 @@ prometheusMetrics sm rtm ts =
deliveredTimes,
smpSubs,
ntfSubs,
loadedCounts
loadedCounts,
fwdWorkerQLength
} = rtm
ServerStatsData
{ _fromTime,
@@ -131,6 +134,9 @@ prometheusMetrics sm rtm ts =
_cmdSlotWaitMs,
_cmdSlotWaitMaxMs,
_cmdSlotWaitTimeouts,
_fwdWorkerCmds,
_fwdWorkerCmdMs,
_fwdWorkerCmdMaxMs,
_qCount,
_msgCount,
_ntfCount,
@@ -395,6 +401,19 @@ prometheusMetrics sm rtm ts =
\# HELP simplex_smp_relay_messages_received Relay messages statistics.\n\
\# TYPE simplex_smp_relay_messages_received counter\n\
\simplex_smp_relay_messages_received " <> mshow _pMsgFwdsRecv <> "\n# pMsgFwdsRecv\n\
\\n\
\# HELP simplex_smp_forward_worker_cmd_ms Time from the start of processing a forwarded command to queueing its response, for commands queued to forward workers, including the wait in the worker queue and excluding AUTH error delay, in ms.\n\
\# TYPE simplex_smp_forward_worker_cmd_ms summary\n\
\simplex_smp_forward_worker_cmd_ms_sum " <> mshow _fwdWorkerCmdMs <> "\n# fwdWorkerCmdMs\n\
\simplex_smp_forward_worker_cmd_ms_count " <> mshow _fwdWorkerCmds <> "\n# fwdWorkerCmds\n\
\\n\
\# HELP simplex_smp_forward_worker_cmd_ms_max The longest time of simplex_smp_forward_worker_cmd_ms since the server start, in ms.\n\
\# TYPE simplex_smp_forward_worker_cmd_ms_max gauge\n\
\simplex_smp_forward_worker_cmd_ms_max " <> mshow _fwdWorkerCmdMaxMs <> "\n# fwdWorkerCmdMaxMs\n\
\\n\
\# HELP simplex_smp_forward_worker_queue_length Forwarded commands waiting in forward worker queues.\n\
\# TYPE simplex_smp_forward_worker_queue_length gauge\n\
\simplex_smp_forward_worker_queue_length " <> mshow fwdWorkerQLength <> "\n# fwdWorkerQLength\n\
\\n"
services =
"# Services\n\
+24
View File
@@ -134,6 +134,9 @@ data ServerStats = ServerStats
cmdSlotWaitMs :: IORef Int,
cmdSlotWaitMaxMs :: IORef Int,
cmdSlotWaitTimeouts :: IORef Int,
fwdWorkerCmds :: IORef Int,
fwdWorkerCmdMs :: IORef Int,
fwdWorkerCmdMaxMs :: IORef Int,
qCount :: IORef Int,
msgCount :: IORef Int,
ntfCount :: IORef Int,
@@ -201,6 +204,9 @@ data ServerStatsData = ServerStatsData
_cmdSlotWaitMs :: Int,
_cmdSlotWaitMaxMs :: Int,
_cmdSlotWaitTimeouts :: Int,
_fwdWorkerCmds :: Int,
_fwdWorkerCmdMs :: Int,
_fwdWorkerCmdMaxMs :: Int,
_qCount :: Int,
_msgCount :: Int,
_ntfCount :: Int,
@@ -270,6 +276,9 @@ newServerStats ts = do
cmdSlotWaitMs <- newIORef 0
cmdSlotWaitMaxMs <- newIORef 0
cmdSlotWaitTimeouts <- newIORef 0
fwdWorkerCmds <- newIORef 0
fwdWorkerCmdMs <- newIORef 0
fwdWorkerCmdMaxMs <- newIORef 0
qCount <- newIORef 0
msgCount <- newIORef 0
ntfCount <- newIORef 0
@@ -336,6 +345,9 @@ newServerStats ts = do
cmdSlotWaitMs,
cmdSlotWaitMaxMs,
cmdSlotWaitTimeouts,
fwdWorkerCmds,
fwdWorkerCmdMs,
fwdWorkerCmdMaxMs,
qCount,
msgCount,
ntfCount,
@@ -404,6 +416,9 @@ getServerStatsData s = do
_cmdSlotWaitMs <- readIORef $ cmdSlotWaitMs s
_cmdSlotWaitMaxMs <- readIORef $ cmdSlotWaitMaxMs s
_cmdSlotWaitTimeouts <- readIORef $ cmdSlotWaitTimeouts s
_fwdWorkerCmds <- readIORef $ fwdWorkerCmds s
_fwdWorkerCmdMs <- readIORef $ fwdWorkerCmdMs s
_fwdWorkerCmdMaxMs <- readIORef $ fwdWorkerCmdMaxMs s
_qCount <- readIORef $ qCount s
_msgCount <- readIORef $ msgCount s
_ntfCount <- readIORef $ ntfCount s
@@ -470,6 +485,9 @@ getServerStatsData s = do
_cmdSlotWaitMs,
_cmdSlotWaitMaxMs,
_cmdSlotWaitTimeouts,
_fwdWorkerCmds,
_fwdWorkerCmdMs,
_fwdWorkerCmdMaxMs,
_qCount,
_msgCount,
_ntfCount,
@@ -539,6 +557,9 @@ setServerStats s d = do
writeIORef (cmdSlotWaitMs s) $! _cmdSlotWaitMs d
writeIORef (cmdSlotWaitMaxMs s) $! _cmdSlotWaitMaxMs d
writeIORef (cmdSlotWaitTimeouts s) $! _cmdSlotWaitTimeouts d
writeIORef (fwdWorkerCmds s) $! _fwdWorkerCmds d
writeIORef (fwdWorkerCmdMs s) $! _fwdWorkerCmdMs d
writeIORef (fwdWorkerCmdMaxMs s) $! _fwdWorkerCmdMaxMs d
writeIORef (qCount s) $! _qCount d
writeIORef (msgCount s) $! _msgCount d
writeIORef (ntfCount s) $! _ntfCount d
@@ -746,6 +767,9 @@ instance StrEncoding ServerStatsData where
_cmdSlotWaitMs = 0,
_cmdSlotWaitMaxMs = 0,
_cmdSlotWaitTimeouts = 0,
_fwdWorkerCmds = 0,
_fwdWorkerCmdMs = 0,
_fwdWorkerCmdMaxMs = 0,
_qCount,
_msgCount = 0,
_ntfCount = 0,