mirror of
https://github.com/simplex-chat/simplexmq.git
synced 2026-11-22 19:58:39 +00:00
smp-server: forward worker queue length and command time metrics (#2048)
* smp-server: forward worker queue length and command time metrics * smp-server: describe forward worker time metric start, test queue wait * smp-server: state worker queue wait in forward worker time metric * smp-server: state counted commands and max period in forward worker metrics
This commit is contained in:
1 parent
e0afca78d3
commit
ca43321733
4 files changed
+76
-8
No files matched your search
@@ -695,8 +695,10 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOpt
|
||||
smpSubs <- getSubscribersMetrics subscribers
|
||||
ntfSubs <- getSubscribersMetrics ntfSubscribers
|
||||
loadedCounts <- loadedQueueCounts $ fromMsgStore ms
|
||||
pure RealTimeMetrics {socketStats, threadsCount, clientsCount, deliveredSubs, deliveredTimes, smpSubs, ntfSubs, loadedCounts}
|
||||
fwdWorkerQLength <- fmap sum . mapM fwdQLength =<< getServerClients srv
|
||||
pure RealTimeMetrics {socketStats, threadsCount, clientsCount, deliveredSubs, deliveredTimes, smpSubs, ntfSubs, loadedCounts, fwdWorkerQLength}
|
||||
where
|
||||
fwdQLength Client {fwdWorkers} = atomically $ readTVar fwdWorkers >>= fmap sum . mapM lengthTBQueue
|
||||
getSubscribersMetrics ServerSubscribers {queueSubscribers, serviceSubscribers, totalServiceSubs, subClients} = do
|
||||
subsCount <- M.size <$> getSubscribedClients queueSubscribers
|
||||
subClientsCount <- IS.size <$> readTVarIO subClients
|
||||
@@ -1208,6 +1210,13 @@ updateSlotWaitStats ServerStats {cmdSlotWaits, cmdSlotWaitMs, cmdSlotWaitMaxMs,
|
||||
atomicModifyIORef'_ cmdSlotWaitMaxMs (max waitMs)
|
||||
unless taken $ incStat cmdSlotWaitTimeouts
|
||||
|
||||
updateFwdWorkerStats :: ServerStats -> SystemTime -> IO ()
|
||||
updateFwdWorkerStats ServerStats {fwdWorkerCmds, fwdWorkerCmdMs, fwdWorkerCmdMaxMs} since = do
|
||||
cmdMs <- fromIntegral . (`div` 1000) . subtract (microseconds since) . microseconds <$> getSystemTime
|
||||
incStat fwdWorkerCmds
|
||||
atomicModifyIORef'_ fwdWorkerCmdMs (+ cmdMs)
|
||||
atomicModifyIORef'_ fwdWorkerCmdMaxMs (max cmdMs)
|
||||
|
||||
rslvTooManyRequests :: BrokerMsg
|
||||
rslvTooManyRequests = ERR $ NAME $ RESOLVER "too many requests"
|
||||
|
||||
@@ -2219,10 +2228,11 @@ client
|
||||
rres <- either ERR id <$> runExceptT (encodeResp r)
|
||||
stats <- asks serverStats
|
||||
authDelay <- asks $ authErrorDelay . config
|
||||
liftIO $
|
||||
liftIO $ do
|
||||
if isAuthError r
|
||||
then sendAfterDelay clnt stats authDelay startedAt ([(corrId, NoEntity, rres)], [])
|
||||
else atomically $ writeTBQueue sndQ ([(corrId, NoEntity, rres)], [])
|
||||
updateFwdWorkerStats stats startedAt
|
||||
pure Nothing
|
||||
Nothing -> pure $ Just (corrId', entId', ERR $ CMD PROHIBITED)
|
||||
stats <- asks serverStats
|
||||
|
||||
@@ -20,6 +20,7 @@ import Data.Time.Clock (UTCTime (..), diffUTCTime)
|
||||
import Data.Time.Clock.System (systemEpochDay)
|
||||
import Data.Time.Format.ISO8601 (iso8601Show)
|
||||
import Network.Socket (ServiceName)
|
||||
import Numeric.Natural (Natural)
|
||||
import Simplex.Messaging.Server.MsgStore.Types (LoadedQueueCounts (..))
|
||||
import Simplex.Messaging.Server.QueueStore.Types (EntityCounts (..))
|
||||
import Simplex.Messaging.Server.Stats
|
||||
@@ -46,7 +47,8 @@ data RealTimeMetrics = RealTimeMetrics
|
||||
deliveredTimes :: TimeBuckets,
|
||||
smpSubs :: RTSubscriberMetrics,
|
||||
ntfSubs :: RTSubscriberMetrics,
|
||||
loadedCounts :: LoadedQueueCounts
|
||||
loadedCounts :: LoadedQueueCounts,
|
||||
fwdWorkerQLength :: Natural
|
||||
}
|
||||
|
||||
data RTSubscriberMetrics = RTSubscriberMetrics
|
||||
@@ -70,7 +72,8 @@ prometheusMetrics sm rtm ts =
|
||||
deliveredTimes,
|
||||
smpSubs,
|
||||
ntfSubs,
|
||||
loadedCounts
|
||||
loadedCounts,
|
||||
fwdWorkerQLength
|
||||
} = rtm
|
||||
ServerStatsData
|
||||
{ _fromTime,
|
||||
@@ -131,6 +134,9 @@ prometheusMetrics sm rtm ts =
|
||||
_cmdSlotWaitMs,
|
||||
_cmdSlotWaitMaxMs,
|
||||
_cmdSlotWaitTimeouts,
|
||||
_fwdWorkerCmds,
|
||||
_fwdWorkerCmdMs,
|
||||
_fwdWorkerCmdMaxMs,
|
||||
_qCount,
|
||||
_msgCount,
|
||||
_ntfCount,
|
||||
@@ -395,6 +401,19 @@ prometheusMetrics sm rtm ts =
|
||||
\# HELP simplex_smp_relay_messages_received Relay messages statistics.\n\
|
||||
\# TYPE simplex_smp_relay_messages_received counter\n\
|
||||
\simplex_smp_relay_messages_received " <> mshow _pMsgFwdsRecv <> "\n# pMsgFwdsRecv\n\
|
||||
\\n\
|
||||
\# HELP simplex_smp_forward_worker_cmd_ms Time from the start of processing a forwarded command to queueing its response, for commands queued to forward workers, including the wait in the worker queue and excluding AUTH error delay, in ms.\n\
|
||||
\# TYPE simplex_smp_forward_worker_cmd_ms summary\n\
|
||||
\simplex_smp_forward_worker_cmd_ms_sum " <> mshow _fwdWorkerCmdMs <> "\n# fwdWorkerCmdMs\n\
|
||||
\simplex_smp_forward_worker_cmd_ms_count " <> mshow _fwdWorkerCmds <> "\n# fwdWorkerCmds\n\
|
||||
\\n\
|
||||
\# HELP simplex_smp_forward_worker_cmd_ms_max The longest time of simplex_smp_forward_worker_cmd_ms since the server start, in ms.\n\
|
||||
\# TYPE simplex_smp_forward_worker_cmd_ms_max gauge\n\
|
||||
\simplex_smp_forward_worker_cmd_ms_max " <> mshow _fwdWorkerCmdMaxMs <> "\n# fwdWorkerCmdMaxMs\n\
|
||||
\\n\
|
||||
\# HELP simplex_smp_forward_worker_queue_length Forwarded commands waiting in forward worker queues.\n\
|
||||
\# TYPE simplex_smp_forward_worker_queue_length gauge\n\
|
||||
\simplex_smp_forward_worker_queue_length " <> mshow fwdWorkerQLength <> "\n# fwdWorkerQLength\n\
|
||||
\\n"
|
||||
services =
|
||||
"# Services\n\
|
||||
|
||||
@@ -134,6 +134,9 @@ data ServerStats = ServerStats
|
||||
cmdSlotWaitMs :: IORef Int,
|
||||
cmdSlotWaitMaxMs :: IORef Int,
|
||||
cmdSlotWaitTimeouts :: IORef Int,
|
||||
fwdWorkerCmds :: IORef Int,
|
||||
fwdWorkerCmdMs :: IORef Int,
|
||||
fwdWorkerCmdMaxMs :: IORef Int,
|
||||
qCount :: IORef Int,
|
||||
msgCount :: IORef Int,
|
||||
ntfCount :: IORef Int,
|
||||
@@ -201,6 +204,9 @@ data ServerStatsData = ServerStatsData
|
||||
_cmdSlotWaitMs :: Int,
|
||||
_cmdSlotWaitMaxMs :: Int,
|
||||
_cmdSlotWaitTimeouts :: Int,
|
||||
_fwdWorkerCmds :: Int,
|
||||
_fwdWorkerCmdMs :: Int,
|
||||
_fwdWorkerCmdMaxMs :: Int,
|
||||
_qCount :: Int,
|
||||
_msgCount :: Int,
|
||||
_ntfCount :: Int,
|
||||
@@ -270,6 +276,9 @@ newServerStats ts = do
|
||||
cmdSlotWaitMs <- newIORef 0
|
||||
cmdSlotWaitMaxMs <- newIORef 0
|
||||
cmdSlotWaitTimeouts <- newIORef 0
|
||||
fwdWorkerCmds <- newIORef 0
|
||||
fwdWorkerCmdMs <- newIORef 0
|
||||
fwdWorkerCmdMaxMs <- newIORef 0
|
||||
qCount <- newIORef 0
|
||||
msgCount <- newIORef 0
|
||||
ntfCount <- newIORef 0
|
||||
@@ -336,6 +345,9 @@ newServerStats ts = do
|
||||
cmdSlotWaitMs,
|
||||
cmdSlotWaitMaxMs,
|
||||
cmdSlotWaitTimeouts,
|
||||
fwdWorkerCmds,
|
||||
fwdWorkerCmdMs,
|
||||
fwdWorkerCmdMaxMs,
|
||||
qCount,
|
||||
msgCount,
|
||||
ntfCount,
|
||||
@@ -404,6 +416,9 @@ getServerStatsData s = do
|
||||
_cmdSlotWaitMs <- readIORef $ cmdSlotWaitMs s
|
||||
_cmdSlotWaitMaxMs <- readIORef $ cmdSlotWaitMaxMs s
|
||||
_cmdSlotWaitTimeouts <- readIORef $ cmdSlotWaitTimeouts s
|
||||
_fwdWorkerCmds <- readIORef $ fwdWorkerCmds s
|
||||
_fwdWorkerCmdMs <- readIORef $ fwdWorkerCmdMs s
|
||||
_fwdWorkerCmdMaxMs <- readIORef $ fwdWorkerCmdMaxMs s
|
||||
_qCount <- readIORef $ qCount s
|
||||
_msgCount <- readIORef $ msgCount s
|
||||
_ntfCount <- readIORef $ ntfCount s
|
||||
@@ -470,6 +485,9 @@ getServerStatsData s = do
|
||||
_cmdSlotWaitMs,
|
||||
_cmdSlotWaitMaxMs,
|
||||
_cmdSlotWaitTimeouts,
|
||||
_fwdWorkerCmds,
|
||||
_fwdWorkerCmdMs,
|
||||
_fwdWorkerCmdMaxMs,
|
||||
_qCount,
|
||||
_msgCount,
|
||||
_ntfCount,
|
||||
@@ -539,6 +557,9 @@ setServerStats s d = do
|
||||
writeIORef (cmdSlotWaitMs s) $! _cmdSlotWaitMs d
|
||||
writeIORef (cmdSlotWaitMaxMs s) $! _cmdSlotWaitMaxMs d
|
||||
writeIORef (cmdSlotWaitTimeouts s) $! _cmdSlotWaitTimeouts d
|
||||
writeIORef (fwdWorkerCmds s) $! _fwdWorkerCmds d
|
||||
writeIORef (fwdWorkerCmdMs s) $! _fwdWorkerCmdMs d
|
||||
writeIORef (fwdWorkerCmdMaxMs s) $! _fwdWorkerCmdMaxMs d
|
||||
writeIORef (qCount s) $! _qCount d
|
||||
writeIORef (msgCount s) $! _msgCount d
|
||||
writeIORef (ntfCount s) $! _ntfCount d
|
||||
@@ -746,6 +767,9 @@ instance StrEncoding ServerStatsData where
|
||||
_cmdSlotWaitMs = 0,
|
||||
_cmdSlotWaitMaxMs = 0,
|
||||
_cmdSlotWaitTimeouts = 0,
|
||||
_fwdWorkerCmds = 0,
|
||||
_fwdWorkerCmdMs = 0,
|
||||
_fwdWorkerCmdMaxMs = 0,
|
||||
_qCount,
|
||||
_msgCount = 0,
|
||||
_ntfCount = 0,
|
||||
|
||||
+19
-4
@@ -50,6 +50,7 @@ import UnliftIO
|
||||
import Util
|
||||
|
||||
#if defined(dbServerPostgres)
|
||||
import qualified Data.ByteString.Char8 as B
|
||||
import Data.Hashable (hash)
|
||||
import Data.Int (Int64)
|
||||
import Database.PostgreSQL.Simple (Binary (..), Only (..))
|
||||
@@ -530,7 +531,7 @@ smpProxyPostgresTests =
|
||||
testForwardedQueueLocked :: HasTestEnv => IO ()
|
||||
testForwardedQueueLocked =
|
||||
withSmpServerConfigOn (transport @TLS) proxyCfg testPort $ \_ ->
|
||||
withSmpServerConfigOn (transport @TLS) (cfgS2QS SQSPostgres) testPort2 $ \_ -> do
|
||||
withSmpServerConfigOn (transport @TLS) (updateCfg (cfgS2QS SQSPostgres) $ \cfg_ -> cfg_ {prometheusInterval = Just 1}) testPort2 $ \_ -> do
|
||||
g <- C.newRandom
|
||||
ts <- getCurrentTime
|
||||
rc <- either (fail . show) pure =<< getProtocolClient g NRMInteractive (2, testSMPServer2, Nothing) defaultSMPClientConfig [] Nothing ts (\_ -> pure ())
|
||||
@@ -557,10 +558,24 @@ testForwardedQueueLocked =
|
||||
_ -> threadDelay 10000 >> waitBlocked
|
||||
timeout 10000000 waitBlocked `shouldReturn` Just ()
|
||||
timeout 10000000 (proxySend sIdB) `shouldReturn` Just (Right ())
|
||||
Nothing <- poll sendA
|
||||
PSQL.rollback locker
|
||||
wait sendA `shouldReturn` Right ()
|
||||
withAsync (proxySend sIdA) $ \sendA' -> do
|
||||
threadDelay 1500000
|
||||
metric "simplex_smp_forward_worker_queue_length" `shouldReturn` 1
|
||||
Nothing <- poll sendA
|
||||
PSQL.rollback locker
|
||||
wait sendA `shouldReturn` Right ()
|
||||
wait sendA' `shouldReturn` Right ()
|
||||
threadDelay 1500000
|
||||
metric "simplex_smp_forward_worker_cmd_ms_count" `shouldReturn` 3
|
||||
maxMs <- metric "simplex_smp_forward_worker_cmd_ms_max"
|
||||
maxMs `shouldSatisfy` (>= 1500)
|
||||
metric "simplex_smp_forward_worker_cmd_ms_sum" >>= (`shouldSatisfy` (>= maxMs + 1000))
|
||||
where
|
||||
metric name = do
|
||||
ms <- map B.words . B.lines <$> B.readFile testPrometheusMetricsFile
|
||||
case [v | n : v : _ <- ms, n == name] of
|
||||
[v] -> pure (read $ B.unpack v :: Int)
|
||||
_ -> fail $ "no metric " <> B.unpack name
|
||||
withPostgres = bracket connectPostgres PSQL.close
|
||||
connectPostgres = do
|
||||
db <- PSQL.connectPostgreSQL testServerDBConnstr
|
||||
|
||||
Reference in new issue
Block a user