smp-server: count forwards above relay limit

src/Simplex/Messaging/Server.hs

@@ -1455,7 +1455,7 @@ client
                       _ -> ERR $ transportErr TEVersion
       PFWD fwdV pubKey encBlock -> do
         ProxyAgent {smpAgent = a, relayForwards} <- asks proxyAgent
-        ServerStats {pMsgFwds, pMsgFwdsOwn} <- asks serverStats
+        ServerStats {pMsgFwds, pMsgFwdsOwn, pMsgFwdsRelayLimit} <- asks serverStats
         let inc = mkIncProxyStats pMsgFwds pMsgFwdsOwn
         liftIO (lookupSMPServerClient a sessId) >>= \case
           Just (own, smp) -> do
@@ -1465,7 +1465,7 @@ client
             forkProxiedCmd $
               bracket (atomically $ reserve maxFwds relayForwards) (\r -> when r $ atomically $ release relayForwards) $ \case
                 -- the relay is not keeping up, the client retries TIMEOUT later
-                False -> ERR (smpProxyError PCEResponseTimeout) <$ inc own pErrorsOther
+                False -> ERR (smpProxyError PCEResponseTimeout) <$ (inc own pErrorsOther >> incStat pMsgFwdsRelayLimit)
                 True ->
                   liftIO (runExceptT (forwardSMPTransmission smp corrId fwdV pubKey encBlock) `E.catches` clientHandlers) >>= \case
                     Right r -> PRES r <$ inc own pSuccesses

src/Simplex/Messaging/Server/Prometheus.hs

@@ -122,6 +122,7 @@ prometheusMetrics sm rtm ts =
         _pMsgFwds,
         _pMsgFwdsOwn,
         _pMsgFwdsRecv,
+        _pMsgFwdsRelayLimit,
         _rcvServices,
         _ntfServices,
         _rcvServicesSubMsg,
@@ -370,6 +371,10 @@ prometheusMetrics sm rtm ts =
       \simplex_smp_relay_messages_errors{source=\"own\",type=\"compat\"} " <> mshow (_pErrorsCompat _pMsgFwdsOwn) <> "\n# pMsgFwdsOwn_pErrorsCompat\n\
       \simplex_smp_relay_messages_errors{source=\"own\",type=\"other\"} " <> mshow (_pErrorsOther _pMsgFwdsOwn) <> "\n# pMsgFwdsOwn_pErrorsOther\n\
       \\n\
+      \# HELP simplex_smp_relay_messages_concurrency_limit Messages not sent through relay, as the limit of messages in flight to relay was reached.\n\
+      \# TYPE simplex_smp_relay_messages_concurrency_limit counter\n\
+      \simplex_smp_relay_messages_concurrency_limit " <> mshow _pMsgFwdsRelayLimit <> "\n# pMsgFwdsRelayLimit\n\
+      \\n\
       \# HELP simplex_smp_relay_messages_received Relay messages statistics.\n\
       \# TYPE simplex_smp_relay_messages_received counter\n\
       \simplex_smp_relay_messages_received " <> mshow _pMsgFwdsRecv <> "\n# pMsgFwdsRecv\n\

src/Simplex/Messaging/Server/Stats.hs

@@ -124,6 +124,7 @@ data ServerStats = ServerStats
     pMsgFwds :: ProxyStats,
     pMsgFwdsOwn :: ProxyStats,
     pMsgFwdsRecv :: IORef Int,
+    pMsgFwdsRelayLimit :: IORef Int,
     rcvServices :: ServiceStats,
     ntfServices :: ServiceStats,
     rcvServicesSubMsg :: IORef Int,
@@ -186,6 +187,7 @@ data ServerStatsData = ServerStatsData
     _pMsgFwds :: ProxyStatsData,
     _pMsgFwdsOwn :: ProxyStatsData,
     _pMsgFwdsRecv :: Int,
+    _pMsgFwdsRelayLimit :: Int,
     _ntfServices :: ServiceStatsData,
     _rcvServices :: ServiceStatsData,
     _rcvServicesSubMsg :: Int,
@@ -250,6 +252,7 @@ newServerStats ts = do
   pMsgFwds <- newProxyStats
   pMsgFwdsOwn <- newProxyStats
   pMsgFwdsRecv <- newIORef 0
+  pMsgFwdsRelayLimit <- newIORef 0
   rcvServices <- newServiceStats
   ntfServices <- newServiceStats
   rcvServicesSubMsg <- newIORef 0
@@ -311,6 +314,7 @@ newServerStats ts = do
         pMsgFwds,
         pMsgFwdsOwn,
         pMsgFwdsRecv,
+        pMsgFwdsRelayLimit,
         rcvServices,
         ntfServices,
         rcvServicesSubMsg,
@@ -374,6 +378,7 @@ getServerStatsData s = do
   _pMsgFwds <- getProxyStatsData $ pMsgFwds s
   _pMsgFwdsOwn <- getProxyStatsData $ pMsgFwdsOwn s
   _pMsgFwdsRecv <- readIORef $ pMsgFwdsRecv s
+  _pMsgFwdsRelayLimit <- readIORef $ pMsgFwdsRelayLimit s
   _rcvServices <- getServiceStatsData $ rcvServices s
   _ntfServices <- getServiceStatsData $ ntfServices s
   _rcvServicesSubMsg <- readIORef $ rcvServicesSubMsg s
@@ -435,6 +440,7 @@ getServerStatsData s = do
         _pMsgFwds,
         _pMsgFwdsOwn,
         _pMsgFwdsRecv,
+        _pMsgFwdsRelayLimit,
         _rcvServices,
         _ntfServices,
         _rcvServicesSubMsg,
@@ -499,6 +505,7 @@ setServerStats s d = do
   setProxyStats (pMsgFwds s) $! _pMsgFwds d
   setProxyStats (pMsgFwdsOwn s) $! _pMsgFwdsOwn d
   writeIORef (pMsgFwdsRecv s) $! _pMsgFwdsRecv d
+  writeIORef (pMsgFwdsRelayLimit s) $! _pMsgFwdsRelayLimit d
   setServiceStats (rcvServices s) $! _rcvServices d
   setServiceStats (ntfServices s) $! _ntfServices d
   writeIORef (rcvServicesSubMsg s) $! _rcvServicesSubMsg d
@@ -701,6 +708,7 @@ instance StrEncoding ServerStatsData where
           _pMsgFwds,
           _pMsgFwdsOwn,
           _pMsgFwdsRecv,
+          _pMsgFwdsRelayLimit = 0,
           _rcvServices,
           _ntfServices,
           _rcvServicesSubMsg = 0,