From 09fa5b6449d4737ea12fda782a526b36b1949328 Mon Sep 17 00:00:00 2001 From: "Evgeny @ SimpleX Chat" <259188159+evgeny-simplex@users.noreply.github.com> Date: Tue, 13 Oct 2026 09:32:07 +0000 Subject: [PATCH] fixes --- plans/2026-09-26-client-usage-stats.md | 66 ++++++++++--------- src/Simplex/FileTransfer/Server.hs | 16 ++--- src/Simplex/Messaging/Notifications/Server.hs | 13 ++-- src/Simplex/Messaging/Server.hs | 9 +-- src/Simplex/Messaging/Server/CLI.hs | 3 +- src/Simplex/Messaging/Server/UsageStats.hs | 26 ++++---- tests/CoreTests/UsageStatsTests.hs | 25 ++++--- tests/NtfClient.hs | 4 ++ tests/NtfServerTests.hs | 23 +++++++ tests/SMPClient.hs | 7 ++ tests/SMPProxyTests.hs | 9 +-- tests/ServerTests.hs | 6 +- tests/XFTPServerTests.hs | 23 ++++--- 13 files changed, 134 insertions(+), 96 deletions(-) diff --git a/plans/2026-09-26-client-usage-stats.md b/plans/2026-09-26-client-usage-stats.md index 2f3f1fb68..d370e9cd1 100644 --- a/plans/2026-09-26-client-usage-stats.md +++ b/plans/2026-09-26-client-usage-stats.md @@ -78,14 +78,14 @@ Commands with their own counter: SMP proxy counters: -- `PRXY_new`: `PRXY` without a connected relay. -- `PRXY_failed`: `PRXY` with an error. +- `PRXY_new`: `PRXY` accepted by `allowProxy`, without a connected relay or an unexpired relay connection error. +- `PRXY_failed`: `PRXY` accepted by `allowProxy`, with a response other than `PKEY`. - `PFWD_failed`: `PFWD` with an error other than a protocol error, or without a session. XFTP counters: -- `upload_kb`: kilobytes of chunks stored by successful `FPUT`, rounded up. -- `download_kb`: kilobytes of chunks sent for `FGET`, rounded up. +- `upload_kb`: kilobytes of chunks stored by successful `FPUT`. +- `download_kb`: kilobytes of chunks in prepared `FGET` responses with the file. - `recipients`: recipient keys added by successful `FNEW` and `FADD`. Bounds: @@ -95,8 +95,8 @@ Bounds: ## Periods -- A period lasts `period` seconds. The first period starts at server start. -- Current counts are accumulated as events occur. +- Periods are closed every `period` seconds from server start. +- Current counts are increased by `addUsage`. - Previous counts are the counts of the last closed period. - The counts of closed periods are accumulated in histograms. @@ -116,15 +116,14 @@ data ClientCounts c = ClientCounts data ClientUsage c = ClientUsage (UsageStats c) ClientAddr data UsageHistogram = UsageHistogram - { bucketCounts :: [Int], + { bucketCounts :: IM.IntMap Int, usageSum :: Int, - clientCount :: Int, lastPeriodMax :: Int } ``` -- `current`, `previous`: counts by counter. A counter without a key has count 0. -- `bucketCounts`: cumulative client counts per bound, from server start. +- `current`, `previous`: counts by counter. The count of a counter without a key is 0. +- `bucketCounts`: client counts by bucket index, from server start. The bucket index of a count is the number of bounds below it. Bounds: @@ -137,15 +136,16 @@ Functions: - `newUsageStats :: UsageCounter c => IO (UsageStats c)` - `counterByName :: UsageCounter c => Text -> Maybe c` - `clientConnected :: UsageCounter c => Socket -> Maybe (UsageStats c) -> IO (Maybe (ClientUsage c))`: - 1. The peer address is read with `getPeerName`. - 2. On an exception or an address type without `ClientAddr`: `Nothing`. - 3. `connectionsCounter` is incremented. + 1. Without `UsageStats`: `Nothing`. + 2. The peer address is read with `getPeerName`. + 3. On an exception or an address type without `ClientAddr`: `Nothing`. + 4. `connectionsCounter` is incremented. - `incUsage :: UsageCounter c => ClientUsage c -> c -> IO ()` - `addUsage :: UsageCounter c => ClientUsage c -> c -> Int -> IO ()`: 1. The client entry is read with `TM.lookupIO`. 2. An absent entry is created, then looked up or inserted in one transaction. 3. The counter is read from `current` with `TM.lookupIO`. - 4. An absent counter is looked up or inserted with the amount in one transaction. + 4. An absent counter is looked up or inserted with the evaluated amount in one transaction. 5. A present counter is increased with `modifyTVar'`. The amount is positive at every call site. @@ -155,10 +155,9 @@ Functions: - `previous` is set to the closed counts. 2. The entry is deleted when the closed counts are empty. - Then, per counter, the closed counts are added to the histogram: - - to each bucket with a bound at or above the count - - to `usageSum` - - to `clientCount` + Then, per counter, the histogram is updated with the closed counts: + - `bucketCounts`: 1 is added at the bucket index of each count + - `usageSum`: the counts are added `lastPeriodMax` is set to the largest closed count, 0 without closed counts. - `usagePeriodThread :: UsageCounter c => Int64 -> UsageStats c -> IO ()`: `closeUsagePeriod` every `period` seconds. @@ -168,11 +167,11 @@ Functions: - clients with sum 0 excluded - limited to the given number - `printTopClients :: UsageCounter c => Handle -> Text -> Maybe Int -> Maybe (UsageStats c) -> IO ()`: control port output. -- `usageMetrics :: UsageCounter c => Text -> Text -> Map c UsageHistogram -> Text`: Prometheus text for a metric prefix and a timestamp. +- `usageMetrics :: UsageCounter c => Text -> Text -> Map c UsageHistogram -> Text`: Prometheus text for a metric prefix and a timestamp. The `_bucket` sample of the bound at index i is the sum of `bucketCounts` at indexes 0 to i. The `+Inf` and `_count` samples are the sum of `bucketCounts`. Counts lost in `closeUsagePeriod`: -- a count added to an entry between its read and its deletion; +- a count added after the closing transaction to an entry deleted after it; - a count added after the closing transaction to a counter read with `TM.lookupIO` before it. ## Prometheus @@ -287,12 +286,12 @@ In `Simplex.Messaging.Server`: - `runClient`: `clientConnected` is called before the handshake. Its result is passed to `runClientTransport`. - `clientUsage :: Maybe (ClientUsage SMPUsageCounter)` is added to `Client`, as the last parameter of `newClient`. - `receive`: - - The counter of each parsed transmission is incremented. - - `errors` is increased by the number of parse errors. - - `errors` is incremented for each failed verification. + - Before verification, the counter of each parsed command is incremented. + - After verification, `errors` is increased by the number of error responses. - `processProxiedCmd`: - `PRXY_new` is incremented before the connection to the relay is forked. - `incFailed own sel`: the proxy stat `sel` and `PRXY_failed` or `PFWD_failed` are incremented. +- `processForwardedCommand`: the command forwarded in `RFWD` and its errors are not counted. ## NTF server @@ -301,9 +300,9 @@ In `Simplex.Messaging.Notifications.Server`: - `runServer`: `runTransportServerState_` is called with a new `SocketState` and `TLSServerCredential {credential = srvCreds, sniCredential = Nothing}`. The socket is passed to `runClient`. - `runClient`: `clientConnected` is called before the handshake. Its result is passed to `runNtfClientTransport`. - `clientUsage :: Maybe (ClientUsage NtfUsageCounter)` is added to `NtfServerClient`, as the last parameter of `newNtfServerClient`. -- `receive`: +- `receive`, after verification: + - `errors` is increased by the number of error responses. - The counter of each parsed command is incremented. - - `errors` is incremented for each parse error and each failed verification. ## XFTP server @@ -321,7 +320,7 @@ Other changes: - The counter of each decoded command is incremented. - `errors` is incremented for each block size error, decoding error and failed verification. - `processXFTPRequest` has the `ClientUsage` as a parameter: - - `upload_kb` is increased after a successful `FPUT`. + - `upload_kb` is increased after the chunk is stored by a successful `FPUT`. - `download_kb` is increased when the `FGET` response with the file is prepared. - `recipients` is increased after a successful `FNEW` or `FADD`. @@ -338,13 +337,16 @@ Other changes: - bounds: - `countBounds` values - `kilobyteBounds` values -- `counterByName` of each counter name +- `counterByName`: + - each counter name + - unknown name: `Nothing` - `addUsage`: one entry per address - `closeUsagePeriod`: - - histogram buckets, sum, client count and max + - histogram buckets, sum and max - counters without counts excluded from histograms - previous counts set to closed counts - clients without counts in the closed period deleted + - a period with counts of 40000 clients closed within 5 seconds - `topClients`: - order by the sum of previous and current counts - number of clients limited @@ -357,14 +359,16 @@ Other changes: Control port tests, with `usageStatsPeriod = Just 300`: - SMP server: - - output of `usage SEND` and `usage connections`: the test client address + - output of `usage SEND`, `usage errors` and `usage connections`: the test client address + - output of `usage SEND 0`: the header only - output of `usage` without authentication: `AUTH` - output of `usage` with an unknown counter: `error: unknown counter` - SMP proxy: `other`, `PRXY_new`, `PRXY_failed`, `PFWD` and `PFWD_failed`. -- XFTP server: command, transfer and recipient counters. +- XFTP server: command, error, connection, transfer and recipient counters. +- NTF server: command, error and connection counters. Other tests: -- `testPrometheusMetrics`: the metrics file contains the histogram type line. +- `testPrometheusMetrics`: the histogram type line is checked in the metrics file. - `testServiceSubsTotalCount`: the metrics file is read strictly. - `CLITests`: `enable = off` and `period = 300` are checked in `[USAGE_STATS]` of each generated INI. diff --git a/src/Simplex/FileTransfer/Server.hs b/src/Simplex/FileTransfer/Server.hs index 334e435e3..284e37e61 100644 --- a/src/Simplex/FileTransfer/Server.hs +++ b/src/Simplex/FileTransfer/Server.hs @@ -49,6 +49,7 @@ import qualified Network.HTTP.Types as N import Network.HPACK.Token (tokenKey) import qualified Network.HTTP2.Server as H import Network.Socket +import Simplex.FileTransfer.Chunks (toKB) import Simplex.FileTransfer.Protocol import Simplex.FileTransfer.Server.Control (ControlProtocol (..)) import Simplex.FileTransfer.Server.Env @@ -437,7 +438,7 @@ data ServerFile = ServerFile processRequest :: FileStoreClass s => XFTPTransportRequest -> M s () processRequest XFTPTransportRequest {thParams, reqBody = body@HTTP2Body {bodyHead}, sendResponse, addCORS, clientUsage} - | B.length bodyHead /= xftpBlockSize = incClientUsage XUCErrors >> sendXFTPResponse ("", NoEntity, FRErr BLOCK) Nothing + | B.length bodyHead /= xftpBlockSize = sendError "" NoEntity BLOCK | otherwise = case xftpDecodeTServer thParams bodyHead of Right (Right t@(_, _, (corrId, fId, cmd))) -> do @@ -446,13 +447,14 @@ processRequest XFTPTransportRequest {thParams, reqBody = body@HTTP2Body {bodyHea ent = peerEntitlement =<< thAuth verifyXFTPTransmission thAuth t >>= \case VRVerified req -> uncurry send =<< processXFTPRequest clientUsage ent body req - VRFailed e -> incClientUsage XUCErrors >> send (FRErr e) Nothing + VRFailed e -> sendError corrId fId e where send resp = sendXFTPResponse (corrId, fId, resp) - Right (Left (corrId, fId, e)) -> incClientUsage XUCErrors >> sendXFTPResponse (corrId, fId, FRErr e) Nothing - Left e -> incClientUsage XUCErrors >> sendXFTPResponse ("", NoEntity, FRErr e) Nothing + Right (Left (corrId, fId, e)) -> sendError corrId fId e + Left e -> sendError "" NoEntity e where incClientUsage c = liftIO $ forM_ clientUsage (`incUsage` c) + sendError corrId fId e = incClientUsage XUCErrors >> sendXFTPResponse (corrId, fId, FRErr e) Nothing sendXFTPResponse t' serverFile_ = do let t_ = xftpEncodeTransmission thParams t' #ifdef slow_servers @@ -528,8 +530,6 @@ processXFTPRequest clientUsage ent HTTP2Body {bodyPart} = \case noFile resp = pure (resp, Nothing) addClientUsage :: XFTPUsageCounter -> Int -> M s () addClientUsage c n = liftIO $ forM_ clientUsage $ \usage -> addUsage usage c n - kilobytes :: Word32 -> Int - kilobytes bytes = (fromIntegral bytes + 1023) `div` 1024 createFile :: FileInfo -> NonEmpty RcvPublicAuthKey -> Maybe Word32 -> M s FileResponse createFile file rks storageHours = do st <- asks fileStore @@ -628,7 +628,7 @@ processXFTPRequest clientUsage ent HTTP2Body {bodyPart} = \case incFileStat filesUploaded incFileStat filesCount liftIO $ atomicModifyIORef'_ (filesSize stats) (+ fromIntegral size) - addClientUsage XUCUploadKb $ kilobytes size + addClientUsage XUCUploadKb $ fromIntegral $ toKB size pure FROk Left _e -> do us <- asks usedStorage @@ -658,7 +658,7 @@ processXFTPRequest clientUsage ent HTTP2Body {bodyPart} = \case stats <- asks serverStats incFileStat fileDownloads liftIO $ updatePeriodStats (filesDownloaded stats) senderId - addClientUsage XUCDownloadKb $ kilobytes size + addClientUsage XUCDownloadKb $ fromIntegral $ toKB size pure (FRFile sDhKey cbNonce, Just ServerFile {filePath = path, fileSize = size, sbState}) _ -> pure (FRErr INTERNAL, Nothing) _ -> pure (FRErr NO_FILE, Nothing) diff --git a/src/Simplex/Messaging/Notifications/Server.hs b/src/Simplex/Messaging/Notifications/Server.hs index c3a9431f7..18844af20 100644 --- a/src/Simplex/Messaging/Notifications/Server.hs +++ b/src/Simplex/Messaging/Notifications/Server.hs @@ -32,7 +32,7 @@ import Control.Monad.Reader import Data.Bifunctor (first) import Data.ByteString.Char8 (ByteString) import qualified Data.ByteString.Char8 as B -import Data.Either (partitionEithers) +import Data.Either (partitionEithers, rights) import Data.Functor (($>)) import Data.Hashable (hash) import Data.IORef @@ -75,7 +75,7 @@ import Simplex.Messaging.Server import Simplex.Messaging.Server.Control (CPClientRole (..)) import Simplex.Messaging.Server.Env.STM (StartOptions (..)) import Simplex.Messaging.Server.Stats (PeriodStats (..), PeriodStatCounts (..), periodStatCounts, periodStatDataCounts, updatePeriodStats) -import Simplex.Messaging.Server.UsageStats (ClientUsage, UsageStats (..), clientConnected, incUsage, printTopClients, usagePeriodThread) +import Simplex.Messaging.Server.UsageStats (ClientUsage, UsageStats (..), addUsage, clientConnected, incUsage, printTopClients, usagePeriodThread) import Simplex.Messaging.Session import Simplex.Messaging.SystemTime import Simplex.Messaging.TMap (TMap) @@ -791,26 +791,25 @@ receive st th@THandle {params = THandleParams {thAuth}} NtfServerClient {rcvQ, s ts <- L.toList <$> tGetServer th atomically . (writeTVar rcvActiveAt $!) =<< getSystemTime (errs, cmds) <- partitionEithers <$> mapM cmdAction ts + forM_ clientUsage $ \usage -> do + unless (null errs) $ addUsage usage NUCErrors $ length errs + forM_ (rights ts) $ \(_, _, (_, _, cmd)) -> incUsage usage $ ntfCmdCounter cmd write sndQ errs write rcvQ cmds where cmdAction = \case Left (corrId, entId, e) -> do - incClientUsage NUCErrors logError $ "invalid client request: " <> tshow e pure $ Left (corrId, entId, NRErr e) - Right t@(_, _, (corrId, entId, cmd)) -> do - incClientUsage $ ntfCmdCounter cmd + Right t@(_, _, (corrId, entId, _)) -> verified =<< verifyNtfTransmission st thAuth t where verified = \case VRVerified req -> pure $ Right req VRFailed e -> do - incClientUsage NUCErrors logError "unauthorized client request" pure $ Left (corrId, entId, NRErr e) write q = mapM_ (atomically . writeTBQueue q) . L.nonEmpty - incClientUsage c = forM_ clientUsage (`incUsage` c) send :: Transport c => THandleNTF c 'TServer -> NtfServerClient -> IO () send h@THandle {params} NtfServerClient {sndQ, sndActiveAt} = forever $ do diff --git a/src/Simplex/Messaging/Server.hs b/src/Simplex/Messaging/Server.hs index 39e2d28ef..eb31831d9 100644 --- a/src/Simplex/Messaging/Server.hs +++ b/src/Simplex/Messaging/Server.hs @@ -1164,9 +1164,7 @@ receive h@THandle {params = THandleParams {thAuth, sessionId}} ms Client {rcvQ, atomically . (writeTVar rcvActiveAt $!) =<< getSystemTime let (es, ts') = partitionEithers $ L.toList ts errs = map (second ERR) es - forM_ clientUsage $ \usage -> do - unless (null es) $ addUsage usage SUCErrors $ length es - forM_ ts' $ \(_, _, (_, _, cmd)) -> incUsage usage $ smpCmdCounter cmd + forM_ clientUsage $ \usage -> forM_ ts' $ \(_, _, (_, _, cmd)) -> incUsage usage $ smpCmdCounter cmd errs' <- case ts' of (_, _, (_, _, Cmd p cmd)) : rest -> do let service = peerClientService =<< thAuth @@ -1180,6 +1178,7 @@ receive h@THandle {params = THandleParams {thAuth, sessionId}} ms Client {rcvQ, mapM_ (atomically . writeTBQueue rcvQ) $ L.nonEmpty cmds pure $ errs ++ errs' [] -> pure errs + forM_ clientUsage $ \usage -> unless (null errs') $ addUsage usage SUCErrors $ length errs' mapM_ (atomically . writeTBQueue sndQ . (,[])) $ L.nonEmpty errs' where sameParty :: SParty p -> SignedTransmission Cmd -> Bool @@ -1194,9 +1193,7 @@ receive h@THandle {params = THandleParams {thAuth, sessionId}} ms Client {rcvQ, verified :: ServerStats -> SignedTransmission Cmd -> VerificationResult s -> IO (VerifiedTransmissionOrError s) verified stats (_, _, t@(corrId, entId, Cmd _ command)) = \case VRVerified q -> pure $ Right (q, t) - VRFailed e -> do - forM_ clientUsage (`incUsage` SUCErrors) - Left (corrId, entId, ERR e) <$ when (e == AUTH) incAuthStat + VRFailed e -> Left (corrId, entId, ERR e) <$ when (e == AUTH) incAuthStat where incAuthStat = case command of SEND {} -> incStat $ msgSentAuth stats diff --git a/src/Simplex/Messaging/Server/CLI.hs b/src/Simplex/Messaging/Server/CLI.hs index 0af47a2fd..4a8175adb 100644 --- a/src/Simplex/Messaging/Server/CLI.hs +++ b/src/Simplex/Messaging/Server/CLI.hs @@ -61,6 +61,7 @@ import Data.ASN1.Types (asn1CharacterToString) import Data.ByteString.Char8 (ByteString) import qualified Data.ByteString.Char8 as B import Data.Either (fromRight) +import Data.Functor (($>)) import Data.Ini (Ini, lookupValue) import Data.Int (Int64) import Data.List ((\\)) @@ -440,7 +441,7 @@ iniDBOptions ini _default@DBOpts {connstr, schema, poolSize} = } iniUsageStatsPeriod :: Ini -> Maybe Int64 -iniUsageStatsPeriod ini = readIniDefault defaultUsagePeriod "USAGE_STATS" "period" ini <$ settingIsOn "USAGE_STATS" "enable" ini +iniUsageStatsPeriod ini = settingIsOn "USAGE_STATS" "enable" ini $> readIniDefault defaultUsagePeriod "USAGE_STATS" "period" ini usageStatsIniContent :: Text usageStatsIniContent = diff --git a/src/Simplex/Messaging/Server/UsageStats.hs b/src/Simplex/Messaging/Server/UsageStats.hs index fdd2e6489..d8c3c91d0 100644 --- a/src/Simplex/Messaging/Server/UsageStats.hs +++ b/src/Simplex/Messaging/Server/UsageStats.hs @@ -37,7 +37,7 @@ import qualified Data.IntMap.Strict as IM import Data.List (find, sortOn) import Data.Map.Strict (Map) import qualified Data.Map.Strict as M -import Data.Maybe (fromMaybe) +import Data.Maybe (fromMaybe, mapMaybe) import Data.Ord (Down (..)) import Data.Text (Text) import qualified Data.Text as T @@ -69,9 +69,8 @@ data ClientCounts c = ClientCounts data ClientUsage c = ClientUsage (UsageStats c) ClientAddr data UsageHistogram = UsageHistogram - { bucketCounts :: [Int], + { bucketCounts :: IM.IntMap Int, usageSum :: Int, - clientCount :: Int, lastPeriodMax :: Int } deriving (Eq, Show) @@ -113,9 +112,9 @@ counterByName :: UsageCounter c => Text -> Maybe c counterByName name = find ((name ==) . counterName) [minBound .. maxBound] newUsageStats :: UsageCounter c => IO (UsageStats c) -newUsageStats = UsageStats <$> TM.emptyIO <*> newIORef (M.fromList $ map (\c -> (c, emptyHistogram c)) [minBound .. maxBound]) +newUsageStats = UsageStats <$> TM.emptyIO <*> newIORef (M.fromList $ map (,emptyHistogram) [minBound .. maxBound]) where - emptyHistogram c = UsageHistogram {bucketCounts = map (const 0) (counterBounds c), usageSum = 0, clientCount = 0, lastPeriodMax = 0} + emptyHistogram = UsageHistogram {bucketCounts = IM.empty, usageSum = 0, lastPeriodMax = 0} clientConnected :: UsageCounter c => Socket -> Maybe (UsageStats c) -> IO (Maybe (ClientUsage c)) clientConnected sock stats_ = @@ -135,7 +134,7 @@ addUsage (ClientUsage UsageStats {clientCounts} addr) c n = do TM.lookupIO c current >>= atomically . maybe (newCount current) add where add v = modifyTVar' v (+ n) - newCount current = TM.lookup c current >>= maybe (TM.insertM c (newTVar n) current) add + newCount current = TM.lookup c current >>= maybe (TM.insertM c (newTVar $! n) current) add newCounts = do counts <- ClientCounts <$> TM.emptyIO <*> newTVarIO M.empty atomically $ TM.lookup addr clientCounts >>= maybe (counts <$ TM.insert addr counts clientCounts) pure @@ -143,8 +142,7 @@ addUsage (ClientUsage UsageStats {clientCounts} addr) c n = do closeUsagePeriod :: UsageCounter c => UsageStats c -> IO () closeUsagePeriod UsageStats {clientCounts, histograms} = do closedCounts <- mapM closeClientPeriod . M.toList =<< readTVarIO clientCounts - let periodCounts = M.unionsWith (<>) $ map (M.map (: [])) closedCounts - atomicModifyIORef'_ histograms $ M.mapWithKey $ \c -> addPeriodCounts (counterBounds c) (M.findWithDefault [] c periodCounts) + atomicModifyIORef'_ histograms $ M.mapWithKey $ \c -> addPeriodCounts (counterBounds c) (mapMaybe (M.lookup c) closedCounts) where closeClientPeriod (addr, ClientCounts {current, previous}) = do counts <- atomically $ do @@ -154,11 +152,10 @@ closeUsagePeriod UsageStats {clientCounts, histograms} = do pure counts addPeriodCounts :: [Int] -> [Int] -> UsageHistogram -> UsageHistogram -addPeriodCounts bounds ns UsageHistogram {bucketCounts, usageSum, clientCount} = +addPeriodCounts bounds ns UsageHistogram {bucketCounts, usageSum} = UsageHistogram - { bucketCounts = zipWith (+) bucketCounts $ scanl1 (+) $ map (\i -> IM.findWithDefault 0 i boundCounts) [0 .. length bounds - 1], + { bucketCounts = IM.unionWith (+) bucketCounts boundCounts, usageSum = usageSum + sum ns, - clientCount = clientCount + length ns, lastPeriodMax = maximum (0 : ns) } where @@ -204,10 +201,13 @@ usageMetrics prefix tsEpoch usageHistograms = hs = M.toList usageHistograms usageMetric = prefix <> "_client_usage" maxMetric = prefix <> "_client_usage_max" - histogramLines (c, UsageHistogram {bucketCounts, usageSum, clientCount}) = - T.concat (zipWith (\b cnt -> sample (usageMetric <> "_bucket") c [("le", tshow b)] cnt) (counterBounds c) bucketCounts) + histogramLines (c, UsageHistogram {bucketCounts, usageSum}) = + T.concat (zipWith (\b cnt -> sample (usageMetric <> "_bucket") c [("le", tshow b)] cnt) (counterBounds c) cumulativeCounts) <> sample (usageMetric <> "_bucket") c [("le", "+Inf")] clientCount <> sample (usageMetric <> "_sum") c [] usageSum <> sample (usageMetric <> "_count") c [] clientCount + where + cumulativeCounts = scanl1 (+) $ map (\i -> IM.findWithDefault 0 i bucketCounts) [0 ..] + clientCount = sum bucketCounts sample metric c labels v = metric <> "{" <> T.intercalate "," (map (\(l, lv) -> l <> "=\"" <> lv <> "\"") (("counter", counterName c) : labels)) <> "} " <> tshow v <> " " <> tsEpoch <> "\n" diff --git a/tests/CoreTests/UsageStatsTests.hs b/tests/CoreTests/UsageStatsTests.hs index 1d68ac75b..25db32cb8 100644 --- a/tests/CoreTests/UsageStatsTests.hs +++ b/tests/CoreTests/UsageStatsTests.hs @@ -6,10 +6,11 @@ module CoreTests.UsageStatsTests where -import Control.Monad ((<=<)) +import Control.Monad (forM_, (<=<)) import Data.ByteString.Char8 (ByteString) import qualified Data.ByteString.Char8 as B import Data.IORef (readIORef) +import qualified Data.IntMap.Strict as IM import Data.List.NonEmpty (NonEmpty (..)) import qualified Data.Map.Strict as M import qualified Data.Set as S @@ -28,6 +29,7 @@ import Simplex.Messaging.SimplexName (SimplexDomain (..), SimplexTLD (..)) import qualified Simplex.Messaging.TMap as TM import Simplex.Messaging.Transport (supportedClientSMPRelayVRange, supportedServerSMPRelayVRange) import Simplex.Messaging.Version (VersionRange (..)) +import System.Timeout (timeout) import Test.Hspec hiding (fit, it) import UnliftIO.STM import Util @@ -138,17 +140,17 @@ closeUsagePeriodTests = do closeUsagePeriod stats readIORef histograms `shouldReturn` M.fromList - [ (TCConnections, UsageHistogram [2, 2, 2, 2] 2 2 1), - (TCSend, UsageHistogram [0, 0, 1, 2] 10 2 7), - (TCGet, UsageHistogram [0, 0, 0, 0] 0 0 0) + [ (TCConnections, UsageHistogram (IM.fromList [(0, 2)]) 2 1), + (TCSend, UsageHistogram (IM.fromList [(2, 1), (3, 1)]) 10 7), + (TCGet, UsageHistogram IM.empty 0 0) ] addUsage (usage addr1) TCSend 20 closeUsagePeriod stats readIORef histograms `shouldReturn` M.fromList - [ (TCConnections, UsageHistogram [2, 2, 2, 2] 2 2 0), - (TCSend, UsageHistogram [0, 0, 1, 2] 30 3 20), - (TCGet, UsageHistogram [0, 0, 0, 0] 0 0 0) + [ (TCConnections, UsageHistogram (IM.fromList [(0, 2)]) 2 0), + (TCSend, UsageHistogram (IM.fromList [(2, 1), (3, 1), (4, 1)]) 30 20), + (TCGet, UsageHistogram IM.empty 0 0) ] it "should set previous counts to closed counts" $ do stats <- newUsageStats @@ -175,6 +177,11 @@ closeUsagePeriodTests = do M.keys <$> readTVarIO clientCounts `shouldReturn` [addr1] closeUsagePeriod stats M.keys <$> readTVarIO clientCounts `shouldReturn` [] + it "should close a period with counts of 40000 clients within 5 seconds" $ do + stats@UsageStats {histograms} <- newUsageStats + forM_ [1 .. 40000] $ \a -> incUsage (ClientUsage stats $ ClientIPv4 $ toEnum a) TCSend + timeout 5000000 (closeUsagePeriod stats) `shouldReturn` Just () + M.lookup TCSend <$> readIORef histograms `shouldReturn` Just (UsageHistogram (IM.fromList [(0, 40000)]) 40000 1) topClientsTests :: Spec topClientsTests = @@ -224,8 +231,8 @@ metricsTests = where usageHistograms = M.fromList - [ (TCSend, UsageHistogram [0, 0, 1, 2] 10 2 7), - (TCGet, UsageHistogram [0, 0, 0, 0] 12 1 12) + [ (TCSend, UsageHistogram (IM.fromList [(2, 1), (3, 1)]) 10 7), + (TCGet, UsageHistogram (IM.fromList [(4, 1)]) 12 12) ] commandCounterTests :: Spec diff --git a/tests/NtfClient.hs b/tests/NtfClient.hs index 91a3155f5..bbf18be0c 100644 --- a/tests/NtfClient.hs +++ b/tests/NtfClient.hs @@ -119,6 +119,10 @@ ntfTestDBCfg2 = ntfTestDBCfg {dbOpts = ntfTestStoreDBOpts2, dbStoreLogPath = Jus testNtfClient :: Transport c => (THandleNTF c 'TClient -> IO a) -> IO a testNtfClient client = do Right host <- pure $ chooseTransportHost defaultNetworkConfig testHost + testNtfClient_ host client + +testNtfClient_ :: Transport c => TransportHost -> (THandleNTF c 'TClient -> IO a) -> IO a +testNtfClient_ host client = runTransportClient defaultTransportClientConfig Nothing host ntfTestPort (Just testKeyHash) $ \h -> runExceptT (ntfClientHandshake h testKeyHash supportedClientNTFVRange False Nothing) >>= \case Right th -> client th diff --git a/tests/NtfServerTests.hs b/tests/NtfServerTests.hs index 7fd430fef..67f5bff1b 100644 --- a/tests/NtfServerTests.hs +++ b/tests/NtfServerTests.hs @@ -58,6 +58,8 @@ ntfServerTests ps@(t, _) = do describe "Notification subscriptions (NKEY)" $ testNotificationSubscription ps createNtfQueueNKEY describe "Notification subscriptions (NEW with ntf creds)" $ testNotificationSubscription ps createNtfQueueNEW describe "Retried notification subscription" $ testRetriedNtfSubscription ps + describe "Client usage statistics" $ + it "should count commands and errors by client address" $ testNtfUsageStats t ntfSyntaxTests :: ASrvTransport -> Spec ntfSyntaxTests (ATransport t) = do @@ -88,6 +90,27 @@ testApnsTestProviderRejected (ATransport (t :: TProxy c 'TServer)) = do signSendRecvNtf nh tknKey ("1", NoEntity, TNEW $ NewNtfTkn tkn tknPub dhPub) pure () +testNtfUsageStats :: ASrvTransport -> Expectation +testNtfUsageStats (ATransport (t :: TProxy c 'TServer)) = do + g <- C.newRandom + (tknPub, tknKey) <- atomically $ C.generateAuthKeyPair C.SEd25519 g + (dhPub, _dhPriv :: C.PrivateKeyX25519) <- atomically $ C.generateKeyPair g + let tkn = DeviceToken PPApnsTest "abcd" + ntfCfg = + ntfServerCfg + { allowTestPushProvider = False, + transports = [(ntfTestPort, ATransport t, False)], + usageStatsPeriod = Just 300, + controlPort = Just testControlPort, + controlPortUserAuth = Just testControlPortUserAuth + } + withNtfServerCfg ntfCfg $ \_ -> + testNtfClient_ "127.0.0.1" $ \(nh :: THandleNTF c 'TClient) -> do + RespNtf "1" NoEntity (NRErr (CMD PROHIBITED)) <- signSendRecvNtf nh tknKey ("1", NoEntity, TNEW $ NewNtfTkn tkn tknPub dhPub) + RespNtf "2" _ (NRErr AUTH) <- signSendRecvNtf nh tknKey ("2", EntityId "1234", TCHK) + RespNtf "3" NoEntity (NRErr (CMD NO_AUTH)) <- signSendRecvNtf nh tknKey ("3", NoEntity, TCHK) + checkUsageCounts [("connections", "1"), ("TNEW", "1"), ("other", "1"), ("errors", "2")] + pattern RespNtf :: CorrId -> QueueId -> NtfResponse -> Transmission (Either ErrorType NtfResponse) pattern RespNtf corrId queueId command <- (corrId, queueId, Right command) diff --git a/tests/SMPClient.hs b/tests/SMPClient.hs index 674368c34..686e28913 100644 --- a/tests/SMPClient.hs +++ b/tests/SMPClient.hs @@ -422,6 +422,13 @@ controlPortCommand h cmd n = do hFlush h replicateM n $ trimCR <$> B.hGetLine h +checkUsageCounts :: HasCallStack => [(ByteString, ByteString)] -> Expectation +checkUsageCounts counts = + withControlPort $ \h -> do + controlPortCommand h ("auth " <> unBasicAuth testControlPortUserAuth) 1 `shouldReturn` ["Current role is CPRUser"] + forM_ counts $ \(counter, n) -> + controlPortCommand h ("usage " <> counter) 2 `shouldReturn` ["address,previous,current", "127.0.0.1,0," <> n] + withSmpServerOn :: HasCallStack => (ASrvTransport, AStoreType) -> ServiceName -> IO a -> IO a withSmpServerOn ps port' = withSmpServerThreadOn ps port' . const diff --git a/tests/SMPProxyTests.hs b/tests/SMPProxyTests.hs index 33223d950..9be6f6617 100644 --- a/tests/SMPProxyTests.hs +++ b/tests/SMPProxyTests.hs @@ -164,13 +164,8 @@ smpProxyTests = do } testProxyCounts = do deliverMessageViaProxy (SMPServer testHost2 testPort testKeyHash) (SMPServer testHost2 testPort2 testKeyHash) C.SEd448 "hello 1" "hello 2" - withControlPort $ \h -> do - controlPortCommand h ("auth " <> SMP.unBasicAuth testControlPortUserAuth) 1 `shouldReturn` ["Current role is CPRUser"] - controlPortCommand h "usage PRXY_failed" 1 `shouldReturn` ["address,previous,current"] - forM_ counts $ \(counter, n) -> - controlPortCommand h ("usage " <> counter) 2 `shouldReturn` ["address,previous,current", "127.0.0.1,0," <> n] - counts :: [(ByteString, ByteString)] - counts = [("other", "2"), ("PRXY_new", "1"), ("PFWD", "3"), ("PFWD_failed", "1")] + proxyConnectDeadRelay 1 0 (SMPServer testHost2 testPort testKeyHash) + checkUsageCounts [("other", "3"), ("PRXY_new", "2"), ("PRXY_failed", "1"), ("PFWD", "3"), ("PFWD_failed", "1")] oneServer test msType = withSmpServerConfigOn (transport @TLS) (updateCfg (proxyCfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 128, maxJournalMsgCount = 256}) testPort $ const test twoServers test msType = twoServers_ (proxyCfgMS msType) (proxyCfgMS msType) test msType twoServersFirstProxy test msType = twoServers_ (proxyCfgMS msType) (updateCfg (cfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 128, maxJournalMsgCount = 256}) test msType diff --git a/tests/ServerTests.hs b/tests/ServerTests.hs index 40e405ae7..b221dd056 100644 --- a/tests/ServerTests.hs +++ b/tests/ServerTests.hs @@ -1270,12 +1270,14 @@ testUsageStatsControlPort = (sId, _, _, _) <- createAndSecureQueue r sPub Resp "bcda" _ OK <- signSendRecv s sKey ("bcda", sId, _SEND "hello") Resp "cdab" _ OK <- signSendRecv s sKey ("cdab", sId, _SEND "hello again") + Resp "dabc" _ (ERR AUTH) <- signSendRecv s sKey ("dabc", EntityId "1234", _SEND "hello") + Resp "abcd" _ (ERR (CMD NO_ENTITY)) <- sendRecv s ("", "abcd", NoEntity, _SEND "hello") withControlPort $ \h -> do controlPortCommand h "usage SEND" 1 `shouldReturn` ["AUTH"] controlPortCommand h ("auth " <> unBasicAuth testControlPortUserAuth) 1 `shouldReturn` ["Current role is CPRUser"] + controlPortCommand h "usage SEND 0" 1 `shouldReturn` ["address,previous,current"] controlPortCommand h "usage unknown" 1 `shouldReturn` ["error: unknown counter"] - controlPortCommand h "usage SEND" 2 `shouldReturn` ["address,previous,current", "127.0.0.1,0,2"] - controlPortCommand h "usage connections" 2 `shouldReturn` ["address,previous,current", "127.0.0.1,0,2"] + checkUsageCounts [("SEND", "3"), ("errors", "2"), ("connections", "2")] createAndSecureQueue :: Transport c => THandleSMP c 'TClient -> SndPublicAuthKey -> IO (SenderId, RecipientId, RcvPrivateAuthKey, RcvDhSecret) createAndSecureQueue h sPub = do diff --git a/tests/XFTPServerTests.hs b/tests/XFTPServerTests.hs index 62e6b761b..d8f282e01 100644 --- a/tests/XFTPServerTests.hs +++ b/tests/XFTPServerTests.hs @@ -28,7 +28,7 @@ import qualified Data.X509 as X import Data.X509.Validation (Fingerprint (..), getFingerprint) import Network.HPACK.Token (tokenKey) import qualified Network.HTTP2.Client as H2 -import SMPClient (controlPortCommand, testControlPort, testControlPortUserAuth, withControlPort) +import SMPClient (checkUsageCounts, testControlPort, testControlPortUserAuth, testHost2, testKeyHash) import ServerTests (logSize) import Simplex.FileTransfer.Client hiding (createXFTPChunk) import qualified Simplex.FileTransfer.Client as A @@ -40,7 +40,7 @@ import Simplex.Messaging.Client (ProtocolClientError (..)) import qualified Simplex.Messaging.Crypto as C import qualified Simplex.Messaging.Crypto.Lazy as LC import Simplex.Messaging.Encoding (smpDecode, smpEncode) -import Simplex.Messaging.Protocol (BasicAuth (..), EntityId (..), RecipientId, SenderId, pattern NoEntity) +import Simplex.Messaging.Protocol (BasicAuth, CommandError (..), EntityId (..), RecipientId, SenderId, pattern NoEntity, pattern XFTPServer) import Simplex.Messaging.Server.Expiration (ExpirationConfig (..)) import Simplex.Messaging.Transport (CertChainPubKey (..), TLS (..), TransportPeer (..), defaultSupportedParams, defaultSupportedParamsHTTPS) import Simplex.Messaging.Transport.Client (TransportClientConfig (..), TransportHost (..), defaultTransportClientConfig, runTLSTransportClient) @@ -69,7 +69,7 @@ xftpServerTests = it "should disconnect inactive clients" testInactiveClientExpiration it "should not allow uploading chunks after specified storage quota" testFileStorageQuota it "should store file records to log and restore them after server restart" testFileLog - it "should count commands, transfers and recipients by client address" testFileUsageStats + it "should count commands, errors, transfers and recipients by client address" testFileUsageStats describe "XFTP basic auth" $ do -- allow FNEW | server auth | clnt auth | success it "prohibited without basic auth" $ testFileBasicAuth True (Just "pwd") Nothing False @@ -341,16 +341,15 @@ testFileUsageStats fsType = withXFTPServerConfigOn (updateXFTPCfg (cfgFS fsType) $ \c -> c {usageStatsPeriod = Just 300, controlPort = Just testControlPort, controlPortUserAuth = Just testControlPortUserAuth}) $ \_ -> do ts <- getCurrentTime runRight_ $ do - c <- ExceptT $ getXFTPClient (1, "xftp://LcJUMfVhwD8yxjAiSaDzzGF3-kLG4Uh0Fl_ZIjrRwjI=@127.0.0.1:8000", Nothing) testXFTPClientConfig [] ts (\_ -> pure Nothing) (\_ -> pure ()) + c <- ExceptT $ getXFTPClient (1, XFTPServer testHost2 xftpTestPort testKeyHash, Nothing) testXFTPClientConfig [] ts (\_ -> pure Nothing) (\_ -> pure ()) runTestFileChunkDelivery c c - withControlPort $ \h -> do - controlPortCommand h ("auth " <> unBasicAuth testControlPortUserAuth) 1 `shouldReturn` ["Current role is CPRUser"] - controlPortCommand h "usage errors" 1 `shouldReturn` ["address,previous,current"] - forM_ counts $ \(counter, n) -> - controlPortCommand h ("usage " <> counter) 2 `shouldReturn` ["address,previous,current", "127.0.0.1,0," <> n] - where - counts :: [(ByteString, ByteString)] - counts = [("connections", "1"), ("FNEW", "2"), ("other", "4"), ("recipients", "2"), ("upload_kb", "128"), ("download_kb", "256")] + g <- liftIO C.newRandom + (_, spKey) <- atomically $ C.generateAuthKeyPair C.SEd25519 g + deleteXFTPChunk c spKey (EntityId "1234") + `catchError` (liftIO . (`shouldBe` PCEProtocolError AUTH)) + deleteXFTPChunk c spKey NoEntity + `catchError` (liftIO . (`shouldBe` PCEProtocolError (CMD NO_AUTH))) + checkUsageCounts [("connections", "1"), ("FNEW", "2"), ("other", "5"), ("errors", "2"), ("recipients", "2"), ("upload_kb", "128"), ("download_kb", "256")] testFileBasicAuth :: Bool -> Maybe BasicAuth -> Maybe BasicAuth -> Bool -> AFStoreType -> IO () testFileBasicAuth allowNewFiles newFileBasicAuth clntAuth success fsType =