mirror of
https://github.com/simplex-chat/simplexmq.git
synced 2026-11-22 15:38:34 +00:00
fixes
This commit is contained in:
1 parent
360424b090
commit
09fa5b6449
13 files changed
+134
-96
No files matched your search
@@ -78,14 +78,14 @@ Commands with their own counter:
|
||||
|
||||
SMP proxy counters:
|
||||
|
||||
- `PRXY_new`: `PRXY` without a connected relay.
|
||||
- `PRXY_failed`: `PRXY` with an error.
|
||||
- `PRXY_new`: `PRXY` accepted by `allowProxy`, without a connected relay or an unexpired relay connection error.
|
||||
- `PRXY_failed`: `PRXY` accepted by `allowProxy`, with a response other than `PKEY`.
|
||||
- `PFWD_failed`: `PFWD` with an error other than a protocol error, or without a session.
|
||||
|
||||
XFTP counters:
|
||||
|
||||
- `upload_kb`: kilobytes of chunks stored by successful `FPUT`, rounded up.
|
||||
- `download_kb`: kilobytes of chunks sent for `FGET`, rounded up.
|
||||
- `upload_kb`: kilobytes of chunks stored by successful `FPUT`.
|
||||
- `download_kb`: kilobytes of chunks in prepared `FGET` responses with the file.
|
||||
- `recipients`: recipient keys added by successful `FNEW` and `FADD`.
|
||||
|
||||
Bounds:
|
||||
@@ -95,8 +95,8 @@ Bounds:
|
||||
|
||||
## Periods
|
||||
|
||||
- A period lasts `period` seconds. The first period starts at server start.
|
||||
- Current counts are accumulated as events occur.
|
||||
- Periods are closed every `period` seconds from server start.
|
||||
- Current counts are increased by `addUsage`.
|
||||
- Previous counts are the counts of the last closed period.
|
||||
- The counts of closed periods are accumulated in histograms.
|
||||
|
||||
@@ -116,15 +116,14 @@ data ClientCounts c = ClientCounts
|
||||
data ClientUsage c = ClientUsage (UsageStats c) ClientAddr
|
||||
|
||||
data UsageHistogram = UsageHistogram
|
||||
{ bucketCounts :: [Int],
|
||||
{ bucketCounts :: IM.IntMap Int,
|
||||
usageSum :: Int,
|
||||
clientCount :: Int,
|
||||
lastPeriodMax :: Int
|
||||
}
|
||||
```
|
||||
|
||||
- `current`, `previous`: counts by counter. A counter without a key has count 0.
|
||||
- `bucketCounts`: cumulative client counts per bound, from server start.
|
||||
- `current`, `previous`: counts by counter. The count of a counter without a key is 0.
|
||||
- `bucketCounts`: client counts by bucket index, from server start. The bucket index of a count is the number of bounds below it.
|
||||
|
||||
Bounds:
|
||||
|
||||
@@ -137,15 +136,16 @@ Functions:
|
||||
- `newUsageStats :: UsageCounter c => IO (UsageStats c)`
|
||||
- `counterByName :: UsageCounter c => Text -> Maybe c`
|
||||
- `clientConnected :: UsageCounter c => Socket -> Maybe (UsageStats c) -> IO (Maybe (ClientUsage c))`:
|
||||
1. The peer address is read with `getPeerName`.
|
||||
2. On an exception or an address type without `ClientAddr`: `Nothing`.
|
||||
3. `connectionsCounter` is incremented.
|
||||
1. Without `UsageStats`: `Nothing`.
|
||||
2. The peer address is read with `getPeerName`.
|
||||
3. On an exception or an address type without `ClientAddr`: `Nothing`.
|
||||
4. `connectionsCounter` is incremented.
|
||||
- `incUsage :: UsageCounter c => ClientUsage c -> c -> IO ()`
|
||||
- `addUsage :: UsageCounter c => ClientUsage c -> c -> Int -> IO ()`:
|
||||
1. The client entry is read with `TM.lookupIO`.
|
||||
2. An absent entry is created, then looked up or inserted in one transaction.
|
||||
3. The counter is read from `current` with `TM.lookupIO`.
|
||||
4. An absent counter is looked up or inserted with the amount in one transaction.
|
||||
4. An absent counter is looked up or inserted with the evaluated amount in one transaction.
|
||||
5. A present counter is increased with `modifyTVar'`.
|
||||
|
||||
The amount is positive at every call site.
|
||||
@@ -155,10 +155,9 @@ Functions:
|
||||
- `previous` is set to the closed counts.
|
||||
2. The entry is deleted when the closed counts are empty.
|
||||
|
||||
Then, per counter, the closed counts are added to the histogram:
|
||||
- to each bucket with a bound at or above the count
|
||||
- to `usageSum`
|
||||
- to `clientCount`
|
||||
Then, per counter, the histogram is updated with the closed counts:
|
||||
- `bucketCounts`: 1 is added at the bucket index of each count
|
||||
- `usageSum`: the counts are added
|
||||
|
||||
`lastPeriodMax` is set to the largest closed count, 0 without closed counts.
|
||||
- `usagePeriodThread :: UsageCounter c => Int64 -> UsageStats c -> IO ()`: `closeUsagePeriod` every `period` seconds.
|
||||
@@ -168,11 +167,11 @@ Functions:
|
||||
- clients with sum 0 excluded
|
||||
- limited to the given number
|
||||
- `printTopClients :: UsageCounter c => Handle -> Text -> Maybe Int -> Maybe (UsageStats c) -> IO ()`: control port output.
|
||||
- `usageMetrics :: UsageCounter c => Text -> Text -> Map c UsageHistogram -> Text`: Prometheus text for a metric prefix and a timestamp.
|
||||
- `usageMetrics :: UsageCounter c => Text -> Text -> Map c UsageHistogram -> Text`: Prometheus text for a metric prefix and a timestamp. The `_bucket` sample of the bound at index i is the sum of `bucketCounts` at indexes 0 to i. The `+Inf` and `_count` samples are the sum of `bucketCounts`.
|
||||
|
||||
Counts lost in `closeUsagePeriod`:
|
||||
|
||||
- a count added to an entry between its read and its deletion;
|
||||
- a count added after the closing transaction to an entry deleted after it;
|
||||
- a count added after the closing transaction to a counter read with `TM.lookupIO` before it.
|
||||
|
||||
## Prometheus
|
||||
@@ -287,12 +286,12 @@ In `Simplex.Messaging.Server`:
|
||||
- `runClient`: `clientConnected` is called before the handshake. Its result is passed to `runClientTransport`.
|
||||
- `clientUsage :: Maybe (ClientUsage SMPUsageCounter)` is added to `Client`, as the last parameter of `newClient`.
|
||||
- `receive`:
|
||||
- The counter of each parsed transmission is incremented.
|
||||
- `errors` is increased by the number of parse errors.
|
||||
- `errors` is incremented for each failed verification.
|
||||
- Before verification, the counter of each parsed command is incremented.
|
||||
- After verification, `errors` is increased by the number of error responses.
|
||||
- `processProxiedCmd`:
|
||||
- `PRXY_new` is incremented before the connection to the relay is forked.
|
||||
- `incFailed own sel`: the proxy stat `sel` and `PRXY_failed` or `PFWD_failed` are incremented.
|
||||
- `processForwardedCommand`: the command forwarded in `RFWD` and its errors are not counted.
|
||||
|
||||
## NTF server
|
||||
|
||||
@@ -301,9 +300,9 @@ In `Simplex.Messaging.Notifications.Server`:
|
||||
- `runServer`: `runTransportServerState_` is called with a new `SocketState` and `TLSServerCredential {credential = srvCreds, sniCredential = Nothing}`. The socket is passed to `runClient`.
|
||||
- `runClient`: `clientConnected` is called before the handshake. Its result is passed to `runNtfClientTransport`.
|
||||
- `clientUsage :: Maybe (ClientUsage NtfUsageCounter)` is added to `NtfServerClient`, as the last parameter of `newNtfServerClient`.
|
||||
- `receive`:
|
||||
- `receive`, after verification:
|
||||
- `errors` is increased by the number of error responses.
|
||||
- The counter of each parsed command is incremented.
|
||||
- `errors` is incremented for each parse error and each failed verification.
|
||||
|
||||
## XFTP server
|
||||
|
||||
@@ -321,7 +320,7 @@ Other changes:
|
||||
- The counter of each decoded command is incremented.
|
||||
- `errors` is incremented for each block size error, decoding error and failed verification.
|
||||
- `processXFTPRequest` has the `ClientUsage` as a parameter:
|
||||
- `upload_kb` is increased after a successful `FPUT`.
|
||||
- `upload_kb` is increased after the chunk is stored by a successful `FPUT`.
|
||||
- `download_kb` is increased when the `FGET` response with the file is prepared.
|
||||
- `recipients` is increased after a successful `FNEW` or `FADD`.
|
||||
|
||||
@@ -338,13 +337,16 @@ Other changes:
|
||||
- bounds:
|
||||
- `countBounds` values
|
||||
- `kilobyteBounds` values
|
||||
- `counterByName` of each counter name
|
||||
- `counterByName`:
|
||||
- each counter name
|
||||
- unknown name: `Nothing`
|
||||
- `addUsage`: one entry per address
|
||||
- `closeUsagePeriod`:
|
||||
- histogram buckets, sum, client count and max
|
||||
- histogram buckets, sum and max
|
||||
- counters without counts excluded from histograms
|
||||
- previous counts set to closed counts
|
||||
- clients without counts in the closed period deleted
|
||||
- a period with counts of 40000 clients closed within 5 seconds
|
||||
- `topClients`:
|
||||
- order by the sum of previous and current counts
|
||||
- number of clients limited
|
||||
@@ -357,14 +359,16 @@ Other changes:
|
||||
Control port tests, with `usageStatsPeriod = Just 300`:
|
||||
|
||||
- SMP server:
|
||||
- output of `usage SEND` and `usage connections`: the test client address
|
||||
- output of `usage SEND`, `usage errors` and `usage connections`: the test client address
|
||||
- output of `usage SEND 0`: the header only
|
||||
- output of `usage` without authentication: `AUTH`
|
||||
- output of `usage` with an unknown counter: `error: unknown counter`
|
||||
- SMP proxy: `other`, `PRXY_new`, `PRXY_failed`, `PFWD` and `PFWD_failed`.
|
||||
- XFTP server: command, transfer and recipient counters.
|
||||
- XFTP server: command, error, connection, transfer and recipient counters.
|
||||
- NTF server: command, error and connection counters.
|
||||
|
||||
Other tests:
|
||||
|
||||
- `testPrometheusMetrics`: the metrics file contains the histogram type line.
|
||||
- `testPrometheusMetrics`: the histogram type line is checked in the metrics file.
|
||||
- `testServiceSubsTotalCount`: the metrics file is read strictly.
|
||||
- `CLITests`: `enable = off` and `period = 300` are checked in `[USAGE_STATS]` of each generated INI.
|
||||
@@ -49,6 +49,7 @@ import qualified Network.HTTP.Types as N
|
||||
import Network.HPACK.Token (tokenKey)
|
||||
import qualified Network.HTTP2.Server as H
|
||||
import Network.Socket
|
||||
import Simplex.FileTransfer.Chunks (toKB)
|
||||
import Simplex.FileTransfer.Protocol
|
||||
import Simplex.FileTransfer.Server.Control (ControlProtocol (..))
|
||||
import Simplex.FileTransfer.Server.Env
|
||||
@@ -437,7 +438,7 @@ data ServerFile = ServerFile
|
||||
|
||||
processRequest :: FileStoreClass s => XFTPTransportRequest -> M s ()
|
||||
processRequest XFTPTransportRequest {thParams, reqBody = body@HTTP2Body {bodyHead}, sendResponse, addCORS, clientUsage}
|
||||
| B.length bodyHead /= xftpBlockSize = incClientUsage XUCErrors >> sendXFTPResponse ("", NoEntity, FRErr BLOCK) Nothing
|
||||
| B.length bodyHead /= xftpBlockSize = sendError "" NoEntity BLOCK
|
||||
| otherwise =
|
||||
case xftpDecodeTServer thParams bodyHead of
|
||||
Right (Right t@(_, _, (corrId, fId, cmd))) -> do
|
||||
@@ -446,13 +447,14 @@ processRequest XFTPTransportRequest {thParams, reqBody = body@HTTP2Body {bodyHea
|
||||
ent = peerEntitlement =<< thAuth
|
||||
verifyXFTPTransmission thAuth t >>= \case
|
||||
VRVerified req -> uncurry send =<< processXFTPRequest clientUsage ent body req
|
||||
VRFailed e -> incClientUsage XUCErrors >> send (FRErr e) Nothing
|
||||
VRFailed e -> sendError corrId fId e
|
||||
where
|
||||
send resp = sendXFTPResponse (corrId, fId, resp)
|
||||
Right (Left (corrId, fId, e)) -> incClientUsage XUCErrors >> sendXFTPResponse (corrId, fId, FRErr e) Nothing
|
||||
Left e -> incClientUsage XUCErrors >> sendXFTPResponse ("", NoEntity, FRErr e) Nothing
|
||||
Right (Left (corrId, fId, e)) -> sendError corrId fId e
|
||||
Left e -> sendError "" NoEntity e
|
||||
where
|
||||
incClientUsage c = liftIO $ forM_ clientUsage (`incUsage` c)
|
||||
sendError corrId fId e = incClientUsage XUCErrors >> sendXFTPResponse (corrId, fId, FRErr e) Nothing
|
||||
sendXFTPResponse t' serverFile_ = do
|
||||
let t_ = xftpEncodeTransmission thParams t'
|
||||
#ifdef slow_servers
|
||||
@@ -528,8 +530,6 @@ processXFTPRequest clientUsage ent HTTP2Body {bodyPart} = \case
|
||||
noFile resp = pure (resp, Nothing)
|
||||
addClientUsage :: XFTPUsageCounter -> Int -> M s ()
|
||||
addClientUsage c n = liftIO $ forM_ clientUsage $ \usage -> addUsage usage c n
|
||||
kilobytes :: Word32 -> Int
|
||||
kilobytes bytes = (fromIntegral bytes + 1023) `div` 1024
|
||||
createFile :: FileInfo -> NonEmpty RcvPublicAuthKey -> Maybe Word32 -> M s FileResponse
|
||||
createFile file rks storageHours = do
|
||||
st <- asks fileStore
|
||||
@@ -628,7 +628,7 @@ processXFTPRequest clientUsage ent HTTP2Body {bodyPart} = \case
|
||||
incFileStat filesUploaded
|
||||
incFileStat filesCount
|
||||
liftIO $ atomicModifyIORef'_ (filesSize stats) (+ fromIntegral size)
|
||||
addClientUsage XUCUploadKb $ kilobytes size
|
||||
addClientUsage XUCUploadKb $ fromIntegral $ toKB size
|
||||
pure FROk
|
||||
Left _e -> do
|
||||
us <- asks usedStorage
|
||||
@@ -658,7 +658,7 @@ processXFTPRequest clientUsage ent HTTP2Body {bodyPart} = \case
|
||||
stats <- asks serverStats
|
||||
incFileStat fileDownloads
|
||||
liftIO $ updatePeriodStats (filesDownloaded stats) senderId
|
||||
addClientUsage XUCDownloadKb $ kilobytes size
|
||||
addClientUsage XUCDownloadKb $ fromIntegral $ toKB size
|
||||
pure (FRFile sDhKey cbNonce, Just ServerFile {filePath = path, fileSize = size, sbState})
|
||||
_ -> pure (FRErr INTERNAL, Nothing)
|
||||
_ -> pure (FRErr NO_FILE, Nothing)
|
||||
|
||||
@@ -32,7 +32,7 @@ import Control.Monad.Reader
|
||||
import Data.Bifunctor (first)
|
||||
import Data.ByteString.Char8 (ByteString)
|
||||
import qualified Data.ByteString.Char8 as B
|
||||
import Data.Either (partitionEithers)
|
||||
import Data.Either (partitionEithers, rights)
|
||||
import Data.Functor (($>))
|
||||
import Data.Hashable (hash)
|
||||
import Data.IORef
|
||||
@@ -75,7 +75,7 @@ import Simplex.Messaging.Server
|
||||
import Simplex.Messaging.Server.Control (CPClientRole (..))
|
||||
import Simplex.Messaging.Server.Env.STM (StartOptions (..))
|
||||
import Simplex.Messaging.Server.Stats (PeriodStats (..), PeriodStatCounts (..), periodStatCounts, periodStatDataCounts, updatePeriodStats)
|
||||
import Simplex.Messaging.Server.UsageStats (ClientUsage, UsageStats (..), clientConnected, incUsage, printTopClients, usagePeriodThread)
|
||||
import Simplex.Messaging.Server.UsageStats (ClientUsage, UsageStats (..), addUsage, clientConnected, incUsage, printTopClients, usagePeriodThread)
|
||||
import Simplex.Messaging.Session
|
||||
import Simplex.Messaging.SystemTime
|
||||
import Simplex.Messaging.TMap (TMap)
|
||||
@@ -791,26 +791,25 @@ receive st th@THandle {params = THandleParams {thAuth}} NtfServerClient {rcvQ, s
|
||||
ts <- L.toList <$> tGetServer th
|
||||
atomically . (writeTVar rcvActiveAt $!) =<< getSystemTime
|
||||
(errs, cmds) <- partitionEithers <$> mapM cmdAction ts
|
||||
forM_ clientUsage $ \usage -> do
|
||||
unless (null errs) $ addUsage usage NUCErrors $ length errs
|
||||
forM_ (rights ts) $ \(_, _, (_, _, cmd)) -> incUsage usage $ ntfCmdCounter cmd
|
||||
write sndQ errs
|
||||
write rcvQ cmds
|
||||
where
|
||||
cmdAction = \case
|
||||
Left (corrId, entId, e) -> do
|
||||
incClientUsage NUCErrors
|
||||
logError $ "invalid client request: " <> tshow e
|
||||
pure $ Left (corrId, entId, NRErr e)
|
||||
Right t@(_, _, (corrId, entId, cmd)) -> do
|
||||
incClientUsage $ ntfCmdCounter cmd
|
||||
Right t@(_, _, (corrId, entId, _)) ->
|
||||
verified =<< verifyNtfTransmission st thAuth t
|
||||
where
|
||||
verified = \case
|
||||
VRVerified req -> pure $ Right req
|
||||
VRFailed e -> do
|
||||
incClientUsage NUCErrors
|
||||
logError "unauthorized client request"
|
||||
pure $ Left (corrId, entId, NRErr e)
|
||||
write q = mapM_ (atomically . writeTBQueue q) . L.nonEmpty
|
||||
incClientUsage c = forM_ clientUsage (`incUsage` c)
|
||||
|
||||
send :: Transport c => THandleNTF c 'TServer -> NtfServerClient -> IO ()
|
||||
send h@THandle {params} NtfServerClient {sndQ, sndActiveAt} = forever $ do
|
||||
|
||||
@@ -1164,9 +1164,7 @@ receive h@THandle {params = THandleParams {thAuth, sessionId}} ms Client {rcvQ,
|
||||
atomically . (writeTVar rcvActiveAt $!) =<< getSystemTime
|
||||
let (es, ts') = partitionEithers $ L.toList ts
|
||||
errs = map (second ERR) es
|
||||
forM_ clientUsage $ \usage -> do
|
||||
unless (null es) $ addUsage usage SUCErrors $ length es
|
||||
forM_ ts' $ \(_, _, (_, _, cmd)) -> incUsage usage $ smpCmdCounter cmd
|
||||
forM_ clientUsage $ \usage -> forM_ ts' $ \(_, _, (_, _, cmd)) -> incUsage usage $ smpCmdCounter cmd
|
||||
errs' <- case ts' of
|
||||
(_, _, (_, _, Cmd p cmd)) : rest -> do
|
||||
let service = peerClientService =<< thAuth
|
||||
@@ -1180,6 +1178,7 @@ receive h@THandle {params = THandleParams {thAuth, sessionId}} ms Client {rcvQ,
|
||||
mapM_ (atomically . writeTBQueue rcvQ) $ L.nonEmpty cmds
|
||||
pure $ errs ++ errs'
|
||||
[] -> pure errs
|
||||
forM_ clientUsage $ \usage -> unless (null errs') $ addUsage usage SUCErrors $ length errs'
|
||||
mapM_ (atomically . writeTBQueue sndQ . (,[])) $ L.nonEmpty errs'
|
||||
where
|
||||
sameParty :: SParty p -> SignedTransmission Cmd -> Bool
|
||||
@@ -1194,9 +1193,7 @@ receive h@THandle {params = THandleParams {thAuth, sessionId}} ms Client {rcvQ,
|
||||
verified :: ServerStats -> SignedTransmission Cmd -> VerificationResult s -> IO (VerifiedTransmissionOrError s)
|
||||
verified stats (_, _, t@(corrId, entId, Cmd _ command)) = \case
|
||||
VRVerified q -> pure $ Right (q, t)
|
||||
VRFailed e -> do
|
||||
forM_ clientUsage (`incUsage` SUCErrors)
|
||||
Left (corrId, entId, ERR e) <$ when (e == AUTH) incAuthStat
|
||||
VRFailed e -> Left (corrId, entId, ERR e) <$ when (e == AUTH) incAuthStat
|
||||
where
|
||||
incAuthStat = case command of
|
||||
SEND {} -> incStat $ msgSentAuth stats
|
||||
|
||||
@@ -61,6 +61,7 @@ import Data.ASN1.Types (asn1CharacterToString)
|
||||
import Data.ByteString.Char8 (ByteString)
|
||||
import qualified Data.ByteString.Char8 as B
|
||||
import Data.Either (fromRight)
|
||||
import Data.Functor (($>))
|
||||
import Data.Ini (Ini, lookupValue)
|
||||
import Data.Int (Int64)
|
||||
import Data.List ((\\))
|
||||
@@ -440,7 +441,7 @@ iniDBOptions ini _default@DBOpts {connstr, schema, poolSize} =
|
||||
}
|
||||
|
||||
iniUsageStatsPeriod :: Ini -> Maybe Int64
|
||||
iniUsageStatsPeriod ini = readIniDefault defaultUsagePeriod "USAGE_STATS" "period" ini <$ settingIsOn "USAGE_STATS" "enable" ini
|
||||
iniUsageStatsPeriod ini = settingIsOn "USAGE_STATS" "enable" ini $> readIniDefault defaultUsagePeriod "USAGE_STATS" "period" ini
|
||||
|
||||
usageStatsIniContent :: Text
|
||||
usageStatsIniContent =
|
||||
|
||||
@@ -37,7 +37,7 @@ import qualified Data.IntMap.Strict as IM
|
||||
import Data.List (find, sortOn)
|
||||
import Data.Map.Strict (Map)
|
||||
import qualified Data.Map.Strict as M
|
||||
import Data.Maybe (fromMaybe)
|
||||
import Data.Maybe (fromMaybe, mapMaybe)
|
||||
import Data.Ord (Down (..))
|
||||
import Data.Text (Text)
|
||||
import qualified Data.Text as T
|
||||
@@ -69,9 +69,8 @@ data ClientCounts c = ClientCounts
|
||||
data ClientUsage c = ClientUsage (UsageStats c) ClientAddr
|
||||
|
||||
data UsageHistogram = UsageHistogram
|
||||
{ bucketCounts :: [Int],
|
||||
{ bucketCounts :: IM.IntMap Int,
|
||||
usageSum :: Int,
|
||||
clientCount :: Int,
|
||||
lastPeriodMax :: Int
|
||||
}
|
||||
deriving (Eq, Show)
|
||||
@@ -113,9 +112,9 @@ counterByName :: UsageCounter c => Text -> Maybe c
|
||||
counterByName name = find ((name ==) . counterName) [minBound .. maxBound]
|
||||
|
||||
newUsageStats :: UsageCounter c => IO (UsageStats c)
|
||||
newUsageStats = UsageStats <$> TM.emptyIO <*> newIORef (M.fromList $ map (\c -> (c, emptyHistogram c)) [minBound .. maxBound])
|
||||
newUsageStats = UsageStats <$> TM.emptyIO <*> newIORef (M.fromList $ map (,emptyHistogram) [minBound .. maxBound])
|
||||
where
|
||||
emptyHistogram c = UsageHistogram {bucketCounts = map (const 0) (counterBounds c), usageSum = 0, clientCount = 0, lastPeriodMax = 0}
|
||||
emptyHistogram = UsageHistogram {bucketCounts = IM.empty, usageSum = 0, lastPeriodMax = 0}
|
||||
|
||||
clientConnected :: UsageCounter c => Socket -> Maybe (UsageStats c) -> IO (Maybe (ClientUsage c))
|
||||
clientConnected sock stats_ =
|
||||
@@ -135,7 +134,7 @@ addUsage (ClientUsage UsageStats {clientCounts} addr) c n = do
|
||||
TM.lookupIO c current >>= atomically . maybe (newCount current) add
|
||||
where
|
||||
add v = modifyTVar' v (+ n)
|
||||
newCount current = TM.lookup c current >>= maybe (TM.insertM c (newTVar n) current) add
|
||||
newCount current = TM.lookup c current >>= maybe (TM.insertM c (newTVar $! n) current) add
|
||||
newCounts = do
|
||||
counts <- ClientCounts <$> TM.emptyIO <*> newTVarIO M.empty
|
||||
atomically $ TM.lookup addr clientCounts >>= maybe (counts <$ TM.insert addr counts clientCounts) pure
|
||||
@@ -143,8 +142,7 @@ addUsage (ClientUsage UsageStats {clientCounts} addr) c n = do
|
||||
closeUsagePeriod :: UsageCounter c => UsageStats c -> IO ()
|
||||
closeUsagePeriod UsageStats {clientCounts, histograms} = do
|
||||
closedCounts <- mapM closeClientPeriod . M.toList =<< readTVarIO clientCounts
|
||||
let periodCounts = M.unionsWith (<>) $ map (M.map (: [])) closedCounts
|
||||
atomicModifyIORef'_ histograms $ M.mapWithKey $ \c -> addPeriodCounts (counterBounds c) (M.findWithDefault [] c periodCounts)
|
||||
atomicModifyIORef'_ histograms $ M.mapWithKey $ \c -> addPeriodCounts (counterBounds c) (mapMaybe (M.lookup c) closedCounts)
|
||||
where
|
||||
closeClientPeriod (addr, ClientCounts {current, previous}) = do
|
||||
counts <- atomically $ do
|
||||
@@ -154,11 +152,10 @@ closeUsagePeriod UsageStats {clientCounts, histograms} = do
|
||||
pure counts
|
||||
|
||||
addPeriodCounts :: [Int] -> [Int] -> UsageHistogram -> UsageHistogram
|
||||
addPeriodCounts bounds ns UsageHistogram {bucketCounts, usageSum, clientCount} =
|
||||
addPeriodCounts bounds ns UsageHistogram {bucketCounts, usageSum} =
|
||||
UsageHistogram
|
||||
{ bucketCounts = zipWith (+) bucketCounts $ scanl1 (+) $ map (\i -> IM.findWithDefault 0 i boundCounts) [0 .. length bounds - 1],
|
||||
{ bucketCounts = IM.unionWith (+) bucketCounts boundCounts,
|
||||
usageSum = usageSum + sum ns,
|
||||
clientCount = clientCount + length ns,
|
||||
lastPeriodMax = maximum (0 : ns)
|
||||
}
|
||||
where
|
||||
@@ -204,10 +201,13 @@ usageMetrics prefix tsEpoch usageHistograms =
|
||||
hs = M.toList usageHistograms
|
||||
usageMetric = prefix <> "_client_usage"
|
||||
maxMetric = prefix <> "_client_usage_max"
|
||||
histogramLines (c, UsageHistogram {bucketCounts, usageSum, clientCount}) =
|
||||
T.concat (zipWith (\b cnt -> sample (usageMetric <> "_bucket") c [("le", tshow b)] cnt) (counterBounds c) bucketCounts)
|
||||
histogramLines (c, UsageHistogram {bucketCounts, usageSum}) =
|
||||
T.concat (zipWith (\b cnt -> sample (usageMetric <> "_bucket") c [("le", tshow b)] cnt) (counterBounds c) cumulativeCounts)
|
||||
<> sample (usageMetric <> "_bucket") c [("le", "+Inf")] clientCount
|
||||
<> sample (usageMetric <> "_sum") c [] usageSum
|
||||
<> sample (usageMetric <> "_count") c [] clientCount
|
||||
where
|
||||
cumulativeCounts = scanl1 (+) $ map (\i -> IM.findWithDefault 0 i bucketCounts) [0 ..]
|
||||
clientCount = sum bucketCounts
|
||||
sample metric c labels v =
|
||||
metric <> "{" <> T.intercalate "," (map (\(l, lv) -> l <> "=\"" <> lv <> "\"") (("counter", counterName c) : labels)) <> "} " <> tshow v <> " " <> tsEpoch <> "\n"
|
||||
@@ -6,10 +6,11 @@
|
||||
|
||||
module CoreTests.UsageStatsTests where
|
||||
|
||||
import Control.Monad ((<=<))
|
||||
import Control.Monad (forM_, (<=<))
|
||||
import Data.ByteString.Char8 (ByteString)
|
||||
import qualified Data.ByteString.Char8 as B
|
||||
import Data.IORef (readIORef)
|
||||
import qualified Data.IntMap.Strict as IM
|
||||
import Data.List.NonEmpty (NonEmpty (..))
|
||||
import qualified Data.Map.Strict as M
|
||||
import qualified Data.Set as S
|
||||
@@ -28,6 +29,7 @@ import Simplex.Messaging.SimplexName (SimplexDomain (..), SimplexTLD (..))
|
||||
import qualified Simplex.Messaging.TMap as TM
|
||||
import Simplex.Messaging.Transport (supportedClientSMPRelayVRange, supportedServerSMPRelayVRange)
|
||||
import Simplex.Messaging.Version (VersionRange (..))
|
||||
import System.Timeout (timeout)
|
||||
import Test.Hspec hiding (fit, it)
|
||||
import UnliftIO.STM
|
||||
import Util
|
||||
@@ -138,17 +140,17 @@ closeUsagePeriodTests = do
|
||||
closeUsagePeriod stats
|
||||
readIORef histograms
|
||||
`shouldReturn` M.fromList
|
||||
[ (TCConnections, UsageHistogram [2, 2, 2, 2] 2 2 1),
|
||||
(TCSend, UsageHistogram [0, 0, 1, 2] 10 2 7),
|
||||
(TCGet, UsageHistogram [0, 0, 0, 0] 0 0 0)
|
||||
[ (TCConnections, UsageHistogram (IM.fromList [(0, 2)]) 2 1),
|
||||
(TCSend, UsageHistogram (IM.fromList [(2, 1), (3, 1)]) 10 7),
|
||||
(TCGet, UsageHistogram IM.empty 0 0)
|
||||
]
|
||||
addUsage (usage addr1) TCSend 20
|
||||
closeUsagePeriod stats
|
||||
readIORef histograms
|
||||
`shouldReturn` M.fromList
|
||||
[ (TCConnections, UsageHistogram [2, 2, 2, 2] 2 2 0),
|
||||
(TCSend, UsageHistogram [0, 0, 1, 2] 30 3 20),
|
||||
(TCGet, UsageHistogram [0, 0, 0, 0] 0 0 0)
|
||||
[ (TCConnections, UsageHistogram (IM.fromList [(0, 2)]) 2 0),
|
||||
(TCSend, UsageHistogram (IM.fromList [(2, 1), (3, 1), (4, 1)]) 30 20),
|
||||
(TCGet, UsageHistogram IM.empty 0 0)
|
||||
]
|
||||
it "should set previous counts to closed counts" $ do
|
||||
stats <- newUsageStats
|
||||
@@ -175,6 +177,11 @@ closeUsagePeriodTests = do
|
||||
M.keys <$> readTVarIO clientCounts `shouldReturn` [addr1]
|
||||
closeUsagePeriod stats
|
||||
M.keys <$> readTVarIO clientCounts `shouldReturn` []
|
||||
it "should close a period with counts of 40000 clients within 5 seconds" $ do
|
||||
stats@UsageStats {histograms} <- newUsageStats
|
||||
forM_ [1 .. 40000] $ \a -> incUsage (ClientUsage stats $ ClientIPv4 $ toEnum a) TCSend
|
||||
timeout 5000000 (closeUsagePeriod stats) `shouldReturn` Just ()
|
||||
M.lookup TCSend <$> readIORef histograms `shouldReturn` Just (UsageHistogram (IM.fromList [(0, 40000)]) 40000 1)
|
||||
|
||||
topClientsTests :: Spec
|
||||
topClientsTests =
|
||||
@@ -224,8 +231,8 @@ metricsTests =
|
||||
where
|
||||
usageHistograms =
|
||||
M.fromList
|
||||
[ (TCSend, UsageHistogram [0, 0, 1, 2] 10 2 7),
|
||||
(TCGet, UsageHistogram [0, 0, 0, 0] 12 1 12)
|
||||
[ (TCSend, UsageHistogram (IM.fromList [(2, 1), (3, 1)]) 10 7),
|
||||
(TCGet, UsageHistogram (IM.fromList [(4, 1)]) 12 12)
|
||||
]
|
||||
|
||||
commandCounterTests :: Spec
|
||||
|
||||
@@ -119,6 +119,10 @@ ntfTestDBCfg2 = ntfTestDBCfg {dbOpts = ntfTestStoreDBOpts2, dbStoreLogPath = Jus
|
||||
testNtfClient :: Transport c => (THandleNTF c 'TClient -> IO a) -> IO a
|
||||
testNtfClient client = do
|
||||
Right host <- pure $ chooseTransportHost defaultNetworkConfig testHost
|
||||
testNtfClient_ host client
|
||||
|
||||
testNtfClient_ :: Transport c => TransportHost -> (THandleNTF c 'TClient -> IO a) -> IO a
|
||||
testNtfClient_ host client =
|
||||
runTransportClient defaultTransportClientConfig Nothing host ntfTestPort (Just testKeyHash) $ \h ->
|
||||
runExceptT (ntfClientHandshake h testKeyHash supportedClientNTFVRange False Nothing) >>= \case
|
||||
Right th -> client th
|
||||
|
||||
@@ -58,6 +58,8 @@ ntfServerTests ps@(t, _) = do
|
||||
describe "Notification subscriptions (NKEY)" $ testNotificationSubscription ps createNtfQueueNKEY
|
||||
describe "Notification subscriptions (NEW with ntf creds)" $ testNotificationSubscription ps createNtfQueueNEW
|
||||
describe "Retried notification subscription" $ testRetriedNtfSubscription ps
|
||||
describe "Client usage statistics" $
|
||||
it "should count commands and errors by client address" $ testNtfUsageStats t
|
||||
|
||||
ntfSyntaxTests :: ASrvTransport -> Spec
|
||||
ntfSyntaxTests (ATransport t) = do
|
||||
@@ -88,6 +90,27 @@ testApnsTestProviderRejected (ATransport (t :: TProxy c 'TServer)) = do
|
||||
signSendRecvNtf nh tknKey ("1", NoEntity, TNEW $ NewNtfTkn tkn tknPub dhPub)
|
||||
pure ()
|
||||
|
||||
testNtfUsageStats :: ASrvTransport -> Expectation
|
||||
testNtfUsageStats (ATransport (t :: TProxy c 'TServer)) = do
|
||||
g <- C.newRandom
|
||||
(tknPub, tknKey) <- atomically $ C.generateAuthKeyPair C.SEd25519 g
|
||||
(dhPub, _dhPriv :: C.PrivateKeyX25519) <- atomically $ C.generateKeyPair g
|
||||
let tkn = DeviceToken PPApnsTest "abcd"
|
||||
ntfCfg =
|
||||
ntfServerCfg
|
||||
{ allowTestPushProvider = False,
|
||||
transports = [(ntfTestPort, ATransport t, False)],
|
||||
usageStatsPeriod = Just 300,
|
||||
controlPort = Just testControlPort,
|
||||
controlPortUserAuth = Just testControlPortUserAuth
|
||||
}
|
||||
withNtfServerCfg ntfCfg $ \_ ->
|
||||
testNtfClient_ "127.0.0.1" $ \(nh :: THandleNTF c 'TClient) -> do
|
||||
RespNtf "1" NoEntity (NRErr (CMD PROHIBITED)) <- signSendRecvNtf nh tknKey ("1", NoEntity, TNEW $ NewNtfTkn tkn tknPub dhPub)
|
||||
RespNtf "2" _ (NRErr AUTH) <- signSendRecvNtf nh tknKey ("2", EntityId "1234", TCHK)
|
||||
RespNtf "3" NoEntity (NRErr (CMD NO_AUTH)) <- signSendRecvNtf nh tknKey ("3", NoEntity, TCHK)
|
||||
checkUsageCounts [("connections", "1"), ("TNEW", "1"), ("other", "1"), ("errors", "2")]
|
||||
|
||||
pattern RespNtf :: CorrId -> QueueId -> NtfResponse -> Transmission (Either ErrorType NtfResponse)
|
||||
pattern RespNtf corrId queueId command <- (corrId, queueId, Right command)
|
||||
|
||||
|
||||
@@ -422,6 +422,13 @@ controlPortCommand h cmd n = do
|
||||
hFlush h
|
||||
replicateM n $ trimCR <$> B.hGetLine h
|
||||
|
||||
checkUsageCounts :: HasCallStack => [(ByteString, ByteString)] -> Expectation
|
||||
checkUsageCounts counts =
|
||||
withControlPort $ \h -> do
|
||||
controlPortCommand h ("auth " <> unBasicAuth testControlPortUserAuth) 1 `shouldReturn` ["Current role is CPRUser"]
|
||||
forM_ counts $ \(counter, n) ->
|
||||
controlPortCommand h ("usage " <> counter) 2 `shouldReturn` ["address,previous,current", "127.0.0.1,0," <> n]
|
||||
|
||||
withSmpServerOn :: HasCallStack => (ASrvTransport, AStoreType) -> ServiceName -> IO a -> IO a
|
||||
withSmpServerOn ps port' = withSmpServerThreadOn ps port' . const
|
||||
|
||||
|
||||
@@ -164,13 +164,8 @@ smpProxyTests = do
|
||||
}
|
||||
testProxyCounts = do
|
||||
deliverMessageViaProxy (SMPServer testHost2 testPort testKeyHash) (SMPServer testHost2 testPort2 testKeyHash) C.SEd448 "hello 1" "hello 2"
|
||||
withControlPort $ \h -> do
|
||||
controlPortCommand h ("auth " <> SMP.unBasicAuth testControlPortUserAuth) 1 `shouldReturn` ["Current role is CPRUser"]
|
||||
controlPortCommand h "usage PRXY_failed" 1 `shouldReturn` ["address,previous,current"]
|
||||
forM_ counts $ \(counter, n) ->
|
||||
controlPortCommand h ("usage " <> counter) 2 `shouldReturn` ["address,previous,current", "127.0.0.1,0," <> n]
|
||||
counts :: [(ByteString, ByteString)]
|
||||
counts = [("other", "2"), ("PRXY_new", "1"), ("PFWD", "3"), ("PFWD_failed", "1")]
|
||||
proxyConnectDeadRelay 1 0 (SMPServer testHost2 testPort testKeyHash)
|
||||
checkUsageCounts [("other", "3"), ("PRXY_new", "2"), ("PRXY_failed", "1"), ("PFWD", "3"), ("PFWD_failed", "1")]
|
||||
oneServer test msType = withSmpServerConfigOn (transport @TLS) (updateCfg (proxyCfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 128, maxJournalMsgCount = 256}) testPort $ const test
|
||||
twoServers test msType = twoServers_ (proxyCfgMS msType) (proxyCfgMS msType) test msType
|
||||
twoServersFirstProxy test msType = twoServers_ (proxyCfgMS msType) (updateCfg (cfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 128, maxJournalMsgCount = 256}) test msType
|
||||
|
||||
@@ -1270,12 +1270,14 @@ testUsageStatsControlPort =
|
||||
(sId, _, _, _) <- createAndSecureQueue r sPub
|
||||
Resp "bcda" _ OK <- signSendRecv s sKey ("bcda", sId, _SEND "hello")
|
||||
Resp "cdab" _ OK <- signSendRecv s sKey ("cdab", sId, _SEND "hello again")
|
||||
Resp "dabc" _ (ERR AUTH) <- signSendRecv s sKey ("dabc", EntityId "1234", _SEND "hello")
|
||||
Resp "abcd" _ (ERR (CMD NO_ENTITY)) <- sendRecv s ("", "abcd", NoEntity, _SEND "hello")
|
||||
withControlPort $ \h -> do
|
||||
controlPortCommand h "usage SEND" 1 `shouldReturn` ["AUTH"]
|
||||
controlPortCommand h ("auth " <> unBasicAuth testControlPortUserAuth) 1 `shouldReturn` ["Current role is CPRUser"]
|
||||
controlPortCommand h "usage SEND 0" 1 `shouldReturn` ["address,previous,current"]
|
||||
controlPortCommand h "usage unknown" 1 `shouldReturn` ["error: unknown counter"]
|
||||
controlPortCommand h "usage SEND" 2 `shouldReturn` ["address,previous,current", "127.0.0.1,0,2"]
|
||||
controlPortCommand h "usage connections" 2 `shouldReturn` ["address,previous,current", "127.0.0.1,0,2"]
|
||||
checkUsageCounts [("SEND", "3"), ("errors", "2"), ("connections", "2")]
|
||||
|
||||
createAndSecureQueue :: Transport c => THandleSMP c 'TClient -> SndPublicAuthKey -> IO (SenderId, RecipientId, RcvPrivateAuthKey, RcvDhSecret)
|
||||
createAndSecureQueue h sPub = do
|
||||
|
||||
+11
-12
@@ -28,7 +28,7 @@ import qualified Data.X509 as X
|
||||
import Data.X509.Validation (Fingerprint (..), getFingerprint)
|
||||
import Network.HPACK.Token (tokenKey)
|
||||
import qualified Network.HTTP2.Client as H2
|
||||
import SMPClient (controlPortCommand, testControlPort, testControlPortUserAuth, withControlPort)
|
||||
import SMPClient (checkUsageCounts, testControlPort, testControlPortUserAuth, testHost2, testKeyHash)
|
||||
import ServerTests (logSize)
|
||||
import Simplex.FileTransfer.Client hiding (createXFTPChunk)
|
||||
import qualified Simplex.FileTransfer.Client as A
|
||||
@@ -40,7 +40,7 @@ import Simplex.Messaging.Client (ProtocolClientError (..))
|
||||
import qualified Simplex.Messaging.Crypto as C
|
||||
import qualified Simplex.Messaging.Crypto.Lazy as LC
|
||||
import Simplex.Messaging.Encoding (smpDecode, smpEncode)
|
||||
import Simplex.Messaging.Protocol (BasicAuth (..), EntityId (..), RecipientId, SenderId, pattern NoEntity)
|
||||
import Simplex.Messaging.Protocol (BasicAuth, CommandError (..), EntityId (..), RecipientId, SenderId, pattern NoEntity, pattern XFTPServer)
|
||||
import Simplex.Messaging.Server.Expiration (ExpirationConfig (..))
|
||||
import Simplex.Messaging.Transport (CertChainPubKey (..), TLS (..), TransportPeer (..), defaultSupportedParams, defaultSupportedParamsHTTPS)
|
||||
import Simplex.Messaging.Transport.Client (TransportClientConfig (..), TransportHost (..), defaultTransportClientConfig, runTLSTransportClient)
|
||||
@@ -69,7 +69,7 @@ xftpServerTests =
|
||||
it "should disconnect inactive clients" testInactiveClientExpiration
|
||||
it "should not allow uploading chunks after specified storage quota" testFileStorageQuota
|
||||
it "should store file records to log and restore them after server restart" testFileLog
|
||||
it "should count commands, transfers and recipients by client address" testFileUsageStats
|
||||
it "should count commands, errors, transfers and recipients by client address" testFileUsageStats
|
||||
describe "XFTP basic auth" $ do
|
||||
-- allow FNEW | server auth | clnt auth | success
|
||||
it "prohibited without basic auth" $ testFileBasicAuth True (Just "pwd") Nothing False
|
||||
@@ -341,16 +341,15 @@ testFileUsageStats fsType =
|
||||
withXFTPServerConfigOn (updateXFTPCfg (cfgFS fsType) $ \c -> c {usageStatsPeriod = Just 300, controlPort = Just testControlPort, controlPortUserAuth = Just testControlPortUserAuth}) $ \_ -> do
|
||||
ts <- getCurrentTime
|
||||
runRight_ $ do
|
||||
c <- ExceptT $ getXFTPClient (1, "xftp://LcJUMfVhwD8yxjAiSaDzzGF3-kLG4Uh0Fl_ZIjrRwjI=@127.0.0.1:8000", Nothing) testXFTPClientConfig [] ts (\_ -> pure Nothing) (\_ -> pure ())
|
||||
c <- ExceptT $ getXFTPClient (1, XFTPServer testHost2 xftpTestPort testKeyHash, Nothing) testXFTPClientConfig [] ts (\_ -> pure Nothing) (\_ -> pure ())
|
||||
runTestFileChunkDelivery c c
|
||||
withControlPort $ \h -> do
|
||||
controlPortCommand h ("auth " <> unBasicAuth testControlPortUserAuth) 1 `shouldReturn` ["Current role is CPRUser"]
|
||||
controlPortCommand h "usage errors" 1 `shouldReturn` ["address,previous,current"]
|
||||
forM_ counts $ \(counter, n) ->
|
||||
controlPortCommand h ("usage " <> counter) 2 `shouldReturn` ["address,previous,current", "127.0.0.1,0," <> n]
|
||||
where
|
||||
counts :: [(ByteString, ByteString)]
|
||||
counts = [("connections", "1"), ("FNEW", "2"), ("other", "4"), ("recipients", "2"), ("upload_kb", "128"), ("download_kb", "256")]
|
||||
g <- liftIO C.newRandom
|
||||
(_, spKey) <- atomically $ C.generateAuthKeyPair C.SEd25519 g
|
||||
deleteXFTPChunk c spKey (EntityId "1234")
|
||||
`catchError` (liftIO . (`shouldBe` PCEProtocolError AUTH))
|
||||
deleteXFTPChunk c spKey NoEntity
|
||||
`catchError` (liftIO . (`shouldBe` PCEProtocolError (CMD NO_AUTH)))
|
||||
checkUsageCounts [("connections", "1"), ("FNEW", "2"), ("other", "5"), ("errors", "2"), ("recipients", "2"), ("upload_kb", "128"), ("download_kb", "256")]
|
||||
|
||||
testFileBasicAuth :: Bool -> Maybe BasicAuth -> Maybe BasicAuth -> Bool -> AFStoreType -> IO ()
|
||||
testFileBasicAuth allowNewFiles newFileBasicAuth clntAuth success fsType =
|
||||
|
||||
Reference in new issue
Block a user