mirror of
https://github.com/simplex-chat/simplexmq.git
synced 2026-07-28 20:50:10 +00:00
xftp server: prometheus metrics (#1595)
* xftp server: prometheus metrics * update Co-authored-by: sh <37271604+shumvgolove@users.noreply.github.com> --------- Co-authored-by: sh <37271604+shumvgolove@users.noreply.github.com>
This commit is contained in:
Evgeny
@@ -224,6 +224,7 @@ library
|
||||
Simplex.FileTransfer.Server.Control
|
||||
Simplex.FileTransfer.Server.Env
|
||||
Simplex.FileTransfer.Server.Main
|
||||
Simplex.FileTransfer.Server.Prometheus
|
||||
Simplex.FileTransfer.Server.Stats
|
||||
Simplex.FileTransfer.Server.Store
|
||||
Simplex.FileTransfer.Server.StoreLog
|
||||
|
||||
@@ -45,6 +45,7 @@ import Network.Socket
|
||||
import Simplex.FileTransfer.Protocol
|
||||
import Simplex.FileTransfer.Server.Control
|
||||
import Simplex.FileTransfer.Server.Env
|
||||
import Simplex.FileTransfer.Server.Prometheus
|
||||
import Simplex.FileTransfer.Server.Stats
|
||||
import Simplex.FileTransfer.Server.Store
|
||||
import Simplex.FileTransfer.Server.StoreLog
|
||||
@@ -69,6 +70,7 @@ import Simplex.Messaging.Transport.HTTP2.Server
|
||||
import Simplex.Messaging.Transport.Server (runLocalTCPServer)
|
||||
import Simplex.Messaging.Util
|
||||
import Simplex.Messaging.Version
|
||||
import System.Environment (lookupEnv)
|
||||
import System.Exit (exitFailure)
|
||||
import System.FilePath ((</>))
|
||||
import System.IO (hPrint, hPutStrLn, universalNewlineMode)
|
||||
@@ -105,7 +107,14 @@ xftpServer :: XFTPServerConfig -> TMVar Bool -> M ()
|
||||
xftpServer cfg@XFTPServerConfig {xftpPort, transportConfig, inactiveClientExpiration, fileExpiration, xftpServerVRange} started = do
|
||||
mapM_ (expireServerFiles Nothing) fileExpiration
|
||||
restoreServerStats
|
||||
raceAny_ (runServer : expireFilesThread_ cfg <> serverStatsThread_ cfg <> controlPortThread_ cfg) `finally` stopServer
|
||||
raceAny_
|
||||
( runServer
|
||||
: expireFilesThread_ cfg
|
||||
<> serverStatsThread_ cfg
|
||||
<> prometheusMetricsThread_ cfg
|
||||
<> controlPortThread_ cfg
|
||||
)
|
||||
`finally` stopServer
|
||||
where
|
||||
runServer :: M ()
|
||||
runServer = do
|
||||
@@ -240,6 +249,30 @@ xftpServer cfg@XFTPServerConfig {xftpPort, transportConfig, inactiveClientExpira
|
||||
]
|
||||
liftIO $ threadDelay' interval
|
||||
|
||||
prometheusMetricsThread_ :: XFTPServerConfig -> [M ()]
|
||||
prometheusMetricsThread_ XFTPServerConfig {prometheusInterval = Just interval, prometheusMetricsFile} =
|
||||
[savePrometheusMetrics interval prometheusMetricsFile]
|
||||
prometheusMetricsThread_ _ = []
|
||||
|
||||
savePrometheusMetrics :: Int -> FilePath -> M ()
|
||||
savePrometheusMetrics saveInterval metricsFile = do
|
||||
labelMyThread "savePrometheusMetrics"
|
||||
liftIO $ putStrLn $ "Prometheus metrics saved every " <> show saveInterval <> " seconds to " <> metricsFile
|
||||
ss <- asks serverStats
|
||||
rtsOpts <- liftIO $ maybe ("set " <> rtsOptionsEnv) T.pack <$> lookupEnv (T.unpack rtsOptionsEnv)
|
||||
let interval = 1000000 * saveInterval
|
||||
liftIO $ forever $ do
|
||||
threadDelay interval
|
||||
ts <- getCurrentTime
|
||||
sm <- getFileServerMetrics ss rtsOpts
|
||||
T.writeFile metricsFile $ xftpPrometheusMetrics sm ts
|
||||
|
||||
getFileServerMetrics :: FileServerStats -> T.Text -> IO FileServerMetrics
|
||||
getFileServerMetrics ss rtsOptions = do
|
||||
d <- getFileServerStatsData ss
|
||||
let fd = periodStatDataCounts $ _filesDownloaded d
|
||||
pure FileServerMetrics {statsData = d, filesDownloadedPeriods = fd, rtsOptions}
|
||||
|
||||
controlPortThread_ :: XFTPServerConfig -> [M ()]
|
||||
controlPortThread_ XFTPServerConfig {controlPort = Just port} = [runCPServer port]
|
||||
controlPortThread_ _ = []
|
||||
|
||||
@@ -64,6 +64,8 @@ data XFTPServerConfig = XFTPServerConfig
|
||||
logStatsStartTime :: Int64,
|
||||
serverStatsLogFile :: FilePath,
|
||||
serverStatsBackupFile :: Maybe FilePath,
|
||||
prometheusInterval :: Maybe Int,
|
||||
prometheusMetricsFile :: FilePath,
|
||||
transportConfig :: TransportServerConfig,
|
||||
responseDelay :: Int
|
||||
}
|
||||
|
||||
@@ -30,7 +30,7 @@ import Simplex.Messaging.Server.Expiration
|
||||
import Simplex.Messaging.Transport (simplexMQVersion)
|
||||
import Simplex.Messaging.Transport.Client (TransportHost (..))
|
||||
import Simplex.Messaging.Transport.Server (ServerCredentials (..), mkTransportServerConfig)
|
||||
import Simplex.Messaging.Util (safeDecodeUtf8, tshow)
|
||||
import Simplex.Messaging.Util (eitherToMaybe, safeDecodeUtf8, tshow)
|
||||
import System.Directory (createDirectoryIfMissing, doesFileExist)
|
||||
import System.FilePath (combine)
|
||||
import System.IO (BufferMode (..), hSetBuffering, stderr, stdout)
|
||||
@@ -89,6 +89,9 @@ xftpServerCLI cfgPath logPath = do
|
||||
<> "# Expire files after the specified number of hours.\n"
|
||||
<> ("expire_files_hours: " <> tshow defFileExpirationHours <> "\n\n")
|
||||
<> "log_stats: off\n\
|
||||
\\n\
|
||||
\# Log interval for real-time Prometheus metrics\n\
|
||||
\# prometheus_interval: 60\n\
|
||||
\\n\
|
||||
\[AUTH]\n\
|
||||
\# Set new_files option to off to completely prohibit uploading new files.\n\
|
||||
@@ -188,6 +191,8 @@ xftpServerCLI cfgPath logPath = do
|
||||
logStatsStartTime = 0, -- seconds from 00:00 UTC
|
||||
serverStatsLogFile = combine logPath "file-server-stats.daily.log",
|
||||
serverStatsBackupFile = logStats $> combine logPath "file-server-stats.log",
|
||||
prometheusInterval = eitherToMaybe $ read . T.unpack <$> lookupValue "STORE_LOG" "prometheus_interval" ini,
|
||||
prometheusMetricsFile = combine logPath "xftp-server-metrics.txt",
|
||||
transportConfig =
|
||||
mkTransportServerConfig
|
||||
(fromMaybe False $ iniOnOff "TRANSPORT" "log_tls_errors" ini)
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
{-# LANGUAGE DuplicateRecordFields #-}
|
||||
{-# LANGUAGE NamedFieldPuns #-}
|
||||
{-# LANGUAGE OverloadedStrings #-}
|
||||
{-# LANGUAGE TypeApplications #-}
|
||||
{-# OPTIONS_GHC -fno-warn-unrecognised-pragmas #-}
|
||||
|
||||
module Simplex.FileTransfer.Server.Prometheus where
|
||||
|
||||
import Data.Int (Int64)
|
||||
import Data.Text (Text)
|
||||
import qualified Data.Text as T
|
||||
import Data.Time.Clock (UTCTime (..), diffUTCTime)
|
||||
import Data.Time.Clock.System (systemEpochDay)
|
||||
import Data.Time.Format.ISO8601 (iso8601Show)
|
||||
import Simplex.FileTransfer.Server.Stats
|
||||
import Simplex.Messaging.Server.Stats (PeriodStatCounts (..))
|
||||
import Simplex.Messaging.Transport (simplexMQVersion)
|
||||
import Simplex.Messaging.Util (tshow)
|
||||
|
||||
data FileServerMetrics = FileServerMetrics
|
||||
{ statsData :: FileServerStatsData,
|
||||
filesDownloadedPeriods :: PeriodStatCounts,
|
||||
rtsOptions :: Text
|
||||
}
|
||||
|
||||
rtsOptionsEnv :: Text
|
||||
rtsOptionsEnv = "XFTP_RTS_OPTIONS"
|
||||
|
||||
{-# FOURMOLU_DISABLE\n#-}
|
||||
xftpPrometheusMetrics :: FileServerMetrics -> UTCTime -> Text
|
||||
xftpPrometheusMetrics sm ts =
|
||||
time <> files <> info
|
||||
where
|
||||
FileServerMetrics {statsData, filesDownloadedPeriods, rtsOptions} = sm
|
||||
FileServerStatsData
|
||||
{ _fromTime,
|
||||
_filesCreated,
|
||||
_fileRecipients,
|
||||
_filesUploaded,
|
||||
_filesExpired,
|
||||
_filesDeleted,
|
||||
_filesBlocked,
|
||||
_fileDownloads,
|
||||
_fileDownloadAcks,
|
||||
_filesCount,
|
||||
_filesSize
|
||||
} = statsData
|
||||
time =
|
||||
"# Recorded at: " <> T.pack (iso8601Show ts) <> "\n\
|
||||
\# Stats from: " <> T.pack (iso8601Show _fromTime) <> "\n\
|
||||
\\n"
|
||||
files =
|
||||
"# Files\n\
|
||||
\# -----\n\
|
||||
\\n\
|
||||
\# HELP simplex_xftp_files_created Created files\n\
|
||||
\# TYPE simplex_xftp_files_created counter\n\
|
||||
\simplex_xftp_files_created " <> mshow _filesCreated <> "\n\
|
||||
\# filesCreated\n\
|
||||
\\n\
|
||||
\# HELP simplex_xftp_files_recipients Files recipients\n\
|
||||
\# TYPE simplex_xftp_files_recipients counter\n\
|
||||
\simplex_xftp_files_recipients " <> mshow _fileRecipients <> "\n\
|
||||
\# fileRecipients\n\
|
||||
\\n\
|
||||
\# HELP simplex_xftp_files_uploaded Uploaded files\n\
|
||||
\# TYPE simplex_xftp_files_uploaded counter\n\
|
||||
\simplex_xftp_files_uploaded " <> mshow _filesUploaded <> "\n\
|
||||
\# filesUploaded\n\
|
||||
\\n\
|
||||
\# HELP simplex_xftp_files_expired Expired files\n\
|
||||
\# TYPE simplex_xftp_files_expired counter\n\
|
||||
\simplex_xftp_files_expired " <> mshow _filesExpired <> "\n\
|
||||
\# filesExpired\n\
|
||||
\\n\
|
||||
\# HELP simplex_xftp_files_deleted Deleted files\n\
|
||||
\# TYPE simplex_xftp_files_deleted counter\n\
|
||||
\simplex_xftp_files_deleted " <> mshow _filesDeleted <> "\n\
|
||||
\# filesDeleted\n\
|
||||
\\n\
|
||||
\# HELP simplex_xftp_files_blocked Blocked files\n\
|
||||
\# TYPE simplex_xftp_files_blocked counter\n\
|
||||
\simplex_xftp_files_blocked " <> mshow _filesBlocked <> "\n\
|
||||
\# filesBlocked\n\
|
||||
\\n\
|
||||
\# HELP simplex_xftp_file_downloads File downloads\n\
|
||||
\# TYPE simplex_xftp_file_downloads counter\n\
|
||||
\simplex_xftp_file_downloads " <> mshow _fileDownloads <> "\n\
|
||||
\# fileDownloads\n\
|
||||
\\n\
|
||||
\# HELP simplex_xftp_file_download_acks File download ACKs\n\
|
||||
\# TYPE simplex_xftp_file_download_acks counter\n\
|
||||
\simplex_xftp_file_download_acks " <> mshow _fileDownloadAcks <> "\n\
|
||||
\# fileDownloadAcks\n\
|
||||
\\n\
|
||||
\# HELP simplex_xftp_files_count_total Total files count \n\
|
||||
\# TYPE simplex_xftp_files_count_total gauge\n\
|
||||
\simplex_xftp_files_count_total " <> mshow _filesCount <> "\n\
|
||||
\# filesCount\n\
|
||||
\\n\
|
||||
\# HELP simplex_xftp_files_size Size of files \n\
|
||||
\# TYPE simplex_xftp_files_size gauge\n\
|
||||
\simplex_xftp_files_size " <> mshow _filesSize <> "\n\
|
||||
\# filesSize \n\
|
||||
\\n\
|
||||
\# HELP simplex_xftp_files_count_daily Daily files count\n\
|
||||
\# TYPE simplex_xftp_files_count_daily gauge\n\
|
||||
\simplex_xftp_files_count_daily " <> mstr (dayCount filesDownloadedPeriods) <> "\n\
|
||||
\# filesDownloaded.dayCount\n\
|
||||
\\n\
|
||||
\# HELP simplex_xftp_files_count_weekly Weekly files count\n\
|
||||
\# TYPE simplex_xftp_files_count_weekly gauge\n\
|
||||
\simplex_xftp_files_count_weekly " <> mstr (weekCount filesDownloadedPeriods) <> "\n\
|
||||
\# filesDownloaded.weekCount\n\
|
||||
\\n\
|
||||
\# HELP simplex_xftp_files_count_monthly Monthly files count\n\
|
||||
\# TYPE simplex_xftp_files_count_monthly gauge\n\
|
||||
\simplex_xftp_files_count_monthly " <> mstr (monthCount filesDownloadedPeriods) <> "\n\
|
||||
\# filesDownloaded.monthCount\n\
|
||||
\\n"
|
||||
info =
|
||||
"# Info\n\
|
||||
\# ----\n\
|
||||
\\n\
|
||||
\# HELP simplex_xftp_info Server information. RTS options have to be passed via " <> rtsOptionsEnv <> " env var\n\
|
||||
\# TYPE simplex_xftp_info gauge\n\
|
||||
\simplex_xftp_info{version=\"" <> T.pack simplexMQVersion <> "\",rts_options=\"" <> rtsOptions <> "\"} 1\n\
|
||||
\\n"
|
||||
mstr a = a <> " " <> tsEpoch
|
||||
mshow :: Show a => a -> Text
|
||||
mshow = mstr . tshow
|
||||
tsEpoch = tshow @Int64 $ floor @Double $ realToFrac (ts `diffUTCTime` epoch) * 1000
|
||||
epoch = UTCTime systemEpochDay 0
|
||||
{-# FOURMOLU_ENABLE\n#-}
|
||||
@@ -167,6 +167,9 @@ ntfServerCLI cfgPath logPath =
|
||||
<> "Time to retain deleted entities in the database, days.\n"
|
||||
<> ("# db_deleted_ttl: " <> tshow defaultDeletedTTL <> "\n\n")
|
||||
<> "log_stats: off\n\n\
|
||||
\# Log interval for real-time Prometheus metrics\n\
|
||||
\# prometheus_interval: 60\n\
|
||||
\\n\
|
||||
\[AUTH]\n\
|
||||
\# control_port_admin_password:\n\
|
||||
\# control_port_user_password:\n\
|
||||
|
||||
@@ -92,7 +92,7 @@ iniFileContent cfgPath logPath opts host basicAuth controlPortPwds =
|
||||
<> "# Log daily server statistics to CSV file\n"
|
||||
<> ("log_stats: " <> onOff logStats <> "\n\n")
|
||||
<> "# Log interval for real-time Prometheus metrics\n\
|
||||
\# prometheus_interval: 300\n\n\
|
||||
\# prometheus_interval: 60\n\n\
|
||||
\[AUTH]\n\
|
||||
\# Set new_queues option to off to completely prohibit creating new messaging queues.\n\
|
||||
\# This can be useful when you want to decommission the server, but not all connections are switched yet.\n\
|
||||
|
||||
@@ -99,6 +99,9 @@ testXFTPLogFile = "tests/tmp/xftp-server-store.log"
|
||||
testXFTPStatsBackupFile :: FilePath
|
||||
testXFTPStatsBackupFile = "tests/tmp/xftp-server-stats.log"
|
||||
|
||||
xftpTestPrometheusMetricsFile :: FilePath
|
||||
xftpTestPrometheusMetricsFile = "tests/tmp/xftp-server-metrics.txt"
|
||||
|
||||
testXFTPServerConfig :: XFTPServerConfig
|
||||
testXFTPServerConfig =
|
||||
XFTPServerConfig
|
||||
@@ -127,6 +130,8 @@ testXFTPServerConfig =
|
||||
logStatsStartTime = 0,
|
||||
serverStatsLogFile = "tests/tmp/xftp-server-stats.daily.log",
|
||||
serverStatsBackupFile = Nothing,
|
||||
prometheusInterval = Nothing,
|
||||
prometheusMetricsFile = xftpTestPrometheusMetricsFile,
|
||||
transportConfig = mkTransportServerConfig True (Just alpnSupportedXFTPhandshakes) False,
|
||||
responseDelay = 0
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user