diff --git a/simplexmq.cabal b/simplexmq.cabal index eab76a9b2..f57d433f8 100644 --- a/simplexmq.cabal +++ b/simplexmq.cabal @@ -224,6 +224,7 @@ library Simplex.FileTransfer.Server.Control Simplex.FileTransfer.Server.Env Simplex.FileTransfer.Server.Main + Simplex.FileTransfer.Server.Prometheus Simplex.FileTransfer.Server.Stats Simplex.FileTransfer.Server.Store Simplex.FileTransfer.Server.StoreLog diff --git a/src/Simplex/FileTransfer/Server.hs b/src/Simplex/FileTransfer/Server.hs index c12c2b5f6..48e855f69 100644 --- a/src/Simplex/FileTransfer/Server.hs +++ b/src/Simplex/FileTransfer/Server.hs @@ -45,6 +45,7 @@ import Network.Socket import Simplex.FileTransfer.Protocol import Simplex.FileTransfer.Server.Control import Simplex.FileTransfer.Server.Env +import Simplex.FileTransfer.Server.Prometheus import Simplex.FileTransfer.Server.Stats import Simplex.FileTransfer.Server.Store import Simplex.FileTransfer.Server.StoreLog @@ -69,6 +70,7 @@ import Simplex.Messaging.Transport.HTTP2.Server import Simplex.Messaging.Transport.Server (runLocalTCPServer) import Simplex.Messaging.Util import Simplex.Messaging.Version +import System.Environment (lookupEnv) import System.Exit (exitFailure) import System.FilePath (()) import System.IO (hPrint, hPutStrLn, universalNewlineMode) @@ -105,7 +107,14 @@ xftpServer :: XFTPServerConfig -> TMVar Bool -> M () xftpServer cfg@XFTPServerConfig {xftpPort, transportConfig, inactiveClientExpiration, fileExpiration, xftpServerVRange} started = do mapM_ (expireServerFiles Nothing) fileExpiration restoreServerStats - raceAny_ (runServer : expireFilesThread_ cfg <> serverStatsThread_ cfg <> controlPortThread_ cfg) `finally` stopServer + raceAny_ + ( runServer + : expireFilesThread_ cfg + <> serverStatsThread_ cfg + <> prometheusMetricsThread_ cfg + <> controlPortThread_ cfg + ) + `finally` stopServer where runServer :: M () runServer = do @@ -240,6 +249,30 @@ xftpServer cfg@XFTPServerConfig {xftpPort, transportConfig, inactiveClientExpira ] liftIO $ threadDelay' interval + prometheusMetricsThread_ :: XFTPServerConfig -> [M ()] + prometheusMetricsThread_ XFTPServerConfig {prometheusInterval = Just interval, prometheusMetricsFile} = + [savePrometheusMetrics interval prometheusMetricsFile] + prometheusMetricsThread_ _ = [] + + savePrometheusMetrics :: Int -> FilePath -> M () + savePrometheusMetrics saveInterval metricsFile = do + labelMyThread "savePrometheusMetrics" + liftIO $ putStrLn $ "Prometheus metrics saved every " <> show saveInterval <> " seconds to " <> metricsFile + ss <- asks serverStats + rtsOpts <- liftIO $ maybe ("set " <> rtsOptionsEnv) T.pack <$> lookupEnv (T.unpack rtsOptionsEnv) + let interval = 1000000 * saveInterval + liftIO $ forever $ do + threadDelay interval + ts <- getCurrentTime + sm <- getFileServerMetrics ss rtsOpts + T.writeFile metricsFile $ xftpPrometheusMetrics sm ts + + getFileServerMetrics :: FileServerStats -> T.Text -> IO FileServerMetrics + getFileServerMetrics ss rtsOptions = do + d <- getFileServerStatsData ss + let fd = periodStatDataCounts $ _filesDownloaded d + pure FileServerMetrics {statsData = d, filesDownloadedPeriods = fd, rtsOptions} + controlPortThread_ :: XFTPServerConfig -> [M ()] controlPortThread_ XFTPServerConfig {controlPort = Just port} = [runCPServer port] controlPortThread_ _ = [] diff --git a/src/Simplex/FileTransfer/Server/Env.hs b/src/Simplex/FileTransfer/Server/Env.hs index 21e6d06dd..206a5b375 100644 --- a/src/Simplex/FileTransfer/Server/Env.hs +++ b/src/Simplex/FileTransfer/Server/Env.hs @@ -64,6 +64,8 @@ data XFTPServerConfig = XFTPServerConfig logStatsStartTime :: Int64, serverStatsLogFile :: FilePath, serverStatsBackupFile :: Maybe FilePath, + prometheusInterval :: Maybe Int, + prometheusMetricsFile :: FilePath, transportConfig :: TransportServerConfig, responseDelay :: Int } diff --git a/src/Simplex/FileTransfer/Server/Main.hs b/src/Simplex/FileTransfer/Server/Main.hs index e8b818f5b..fcac06cd4 100644 --- a/src/Simplex/FileTransfer/Server/Main.hs +++ b/src/Simplex/FileTransfer/Server/Main.hs @@ -30,7 +30,7 @@ import Simplex.Messaging.Server.Expiration import Simplex.Messaging.Transport (simplexMQVersion) import Simplex.Messaging.Transport.Client (TransportHost (..)) import Simplex.Messaging.Transport.Server (ServerCredentials (..), mkTransportServerConfig) -import Simplex.Messaging.Util (safeDecodeUtf8, tshow) +import Simplex.Messaging.Util (eitherToMaybe, safeDecodeUtf8, tshow) import System.Directory (createDirectoryIfMissing, doesFileExist) import System.FilePath (combine) import System.IO (BufferMode (..), hSetBuffering, stderr, stdout) @@ -89,6 +89,9 @@ xftpServerCLI cfgPath logPath = do <> "# Expire files after the specified number of hours.\n" <> ("expire_files_hours: " <> tshow defFileExpirationHours <> "\n\n") <> "log_stats: off\n\ + \\n\ + \# Log interval for real-time Prometheus metrics\n\ + \# prometheus_interval: 60\n\ \\n\ \[AUTH]\n\ \# Set new_files option to off to completely prohibit uploading new files.\n\ @@ -188,6 +191,8 @@ xftpServerCLI cfgPath logPath = do logStatsStartTime = 0, -- seconds from 00:00 UTC serverStatsLogFile = combine logPath "file-server-stats.daily.log", serverStatsBackupFile = logStats $> combine logPath "file-server-stats.log", + prometheusInterval = eitherToMaybe $ read . T.unpack <$> lookupValue "STORE_LOG" "prometheus_interval" ini, + prometheusMetricsFile = combine logPath "xftp-server-metrics.txt", transportConfig = mkTransportServerConfig (fromMaybe False $ iniOnOff "TRANSPORT" "log_tls_errors" ini) diff --git a/src/Simplex/FileTransfer/Server/Prometheus.hs b/src/Simplex/FileTransfer/Server/Prometheus.hs new file mode 100644 index 000000000..b540807aa --- /dev/null +++ b/src/Simplex/FileTransfer/Server/Prometheus.hs @@ -0,0 +1,134 @@ +{-# LANGUAGE DuplicateRecordFields #-} +{-# LANGUAGE NamedFieldPuns #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TypeApplications #-} +{-# OPTIONS_GHC -fno-warn-unrecognised-pragmas #-} + +module Simplex.FileTransfer.Server.Prometheus where + +import Data.Int (Int64) +import Data.Text (Text) +import qualified Data.Text as T +import Data.Time.Clock (UTCTime (..), diffUTCTime) +import Data.Time.Clock.System (systemEpochDay) +import Data.Time.Format.ISO8601 (iso8601Show) +import Simplex.FileTransfer.Server.Stats +import Simplex.Messaging.Server.Stats (PeriodStatCounts (..)) +import Simplex.Messaging.Transport (simplexMQVersion) +import Simplex.Messaging.Util (tshow) + +data FileServerMetrics = FileServerMetrics + { statsData :: FileServerStatsData, + filesDownloadedPeriods :: PeriodStatCounts, + rtsOptions :: Text + } + +rtsOptionsEnv :: Text +rtsOptionsEnv = "XFTP_RTS_OPTIONS" + +{-# FOURMOLU_DISABLE\n#-} +xftpPrometheusMetrics :: FileServerMetrics -> UTCTime -> Text +xftpPrometheusMetrics sm ts = + time <> files <> info + where + FileServerMetrics {statsData, filesDownloadedPeriods, rtsOptions} = sm + FileServerStatsData + { _fromTime, + _filesCreated, + _fileRecipients, + _filesUploaded, + _filesExpired, + _filesDeleted, + _filesBlocked, + _fileDownloads, + _fileDownloadAcks, + _filesCount, + _filesSize + } = statsData + time = + "# Recorded at: " <> T.pack (iso8601Show ts) <> "\n\ + \# Stats from: " <> T.pack (iso8601Show _fromTime) <> "\n\ + \\n" + files = + "# Files\n\ + \# -----\n\ + \\n\ + \# HELP simplex_xftp_files_created Created files\n\ + \# TYPE simplex_xftp_files_created counter\n\ + \simplex_xftp_files_created " <> mshow _filesCreated <> "\n\ + \# filesCreated\n\ + \\n\ + \# HELP simplex_xftp_files_recipients Files recipients\n\ + \# TYPE simplex_xftp_files_recipients counter\n\ + \simplex_xftp_files_recipients " <> mshow _fileRecipients <> "\n\ + \# fileRecipients\n\ + \\n\ + \# HELP simplex_xftp_files_uploaded Uploaded files\n\ + \# TYPE simplex_xftp_files_uploaded counter\n\ + \simplex_xftp_files_uploaded " <> mshow _filesUploaded <> "\n\ + \# filesUploaded\n\ + \\n\ + \# HELP simplex_xftp_files_expired Expired files\n\ + \# TYPE simplex_xftp_files_expired counter\n\ + \simplex_xftp_files_expired " <> mshow _filesExpired <> "\n\ + \# filesExpired\n\ + \\n\ + \# HELP simplex_xftp_files_deleted Deleted files\n\ + \# TYPE simplex_xftp_files_deleted counter\n\ + \simplex_xftp_files_deleted " <> mshow _filesDeleted <> "\n\ + \# filesDeleted\n\ + \\n\ + \# HELP simplex_xftp_files_blocked Blocked files\n\ + \# TYPE simplex_xftp_files_blocked counter\n\ + \simplex_xftp_files_blocked " <> mshow _filesBlocked <> "\n\ + \# filesBlocked\n\ + \\n\ + \# HELP simplex_xftp_file_downloads File downloads\n\ + \# TYPE simplex_xftp_file_downloads counter\n\ + \simplex_xftp_file_downloads " <> mshow _fileDownloads <> "\n\ + \# fileDownloads\n\ + \\n\ + \# HELP simplex_xftp_file_download_acks File download ACKs\n\ + \# TYPE simplex_xftp_file_download_acks counter\n\ + \simplex_xftp_file_download_acks " <> mshow _fileDownloadAcks <> "\n\ + \# fileDownloadAcks\n\ + \\n\ + \# HELP simplex_xftp_files_count_total Total files count \n\ + \# TYPE simplex_xftp_files_count_total gauge\n\ + \simplex_xftp_files_count_total " <> mshow _filesCount <> "\n\ + \# filesCount\n\ + \\n\ + \# HELP simplex_xftp_files_size Size of files \n\ + \# TYPE simplex_xftp_files_size gauge\n\ + \simplex_xftp_files_size " <> mshow _filesSize <> "\n\ + \# filesSize \n\ + \\n\ + \# HELP simplex_xftp_files_count_daily Daily files count\n\ + \# TYPE simplex_xftp_files_count_daily gauge\n\ + \simplex_xftp_files_count_daily " <> mstr (dayCount filesDownloadedPeriods) <> "\n\ + \# filesDownloaded.dayCount\n\ + \\n\ + \# HELP simplex_xftp_files_count_weekly Weekly files count\n\ + \# TYPE simplex_xftp_files_count_weekly gauge\n\ + \simplex_xftp_files_count_weekly " <> mstr (weekCount filesDownloadedPeriods) <> "\n\ + \# filesDownloaded.weekCount\n\ + \\n\ + \# HELP simplex_xftp_files_count_monthly Monthly files count\n\ + \# TYPE simplex_xftp_files_count_monthly gauge\n\ + \simplex_xftp_files_count_monthly " <> mstr (monthCount filesDownloadedPeriods) <> "\n\ + \# filesDownloaded.monthCount\n\ + \\n" + info = + "# Info\n\ + \# ----\n\ + \\n\ + \# HELP simplex_xftp_info Server information. RTS options have to be passed via " <> rtsOptionsEnv <> " env var\n\ + \# TYPE simplex_xftp_info gauge\n\ + \simplex_xftp_info{version=\"" <> T.pack simplexMQVersion <> "\",rts_options=\"" <> rtsOptions <> "\"} 1\n\ + \\n" + mstr a = a <> " " <> tsEpoch + mshow :: Show a => a -> Text + mshow = mstr . tshow + tsEpoch = tshow @Int64 $ floor @Double $ realToFrac (ts `diffUTCTime` epoch) * 1000 + epoch = UTCTime systemEpochDay 0 +{-# FOURMOLU_ENABLE\n#-} diff --git a/src/Simplex/Messaging/Notifications/Server/Main.hs b/src/Simplex/Messaging/Notifications/Server/Main.hs index e5b085a7c..f79ad12b7 100644 --- a/src/Simplex/Messaging/Notifications/Server/Main.hs +++ b/src/Simplex/Messaging/Notifications/Server/Main.hs @@ -167,6 +167,9 @@ ntfServerCLI cfgPath logPath = <> "Time to retain deleted entities in the database, days.\n" <> ("# db_deleted_ttl: " <> tshow defaultDeletedTTL <> "\n\n") <> "log_stats: off\n\n\ + \# Log interval for real-time Prometheus metrics\n\ + \# prometheus_interval: 60\n\ + \\n\ \[AUTH]\n\ \# control_port_admin_password:\n\ \# control_port_user_password:\n\ diff --git a/src/Simplex/Messaging/Server/Main/Init.hs b/src/Simplex/Messaging/Server/Main/Init.hs index 9f52d0797..2823df74f 100644 --- a/src/Simplex/Messaging/Server/Main/Init.hs +++ b/src/Simplex/Messaging/Server/Main/Init.hs @@ -92,7 +92,7 @@ iniFileContent cfgPath logPath opts host basicAuth controlPortPwds = <> "# Log daily server statistics to CSV file\n" <> ("log_stats: " <> onOff logStats <> "\n\n") <> "# Log interval for real-time Prometheus metrics\n\ - \# prometheus_interval: 300\n\n\ + \# prometheus_interval: 60\n\n\ \[AUTH]\n\ \# Set new_queues option to off to completely prohibit creating new messaging queues.\n\ \# This can be useful when you want to decommission the server, but not all connections are switched yet.\n\ diff --git a/tests/XFTPClient.hs b/tests/XFTPClient.hs index d3215ea95..0f7310073 100644 --- a/tests/XFTPClient.hs +++ b/tests/XFTPClient.hs @@ -99,6 +99,9 @@ testXFTPLogFile = "tests/tmp/xftp-server-store.log" testXFTPStatsBackupFile :: FilePath testXFTPStatsBackupFile = "tests/tmp/xftp-server-stats.log" +xftpTestPrometheusMetricsFile :: FilePath +xftpTestPrometheusMetricsFile = "tests/tmp/xftp-server-metrics.txt" + testXFTPServerConfig :: XFTPServerConfig testXFTPServerConfig = XFTPServerConfig @@ -127,6 +130,8 @@ testXFTPServerConfig = logStatsStartTime = 0, serverStatsLogFile = "tests/tmp/xftp-server-stats.daily.log", serverStatsBackupFile = Nothing, + prometheusInterval = Nothing, + prometheusMetricsFile = xftpTestPrometheusMetricsFile, transportConfig = mkTransportServerConfig True (Just alpnSupportedXFTPhandshakes) False, responseDelay = 0 }