From 1848a21eda741f5ba9f5054ae3931cd4eeab1e2d Mon Sep 17 00:00:00 2001 From: Dan McFaul <55854809+real-danm@users.noreply.github.com> Date: Wed, 15 Feb 2023 14:41:44 -0700 Subject: [PATCH] add configurable environment value (#1421) * add configurable prometheus env label * Update pkg/config/config.go Co-authored-by: Mathew Kamkar <578302+matkam@users.noreply.github.com> * Update cmd/server/main.go Co-authored-by: Mathew Kamkar <578302+matkam@users.noreply.github.com> * Update config-sample.yaml Co-authored-by: Mathew Kamkar <578302+matkam@users.noreply.github.com> * set config.Environment value to dev when in dev mode * be more precise for config-sample --------- Co-authored-by: Mathew Kamkar <578302+matkam@users.noreply.github.com> --- cmd/server/main.go | 2 +- config-sample.yaml | 2 ++ pkg/config/config.go | 5 +++++ pkg/rtc/helper_test.go | 2 +- pkg/telemetry/prometheus/node.go | 16 ++++++++-------- pkg/telemetry/prometheus/packets.go | 24 ++++++++++++------------ pkg/telemetry/prometheus/rooms.go | 16 ++++++++-------- pkg/telemetry/stats_test.go | 2 +- test/integration_helpers.go | 2 +- 9 files changed, 39 insertions(+), 32 deletions(-) diff --git a/cmd/server/main.go b/cmd/server/main.go index 695fd38d0..994e71b34 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -250,7 +250,7 @@ func startServer(c *cli.Context) error { return err } - prometheus.Init(currentNode.Id, currentNode.Type) + prometheus.Init(currentNode.Id, currentNode.Type, conf.Environment) server, err := service.InitializeServer(conf, currentNode) if err != nil { diff --git a/config-sample.yaml b/config-sample.yaml index 2dddb1238..c8340341c 100644 --- a/config-sample.yaml +++ b/config-sample.yaml @@ -110,6 +110,8 @@ rtc: # when enabled, LiveKit will expose prometheus metrics on :6789/metrics # prometheus_port: 6789 +# set a custom environment variable. prometheus metrics will be labeled with this value. defaults to an empty string +# environment: custom-value # API key / secret pairs. # Keys are used for JWT authentication, server APIs would require a keypair in order to generate access tokens diff --git a/pkg/config/config.go b/pkg/config/config.go index 699ee630b..e6bbae9e1 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -46,6 +46,7 @@ type Config struct { Port uint32 `yaml:"port"` BindAddresses []string `yaml:"bind_addresses"` PrometheusPort uint32 `yaml:"prometheus_port,omitempty"` + Environment string `yaml:"environment,omitempty"` RTC RTCConfig `yaml:"rtc,omitempty"` Redis redisLiveKit.RedisConfig `yaml:"redis,omitempty"` Audio AudioConfig `yaml:"audio,omitempty"` @@ -458,6 +459,10 @@ func NewConfig(confString string, strictMode bool, c *cli.Context, baseFlags []c conf.Logging.Level = "debug" } + if conf.Development { + conf.Environment = "dev" + } + return conf, nil } diff --git a/pkg/rtc/helper_test.go b/pkg/rtc/helper_test.go index b152341f1..dff552233 100644 --- a/pkg/rtc/helper_test.go +++ b/pkg/rtc/helper_test.go @@ -10,7 +10,7 @@ import ( ) func init() { - prometheus.Init("test", livekit.NodeType_SERVER) + prometheus.Init("test", livekit.NodeType_SERVER, "test") } func newMockParticipant(identity livekit.ParticipantIdentity, protocol types.ProtocolVersion, hidden bool, publisher bool) *typesfakes.FakeLocalParticipant { diff --git a/pkg/telemetry/prometheus/node.go b/pkg/telemetry/prometheus/node.go index 6259f6b00..9393052c1 100644 --- a/pkg/telemetry/prometheus/node.go +++ b/pkg/telemetry/prometheus/node.go @@ -29,7 +29,7 @@ var ( promSysDroppedPacketPctGauge prometheus.Gauge ) -func Init(nodeID string, nodeType livekit.NodeType) { +func Init(nodeID string, nodeType livekit.NodeType, env string) { if initialized.Swap(true) { return } @@ -39,7 +39,7 @@ func Init(nodeID string, nodeType livekit.NodeType) { Namespace: livekitNamespace, Subsystem: "node", Name: "messages", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, }, []string{"type", "status"}, ) @@ -49,7 +49,7 @@ func Init(nodeID string, nodeType livekit.NodeType) { Namespace: livekitNamespace, Subsystem: "node", Name: "service_operation", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, }, []string{"type", "status", "error_type"}, ) @@ -59,7 +59,7 @@ func Init(nodeID string, nodeType livekit.NodeType) { Namespace: livekitNamespace, Subsystem: "node", Name: "twirp_request_status", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, }, []string{"service", "method", "status", "code"}, ) @@ -69,7 +69,7 @@ func Init(nodeID string, nodeType livekit.NodeType) { Namespace: livekitNamespace, Subsystem: "node", Name: "packet_total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, Help: "System level packet count. Count starts at 0 when service is first started.", }, []string{"type"}, @@ -80,7 +80,7 @@ func Init(nodeID string, nodeType livekit.NodeType) { Namespace: livekitNamespace, Subsystem: "node", Name: "dropped_packets", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, Help: "System level dropped outgoing packet percentage.", }, ) @@ -93,8 +93,8 @@ func Init(nodeID string, nodeType livekit.NodeType) { sysPacketsStart, sysDroppedPacketsStart, _ = getTCStats() - initPacketStats(nodeID, nodeType) - initRoomStats(nodeID, nodeType) + initPacketStats(nodeID, nodeType, env) + initRoomStats(nodeID, nodeType, env) } func GetUpdatedNodeStats(prev *livekit.NodeStats, prevAverage *livekit.NodeStats) (*livekit.NodeStats, bool, error) { diff --git a/pkg/telemetry/prometheus/packets.go b/pkg/telemetry/prometheus/packets.go index ad7bcfd82..09525053c 100644 --- a/pkg/telemetry/prometheus/packets.go +++ b/pkg/telemetry/prometheus/packets.go @@ -44,75 +44,75 @@ var ( promConnections *prometheus.GaugeVec ) -func initPacketStats(nodeID string, nodeType livekit.NodeType) { +func initPacketStats(nodeID string, nodeType livekit.NodeType, env string) { promPacketTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "packet", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, }, promPacketLabels) promPacketBytes = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "packet", Name: "bytes", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, }, promPacketLabels) promNackTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "nack", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, }, promRTCPLabels) promPliTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "pli", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, }, promRTCPLabels) promFirTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "fir", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, }, promRTCPLabels) promPacketLossTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "packet_loss", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, }, promStreamLabels) promPacketLoss = prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: livekitNamespace, Subsystem: "packet_loss", Name: "percent", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, Buckets: []float64{0.0, 0.1, 0.3, 0.5, 0.7, 1, 5, 10, 40, 100}, }, promStreamLabels) promJitter = prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: livekitNamespace, Subsystem: "jitter", Name: "us", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, Buckets: []float64{100, 500, 1500, 3000, 6000, 12000, 24000, 48000, 96000, 192000}, }, promStreamLabels) promRTT = prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: livekitNamespace, Subsystem: "rtt", Name: "ms", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, Buckets: []float64{50, 100, 150, 200, 250, 500, 750, 1000, 5000, 10000}, }, promStreamLabels) promParticipantJoin = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "participant_join", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, }, []string{"state"}) promConnections = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: livekitNamespace, Subsystem: "connection", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, }, []string{"kind"}) prometheus.MustRegister(promPacketTotal) diff --git a/pkg/telemetry/prometheus/rooms.go b/pkg/telemetry/prometheus/rooms.go index d8e1368f5..d50ce2d15 100644 --- a/pkg/telemetry/prometheus/rooms.go +++ b/pkg/telemetry/prometheus/rooms.go @@ -31,18 +31,18 @@ var ( promTrackSubscribeCounter *prometheus.CounterVec ) -func initRoomStats(nodeID string, nodeType livekit.NodeType) { +func initRoomStats(nodeID string, nodeType livekit.NodeType, env string) { promRoomCurrent = prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: livekitNamespace, Subsystem: "room", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, }) promRoomDuration = prometheus.NewHistogram(prometheus.HistogramOpts{ Namespace: livekitNamespace, Subsystem: "room", Name: "duration_seconds", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, Buckets: []float64{ 5, 10, 60, 5 * 60, 10 * 60, 30 * 60, 60 * 60, 2 * 60 * 60, 5 * 60 * 60, 10 * 60 * 60, }, @@ -51,31 +51,31 @@ func initRoomStats(nodeID string, nodeType livekit.NodeType) { Namespace: livekitNamespace, Subsystem: "participant", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, }) promTrackPublishedCurrent = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: livekitNamespace, Subsystem: "track", Name: "published_total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, }, []string{"kind"}) promTrackSubscribedCurrent = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: livekitNamespace, Subsystem: "track", Name: "subscribed_total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, }, []string{"kind"}) promTrackPublishCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "track", Name: "publish_counter", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, }, []string{"kind", "state"}) promTrackSubscribeCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "track", Name: "subscribe_counter", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, }, []string{"state", "error"}) prometheus.MustRegister(promRoomCurrent) diff --git a/pkg/telemetry/stats_test.go b/pkg/telemetry/stats_test.go index f8cbe280c..5dd8b82c3 100644 --- a/pkg/telemetry/stats_test.go +++ b/pkg/telemetry/stats_test.go @@ -15,7 +15,7 @@ import ( ) func init() { - prometheus.Init("test", livekit.NodeType_SERVER) + prometheus.Init("test", livekit.NodeType_SERVER, "test") } type telemetryServiceFixture struct { diff --git a/test/integration_helpers.go b/test/integration_helpers.go index 9a997c025..ab2f10a23 100644 --- a/test/integration_helpers.go +++ b/test/integration_helpers.go @@ -46,7 +46,7 @@ func init() { Config: logger.Config{Level: "debug"}, }) - prometheus.Init("test", livekit.NodeType_SERVER) + prometheus.Init("test", livekit.NodeType_SERVER, "test") } func setupSingleNodeTest(name string) (*service.LivekitServer, func()) {