diff --git a/cmd/server/main.go b/cmd/server/main.go index 88104dec0..bb6fcae83 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -239,7 +239,7 @@ func startServer(c *cli.Context) error { return err } - prometheus.Init(currentNode.Id) + prometheus.Init(currentNode.Id, currentNode.Type) server, err := service.InitializeServer(conf, currentNode) if err != nil { diff --git a/pkg/rtc/helper_test.go b/pkg/rtc/helper_test.go index 4ddbf14a2..b152341f1 100644 --- a/pkg/rtc/helper_test.go +++ b/pkg/rtc/helper_test.go @@ -10,7 +10,7 @@ import ( ) func init() { - prometheus.Init("test") + prometheus.Init("test", livekit.NodeType_SERVER) } func newMockParticipant(identity livekit.ParticipantIdentity, protocol types.ProtocolVersion, hidden bool, publisher bool) *typesfakes.FakeLocalParticipant { diff --git a/pkg/telemetry/prometheus/node.go b/pkg/telemetry/prometheus/node.go index 78e2e67cb..aef4155ff 100644 --- a/pkg/telemetry/prometheus/node.go +++ b/pkg/telemetry/prometheus/node.go @@ -28,7 +28,7 @@ var ( promSysDroppedPacketPctGauge prometheus.Gauge ) -func Init(nodeID string) { +func Init(nodeID string, nodeType livekit.NodeType) { if initialized.Swap(true) { return } @@ -38,7 +38,7 @@ func Init(nodeID string) { Namespace: livekitNamespace, Subsystem: "node", Name: "messages", - ConstLabels: prometheus.Labels{"node_id": nodeID}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, []string{"type", "status"}, ) @@ -48,7 +48,7 @@ func Init(nodeID string) { Namespace: livekitNamespace, Subsystem: "node", Name: "service_operation", - ConstLabels: prometheus.Labels{"node_id": nodeID}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, []string{"type", "status", "error_type"}, ) @@ -58,7 +58,7 @@ func Init(nodeID string) { Namespace: livekitNamespace, Subsystem: "node", Name: "packet_total", - ConstLabels: prometheus.Labels{"node_id": nodeID}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, Help: "System level packet count. Count starts at 0 when service is first started.", }, []string{"type"}, @@ -69,7 +69,7 @@ func Init(nodeID string) { Namespace: livekitNamespace, Subsystem: "node", Name: "dropped_packets", - ConstLabels: prometheus.Labels{"node_id": nodeID}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, Help: "System level dropped outgoing packet percentage.", }, ) @@ -81,8 +81,8 @@ func Init(nodeID string) { sysPacketsStart, sysDroppedPacketsStart, _ = getTCStats() - initPacketStats(nodeID) - initRoomStats(nodeID) + initPacketStats(nodeID, nodeType) + initRoomStats(nodeID, nodeType) } func getMemoryStats() (memoryLoad float32, err error) { diff --git a/pkg/telemetry/prometheus/packets.go b/pkg/telemetry/prometheus/packets.go index 3097e917f..b195950b8 100644 --- a/pkg/telemetry/prometheus/packets.go +++ b/pkg/telemetry/prometheus/packets.go @@ -3,6 +3,8 @@ package prometheus import ( "github.com/prometheus/client_golang/prometheus" "go.uber.org/atomic" + + "github.com/livekit/protocol/livekit" ) type Direction string @@ -35,48 +37,48 @@ var ( promConnections *prometheus.GaugeVec ) -func initPacketStats(nodeID string) { +func initPacketStats(nodeID string, nodeType livekit.NodeType) { promPacketTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "packet", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, promPacketLabels) promPacketBytes = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "packet", Name: "bytes", - ConstLabels: prometheus.Labels{"node_id": nodeID}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, promPacketLabels) promNackTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "nack", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, promRTCPLabels) promPliTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "pli", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, promRTCPLabels) promFirTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "fir", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, promRTCPLabels) promParticipantJoin = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "participant_join", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, nil) promConnections = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: livekitNamespace, Subsystem: "connection", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, []string{"kind"}) prometheus.MustRegister(promPacketTotal) diff --git a/pkg/telemetry/prometheus/rooms.go b/pkg/telemetry/prometheus/rooms.go index b41066f3c..2bbca6476 100644 --- a/pkg/telemetry/prometheus/rooms.go +++ b/pkg/telemetry/prometheus/rooms.go @@ -5,6 +5,8 @@ import ( "github.com/prometheus/client_golang/prometheus" "go.uber.org/atomic" + + "github.com/livekit/protocol/livekit" ) var ( @@ -20,18 +22,18 @@ var ( promTrackSubscribedTotal *prometheus.GaugeVec ) -func initRoomStats(nodeID string) { +func initRoomStats(nodeID string, nodeType livekit.NodeType) { promRoomTotal = prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: livekitNamespace, Subsystem: "room", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }) promRoomDuration = prometheus.NewHistogram(prometheus.HistogramOpts{ Namespace: livekitNamespace, Subsystem: "room", Name: "duration_seconds", - ConstLabels: prometheus.Labels{"node_id": nodeID}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, Buckets: []float64{ 5, 10, 60, 5 * 60, 10 * 60, 30 * 60, 60 * 60, 2 * 60 * 60, 5 * 60 * 60, 10 * 60 * 60, }, @@ -40,19 +42,19 @@ func initRoomStats(nodeID string) { Namespace: livekitNamespace, Subsystem: "participant", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }) promTrackPublishedTotal = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: livekitNamespace, Subsystem: "track", Name: "published_total", - ConstLabels: prometheus.Labels{"node_id": nodeID}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, []string{"kind"}) promTrackSubscribedTotal = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: livekitNamespace, Subsystem: "track", Name: "subscribed_total", - ConstLabels: prometheus.Labels{"node_id": nodeID}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, []string{"kind"}) prometheus.MustRegister(promRoomTotal) diff --git a/pkg/telemetry/stats_test.go b/pkg/telemetry/stats_test.go index 65029c429..86e6b60fb 100644 --- a/pkg/telemetry/stats_test.go +++ b/pkg/telemetry/stats_test.go @@ -15,7 +15,7 @@ import ( ) func init() { - prometheus.Init("test") + prometheus.Init("test", livekit.NodeType_SERVER) } type telemetryServiceFixture struct { diff --git a/test/integration_helpers.go b/test/integration_helpers.go index aff574b0d..5f059e9d5 100644 --- a/test/integration_helpers.go +++ b/test/integration_helpers.go @@ -47,7 +47,7 @@ func init() { Config: logger.Config{Level: "debug"}, }) - prometheus.Init("test") + prometheus.Init("test", livekit.NodeType_SERVER) } func setupSingleNodeTest(name string) (*service.LivekitServer, func()) {