From 508aa471a916c74ee94468afef85931e26c9267d Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 30 May 2022 15:58:30 +0530 Subject: [PATCH] Track participant join total + rate in node stats (#741) * Track participant join total + rate in node stats * update protocol --- go.mod | 2 +- go.sum | 4 +-- pkg/telemetry/prometheus/node.go | 7 ++++- pkg/telemetry/prometheus/packets.go | 30 ++++++++++++++----- .../telemetryserviceinternalevents.go | 2 ++ 5 files changed, 34 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 01a3fd59f..c8281d350 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/google/wire v0.5.0 github.com/gorilla/websocket v1.4.2 github.com/hashicorp/golang-lru v0.5.4 - github.com/livekit/protocol v0.13.3-0.20220526091938-2ecd4805e7e2 + github.com/livekit/protocol v0.13.3-0.20220530102138-51a8116f88b2 github.com/livekit/rtcscore-go v0.0.0-20220524203225-dfd1ba40744a github.com/mackerelio/go-osstat v0.2.1 github.com/magefile/mage v1.11.0 diff --git a/go.sum b/go.sum index ee74e14f4..66fd7a88d 100644 --- a/go.sum +++ b/go.sum @@ -133,8 +133,8 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lithammer/shortuuid/v3 v3.0.6 h1:pr15YQyvhiSX/qPxncFtqk+v4xLEpOZObbsY/mKrcvA= github.com/lithammer/shortuuid/v3 v3.0.6/go.mod h1:vMk8ke37EmiewwolSO1NLW8vP4ZaKlRuDIi8tWWmAts= -github.com/livekit/protocol v0.13.3-0.20220526091938-2ecd4805e7e2 h1:ufUi4uR5OgfnrweGTWXkplxudLeZw9/gtkgws2oszAo= -github.com/livekit/protocol v0.13.3-0.20220526091938-2ecd4805e7e2/go.mod h1:BLtSeVmn2rLP37xjzw7gHgaAmkWl3L/L9bPvgSbaOfo= +github.com/livekit/protocol v0.13.3-0.20220530102138-51a8116f88b2 h1:ioAZAmtoQnHELXBKOQr041ro8GwKcZ7lgNuDNbLG1vo= +github.com/livekit/protocol v0.13.3-0.20220530102138-51a8116f88b2/go.mod h1:BLtSeVmn2rLP37xjzw7gHgaAmkWl3L/L9bPvgSbaOfo= github.com/livekit/rtcscore-go v0.0.0-20220524203225-dfd1ba40744a h1:cENjhGfslLSDV07gt8ASy47Wd12Q0kBS7hsdunyQ62I= github.com/livekit/rtcscore-go v0.0.0-20220524203225-dfd1ba40744a/go.mod h1:116ych8UaEs9vfIE8n6iZCZ30iagUFTls0vRmC+Ix5U= github.com/mackerelio/go-osstat v0.2.1 h1:5AeAcBEutEErAOlDz6WCkEvm6AKYgHTUQrfwm5RbeQc= diff --git a/pkg/telemetry/prometheus/node.go b/pkg/telemetry/prometheus/node.go index d07b1d451..046336328 100644 --- a/pkg/telemetry/prometheus/node.go +++ b/pkg/telemetry/prometheus/node.go @@ -67,6 +67,7 @@ func GetUpdatedNodeStats(prev *livekit.NodeStats, prevAverage *livekit.NodeStats nackTotalNow := nackTotal.Load() retransmitBytesNow := retransmitBytes.Load() retransmitPacketsNow := retransmitPackets.Load() + participantJoinNow := participantJoin.Load() updatedAt := time.Now().Unix() elapsed := updatedAt - prevAverage.UpdatedAt @@ -77,7 +78,8 @@ func GetUpdatedNodeStats(prev *livekit.NodeStats, prevAverage *livekit.NodeStats packetsInNow != prevAverage.PacketsIn || packetsOutNow != prevAverage.PacketsOut || retransmitBytesNow != prevAverage.RetransmitBytesOut || - retransmitPacketsNow != prevAverage.RetransmitPacketsOut { + retransmitPacketsNow != prevAverage.RetransmitPacketsOut || + participantJoinNow != prevAverage.ParticipantJoin { computeAverage = true } @@ -95,6 +97,7 @@ func GetUpdatedNodeStats(prev *livekit.NodeStats, prevAverage *livekit.NodeStats RetransmitBytesOut: retransmitBytesNow, RetransmitPacketsOut: retransmitPacketsNow, NackTotal: nackTotalNow, + ParticipantJoin: participantJoinNow, BytesInPerSec: prevAverage.BytesInPerSec, BytesOutPerSec: prevAverage.BytesOutPerSec, PacketsInPerSec: prevAverage.PacketsInPerSec, @@ -102,6 +105,7 @@ func GetUpdatedNodeStats(prev *livekit.NodeStats, prevAverage *livekit.NodeStats RetransmitBytesOutPerSec: prevAverage.RetransmitBytesOutPerSec, RetransmitPacketsOutPerSec: prevAverage.RetransmitPacketsOutPerSec, NackPerSec: prevAverage.NackPerSec, + ParticipantJoinPerSec: prevAverage.ParticipantJoinPerSec, NumCpus: numCPUs, CpuLoad: cpuLoad, LoadAvgLast1Min: float32(loadAvg.Loadavg1), @@ -118,6 +122,7 @@ func GetUpdatedNodeStats(prev *livekit.NodeStats, prevAverage *livekit.NodeStats stats.RetransmitBytesOutPerSec = perSec(prevAverage.RetransmitBytesOut, retransmitBytesNow, elapsed) stats.RetransmitPacketsOutPerSec = perSec(prevAverage.RetransmitPacketsOut, retransmitPacketsNow, elapsed) stats.NackPerSec = perSec(prevAverage.NackTotal, nackTotalNow, elapsed) + stats.ParticipantJoinPerSec = perSec(prevAverage.ParticipantJoin, participantJoinNow, elapsed) } return stats, computeAverage, nil diff --git a/pkg/telemetry/prometheus/packets.go b/pkg/telemetry/prometheus/packets.go index 325a3bbfd..d939c5bb8 100644 --- a/pkg/telemetry/prometheus/packets.go +++ b/pkg/telemetry/prometheus/packets.go @@ -22,14 +22,16 @@ var ( nackTotal atomic.Uint64 retransmitBytes atomic.Uint64 retransmitPackets atomic.Uint64 + participantJoin atomic.Uint64 - promPacketLabels = []string{"direction", "transmission"} - promPacketTotal *prometheus.CounterVec - promPacketBytes *prometheus.CounterVec - promRTCPLabels = []string{"direction"} - promNackTotal *prometheus.CounterVec - promPliTotal *prometheus.CounterVec - promFirTotal *prometheus.CounterVec + promPacketLabels = []string{"direction", "transmission"} + promPacketTotal *prometheus.CounterVec + promPacketBytes *prometheus.CounterVec + promRTCPLabels = []string{"direction"} + promNackTotal *prometheus.CounterVec + promPliTotal *prometheus.CounterVec + promFirTotal *prometheus.CounterVec + promParticipantJoin *prometheus.CounterVec ) func initPacketStats(nodeID string) { @@ -63,12 +65,19 @@ func initPacketStats(nodeID string) { Name: "total", ConstLabels: prometheus.Labels{"node_id": nodeID}, }, promRTCPLabels) + promParticipantJoin = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: livekitNamespace, + Subsystem: "participant_join", + Name: "total", + ConstLabels: prometheus.Labels{"node_id": nodeID}, + }, nil) prometheus.MustRegister(promPacketTotal) prometheus.MustRegister(promPacketBytes) prometheus.MustRegister(promNackTotal) prometheus.MustRegister(promPliTotal) prometheus.MustRegister(promFirTotal) + prometheus.MustRegister(promParticipantJoin) } func IncrementPackets(direction Direction, count uint64, retransmit bool) { @@ -114,6 +123,13 @@ func IncrementRTCP(direction Direction, nack, pli, fir uint32) { } } +func IncrementParticipantJoin(join uint32) { + if join > 0 { + promParticipantJoin.WithLabelValues().Add(float64(join)) + participantJoin.Add(uint64(join)) + } +} + func transmissionLabel(retransmit bool) string { if !retransmit { return transmissionInitial diff --git a/pkg/telemetry/telemetryserviceinternalevents.go b/pkg/telemetry/telemetryserviceinternalevents.go index f5405fe1a..9f65b95ff 100644 --- a/pkg/telemetry/telemetryserviceinternalevents.go +++ b/pkg/telemetry/telemetryserviceinternalevents.go @@ -52,6 +52,8 @@ func (t *telemetryServiceInternal) ParticipantJoined( clientInfo *livekit.ClientInfo, clientMeta *livekit.AnalyticsClientMeta, ) { + prometheus.IncrementParticipantJoin(1) + newWorker := newStatsWorker( ctx, t,