From 7f10e18bac9ce1ff95cb8e8789205542bbf11425 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 25 Nov 2025 14:06:02 +0530 Subject: [PATCH] Record join/publish/subscribe cancellations. (#4102) To get better picture of success/failure rate. --- go.mod | 16 ++++++------- go.sum | 36 ++++++++++++++++------------- pkg/rtc/participant.go | 12 ++++++++-- pkg/rtc/subscriptionmanager.go | 19 +++++++++++++++ pkg/service/roommanager.go | 2 ++ pkg/telemetry/events.go | 4 ++-- pkg/telemetry/prometheus/node.go | 6 +++++ pkg/telemetry/prometheus/packets.go | 8 +++++++ pkg/telemetry/prometheus/rooms.go | 17 ++++++++++++-- 9 files changed, 90 insertions(+), 30 deletions(-) diff --git a/go.mod b/go.mod index 627c22ab6..84180660a 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731 github.com/livekit/mediatransportutil v0.0.0-20250922175932-f537f0880397 - github.com/livekit/protocol v1.43.1-0.20251118083515-f5993a2691d1 + github.com/livekit/protocol v1.43.3-0.20251125082437-3bc7a74d3d5e github.com/livekit/psrpc v0.7.1 github.com/mackerelio/go-osstat v0.2.6 github.com/magefile/mage v1.15.0 @@ -44,7 +44,7 @@ require ( github.com/pion/webrtc/v4 v4.1.6 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.23.0 - github.com/redis/go-redis/v9 v9.16.0 + github.com/redis/go-redis/v9 v9.17.0 github.com/rs/cors v1.11.1 github.com/stretchr/testify v1.11.1 github.com/thoas/go-funk v0.9.3 @@ -54,7 +54,7 @@ require ( github.com/urfave/negroni/v3 v3.1.1 go.uber.org/atomic v1.11.0 go.uber.org/multierr v1.11.0 - go.uber.org/zap v1.27.0 + go.uber.org/zap v1.27.1 golang.org/x/exp v0.0.0-20251113190631-e25ba8c21ef6 golang.org/x/mod v0.30.0 golang.org/x/sync v0.18.0 @@ -75,7 +75,7 @@ require ( require ( buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.10-20250912141014-52f32327d4b0.1 // indirect - buf.build/go/protovalidate v1.0.0 // indirect + buf.build/go/protovalidate v1.0.1 // indirect buf.build/go/protoyaml v0.6.0 // indirect cel.dev/expr v0.25.1 // indirect dario.cat/mergo v1.0.0 // indirect @@ -116,7 +116,7 @@ require ( github.com/moby/term v0.5.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/nats-io/nats.go v1.47.0 // indirect - github.com/nats-io/nkeys v0.4.11 // indirect + github.com/nats-io/nkeys v0.4.12 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect @@ -140,13 +140,13 @@ require ( github.com/xeipuuv/gojsonschema v1.2.0 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect go.uber.org/zap/exp v0.3.0 // indirect - golang.org/x/crypto v0.44.0 // indirect + golang.org/x/crypto v0.45.0 // indirect golang.org/x/net v0.47.0 // indirect golang.org/x/sys v0.38.0 // indirect golang.org/x/text v0.31.0 // indirect golang.org/x/tools v0.39.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20251111163417-95abcf5c77ba // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20251124214823-79d6a2a48846 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251124214823-79d6a2a48846 // indirect google.golang.org/grpc v1.77.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 5f9f73b7e..b8dff1c4c 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.10-20250912141014-52f32327d4b0.1 h1:31on4W/yPcV4nZHL4+UCiCvLPsMqe/vJcNg8Rci0scc= buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.10-20250912141014-52f32327d4b0.1/go.mod h1:fUl8CEN/6ZAMk6bP8ahBJPUJw7rbp+j4x+wCcYi2IG4= -buf.build/go/protovalidate v1.0.0 h1:IAG1etULddAy93fiBsFVhpj7es5zL53AfB/79CVGtyY= -buf.build/go/protovalidate v1.0.0/go.mod h1:KQmEUrcQuC99hAw+juzOEAmILScQiKBP1Oc36vvCLW8= +buf.build/go/protovalidate v1.0.1 h1:Fwmf08OOUuKVeMvEnDmcKxQam4PJc/zFgvVX64BhTms= +buf.build/go/protovalidate v1.0.1/go.mod h1:SoZmvk/3ZzOVg9YSkTdm4grMAByjf8zgZq4ZNaLZXoQ= buf.build/go/protoyaml v0.6.0 h1:Nzz1lvcXF8YgNZXk+voPPwdU8FjDPTUV4ndNTXN0n2w= buf.build/go/protoyaml v0.6.0/go.mod h1:RgUOsBu/GYKLDSIRgQXniXbNgFlGEZnQpRAUdLAFV2Q= cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= @@ -24,6 +24,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bep/debounce v1.2.1 h1:v67fRdBA9UQu2NhLFXrSg0Brw7CexQekrBwDMM8bzeY= github.com/bep/debounce v1.2.1/go.mod h1:H8yggRPQKLUhUoqrJC1bO2xNya7vanpDl7xR3ISbCJ0= +github.com/brianvoe/gofakeit/v6 v6.28.0 h1:Xib46XXuQfmlLS2EXRuJpqcw8St6qSZz75OUo0tgAW4= +github.com/brianvoe/gofakeit/v6 v6.28.0/go.mod h1:Xj58BMSnFqcn/fAQeSK+/PLtC5kSb7FJIq4JyGa8vEs= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= @@ -171,8 +173,8 @@ github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731 h1:9x+U2HGLrSw5AT github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20250922175932-f537f0880397 h1:Z7j2mY+bvG05UC80MpnJkitlJju8sSDWsr0Bb4dPceo= github.com/livekit/mediatransportutil v0.0.0-20250922175932-f537f0880397/go.mod h1:mSNtYzSf6iY9xM3UX42VEI+STHvMgHmrYzEHPcdhB8A= -github.com/livekit/protocol v1.43.1-0.20251118083515-f5993a2691d1 h1:4JP0s3dtkwIKNwT4K9rFsrhIuVrCKkXAbuZJtfceH5E= -github.com/livekit/protocol v1.43.1-0.20251118083515-f5993a2691d1/go.mod h1:yjkL2/HcaCRyHykP9rLgKST2099AGd8laaU8EuHMnfw= +github.com/livekit/protocol v1.43.3-0.20251125082437-3bc7a74d3d5e h1:Fsdj1LoPI5LrzGwOxTGfP7SbGsK+/4SgFCbt83NAiws= +github.com/livekit/protocol v1.43.3-0.20251125082437-3bc7a74d3d5e/go.mod h1:yjkL2/HcaCRyHykP9rLgKST2099AGd8laaU8EuHMnfw= github.com/livekit/psrpc v0.7.1 h1:ms37az0QTD3UXIWuUC5D/SkmKOlRMVRsI261eBWu/Vw= github.com/livekit/psrpc v0.7.1/go.mod h1:bZ4iHFQptTkbPnB0LasvRNu/OBYXEu1NA6O5BMFo9kk= github.com/mackerelio/go-osstat v0.2.6 h1:gs4U8BZeS1tjrL08tt5VUliVvSWP26Ai2Ob8Lr7f2i0= @@ -218,8 +220,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/nats-io/nats.go v1.47.0 h1:YQdADw6J/UfGUd2Oy6tn4Hq6YHxCaJrVKayxxFqYrgM= github.com/nats-io/nats.go v1.47.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g= -github.com/nats-io/nkeys v0.4.11 h1:q44qGV008kYd9W1b1nEBkNzvnWxtRSQ7A8BoqRrcfa0= -github.com/nats-io/nkeys v0.4.11/go.mod h1:szDimtgmfOi9n25JpfIdGw12tZFYXqhGxjhVxsatHVE= +github.com/nats-io/nkeys v0.4.12 h1:nssm7JKOG9/x4J8II47VWCL1Ds29avyiQDRn0ckMvDc= +github.com/nats-io/nkeys v0.4.12/go.mod h1:MT59A1HYcjIcyQDJStTfaOY6vhy9XTUjOFo+SVsvpBg= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/nyaruka/phonenumbers v1.6.5 h1:aBCaUhfpRA7hU6fsXk+p7KF1aNx4nQlq9hGeo2qdFg8= @@ -282,8 +284,10 @@ github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzM github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/puzpuzpuz/xsync/v3 v3.5.1 h1:GJYJZwO6IdxN/IKbneznS6yPkVC+c3zyY/j19c++5Fg= github.com/puzpuzpuz/xsync/v3 v3.5.1/go.mod h1:VjzYrABPabuM4KyBh1Ftq6u8nhwY5tBPKP9jpmh0nnA= -github.com/redis/go-redis/v9 v9.16.0 h1:OotgqgLSRCmzfqChbQyG1PHC3tLNR89DG4jdOERSEP4= -github.com/redis/go-redis/v9 v9.16.0/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370= +github.com/redis/go-redis/v9 v9.17.0 h1:K6E+ZlYN95KSMmZeEQPbU/c++wfmEvfFB17yEAq/VhM= +github.com/redis/go-redis/v9 v9.17.0/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370= +github.com/rodaine/protogofakeit v0.1.1 h1:ZKouljuRM3A+TArppfBqnH8tGZHOwM/pjvtXe9DaXH8= +github.com/rodaine/protogofakeit v0.1.1/go.mod h1:pXn/AstBYMaSfc1/RqH3N82pBuxtWgejz1AlYpY1mI0= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= @@ -352,8 +356,8 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= -go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= +go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.uber.org/zap/exp v0.3.0 h1:6JYzdifzYkGmTdRR59oYH+Ng7k49H9qVpWwNSsGJj3U= go.uber.org/zap/exp v0.3.0/go.mod h1:5I384qq7XGxYyByIhHm6jg5CHkGY0nsTfbDLgDDlgJQ= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= @@ -365,8 +369,8 @@ golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632 golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8= -golang.org/x/crypto v0.44.0 h1:A97SsFvM3AIwEEmTBiaxPPTYpDC47w720rdiiUvgoAU= -golang.org/x/crypto v0.44.0/go.mod h1:013i+Nw79BMiQiMsOPcVCB5ZIJbYkerPrGnOa00tvmc= +golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= +golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= golang.org/x/exp v0.0.0-20251113190631-e25ba8c21ef6 h1:zfMcR1Cs4KNuomFFgGefv5N0czO2XZpUbxGUy8i8ug0= golang.org/x/exp v0.0.0-20251113190631-e25ba8c21ef6/go.mod h1:46edojNIoXTNOhySWIWdix628clX9ODXwPsQuG6hsK0= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= @@ -466,10 +470,10 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/genproto/googleapis/api v0.0.0-20251111163417-95abcf5c77ba h1:B14OtaXuMaCQsl2deSvNkyPKIzq3BjfxQp8d00QyWx4= -google.golang.org/genproto/googleapis/api v0.0.0-20251111163417-95abcf5c77ba/go.mod h1:G5IanEx8/PgI9w6CFcYQf7jMtHQhZruvfM1i3qOqk5U= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba h1:UKgtfRM7Yh93Sya0Fo8ZzhDP4qBckrrxEr2oF5UIVb8= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= +google.golang.org/genproto/googleapis/api v0.0.0-20251124214823-79d6a2a48846 h1:ZdyUkS9po3H7G0tuh955QVyyotWvOD4W0aEapeGeUYk= +google.golang.org/genproto/googleapis/api v0.0.0-20251124214823-79d6a2a48846/go.mod h1:Fk4kyraUvqD7i5H6S43sj2W98fbZa75lpZz/eUyhfO0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251124214823-79d6a2a48846 h1:Wgl1rcDNThT+Zn47YyCXOXyX/COgMTIdhJ717F0l4xk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251124214823-79d6a2a48846/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= google.golang.org/grpc v1.77.0 h1:wVVY6/8cGA6vvffn+wWK5ToddbgdU3d8MNENr4evgXM= google.golang.org/grpc v1.77.0/go.mod h1:z0BY1iVj0q8E1uSQCjL9cppRj+gnZjzDnzV0dHhrNig= google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 3b423da20..2aa199edb 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -1525,6 +1525,12 @@ func (p *ParticipantImpl) Close(sendLeave bool, reason types.ParticipantCloseRea } p.pendingTracksLock.Lock() + for _, pti := range p.pendingTracks { + if len(pti.trackInfos) == 0 { + continue + } + prometheus.RecordTrackPublishCancels(pti.trackInfos[0].Type.String(), int32(len(pti.trackInfos))) + } p.pendingTracks = make(map[string]*pendingTrackInfo) p.pendingPublishingTracks = make(map[livekit.TrackID]*pendingTrackInfo) p.pendingTracksLock.Unlock() @@ -3006,7 +3012,8 @@ func (p *ParticipantImpl) addPendingTrackLocked(req *livekit.AddTrackRequest) *l mimeType := codec.Codec videoLayerMode := codec.VideoLayerMode - if req.Type == livekit.TrackType_VIDEO { + switch req.Type { + case livekit.TrackType_VIDEO: if !mime.IsMimeTypeStringVideo(mimeType) { mimeType = mime.MimeTypePrefixVideo + mimeType } @@ -3032,7 +3039,8 @@ func (p *ParticipantImpl) addPendingTrackLocked(req *livekit.AddTrackRequest) *l } } } - } else if req.Type == livekit.TrackType_AUDIO { + + case livekit.TrackType_AUDIO: if !mime.IsMimeTypeStringAudio(mimeType) { mimeType = mime.MimeTypePrefixAudio + mimeType } diff --git a/pkg/rtc/subscriptionmanager.go b/pkg/rtc/subscriptionmanager.go index b75b5cce8..a81e0a909 100644 --- a/pkg/rtc/subscriptionmanager.go +++ b/pkg/rtc/subscriptionmanager.go @@ -120,6 +120,8 @@ func (m *SubscriptionManager) Close(isExpectedToResume bool) { <-m.doneCh done.Store(true) + prometheus.RecordTrackSubscribeCancels(int32(m.getNumCancellations())) + subTracks := m.GetSubscribedTracks() downTracksToClose := make([]*sfu.DownTrack, 0, len(subTracks)) for _, st := range subTracks { @@ -891,6 +893,19 @@ func (m *SubscriptionManager) handleSubscribedTrackClose(s *trackSubscription, i } } +func (m *SubscriptionManager) getNumCancellations() int { + m.lock.RLock() + defer m.lock.RUnlock() + + numCancellations := 0 + for _, sub := range m.subscriptions { + if sub.isCanceled() { + numCancellations++ + } + } + return numCancellations +} + // -------------------------------------------------------------------------------------- type trackSubscription struct { @@ -1173,6 +1188,10 @@ func (s *trackSubscription) maybeRecordSuccess(ts telemetry.TelemetryService, pI ts.TrackSubscribed(context.Background(), pID, mediaTrack.ToProto(), pi, !eventSent) } +func (s *trackSubscription) isCanceled() bool { + return !s.eventSent.Load() && s.isDesired() +} + func (s *trackSubscription) durationSinceStart() time.Duration { t := s.subStartedAt.Load() if t == nil { diff --git a/pkg/service/roommanager.go b/pkg/service/roommanager.go index 0d7e67545..97e660af8 100644 --- a/pkg/service/roommanager.go +++ b/pkg/service/roommanager.go @@ -338,6 +338,7 @@ func (r *RoomManager) StartSession( Leave: leave, }, }) + prometheus.IncrementParticipantRtcCanceled(1) return errors.New("could not restart closed participant") } @@ -395,6 +396,7 @@ func (r *RoomManager) StartSession( Leave: leave, }, }) + prometheus.IncrementParticipantRtcCanceled(1) return errors.New("could not restart participant") } diff --git a/pkg/telemetry/events.go b/pkg/telemetry/events.go index 520b6f462..0000c1286 100644 --- a/pkg/telemetry/events.go +++ b/pkg/telemetry/events.go @@ -221,7 +221,7 @@ func (t *telemetryService) TrackPublishRequested( track *livekit.TrackInfo, ) { t.enqueue(func() { - prometheus.AddPublishAttempt(track.Type.String()) + prometheus.RecordTrackPublishAttempt(track.Type.String()) room := t.getRoomDetails(participantID) ev := newTrackEvent(livekit.AnalyticsEventType_TRACK_PUBLISH_REQUESTED, room, participantID, track) if ev.Participant != nil { @@ -239,7 +239,7 @@ func (t *telemetryService) TrackPublished( ) { t.enqueue(func() { prometheus.AddPublishedTrack(track.Type.String()) - prometheus.AddPublishSuccess(track.Type.String()) + prometheus.RecordTrackPublishSuccess(track.Type.String()) room := t.getRoomDetails(participantID) participant := &livekit.ParticipantInfo{ diff --git a/pkg/telemetry/prometheus/node.go b/pkg/telemetry/prometheus/node.go index 5668db3c4..480c95688 100644 --- a/pkg/telemetry/prometheus/node.go +++ b/pkg/telemetry/prometheus/node.go @@ -143,8 +143,10 @@ func GetNodeStats(nodeStartedAt int64, prevStats []*livekit.NodeStats, rateInter NumTracksOut: trackSubscribedCurrent.Load(), NumTrackPublishAttempts: trackPublishAttempts.Load(), NumTrackPublishSuccess: trackPublishSuccess.Load(), + NumTrackPublishCancels: trackPublishCancels.Load(), NumTrackSubscribeAttempts: trackSubscribeAttempts.Load(), NumTrackSubscribeSuccess: trackSubscribeSuccess.Load(), + NumTrackSubscribeCancels: trackSubscribeCancels.Load(), BytesIn: bytesIn.Load(), BytesOut: bytesOut.Load(), PacketsIn: packetsIn.Load(), @@ -155,6 +157,7 @@ func GetNodeStats(nodeStartedAt int64, prevStats []*livekit.NodeStats, rateInter ParticipantSignalConnected: participantSignalConnected.Load(), ParticipantRtcInit: participantRTCInit.Load(), ParticipantRtcConnected: participantRTCConnected.Load(), + ParticipantRtcCanceled: participantRTCCanceled.Load(), ForwardLatency: forwardLatency.Load(), ForwardJitter: forwardJitter.Load(), NumCpus: uint32(cpuStats.NumCPU()), // this will round down to the nearest integer @@ -235,12 +238,15 @@ func getNodeStatsRate(statsHistory []*livekit.NodeStats) *livekit.NodeStatsRate ParticipantSignalConnected: perSec(earlier.ParticipantSignalConnected, later.ParticipantSignalConnected, elapsed), ParticipantRtcInit: perSec(earlier.ParticipantRtcInit, later.ParticipantRtcInit, elapsed), ParticipantRtcConnected: perSec(earlier.ParticipantRtcConnected, later.ParticipantRtcConnected, elapsed), + ParticipantRtcCanceled: perSec(earlier.ParticipantRtcCanceled, later.ParticipantRtcCanceled, elapsed), SysPacketsOut: perSec(uint64(earlier.SysPacketsOut), uint64(later.SysPacketsOut), elapsed), SysPacketsDropped: perSec(uint64(earlier.SysPacketsDropped), uint64(later.SysPacketsDropped), elapsed), TrackPublishAttempts: perSec(uint64(earlier.NumTrackPublishAttempts), uint64(later.NumTrackPublishAttempts), elapsed), TrackPublishSuccess: perSec(uint64(earlier.NumTrackPublishSuccess), uint64(later.NumTrackPublishSuccess), elapsed), + TrackPublishCancels: perSec(uint64(earlier.NumTrackPublishCancels), uint64(later.NumTrackPublishCancels), elapsed), TrackSubscribeAttempts: perSec(uint64(earlier.NumTrackSubscribeAttempts), uint64(later.NumTrackSubscribeAttempts), elapsed), TrackSubscribeSuccess: perSec(uint64(earlier.NumTrackSubscribeSuccess), uint64(later.NumTrackSubscribeSuccess), elapsed), + TrackSubscribeCancels: perSec(uint64(earlier.NumTrackSubscribeCancels), uint64(later.NumTrackSubscribeCancels), elapsed), CpuLoad: cpuLoad / float32(elapsed), MemoryLoad: memoryLoad / float32(elapsed), MemoryUsed: memoryUsed / float32(elapsed), diff --git a/pkg/telemetry/prometheus/packets.go b/pkg/telemetry/prometheus/packets.go index d9681748a..576bb65cb 100644 --- a/pkg/telemetry/prometheus/packets.go +++ b/pkg/telemetry/prometheus/packets.go @@ -46,6 +46,7 @@ var ( participantSignalConnected atomic.Uint64 participantRTCConnected atomic.Uint64 participantRTCInit atomic.Uint64 + participantRTCCanceled atomic.Uint64 forwardLatency atomic.Uint32 forwardJitter atomic.Uint32 @@ -328,6 +329,13 @@ func IncrementParticipantRtcConnected(join uint32) { } } +func IncrementParticipantRtcCanceled(numCancels uint64) { + if numCancels > 0 { + participantRTCConnected.Add(numCancels) + promParticipantJoin.WithLabelValues("rtc_canceled").Add(float64(numCancels)) + } +} + func AddConnection(direction Direction) { promConnections.WithLabelValues(string(direction)).Add(1) } diff --git a/pkg/telemetry/prometheus/rooms.go b/pkg/telemetry/prometheus/rooms.go index 9214c7e42..00ec4b11c 100644 --- a/pkg/telemetry/prometheus/rooms.go +++ b/pkg/telemetry/prometheus/rooms.go @@ -31,8 +31,10 @@ var ( trackSubscribedCurrent atomic.Int32 trackPublishAttempts atomic.Int32 trackPublishSuccess atomic.Int32 + trackPublishCancels atomic.Int32 trackSubscribeAttempts atomic.Int32 trackSubscribeSuccess atomic.Int32 + trackSubscribeCancels atomic.Int32 // count the number of failures that are due to user error (permissions, track doesn't exist), so we could compute // success rate by subtracting this from total attempts trackSubscribeUserError atomic.Int32 @@ -162,16 +164,21 @@ func SubPublishedTrack(kind string) { trackPublishedCurrent.Dec() } -func AddPublishAttempt(kind string) { +func RecordTrackPublishAttempt(kind string) { trackPublishAttempts.Inc() promTrackPublishCounter.WithLabelValues(kind, "attempt").Inc() } -func AddPublishSuccess(kind string) { +func RecordTrackPublishSuccess(kind string) { trackPublishSuccess.Inc() promTrackPublishCounter.WithLabelValues(kind, "success").Inc() } +func RecordTrackPublishCancels(kind string, numCancels int32) { + trackPublishCancels.Add(numCancels) + promTrackPublishCounter.WithLabelValues(kind, "cancel").Add(float64(numCancels)) +} + func RecordPublishTime( country string, source livekit.TrackSource, @@ -246,9 +253,15 @@ func RecordTrackSubscribeFailure(err error, isUserError bool) { if isUserError { trackSubscribeUserError.Inc() + trackSubscribeCancels.Inc() } } +func RecordTrackSubscribeCancels(numCancels int32) { + trackSubscribeCancels.Add(numCancels) + promTrackSubscribeCounter.WithLabelValues("cancel", "").Add(float64(numCancels)) +} + func RecordSessionStartTime(protocolVersion int, d time.Duration) { promSessionStartTime.WithLabelValues(strconv.Itoa(protocolVersion)).Observe(float64(d.Milliseconds())) }