From 07791ffc00e9bd2276b9391192012726d729ee09 Mon Sep 17 00:00:00 2001 From: Denys Smirnov Date: Fri, 26 Apr 2024 13:57:39 +0300 Subject: [PATCH 01/60] Pass new SIP metadata. Update protocol. (#2683) --- go.mod | 2 +- go.sum | 4 ++-- pkg/service/ioservice_sip.go | 36 +++++++++++++++++++++++++++--------- pkg/service/sip.go | 33 +++++++++++++++++++++------------ 4 files changed, 51 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index ed85a6780..42c4858cc 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20240416023643-881d3dc5423e - github.com/livekit/protocol v1.12.1-0.20240426063020-fd19ad24b86b + github.com/livekit/protocol v1.14.1-0.20240426104403-e7962f444464 github.com/livekit/psrpc v0.5.3-0.20240426045048-8ba067a45715 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 diff --git a/go.sum b/go.sum index 28895fb95..d2a138a9c 100644 --- a/go.sum +++ b/go.sum @@ -120,8 +120,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20240416023643-881d3dc5423e h1:ss4VwrouYiDpuNJ9BUTH+WsW+GDdJS70iZp8ii3/0Lc= github.com/livekit/mediatransportutil v0.0.0-20240416023643-881d3dc5423e/go.mod h1:jwKUCmObuiEDH0iiuJHaGMXwRs3RjrB4G6qqgkr/5oE= -github.com/livekit/protocol v1.12.1-0.20240426063020-fd19ad24b86b h1:hPgkp/LJzhx+U2CHOc68yxGIyfFspagsyupAaqx1Ulw= -github.com/livekit/protocol v1.12.1-0.20240426063020-fd19ad24b86b/go.mod h1:pnn0Dv+/0K0OFqKHX6J6SreYO1dZxl6tDuAZ1ns8L/w= +github.com/livekit/protocol v1.14.1-0.20240426104403-e7962f444464 h1:5IxCPDkibpvnAYN6+djltH6Gj4dMOL0hNecHn5jZKmk= +github.com/livekit/protocol v1.14.1-0.20240426104403-e7962f444464/go.mod h1:pnn0Dv+/0K0OFqKHX6J6SreYO1dZxl6tDuAZ1ns8L/w= github.com/livekit/psrpc v0.5.3-0.20240426045048-8ba067a45715 h1:vhDMOe8fxEc/amYTFo799LySPM12Fk3vc+Nc6o4gYZQ= github.com/livekit/psrpc v0.5.3-0.20240426045048-8ba067a45715/go.mod h1:CQUBSPfYYAaevg1TNCc6/aYsa8DJH4jSRFdCeSZk5u0= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= diff --git a/pkg/service/ioservice_sip.go b/pkg/service/ioservice_sip.go index 133044af9..e893dafb5 100644 --- a/pkg/service/ioservice_sip.go +++ b/pkg/service/ioservice_sip.go @@ -47,38 +47,56 @@ func (s *IOInfoService) matchSIPDispatchRule(ctx context.Context, trunk *livekit } func (s *IOInfoService) EvaluateSIPDispatchRules(ctx context.Context, req *rpc.EvaluateSIPDispatchRulesRequest) (*rpc.EvaluateSIPDispatchRulesResponse, error) { + log := logger.GetLogger() + log = log.WithValues("to-user", req.CalledNumber, "from-user", req.CallingNumber) trunk, err := s.matchSIPTrunk(ctx, req.CallingNumber, req.CalledNumber) if err != nil { return nil, err } + trunkID := "" if trunk != nil { - logger.Debugw("SIP trunk matched", "trunkID", trunk.SipTrunkId, "called", req.CalledNumber, "calling", req.CallingNumber) + trunkID = trunk.SipTrunkId + } + log = log.WithValues("sip-trunk", trunkID) + if trunk != nil { + log.Debugw("SIP trunk matched") } else { - logger.Debugw("No SIP trunk matched", "trunkID", "", "called", req.CalledNumber, "calling", req.CallingNumber) + log.Debugw("No SIP trunk matched") } best, err := s.matchSIPDispatchRule(ctx, trunk, req) if err != nil { if e := (*sip.ErrNoDispatchMatched)(nil); errors.As(err, &e) { - return &rpc.EvaluateSIPDispatchRulesResponse{Result: rpc.SIPDispatchResult_DROP}, nil + return &rpc.EvaluateSIPDispatchRulesResponse{ + SipTrunkId: trunkID, + Result: rpc.SIPDispatchResult_DROP, + }, nil } return nil, err } - logger.Debugw("SIP dispatch rule matched", "dispatchRule", best.SipDispatchRuleId, "called", req.CalledNumber, "calling", req.CallingNumber) - return sip.EvaluateDispatchRule(best, req) + log.Debugw("SIP dispatch rule matched", "sip-rule", best.SipDispatchRuleId) + resp, err := sip.EvaluateDispatchRule(best, req) + if err != nil { + return nil, err + } + resp.SipTrunkId = trunkID + return resp, err } func (s *IOInfoService) GetSIPTrunkAuthentication(ctx context.Context, req *rpc.GetSIPTrunkAuthenticationRequest) (*rpc.GetSIPTrunkAuthenticationResponse, error) { + log := logger.GetLogger() + log = log.WithValues("to-user", req.To, "from-user", req.From) trunk, err := s.matchSIPTrunk(ctx, req.From, req.To) if err != nil { return nil, err } if trunk == nil { - logger.Debugw("No SIP trunk matched for auth", "trunkID", "", "called", req.To, "calling", req.From) + log.Debugw("No SIP trunk matched for auth", "sip-trunk", "") return &rpc.GetSIPTrunkAuthenticationResponse{}, nil } - logger.Debugw("SIP trunk matched for auth", "trunkID", trunk.SipTrunkId, "called", req.To, "calling", req.From) + log.Debugw("SIP trunk matched for auth", "sip-trunk", trunk.SipTrunkId) return &rpc.GetSIPTrunkAuthenticationResponse{ - Username: trunk.InboundUsername, - Password: trunk.InboundPassword, + SipTrunkId: trunk.SipTrunkId, + Username: trunk.InboundUsername, + Password: trunk.InboundPassword, }, nil } diff --git a/pkg/service/sip.go b/pkg/service/sip.go index 1bcf8f603..172b46e5c 100644 --- a/pkg/service/sip.go +++ b/pkg/service/sip.go @@ -76,6 +76,8 @@ func (s *SIPService) CreateSIPTrunk(ctx context.Context, req *livekit.CreateSIPT InboundPassword: req.InboundPassword, OutboundUsername: req.OutboundUsername, OutboundPassword: req.OutboundPassword, + Name: req.Name, + Metadata: req.Metadata, } // Validate all trunks including the new one first. @@ -136,6 +138,8 @@ func (s *SIPService) CreateSIPDispatchRule(ctx context.Context, req *livekit.Cre Rule: req.Rule, TrunkIds: req.TrunkIds, HidePhoneNumber: req.HidePhoneNumber, + Name: req.Name, + Metadata: req.Metadata, } // Validate all rules including the new one first. @@ -190,28 +194,32 @@ func (s *SIPService) CreateSIPParticipantWithToken(ctx context.Context, req *liv if s.store == nil { return nil, ErrSIPNotConnected } + callID := sip.NewCallID() + log := logger.GetLogger() + log = log.WithValues("call-id", callID, "roomName", req.RoomName, "sip-trunk", req.SipTrunkId, "to-user", req.SipCallTo) - AppendLogFields(ctx, "room", req.RoomName, "trunk", req.SipTrunkId, "to", req.SipCallTo) ireq := &rpc.InternalCreateSIPParticipantRequest{ + SipCallId: callID, CallTo: req.SipCallTo, RoomName: req.RoomName, ParticipantIdentity: req.ParticipantIdentity, + ParticipantName: req.ParticipantName, + ParticipantMetadata: req.ParticipantMetadata, Dtmf: req.Dtmf, PlayRingtone: req.PlayRingtone, WsUrl: wsUrl, Token: token, } - if req.SipTrunkId != "" { - trunk, err := s.store.LoadSIPTrunk(ctx, req.SipTrunkId) - if err != nil { - logger.Errorw("cannot get trunk to update sip participant", err) - return nil, err - } - ireq.Address = trunk.OutboundAddress - ireq.Number = trunk.OutboundNumber - ireq.Username = trunk.OutboundUsername - ireq.Password = trunk.OutboundPassword + trunk, err := s.store.LoadSIPTrunk(ctx, req.SipTrunkId) + if err != nil { + log.Errorw("cannot get trunk to update sip participant", err) + return nil, err } + log = log.WithValues("from-user", trunk.OutboundNumber, "to-host", trunk.OutboundAddress) + ireq.Address = trunk.OutboundAddress + ireq.Number = trunk.OutboundNumber + ireq.Username = trunk.OutboundUsername + ireq.Password = trunk.OutboundPassword // CreateSIPParticipant will wait for LiveKit Participant to be created and that can take some time. // Thus, we must set a higher deadline for it, if it's not set already. @@ -226,13 +234,14 @@ func (s *SIPService) CreateSIPParticipantWithToken(ctx context.Context, req *liv } resp, err := s.psrpcClient.CreateSIPParticipant(ctx, "", ireq, psrpc.WithRequestTimeout(timeout)) if err != nil { - logger.Errorw("cannot update sip participant", err) + log.Errorw("cannot update sip participant", err) return nil, err } return &livekit.SIPParticipantInfo{ ParticipantId: resp.ParticipantId, ParticipantIdentity: resp.ParticipantIdentity, RoomName: req.RoomName, + SipCallId: callID, }, nil } func (s *SIPService) CreateSIPParticipant(ctx context.Context, req *livekit.CreateSIPParticipantRequest) (*livekit.SIPParticipantInfo, error) { From f0887300074a40c7867c65fef18465622506088c Mon Sep 17 00:00:00 2001 From: Benjamin Pracht Date: Fri, 26 Apr 2024 14:26:00 +0200 Subject: [PATCH 02/60] Add support for EnableTranscoding ingress option (#2681) This changes the default WHIP ingress behavior to skipping transcode. Other inputs are still always transcoded. --- pkg/service/ingress.go | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/pkg/service/ingress.go b/pkg/service/ingress.go index 40b09aaf3..d5b87df81 100644 --- a/pkg/service/ingress.go +++ b/pkg/service/ingress.go @@ -156,7 +156,7 @@ func (s *IngressService) CreateIngressWithUrl(ctx context.Context, urlStr string InputType: req.InputType, Audio: req.Audio, Video: req.Video, - BypassTranscoding: req.BypassTranscoding, + EnableTranscoding: req.EnableTranscoding, RoomName: req.RoomName, ParticipantIdentity: req.ParticipantIdentity, ParticipantName: req.ParticipantName, @@ -179,9 +179,7 @@ func (s *IngressService) CreateIngressWithUrl(ctx context.Context, urlStr string return nil, ingress.ErrInvalidIngressType } - if err := ingress.ValidateForSerialization(info); err != nil { - return nil, err - } + updateEnableTranscoding(info) if req.InputType == livekit.IngressInput_URL_INPUT { retInfo, err := s.launcher.LaunchPullIngress(ctx, info) @@ -221,6 +219,24 @@ func (s *IngressService) LaunchPullIngress(ctx context.Context, info *livekit.In return s.psrpcClient.StartIngress(ctx, req) } +func updateEnableTranscoding(info *livekit.IngressInfo) { + // Set BypassTranscoding as well for backward compatiblity + if info.EnableTranscoding != nil { + info.BypassTranscoding = !*info.EnableTranscoding + return + } + + switch info.InputType { + case livekit.IngressInput_WHIP_INPUT: + f := false + info.EnableTranscoding = &f + info.BypassTranscoding = true + default: + t := true + info.EnableTranscoding = &t + } +} + func updateInfoUsingRequest(req *livekit.UpdateIngressRequest, info *livekit.IngressInfo) error { if req.Name != "" { info.Name = req.Name @@ -234,9 +250,10 @@ func updateInfoUsingRequest(req *livekit.UpdateIngressRequest, info *livekit.Ing if req.ParticipantName != "" { info.ParticipantName = req.ParticipantName } - if req.BypassTranscoding != nil { - info.BypassTranscoding = *req.BypassTranscoding + if req.EnableTranscoding != nil { + info.EnableTranscoding = req.EnableTranscoding } + if req.ParticipantMetadata != "" { info.ParticipantMetadata = req.ParticipantMetadata } @@ -251,6 +268,8 @@ func updateInfoUsingRequest(req *livekit.UpdateIngressRequest, info *livekit.Ing return err } + updateEnableTranscoding(info) + return nil } From 15ffb53df976ce19086a17c9339a107dce038989 Mon Sep 17 00:00:00 2001 From: Benjamin Pracht Date: Fri, 26 Apr 2024 21:07:55 +0200 Subject: [PATCH 03/60] Version 1.6.1 ## Changelog This release changes the default behavior when creating or updating WHIP ingress. WHIP ingress will now default to disabling transcoding and forwarding media unchanged to the LiveKit subscribers. This behavior can be changed by using the new `enable_transcoding` available in updated SDKs. The behavior of existing ingresses is unchanged. ### Added - Add support for "abs-capture-time" extension. (#2640) - Add PropagationDelay API to sender report data (#2646) - Add support for EnableTranscoding ingress option (#2681) - Pass new SIP metadata. Update protocol. (#2683) - Handle UpdateLocalAudioTrack and UpdateLocalVideoTrack. (#2684) - Forward transcription data packets to the room (#2687) ### Fixed - backwards compatability for IsRecorder (#2647) - Reduce RED weight in half. (#2648) - add disconnected chan to participant (#2650) - add typed ops queue (#2655) - ICE config cache module. (#2654) - use typed ops queue in pctransport (#2656) - Use the ingress state updated_at field to ensure that out of order RPC do not overwrite state (#2657) - Log ICE candidates to debug TCP connection issues. (#2658) - Debug logging addition of ICE candidate (#2659) - fix participant, ensure room name matches (#2660) - replace keyframe ticker with timer (#2661) - fix key frame timer (#2662) - Disable dynamic playout delay for screenshare track (#2663) - Don't log dd invalid template index (#2664) - Do codec munging when munging RTP header. (#2665) - Connection quality LOST only if RTCP is also not available. (#2670) - Handle large jumps in RTCP sender report timestamp. (#2674) - Bump golang.org/x/net from 0.22.0 to 0.23.0 (#2673) - do not capture pointers in ops queue closures (#2675) - Fix SubParticipant twice when paticipant left (#2672) - use ttlcache (#2677) - Detach subscriber datachannel to save memory (#2680) - Clean up UpdateVideoLayers (#2685) - Make datachannel optional for publisher (#2686) --- version/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version/version.go b/version/version.go index 96df3705f..b6dc66aa5 100644 --- a/version/version.go +++ b/version/version.go @@ -14,4 +14,4 @@ package version -const Version = "1.6.0" +const Version = "1.6.1" From ac1b0e38caac925c4783c99f230575395ae91cc4 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Fri, 26 Apr 2024 17:24:10 -0700 Subject: [PATCH 04/60] store active stats workers in list (#2690) * store active stats workers in list * test * single node cleanup * cleanup * cleanup * cleanup --- pkg/telemetry/events.go | 3 +- pkg/telemetry/statsworker.go | 28 +++---- pkg/telemetry/telemetryservice.go | 122 ++++++++++++++++-------------- 3 files changed, 81 insertions(+), 72 deletions(-) diff --git a/pkg/telemetry/events.go b/pkg/telemetry/events.go index 1a5d22853..e16f5fadd 100644 --- a/pkg/telemetry/events.go +++ b/pkg/telemetry/events.go @@ -163,10 +163,9 @@ func (t *telemetryService) ParticipantLeft(ctx context.Context, isConnected := false if worker, ok := t.getWorker(livekit.ParticipantID(participant.Sid)); ok { isConnected = worker.IsConnected() - if worker.ClosedAt().IsZero() { + if worker.Close() { prometheus.SubParticipant() } - worker.Close() } if isConnected && shouldSendEvent { diff --git a/pkg/telemetry/statsworker.go b/pkg/telemetry/statsworker.go index 18bde9e55..735821b6c 100644 --- a/pkg/telemetry/statsworker.go +++ b/pkg/telemetry/statsworker.go @@ -29,6 +29,8 @@ import ( // StatsWorker handles participant stats type StatsWorker struct { + next *StatsWorker + ctx context.Context t TelemetryService roomID livekit.RoomID @@ -91,8 +93,8 @@ func (s *StatsWorker) IsConnected() bool { return s.isConnected } -func (s *StatsWorker) Flush() { - ts := timestamppb.Now() +func (s *StatsWorker) Flush(now time.Time) bool { + ts := timestamppb.New(now) s.lock.Lock() stats := make([]*livekit.AnalyticsStat, 0, len(s.incomingPerTrack)+len(s.outgoingPerTrack)) @@ -102,6 +104,8 @@ func (s *StatsWorker) Flush() { outgoingPerTrack := s.outgoingPerTrack s.outgoingPerTrack = make(map[livekit.TrackID][]*livekit.AnalyticsStat) + + closed := !s.closedAt.IsZero() && now.Sub(s.closedAt) > workerCleanupWait s.lock.Unlock() stats = s.collectStats(ts, livekit.StreamType_UPSTREAM, incomingPerTrack, stats) @@ -109,21 +113,19 @@ func (s *StatsWorker) Flush() { if len(stats) > 0 { s.t.SendStats(s.ctx, stats) } + + return closed } -func (s *StatsWorker) Close() { - s.Flush() - +func (s *StatsWorker) Close() bool { s.lock.Lock() - s.closedAt = time.Now() - s.lock.Unlock() -} + defer s.lock.Unlock() -func (s *StatsWorker) ClosedAt() time.Time { - s.lock.RLock() - defer s.lock.RUnlock() - - return s.closedAt + ok := s.closedAt.IsZero() + if ok { + s.closedAt = time.Now() + } + return ok } func (s *StatsWorker) collectStats( diff --git a/pkg/telemetry/telemetryservice.go b/pkg/telemetry/telemetryservice.go index 9695476ab..e20e2a543 100644 --- a/pkg/telemetry/telemetryservice.go +++ b/pkg/telemetry/telemetryservice.go @@ -19,8 +19,6 @@ import ( "sync" "time" - "golang.org/x/exp/maps" - "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/utils" "github.com/livekit/protocol/livekit" @@ -95,9 +93,11 @@ type telemetryService struct { notifier webhook.QueuedNotifier jobsQueue *utils.OpsQueue - lock sync.RWMutex - workers map[livekit.ParticipantID]*StatsWorker - workersShadow []*StatsWorker + workersMu sync.RWMutex + workers map[livekit.ParticipantID]*StatsWorker + workerList *StatsWorker + + flushMu sync.Mutex } func NewTelemetryService(notifier webhook.QueuedNotifier, analytics AnalyticsService) TelemetryService { @@ -121,29 +121,62 @@ func NewTelemetryService(notifier webhook.QueuedNotifier, analytics AnalyticsSer } func (t *telemetryService) FlushStats() { - t.lock.RLock() - workersShadow := t.workersShadow - t.lock.RUnlock() + t.flushMu.Lock() + defer t.flushMu.Unlock() - for _, worker := range workersShadow { - worker.Flush() + t.workersMu.RLock() + worker := t.workerList + t.workersMu.RUnlock() + + now := time.Now() + var prev, reapHead, reapTail *StatsWorker + for worker != nil { + if closed := worker.Flush(now); closed { + if prev == nil { + // this worker was at the head of the list + t.workersMu.Lock() + p := &t.workerList + for *p != worker { + // new workers have been added. scan until we find the one + // immediately before this + prev = *p + p = &prev.next + } + *p = worker.next + t.workersMu.Unlock() + } else { + prev.next = worker.next + } + + if reapHead == nil { + reapHead = worker + reapTail = worker + } else { + reapTail.next = worker + reapTail = worker + } + } else { + prev = worker + } + worker = worker.next + } + + if reapHead != nil { + t.workersMu.Lock() + for { + delete(t.workers, reapHead.participantID) + if reapHead == reapTail { + break + } + reapHead = reapHead.next + } + t.workersMu.Unlock() } } func (t *telemetryService) run() { - ticker := time.NewTicker(config.TelemetryStatsUpdateInterval) - defer ticker.Stop() - - cleanupTicker := time.NewTicker(time.Minute) - defer cleanupTicker.Stop() - - for { - select { - case <-ticker.C: - t.FlushStats() - case <-cleanupTicker.C: - t.cleanupWorkers() - } + for range time.Tick(config.TelemetryStatsUpdateInterval) { + t.FlushStats() } } @@ -152,21 +185,22 @@ func (t *telemetryService) enqueue(op func()) { } func (t *telemetryService) getWorker(participantID livekit.ParticipantID) (worker *StatsWorker, ok bool) { - t.lock.RLock() - defer t.lock.RUnlock() + t.workersMu.RLock() + defer t.workersMu.RUnlock() worker, ok = t.workers[participantID] return } -func (t *telemetryService) getOrCreateWorker(ctx context.Context, +func (t *telemetryService) getOrCreateWorker( + ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, participantIdentity livekit.ParticipantIdentity, ) (*StatsWorker, bool) { - t.lock.Lock() - defer t.lock.Unlock() + t.workersMu.Lock() + defer t.workersMu.Unlock() if worker, ok := t.workers[participantID]; ok { return worker, true @@ -182,39 +216,13 @@ func (t *telemetryService) getOrCreateWorker(ctx context.Context, ) t.workers[participantID] = worker - t.workersShadow = maps.Values(t.workers) + + worker.next = t.workerList + t.workerList = worker return worker, false } -func (t *telemetryService) cleanupWorkers() { - t.lock.RLock() - workersShadow := t.workersShadow - t.lock.RUnlock() - - toReap := make([]livekit.ParticipantID, 0, len(workersShadow)) - for _, worker := range workersShadow { - closedAt := worker.ClosedAt() - if !closedAt.IsZero() && time.Since(closedAt) > workerCleanupWait { - worker.Flush() - - toReap = append(toReap, worker.ParticipantID()) - } - } - - if len(toReap) == 0 { - return - } - - t.lock.Lock() - logger.Debugw("reaping analytics worker for participants", "pID", toReap) - for _, pID := range toReap { - delete(t.workers, pID) - } - t.workersShadow = maps.Values(t.workers) - t.lock.Unlock() -} - func (t *telemetryService) LocalRoomState(ctx context.Context, info *livekit.AnalyticsNodeRooms) { t.enqueue(func() { t.SendNodeRoomStates(ctx, info) From 18b3b7b421a1c68de798d8f20a2db49b80365fbc Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Sat, 27 Apr 2024 04:04:01 -0700 Subject: [PATCH 05/60] use readCond in buffer read (#2691) --- pkg/sfu/buffer/buffer.go | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go index 85a9f07eb..64ff28917 100644 --- a/pkg/sfu/buffer/buffer.go +++ b/pkg/sfu/buffer/buffer.go @@ -336,6 +336,7 @@ func (b *Buffer) Write(pkt []byte) (n int, err error) { arrivalTime: now, }) b.Unlock() + b.readCond.Signal() return } @@ -392,26 +393,24 @@ func (b *Buffer) writeRTX(rtxPkt *rtp.Packet, arrivalTime time.Time) (n int, err } func (b *Buffer) Read(buff []byte) (n int, err error) { + b.Lock() for { if b.closed.Load() { - err = io.EOF - return + b.Unlock() + return 0, io.EOF } - b.Lock() if b.pPackets != nil && len(b.pPackets) > b.lastPacketRead { if len(buff) < len(b.pPackets[b.lastPacketRead].packet) { - err = bucket.ErrBufferTooSmall b.Unlock() - return + return 0, bucket.ErrBufferTooSmall } - n = len(b.pPackets[b.lastPacketRead].packet) - copy(buff, b.pPackets[b.lastPacketRead].packet) + + n = copy(buff, b.pPackets[b.lastPacketRead].packet) b.lastPacketRead++ b.Unlock() return } - b.Unlock() - time.Sleep(25 * time.Millisecond) + b.readCond.Wait() } } From c8b289daa5f538a41b4698b76f93f34e989ce213 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sun, 28 Apr 2024 12:13:52 +0530 Subject: [PATCH 06/60] (Attempted) Simplify time stamp calculation on switches. (#2688) * Simplify time stamp calculation on switches. Trying to simplify time stamp calculation on restarts. The additional checks take effect rarely and it not worth the extra complication. Also, doing the reference time stamp in extended range. The challenge with that is when publisher migrates the extended timestamp could change post migration (i. e. post migration would not know about rollovers). To address that, maintain an offset that is updated on resync. * WIP * Revert to resume threshold * typo * clean up --- pkg/sfu/buffer/rtpstats_base.go | 5 +++ pkg/sfu/buffer/rtpstats_receiver.go | 13 +++++--- pkg/sfu/buffer/rtpstats_sender.go | 4 ++- pkg/sfu/forwarder.go | 47 +++++++++++++++-------------- 4 files changed, 41 insertions(+), 28 deletions(-) diff --git a/pkg/sfu/buffer/rtpstats_base.go b/pkg/sfu/buffer/rtpstats_base.go index 04028465d..0b5139b37 100644 --- a/pkg/sfu/buffer/rtpstats_base.go +++ b/pkg/sfu/buffer/rtpstats_base.go @@ -529,6 +529,11 @@ func (r *rtpStatsBase) maybeAdjustFirstPacketTime(srData *RTCPSenderReportData, "adjustment", r.firstTime.Sub(firstTime).String(), "extNowTS", extNowTS, "extStartTS", extStartTS, + "srData", srData, + "timeSinceReceive", timeSinceReceive.String(), + "timeSinceFirst", timeSinceFirst.String(), + "samplesDiff", samplesDiff, + "samplesDuration", samplesDuration, } } diff --git a/pkg/sfu/buffer/rtpstats_receiver.go b/pkg/sfu/buffer/rtpstats_receiver.go index bd1b16341..979e3a623 100644 --- a/pkg/sfu/buffer/rtpstats_receiver.go +++ b/pkg/sfu/buffer/rtpstats_receiver.go @@ -212,18 +212,21 @@ func (r *RTPStatsReceiver) Update( flowState.ExtSequenceNumber = resSN.ExtendedVal flowState.ExtTimestamp = resTS.ExtendedVal } else { // in-order - if gapSN >= cNumSequenceNumbers/2 { + if gapSN >= cNumSequenceNumbers/2 || resTS.ExtendedVal < resTS.PreExtendedHighest { r.logger.Warnw( - "large sequence number gap", nil, + "large sequence number gap OR time reversed", nil, "extStartSN", r.sequenceNumber.GetExtendedStart(), "extHighestSN", r.sequenceNumber.GetExtendedHighest(), "extStartTS", r.timestamp.GetExtendedStart(), "extHighestTS", r.timestamp.GetExtendedHighest(), "firstTime", r.firstTime.String(), "highestTime", r.highestTime.String(), - "prev", resSN.PreExtendedHighest, - "curr", resSN.ExtendedVal, - "gap", gapSN, + "prevSN", resSN.PreExtendedHighest, + "currSN", resSN.ExtendedVal, + "gapSN", gapSN, + "prevTS", resTS.PreExtendedHighest, + "currTS", resTS.ExtendedVal, + "gapTS", resTS.ExtendedVal-resTS.PreExtendedHighest, "packetTime", packetTime.String(), "sequenceNumber", sequenceNumber, "timestamp", timestamp, diff --git a/pkg/sfu/buffer/rtpstats_sender.go b/pkg/sfu/buffer/rtpstats_sender.go index be3f12990..6dafb1047 100644 --- a/pkg/sfu/buffer/rtpstats_sender.go +++ b/pkg/sfu/buffer/rtpstats_sender.go @@ -33,6 +33,8 @@ const ( cSenderReportInitialWait = time.Second ) +// ------------------------------------------------------------------- + type snInfoFlag byte const ( @@ -625,7 +627,7 @@ func (r *RTPStatsSender) GetExpectedRTPTimestamp(at time.Time) (expectedTSExt ui defer r.lock.RUnlock() if !r.initialized { - err = errors.New("uninitilaized") + err = errors.New("uninitialized") return } diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 92c354c3b..6b8415972 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -191,7 +191,7 @@ type ForwarderState struct { ReferenceLayerSpatial int32 PreStartTime time.Time ExtFirstTS uint64 - RefTSOffset uint64 + DummyStartTSOffset uint64 RTP RTPMungerState Codec interface{} } @@ -202,12 +202,12 @@ func (f ForwarderState) String() string { case codecmunger.VP8State: codecString = codecState.String() } - return fmt.Sprintf("ForwarderState{started: %v, referenceLayerSpatial: %d, preStartTime: %s, extFirstTS: %d, refTSOffset: %d, rtp: %s, codec: %s}", + return fmt.Sprintf("ForwarderState{started: %v, referenceLayerSpatial: %d, preStartTime: %s, extFirstTS: %d, dummyStartTSOffset: %d, rtp: %s, codec: %s}", f.Started, f.ReferenceLayerSpatial, f.PreStartTime.String(), f.ExtFirstTS, - f.RefTSOffset, + f.DummyStartTSOffset, f.RTP.String(), codecString, ) @@ -232,7 +232,7 @@ type Forwarder struct { extFirstTS uint64 lastSSRC uint32 referenceLayerSpatial int32 - refTSOffset uint64 + dummyStartTSOffset uint64 refSenderReports [buffer.DefaultMaxLayerSpatial + 1]*buffer.RTCPSenderReportData refIsSVC bool @@ -383,7 +383,7 @@ func (f *Forwarder) GetState() ForwarderState { ReferenceLayerSpatial: f.referenceLayerSpatial, PreStartTime: f.preStartTime, ExtFirstTS: f.extFirstTS, - RefTSOffset: f.refTSOffset, + DummyStartTSOffset: f.dummyStartTSOffset, RTP: f.rtpMunger.GetLast(), Codec: f.codecMunger.GetState(), } @@ -404,7 +404,7 @@ func (f *Forwarder) SeedState(state ForwarderState) { f.referenceLayerSpatial = state.ReferenceLayerSpatial f.preStartTime = state.PreStartTime f.extFirstTS = state.ExtFirstTS - f.refTSOffset = state.RefTSOffset + f.dummyStartTSOffset = state.DummyStartTSOffset } func (f *Forwarder) Mute(muted bool, isSubscribeMutable bool) bool { @@ -578,7 +578,7 @@ func (f *Forwarder) clearRefSenderReportsLocked() { // This is done to prevent use of potentially stale publisher sender reports. // // It is possible to implement mute using pause/unpause - // which can implemented using a replaceTrack(null)/replaceTrack(track). + // which can be implemented using replaceTrack(null)/replaceTrack(track). // In those cases, the RTP time stamp may not jump across // the mute/pause valley (for the time it is replaced with null track). // So, relying on a report that happened before unmute/unpause @@ -591,7 +591,8 @@ func (f *Forwarder) clearRefSenderReportsLocked() { // 2. Publisher pauses: there are no more reports. // 3. When paused, subscriber can still use the publisher side sender // report to send reports. Although the time since last publisher - // sender report is increasing, the reports are correct though. + // sender report is increasing, the reports would still be correct + // as they referencing a previous (albeit older) correct report. // 4. Publisher unpauses after 20 seconds. But, it may not have advanced // RTP Timestamp by that much. Let us say, it advances only by 5 seconds. // 5. When subscriber starts forwarding packets, it will calculate @@ -1623,7 +1624,7 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e rtpMungerState := f.rtpMunger.GetLast() extLastTS := rtpMungerState.ExtLastTS extExpectedTS := extLastTS - extRefTS := extExpectedTS + extRefTS := extLastTS refTS := uint32(extRefTS) switchingAt := time.Now() if !f.skipReferenceTS { @@ -1639,13 +1640,13 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e } } + // adjust extRefTS to current packet's timestamp mapped to that of reference layer's extRefTS = (extRefTS & 0xFFFF_FFFF_0000_0000) + uint64(refTS) - - expectedTS := uint32(extExpectedTS) - if (refTS-expectedTS) < 1<<31 && refTS < expectedTS { + lastTS := uint32(extLastTS) + if (refTS-lastTS) < 1<<31 && refTS < lastTS { extRefTS += (1 << 32) } - if (expectedTS-refTS) < 1<<31 && expectedTS < refTS && extRefTS >= 1<<32 { + if (lastTS-refTS) < 1<<31 && lastTS < refTS && extRefTS >= 1<<32 { extRefTS -= (1 << 32) } @@ -1658,22 +1659,22 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e timeSinceFirst := time.Since(f.preStartTime) rtpDiff := uint64(timeSinceFirst.Nanoseconds() * int64(f.codec.ClockRate) / 1e9) extExpectedTS = f.extFirstTS + rtpDiff - if f.refTSOffset == 0 { - f.refTSOffset = extExpectedTS - extRefTS + if f.dummyStartTSOffset == 0 { + f.dummyStartTSOffset = extExpectedTS - extRefTS f.logger.Infow( - "calculating refTSOffset", + "calculating dummyStartTSOffset", "preStartTime", f.preStartTime.String(), "extFirstTS", f.extFirstTS, "timeSinceFirst", timeSinceFirst, "rtpDiff", rtpDiff, "extRefTS", extRefTS, - "refTSOffset", f.refTSOffset, + "dummyStartTSOffset", f.dummyStartTSOffset, ) } } } } - extRefTS += f.refTSOffset + extRefTS += f.dummyStartTSOffset var extNextTS uint64 if f.lastSSRC == 0 { @@ -1689,7 +1690,7 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e // timestamp should be used as things will catch up to real time when channel capacity // increases and pacer starts sending at faster rate. // - // But, the challenege is distinguishing between the two cases. As a compromise, the difference + // But, the challenge is distinguishing between the two cases. As a compromise, the difference // between extExpectedTS and extRefTS is thresholded. Difference below the threshold is treated as Case 2 // and above as Case 1. // @@ -1734,14 +1735,16 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e logTransition("layer switch, reference too far behind", extExpectedTS, extRefTS, extLastTS, diffSeconds) return errors.New("switch point too far behind") } + // use a nominal increase to ensure that timestamp is always moving forward logTransition("layer switch, reference is slightly behind", extExpectedTS, extRefTS, extLastTS, diffSeconds) extNextTS = extLastTS + 1 } else { - diffSeconds = float64(int64(extExpectedTS-extRefTS)) / float64(f.codec.ClockRate) - if diffSeconds < 0.0 && math.Abs(diffSeconds) > SwitchAheadThresholdSeconds { + diffSeconds = float64(int64(extRefTS-extExpectedTS)) / float64(f.codec.ClockRate) + if diffSeconds > SwitchAheadThresholdSeconds { logTransition("layer switch, reference too far ahead", extExpectedTS, extRefTS, extLastTS, diffSeconds) } + extNextTS = extRefTS } } @@ -1757,7 +1760,7 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e "layer", layer, "extLastTS", extLastTS, "extRefTS", extRefTS, - "refTSOffset", f.refTSOffset, + "dummyStartTSOffset", f.dummyStartTSOffset, "referenceLayerSpatial", f.referenceLayerSpatial, "extExpectedTS", extExpectedTS, "extNextTS", extNextTS, From 3d22d794e01c6ff2502dddddc97831e730a3e928 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 27 Apr 2024 23:53:53 -0700 Subject: [PATCH 07/60] Update pion deps (#2692) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 42c4858cc..457d3a8e1 100644 --- a/go.mod +++ b/go.mod @@ -34,9 +34,9 @@ require ( github.com/pion/rtp v1.8.6 github.com/pion/sctp v1.8.16 github.com/pion/sdp/v3 v3.0.9 - github.com/pion/transport/v2 v2.2.4 + github.com/pion/transport/v2 v2.2.5 github.com/pion/turn/v2 v2.1.6 - github.com/pion/webrtc/v3 v3.2.38 + github.com/pion/webrtc/v3 v3.2.39 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.19.0 github.com/redis/go-redis/v9 v9.5.1 diff --git a/go.sum b/go.sum index d2a138a9c..ec1a6c8c2 100644 --- a/go.sum +++ b/go.sum @@ -198,16 +198,17 @@ github.com/pion/transport v0.14.1/go.mod h1:4tGmbk00NeYA3rUa9+n+dzCCoKkcy3YlYb99 github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g= github.com/pion/transport/v2 v2.2.2/go.mod h1:OJg3ojoBJopjEeECq2yJdXH9YVrUJ1uQ++NjXLOUorc= github.com/pion/transport/v2 v2.2.3/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= -github.com/pion/transport/v2 v2.2.4 h1:41JJK6DZQYSeVLxILA2+F4ZkKb4Xd/tFJZRFZQ9QAlo= github.com/pion/transport/v2 v2.2.4/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= +github.com/pion/transport/v2 v2.2.5 h1:iyi25i/21gQck4hfRhomF6SktmUQjRsRW4WJdhfc3Kc= +github.com/pion/transport/v2 v2.2.5/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= github.com/pion/transport/v3 v3.0.1/go.mod h1:UY7kiITrlMv7/IKgd5eTUcaahZx5oUN3l9SzK5f5xE0= github.com/pion/transport/v3 v3.0.2 h1:r+40RJR25S9w3jbA6/5uEPTzcdn7ncyU44RWCbHkLg4= github.com/pion/transport/v3 v3.0.2/go.mod h1:nIToODoOlb5If2jF9y2Igfx3PFYWfuXi37m0IlWa/D0= github.com/pion/turn/v2 v2.1.3/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= github.com/pion/turn/v2 v2.1.6 h1:Xr2niVsiPTB0FPtt+yAWKFUkU1eotQbGgpTIld4x1Gc= github.com/pion/turn/v2 v2.1.6/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= -github.com/pion/webrtc/v3 v3.2.38 h1:oA52VJAJhOjSi1JpKjf0CM+cCiZ3b7jBxvsoOiajeDU= -github.com/pion/webrtc/v3 v3.2.38/go.mod h1:AQ8p56OLbm3MjhYovYdgPuyX6oc+JcKx/HFoCGFcYzA= +github.com/pion/webrtc/v3 v3.2.39 h1:Lf2SIMGdE3M9VNm48KpoX5pR8SJ6TsMnktzOkc/oB0o= +github.com/pion/webrtc/v3 v3.2.39/go.mod h1:AQ8p56OLbm3MjhYovYdgPuyX6oc+JcKx/HFoCGFcYzA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= From cfa82c6ad4b6da74596dc7adafb21fde4f9d3439 Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Sun, 28 Apr 2024 15:09:24 +0800 Subject: [PATCH 08/60] increase protocol version to support optional publisher datachannel (#2693) --- pkg/rtc/types/protocol_version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/rtc/types/protocol_version.go b/pkg/rtc/types/protocol_version.go index ae2282085..b82e99f66 100644 --- a/pkg/rtc/types/protocol_version.go +++ b/pkg/rtc/types/protocol_version.go @@ -16,7 +16,7 @@ package types type ProtocolVersion int -const CurrentProtocol = 13 +const CurrentProtocol = 14 func (v ProtocolVersion) SupportsPackedStreamId() bool { return v > 0 From a2a8810734e96696ddf05f4f3a72bb4cad24385e Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 30 Apr 2024 08:29:59 +0530 Subject: [PATCH 09/60] log the correct new propagation delay (#2694) --- pkg/sfu/buffer/rtpstats_receiver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/sfu/buffer/rtpstats_receiver.go b/pkg/sfu/buffer/rtpstats_receiver.go index 979e3a623..da387dc44 100644 --- a/pkg/sfu/buffer/rtpstats_receiver.go +++ b/pkg/sfu/buffer/rtpstats_receiver.go @@ -441,7 +441,7 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *RTCPSenderReportData) } if r.propagationDelayDeltaHighCount >= cPropagationDelayDeltaHighResetNumReports && time.Since(r.propagationDelayDeltaHighStartTime) >= cPropagationDelayDeltaHighResetWait { - r.logger.Debugw("re-initializing propagation delay", append(getPropagationFields(), "newPropagationDelay", propagationDelay.String())...) + r.logger.Debugw("re-initializing propagation delay", append(getPropagationFields(), "newPropagationDelay", r.propagationDelaySpike.String())...) initPropagationDelay(r.propagationDelaySpike) } } else { From 72f847427dbdbcc91d0943258c61b7632bfba4b5 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 30 Apr 2024 08:49:12 -0700 Subject: [PATCH 10/60] Update module github.com/pion/ice/v2 to v2.3.20 (#2695) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 457d3a8e1..63031c75c 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/mitchellh/go-homedir v1.1.0 github.com/olekukonko/tablewriter v0.0.5 github.com/pion/dtls/v2 v2.2.10 - github.com/pion/ice/v2 v2.3.19 + github.com/pion/ice/v2 v2.3.20 github.com/pion/interceptor v0.1.29 github.com/pion/rtcp v1.2.14 github.com/pion/rtp v1.8.6 diff --git a/go.sum b/go.sum index ec1a6c8c2..e8190e1a3 100644 --- a/go.sum +++ b/go.sum @@ -168,8 +168,8 @@ github.com/pion/datachannel v1.5.5/go.mod h1:iMz+lECmfdCMqFRhXhcA/219B0SQlbpoR2V github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= github.com/pion/dtls/v2 v2.2.10 h1:u2Axk+FyIR1VFTPurktB+1zoEPGIW3bmyj3LEFrXjAA= github.com/pion/dtls/v2 v2.2.10/go.mod h1:d9SYc9fch0CqK90mRk1dC7AkzzpwJj6u2GU3u+9pqFE= -github.com/pion/ice/v2 v2.3.19 h1:1GoMRTMnB6bCP4aGy2MjxK3w4laDkk+m7svJb/eqybc= -github.com/pion/ice/v2 v2.3.19/go.mod h1:KXJJcZK7E8WzrBEYnV4UtqEZsGeWfHxsNqhVcVvgjxw= +github.com/pion/ice/v2 v2.3.20 h1:V6TN3E/B0Xh4nRjX4bbRlyyxsbF2Lv8NNdRidZjCxvc= +github.com/pion/ice/v2 v2.3.20/go.mod h1:KXJJcZK7E8WzrBEYnV4UtqEZsGeWfHxsNqhVcVvgjxw= github.com/pion/interceptor v0.1.29 h1:39fsnlP1U8gw2JzOFWdfCU82vHvhW9o0rZnZF56wF+M= github.com/pion/interceptor v0.1.29/go.mod h1:ri+LGNjRUc5xUNtDEPzfdkmSqISixVTBF/z/Zms/6T4= github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= From 2792e7e8342e2ef3a74d1bb34c4a2beab9aa9c53 Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 30 Apr 2024 19:45:58 -0400 Subject: [PATCH 11/60] Remove unsupported environment option (#2696) This option seems to be no longer supported as of 10c8582a6b01c37fb2db7e343b5204c713fbb7cd, and will crash the server if provided in the config. --- config-sample.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/config-sample.yaml b/config-sample.yaml index 96eadb44b..9370819a4 100644 --- a/config-sample.yaml +++ b/config-sample.yaml @@ -143,8 +143,6 @@ rtc: # when enabled, LiveKit will expose prometheus metrics on :6789/metrics # prometheus_port: 6789 -# set a custom environment variable. prometheus metrics will be labeled with this value. defaults to an empty string -# environment: custom-value # API key / secret pairs. # Keys are used for JWT authentication, server APIs would require a keypair in order to generate access tokens From 674550ea16a1cdd4f7e7e2267fa7f70ab1d9a5a8 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 2 May 2024 19:09:12 +0530 Subject: [PATCH 12/60] Option to disable traffic load tracking. (#2698) --- pkg/rtc/participant.go | 17 +++++++++++------ pkg/rtc/participant_traffic_load.go | 8 ++++++++ pkg/sfu/buffer/rtpstats_base.go | 1 + 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 0c3481449..7cb7db17f 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -140,6 +140,7 @@ type ParticipantParams struct { SubscriptionLimitVideo int32 PlayoutDelay *livekit.PlayoutDelay SyncStreams bool + EnableTrafficLoadTracking bool } type ParticipantImpl struct { @@ -868,7 +869,9 @@ func (p *ParticipantImpl) Close(sendLeave bool, reason types.ParticipantCloseRea go func() { p.SubscriptionManager.Close(isExpectedToResume) p.TransportManager.Close() - p.ParticipantTrafficLoad.Close() + if p.ParticipantTrafficLoad != nil { + p.ParticipantTrafficLoad.Close() + } }() p.dataChannelStats.Stop() @@ -1383,11 +1386,13 @@ func (p *ParticipantImpl) setupSubscriptionManager() { } func (p *ParticipantImpl) setupParticipantTrafficLoad() { - p.ParticipantTrafficLoad = NewParticipantTrafficLoad(ParticipantTrafficLoadParams{ - Participant: p, - DataChannelStats: p.dataChannelStats, - Logger: p.params.Logger, - }) + if p.params.EnableTrafficLoadTracking { + p.ParticipantTrafficLoad = NewParticipantTrafficLoad(ParticipantTrafficLoadParams{ + Participant: p, + DataChannelStats: p.dataChannelStats, + Logger: p.params.Logger, + }) + } } func (p *ParticipantImpl) updateState(state livekit.ParticipantInfo_State) { diff --git a/pkg/rtc/participant_traffic_load.go b/pkg/rtc/participant_traffic_load.go index e0fa7a4e5..28f6867d1 100644 --- a/pkg/rtc/participant_traffic_load.go +++ b/pkg/rtc/participant_traffic_load.go @@ -62,6 +62,10 @@ func (p *ParticipantTrafficLoad) Close() { } func (p *ParticipantTrafficLoad) OnTrafficLoad(f func(trafficLoad *types.TrafficLoad)) { + if p == nil { + return + } + p.lock.Lock() p.onTrafficLoad = f p.lock.Unlock() @@ -75,6 +79,10 @@ func (p *ParticipantTrafficLoad) getOnTrafficLoad() func(trafficLoad *types.Traf } func (p *ParticipantTrafficLoad) GetTrafficLoad() *types.TrafficLoad { + if p == nil { + return nil + } + p.lock.RLock() defer p.lock.RUnlock() diff --git a/pkg/sfu/buffer/rtpstats_base.go b/pkg/sfu/buffer/rtpstats_base.go index 0b5139b37..eba2425b9 100644 --- a/pkg/sfu/buffer/rtpstats_base.go +++ b/pkg/sfu/buffer/rtpstats_base.go @@ -530,6 +530,7 @@ func (r *rtpStatsBase) maybeAdjustFirstPacketTime(srData *RTCPSenderReportData, "extNowTS", extNowTS, "extStartTS", extStartTS, "srData", srData, + "tsOffset", tsOffset, "timeSinceReceive", timeSinceReceive.String(), "timeSinceFirst", timeSinceFirst.String(), "samplesDiff", samplesDiff, From 26c2f7cd9a1217852a8c61afd827ed7f856800db Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 3 May 2024 18:56:01 +0530 Subject: [PATCH 13/60] Do not use LastTS for dummy offset. (#2700) * Do not use LastTS for dummy offset. LastTS could be random when using dummy start. That should not be used in calculating offsets. Also, do not push padding into sequence before init. Could have heppened with dummy start. * apply dummy offset before comparing to last * refresh ref TS * initialize codec munger on catch up forwarding --- pkg/sfu/forwarder.go | 13 +++++++++---- pkg/sfu/sequencer.go | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 6b8415972..9fb66e020 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -1589,6 +1589,7 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e return nil } else if f.referenceLayerSpatial == buffer.InvalidLayerSpatial { f.referenceLayerSpatial = layer + f.codecMunger.SetLast(extPkt) f.logger.Debugw( "catch up forwarding", "sequenceNumber", extPkt.Packet.SequenceNumber, @@ -1641,8 +1642,9 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e } // adjust extRefTS to current packet's timestamp mapped to that of reference layer's - extRefTS = (extRefTS & 0xFFFF_FFFF_0000_0000) + uint64(refTS) + extRefTS = (extRefTS & 0xFFFF_FFFF_0000_0000) + uint64(refTS) + f.dummyStartTSOffset lastTS := uint32(extLastTS) + refTS = uint32(extRefTS) if (refTS-lastTS) < 1<<31 && refTS < lastTS { extRefTS += (1 << 32) } @@ -1660,21 +1662,23 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e rtpDiff := uint64(timeSinceFirst.Nanoseconds() * int64(f.codec.ClockRate) / 1e9) extExpectedTS = f.extFirstTS + rtpDiff if f.dummyStartTSOffset == 0 { - f.dummyStartTSOffset = extExpectedTS - extRefTS + f.dummyStartTSOffset = extExpectedTS - uint64(refTS) + extRefTS = extExpectedTS f.logger.Infow( "calculating dummyStartTSOffset", "preStartTime", f.preStartTime.String(), "extFirstTS", f.extFirstTS, - "timeSinceFirst", timeSinceFirst, + "timeSinceFirst", timeSinceFirst.String(), "rtpDiff", rtpDiff, "extRefTS", extRefTS, + "incomingTS", extPkt.Packet.Timestamp, + "referenceLayerSpatial", f.referenceLayerSpatial, "dummyStartTSOffset", f.dummyStartTSOffset, ) } } } } - extRefTS += f.dummyStartTSOffset var extNextTS uint64 if f.lastSSRC == 0 { @@ -1759,6 +1763,7 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e "switchingAt", switchingAt.String(), "layer", layer, "extLastTS", extLastTS, + "lastMarker", rtpMungerState.LastMarker, "extRefTS", extRefTS, "dummyStartTSOffset", f.dummyStartTSOffset, "referenceLayerSpatial", f.referenceLayerSpatial, diff --git a/pkg/sfu/sequencer.go b/pkg/sfu/sequencer.go index a78b23eb6..c876b63c9 100644 --- a/pkg/sfu/sequencer.go +++ b/pkg/sfu/sequencer.go @@ -237,7 +237,7 @@ func (s *sequencer) pushPadding(extStartSNInclusive uint64, extEndSNInclusive ui s.Lock() defer s.Unlock() - if s.snRangeMap == nil { + if s.snRangeMap == nil || !s.initialized { return } From 45ed030ce673d9d496588ba97a7125f70c5ce201 Mon Sep 17 00:00:00 2001 From: Antti Tapaninen Date: Fri, 3 May 2024 23:17:29 -0700 Subject: [PATCH 14/60] add missing strings.EqualFold for some mimeType comparisons (#2701) --- pkg/rtc/mediatrackreceiver.go | 2 +- pkg/rtc/mediatracksubscriptions.go | 5 +++-- pkg/rtc/wrappedreceiver.go | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/rtc/mediatrackreceiver.go b/pkg/rtc/mediatrackreceiver.go index ec4c9adf0..5453e4300 100644 --- a/pkg/rtc/mediatrackreceiver.go +++ b/pkg/rtc/mediatrackreceiver.go @@ -469,7 +469,7 @@ func (t *MediaTrackReceiver) AddSubscriber(sub types.LocalParticipant) (types.Su codec := receiver.Codec() var found bool for _, pc := range potentialCodecs { - if codec.MimeType == pc.MimeType { + if strings.EqualFold(codec.MimeType, pc.MimeType) { found = true break } diff --git a/pkg/rtc/mediatracksubscriptions.go b/pkg/rtc/mediatracksubscriptions.go index 158cafe10..78c045e88 100644 --- a/pkg/rtc/mediatracksubscriptions.go +++ b/pkg/rtc/mediatracksubscriptions.go @@ -16,6 +16,7 @@ package rtc import ( "errors" + "strings" "sync" "github.com/pion/rtcp" @@ -258,7 +259,7 @@ func (t *MediaTrackSubscriptions) AddSubscriber(sub types.LocalParticipant, wr * Stereo: info.Stereo, Red: !info.DisableRed, } - if addTrackParams.Red && (len(codecs) == 1 && codecs[0].MimeType == webrtc.MimeTypeOpus) { + if addTrackParams.Red && (len(codecs) == 1 && strings.EqualFold(codecs[0].MimeType, webrtc.MimeTypeOpus)) { addTrackParams.Red = false } @@ -354,7 +355,7 @@ func (t *MediaTrackSubscriptions) GetAllSubscribersForMime(mime string) []liveki subs := make([]livekit.ParticipantID, 0, len(t.subscribedTracks)) for id, subTrack := range t.subscribedTracks { - if subTrack.DownTrack().Codec().MimeType != mime { + if !strings.EqualFold(subTrack.DownTrack().Codec().MimeType, mime) { continue } diff --git a/pkg/rtc/wrappedreceiver.go b/pkg/rtc/wrappedreceiver.go index 42fe9d707..1d1a7cd9c 100644 --- a/pkg/rtc/wrappedreceiver.go +++ b/pkg/rtc/wrappedreceiver.go @@ -90,7 +90,7 @@ func (r *WrappedReceiver) StreamID() string { func (r *WrappedReceiver) DetermineReceiver(codec webrtc.RTPCodecCapability) { r.determinedCodec = codec for _, receiver := range r.receivers { - if c := receiver.Codec(); c.MimeType == codec.MimeType { + if c := receiver.Codec(); strings.EqualFold(c.MimeType, codec.MimeType) { r.TrackReceiver = receiver break } else if strings.EqualFold(c.MimeType, sfu.MimeTypeAudioRed) && strings.EqualFold(codec.MimeType, webrtc.MimeTypeOpus) { From 4d7586bda88a0b54aad431cffdc38c064b198358 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sat, 4 May 2024 15:00:19 +0530 Subject: [PATCH 15/60] Simplify layer roll back. (#2702) * Cache TS offset along with sender report. * set offset after clearing * simplify roll back --- pkg/sfu/forwarder.go | 88 ++++++++++--------- pkg/sfu/rtpmunger.go | 8 +- pkg/sfu/videolayerselector/base.go | 14 ++- .../videolayerselector/videolayerselector.go | 2 +- 4 files changed, 55 insertions(+), 57 deletions(-) diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 9fb66e020..feb852642 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -215,6 +215,11 @@ func (f ForwarderState) String() string { // ------------------------------------------------------------------- +type refInfo struct { + senderReport *buffer.RTCPSenderReportData + tsOffset uint64 +} + type Forwarder struct { lock sync.RWMutex codec webrtc.RTPCodecCapability @@ -233,7 +238,7 @@ type Forwarder struct { lastSSRC uint32 referenceLayerSpatial int32 dummyStartTSOffset uint64 - refSenderReports [buffer.DefaultMaxLayerSpatial + 1]*buffer.RTCPSenderReportData + refInfos [buffer.DefaultMaxLayerSpatial + 1]refInfo refIsSVC bool provisional *VideoAllocationProvisional @@ -326,15 +331,17 @@ func (f *Forwarder) DetermineCodec(codec webrtc.RTPCodecCapability, extensions [ f.vls = videolayerselector.NewSimulcast(f.logger) } f.vls.SetTemporalLayerSelector(temporallayerselector.NewVP8(f.logger)) + case "video/h264": if f.vls != nil { f.vls = videolayerselector.NewSimulcastFromNull(f.vls) } else { f.vls = videolayerselector.NewSimulcast(f.logger) } - case "video/vp9": - isDDAvailable := ddAvailable(extensions) + case "video/vp9": + // DD-TODO : we only enable dd layer selector for av1/vp9 now, in the future we can enable it for vp8 too + isDDAvailable := ddAvailable(extensions) if isDDAvailable { if f.vls != nil { f.vls = videolayerselector.NewDependencyDescriptorFromNull(f.vls) @@ -349,9 +356,9 @@ func (f *Forwarder) DetermineCodec(codec webrtc.RTPCodecCapability, extensions [ } } // SVC-TODO: Support for VP9 simulcast. When DD is not available, have to pick selector based on VP9 SVC or Simulcast + case "video/av1": // DD-TODO : we only enable dd layer selector for av1/vp9 now, in the future we can enable it for vp8 too - isDDAvailable := ddAvailable(extensions) if isDDAvailable { if f.vls != nil { @@ -568,8 +575,8 @@ func (f *Forwarder) SetRefSenderReport(isSVC bool, layer int32, srData *buffer.R if isSVC { layer = 0 } - if layer >= 0 && int(layer) < len(f.refSenderReports) { - f.refSenderReports[layer] = srData + if layer >= 0 && int(layer) < len(f.refInfos) { + f.refInfos[layer].senderReport = srData } } @@ -604,7 +611,7 @@ func (f *Forwarder) clearRefSenderReportsLocked() { // By clearing sender report on (re)start of a stream, subscribers will wait for a fresh report // after unmute to send sender report. for layer := int32(0); layer < buffer.DefaultMaxLayerSpatial+1; layer++ { - f.refSenderReports[layer] = nil + f.refInfos[layer] = refInfo{nil, 0} } } @@ -613,20 +620,22 @@ func (f *Forwarder) GetSenderReportParams() (int32, uint64, *buffer.RTCPSenderRe defer f.lock.RUnlock() if f.kind == webrtc.RTPCodecTypeAudio { - return 0, f.rtpMunger.GetPinnedTSOffset(), f.refSenderReports[0] + return 0, f.refInfos[0].tsOffset, f.refInfos[0].senderReport } currentLayerSpatial := f.vls.GetCurrent().Spatial if currentLayerSpatial < 0 || currentLayerSpatial > buffer.DefaultMaxLayerSpatial { - return currentLayerSpatial, f.rtpMunger.GetPinnedTSOffset(), nil + return currentLayerSpatial, 0, nil } - refSenderReport := f.refSenderReports[currentLayerSpatial] + refSenderReport := f.refInfos[currentLayerSpatial].senderReport + tsOffset := f.refInfos[currentLayerSpatial].tsOffset if f.refIsSVC { - refSenderReport = f.refSenderReports[0] + refSenderReport = f.refInfos[0].senderReport + tsOffset = f.refInfos[0].tsOffset } - return currentLayerSpatial, f.rtpMunger.GetPinnedTSOffset(), refSenderReport + return currentLayerSpatial, tsOffset, refSenderReport } func (f *Forwarder) isDeficientLocked() bool { @@ -1534,6 +1543,7 @@ func (f *Forwarder) GetTranslationParams(extPkt *buffer.ExtPacket, layer int32) switch f.kind { case webrtc.RTPCodecTypeAudio: return f.getTranslationParamsAudio(extPkt, layer) + case webrtc.RTPCodecTypeVideo: return f.getTranslationParamsVideo(extPkt, layer) } @@ -1544,7 +1554,7 @@ func (f *Forwarder) GetTranslationParams(extPkt *buffer.ExtPacket, layer int32) } func (f *Forwarder) getReferenceLayerRTPTimestamp(ts uint32, refLayer, targetLayer int32) (uint32, error) { - if refLayer < 0 || int(refLayer) > len(f.refSenderReports) || targetLayer < 0 || int(targetLayer) > len(f.refSenderReports) { + if refLayer < 0 || int(refLayer) > len(f.refInfos) || targetLayer < 0 || int(targetLayer) > len(f.refInfos) { return 0, fmt.Errorf("invalid layer(s), refLayer: %d, targetLayer: %d", refLayer, targetLayer) } @@ -1552,8 +1562,8 @@ func (f *Forwarder) getReferenceLayerRTPTimestamp(ts uint32, refLayer, targetLay return ts, nil } - srRef := f.refSenderReports[refLayer] - srTarget := f.refSenderReports[targetLayer] + srRef := f.refInfos[refLayer].senderReport + srTarget := f.refInfos[targetLayer].senderReport if srRef == nil || srRef.NTPTimestamp == 0 || srTarget == nil || srTarget.NTPTimestamp == 0 { return 0, fmt.Errorf("unavailable layer(s), refLayer: %d, targetLayer: %d", refLayer, targetLayer) } @@ -1576,7 +1586,10 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e f.referenceLayerSpatial = layer f.rtpMunger.SetLastSnTs(extPkt) f.codecMunger.SetLast(extPkt) + f.clearRefSenderReportsLocked() + f.refInfos[layer].tsOffset = f.rtpMunger.GetTSOffset() + f.logger.Debugw( "starting forwarding", "sequenceNumber", extPkt.Packet.SequenceNumber, @@ -1776,16 +1789,18 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e ) f.rtpMunger.UpdateSnTsOffsets(extPkt, 1, extNextTS-extLastTS) + f.refInfos[layer].tsOffset = f.rtpMunger.GetTSOffset() f.codecMunger.UpdateOffsets(extPkt) return nil } // should be called with lock held -func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer int32, tp *TranslationParams) (bool, error) { +func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer int32, tp *TranslationParams) error { if f.lastSSRC != extPkt.Packet.SSRC { if err := f.processSourceSwitch(extPkt, layer); err != nil { tp.shouldDrop = true - return false, nil + f.vls.Rollback() + return nil } f.logger.Debugw("switching feed", "from", f.lastSSRC, "to", extPkt.Packet.SSRC) f.lastSSRC = extPkt.Packet.SSRC @@ -1795,9 +1810,9 @@ func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer i if err != nil { tp.shouldDrop = true if err == ErrPaddingOnlyPacket || err == ErrDuplicatePacket || err == ErrOutOfOrderSequenceNumberCacheMiss { - return false, nil + return nil } - return false, err + return err } tp.rtp = tpRTP @@ -1806,13 +1821,13 @@ func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer i return f.translateCodecHeader(extPkt, tp) } - return false, nil + return nil } // should be called with lock held func (f *Forwarder) getTranslationParamsAudio(extPkt *buffer.ExtPacket, layer int32) (TranslationParams, error) { tp := TranslationParams{} - if _, err := f.getTranslationParamsCommon(extPkt, layer, &tp); err != nil { + if err := f.getTranslationParamsCommon(extPkt, layer, &tp); err != nil { tp.shouldDrop = true return tp, err } @@ -1821,12 +1836,6 @@ func (f *Forwarder) getTranslationParamsAudio(extPkt *buffer.ExtPacket, layer in // should be called with lock held func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer int32) (TranslationParams, error) { - maybeRollback := func(isSwitching bool) { - if isSwitching { - f.vls.Rollback() - } - } - tp := TranslationParams{} if !f.vls.GetTarget().IsValid() { // stream is paused by streamallocator @@ -1852,6 +1861,11 @@ func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer in tp.ddBytes = result.DependencyDescriptorExtension tp.marker = result.RTPMarker + err := f.getTranslationParamsCommon(extPkt, layer, &tp) + if tp.shouldDrop { + return tp, err + } + if FlagPauseOnDowngrade && f.isDeficientLocked() && f.vls.GetTarget().Spatial < f.vls.GetCurrent().Spatial { // // If target layer is lower than both the current and @@ -1872,22 +1886,15 @@ func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer in // To differentiate between the two cases, drop only when in DEFICIENT state. // tp.shouldDrop = true - maybeRollback(result.IsSwitching) return tp, nil } - isTemporalSwitching, err := f.getTranslationParamsCommon(extPkt, layer, &tp) - if tp.shouldDrop { - maybeRollback(result.IsSwitching || isTemporalSwitching) - return tp, err - } - - return tp, err + return tp, nil } -func (f *Forwarder) translateCodecHeader(extPkt *buffer.ExtPacket, tp *TranslationParams) (bool, error) { +func (f *Forwarder) translateCodecHeader(extPkt *buffer.ExtPacket, tp *TranslationParams) error { // codec specific forwarding check and any needed packet munging - tl, isSwitching := f.vls.SelectTemporal(extPkt) + tl := f.vls.SelectTemporal(extPkt) inputSize, codecBytes, err := f.codecMunger.UpdateAndGet( extPkt, tp.rtp.snOrdering == SequenceNumberOrderingOutOfOrder, @@ -1901,15 +1908,14 @@ func (f *Forwarder) translateCodecHeader(extPkt *buffer.ExtPacket, tp *Translati // filtered temporal layer, update sequence number offset to prevent holes f.rtpMunger.PacketDropped(extPkt) } - return isSwitching, nil + return nil } - return isSwitching, err + return err } tp.incomingHeaderSize = inputSize tp.codecBytes = codecBytes - - return isSwitching, nil + return nil } func (f *Forwarder) maybeStart() { diff --git a/pkg/sfu/rtpmunger.go b/pkg/sfu/rtpmunger.go index 47287e84a..0e8fa78ab 100644 --- a/pkg/sfu/rtpmunger.go +++ b/pkg/sfu/rtpmunger.go @@ -83,7 +83,6 @@ type RTPMunger struct { extLastTS uint64 extSecondLastTS uint64 tsOffset uint64 - pinnedTSOffset uint64 lastMarker bool secondLastMarker bool @@ -108,7 +107,6 @@ func (r *RTPMunger) DebugInfo() map[string]interface{} { "ExtLastTS": r.extLastTS, "ExtSecondLastTS": r.extSecondLastTS, "TSOffset": r.tsOffset, - "PinnedTSOffset": r.pinnedTSOffset, "LastMarker": r.lastMarker, "SecondLastMarker": r.secondLastMarker, } @@ -125,8 +123,8 @@ func (r *RTPMunger) GetLast() RTPMungerState { } } -func (r *RTPMunger) GetPinnedTSOffset() uint64 { - return r.pinnedTSOffset +func (r *RTPMunger) GetTSOffset() uint64 { + return r.tsOffset } func (r *RTPMunger) SeedLast(state RTPMungerState) { @@ -149,7 +147,6 @@ func (r *RTPMunger) SetLastSnTs(extPkt *buffer.ExtPacket) { r.extLastTS = extPkt.ExtTimestamp r.extSecondLastTS = extPkt.ExtTimestamp r.tsOffset = 0 - r.pinnedTSOffset = r.tsOffset } func (r *RTPMunger) UpdateSnTsOffsets(extPkt *buffer.ExtPacket, snAdjust uint64, tsAdjust uint64) { @@ -159,7 +156,6 @@ func (r *RTPMunger) UpdateSnTsOffsets(extPkt *buffer.ExtPacket, snAdjust uint64, r.updateSnOffset() r.tsOffset = extPkt.ExtTimestamp - r.extLastTS - tsAdjust - r.pinnedTSOffset = r.tsOffset } func (r *RTPMunger) PacketDropped(extPkt *buffer.ExtPacket) { diff --git a/pkg/sfu/videolayerselector/base.go b/pkg/sfu/videolayerselector/base.go index 08b61e308..91551fbfd 100644 --- a/pkg/sfu/videolayerselector/base.go +++ b/pkg/sfu/videolayerselector/base.go @@ -140,29 +140,25 @@ func (b *Base) Rollback() { b.targetLayer = b.previousTargetLayer } -func (b *Base) SelectTemporal(extPkt *buffer.ExtPacket) (int32, bool) { +func (b *Base) SelectTemporal(extPkt *buffer.ExtPacket) int32 { if b.tls != nil { - isSwitching := false this, next := b.tls.Select(extPkt, b.currentLayer.Temporal, b.targetLayer.Temporal) if next != b.currentLayer.Temporal { - isSwitching = true - - b.previousLayer = b.currentLayer + previousLayer := b.currentLayer b.currentLayer.Temporal = next b.logger.Debugw( "updating temporal layer", - "previous", b.previousLayer, + "previous", previousLayer, "current", b.currentLayer, - "previousTarget", b.previousTargetLayer, "target", b.targetLayer, "max", b.maxLayer, "req", b.requestSpatial, "maxSeen", b.maxSeenLayer, ) } - return this, isSwitching + return this } - return b.currentLayer.Temporal, false + return b.currentLayer.Temporal } diff --git a/pkg/sfu/videolayerselector/videolayerselector.go b/pkg/sfu/videolayerselector/videolayerselector.go index 186c8f64d..46b97fedf 100644 --- a/pkg/sfu/videolayerselector/videolayerselector.go +++ b/pkg/sfu/videolayerselector/videolayerselector.go @@ -55,6 +55,6 @@ type VideoLayerSelector interface { GetCurrent() buffer.VideoLayer Select(extPkt *buffer.ExtPacket, layer int32) VideoLayerSelectorResult - SelectTemporal(extPkt *buffer.ExtPacket) (int32, bool) + SelectTemporal(extPkt *buffer.ExtPacket) int32 Rollback() } From 65f06da1ca027d3bbb7f8138a07e1350bff324c6 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 5 May 2024 21:49:16 -0700 Subject: [PATCH 16/60] Update pion deps (#2697) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 63031c75c..26bf23943 100644 --- a/go.mod +++ b/go.mod @@ -27,8 +27,8 @@ require ( github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1 github.com/mitchellh/go-homedir v1.1.0 github.com/olekukonko/tablewriter v0.0.5 - github.com/pion/dtls/v2 v2.2.10 - github.com/pion/ice/v2 v2.3.20 + github.com/pion/dtls/v2 v2.2.11 + github.com/pion/ice/v2 v2.3.24 github.com/pion/interceptor v0.1.29 github.com/pion/rtcp v1.2.14 github.com/pion/rtp v1.8.6 @@ -36,7 +36,7 @@ require ( github.com/pion/sdp/v3 v3.0.9 github.com/pion/transport/v2 v2.2.5 github.com/pion/turn/v2 v2.1.6 - github.com/pion/webrtc/v3 v3.2.39 + github.com/pion/webrtc/v3 v3.2.40 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.19.0 github.com/redis/go-redis/v9 v9.5.1 diff --git a/go.sum b/go.sum index e8190e1a3..08f1961f4 100644 --- a/go.sum +++ b/go.sum @@ -166,10 +166,10 @@ github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8P github.com/pion/datachannel v1.5.5 h1:10ef4kwdjije+M9d7Xm9im2Y3O6A6ccQb0zcqZcJew8= github.com/pion/datachannel v1.5.5/go.mod h1:iMz+lECmfdCMqFRhXhcA/219B0SQlbpoR2V118yimL0= github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= -github.com/pion/dtls/v2 v2.2.10 h1:u2Axk+FyIR1VFTPurktB+1zoEPGIW3bmyj3LEFrXjAA= -github.com/pion/dtls/v2 v2.2.10/go.mod h1:d9SYc9fch0CqK90mRk1dC7AkzzpwJj6u2GU3u+9pqFE= -github.com/pion/ice/v2 v2.3.20 h1:V6TN3E/B0Xh4nRjX4bbRlyyxsbF2Lv8NNdRidZjCxvc= -github.com/pion/ice/v2 v2.3.20/go.mod h1:KXJJcZK7E8WzrBEYnV4UtqEZsGeWfHxsNqhVcVvgjxw= +github.com/pion/dtls/v2 v2.2.11 h1:9U/dpCYl1ySttROPWJgqWKEylUdT0fXp/xst6JwY5Ks= +github.com/pion/dtls/v2 v2.2.11/go.mod h1:d9SYc9fch0CqK90mRk1dC7AkzzpwJj6u2GU3u+9pqFE= +github.com/pion/ice/v2 v2.3.24 h1:RYgzhH/u5lH0XO+ABatVKCtRd+4U1GEaCXSMjNr13tI= +github.com/pion/ice/v2 v2.3.24/go.mod h1:KXJJcZK7E8WzrBEYnV4UtqEZsGeWfHxsNqhVcVvgjxw= github.com/pion/interceptor v0.1.29 h1:39fsnlP1U8gw2JzOFWdfCU82vHvhW9o0rZnZF56wF+M= github.com/pion/interceptor v0.1.29/go.mod h1:ri+LGNjRUc5xUNtDEPzfdkmSqISixVTBF/z/Zms/6T4= github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= @@ -207,8 +207,8 @@ github.com/pion/transport/v3 v3.0.2/go.mod h1:nIToODoOlb5If2jF9y2Igfx3PFYWfuXi37 github.com/pion/turn/v2 v2.1.3/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= github.com/pion/turn/v2 v2.1.6 h1:Xr2niVsiPTB0FPtt+yAWKFUkU1eotQbGgpTIld4x1Gc= github.com/pion/turn/v2 v2.1.6/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= -github.com/pion/webrtc/v3 v3.2.39 h1:Lf2SIMGdE3M9VNm48KpoX5pR8SJ6TsMnktzOkc/oB0o= -github.com/pion/webrtc/v3 v3.2.39/go.mod h1:AQ8p56OLbm3MjhYovYdgPuyX6oc+JcKx/HFoCGFcYzA= +github.com/pion/webrtc/v3 v3.2.40 h1:Wtfi6AZMQg+624cvCXUuSmrKWepSB7zfgYDOYqsSOVU= +github.com/pion/webrtc/v3 v3.2.40/go.mod h1:M1RAe3TNTD1tzyvqHrbVODfwdPGSXOUo/OgpoGGJqFY= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= From 9a5db132eb9586b5df785f35eda6c562a87cbfeb Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Mon, 6 May 2024 17:25:18 -0700 Subject: [PATCH 17/60] add room/participant name limit (#2704) * add room/participant name limit * defaults * simplify * omitempty * handle 0 config * fix race * unlock * tidy --- pkg/config/config.go | 26 ++++++++------ pkg/service/errors.go | 40 ++++++++++++---------- pkg/service/roomservice.go | 5 +++ pkg/service/rtcservice.go | 6 ++++ pkg/service/wsprotocol.go | 6 ++-- pkg/sfu/streamallocator/streamallocator.go | 7 ++-- pkg/telemetry/telemetryservice.go | 25 +++++--------- 7 files changed, 63 insertions(+), 52 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 78cd0f1e1..67006eb33 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -229,15 +229,17 @@ type VideoConfig struct { type RoomConfig struct { // enable rooms to be automatically created - AutoCreate bool `yaml:"auto_create,omitempty"` - EnabledCodecs []CodecSpec `yaml:"enabled_codecs,omitempty"` - MaxParticipants uint32 `yaml:"max_participants,omitempty"` - EmptyTimeout uint32 `yaml:"empty_timeout,omitempty"` - DepartureTimeout uint32 `yaml:"departure_timeout,omitempty"` - EnableRemoteUnmute bool `yaml:"enable_remote_unmute,omitempty"` - MaxMetadataSize uint32 `yaml:"max_metadata_size,omitempty"` - PlayoutDelay PlayoutDelayConfig `yaml:"playout_delay,omitempty"` - SyncStreams bool `yaml:"sync_streams,omitempty"` + AutoCreate bool `yaml:"auto_create,omitempty"` + EnabledCodecs []CodecSpec `yaml:"enabled_codecs,omitempty"` + MaxParticipants uint32 `yaml:"max_participants,omitempty"` + EmptyTimeout uint32 `yaml:"empty_timeout,omitempty"` + DepartureTimeout uint32 `yaml:"departure_timeout,omitempty"` + EnableRemoteUnmute bool `yaml:"enable_remote_unmute,omitempty"` + MaxMetadataSize uint32 `yaml:"max_metadata_size,omitempty"` + PlayoutDelay PlayoutDelayConfig `yaml:"playout_delay,omitempty"` + SyncStreams bool `yaml:"sync_streams,omitempty"` + MaxRoomNameLength int `yaml:"max_room_name_length,omitempty"` + MaxParticipantIdentityLength int `yaml:"max_participant_identity_length,omitempty"` } type CodecSpec struct { @@ -484,8 +486,10 @@ var DefaultConfig = Config{ {Mime: webrtc.MimeTypeVP9}, {Mime: webrtc.MimeTypeAV1}, }, - EmptyTimeout: 5 * 60, - DepartureTimeout: 20, + EmptyTimeout: 5 * 60, + DepartureTimeout: 20, + MaxRoomNameLength: 256, + MaxParticipantIdentityLength: 256, }, Logging: LoggingConfig{ PionLevel: "error", diff --git a/pkg/service/errors.go b/pkg/service/errors.go index 9a77f5fde..26f24389d 100644 --- a/pkg/service/errors.go +++ b/pkg/service/errors.go @@ -19,23 +19,25 @@ import ( ) var ( - ErrEgressNotFound = psrpc.NewErrorf(psrpc.NotFound, "egress does not exist") - ErrEgressNotConnected = psrpc.NewErrorf(psrpc.Internal, "egress not connected (redis required)") - ErrIdentityEmpty = psrpc.NewErrorf(psrpc.InvalidArgument, "identity cannot be empty") - ErrIngressNotConnected = psrpc.NewErrorf(psrpc.Internal, "ingress not connected (redis required)") - ErrIngressNotFound = psrpc.NewErrorf(psrpc.NotFound, "ingress does not exist") - ErrIngressNonReusable = psrpc.NewErrorf(psrpc.InvalidArgument, "ingress is not reusable and cannot be modified") - ErrMetadataExceedsLimits = psrpc.NewErrorf(psrpc.InvalidArgument, "metadata size exceeds limits") - ErrOperationFailed = psrpc.NewErrorf(psrpc.Internal, "operation cannot be completed") - ErrParticipantNotFound = psrpc.NewErrorf(psrpc.NotFound, "participant does not exist") - ErrRoomNotFound = psrpc.NewErrorf(psrpc.NotFound, "requested room does not exist") - ErrRoomLockFailed = psrpc.NewErrorf(psrpc.Internal, "could not lock room") - ErrRoomUnlockFailed = psrpc.NewErrorf(psrpc.Internal, "could not unlock room, lock token does not match") - ErrRemoteUnmuteNoteEnabled = psrpc.NewErrorf(psrpc.FailedPrecondition, "remote unmute not enabled") - ErrTrackNotFound = psrpc.NewErrorf(psrpc.NotFound, "track is not found") - ErrWebHookMissingAPIKey = psrpc.NewErrorf(psrpc.InvalidArgument, "api_key is required to use webhooks") - ErrSIPNotConnected = psrpc.NewErrorf(psrpc.Internal, "sip not connected (redis required)") - ErrSIPTrunkNotFound = psrpc.NewErrorf(psrpc.NotFound, "requested sip trunk does not exist") - ErrSIPDispatchRuleNotFound = psrpc.NewErrorf(psrpc.NotFound, "requested sip dispatch rule does not exist") - ErrSIPParticipantNotFound = psrpc.NewErrorf(psrpc.NotFound, "requested sip participant does not exist") + ErrEgressNotFound = psrpc.NewErrorf(psrpc.NotFound, "egress does not exist") + ErrEgressNotConnected = psrpc.NewErrorf(psrpc.Internal, "egress not connected (redis required)") + ErrIdentityEmpty = psrpc.NewErrorf(psrpc.InvalidArgument, "identity cannot be empty") + ErrIngressNotConnected = psrpc.NewErrorf(psrpc.Internal, "ingress not connected (redis required)") + ErrIngressNotFound = psrpc.NewErrorf(psrpc.NotFound, "ingress does not exist") + ErrIngressNonReusable = psrpc.NewErrorf(psrpc.InvalidArgument, "ingress is not reusable and cannot be modified") + ErrMetadataExceedsLimits = psrpc.NewErrorf(psrpc.InvalidArgument, "metadata size exceeds limits") + ErrRoomNameExceedsLimits = psrpc.NewErrorf(psrpc.InvalidArgument, "room name length exceeds limits") + ErrParticipantIdentityExceedsLimits = psrpc.NewErrorf(psrpc.InvalidArgument, "participant identity length exceeds limits") + ErrOperationFailed = psrpc.NewErrorf(psrpc.Internal, "operation cannot be completed") + ErrParticipantNotFound = psrpc.NewErrorf(psrpc.NotFound, "participant does not exist") + ErrRoomNotFound = psrpc.NewErrorf(psrpc.NotFound, "requested room does not exist") + ErrRoomLockFailed = psrpc.NewErrorf(psrpc.Internal, "could not lock room") + ErrRoomUnlockFailed = psrpc.NewErrorf(psrpc.Internal, "could not unlock room, lock token does not match") + ErrRemoteUnmuteNoteEnabled = psrpc.NewErrorf(psrpc.FailedPrecondition, "remote unmute not enabled") + ErrTrackNotFound = psrpc.NewErrorf(psrpc.NotFound, "track is not found") + ErrWebHookMissingAPIKey = psrpc.NewErrorf(psrpc.InvalidArgument, "api_key is required to use webhooks") + ErrSIPNotConnected = psrpc.NewErrorf(psrpc.Internal, "sip not connected (redis required)") + ErrSIPTrunkNotFound = psrpc.NewErrorf(psrpc.NotFound, "requested sip trunk does not exist") + ErrSIPDispatchRuleNotFound = psrpc.NewErrorf(psrpc.NotFound, "requested sip dispatch rule does not exist") + ErrSIPParticipantNotFound = psrpc.NewErrorf(psrpc.NotFound, "requested sip participant does not exist") ) diff --git a/pkg/service/roomservice.go b/pkg/service/roomservice.go index 319e06cb5..54ab2c1c9 100644 --- a/pkg/service/roomservice.go +++ b/pkg/service/roomservice.go @@ -16,6 +16,7 @@ package service import ( "context" + "fmt" "strconv" "github.com/avast/retry-go/v4" @@ -83,6 +84,10 @@ func (s *RoomService) CreateRoom(ctx context.Context, req *livekit.CreateRoomReq return nil, ErrEgressNotConnected } + if limit := s.roomConf.MaxRoomNameLength; limit > 0 && len(req.Name) > limit { + return nil, fmt.Errorf("%w: max length %d", ErrRoomNameExceedsLimits, limit) + } + rm, created, err := s.roomAllocator.CreateRoom(ctx, req) if err != nil { err = errors.Wrap(err, "could not create room") diff --git a/pkg/service/rtcservice.go b/pkg/service/rtcservice.go index efe314c5a..c20db4302 100644 --- a/pkg/service/rtcservice.go +++ b/pkg/service/rtcservice.go @@ -120,6 +120,9 @@ func (s *RTCService) validate(r *http.Request) (livekit.RoomName, routing.Partic if claims.Identity == "" { return "", pi, http.StatusBadRequest, ErrIdentityEmpty } + if limit := s.config.Room.MaxParticipantIdentityLength; limit > 0 && len(claims.Identity) > limit { + return "", pi, http.StatusBadRequest, fmt.Errorf("%w: max length %d", ErrParticipantIdentityExceedsLimits, limit) + } roomName := livekit.RoomName(r.FormValue("room")) reconnectParam := r.FormValue("reconnect") @@ -133,6 +136,9 @@ func (s *RTCService) validate(r *http.Request) (livekit.RoomName, routing.Partic if onlyName != "" { roomName = onlyName } + if limit := s.config.Room.MaxRoomNameLength; limit > 0 && len(roomName) > limit { + return "", pi, http.StatusBadRequest, fmt.Errorf("%w: max length %d", ErrRoomNameExceedsLimits, limit) + } // this is new connection for existing participant - with publish only permissions if publishParam != "" { diff --git a/pkg/service/wsprotocol.go b/pkg/service/wsprotocol.go index 5ac7c744f..d94ce3f90 100644 --- a/pkg/service/wsprotocol.go +++ b/pkg/service/wsprotocol.go @@ -162,8 +162,10 @@ func (c *WSSignalConnection) WriteServerMessage(msg *livekit.ServerMessage) (int } func (c *WSSignalConnection) pingWorker() { - for { - <-time.After(pingFrequency) + ticker := time.NewTicker(pingFrequency) + defer ticker.Stop() + + for range ticker.C { err := c.conn.WriteControl(websocket.PingMessage, []byte(""), time.Now().Add(pingTimeout)) if err != nil { return diff --git a/pkg/sfu/streamallocator/streamallocator.go b/pkg/sfu/streamallocator/streamallocator.go index e68d51717..87092b434 100644 --- a/pkg/sfu/streamallocator/streamallocator.go +++ b/pkg/sfu/streamallocator/streamallocator.go @@ -698,13 +698,12 @@ func (s *StreamAllocator) handleSignalProbeClusterDone(event Event) { func (s *StreamAllocator) handleSignalResume(event Event) { s.videoTracksMu.Lock() track := s.videoTracks[event.TrackID] + updated := track != nil && track.SetStreamState(StreamStateActive) s.videoTracksMu.Unlock() - if track != nil { + if updated { update := NewStreamStateUpdate() - if track.SetStreamState(StreamStateActive) { - update.HandleStreamingChange(track, StreamStateActive) - } + update.HandleStreamingChange(track, StreamStateActive) s.maybeSendUpdate(update) } } diff --git a/pkg/telemetry/telemetryservice.go b/pkg/telemetry/telemetryservice.go index e20e2a543..8af2fb6c4 100644 --- a/pkg/telemetry/telemetryservice.go +++ b/pkg/telemetry/telemetryservice.go @@ -129,8 +129,9 @@ func (t *telemetryService) FlushStats() { t.workersMu.RUnlock() now := time.Now() - var prev, reapHead, reapTail *StatsWorker + var prev, reap *StatsWorker for worker != nil { + next := worker.next if closed := worker.Flush(now); closed { if prev == nil { // this worker was at the head of the list @@ -148,27 +149,19 @@ func (t *telemetryService) FlushStats() { prev.next = worker.next } - if reapHead == nil { - reapHead = worker - reapTail = worker - } else { - reapTail.next = worker - reapTail = worker - } + worker.next = reap + reap = worker } else { prev = worker } - worker = worker.next + worker = next } - if reapHead != nil { + if reap != nil { t.workersMu.Lock() - for { - delete(t.workers, reapHead.participantID) - if reapHead == reapTail { - break - } - reapHead = reapHead.next + for reap != nil { + delete(t.workers, reap.participantID) + reap = reap.next } t.workersMu.Unlock() } From 71b5ffed9380abb9d2766e6e5f1c52e4d056754b Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 8 May 2024 16:10:49 +0530 Subject: [PATCH 18/60] Less confusing variable name (#2706) --- pkg/sfu/connectionquality/scorer.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/sfu/connectionquality/scorer.go b/pkg/sfu/connectionquality/scorer.go index facef770a..7364e3dc3 100644 --- a/pkg/sfu/connectionquality/scorer.go +++ b/pkg/sfu/connectionquality/scorer.go @@ -124,18 +124,18 @@ func (w *windowStat) calculatePacketScore(plw float64, includeRTT bool, includeJ return score } -func (w *windowStat) calculateBitrateScore(expectedBitrate int64, isEnabled bool) float64 { - if expectedBitrate == 0 || !isEnabled { +func (w *windowStat) calculateBitrateScore(expectedBits int64, isEnabled bool) float64 { + if expectedBits == 0 || !isEnabled { // unsupported mode OR all layers stopped return cMaxScore } var score float64 if w.bytes != 0 { - // using the ratio of expectedBitrate / actualBitrate + // using the ratio of expectedBits / actualBits // the quality inflection points are approximately // GOOD at ~2.7x, POOR at ~20.1x - score = cMaxScore - 20*math.Log(float64(expectedBitrate)/float64(w.bytes*8)) + score = cMaxScore - 20*math.Log(float64(expectedBits)/float64(w.bytes*8)) if score > cMaxScore { score = cMaxScore } @@ -369,7 +369,7 @@ func (q *qualityScorer) AddLayerTransition(distance float64) { func (q *qualityScorer) updateAtLocked(stat *windowStat, at time.Time) { // always update transitions - expectedBitrate, _, err := q.aggregateBitrate.GetAggregateAndRestartAt(at) + expectedBits, _, err := q.aggregateBitrate.GetAggregateAndRestartAt(at) if err != nil { q.params.Logger.Warnw("error getting expected bitrate", err) } @@ -405,7 +405,7 @@ func (q *qualityScorer) updateAtLocked(stat *windowStat, at time.Time) { } } else { packetScore := stat.calculatePacketScore(plw, q.params.IncludeRTT, q.params.IncludeJitter) - bitrateScore := stat.calculateBitrateScore(expectedBitrate, q.params.EnableBitrateScore) + bitrateScore := stat.calculateBitrateScore(expectedBits, q.params.EnableBitrateScore) layerScore := math.Max(math.Min(cMaxScore, cMaxScore-(expectedDistance*distanceWeight)), 0.0) minScore := math.Min(packetScore, bitrateScore) @@ -451,7 +451,7 @@ func (q *qualityScorer) updateAtLocked(stat *windowStat, at time.Time) { "stat", stat, "packetLossWeight", plw, "maxPPS", q.maxPPS, - "expectedBitrate", expectedBitrate, + "expectedBits", expectedBits, "expectedDistance", expectedDistance, ) } From b54acb0bad6447623996993609974c1a6c9f2a0f Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Thu, 9 May 2024 00:06:41 -0700 Subject: [PATCH 19/60] ensure room is running before attempting to delete (#2705) * ignore api errors when deleting room * start room before deletion * delete from db first * load --- pkg/service/roomservice.go | 39 +++++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/pkg/service/roomservice.go b/pkg/service/roomservice.go index 54ab2c1c9..c138727a4 100644 --- a/pkg/service/roomservice.go +++ b/pkg/service/roomservice.go @@ -29,7 +29,6 @@ import ( "github.com/livekit/livekit-server/pkg/rtc" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/rpc" - "github.com/livekit/psrpc" ) // A rooms service that supports a single node @@ -94,16 +93,11 @@ func (s *RoomService) CreateRoom(ctx context.Context, req *livekit.CreateRoomReq return nil, err } - // actually start the room on an RTC node, to ensure metadata & empty timeout functionality - res, err := s.router.StartParticipantSignal(ctx, - livekit.RoomName(req.Name), - routing.ParticipantInit{}, - ) + done, err := s.startRoom(ctx, livekit.RoomName(req.Name)) if err != nil { return nil, err } - defer res.RequestSink.Close() - defer res.ResponseSource.Close() + defer done() if created { go s.agentClient.LaunchJob(ctx, &agent.JobDescription{ @@ -158,9 +152,20 @@ func (s *RoomService) DeleteRoom(ctx context.Context, req *livekit.DeleteRoomReq return nil, twirpAuthError(err) } - _, err := s.roomClient.DeleteRoom(ctx, s.topicFormatter.RoomTopic(ctx, livekit.RoomName(req.Room)), req) - if !errors.Is(err, psrpc.ErrNoResponse) { - return &livekit.DeleteRoomResponse{}, err + _, _, err := s.roomStore.LoadRoom(ctx, livekit.RoomName(req.Room), false) + if err != nil { + return nil, err + } + + done, err := s.startRoom(ctx, livekit.RoomName(req.Room)) + if err != nil { + return nil, err + } + defer done() + + _, err = s.roomClient.DeleteRoom(ctx, s.topicFormatter.RoomTopic(ctx, livekit.RoomName(req.Room)), req) + if err != nil { + return nil, err } err = s.roomStore.DeleteRoom(ctx, livekit.RoomName(req.Room)) @@ -325,3 +330,15 @@ func (s *RoomService) confirmExecution(ctx context.Context, f func() error) erro retry.DelayType(retry.BackOffDelay), ) } + +// startRoom starts the room on an RTC node, to ensure metadata & empty timeout functionality +func (s *RoomService) startRoom(ctx context.Context, roomName livekit.RoomName) (func(), error) { + res, err := s.router.StartParticipantSignal(ctx, roomName, routing.ParticipantInit{}) + if err != nil { + return nil, err + } + return func() { + res.RequestSink.Close() + res.ResponseSource.Close() + }, nil +} From 66a3a8e02815d40a14cda93c5536f0f6c06a50a7 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 10 May 2024 12:32:28 +0530 Subject: [PATCH 20/60] cond broadcast always. (#2699) With Read and ReadExtended waiting (they are two different goroutines), use Broadcast always. In theory, they both should not be waiting at the same time, but just being safe. --- pkg/sfu/buffer/buffer.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go index 64ff28917..5474c2664 100644 --- a/pkg/sfu/buffer/buffer.go +++ b/pkg/sfu/buffer/buffer.go @@ -336,14 +336,14 @@ func (b *Buffer) Write(pkt []byte) (n int, err error) { arrivalTime: now, }) b.Unlock() - b.readCond.Signal() + b.readCond.Broadcast() return } b.payloadType = rtpPacket.PayloadType b.calc(pkt, &rtpPacket, time.Now(), false) b.Unlock() - b.readCond.Signal() + b.readCond.Broadcast() return } From 80a4d021b95fe08987eeae3f769499f3860c44b7 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 10 May 2024 20:02:25 +0530 Subject: [PATCH 21/60] Add simulate leave request close reason (#2713) --- pkg/rtc/types/interfaces.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index 3b230b0ee..01b874c62 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -97,6 +97,7 @@ const ( ParticipantCloseReasonSimulateMigration ParticipantCloseReasonSimulateNodeFailure ParticipantCloseReasonSimulateServerLeave + ParticipantCloseReasonSimulateLeaveRequest ParticipantCloseReasonNegotiateFailed ParticipantCloseReasonMigrationRequested ParticipantCloseReasonPublicationError @@ -140,6 +141,8 @@ func (p ParticipantCloseReason) String() string { return "SIMULATE_NODE_FAILURE" case ParticipantCloseReasonSimulateServerLeave: return "SIMULATE_SERVER_LEAVE" + case ParticipantCloseReasonSimulateLeaveRequest: + return "SIMULATE_LEAVE_REQUEST" case ParticipantCloseReasonNegotiateFailed: return "NEGOTIATE_FAILED" case ParticipantCloseReasonMigrationRequested: @@ -161,7 +164,7 @@ func (p ParticipantCloseReason) String() string { func (p ParticipantCloseReason) ToDisconnectReason() livekit.DisconnectReason { switch p { - case ParticipantCloseReasonClientRequestLeave: + case ParticipantCloseReasonClientRequestLeave, ParticipantCloseReasonSimulateLeaveRequest: return livekit.DisconnectReason_CLIENT_INITIATED case ParticipantCloseReasonRoomManagerStop: return livekit.DisconnectReason_SERVER_SHUTDOWN From eef3cf0f010a14baafdfc25c2314eed694f56f23 Mon Sep 17 00:00:00 2001 From: Benjamin Pracht Date: Fri, 10 May 2024 10:05:53 -0700 Subject: [PATCH 22/60] Redact egress object in CreateRoom request (#2710) --- go.mod | 2 +- go.sum | 4 ++-- pkg/service/roomservice.go | 27 ++++++++++++++++++++++++++- 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 26bf23943..75b6c9839 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20240416023643-881d3dc5423e - github.com/livekit/protocol v1.14.1-0.20240426104403-e7962f444464 + github.com/livekit/protocol v1.15.1-0.20240510165606-93a26f478d00 github.com/livekit/psrpc v0.5.3-0.20240426045048-8ba067a45715 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 diff --git a/go.sum b/go.sum index 08f1961f4..d0e7da290 100644 --- a/go.sum +++ b/go.sum @@ -120,8 +120,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20240416023643-881d3dc5423e h1:ss4VwrouYiDpuNJ9BUTH+WsW+GDdJS70iZp8ii3/0Lc= github.com/livekit/mediatransportutil v0.0.0-20240416023643-881d3dc5423e/go.mod h1:jwKUCmObuiEDH0iiuJHaGMXwRs3RjrB4G6qqgkr/5oE= -github.com/livekit/protocol v1.14.1-0.20240426104403-e7962f444464 h1:5IxCPDkibpvnAYN6+djltH6Gj4dMOL0hNecHn5jZKmk= -github.com/livekit/protocol v1.14.1-0.20240426104403-e7962f444464/go.mod h1:pnn0Dv+/0K0OFqKHX6J6SreYO1dZxl6tDuAZ1ns8L/w= +github.com/livekit/protocol v1.15.1-0.20240510165606-93a26f478d00 h1:c5VOR2XrAgxjwvWpQIA0lDUX+YcpxGzxXtaRfhu510E= +github.com/livekit/protocol v1.15.1-0.20240510165606-93a26f478d00/go.mod h1:pnn0Dv+/0K0OFqKHX6J6SreYO1dZxl6tDuAZ1ns8L/w= github.com/livekit/psrpc v0.5.3-0.20240426045048-8ba067a45715 h1:vhDMOe8fxEc/amYTFo799LySPM12Fk3vc+Nc6o4gYZQ= github.com/livekit/psrpc v0.5.3-0.20240426045048-8ba067a45715/go.mod h1:CQUBSPfYYAaevg1TNCc6/aYsa8DJH4jSRFdCeSZk5u0= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= diff --git a/pkg/service/roomservice.go b/pkg/service/roomservice.go index c138727a4..285c025d9 100644 --- a/pkg/service/roomservice.go +++ b/pkg/service/roomservice.go @@ -22,11 +22,13 @@ import ( "github.com/avast/retry-go/v4" "github.com/pkg/errors" "github.com/twitchtv/twirp" + "google.golang.org/protobuf/proto" "github.com/livekit/livekit-server/pkg/agent" "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/routing" "github.com/livekit/livekit-server/pkg/rtc" + "github.com/livekit/protocol/egress" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/rpc" ) @@ -76,7 +78,9 @@ func NewRoomService( } func (s *RoomService) CreateRoom(ctx context.Context, req *livekit.CreateRoomRequest) (*livekit.Room, error) { - AppendLogFields(ctx, "room", req.Name, "request", req) + clone := redactCreateRoomRequest(req) + + AppendLogFields(ctx, "room", clone.Name, "request", clone) if err := EnsureCreatePermission(ctx); err != nil { return nil, twirpAuthError(err) } else if req.Egress != nil && s.egressLauncher == nil { @@ -342,3 +346,24 @@ func (s *RoomService) startRoom(ctx context.Context, roomName livekit.RoomName) res.ResponseSource.Close() }, nil } + +func redactCreateRoomRequest(req *livekit.CreateRoomRequest) *livekit.CreateRoomRequest { + if req.Egress == nil { + // nothing to redact + return req + } + + clone := proto.Clone(req).(*livekit.CreateRoomRequest) + + if clone.Egress.Room != nil { + egress.RedactEncodedOutputs(clone.Egress.Room) + } + if clone.Egress.Participant != nil { + egress.RedactAutoEncodedOutput(clone.Egress.Participant) + } + if clone.Egress.Tracks != nil { + egress.RedactUpload(clone.Egress.Tracks) + } + + return clone +} From 9887d32bd23fd79a840a6925fb08f3773076489e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Sat, 11 May 2024 13:39:17 +0200 Subject: [PATCH 23/60] fix: connection reset without any closing handshake on clientside (#2709) --- pkg/service/rtcservice.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/service/rtcservice.go b/pkg/service/rtcservice.go index c20db4302..eaf0b2e87 100644 --- a/pkg/service/rtcservice.go +++ b/pkg/service/rtcservice.go @@ -317,6 +317,8 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { defer func() { // when the source is terminated, this means Participant.Close had been called and RTC connection is done // we would terminate the signal connection as well + closeMsg := websocket.FormatCloseMessage(websocket.CloseNormalClosure, "") + _ = conn.WriteControl(websocket.CloseMessage, closeMsg, time.Now().Add(time.Second)) _ = conn.Close() }() defer func() { From 91520a36e0fe67256d9c1ff060f4944312494e6b Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 13 May 2024 15:11:28 +0530 Subject: [PATCH 24/60] Add a flag to pass through timestamp. (#2714) * Add a flag to psss through timestamp. Can make it a config later if needed. * log both adjusted and non-adjusted --- pkg/sfu/buffer/rtpstats_sender.go | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/pkg/sfu/buffer/rtpstats_sender.go b/pkg/sfu/buffer/rtpstats_sender.go index 6dafb1047..6f32ac0a7 100644 --- a/pkg/sfu/buffer/rtpstats_sender.go +++ b/pkg/sfu/buffer/rtpstats_sender.go @@ -31,6 +31,8 @@ const ( cSnInfoMask = cSnInfoSize - 1 cSenderReportInitialWait = time.Second + + cPassthroughNTPTimestamp = true ) // ------------------------------------------------------------------- @@ -645,10 +647,19 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, publisherSRData *RTCPS return nil } - timeSincePublisherSR := time.Since(publisherSRData.AtAdjusted) - now := publisherSRData.AtAdjusted.Add(timeSincePublisherSR) - nowNTP := mediatransportutil.ToNtpTime(now) - nowRTPExt := publisherSRData.RTPTimestampExt - tsOffset + uint64(timeSincePublisherSR.Nanoseconds()*int64(r.params.ClockRate)/1e9) + timeSincePublisherSRAdjusted := time.Since(publisherSRData.AtAdjusted) + now := publisherSRData.AtAdjusted.Add(timeSincePublisherSRAdjusted) + var ( + nowNTP mediatransportutil.NtpTime + nowRTPExt uint64 + ) + if cPassthroughNTPTimestamp { + nowNTP = publisherSRData.NTPTimestamp + nowRTPExt = publisherSRData.RTPTimestampExt - tsOffset + } else { + nowNTP = mediatransportutil.ToNtpTime(now) + nowRTPExt = publisherSRData.RTPTimestampExt - tsOffset + uint64(timeSincePublisherSRAdjusted.Nanoseconds()*int64(r.params.ClockRate)/1e9) + } srData := &RTCPSenderReportData{ NTPTimestamp: nowNTP, @@ -666,13 +677,15 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, publisherSRData *RTCPS "feed", publisherSRData, "tsOffset", tsOffset, "timeNow", time.Now().String(), + "now", now.String(), "extStartTS", r.extStartTS, "extHighestTS", r.extHighestTS, "highestTime", r.highestTime.String(), "timeSinceHighest", now.Sub(r.highestTime).String(), "firstTime", r.firstTime.String(), "timeSinceFirst", now.Sub(r.firstTime).String(), - "timeSincePublisherSR", timeSincePublisherSR.String(), + "timeSincePublisherSRAdjusted", timeSincePublisherSRAdjusted.String(), + "timeSincePublisherSR", time.Since(publisherSRData.At).String(), "nowRTPExt", nowRTPExt, } } From ef6f205fccccc4a53cf75ff33983a893295bf098 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 15 May 2024 11:41:37 +0530 Subject: [PATCH 25/60] Pass through timestamp in abs capture time (#2715) --- pkg/sfu/buffer/rtpstats_base.go | 6 ++++++ pkg/sfu/buffer/rtpstats_sender.go | 2 -- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/sfu/buffer/rtpstats_base.go b/pkg/sfu/buffer/rtpstats_base.go index eba2425b9..af8ecfe21 100644 --- a/pkg/sfu/buffer/rtpstats_base.go +++ b/pkg/sfu/buffer/rtpstats_base.go @@ -35,6 +35,8 @@ const ( cFirstPacketTimeAdjustWindow = 2 * time.Minute cFirstPacketTimeAdjustThreshold = 15 * time.Second + + cPassthroughNTPTimestamp = true ) // ------------------------------------------------------- @@ -118,6 +120,10 @@ type RTCPSenderReportData struct { } func (r *RTCPSenderReportData) PropagationDelay() time.Duration { + if cPassthroughNTPTimestamp { + return 0 + } + return r.AtAdjusted.Sub(r.NTPTimestamp.Time()) } diff --git a/pkg/sfu/buffer/rtpstats_sender.go b/pkg/sfu/buffer/rtpstats_sender.go index 6f32ac0a7..3074803d0 100644 --- a/pkg/sfu/buffer/rtpstats_sender.go +++ b/pkg/sfu/buffer/rtpstats_sender.go @@ -31,8 +31,6 @@ const ( cSnInfoMask = cSnInfoSize - 1 cSenderReportInitialWait = time.Second - - cPassthroughNTPTimestamp = true ) // ------------------------------------------------------------------- From 86b15c2e82302f968922fda61602572387312db7 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 22 May 2024 10:00:11 -0700 Subject: [PATCH 26/60] Update go deps (#2624) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 26 +++++++++++++------------- go.sum | 51 ++++++++++++++++++++++++++------------------------- 2 files changed, 39 insertions(+), 38 deletions(-) diff --git a/go.mod b/go.mod index 75b6c9839..c52893348 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/livekit/livekit-server go 1.22 require ( - github.com/avast/retry-go/v4 v4.5.1 + github.com/avast/retry-go/v4 v4.6.0 github.com/bep/debounce v1.2.1 github.com/d5/tengo/v2 v2.17.0 github.com/dustin/go-humanize v1.0.1 @@ -38,20 +38,20 @@ require ( github.com/pion/turn/v2 v2.1.6 github.com/pion/webrtc/v3 v3.2.40 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.19.0 + github.com/prometheus/client_golang v1.19.1 github.com/redis/go-redis/v9 v9.5.1 - github.com/rs/cors v1.10.1 + github.com/rs/cors v1.11.0 github.com/stretchr/testify v1.9.0 github.com/thoas/go-funk v0.9.3 github.com/twitchtv/twirp v8.1.3+incompatible github.com/ua-parser/uap-go v0.0.0-20240113215029-33f8e6d47f38 - github.com/urfave/cli/v2 v2.27.1 + github.com/urfave/cli/v2 v2.27.2 github.com/urfave/negroni/v3 v3.1.0 go.uber.org/atomic v1.11.0 go.uber.org/zap v1.27.0 - golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f + golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 golang.org/x/sync v0.7.0 - google.golang.org/protobuf v1.33.0 + google.golang.org/protobuf v1.34.1 gopkg.in/yaml.v3 v3.0.1 ) @@ -59,7 +59,7 @@ require ( github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/eapache/channels v1.1.0 // indirect @@ -95,16 +95,16 @@ require ( github.com/prometheus/procfs v0.12.0 // indirect github.com/puzpuzpuz/xsync/v3 v3.1.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect + github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap/exp v0.2.0 // indirect - golang.org/x/crypto v0.22.0 // indirect + golang.org/x/crypto v0.23.0 // indirect golang.org/x/mod v0.17.0 // indirect - golang.org/x/net v0.24.0 // indirect - golang.org/x/sys v0.19.0 // indirect - golang.org/x/text v0.14.0 // indirect - golang.org/x/tools v0.20.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect + golang.org/x/tools v0.21.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be // indirect google.golang.org/grpc v1.63.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index d0e7da290..9f0c6263c 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -github.com/avast/retry-go/v4 v4.5.1 h1:AxIx0HGi4VZ3I02jr78j5lZ3M6x1E0Ivxa6b0pUUh7o= -github.com/avast/retry-go/v4 v4.5.1/go.mod h1:/sipNsvNB3RRuT5iNcb6h73nw3IBmXJ/H3XrCQYSOpc= +github.com/avast/retry-go/v4 v4.6.0 h1:K9xNA+KeB8HHc2aWFuLb25Offp+0iVRXEvFx8IinRJA= +github.com/avast/retry-go/v4 v4.6.0/go.mod h1:gvWlPhBVsvBbLkVGDg/KwvBv0bEkCOLRRSHKIr2PyOE= github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -16,8 +16,8 @@ github.com/cilium/ebpf v0.5.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJ github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= github.com/cilium/ebpf v0.8.1 h1:bLSSEbBLqGPXxls55pGr5qWZaTqcmfDJHhou7t254ao= github.com/cilium/ebpf v0.8.1/go.mod h1:f5zLIM0FSNuAkSyLAN7X+Hy6yznlF1mNiWUMfxMtrgk= -github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/d5/tengo/v2 v2.17.0 h1:BWUN9NoJzw48jZKiYDXDIF3QrIVZRm1uV1gTzeZ2lqM= github.com/d5/tengo/v2 v2.17.0/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8= @@ -213,8 +213,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= -github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= +github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= +github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= @@ -228,8 +228,8 @@ github.com/redis/go-redis/v9 v9.5.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= -github.com/rs/cors v1.10.1 h1:L0uuZVXIKlI1SShY2nhFfo44TYvDPQ1w4oFkUJNfhyo= -github.com/rs/cors v1.10.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= +github.com/rs/cors v1.11.0 h1:0B9GE/r9Bc2UxRMMtymBkHTenPkHDv0CW4Y98GBY+po= +github.com/rs/cors v1.11.0/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sclevine/spec v1.4.0 h1:z/Q9idDcay5m5irkZ28M7PtQM4aOISzOpj4bUPkDee8= @@ -254,12 +254,12 @@ github.com/twitchtv/twirp v8.1.3+incompatible h1:+F4TdErPgSUbMZMwp13Q/KgDVuI7HJX github.com/twitchtv/twirp v8.1.3+incompatible/go.mod h1:RRJoFSAmTEh2weEqWtpPE3vFK5YBhA6bqp2l1kfCC5A= github.com/ua-parser/uap-go v0.0.0-20240113215029-33f8e6d47f38 h1:F04Na0QJP9GJrwmK3vQDuDrCuGllrrfngW8CIeF1aag= github.com/ua-parser/uap-go v0.0.0-20240113215029-33f8e6d47f38/go.mod h1:BUbeWZiieNxAuuADTBNb3/aeje6on3DhU3rpWsQSB1E= -github.com/urfave/cli/v2 v2.27.1 h1:8xSQ6szndafKVRmfyeUMxkNUJQMjL1F2zmsZ+qHpfho= -github.com/urfave/cli/v2 v2.27.1/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= +github.com/urfave/cli/v2 v2.27.2 h1:6e0H+AkS+zDckwPCUrZkKX38mRaau4nL2uipkJpbkcI= +github.com/urfave/cli/v2 v2.27.2/go.mod h1:g0+79LmHHATl7DAcHO99smiR/T7uGLw84w8Y42x+4eM= github.com/urfave/negroni/v3 v3.1.0 h1:lzmuxGSpnJCT/ujgIAjkU3+LW3NX8alCglO/L6KjIGQ= github.com/urfave/negroni/v3 v3.1.0/go.mod h1:jWvnX03kcSjDBl/ShB0iHvx5uOs7mAzZXW+JvJ5XYAs= -github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= -github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= +github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 h1:+qGGcbkzsfDQNPPe9UDgpxAWQrhbbBXOYJFQDq/dtJw= +github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913/go.mod h1:4aEEwZQutDLsQv2Deui4iYQ6DWTxR14g6m8Wv88+Xqk= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= @@ -284,10 +284,10 @@ golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98y golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= -golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= -golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY= -golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= @@ -318,8 +318,8 @@ golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= -golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= -golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -365,8 +365,8 @@ golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= -golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -388,16 +388,17 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= -golang.org/x/tools v0.20.0 h1:hz/CVckiOxybQvFw6h7b/q80NTr9IUQb4s1IIzW7KNY= -golang.org/x/tools v0.20.0/go.mod h1:WvitBU7JJf6A4jOdg4S1tviW9bhUxkgeCui/0JHctQg= +golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw= +golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -405,8 +406,8 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be h1: google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 96cb829b84fc848b0cf98932eaa20ba91013d25b Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 23 May 2024 13:03:26 +0530 Subject: [PATCH 27/60] Log more info when adjusting start timestamp. (#2722) Seeing some large time stamp jump in relay down track once in a while. Logging more details on time stamp switch to learn more. --- pkg/sfu/buffer/rtpstats_receiver.go | 66 ++++++++------------- pkg/sfu/buffer/rtpstats_sender.go | 91 +++++++++++++---------------- 2 files changed, 65 insertions(+), 92 deletions(-) diff --git a/pkg/sfu/buffer/rtpstats_receiver.go b/pkg/sfu/buffer/rtpstats_receiver.go index da387dc44..9ffc2507e 100644 --- a/pkg/sfu/buffer/rtpstats_receiver.go +++ b/pkg/sfu/buffer/rtpstats_receiver.go @@ -169,27 +169,32 @@ func (r *RTPStatsReceiver) Update( pktSize := uint64(hdrSize + payloadSize + paddingSize) gapSN := int64(resSN.ExtendedVal - resSN.PreExtendedHighest) + getLoggingFields := func() []interface{} { + return []interface{}{ + "extStartSN", r.sequenceNumber.GetExtendedStart(), + "extHighestSN", r.sequenceNumber.GetExtendedHighest(), + "extStartTS", r.timestamp.GetExtendedStart(), + "extHighestTS", r.timestamp.GetExtendedHighest(), + "firstTime", r.firstTime.String(), + "highestTime", r.highestTime.String(), + "prevSN", resSN.PreExtendedHighest, + "currSN", resSN.ExtendedVal, + "gapSN", gapSN, + "prevTS", resTS.PreExtendedHighest, + "currTS", resTS.ExtendedVal, + "gapTS", resTS.ExtendedVal - resTS.PreExtendedHighest, + "packetTime", packetTime.String(), + "sequenceNumber", sequenceNumber, + "timestamp", timestamp, + "marker", marker, + "hdrSize", hdrSize, + "payloadSize", payloadSize, + "paddingSize", paddingSize, + } + } if gapSN <= 0 { // duplicate OR out-of-order if -gapSN >= cNumSequenceNumbers/2 { - r.logger.Warnw( - "large sequence number gap negative", nil, - "extStartSN", r.sequenceNumber.GetExtendedStart(), - "extHighestSN", r.sequenceNumber.GetExtendedHighest(), - "extStartTS", r.timestamp.GetExtendedStart(), - "extHighestTS", r.timestamp.GetExtendedHighest(), - "firstTime", r.firstTime.String(), - "highestTime", r.highestTime.String(), - "prev", resSN.PreExtendedHighest, - "curr", resSN.ExtendedVal, - "gap", gapSN, - "packetTime", packetTime.String(), - "sequenceNumber", sequenceNumber, - "timestamp", timestamp, - "marker", marker, - "hdrSize", hdrSize, - "payloadSize", payloadSize, - "paddingSize", paddingSize, - ) + r.logger.Warnw("large sequence number gap negative", nil, getLoggingFields()...) } if gapSN != 0 { @@ -213,28 +218,7 @@ func (r *RTPStatsReceiver) Update( flowState.ExtTimestamp = resTS.ExtendedVal } else { // in-order if gapSN >= cNumSequenceNumbers/2 || resTS.ExtendedVal < resTS.PreExtendedHighest { - r.logger.Warnw( - "large sequence number gap OR time reversed", nil, - "extStartSN", r.sequenceNumber.GetExtendedStart(), - "extHighestSN", r.sequenceNumber.GetExtendedHighest(), - "extStartTS", r.timestamp.GetExtendedStart(), - "extHighestTS", r.timestamp.GetExtendedHighest(), - "firstTime", r.firstTime.String(), - "highestTime", r.highestTime.String(), - "prevSN", resSN.PreExtendedHighest, - "currSN", resSN.ExtendedVal, - "gapSN", gapSN, - "prevTS", resTS.PreExtendedHighest, - "currTS", resTS.ExtendedVal, - "gapTS", resTS.ExtendedVal-resTS.PreExtendedHighest, - "packetTime", packetTime.String(), - "sequenceNumber", sequenceNumber, - "timestamp", timestamp, - "marker", marker, - "hdrSize", hdrSize, - "payloadSize", payloadSize, - "paddingSize", paddingSize, - ) + r.logger.Warnw("large sequence number gap OR time reversed", nil, getLoggingFields()...) } // update gap histogram diff --git a/pkg/sfu/buffer/rtpstats_sender.go b/pkg/sfu/buffer/rtpstats_sender.go index 3074803d0..a46e4de07 100644 --- a/pkg/sfu/buffer/rtpstats_sender.go +++ b/pkg/sfu/buffer/rtpstats_sender.go @@ -284,33 +284,38 @@ func (r *RTPStatsSender) Update( pktSize := uint64(hdrSize + payloadSize + paddingSize) isDuplicate := false gapSN := int64(extSequenceNumber - r.extHighestSN) + getLoggingFields := func() []interface{} { + return []interface{}{ + "extStartSN", r.extStartSN, + "extHighestSN", r.extHighestSN, + "extStartTS", r.extStartTS, + "extHighestTS", r.extHighestTS, + "firstTime", r.firstTime.String(), + "highestTime", r.highestTime.String(), + "prevSN", r.extHighestSN, + "currSN", extSequenceNumber, + "gapSN", gapSN, + "prevTS", r.extHighestTS, + "currTS", extTimestamp, + "gapTS", extTimestamp - r.extHighestTS, + "packetTime", packetTime.String(), + "sequenceNumber", extSequenceNumber, + "timestamp", extTimestamp, + "marker", marker, + "hdrSize", hdrSize, + "payloadSize", payloadSize, + "paddingSize", paddingSize, + "firstSR", r.srFirst, + "lastSR", r.srNewest, + } + } if gapSN <= 0 { // duplicate OR out-of-order if payloadSize == 0 && extSequenceNumber < r.extStartSN { // do not start on a padding only packet return } if -gapSN >= cNumSequenceNumbers/2 { - r.logger.Warnw( - "large sequence number gap negative", nil, - "extStartSN", r.extStartSN, - "extHighestSN", r.extHighestSN, - "extStartTS", r.extStartTS, - "extHighestTS", r.extHighestTS, - "firstTime", r.firstTime.String(), - "highestTime", r.highestTime.String(), - "prev", r.extHighestSN, - "curr", extSequenceNumber, - "gap", gapSN, - "packetTime", packetTime.String(), - "sequenceNumber", extSequenceNumber, - "timestamp", extTimestamp, - "marker", marker, - "hdrSize", hdrSize, - "payloadSize", payloadSize, - "paddingSize", paddingSize, - "firstSR", r.srFirst, - "lastSR", r.srNewest, - ) + r.logger.Warnw("large sequence number gap negative", nil, getLoggingFields()...) } if extSequenceNumber < r.extStartSN { @@ -335,10 +340,12 @@ func (r *RTPStatsSender) Update( r.logger.Infow( "adjusting start sequence number", - "snBefore", r.extStartSN, - "snAfter", extSequenceNumber, - "tsBefore", r.extStartTS, - "tsAfter", extTimestamp, + append(getLoggingFields(), + "snBefore", r.extStartSN, + "snAfter", extSequenceNumber, + "tsBefore", r.extStartTS, + "tsAfter", extTimestamp, + )..., ) r.extStartSN = extSequenceNumber } @@ -357,28 +364,8 @@ func (r *RTPStatsSender) Update( r.setSnInfo(extSequenceNumber, r.extHighestSN, uint16(pktSize), uint8(hdrSize), uint16(payloadSize), marker, true) } } else { // in-order - if gapSN >= cNumSequenceNumbers/2 { - r.logger.Warnw( - "large sequence number gap", nil, - "extStartSN", r.extStartSN, - "extHighestSN", r.extHighestSN, - "extStartTS", r.extStartTS, - "extHighestTS", r.extHighestTS, - "firstTime", r.firstTime.String(), - "highestTime", r.highestTime.String(), - "prev", r.extHighestSN, - "curr", extSequenceNumber, - "gap", gapSN, - "packetTime", packetTime.String(), - "sequenceNumber", extSequenceNumber, - "timestamp", extTimestamp, - "marker", marker, - "hdrSize", hdrSize, - "payloadSize", payloadSize, - "paddingSize", paddingSize, - "firstSR", r.srFirst, - "lastSR", r.srNewest, - ) + if gapSN >= cNumSequenceNumbers/2 || extTimestamp < r.extHighestTS { + r.logger.Warnw("large sequence number gap OR time reversed", nil, getLoggingFields()...) } // update gap histogram @@ -396,10 +383,12 @@ func (r *RTPStatsSender) Update( if extTimestamp < r.extStartTS { r.logger.Infow( "adjusting start timestamp", - "snBefore", r.extStartSN, - "snAfter", extSequenceNumber, - "tsBefore", r.extStartTS, - "tsAfter", extTimestamp, + append(getLoggingFields(), + "snBefore", r.extStartSN, + "snAfter", extSequenceNumber, + "tsBefore", r.extStartTS, + "tsAfter", extTimestamp, + )..., ) r.extStartTS = extTimestamp } From a444f2477c8dd3f6e68ef48dfaadf59cb571ed79 Mon Sep 17 00:00:00 2001 From: Denys Smirnov Date: Thu, 23 May 2024 20:31:01 +0300 Subject: [PATCH 28/60] Update protocol. Support SIP transports. (#2724) --- go.mod | 2 +- go.sum | 4 ++-- pkg/service/sip.go | 17 +---------------- 3 files changed, 4 insertions(+), 19 deletions(-) diff --git a/go.mod b/go.mod index c52893348..6d94e0d05 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20240416023643-881d3dc5423e - github.com/livekit/protocol v1.15.1-0.20240510165606-93a26f478d00 + github.com/livekit/protocol v1.16.1-0.20240523171447-90dfd668f42f github.com/livekit/psrpc v0.5.3-0.20240426045048-8ba067a45715 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 diff --git a/go.sum b/go.sum index 9f0c6263c..6a8e67882 100644 --- a/go.sum +++ b/go.sum @@ -120,8 +120,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20240416023643-881d3dc5423e h1:ss4VwrouYiDpuNJ9BUTH+WsW+GDdJS70iZp8ii3/0Lc= github.com/livekit/mediatransportutil v0.0.0-20240416023643-881d3dc5423e/go.mod h1:jwKUCmObuiEDH0iiuJHaGMXwRs3RjrB4G6qqgkr/5oE= -github.com/livekit/protocol v1.15.1-0.20240510165606-93a26f478d00 h1:c5VOR2XrAgxjwvWpQIA0lDUX+YcpxGzxXtaRfhu510E= -github.com/livekit/protocol v1.15.1-0.20240510165606-93a26f478d00/go.mod h1:pnn0Dv+/0K0OFqKHX6J6SreYO1dZxl6tDuAZ1ns8L/w= +github.com/livekit/protocol v1.16.1-0.20240523171447-90dfd668f42f h1:T4Twu81WbBQ7HyJWv3j/D8VJ//7M4yo4s3EmdPJqcGY= +github.com/livekit/protocol v1.16.1-0.20240523171447-90dfd668f42f/go.mod h1:pnn0Dv+/0K0OFqKHX6J6SreYO1dZxl6tDuAZ1ns8L/w= github.com/livekit/psrpc v0.5.3-0.20240426045048-8ba067a45715 h1:vhDMOe8fxEc/amYTFo799LySPM12Fk3vc+Nc6o4gYZQ= github.com/livekit/psrpc v0.5.3-0.20240426045048-8ba067a45715/go.mod h1:CQUBSPfYYAaevg1TNCc6/aYsa8DJH4jSRFdCeSZk5u0= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= diff --git a/pkg/service/sip.go b/pkg/service/sip.go index 172b46e5c..1772a5497 100644 --- a/pkg/service/sip.go +++ b/pkg/service/sip.go @@ -198,28 +198,13 @@ func (s *SIPService) CreateSIPParticipantWithToken(ctx context.Context, req *liv log := logger.GetLogger() log = log.WithValues("call-id", callID, "roomName", req.RoomName, "sip-trunk", req.SipTrunkId, "to-user", req.SipCallTo) - ireq := &rpc.InternalCreateSIPParticipantRequest{ - SipCallId: callID, - CallTo: req.SipCallTo, - RoomName: req.RoomName, - ParticipantIdentity: req.ParticipantIdentity, - ParticipantName: req.ParticipantName, - ParticipantMetadata: req.ParticipantMetadata, - Dtmf: req.Dtmf, - PlayRingtone: req.PlayRingtone, - WsUrl: wsUrl, - Token: token, - } trunk, err := s.store.LoadSIPTrunk(ctx, req.SipTrunkId) if err != nil { log.Errorw("cannot get trunk to update sip participant", err) return nil, err } log = log.WithValues("from-user", trunk.OutboundNumber, "to-host", trunk.OutboundAddress) - ireq.Address = trunk.OutboundAddress - ireq.Number = trunk.OutboundNumber - ireq.Username = trunk.OutboundUsername - ireq.Password = trunk.OutboundPassword + ireq := rpc.NewCreateSIPParticipantRequest(callID, wsUrl, token, req, trunk) // CreateSIPParticipant will wait for LiveKit Participant to be created and that can take some time. // Thus, we must set a higher deadline for it, if it's not set already. From e6aa36fdd6ea3689e275faa06c041feaf680fca9 Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Fri, 24 May 2024 17:43:28 +0800 Subject: [PATCH 29/60] Add forward stats (#2725) * Add forward metrics * ignore packets was not forwarded * rename --- go.mod | 10 ++-- go.sum | 20 ++++---- pkg/config/config.go | 8 ++++ pkg/rtc/mediatrack.go | 2 + pkg/rtc/participant.go | 2 + pkg/service/roommanager.go | 10 ++++ pkg/service/wire.go | 11 ++++- pkg/service/wire_gen.go | 11 ++++- pkg/sfu/downtrackspreader.go | 7 ++- pkg/sfu/forwardstats.go | 74 +++++++++++++++++++++++++++++ pkg/sfu/receiver.go | 22 ++++++--- pkg/sfu/redprimaryreceiver.go | 10 ++-- pkg/sfu/redreceiver.go | 8 ++-- pkg/telemetry/prometheus/node.go | 4 ++ pkg/telemetry/prometheus/packets.go | 26 ++++++++++ 15 files changed, 193 insertions(+), 32 deletions(-) create mode 100644 pkg/sfu/forwardstats.go diff --git a/go.mod b/go.mod index 6d94e0d05..c789b5ba9 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20240416023643-881d3dc5423e - github.com/livekit/protocol v1.16.1-0.20240523171447-90dfd668f42f + github.com/livekit/protocol v1.16.1-0.20240524061435-6410d008bf7b github.com/livekit/psrpc v0.5.3-0.20240426045048-8ba067a45715 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 @@ -66,7 +66,7 @@ require ( github.com/eapache/queue v1.1.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-jose/go-jose/v3 v3.0.3 // indirect - github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/subcommands v1.2.0 // indirect github.com/google/uuid v1.6.0 // indirect @@ -80,7 +80,7 @@ require ( github.com/mattn/go-runewidth v0.0.9 // indirect github.com/mdlayher/netlink v1.7.1 // indirect github.com/mdlayher/socket v0.4.0 // indirect - github.com/nats-io/nats.go v1.34.1 // indirect + github.com/nats-io/nats.go v1.35.0 // indirect github.com/nats-io/nkeys v0.4.7 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/pion/datachannel v1.5.5 // indirect @@ -105,7 +105,7 @@ require ( golang.org/x/sys v0.20.0 // indirect golang.org/x/text v0.15.0 // indirect golang.org/x/tools v0.21.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be // indirect - google.golang.org/grpc v1.63.2 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e // indirect + google.golang.org/grpc v1.64.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 6a8e67882..b1f2205bc 100644 --- a/go.sum +++ b/go.sum @@ -48,8 +48,8 @@ github.com/gammazero/workerpool v1.1.3 h1:WixN4xzukFoN0XSeXF6puqEqFTl2mECI9S6W44 github.com/gammazero/workerpool v1.1.3/go.mod h1:wPjyBLDbyKnUn2XwwyD3EEwo9dHutia9/fwNmSHWACc= github.com/go-jose/go-jose/v3 v3.0.3 h1:fFKWeig/irsp7XD2zBxvnmA/XaRWp5V3CBsZXJF7G7k= github.com/go-jose/go-jose/v3 v3.0.3/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -120,8 +120,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20240416023643-881d3dc5423e h1:ss4VwrouYiDpuNJ9BUTH+WsW+GDdJS70iZp8ii3/0Lc= github.com/livekit/mediatransportutil v0.0.0-20240416023643-881d3dc5423e/go.mod h1:jwKUCmObuiEDH0iiuJHaGMXwRs3RjrB4G6qqgkr/5oE= -github.com/livekit/protocol v1.16.1-0.20240523171447-90dfd668f42f h1:T4Twu81WbBQ7HyJWv3j/D8VJ//7M4yo4s3EmdPJqcGY= -github.com/livekit/protocol v1.16.1-0.20240523171447-90dfd668f42f/go.mod h1:pnn0Dv+/0K0OFqKHX6J6SreYO1dZxl6tDuAZ1ns8L/w= +github.com/livekit/protocol v1.16.1-0.20240524061435-6410d008bf7b h1:TQOoMQqruWwgbhMoxY2ZCksge88NesmWSn8eBZuWKFs= +github.com/livekit/protocol v1.16.1-0.20240524061435-6410d008bf7b/go.mod h1:pnn0Dv+/0K0OFqKHX6J6SreYO1dZxl6tDuAZ1ns8L/w= github.com/livekit/psrpc v0.5.3-0.20240426045048-8ba067a45715 h1:vhDMOe8fxEc/amYTFo799LySPM12Fk3vc+Nc6o4gYZQ= github.com/livekit/psrpc v0.5.3-0.20240426045048-8ba067a45715/go.mod h1:CQUBSPfYYAaevg1TNCc6/aYsa8DJH4jSRFdCeSZk5u0= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= @@ -153,8 +153,8 @@ github.com/mdlayher/socket v0.4.0 h1:280wsy40IC9M9q1uPGcLBwXpcTQDtoGwVt+BNoITxIw github.com/mdlayher/socket v0.4.0/go.mod h1:xxFqz5GRCUN3UEOm9CZqEJsAbe1C8OwSK46NlmWuVoc= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/nats-io/nats.go v1.34.1 h1:syWey5xaNHZgicYBemv0nohUPPmaLteiBEUT6Q5+F/4= -github.com/nats-io/nats.go v1.34.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8= +github.com/nats-io/nats.go v1.35.0 h1:XFNqNM7v5B+MQMKqVGAyHwYhyKb48jrenXNxIU20ULk= +github.com/nats-io/nats.go v1.35.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8= github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI= github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= @@ -402,10 +402,10 @@ golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be h1:LG9vZxsWGOmUKieR8wPAUR3u3MpnYFQZROPIMaXh7/A= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= -google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= -google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e h1:Elxv5MwEkCI9f5SkoL6afed6NTdxaGoAo39eANBwHL8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pkg/config/config.go b/pkg/config/config.go index 67006eb33..c1b40cc12 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -117,6 +117,8 @@ type RTCConfig struct { // max number of bytes to buffer for data channel. 0 means unlimited DataChannelMaxBufferedAmount uint64 `yaml:"data_channel_max_buffered_amount,omitempty"` + + ForwardStats ForwardStatsConfig `yaml:"forward_stats,omitempty"` } type TURNServer struct { @@ -318,6 +320,12 @@ type APIConfig struct { MaxCheckInterval time.Duration `yaml:"max_check_interval,omitempty"` } +type ForwardStatsConfig struct { + SummaryInterval time.Duration `yaml:"summary_interval,omitempty"` + ReportInterval time.Duration `yaml:"report_interval,omitempty"` + ReportWindow time.Duration `yaml:"report_window,omitempty"` +} + func DefaultAPIConfig() APIConfig { return APIConfig{ ExecutionTimeout: 2 * time.Second, diff --git a/pkg/rtc/mediatrack.go b/pkg/rtc/mediatrack.go index f575b4e13..21191a924 100644 --- a/pkg/rtc/mediatrack.go +++ b/pkg/rtc/mediatrack.go @@ -70,6 +70,7 @@ type MediaTrackParams struct { Logger logger.Logger SimTracks map[uint32]SimulcastTrackInfo OnRTCP func([]rtcp.Packet) + ForwardStats *sfu.ForwardStats } func NewMediaTrack(params MediaTrackParams, ti *livekit.TrackInfo) *MediaTrack { @@ -281,6 +282,7 @@ func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track *webrtc.Tra sfu.WithAudioConfig(t.params.AudioConfig), sfu.WithLoadBalanceThreshold(20), sfu.WithStreamTrackers(), + sfu.WithForwardStats(t.params.ForwardStats), ) newWR.OnCloseHandler(func() { t.MediaTrackReceiver.SetClosing() diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 7cb7db17f..3042e0843 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -141,6 +141,7 @@ type ParticipantParams struct { PlayoutDelay *livekit.PlayoutDelay SyncStreams bool EnableTrafficLoadTracking bool + ForwardStats *sfu.ForwardStats } type ParticipantImpl struct { @@ -2126,6 +2127,7 @@ func (p *ParticipantImpl) addMediaTrack(signalCid string, sdpCid string, ti *liv PLIThrottleConfig: p.params.PLIThrottleConfig, SimTracks: p.params.SimTracks, OnRTCP: p.postRtcp, + ForwardStats: p.params.ForwardStats, }, ti) mt.OnSubscribedMaxQualityChange(p.onSubscribedMaxQualityChange) diff --git a/pkg/service/roommanager.go b/pkg/service/roommanager.go index 3dfa146d5..16437ff38 100644 --- a/pkg/service/roommanager.go +++ b/pkg/service/roommanager.go @@ -25,6 +25,7 @@ import ( "golang.org/x/exp/maps" "github.com/livekit/livekit-server/pkg/agent" + "github.com/livekit/livekit-server/pkg/sfu" sutils "github.com/livekit/livekit-server/pkg/utils" "github.com/livekit/mediatransportutil/pkg/rtcconfig" "github.com/livekit/protocol/auth" @@ -83,6 +84,8 @@ type RoomManager struct { participantServers utils.MultitonService[rpc.ParticipantTopic] iceConfigCache *sutils.IceConfigCache[iceConfigCacheKey] + + forwardStats *sfu.ForwardStats } func NewLocalRoomManager( @@ -97,6 +100,7 @@ func NewLocalRoomManager( versionGenerator utils.TimedVersionGenerator, turnAuthHandler *TURNAuthHandler, bus psrpc.MessageBus, + forwardStats *sfu.ForwardStats, ) (*RoomManager, error) { rtcConf, err := rtc.NewWebRTCConfig(conf) if err != nil { @@ -116,6 +120,7 @@ func NewLocalRoomManager( versionGenerator: versionGenerator, turnAuthHandler: turnAuthHandler, bus: bus, + forwardStats: forwardStats, rooms: make(map[livekit.RoomName]*rtc.Room), @@ -232,6 +237,10 @@ func (r *RoomManager) Stop() { } r.iceConfigCache.Stop() + + if r.forwardStats != nil { + r.forwardStats.Stop() + } } // StartSession starts WebRTC session when a new participant is connected, takes place on RTC node @@ -440,6 +449,7 @@ func (r *RoomManager) StartSession( SubscriptionLimitVideo: r.config.Limit.SubscriptionLimitVideo, PlayoutDelay: roomInternal.GetPlayoutDelay(), SyncStreams: roomInternal.GetSyncStreams(), + ForwardStats: r.forwardStats, }) if err != nil { return err diff --git a/pkg/service/wire.go b/pkg/service/wire.go index 3eb4a64c6..91d6df7d1 100644 --- a/pkg/service/wire.go +++ b/pkg/service/wire.go @@ -27,10 +27,11 @@ import ( "github.com/redis/go-redis/v9" "gopkg.in/yaml.v3" + "github.com/livekit/livekit-server/pkg/agent" "github.com/livekit/livekit-server/pkg/clientconfiguration" "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/routing" - "github.com/livekit/livekit-server/pkg/agent" + "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/livekit-server/pkg/telemetry" "github.com/livekit/protocol/auth" "github.com/livekit/protocol/livekit" @@ -51,6 +52,7 @@ func InitializeServer(conf *config.Config, currentNode routing.LocalNode) (*Live createKeyProvider, createWebhookNotifier, createClientConfiguration, + createForwardStats, routing.CreateRouter, getRoomConf, config.DefaultAPIConfig, @@ -235,6 +237,13 @@ func getPSRPCClientParams(config rpc.PSRPCConfig, bus psrpc.MessageBus) rpc.Clie return rpc.NewClientParams(config, bus, logger.GetLogger(), rpc.PSRPCMetricsObserver{}) } +func createForwardStats(conf *config.Config) *sfu.ForwardStats { + if conf.RTC.ForwardStats.SummaryInterval == 0 || conf.RTC.ForwardStats.ReportInterval == 0 || conf.RTC.ForwardStats.ReportWindow == 0 { + return nil + } + return sfu.NewForwardStats(conf.RTC.ForwardStats.SummaryInterval, conf.RTC.ForwardStats.ReportInterval, conf.RTC.ForwardStats.ReportWindow) +} + func newInProcessTurnServer(conf *config.Config, authHandler turn.AuthHandler) (*turn.Server, error) { return NewTurnServer(conf, authHandler, false) } diff --git a/pkg/service/wire_gen.go b/pkg/service/wire_gen.go index 08d493b13..f731de75e 100644 --- a/pkg/service/wire_gen.go +++ b/pkg/service/wire_gen.go @@ -12,6 +12,7 @@ import ( "github.com/livekit/livekit-server/pkg/clientconfiguration" "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/routing" + "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/livekit-server/pkg/telemetry" "github.com/livekit/protocol/auth" "github.com/livekit/protocol/livekit" @@ -120,7 +121,8 @@ func InitializeServer(conf *config.Config, currentNode routing.LocalNode) (*Live clientConfigurationManager := createClientConfiguration() timedVersionGenerator := utils.NewDefaultTimedVersionGenerator() turnAuthHandler := NewTURNAuthHandler(keyProvider) - roomManager, err := NewLocalRoomManager(conf, objectStore, currentNode, router, telemetryService, clientConfigurationManager, client, rtcEgressLauncher, timedVersionGenerator, turnAuthHandler, messageBus) + forwardStats := createForwardStats(conf) + roomManager, err := NewLocalRoomManager(conf, objectStore, currentNode, router, telemetryService, clientConfigurationManager, client, rtcEgressLauncher, timedVersionGenerator, turnAuthHandler, messageBus, forwardStats) if err != nil { return nil, err } @@ -286,6 +288,13 @@ func getPSRPCClientParams(config2 rpc.PSRPCConfig, bus psrpc.MessageBus) rpc.Cli return rpc.NewClientParams(config2, bus, logger.GetLogger(), rpc.PSRPCMetricsObserver{}) } +func createForwardStats(conf *config.Config) *sfu.ForwardStats { + if conf.RTC.ForwardStats.SummaryInterval == 0 || conf.RTC.ForwardStats.ReportInterval == 0 || conf.RTC.ForwardStats.ReportWindow == 0 { + return nil + } + return sfu.NewForwardStats(conf.RTC.ForwardStats.SummaryInterval, conf.RTC.ForwardStats.ReportInterval, conf.RTC.ForwardStats.ReportWindow) +} + func newInProcessTurnServer(conf *config.Config, authHandler turn.AuthHandler) (*turn.Server, error) { return NewTurnServer(conf, authHandler, false) } diff --git a/pkg/sfu/downtrackspreader.go b/pkg/sfu/downtrackspreader.go index 68fc7b411..98057a0d4 100644 --- a/pkg/sfu/downtrackspreader.go +++ b/pkg/sfu/downtrackspreader.go @@ -86,8 +86,12 @@ func (d *DownTrackSpreader) HasDownTrack(subscriberID livekit.ParticipantID) boo return ok } -func (d *DownTrackSpreader) Broadcast(writer func(TrackSender)) { +func (d *DownTrackSpreader) Broadcast(writer func(TrackSender)) int { downTracks := d.GetDownTracks() + if len(downTracks) == 0 { + return 0 + } + threshold := uint64(d.params.Threshold) if threshold == 0 { threshold = 1000000 @@ -97,6 +101,7 @@ func (d *DownTrackSpreader) Broadcast(writer func(TrackSender)) { // WriteRTP takes about 50µs on average, so we write to 2 down tracks per loop. step := uint64(2) utils.ParallelExec(downTracks, threshold, step, writer) + return len(downTracks) } func (d *DownTrackSpreader) DownTrackCount() int { diff --git a/pkg/sfu/forwardstats.go b/pkg/sfu/forwardstats.go new file mode 100644 index 000000000..62544e3a8 --- /dev/null +++ b/pkg/sfu/forwardstats.go @@ -0,0 +1,74 @@ +package sfu + +import ( + "sync" + "sync/atomic" + "time" + + "github.com/livekit/livekit-server/pkg/telemetry/prometheus" + "github.com/livekit/protocol/utils" +) + +type ForwardStats struct { + lock sync.Mutex + lastLeftMs atomic.Int64 + latency *utils.LatencyAggregate + closeCh chan struct{} +} + +func NewForwardStats(latencyUpdateInterval, reportInterval, latencyWindowLength time.Duration) *ForwardStats { + s := &ForwardStats{ + latency: utils.NewLatencyAggregate(latencyUpdateInterval, latencyWindowLength), + closeCh: make(chan struct{}), + } + + go s.report(reportInterval) + return s +} + +func (s *ForwardStats) Update(arrival, left time.Time) { + leftMs := left.UnixMilli() + lastMs := s.lastLeftMs.Load() + if leftMs < lastMs || !s.lastLeftMs.CompareAndSwap(lastMs, leftMs) { + return + } + + transit := left.Sub(arrival) + s.lock.Lock() + defer s.lock.Unlock() + s.latency.Update(time.Duration(arrival.UnixNano()), float64(transit)) +} + +func (s *ForwardStats) GetStats() (latency, jitter time.Duration) { + s.lock.Lock() + defer s.lock.Unlock() + w := s.latency.Summarize() + return time.Duration(w.Mean()), time.Duration(w.StdDev()) +} + +func (s *ForwardStats) GetLastStats(duration time.Duration) (latency, jitter time.Duration) { + s.lock.Lock() + defer s.lock.Unlock() + w := s.latency.SummarizeLast(duration) + return time.Duration(w.Mean()), time.Duration(w.StdDev()) +} + +func (s *ForwardStats) Stop() { + close(s.closeCh) +} + +func (s *ForwardStats) report(reportInterval time.Duration) { + ticker := time.NewTicker(reportInterval) + defer ticker.Stop() + for { + select { + case <-s.closeCh: + return + case <-ticker.C: + latency, jitter := s.GetLastStats(reportInterval) + latencySlow, jitterSlow := s.GetStats() + prometheus.RecordForwardJitter(uint32(jitter/time.Millisecond), uint32(jitterSlow/time.Millisecond)) + prometheus.RecordForwardLatency(uint32(latency/time.Millisecond), uint32(latencySlow/time.Millisecond)) + } + } +} diff --git a/pkg/sfu/receiver.go b/pkg/sfu/receiver.go index 0cb210273..337bb3021 100644 --- a/pkg/sfu/receiver.go +++ b/pkg/sfu/receiver.go @@ -27,7 +27,6 @@ import ( "google.golang.org/protobuf/proto" "github.com/livekit/mediatransportutil/pkg/bucket" - "github.com/livekit/mediatransportutil/pkg/twcc" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" @@ -108,8 +107,6 @@ type WebRTCReceiver struct { onRTCP func([]rtcp.Packet) - twcc *twcc.Responder - bufferMu sync.RWMutex buffers [buffer.DefaultMaxLayerSpatial + 1]*buffer.Buffer upTracks [buffer.DefaultMaxLayerSpatial + 1]*webrtc.TrackRemote @@ -128,7 +125,9 @@ type WebRTCReceiver struct { primaryReceiver atomic.Pointer[RedPrimaryReceiver] redReceiver atomic.Pointer[RedReceiver] - redPktWriter func(pkt *buffer.ExtPacket, spatialLayer int32) + redPktWriter func(pkt *buffer.ExtPacket, spatialLayer int32) int + + forwardStats *ForwardStats } // SVC-TODO: Have to use more conditions to differentiate between @@ -187,6 +186,13 @@ func WithLoadBalanceThreshold(downTracks int) ReceiverOpts { } } +func WithForwardStats(forwardStats *ForwardStats) ReceiverOpts { + return func(w *WebRTCReceiver) *WebRTCReceiver { + w.forwardStats = forwardStats + return w + } +} + // NewWebRTCReceiver creates a new webrtc track receiver func NewWebRTCReceiver( receiver *webrtc.RTPReceiver, @@ -709,12 +715,16 @@ func (w *WebRTCReceiver) forwardRTP(layer int32) { } } - w.downTrackSpreader.Broadcast(func(dt TrackSender) { + writeCount := w.downTrackSpreader.Broadcast(func(dt TrackSender) { _ = dt.WriteRTP(pkt, spatialLayer) }) if redPktWriter != nil { - redPktWriter(pkt, spatialLayer) + writeCount += redPktWriter(pkt, spatialLayer) + } + + if writeCount > 0 && w.forwardStats != nil { + w.forwardStats.Update(pkt.Arrival, time.Now()) } if spatialTracker != nil { diff --git a/pkg/sfu/redprimaryreceiver.go b/pkg/sfu/redprimaryreceiver.go index e9e2642f3..70cc1945e 100644 --- a/pkg/sfu/redprimaryreceiver.go +++ b/pkg/sfu/redprimaryreceiver.go @@ -57,18 +57,19 @@ func NewRedPrimaryReceiver(receiver TrackReceiver, dsp DownTrackSpreaderParams) } } -func (r *RedPrimaryReceiver) ForwardRTP(pkt *buffer.ExtPacket, spatialLayer int32) { +func (r *RedPrimaryReceiver) ForwardRTP(pkt *buffer.ExtPacket, spatialLayer int32) int { // extract primary payload from RED and forward to downtracks if r.downTrackSpreader.DownTrackCount() == 0 { - return + return 0 } pkts, err := r.getSendPktsFromRed(pkt.Packet) if err != nil { r.logger.Errorw("get encoding for red failed", err, "payloadtype", pkt.Packet.PayloadType) - return + return 0 } + var count int for i, sendPkt := range pkts { pPkt := *pkt if i != len(pkts)-1 { @@ -81,10 +82,11 @@ func (r *RedPrimaryReceiver) ForwardRTP(pkt *buffer.ExtPacket, spatialLayer int3 // not modify the ExtPacket.RawPacket here for performance since it is not used by the DownTrack, // otherwise it should be set to the correct value (marshal the primary rtp packet) - r.downTrackSpreader.Broadcast(func(dt TrackSender) { + count += r.downTrackSpreader.Broadcast(func(dt TrackSender) { _ = dt.WriteRTP(&pPkt, spatialLayer) }) } + return count } func (r *RedPrimaryReceiver) AddDownTrack(track TrackSender) error { diff --git a/pkg/sfu/redreceiver.go b/pkg/sfu/redreceiver.go index fe254a972..d9b637c89 100644 --- a/pkg/sfu/redreceiver.go +++ b/pkg/sfu/redreceiver.go @@ -55,15 +55,15 @@ func NewRedReceiver(receiver TrackReceiver, dsp DownTrackSpreaderParams) *RedRec } } -func (r *RedReceiver) ForwardRTP(pkt *buffer.ExtPacket, spatialLayer int32) { +func (r *RedReceiver) ForwardRTP(pkt *buffer.ExtPacket, spatialLayer int32) int { // extract primary payload from RED and forward to downtracks if r.downTrackSpreader.DownTrackCount() == 0 { - return + return 0 } redLen, err := r.encodeRedForPrimary(pkt.Packet, r.redPayloadBuf[:]) if err != nil { r.logger.Errorw("red encoding failed", err) - return + return 0 } pPkt := *pkt @@ -73,7 +73,7 @@ func (r *RedReceiver) ForwardRTP(pkt *buffer.ExtPacket, spatialLayer int32) { // not modify the ExtPacket.RawPacket here for performance since it is not used by the DownTrack, // otherwise it should be set to the correct value (marshal the primary rtp packet) - r.downTrackSpreader.Broadcast(func(dt TrackSender) { + return r.downTrackSpreader.Broadcast(func(dt TrackSender) { _ = dt.WriteRTP(&pPkt, spatialLayer) }) } diff --git a/pkg/telemetry/prometheus/node.go b/pkg/telemetry/prometheus/node.go index 703615693..fe186e1dd 100644 --- a/pkg/telemetry/prometheus/node.go +++ b/pkg/telemetry/prometheus/node.go @@ -165,6 +165,8 @@ func GetUpdatedNodeStats(prev *livekit.NodeStats, prevAverage *livekit.NodeStats trackPublishSuccessNow := trackPublishSuccess.Load() trackSubscribeAttemptsNow := trackSubscribeAttempts.Load() trackSubscribeSuccessNow := trackSubscribeSuccess.Load() + forwardLatencyNow := forwardLatency.Load() + forwardJitterNow := forwardJitter.Load() updatedAt := time.Now().Unix() elapsed := updatedAt - prevAverage.UpdatedAt @@ -207,6 +209,8 @@ func GetUpdatedNodeStats(prev *livekit.NodeStats, prevAverage *livekit.NodeStats RetransmitBytesOutPerSec: prevAverage.RetransmitBytesOutPerSec, RetransmitPacketsOutPerSec: prevAverage.RetransmitPacketsOutPerSec, NackPerSec: prevAverage.NackPerSec, + ForwardLatency: forwardLatencyNow, + ForwardJitter: forwardJitterNow, ParticipantSignalConnectedPerSec: prevAverage.ParticipantSignalConnectedPerSec, ParticipantRtcInitPerSec: prevAverage.ParticipantRtcInitPerSec, ParticipantRtcConnectedPerSec: prevAverage.ParticipantRtcConnectedPerSec, diff --git a/pkg/telemetry/prometheus/packets.go b/pkg/telemetry/prometheus/packets.go index 3efd1eb28..7242f803f 100644 --- a/pkg/telemetry/prometheus/packets.go +++ b/pkg/telemetry/prometheus/packets.go @@ -41,6 +41,8 @@ var ( participantSignalConnected atomic.Uint64 participantRTCConnected atomic.Uint64 participantRTCInit atomic.Uint64 + forwardLatency atomic.Uint32 + forwardJitter atomic.Uint32 promPacketLabels = []string{"direction", "transmission"} promPacketTotal *prometheus.CounterVec @@ -56,6 +58,8 @@ var ( promRTT *prometheus.HistogramVec promParticipantJoin *prometheus.CounterVec promConnections *prometheus.GaugeVec + promForwardLatency prometheus.Gauge + promForwardJitter prometheus.Gauge promPacketTotalIncomingInitial prometheus.Counter promPacketTotalIncomingRetransmit prometheus.Counter @@ -139,6 +143,18 @@ func initPacketStats(nodeID string, nodeType livekit.NodeType) { Name: "total", ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, []string{"kind"}) + promForwardLatency = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: livekitNamespace, + Subsystem: "forward", + Name: "latency", + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, + }) + promForwardJitter = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: livekitNamespace, + Subsystem: "forward", + Name: "jitter", + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, + }) prometheus.MustRegister(promPacketTotal) prometheus.MustRegister(promPacketBytes) @@ -280,3 +296,13 @@ func AddConnection(direction Direction) { func SubConnection(direction Direction) { promConnections.WithLabelValues(string(direction)).Sub(1) } + +func RecordForwardLatency(latencyInstant, latencyAvg uint32) { + forwardLatency.Store(latencyAvg) + promForwardLatency.Set(float64(latencyInstant)) +} + +func RecordForwardJitter(jitterInstant, jitterAvg uint32) { + forwardJitter.Store(jitterAvg) + promForwardJitter.Set(float64(jitterInstant)) +} From cf8c4628050eae3a708ca8a796d83c634ccc0ec7 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 24 May 2024 18:11:28 -0700 Subject: [PATCH 30/60] fix(deps): update module github.com/hashicorp/go-version to v1.7.0 (#2728) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index c789b5ba9..95845ef3f 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/gammazero/workerpool v1.1.3 github.com/google/wire v0.6.0 github.com/gorilla/websocket v1.5.1 - github.com/hashicorp/go-version v1.6.0 + github.com/hashicorp/go-version v1.7.0 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/jellydator/ttlcache/v3 v3.2.0 github.com/jxskiss/base62 v1.1.0 diff --git a/go.sum b/go.sum index b1f2205bc..546a66014 100644 --- a/go.sum +++ b/go.sum @@ -77,8 +77,8 @@ github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxC github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-retryablehttp v0.7.5 h1:bJj+Pj19UZMIweq/iie+1u5YCdGrnxCT9yvm0e+Nd5M= github.com/hashicorp/go-retryablehttp v0.7.5/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= -github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= -github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= +github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= From 8be2005e0ffe8181150fe15484e65503c7b2dd0e Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sat, 25 May 2024 18:34:55 +0530 Subject: [PATCH 31/60] More detailed logging to understand old packets. (#2730) --- pkg/sfu/buffer/buffer.go | 9 +++- pkg/sfu/buffer/rtpstats_base.go | 75 +++++++++++++++++++++++++++++ pkg/sfu/buffer/rtpstats_receiver.go | 56 ++++++++++++++++++++- pkg/sfu/buffer/rtpstats_sender.go | 42 +++++++++++++++- 4 files changed, 177 insertions(+), 5 deletions(-) diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go index 5474c2664..dd0916d82 100644 --- a/pkg/sfu/buffer/buffer.go +++ b/pkg/sfu/buffer/buffer.go @@ -570,7 +570,14 @@ func (b *Buffer) calc(rawPkt []byte, rtpPacket *rtp.Packet, arrivalTime time.Tim if errors.Is(err, bucket.ErrPacketTooOld) { packetTooOldCount := b.packetTooOldCount.Inc() if (packetTooOldCount-1)%100 == 0 { - b.logger.Warnw("could not add packet to bucket", err, "count", packetTooOldCount) + b.logger.Warnw( + "could not add packet to bucket", err, + "count", packetTooOldCount, + "flowState", &flowState, + "snAdjustment", snAdjustment, + "incomingSequenceNumber", flowState.ExtSequenceNumber+snAdjustment, + "rtpStats", b.rtpStats, + ) } } else if err != bucket.ErrRTXPacket { b.logger.Warnw("could not add packet to bucket", err) diff --git a/pkg/sfu/buffer/rtpstats_base.go b/pkg/sfu/buffer/rtpstats_base.go index af8ecfe21..51b06c8d0 100644 --- a/pkg/sfu/buffer/rtpstats_base.go +++ b/pkg/sfu/buffer/rtpstats_base.go @@ -635,6 +635,81 @@ func (r *rtpStatsBase) deltaInfo(snapshotID uint32, extStartSN uint64, extHighes } } +func (r *rtpStatsBase) MarshalLogObject(e zapcore.ObjectEncoder) error { + if r == nil { + return nil + } + + e.AddTime("startTime", r.startTime) + e.AddTime("firstTime", r.firstTime) + e.AddTime("highestTime", r.highestTime) + + e.AddUint64("bytes", r.bytes) + e.AddUint64("headerBytes", r.headerBytes) + + e.AddUint64("packetsDuplicate", r.packetsDuplicate) + e.AddUint64("bytesDuplicate", r.bytesDuplicate) + e.AddUint64("headerBytesDuplicate", r.headerBytesDuplicate) + + e.AddUint64("packetsPadding", r.packetsPadding) + e.AddUint64("bytesPadding", r.bytesPadding) + e.AddUint64("headerBytesPadding", r.headerBytesPadding) + + e.AddUint64("packetsOutOfOrder", r.packetsOutOfOrder) + + e.AddUint64("packetsLost", r.packetsLost) + + e.AddUint32("frames", r.frames) + + e.AddFloat64("jitter", r.jitter) + e.AddFloat64("maxJitter", r.maxJitter) + + hasLoss := false + first := true + str := "[" + for burst, count := range r.gapHistogram { + if count == 0 { + continue + } + + hasLoss = true + + if !first { + str += ", " + } + first = false + str += fmt.Sprintf("%d:%d", burst+1, count) + } + str += "]" + if hasLoss { + e.AddString("gapHistogram", str) + } + + e.AddUint32("nacks", r.nacks) + e.AddUint32("nackAcks", r.nackAcks) + e.AddUint32("nackMisses", r.nackMisses) + e.AddUint32("nackRepeated", r.nackRepeated) + + e.AddUint32("plis", r.plis) + e.AddTime("lastPli", r.lastPli) + + e.AddUint32("layerLockPlis", r.layerLockPlis) + e.AddTime("lastLayerLockPli", r.lastLayerLockPli) + + e.AddUint32("firs", r.firs) + e.AddTime("lastFir", r.lastFir) + + e.AddUint32("keyFrames", r.keyFrames) + e.AddTime("lastKeyFrame", r.lastKeyFrame) + + e.AddUint32("rtt", r.rtt) + e.AddUint32("maxRtt", r.maxRtt) + + e.AddObject("srFirst", r.srFirst) + e.AddObject("srNewest", r.srNewest) + return nil +} + func (r *rtpStatsBase) toString( extStartSN, extHighestSN, extStartTS, extHighestTS uint64, packetsLost uint64, diff --git a/pkg/sfu/buffer/rtpstats_receiver.go b/pkg/sfu/buffer/rtpstats_receiver.go index 9ffc2507e..a82ba4b4a 100644 --- a/pkg/sfu/buffer/rtpstats_receiver.go +++ b/pkg/sfu/buffer/rtpstats_receiver.go @@ -20,6 +20,7 @@ import ( "time" "github.com/pion/rtcp" + "go.uber.org/zap/zapcore" "github.com/livekit/livekit-server/pkg/sfu/utils" "github.com/livekit/protocol/livekit" @@ -61,6 +62,8 @@ const ( cReportSlack = float64(60.0) ) +// --------------------------------------------------------------------- + type RTPFlowState struct { IsNotHandled bool @@ -75,6 +78,24 @@ type RTPFlowState struct { ExtTimestamp uint64 } +func (r *RTPFlowState) MarshalLogObject(e zapcore.ObjectEncoder) error { + if r == nil { + return nil + } + + e.AddBool("IsNotHandled", r.IsNotHandled) + e.AddBool("HasLoss", r.HasLoss) + e.AddUint64("LossStartInclusive", r.LossStartInclusive) + e.AddUint64("LossEndExclusive", r.LossEndExclusive) + e.AddBool("IsDuplicate", r.IsDuplicate) + e.AddBool("IsOutOfOrder", r.IsOutOfOrder) + e.AddUint64("ExtSequenceNumber", r.ExtSequenceNumber) + e.AddUint64("ExtTimestamp", r.ExtTimestamp) + return nil +} + +// --------------------------------------------------------------------- + type RTPStatsReceiver struct { *rtpStatsBase @@ -92,6 +113,8 @@ type RTPStatsReceiver struct { clockSkewCount int outOfOrderSsenderReportCount int + largeJumpCount int + largeJumpNegativeCount int } func NewRTPStatsReceiver(params RTPStatsParams) *RTPStatsReceiver { @@ -175,6 +198,7 @@ func (r *RTPStatsReceiver) Update( "extHighestSN", r.sequenceNumber.GetExtendedHighest(), "extStartTS", r.timestamp.GetExtendedStart(), "extHighestTS", r.timestamp.GetExtendedHighest(), + "startTime", r.startTime.String(), "firstTime", r.firstTime.String(), "highestTime", r.highestTime.String(), "prevSN", resSN.PreExtendedHighest, @@ -194,7 +218,13 @@ func (r *RTPStatsReceiver) Update( } if gapSN <= 0 { // duplicate OR out-of-order if -gapSN >= cNumSequenceNumbers/2 { - r.logger.Warnw("large sequence number gap negative", nil, getLoggingFields()...) + if r.largeJumpNegativeCount%100 == 0 { + r.logger.Warnw( + "large sequence number gap negative", nil, + append(getLoggingFields(), "count", r.largeJumpNegativeCount)..., + ) + } + r.largeJumpNegativeCount++ } if gapSN != 0 { @@ -218,7 +248,13 @@ func (r *RTPStatsReceiver) Update( flowState.ExtTimestamp = resTS.ExtendedVal } else { // in-order if gapSN >= cNumSequenceNumbers/2 || resTS.ExtendedVal < resTS.PreExtendedHighest { - r.logger.Warnw("large sequence number gap OR time reversed", nil, getLoggingFields()...) + if r.largeJumpCount%100 == 0 { + r.logger.Warnw( + "large sequence number gap OR time reversed", nil, + append(getLoggingFields(), "count", r.largeJumpCount)..., + ) + } + r.largeJumpCount++ } // update gap histogram @@ -548,6 +584,22 @@ func (r *RTPStatsReceiver) DeltaInfo(snapshotID uint32) *RTPDeltaInfo { return r.deltaInfo(snapshotID, r.sequenceNumber.GetExtendedStart(), r.sequenceNumber.GetExtendedHighest()) } +func (r *RTPStatsReceiver) MarshalLogObject(e zapcore.ObjectEncoder) error { + if r == nil { + return nil + } + + r.lock.RLock() + defer r.lock.RUnlock() + + e.AddObject("base", r.rtpStatsBase) + e.AddUint64("extendedStartSN", r.sequenceNumber.GetExtendedStart()) + e.AddUint64("extHighestSN", r.sequenceNumber.GetExtendedHighest()) + e.AddUint64("extStartTS", r.timestamp.GetExtendedStart()) + e.AddUint64("extHighestTS", r.timestamp.GetExtendedHighest()) + return nil +} + func (r *RTPStatsReceiver) String() string { r.lock.RLock() defer r.lock.RUnlock() diff --git a/pkg/sfu/buffer/rtpstats_sender.go b/pkg/sfu/buffer/rtpstats_sender.go index a46e4de07..64aca2fa8 100644 --- a/pkg/sfu/buffer/rtpstats_sender.go +++ b/pkg/sfu/buffer/rtpstats_sender.go @@ -21,6 +21,7 @@ import ( "time" "github.com/pion/rtcp" + "go.uber.org/zap/zapcore" "github.com/livekit/mediatransportutil" "github.com/livekit/protocol/livekit" @@ -163,6 +164,8 @@ type RTPStatsSender struct { clockSkewCount int metadataCacheOverflowCount int + largeJumpNegativeCount int + largeJumpCount int } func NewRTPStatsSender(params RTPStatsParams) *RTPStatsSender { @@ -290,6 +293,7 @@ func (r *RTPStatsSender) Update( "extHighestSN", r.extHighestSN, "extStartTS", r.extStartTS, "extHighestTS", r.extHighestTS, + "startTime", r.startTime.String(), "firstTime", r.firstTime.String(), "highestTime", r.highestTime.String(), "prevSN", r.extHighestSN, @@ -315,7 +319,13 @@ func (r *RTPStatsSender) Update( return } if -gapSN >= cNumSequenceNumbers/2 { - r.logger.Warnw("large sequence number gap negative", nil, getLoggingFields()...) + if r.largeJumpNegativeCount%100 == 0 { + r.logger.Warnw( + "large sequence number gap negative", nil, + append(getLoggingFields(), "count", r.largeJumpNegativeCount)..., + ) + } + r.largeJumpNegativeCount++ } if extSequenceNumber < r.extStartSN { @@ -365,7 +375,13 @@ func (r *RTPStatsSender) Update( } } else { // in-order if gapSN >= cNumSequenceNumbers/2 || extTimestamp < r.extHighestTS { - r.logger.Warnw("large sequence number gap OR time reversed", nil, getLoggingFields()...) + if r.largeJumpCount%100 == 0 { + r.logger.Warnw( + "large sequence number gap OR time reversed", nil, + append(getLoggingFields(), "count", r.largeJumpCount)..., + ) + } + r.largeJumpCount++ } // update gap histogram @@ -812,6 +828,28 @@ func (r *RTPStatsSender) DeltaInfoSender(senderSnapshotID uint32) *RTPDeltaInfo } } +func (r *RTPStatsSender) MarshalLogObject(e zapcore.ObjectEncoder) error { + if r == nil { + return nil + } + + r.lock.RLock() + defer r.lock.RUnlock() + + e.AddObject("base", r.rtpStatsBase) + e.AddUint64("extStartSN", r.extStartSN) + e.AddUint64("extHighestSN", r.extHighestSN) + e.AddUint64("extStartTS", r.extStartTS) + e.AddUint64("extHighestTS", r.extHighestTS) + e.AddTime("lastRRTime", r.lastRRTime) + e.AddReflected("lastRR", r.lastRR) + e.AddUint64("extHighestSNFromRR", r.extHighestSNFromRR) + e.AddUint64("packetsLostFromRR", r.packetsLostFromRR) + e.AddFloat64("jitterFromRR", r.jitterFromRR) + e.AddFloat64("maxJitterFromRR", r.maxJitterFromRR) + return nil +} + func (r *RTPStatsSender) String() string { r.lock.RLock() defer r.lock.RUnlock() From 8948562572d5950980ec12b5acec43af11fd620a Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 26 May 2024 14:01:02 -0700 Subject: [PATCH 32/60] fix(deps): update livekit deps (#2671) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 95845ef3f..c6b150fcd 100644 --- a/go.mod +++ b/go.mod @@ -19,8 +19,8 @@ require ( github.com/jellydator/ttlcache/v3 v3.2.0 github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 - github.com/livekit/mediatransportutil v0.0.0-20240416023643-881d3dc5423e - github.com/livekit/protocol v1.16.1-0.20240524061435-6410d008bf7b + github.com/livekit/mediatransportutil v0.0.0-20240501132628-6105557bbb9a + github.com/livekit/protocol v1.16.1-0.20240524144050-2ec622e46899 github.com/livekit/psrpc v0.5.3-0.20240426045048-8ba067a45715 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 diff --git a/go.sum b/go.sum index 546a66014..a1574c1d6 100644 --- a/go.sum +++ b/go.sum @@ -118,10 +118,10 @@ github.com/lithammer/shortuuid/v4 v4.0.0 h1:QRbbVkfgNippHOS8PXDkti4NaWeyYfcBTHtw github.com/lithammer/shortuuid/v4 v4.0.0/go.mod h1:Zs8puNcrvf2rV9rTH51ZLLcj7ZXqQI3lv67aw4KiB1Y= github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkDaKb5iXdynYrzB84ErPPO4LbRASk58= github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= -github.com/livekit/mediatransportutil v0.0.0-20240416023643-881d3dc5423e h1:ss4VwrouYiDpuNJ9BUTH+WsW+GDdJS70iZp8ii3/0Lc= -github.com/livekit/mediatransportutil v0.0.0-20240416023643-881d3dc5423e/go.mod h1:jwKUCmObuiEDH0iiuJHaGMXwRs3RjrB4G6qqgkr/5oE= -github.com/livekit/protocol v1.16.1-0.20240524061435-6410d008bf7b h1:TQOoMQqruWwgbhMoxY2ZCksge88NesmWSn8eBZuWKFs= -github.com/livekit/protocol v1.16.1-0.20240524061435-6410d008bf7b/go.mod h1:pnn0Dv+/0K0OFqKHX6J6SreYO1dZxl6tDuAZ1ns8L/w= +github.com/livekit/mediatransportutil v0.0.0-20240501132628-6105557bbb9a h1:ATbv0x7G5tW2HgiouQ57csFE/G4gekl2oV1cxb2Dy24= +github.com/livekit/mediatransportutil v0.0.0-20240501132628-6105557bbb9a/go.mod h1:jwKUCmObuiEDH0iiuJHaGMXwRs3RjrB4G6qqgkr/5oE= +github.com/livekit/protocol v1.16.1-0.20240524144050-2ec622e46899 h1:RBcTg+0C+k3aHGbPbsoogRtBMVp29OWMx+qD9ck3NUQ= +github.com/livekit/protocol v1.16.1-0.20240524144050-2ec622e46899/go.mod h1:pnn0Dv+/0K0OFqKHX6J6SreYO1dZxl6tDuAZ1ns8L/w= github.com/livekit/psrpc v0.5.3-0.20240426045048-8ba067a45715 h1:vhDMOe8fxEc/amYTFo799LySPM12Fk3vc+Nc6o4gYZQ= github.com/livekit/psrpc v0.5.3-0.20240426045048-8ba067a45715/go.mod h1:CQUBSPfYYAaevg1TNCc6/aYsa8DJH4jSRFdCeSZk5u0= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= From 38470f378bb0cac735f3a47bea17647ba6a79e73 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Sun, 26 May 2024 14:01:13 -0700 Subject: [PATCH 33/60] add message bytes metric (#2731) --- pkg/telemetry/prometheus/node.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pkg/telemetry/prometheus/node.go b/pkg/telemetry/prometheus/node.go index fe186e1dd..514220ca4 100644 --- a/pkg/telemetry/prometheus/node.go +++ b/pkg/telemetry/prometheus/node.go @@ -35,6 +35,7 @@ var ( initialized atomic.Bool MessageCounter *prometheus.CounterVec + MessageBytes *prometheus.CounterVec ServiceOperationCounter *prometheus.CounterVec TwirpRequestStatusCounter *prometheus.CounterVec @@ -61,6 +62,16 @@ func Init(nodeID string, nodeType livekit.NodeType) error { []string{"type", "status"}, ) + MessageBytes = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: livekitNamespace, + Subsystem: "node", + Name: "message_bytes", + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, + }, + []string{"type", "message_type"}, + ) + ServiceOperationCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: livekitNamespace, @@ -103,6 +114,7 @@ func Init(nodeID string, nodeType livekit.NodeType) error { ) prometheus.MustRegister(MessageCounter) + prometheus.MustRegister(MessageBytes) prometheus.MustRegister(ServiceOperationCounter) prometheus.MustRegister(TwirpRequestStatusCounter) prometheus.MustRegister(promSysPacketGauge) From 5e3b1e0f67798369b97cd85efbf128faea43174d Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Sun, 26 May 2024 14:01:23 -0700 Subject: [PATCH 34/60] reduce participant lock scope (#2732) --- pkg/rtc/participant.go | 77 ++++++++++++++++++------------------------ 1 file changed, 32 insertions(+), 45 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 3042e0843..4ec917c6f 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -160,8 +160,7 @@ type ParticipantImpl struct { resSinkMu sync.Mutex resSink routing.MessageSink - grants *auth.ClaimGrants - hidden atomic.Bool + grants atomic.Pointer[auth.ClaimGrants] isPublisher atomic.Bool sessionStartRecorded atomic.Bool @@ -277,8 +276,7 @@ func NewParticipant(params ParticipantParams) (*ParticipantImpl, error) { p.timedVersion.Update(params.VersionGenerator.Next()) p.migrateState.Store(types.MigrateStateInit) p.state.Store(livekit.ParticipantInfo_JOINING) - p.grants = params.Grants - p.hidden.Store(p.grants.Video.Hidden) + p.grants.Store(params.Grants) p.SetResponseSink(params.Sink) p.setupEnabledCodecs(params.PublishEnabledCodecs, params.SubscribeEnabledCodecs, params.ClientConf.GetDisabledCodecs()) @@ -335,28 +333,21 @@ func (p *ParticipantImpl) State() livekit.ParticipantInfo_State { } func (p *ParticipantImpl) Kind() livekit.ParticipantInfo_Kind { - p.lock.RLock() - defer p.lock.RUnlock() - - return p.grants.GetParticipantKind() + return p.grants.Load().GetParticipantKind() } func (p *ParticipantImpl) IsRecorder() bool { - p.lock.RLock() - defer p.lock.RUnlock() - - return p.grants.GetParticipantKind() == livekit.ParticipantInfo_EGRESS || p.grants.Video.Recorder + grants := p.grants.Load() + return grants.GetParticipantKind() == livekit.ParticipantInfo_EGRESS || grants.Video.Recorder } func (p *ParticipantImpl) IsDependent() bool { - p.lock.RLock() - defer p.lock.RUnlock() - - switch p.grants.GetParticipantKind() { + grants := p.grants.Load() + switch grants.GetParticipantKind() { case livekit.ParticipantInfo_AGENT, livekit.ParticipantInfo_EGRESS: return true default: - return p.grants.Video.Agent || p.grants.Video.Recorder + return grants.Video.Agent || grants.Video.Recorder } } @@ -419,12 +410,15 @@ func (p *ParticipantImpl) GetBufferFactory() *buffer.Factory { // SetName attaches name to the participant func (p *ParticipantImpl) SetName(name string) { p.lock.Lock() - if p.grants.Name == name { + grants := p.grants.Load() + if grants.Name == name { p.lock.Unlock() return } - p.grants.Name = name + grants = grants.Clone() + grants.Name = name + p.grants.Store(grants) p.dirty.Store(true) onParticipantUpdate := p.onParticipantUpdate @@ -442,12 +436,15 @@ func (p *ParticipantImpl) SetName(name string) { // SetMetadata attaches metadata to the participant func (p *ParticipantImpl) SetMetadata(metadata string) { p.lock.Lock() - if p.grants.Metadata == metadata { + grants := p.grants.Load() + if grants.Metadata == metadata { p.lock.Unlock() return } - p.grants.Metadata = metadata + grants = grants.Clone() + grants.Metadata = metadata + p.grants.Store(grants) p.requireBroadcast = p.requireBroadcast || metadata != "" p.dirty.Store(true) @@ -464,10 +461,7 @@ func (p *ParticipantImpl) SetMetadata(metadata string) { } func (p *ParticipantImpl) ClaimGrants() *auth.ClaimGrants { - p.lock.RLock() - defer p.lock.RUnlock() - - return p.grants.Clone() + return p.grants.Load() } func (p *ParticipantImpl) SetPermission(permission *livekit.ParticipantPermission) bool { @@ -475,21 +469,22 @@ func (p *ParticipantImpl) SetPermission(permission *livekit.ParticipantPermissio return false } p.lock.Lock() - video := p.grants.Video + grants := p.grants.Load() - if video.MatchesPermission(permission) { + if grants.Video.MatchesPermission(permission) { p.lock.Unlock() return false } p.params.Logger.Infow("updating participant permission", "permission", permission) - video.UpdateFromPermission(permission) - p.hidden.Store(permission.Hidden) + grants = grants.Clone() + grants.Video.UpdateFromPermission(permission) + p.grants.Store(grants) p.dirty.Store(true) - canPublish := video.GetCanPublish() - canSubscribe := video.GetCanSubscribe() + canPublish := grants.Video.GetCanPublish() + canSubscribe := grants.Video.GetCanSubscribe() onParticipantUpdate := p.onParticipantUpdate onClaimsChanged := p.onClaimsChanged @@ -500,7 +495,7 @@ func (p *ParticipantImpl) SetPermission(permission *livekit.ParticipantPermissio // publish permission has been revoked then remove offending tracks for _, track := range p.GetPublishedTracks() { - if !video.GetCanPublishSource(track.Source()) { + if !grants.Video.GetCanPublishSource(track.Source()) { p.removePublishedTrack(track) } } @@ -543,8 +538,8 @@ func (p *ParticipantImpl) ToProtoWithVersion() (*livekit.ParticipantInfo, utils. p.lock.Unlock() } - grants := p.ClaimGrants() p.lock.RLock() + grants := p.grants.Load() v := p.version.Load() piv := p.timedVersion @@ -1108,27 +1103,19 @@ func (p *ParticipantImpl) IsPublisher() bool { } func (p *ParticipantImpl) CanPublishSource(source livekit.TrackSource) bool { - p.lock.RLock() - defer p.lock.RUnlock() - return p.grants.Video.GetCanPublishSource(source) + return p.grants.Load().Video.GetCanPublishSource(source) } func (p *ParticipantImpl) CanSubscribe() bool { - p.lock.RLock() - defer p.lock.RUnlock() - - return p.grants.Video.GetCanSubscribe() + return p.grants.Load().Video.GetCanSubscribe() } func (p *ParticipantImpl) CanPublishData() bool { - p.lock.RLock() - defer p.lock.RUnlock() - - return p.grants.Video.GetCanPublishData() + return p.grants.Load().Video.GetCanPublishData() } func (p *ParticipantImpl) Hidden() bool { - return p.hidden.Load() + return p.grants.Load().Video.Hidden } func (p *ParticipantImpl) VerifySubscribeParticipantInfo(pID livekit.ParticipantID, version uint32) { From 721c36fbc09f47f9ef54070e9d1a07f18c3cb29d Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 27 May 2024 10:46:26 +0530 Subject: [PATCH 35/60] Send telemetry events on track feature updates. (#2723) * Send telemetry events on track feature updates. * Do not process no change --- pkg/rtc/mediatrackreceiver.go | 38 +++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/pkg/rtc/mediatrackreceiver.go b/pkg/rtc/mediatrackreceiver.go index 5453e4300..b346eb37f 100644 --- a/pkg/rtc/mediatrackreceiver.go +++ b/pkg/rtc/mediatrackreceiver.go @@ -673,20 +673,29 @@ func (t *MediaTrackReceiver) UpdateAudioTrack(update *livekit.UpdateLocalAudioTr } t.lock.Lock() - t.trackInfo.AudioFeatures = update.Features - t.trackInfo.Stereo = false - t.trackInfo.DisableDtx = false + clonedInfo := proto.Clone(t.trackInfo).(*livekit.TrackInfo) + clonedInfo.AudioFeatures = update.Features + clonedInfo.Stereo = false + clonedInfo.DisableDtx = false for _, feature := range update.Features { switch feature { case livekit.AudioTrackFeature_TF_STEREO: - t.trackInfo.Stereo = true + clonedInfo.Stereo = true case livekit.AudioTrackFeature_TF_NO_DTX: - t.trackInfo.DisableDtx = true + clonedInfo.DisableDtx = true } } + if proto.Equal(t.trackInfo, clonedInfo) { + t.lock.Unlock() + return + } + + t.trackInfo = clonedInfo t.lock.Unlock() t.updateTrackInfoOfReceivers() + + t.params.Telemetry.TrackPublishedUpdate(context.Background(), t.PublisherID(), clonedInfo) } func (t *MediaTrackReceiver) UpdateVideoTrack(update *livekit.UpdateLocalVideoTrack) { @@ -695,11 +704,20 @@ func (t *MediaTrackReceiver) UpdateVideoTrack(update *livekit.UpdateLocalVideoTr } t.lock.Lock() - t.trackInfo.Width = update.Width - t.trackInfo.Height = update.Height + clonedInfo := proto.Clone(t.trackInfo).(*livekit.TrackInfo) + clonedInfo.Width = update.Width + clonedInfo.Height = update.Height + if proto.Equal(t.trackInfo, clonedInfo) { + t.lock.Unlock() + return + } + + t.trackInfo = clonedInfo t.lock.Unlock() t.updateTrackInfoOfReceivers() + + t.params.Telemetry.TrackPublishedUpdate(context.Background(), t.PublisherID(), clonedInfo) } func (t *MediaTrackReceiver) TrackInfo() *livekit.TrackInfo { @@ -709,11 +727,15 @@ func (t *MediaTrackReceiver) TrackInfo() *livekit.TrackInfo { return t.trackInfo } +func (t *MediaTrackReceiver) trackInfoCloneLocked() *livekit.TrackInfo { + return proto.Clone(t.trackInfo).(*livekit.TrackInfo) +} + func (t *MediaTrackReceiver) TrackInfoClone() *livekit.TrackInfo { t.lock.RLock() defer t.lock.RUnlock() - return proto.Clone(t.trackInfo).(*livekit.TrackInfo) + return t.trackInfoCloneLocked() } func (t *MediaTrackReceiver) NotifyMaxLayerChange(maxLayer int32) { From 991ad5a0ead6fd523612cedaab35995fc5fa7458 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 27 May 2024 10:53:23 +0530 Subject: [PATCH 36/60] Remove unused traffic load (#2734) --- pkg/rtc/participant.go | 16 -- pkg/rtc/participant_traffic_load.go | 219 ------------------ pkg/rtc/types/interfaces.go | 3 - pkg/rtc/types/trafficstats.go | 161 ------------- .../typesfakes/fake_local_participant.go | 104 --------- 5 files changed, 503 deletions(-) delete mode 100644 pkg/rtc/participant_traffic_load.go delete mode 100644 pkg/rtc/types/trafficstats.go diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 4ec917c6f..05a2f27b2 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -140,7 +140,6 @@ type ParticipantParams struct { SubscriptionLimitVideo int32 PlayoutDelay *livekit.PlayoutDelay SyncStreams bool - EnableTrafficLoadTracking bool ForwardStats *sfu.ForwardStats } @@ -187,7 +186,6 @@ type ParticipantImpl struct { *TransportManager *UpTrackManager *SubscriptionManager - *ParticipantTrafficLoad // keeps track of unpublished tracks in order to reuse trackID unpublishedTracks []*livekit.TrackInfo @@ -297,7 +295,6 @@ func NewParticipant(params ParticipantParams) (*ParticipantImpl, error) { p.setupUpTrackManager() p.setupSubscriptionManager() - p.setupParticipantTrafficLoad() return p, nil } @@ -865,9 +862,6 @@ func (p *ParticipantImpl) Close(sendLeave bool, reason types.ParticipantCloseRea go func() { p.SubscriptionManager.Close(isExpectedToResume) p.TransportManager.Close() - if p.ParticipantTrafficLoad != nil { - p.ParticipantTrafficLoad.Close() - } }() p.dataChannelStats.Stop() @@ -1373,16 +1367,6 @@ func (p *ParticipantImpl) setupSubscriptionManager() { }) } -func (p *ParticipantImpl) setupParticipantTrafficLoad() { - if p.params.EnableTrafficLoadTracking { - p.ParticipantTrafficLoad = NewParticipantTrafficLoad(ParticipantTrafficLoadParams{ - Participant: p, - DataChannelStats: p.dataChannelStats, - Logger: p.params.Logger, - }) - } -} - func (p *ParticipantImpl) updateState(state livekit.ParticipantInfo_State) { oldState := p.state.Swap(state).(livekit.ParticipantInfo_State) if oldState == state { diff --git a/pkg/rtc/participant_traffic_load.go b/pkg/rtc/participant_traffic_load.go deleted file mode 100644 index 28f6867d1..000000000 --- a/pkg/rtc/participant_traffic_load.go +++ /dev/null @@ -1,219 +0,0 @@ -// Copyright 2023 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package rtc - -import ( - "sync" - "time" - - "github.com/frostbyte73/core" - "github.com/livekit/protocol/livekit" - "github.com/livekit/protocol/logger" - - "github.com/livekit/livekit-server/pkg/rtc/types" - "github.com/livekit/livekit-server/pkg/telemetry" -) - -const ( - reportInterval = 10 * time.Second -) - -type ParticipantTrafficLoadParams struct { - Participant *ParticipantImpl - DataChannelStats *telemetry.BytesTrackStats - Logger logger.Logger -} - -type ParticipantTrafficLoad struct { - params ParticipantTrafficLoadParams - - lock sync.RWMutex - onTrafficLoad func(trafficLoad *types.TrafficLoad) - tracksStatsMedia map[livekit.TrackID]*livekit.RTPStats - dataChannelTraffic *telemetry.TrafficTotals - trafficLoad *types.TrafficLoad - - closed core.Fuse -} - -func NewParticipantTrafficLoad(params ParticipantTrafficLoadParams) *ParticipantTrafficLoad { - p := &ParticipantTrafficLoad{ - params: params, - tracksStatsMedia: make(map[livekit.TrackID]*livekit.RTPStats), - } - go p.reporter() - return p -} - -func (p *ParticipantTrafficLoad) Close() { - p.closed.Break() -} - -func (p *ParticipantTrafficLoad) OnTrafficLoad(f func(trafficLoad *types.TrafficLoad)) { - if p == nil { - return - } - - p.lock.Lock() - p.onTrafficLoad = f - p.lock.Unlock() -} - -func (p *ParticipantTrafficLoad) getOnTrafficLoad() func(trafficLoad *types.TrafficLoad) { - p.lock.RLock() - defer p.lock.RUnlock() - - return p.onTrafficLoad -} - -func (p *ParticipantTrafficLoad) GetTrafficLoad() *types.TrafficLoad { - if p == nil { - return nil - } - - p.lock.RLock() - defer p.lock.RUnlock() - - return p.trafficLoad -} - -func (p *ParticipantTrafficLoad) updateTrafficLoad() *types.TrafficLoad { - publishedTracks := p.params.Participant.GetPublishedTracks() - subscribedTracks := p.params.Participant.SubscriptionManager.GetSubscribedTracks() - - availableTracks := make(map[livekit.TrackID]bool, len(publishedTracks)+len(subscribedTracks)) - - upstreamAudioStats := make([]*types.TrafficStats, 0, len(publishedTracks)) - upstreamVideoStats := make([]*types.TrafficStats, 0, len(publishedTracks)) - - downstreamAudioStats := make([]*types.TrafficStats, 0, len(subscribedTracks)) - downstreamVideoStats := make([]*types.TrafficStats, 0, len(subscribedTracks)) - - p.lock.Lock() - defer p.lock.Unlock() - for _, pt := range publishedTracks { - lmt, ok := pt.(types.LocalMediaTrack) - if !ok { - continue - } - trackID := lmt.ID() - stats := lmt.GetTrackStats() - trafficStats := types.RTPStatsDiffToTrafficStats(p.tracksStatsMedia[trackID], stats) - if stats != nil { - p.tracksStatsMedia[trackID] = stats - availableTracks[trackID] = true - } - if trafficStats != nil { - switch lmt.Kind() { - case livekit.TrackType_AUDIO: - upstreamAudioStats = append(upstreamAudioStats, trafficStats) - case livekit.TrackType_VIDEO: - upstreamVideoStats = append(upstreamVideoStats, trafficStats) - } - } - } - - for _, st := range subscribedTracks { - trackID := st.ID() - stats := st.DownTrack().GetTrackStats() - trafficStats := types.RTPStatsDiffToTrafficStats(p.tracksStatsMedia[trackID], stats) - if stats != nil { - p.tracksStatsMedia[trackID] = stats - availableTracks[trackID] = true - } - if trafficStats != nil { - switch st.MediaTrack().Kind() { - case livekit.TrackType_AUDIO: - downstreamAudioStats = append(downstreamAudioStats, trafficStats) - case livekit.TrackType_VIDEO: - downstreamVideoStats = append(downstreamVideoStats, trafficStats) - } - } - } - - // remove unavailable tracks from track stats cache - for trackID := range p.tracksStatsMedia { - if !availableTracks[trackID] { - delete(p.tracksStatsMedia, trackID) - } - } - - trafficTypeStats := make([]*types.TrafficTypeStats, 0, 6) - addTypeStats := func(statsList []*types.TrafficStats, trackType livekit.TrackType, streamType livekit.StreamType) { - agg := types.AggregateTrafficStats(statsList...) - if agg != nil { - trafficTypeStats = append(trafficTypeStats, &types.TrafficTypeStats{ - TrackType: trackType, - StreamType: streamType, - TrafficStats: agg, - }) - } - } - addTypeStats(upstreamAudioStats, livekit.TrackType_AUDIO, livekit.StreamType_UPSTREAM) - addTypeStats(upstreamVideoStats, livekit.TrackType_VIDEO, livekit.StreamType_UPSTREAM) - addTypeStats(downstreamAudioStats, livekit.TrackType_VIDEO, livekit.StreamType_DOWNSTREAM) - addTypeStats(downstreamVideoStats, livekit.TrackType_VIDEO, livekit.StreamType_DOWNSTREAM) - - if p.params.DataChannelStats != nil { - dataChannelTraffic := p.params.DataChannelStats.GetTrafficTotals() - if p.dataChannelTraffic != nil { - trafficTypeStats = append(trafficTypeStats, &types.TrafficTypeStats{ - TrackType: livekit.TrackType_DATA, - StreamType: livekit.StreamType_UPSTREAM, - TrafficStats: &types.TrafficStats{ - StartTime: p.dataChannelTraffic.At, - EndTime: dataChannelTraffic.At, - Packets: dataChannelTraffic.RecvMessages - p.dataChannelTraffic.RecvMessages, - Bytes: dataChannelTraffic.RecvBytes - p.dataChannelTraffic.RecvBytes, - }, - }) - - trafficTypeStats = append(trafficTypeStats, &types.TrafficTypeStats{ - TrackType: livekit.TrackType_DATA, - StreamType: livekit.StreamType_DOWNSTREAM, - TrafficStats: &types.TrafficStats{ - StartTime: p.dataChannelTraffic.At, - EndTime: dataChannelTraffic.At, - Packets: dataChannelTraffic.SendMessages - p.dataChannelTraffic.SendMessages, - Bytes: dataChannelTraffic.SendBytes - p.dataChannelTraffic.SendBytes, - }, - }) - } - p.dataChannelTraffic = dataChannelTraffic - } - - p.trafficLoad = &types.TrafficLoad{ - TrafficTypeStats: trafficTypeStats, - } - return p.trafficLoad -} - -func (p *ParticipantTrafficLoad) reporter() { - ticker := time.NewTicker(reportInterval) - defer ticker.Stop() - - for { - select { - case <-p.closed.Watch(): - return - - case <-ticker.C: - trafficLoad := p.updateTrafficLoad() - if onTrafficLoad := p.getOnTrafficLoad(); onTrafficLoad != nil { - onTrafficLoad(trafficLoad) - } - } - } -} diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index 01b874c62..b51aaa784 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -394,7 +394,6 @@ type LocalParticipant interface { OnSubscribeStatusChanged(fn func(publisherID livekit.ParticipantID, subscribed bool)) OnClose(callback func(LocalParticipant)) OnClaimsChanged(callback func(LocalParticipant)) - OnTrafficLoad(callback func(trafficLoad *TrafficLoad)) HandleReceiverReport(dt *sfu.DownTrack, report *rtcp.ReceiverReport) @@ -423,8 +422,6 @@ type LocalParticipant interface { SetSubscriberChannelCapacity(channelCapacity int64) GetPacer() pacer.Pacer - - GetTrafficLoad() *TrafficLoad } // Room is a container of participants, and can provide room-level actions diff --git a/pkg/rtc/types/trafficstats.go b/pkg/rtc/types/trafficstats.go deleted file mode 100644 index f5c0b386d..000000000 --- a/pkg/rtc/types/trafficstats.go +++ /dev/null @@ -1,161 +0,0 @@ -// Copyright 2023 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package types - -import ( - "time" - - "github.com/livekit/protocol/livekit" -) - -type TrafficStats struct { - StartTime time.Time - EndTime time.Time - Packets uint32 - PacketsLost uint32 - PacketsPadding uint32 - PacketsOutOfOrder uint32 - Bytes uint64 -} - -type TrafficTypeStats struct { - TrackType livekit.TrackType - StreamType livekit.StreamType - TrafficStats *TrafficStats -} - -type TrafficLoad struct { - TrafficTypeStats []*TrafficTypeStats -} - -func RTPStatsDiffToTrafficStats(before, after *livekit.RTPStats) *TrafficStats { - if after == nil { - return nil - } - - startTime := after.StartTime - if before != nil { - startTime = before.EndTime - } - - getAfter := func() *TrafficStats { - return &TrafficStats{ - StartTime: startTime.AsTime(), - EndTime: after.EndTime.AsTime(), - Packets: after.Packets, - PacketsLost: after.PacketsLost, - PacketsPadding: after.PacketsPadding, - PacketsOutOfOrder: after.PacketsOutOfOrder, - Bytes: after.Bytes + after.BytesDuplicate + after.BytesPadding, - } - } - - if before == nil { - return getAfter() - } - - if (after.Packets - before.Packets) > (1 << 31) { - // after packets < before packets, probably got reset, just return after - return getAfter() - } - if ((after.Bytes + after.BytesDuplicate + after.BytesPadding) - (before.Bytes + before.BytesDuplicate + before.BytesPadding)) > (1 << 63) { - // after bytes < before bytes, probably got reset, just return after - return getAfter() - } - - packetsLost := uint32(0) - if after.PacketsLost >= before.PacketsLost { - packetsLost = after.PacketsLost - before.PacketsLost - } - return &TrafficStats{ - StartTime: startTime.AsTime(), - EndTime: after.EndTime.AsTime(), - Packets: after.Packets - before.Packets, - PacketsLost: packetsLost, - PacketsPadding: after.PacketsPadding - before.PacketsPadding, - PacketsOutOfOrder: after.PacketsOutOfOrder - before.PacketsOutOfOrder, - Bytes: (after.Bytes + after.BytesDuplicate + after.BytesPadding) - (before.Bytes + before.BytesDuplicate + before.BytesPadding), - } -} - -func AggregateTrafficStats(statsList ...*TrafficStats) *TrafficStats { - if len(statsList) == 0 { - return nil - } - - startTime := time.Time{} - endTime := time.Time{} - - packets := uint32(0) - packetsLost := uint32(0) - packetsPadding := uint32(0) - packetsOutOfOrder := uint32(0) - bytes := uint64(0) - - for _, stats := range statsList { - if startTime.IsZero() || startTime.After(stats.StartTime) { - startTime = stats.StartTime - } - - if endTime.IsZero() || endTime.Before(stats.EndTime) { - endTime = stats.EndTime - } - - packets += stats.Packets - packetsLost += stats.PacketsLost - packetsPadding += stats.PacketsPadding - packetsOutOfOrder += stats.PacketsOutOfOrder - bytes += stats.Bytes - } - - if endTime.IsZero() { - endTime = time.Now() - } - return &TrafficStats{ - StartTime: startTime, - EndTime: endTime, - Packets: packets, - PacketsLost: packetsLost, - PacketsPadding: packetsPadding, - PacketsOutOfOrder: packetsOutOfOrder, - Bytes: bytes, - } -} - -func TrafficLoadToTrafficRate(trafficLoad *TrafficLoad) ( - packetRateIn float64, - byteRateIn float64, - packetRateOut float64, - byteRateOut float64, -) { - if trafficLoad == nil { - return - } - - for _, trafficTypeStat := range trafficLoad.TrafficTypeStats { - elapsed := trafficTypeStat.TrafficStats.EndTime.Sub(trafficTypeStat.TrafficStats.StartTime).Seconds() - packetRate := float64(trafficTypeStat.TrafficStats.Packets) / elapsed - byteRate := float64(trafficTypeStat.TrafficStats.Bytes) / elapsed - switch trafficTypeStat.StreamType { - case livekit.StreamType_UPSTREAM: - packetRateIn += packetRate - byteRateIn += byteRate - case livekit.StreamType_DOWNSTREAM: - packetRateOut += packetRate - byteRateOut += byteRate - } - } - return -} diff --git a/pkg/rtc/types/typesfakes/fake_local_participant.go b/pkg/rtc/types/typesfakes/fake_local_participant.go index 7705cda47..8a4bf1b86 100644 --- a/pkg/rtc/types/typesfakes/fake_local_participant.go +++ b/pkg/rtc/types/typesfakes/fake_local_participant.go @@ -345,16 +345,6 @@ type FakeLocalParticipant struct { getSubscribedTracksReturnsOnCall map[int]struct { result1 []types.SubscribedTrack } - GetTrafficLoadStub func() *types.TrafficLoad - getTrafficLoadMutex sync.RWMutex - getTrafficLoadArgsForCall []struct { - } - getTrafficLoadReturns struct { - result1 *types.TrafficLoad - } - getTrafficLoadReturnsOnCall map[int]struct { - result1 *types.TrafficLoad - } GetTrailerStub func() []byte getTrailerMutex sync.RWMutex getTrailerArgsForCall []struct { @@ -636,11 +626,6 @@ type FakeLocalParticipant struct { onTrackUpdatedArgsForCall []struct { arg1 func(types.LocalParticipant, types.MediaTrack) } - OnTrafficLoadStub func(func(trafficLoad *types.TrafficLoad)) - onTrafficLoadMutex sync.RWMutex - onTrafficLoadArgsForCall []struct { - arg1 func(trafficLoad *types.TrafficLoad) - } ProtocolVersionStub func() types.ProtocolVersion protocolVersionMutex sync.RWMutex protocolVersionArgsForCall []struct { @@ -2722,59 +2707,6 @@ func (fake *FakeLocalParticipant) GetSubscribedTracksReturnsOnCall(i int, result }{result1} } -func (fake *FakeLocalParticipant) GetTrafficLoad() *types.TrafficLoad { - fake.getTrafficLoadMutex.Lock() - ret, specificReturn := fake.getTrafficLoadReturnsOnCall[len(fake.getTrafficLoadArgsForCall)] - fake.getTrafficLoadArgsForCall = append(fake.getTrafficLoadArgsForCall, struct { - }{}) - stub := fake.GetTrafficLoadStub - fakeReturns := fake.getTrafficLoadReturns - fake.recordInvocation("GetTrafficLoad", []interface{}{}) - fake.getTrafficLoadMutex.Unlock() - if stub != nil { - return stub() - } - if specificReturn { - return ret.result1 - } - return fakeReturns.result1 -} - -func (fake *FakeLocalParticipant) GetTrafficLoadCallCount() int { - fake.getTrafficLoadMutex.RLock() - defer fake.getTrafficLoadMutex.RUnlock() - return len(fake.getTrafficLoadArgsForCall) -} - -func (fake *FakeLocalParticipant) GetTrafficLoadCalls(stub func() *types.TrafficLoad) { - fake.getTrafficLoadMutex.Lock() - defer fake.getTrafficLoadMutex.Unlock() - fake.GetTrafficLoadStub = stub -} - -func (fake *FakeLocalParticipant) GetTrafficLoadReturns(result1 *types.TrafficLoad) { - fake.getTrafficLoadMutex.Lock() - defer fake.getTrafficLoadMutex.Unlock() - fake.GetTrafficLoadStub = nil - fake.getTrafficLoadReturns = struct { - result1 *types.TrafficLoad - }{result1} -} - -func (fake *FakeLocalParticipant) GetTrafficLoadReturnsOnCall(i int, result1 *types.TrafficLoad) { - fake.getTrafficLoadMutex.Lock() - defer fake.getTrafficLoadMutex.Unlock() - fake.GetTrafficLoadStub = nil - if fake.getTrafficLoadReturnsOnCall == nil { - fake.getTrafficLoadReturnsOnCall = make(map[int]struct { - result1 *types.TrafficLoad - }) - } - fake.getTrafficLoadReturnsOnCall[i] = struct { - result1 *types.TrafficLoad - }{result1} -} - func (fake *FakeLocalParticipant) GetTrailer() []byte { fake.getTrailerMutex.Lock() ret, specificReturn := fake.getTrailerReturnsOnCall[len(fake.getTrailerArgsForCall)] @@ -4357,38 +4289,6 @@ func (fake *FakeLocalParticipant) OnTrackUpdatedArgsForCall(i int) func(types.Lo return argsForCall.arg1 } -func (fake *FakeLocalParticipant) OnTrafficLoad(arg1 func(trafficLoad *types.TrafficLoad)) { - fake.onTrafficLoadMutex.Lock() - fake.onTrafficLoadArgsForCall = append(fake.onTrafficLoadArgsForCall, struct { - arg1 func(trafficLoad *types.TrafficLoad) - }{arg1}) - stub := fake.OnTrafficLoadStub - fake.recordInvocation("OnTrafficLoad", []interface{}{arg1}) - fake.onTrafficLoadMutex.Unlock() - if stub != nil { - fake.OnTrafficLoadStub(arg1) - } -} - -func (fake *FakeLocalParticipant) OnTrafficLoadCallCount() int { - fake.onTrafficLoadMutex.RLock() - defer fake.onTrafficLoadMutex.RUnlock() - return len(fake.onTrafficLoadArgsForCall) -} - -func (fake *FakeLocalParticipant) OnTrafficLoadCalls(stub func(func(trafficLoad *types.TrafficLoad))) { - fake.onTrafficLoadMutex.Lock() - defer fake.onTrafficLoadMutex.Unlock() - fake.OnTrafficLoadStub = stub -} - -func (fake *FakeLocalParticipant) OnTrafficLoadArgsForCall(i int) func(trafficLoad *types.TrafficLoad) { - fake.onTrafficLoadMutex.RLock() - defer fake.onTrafficLoadMutex.RUnlock() - argsForCall := fake.onTrafficLoadArgsForCall[i] - return argsForCall.arg1 -} - func (fake *FakeLocalParticipant) ProtocolVersion() types.ProtocolVersion { fake.protocolVersionMutex.Lock() ret, specificReturn := fake.protocolVersionReturnsOnCall[len(fake.protocolVersionArgsForCall)] @@ -6577,8 +6477,6 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.getSubscribedParticipantsMutex.RUnlock() fake.getSubscribedTracksMutex.RLock() defer fake.getSubscribedTracksMutex.RUnlock() - fake.getTrafficLoadMutex.RLock() - defer fake.getTrafficLoadMutex.RUnlock() fake.getTrailerMutex.RLock() defer fake.getTrailerMutex.RUnlock() fake.handleAnswerMutex.RLock() @@ -6653,8 +6551,6 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.onTrackUnpublishedMutex.RUnlock() fake.onTrackUpdatedMutex.RLock() defer fake.onTrackUpdatedMutex.RUnlock() - fake.onTrafficLoadMutex.RLock() - defer fake.onTrafficLoadMutex.RUnlock() fake.protocolVersionMutex.RLock() defer fake.protocolVersionMutex.RUnlock() fake.removePublishedTrackMutex.RLock() From 2288e402ac59f4e0e85983e2e331c562b18f1092 Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Mon, 27 May 2024 15:47:01 +0800 Subject: [PATCH 37/60] register forward metrics (#2735) --- pkg/telemetry/prometheus/packets.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/telemetry/prometheus/packets.go b/pkg/telemetry/prometheus/packets.go index 7242f803f..02dc37074 100644 --- a/pkg/telemetry/prometheus/packets.go +++ b/pkg/telemetry/prometheus/packets.go @@ -167,6 +167,8 @@ func initPacketStats(nodeID string, nodeType livekit.NodeType) { prometheus.MustRegister(promRTT) prometheus.MustRegister(promParticipantJoin) prometheus.MustRegister(promConnections) + prometheus.MustRegister(promForwardLatency) + prometheus.MustRegister(promForwardJitter) promPacketTotalIncomingInitial = promPacketTotal.WithLabelValues(string(Incoming), transmissionInitial) promPacketTotalIncomingRetransmit = promPacketTotal.WithLabelValues(string(Incoming), transmissionRetransmit) From d9910ead61be1c46d016ecc97060199739a101b0 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 27 May 2024 13:43:20 +0530 Subject: [PATCH 38/60] Add back TrafficStats used by cloud. (#2736) --- pkg/rtc/types/trafficstats.go | 161 ++++++++++++++++++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 pkg/rtc/types/trafficstats.go diff --git a/pkg/rtc/types/trafficstats.go b/pkg/rtc/types/trafficstats.go new file mode 100644 index 000000000..f5c0b386d --- /dev/null +++ b/pkg/rtc/types/trafficstats.go @@ -0,0 +1,161 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import ( + "time" + + "github.com/livekit/protocol/livekit" +) + +type TrafficStats struct { + StartTime time.Time + EndTime time.Time + Packets uint32 + PacketsLost uint32 + PacketsPadding uint32 + PacketsOutOfOrder uint32 + Bytes uint64 +} + +type TrafficTypeStats struct { + TrackType livekit.TrackType + StreamType livekit.StreamType + TrafficStats *TrafficStats +} + +type TrafficLoad struct { + TrafficTypeStats []*TrafficTypeStats +} + +func RTPStatsDiffToTrafficStats(before, after *livekit.RTPStats) *TrafficStats { + if after == nil { + return nil + } + + startTime := after.StartTime + if before != nil { + startTime = before.EndTime + } + + getAfter := func() *TrafficStats { + return &TrafficStats{ + StartTime: startTime.AsTime(), + EndTime: after.EndTime.AsTime(), + Packets: after.Packets, + PacketsLost: after.PacketsLost, + PacketsPadding: after.PacketsPadding, + PacketsOutOfOrder: after.PacketsOutOfOrder, + Bytes: after.Bytes + after.BytesDuplicate + after.BytesPadding, + } + } + + if before == nil { + return getAfter() + } + + if (after.Packets - before.Packets) > (1 << 31) { + // after packets < before packets, probably got reset, just return after + return getAfter() + } + if ((after.Bytes + after.BytesDuplicate + after.BytesPadding) - (before.Bytes + before.BytesDuplicate + before.BytesPadding)) > (1 << 63) { + // after bytes < before bytes, probably got reset, just return after + return getAfter() + } + + packetsLost := uint32(0) + if after.PacketsLost >= before.PacketsLost { + packetsLost = after.PacketsLost - before.PacketsLost + } + return &TrafficStats{ + StartTime: startTime.AsTime(), + EndTime: after.EndTime.AsTime(), + Packets: after.Packets - before.Packets, + PacketsLost: packetsLost, + PacketsPadding: after.PacketsPadding - before.PacketsPadding, + PacketsOutOfOrder: after.PacketsOutOfOrder - before.PacketsOutOfOrder, + Bytes: (after.Bytes + after.BytesDuplicate + after.BytesPadding) - (before.Bytes + before.BytesDuplicate + before.BytesPadding), + } +} + +func AggregateTrafficStats(statsList ...*TrafficStats) *TrafficStats { + if len(statsList) == 0 { + return nil + } + + startTime := time.Time{} + endTime := time.Time{} + + packets := uint32(0) + packetsLost := uint32(0) + packetsPadding := uint32(0) + packetsOutOfOrder := uint32(0) + bytes := uint64(0) + + for _, stats := range statsList { + if startTime.IsZero() || startTime.After(stats.StartTime) { + startTime = stats.StartTime + } + + if endTime.IsZero() || endTime.Before(stats.EndTime) { + endTime = stats.EndTime + } + + packets += stats.Packets + packetsLost += stats.PacketsLost + packetsPadding += stats.PacketsPadding + packetsOutOfOrder += stats.PacketsOutOfOrder + bytes += stats.Bytes + } + + if endTime.IsZero() { + endTime = time.Now() + } + return &TrafficStats{ + StartTime: startTime, + EndTime: endTime, + Packets: packets, + PacketsLost: packetsLost, + PacketsPadding: packetsPadding, + PacketsOutOfOrder: packetsOutOfOrder, + Bytes: bytes, + } +} + +func TrafficLoadToTrafficRate(trafficLoad *TrafficLoad) ( + packetRateIn float64, + byteRateIn float64, + packetRateOut float64, + byteRateOut float64, +) { + if trafficLoad == nil { + return + } + + for _, trafficTypeStat := range trafficLoad.TrafficTypeStats { + elapsed := trafficTypeStat.TrafficStats.EndTime.Sub(trafficTypeStat.TrafficStats.StartTime).Seconds() + packetRate := float64(trafficTypeStat.TrafficStats.Packets) / elapsed + byteRate := float64(trafficTypeStat.TrafficStats.Bytes) / elapsed + switch trafficTypeStat.StreamType { + case livekit.StreamType_UPSTREAM: + packetRateIn += packetRate + byteRateIn += byteRate + case livekit.StreamType_DOWNSTREAM: + packetRateOut += packetRate + byteRateOut += byteRate + } + } + return +} From 7ed1284b96f4babbf740dbd551710bf048c9d267 Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Tue, 28 May 2024 17:03:18 +0800 Subject: [PATCH 39/60] report average forward metrics (#2737) * report average forward metrics * unused parameter --- pkg/telemetry/prometheus/packets.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/telemetry/prometheus/packets.go b/pkg/telemetry/prometheus/packets.go index 02dc37074..b0354bf3a 100644 --- a/pkg/telemetry/prometheus/packets.go +++ b/pkg/telemetry/prometheus/packets.go @@ -299,12 +299,12 @@ func SubConnection(direction Direction) { promConnections.WithLabelValues(string(direction)).Sub(1) } -func RecordForwardLatency(latencyInstant, latencyAvg uint32) { +func RecordForwardLatency(_, latencyAvg uint32) { forwardLatency.Store(latencyAvg) - promForwardLatency.Set(float64(latencyInstant)) + promForwardLatency.Set(float64(latencyAvg)) } -func RecordForwardJitter(jitterInstant, jitterAvg uint32) { +func RecordForwardJitter(_, jitterAvg uint32) { forwardJitter.Store(jitterAvg) - promForwardJitter.Set(float64(jitterInstant)) + promForwardJitter.Set(float64(jitterAvg)) } From fa65712a0e6d54bc17434d3e0f34af9ee7cdcc2d Mon Sep 17 00:00:00 2001 From: Denys Smirnov Date: Tue, 28 May 2024 13:43:22 +0300 Subject: [PATCH 40/60] Use camel case in SIP logs. (#2738) --- pkg/service/ioservice_sip.go | 12 ++++++------ pkg/service/sip.go | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/service/ioservice_sip.go b/pkg/service/ioservice_sip.go index e893dafb5..8733be07a 100644 --- a/pkg/service/ioservice_sip.go +++ b/pkg/service/ioservice_sip.go @@ -48,7 +48,7 @@ func (s *IOInfoService) matchSIPDispatchRule(ctx context.Context, trunk *livekit func (s *IOInfoService) EvaluateSIPDispatchRules(ctx context.Context, req *rpc.EvaluateSIPDispatchRulesRequest) (*rpc.EvaluateSIPDispatchRulesResponse, error) { log := logger.GetLogger() - log = log.WithValues("to-user", req.CalledNumber, "from-user", req.CallingNumber) + log = log.WithValues("toUser", req.CalledNumber, "fromUser", req.CallingNumber) trunk, err := s.matchSIPTrunk(ctx, req.CallingNumber, req.CalledNumber) if err != nil { return nil, err @@ -57,7 +57,7 @@ func (s *IOInfoService) EvaluateSIPDispatchRules(ctx context.Context, req *rpc.E if trunk != nil { trunkID = trunk.SipTrunkId } - log = log.WithValues("sip-trunk", trunkID) + log = log.WithValues("sipTrunk", trunkID) if trunk != nil { log.Debugw("SIP trunk matched") } else { @@ -73,7 +73,7 @@ func (s *IOInfoService) EvaluateSIPDispatchRules(ctx context.Context, req *rpc.E } return nil, err } - log.Debugw("SIP dispatch rule matched", "sip-rule", best.SipDispatchRuleId) + log.Debugw("SIP dispatch rule matched", "sipRule", best.SipDispatchRuleId) resp, err := sip.EvaluateDispatchRule(best, req) if err != nil { return nil, err @@ -84,16 +84,16 @@ func (s *IOInfoService) EvaluateSIPDispatchRules(ctx context.Context, req *rpc.E func (s *IOInfoService) GetSIPTrunkAuthentication(ctx context.Context, req *rpc.GetSIPTrunkAuthenticationRequest) (*rpc.GetSIPTrunkAuthenticationResponse, error) { log := logger.GetLogger() - log = log.WithValues("to-user", req.To, "from-user", req.From) + log = log.WithValues("toUser", req.To, "fromUser", req.From) trunk, err := s.matchSIPTrunk(ctx, req.From, req.To) if err != nil { return nil, err } if trunk == nil { - log.Debugw("No SIP trunk matched for auth", "sip-trunk", "") + log.Debugw("No SIP trunk matched for auth", "sipTrunk", "") return &rpc.GetSIPTrunkAuthenticationResponse{}, nil } - log.Debugw("SIP trunk matched for auth", "sip-trunk", trunk.SipTrunkId) + log.Debugw("SIP trunk matched for auth", "sipTrunk", trunk.SipTrunkId) return &rpc.GetSIPTrunkAuthenticationResponse{ SipTrunkId: trunk.SipTrunkId, Username: trunk.InboundUsername, diff --git a/pkg/service/sip.go b/pkg/service/sip.go index 1772a5497..417457c8f 100644 --- a/pkg/service/sip.go +++ b/pkg/service/sip.go @@ -196,14 +196,14 @@ func (s *SIPService) CreateSIPParticipantWithToken(ctx context.Context, req *liv } callID := sip.NewCallID() log := logger.GetLogger() - log = log.WithValues("call-id", callID, "roomName", req.RoomName, "sip-trunk", req.SipTrunkId, "to-user", req.SipCallTo) + log = log.WithValues("callId", callID, "roomName", req.RoomName, "sipTrunk", req.SipTrunkId, "toUser", req.SipCallTo) trunk, err := s.store.LoadSIPTrunk(ctx, req.SipTrunkId) if err != nil { log.Errorw("cannot get trunk to update sip participant", err) return nil, err } - log = log.WithValues("from-user", trunk.OutboundNumber, "to-host", trunk.OutboundAddress) + log = log.WithValues("fromUser", trunk.OutboundNumber, "toHost", trunk.OutboundAddress) ireq := rpc.NewCreateSIPParticipantRequest(callID, wsUrl, token, req, trunk) // CreateSIPParticipant will wait for LiveKit Participant to be created and that can take some time. From 9781d30611333a810490734c57dcb520b7957361 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 28 May 2024 19:29:54 +0530 Subject: [PATCH 41/60] Do not propagate RTCP if report is not processed. (#2739) --- pkg/sfu/buffer/buffer.go | 5 +++-- pkg/sfu/buffer/rtpstats_base.go | 1 + pkg/sfu/buffer/rtpstats_receiver.go | 23 ++++++++++++----------- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go index dd0916d82..2b82f3760 100644 --- a/pkg/sfu/buffer/buffer.go +++ b/pkg/sfu/buffer/buffer.go @@ -872,12 +872,13 @@ func (b *Buffer) SetSenderReportData(rtpTime uint32, ntpTime uint64) { At: time.Now(), } + didSet := false if b.rtpStats != nil { - b.rtpStats.SetRtcpSenderReportData(srData) + didSet = b.rtpStats.SetRtcpSenderReportData(srData) } b.RUnlock() - if b.onRtcpSenderReport != nil { + if didSet && b.onRtcpSenderReport != nil { b.onRtcpSenderReport() } } diff --git a/pkg/sfu/buffer/rtpstats_base.go b/pkg/sfu/buffer/rtpstats_base.go index 51b06c8d0..1fcef9a23 100644 --- a/pkg/sfu/buffer/rtpstats_base.go +++ b/pkg/sfu/buffer/rtpstats_base.go @@ -641,6 +641,7 @@ func (r *rtpStatsBase) MarshalLogObject(e zapcore.ObjectEncoder) error { } e.AddTime("startTime", r.startTime) + e.AddTime("endTime", r.endTime) e.AddTime("firstTime", r.firstTime) e.AddTime("highestTime", r.highestTime) diff --git a/pkg/sfu/buffer/rtpstats_receiver.go b/pkg/sfu/buffer/rtpstats_receiver.go index a82ba4b4a..d09b07cb7 100644 --- a/pkg/sfu/buffer/rtpstats_receiver.go +++ b/pkg/sfu/buffer/rtpstats_receiver.go @@ -111,10 +111,10 @@ type RTPStatsReceiver struct { propagationDelayDeltaHighStartTime time.Time propagationDelaySpike time.Duration - clockSkewCount int - outOfOrderSsenderReportCount int - largeJumpCount int - largeJumpNegativeCount int + clockSkewCount int + outOfOrderSenderReportCount int + largeJumpCount int + largeJumpNegativeCount int } func NewRTPStatsReceiver(params RTPStatsParams) *RTPStatsReceiver { @@ -300,12 +300,12 @@ func (r *RTPStatsReceiver) Update( return } -func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *RTCPSenderReportData) { +func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *RTCPSenderReportData) bool { r.lock.Lock() defer r.lock.Unlock() if srData == nil || !r.initialized { - return + return false } // prevent against extreme case of anachronous sender reports @@ -316,7 +316,7 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *RTCPSenderReportData) "last", r.srNewest, "current", srData, ) - return + return false } tsCycles := uint64(0) @@ -365,17 +365,17 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *RTCPSenderReportData) // i. e. muting replacing with null and unmute restoring the original track. // Or it could be due bad report generation. // In any case, ignore out-of-order reports. - if r.outOfOrderSsenderReportCount%10 == 0 { + if r.outOfOrderSenderReportCount%10 == 0 { r.logger.Infow( "received sender report, out-of-order, skipping", "first", r.srFirst, "last", r.srNewest, "current", &srDataCopy, - "count", r.outOfOrderSsenderReportCount, + "count", r.outOfOrderSenderReportCount, ) } - r.outOfOrderSsenderReportCount++ - return + r.outOfOrderSenderReportCount++ + return false } if r.srNewest != nil { @@ -494,6 +494,7 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *RTCPSenderReportData) r.srNewest = &srDataCopy r.maybeAdjustFirstPacketTime(r.srNewest, 0, r.timestamp.GetExtendedStart()) + return true } func (r *RTPStatsReceiver) GetRtcpSenderReportData() *RTCPSenderReportData { From bc6dbd98c799fc2d513759f388963eb447733515 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Tue, 28 May 2024 19:04:06 -0700 Subject: [PATCH 42/60] fix sip MatchTrunk call (#2740) --- go.mod | 4 ++-- go.sum | 8 ++++---- pkg/service/ioservice_sip.go | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index c6b150fcd..705eb4624 100644 --- a/go.mod +++ b/go.mod @@ -20,8 +20,8 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20240501132628-6105557bbb9a - github.com/livekit/protocol v1.16.1-0.20240524144050-2ec622e46899 - github.com/livekit/psrpc v0.5.3-0.20240426045048-8ba067a45715 + github.com/livekit/protocol v1.16.1-0.20240529013258-ec704e0cf263 + github.com/livekit/psrpc v0.5.3-0.20240526192918-fbdaf10e6aa5 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1 diff --git a/go.sum b/go.sum index a1574c1d6..53a41359b 100644 --- a/go.sum +++ b/go.sum @@ -120,10 +120,10 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20240501132628-6105557bbb9a h1:ATbv0x7G5tW2HgiouQ57csFE/G4gekl2oV1cxb2Dy24= github.com/livekit/mediatransportutil v0.0.0-20240501132628-6105557bbb9a/go.mod h1:jwKUCmObuiEDH0iiuJHaGMXwRs3RjrB4G6qqgkr/5oE= -github.com/livekit/protocol v1.16.1-0.20240524144050-2ec622e46899 h1:RBcTg+0C+k3aHGbPbsoogRtBMVp29OWMx+qD9ck3NUQ= -github.com/livekit/protocol v1.16.1-0.20240524144050-2ec622e46899/go.mod h1:pnn0Dv+/0K0OFqKHX6J6SreYO1dZxl6tDuAZ1ns8L/w= -github.com/livekit/psrpc v0.5.3-0.20240426045048-8ba067a45715 h1:vhDMOe8fxEc/amYTFo799LySPM12Fk3vc+Nc6o4gYZQ= -github.com/livekit/psrpc v0.5.3-0.20240426045048-8ba067a45715/go.mod h1:CQUBSPfYYAaevg1TNCc6/aYsa8DJH4jSRFdCeSZk5u0= +github.com/livekit/protocol v1.16.1-0.20240529013258-ec704e0cf263 h1:5Iat/a2LYG4HG+4voZrpnNxhfiVd1G2M8dOVnK/uRIA= +github.com/livekit/protocol v1.16.1-0.20240529013258-ec704e0cf263/go.mod h1:cN8WmGQR+kWz1+UWcAQdFFUcbW76PnfZDdkLAbYIqd4= +github.com/livekit/psrpc v0.5.3-0.20240526192918-fbdaf10e6aa5 h1:mTZyrjk5WEWMsvaYtJ42pG7DuxysKj21DKPINpGSIto= +github.com/livekit/psrpc v0.5.3-0.20240526192918-fbdaf10e6aa5/go.mod h1:CQUBSPfYYAaevg1TNCc6/aYsa8DJH4jSRFdCeSZk5u0= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= github.com/mackerelio/go-osstat v0.2.4/go.mod h1:Zy+qzGdZs3A9cuIqmgbJvwbmLQH9dJvtio5ZjJTbdlQ= github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= diff --git a/pkg/service/ioservice_sip.go b/pkg/service/ioservice_sip.go index 8733be07a..f3c686bfb 100644 --- a/pkg/service/ioservice_sip.go +++ b/pkg/service/ioservice_sip.go @@ -31,7 +31,7 @@ func (s *IOInfoService) matchSIPTrunk(ctx context.Context, calling, called strin if err != nil { return nil, err } - return sip.MatchTrunk(trunks, calling, called) + return sip.MatchTrunk(trunks, "", calling, called) } // matchSIPDispatchRule finds the best dispatch rule matching the request parameters. Returns an error if no rule matched. From d944faf07874b78e9a592533c9c81ae3d286ad70 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 29 May 2024 10:51:17 +0530 Subject: [PATCH 43/60] Selectively log big jumps in timestamp on source switch. (#2741) --- pkg/sfu/forwarder.go | 76 +++++++++++++++++++++++++++++++++----------- pkg/sfu/rtpmunger.go | 15 +++++++++ 2 files changed, 72 insertions(+), 19 deletions(-) diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index feb852642..bc0e7f31c 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -1619,6 +1619,21 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e message, "layer", layer, "extExpectedTS", extExpectedTS, + "incomingTS", extPkt.Packet.Timestamp, + "extIncomingTS", extPkt.ExtTimestamp, + "extRefTS", extRefTS, + "extLastTS", extLastTS, + "diffSeconds", math.Abs(diffSeconds), + ) + } + // TODO-REMOVE-AFTER-DATA-COLLECTION + logTransitionInfo := func(message string, extExpectedTS, extRefTS, extLastTS uint64, diffSeconds float64) { + f.logger.Infow( + message, + "layer", layer, + "extExpectedTS", extExpectedTS, + "incomingTS", extPkt.Packet.Timestamp, + "extIncomingTS", extPkt.ExtTimestamp, "extRefTS", extRefTS, "extLastTS", extLastTS, "diffSeconds", math.Abs(diffSeconds), @@ -1693,6 +1708,7 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e } } + bigJump := false var extNextTS uint64 if f.lastSSRC == 0 { // If resuming (e. g. on unmute), keep next timestamp close to expected timestamp. @@ -1718,12 +1734,14 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e diffSeconds := float64(int64(extExpectedTS-extRefTS)) / float64(f.codec.ClockRate) if diffSeconds >= 0.0 { if f.resumeBehindThreshold > 0 && diffSeconds > f.resumeBehindThreshold { - logTransition("resume, reference too far behind", extExpectedTS, extRefTS, extLastTS, diffSeconds) + logTransitionInfo("resume, reference too far behind", extExpectedTS, extRefTS, extLastTS, diffSeconds) extNextTS = extExpectedTS + bigJump = true } else if diffSeconds > ResumeBehindHighThresholdSeconds { - // could be due to incorrect reference calculation - logTransition("resume, reference very far behind", extExpectedTS, extRefTS, extLastTS, diffSeconds) + // could be due to incoming time stamp lagging a lot, like an unpause of the track + logTransitionInfo("resume, reference very far behind", extExpectedTS, extRefTS, extLastTS, diffSeconds) extNextTS = extExpectedTS + bigJump = true } else { extNextTS = extRefTS } @@ -1771,22 +1789,42 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e // nominal increase extNextTS = extLastTS + 1 } - f.logger.Debugw( - "next timestamp on switch", - "switchingAt", switchingAt.String(), - "layer", layer, - "extLastTS", extLastTS, - "lastMarker", rtpMungerState.LastMarker, - "extRefTS", extRefTS, - "dummyStartTSOffset", f.dummyStartTSOffset, - "referenceLayerSpatial", f.referenceLayerSpatial, - "extExpectedTS", extExpectedTS, - "extNextTS", extNextTS, - "tsJump", extNextTS-extLastTS, - "nextSN", rtpMungerState.ExtLastSN+1, - "extIncomingSN", extPkt.ExtSequenceNumber, - "extIncomingTS", extPkt.ExtTimestamp, - ) + if bigJump { // TODO-REMOVE-AFTER-DATA-COLLECTION + f.logger.Infow( + "next timestamp on switch", + "switchingAt", switchingAt.String(), + "layer", layer, + "extLastTS", extLastTS, + "lastMarker", rtpMungerState.LastMarker, + "extRefTS", extRefTS, + "dummyStartTSOffset", f.dummyStartTSOffset, + "referenceLayerSpatial", f.referenceLayerSpatial, + "extExpectedTS", extExpectedTS, + "extNextTS", extNextTS, + "tsJump", extNextTS-extLastTS, + "nextSN", rtpMungerState.ExtLastSN+1, + "extIncomingSN", extPkt.ExtSequenceNumber, + "incomingTS", extPkt.Packet.Timestamp, + "extIncomingTS", extPkt.ExtTimestamp, + ) + } else { + f.logger.Debugw( + "next timestamp on switch", + "switchingAt", switchingAt.String(), + "layer", layer, + "extLastTS", extLastTS, + "lastMarker", rtpMungerState.LastMarker, + "extRefTS", extRefTS, + "dummyStartTSOffset", f.dummyStartTSOffset, + "referenceLayerSpatial", f.referenceLayerSpatial, + "extExpectedTS", extExpectedTS, + "extNextTS", extNextTS, + "tsJump", extNextTS-extLastTS, + "nextSN", rtpMungerState.ExtLastSN+1, + "extIncomingSN", extPkt.ExtSequenceNumber, + "extIncomingTS", extPkt.ExtTimestamp, + ) + } f.rtpMunger.UpdateSnTsOffsets(extPkt, 1, extNextTS-extLastTS) f.refInfos[layer].tsOffset = f.rtpMunger.GetTSOffset() diff --git a/pkg/sfu/rtpmunger.go b/pkg/sfu/rtpmunger.go index 0e8fa78ab..c1fff7cd3 100644 --- a/pkg/sfu/rtpmunger.go +++ b/pkg/sfu/rtpmunger.go @@ -75,6 +75,7 @@ type RTPMunger struct { extHighestIncomingSN uint64 snRangeMap *utils.RangeMap[uint64, uint64] + extHighestIncomingTS uint64 // TODO-REMOVE-AFTER-DATA-COLLECTION extLastSN uint64 extSecondLastSN uint64 @@ -138,6 +139,7 @@ func (r *RTPMunger) SeedLast(state RTPMungerState) { func (r *RTPMunger) SetLastSnTs(extPkt *buffer.ExtPacket) { r.extHighestIncomingSN = extPkt.ExtSequenceNumber - 1 + r.extHighestIncomingTS = extPkt.ExtTimestamp - 1 r.extLastSN = extPkt.ExtSequenceNumber r.extSecondLastSN = r.extLastSN - 1 @@ -151,6 +153,7 @@ func (r *RTPMunger) SetLastSnTs(extPkt *buffer.ExtPacket) { func (r *RTPMunger) UpdateSnTsOffsets(extPkt *buffer.ExtPacket, snAdjust uint64, tsAdjust uint64) { r.extHighestIncomingSN = extPkt.ExtSequenceNumber - 1 + r.extHighestIncomingTS = extPkt.ExtTimestamp - 1 r.snRangeMap.ClearAndResetValue(extPkt.ExtSequenceNumber, extPkt.ExtSequenceNumber-r.extLastSN-snAdjust) r.updateSnOffset() @@ -191,6 +194,18 @@ func (r *RTPMunger) UpdateAndGetSnTs(extPkt *buffer.ExtPacket, marker bool) (Tra // in-order - either contiguous packet with payload OR packet following a gap, may or may not have payload r.extHighestIncomingSN = extPkt.ExtSequenceNumber + // TODO-REMOVE-AFTER-DATA-COLLECTION + tsDiff := int64(extPkt.ExtTimestamp - r.extHighestIncomingTS) + if tsDiff > 24000 { // 1/2 second at audio clock rate + r.logger.Infow( + "big jump in incoming timestamp", + "last", r.extHighestIncomingTS, + "current", extPkt.ExtTimestamp, + "diff", tsDiff, + ) + } + r.extHighestIncomingTS = extPkt.ExtTimestamp + ordering := SequenceNumberOrderingContiguous if diff > 1 { ordering = SequenceNumberOrderingGap From 447793d077d2d49cfac57330c590310dc335c28a Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 29 May 2024 11:26:30 +0530 Subject: [PATCH 44/60] Move RTT errors to Debugw. (#2742) With the move to forwarding NTP timestamp as is, we get a bunch more of this error logged as the remote is basing it off of previous report and local (i. e. server-side) bases it off of a more recent report. Anyhow, this code has been around for a long time and there is nothing new to learn from those errors. Just log it at Debugw in case we can learn something from it for specific projects or environments where Debugw is okay. --- pkg/sfu/buffer/rtpstats_sender.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/sfu/buffer/rtpstats_sender.go b/pkg/sfu/buffer/rtpstats_sender.go index 64aca2fa8..a1b209a80 100644 --- a/pkg/sfu/buffer/rtpstats_sender.go +++ b/pkg/sfu/buffer/rtpstats_sender.go @@ -491,9 +491,7 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt if err == nil { isRttChanged = rtt != r.rtt } else { - if !errors.Is(err, mediatransportutil.ErrRttNotLastSenderReport) && !errors.Is(err, mediatransportutil.ErrRttNoLastSenderReport) { - r.logger.Warnw("error getting rtt", err) - } + r.logger.Debugw("error getting rtt", "error", err) } } From f9f761b223d053535d00b463539d01dc7c169815 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 29 May 2024 12:05:18 +0530 Subject: [PATCH 45/60] Demote some less useful/noisy logs. (#2743) --- pkg/rtc/participant_signal.go | 4 +-- pkg/sfu/connectionquality/scorer.go | 2 +- pkg/sfu/streamallocator/probe_controller.go | 4 +-- pkg/sfu/streamallocator/prober.go | 2 +- pkg/sfu/streamallocator/streamallocator.go | 33 ++++++++++++++------- 5 files changed, 28 insertions(+), 17 deletions(-) diff --git a/pkg/rtc/participant_signal.go b/pkg/rtc/participant_signal.go index dc0a2405a..6f84eadd9 100644 --- a/pkg/rtc/participant_signal.go +++ b/pkg/rtc/participant_signal.go @@ -314,9 +314,7 @@ func (p *ParticipantImpl) writeMessage(msg *livekit.SignalResponse) error { func (p *ParticipantImpl) CloseSignalConnection(reason types.SignallingCloseReason) { sink := p.getResponseSink() if sink != nil { - if reason != types.SignallingCloseReasonParticipantClose { - p.params.Logger.Infow("closing signal connection", "reason", reason, "connID", sink.ConnectionID()) - } + p.params.Logger.Debugw("closing signal connection", "reason", reason, "connID", sink.ConnectionID()) sink.Close() p.SetResponseSink(nil) } diff --git a/pkg/sfu/connectionquality/scorer.go b/pkg/sfu/connectionquality/scorer.go index 7364e3dc3..5c960c8e8 100644 --- a/pkg/sfu/connectionquality/scorer.go +++ b/pkg/sfu/connectionquality/scorer.go @@ -440,7 +440,7 @@ func (q *qualityScorer) updateAtLocked(stat *windowStat, at time.Time) { prevCQ := scoreToConnectionQuality(q.score) currCQ := scoreToConnectionQuality(score) if utils.IsConnectionQualityLower(prevCQ, currCQ) { - q.params.Logger.Infow( + q.params.Logger.Debugw( "quality drop", "reason", reason, "prevScore", q.score, diff --git a/pkg/sfu/streamallocator/probe_controller.go b/pkg/sfu/streamallocator/probe_controller.go index 3796e3bfa..7426683c5 100644 --- a/pkg/sfu/streamallocator/probe_controller.go +++ b/pkg/sfu/streamallocator/probe_controller.go @@ -99,12 +99,12 @@ func (p *ProbeController) CheckProbe(trend ChannelTrend, highestEstimate int64) // In rare cases, the estimate gets stuck. Prevent from probe running amok // STREAM-ALLOCATOR-TODO: Need more testing here to ensure that probe does not cause a lot of damage // - p.params.Logger.Infow("stream allocator: probe: aborting, no trend", "cluster", p.probeClusterId) + p.params.Logger.Debugw("stream allocator: probe: aborting, no trend", "cluster", p.probeClusterId) p.abortProbeLocked() case trend == ChannelTrendCongesting: // stop immediately if the probe is congesting channel more - p.params.Logger.Infow("stream allocator: probe: aborting, channel is congesting", "cluster", p.probeClusterId) + p.params.Logger.Debugw("stream allocator: probe: aborting, channel is congesting", "cluster", p.probeClusterId) p.abortProbeLocked() case highestEstimate > p.probeGoalBps: diff --git a/pkg/sfu/streamallocator/prober.go b/pkg/sfu/streamallocator/prober.go index c30202454..d2fca217e 100644 --- a/pkg/sfu/streamallocator/prober.go +++ b/pkg/sfu/streamallocator/prober.go @@ -189,7 +189,7 @@ func (p *Prober) Reset() { p.clustersMu.Lock() if p.activeCluster != nil { - p.logger.Infow("prober: resetting active cluster", "cluster", p.activeCluster.String()) + p.logger.Debugw("prober: resetting active cluster", "cluster", p.activeCluster.String()) reset = true info = p.activeCluster.GetInfo() } diff --git a/pkg/sfu/streamallocator/streamallocator.go b/pkg/sfu/streamallocator/streamallocator.go index 87092b434..caeb594db 100644 --- a/pkg/sfu/streamallocator/streamallocator.go +++ b/pkg/sfu/streamallocator/streamallocator.go @@ -818,16 +818,29 @@ func (s *StreamAllocator) handleNewEstimateInNonProbe() { action = "skipping" } - s.params.Logger.Infow( - fmt.Sprintf("stream allocator: channel congestion detected, %s channel capacity update", action), - "reason", reason, - "old(bps)", s.committedChannelCapacity, - "new(bps)", estimateToCommit, - "lastReceived(bps)", s.lastReceivedEstimate, - "expectedUsage(bps)", expectedBandwidthUsage, - "commitThreshold(bps)", commitThreshold, - "channel", s.channelObserver.ToString(), - ) + if action == "applying" { + s.params.Logger.Infow( + fmt.Sprintf("stream allocator: channel congestion detected, %s channel capacity update", action), + "reason", reason, + "old(bps)", s.committedChannelCapacity, + "new(bps)", estimateToCommit, + "lastReceived(bps)", s.lastReceivedEstimate, + "expectedUsage(bps)", expectedBandwidthUsage, + "commitThreshold(bps)", commitThreshold, + "channel", s.channelObserver.ToString(), + ) + } else { + s.params.Logger.Debugw( + fmt.Sprintf("stream allocator: channel congestion detected, %s channel capacity update", action), + "reason", reason, + "old(bps)", s.committedChannelCapacity, + "new(bps)", estimateToCommit, + "lastReceived(bps)", s.lastReceivedEstimate, + "expectedUsage(bps)", expectedBandwidthUsage, + "commitThreshold(bps)", commitThreshold, + "channel", s.channelObserver.ToString(), + ) + } /* STREAM-ALLOCATOR-DATA s.params.Logger.Debugw( fmt.Sprintf("stream allocator: channel congestion detected, %s channel capacity: experimental", action), From 410bf7c6ec88ce0eddb075c9d2baa71648954c8f Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 28 May 2024 23:37:02 -0700 Subject: [PATCH 46/60] fix(deps): update golang.org/x/exp digest to 23cca88 (#2729) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 705eb4624..e62ed307a 100644 --- a/go.mod +++ b/go.mod @@ -49,7 +49,7 @@ require ( github.com/urfave/negroni/v3 v3.1.0 go.uber.org/atomic v1.11.0 go.uber.org/zap v1.27.0 - golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 + golang.org/x/exp v0.0.0-20240529005216-23cca8864a10 golang.org/x/sync v0.7.0 google.golang.org/protobuf v1.34.1 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index 53a41359b..0c9273992 100644 --- a/go.sum +++ b/go.sum @@ -286,8 +286,8 @@ golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1m golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= -golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= -golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= +golang.org/x/exp v0.0.0-20240529005216-23cca8864a10 h1:vpzMC/iZhYFAjJzHU0Cfuq+w1vLLsF2vLkDrPjzKYck= +golang.org/x/exp v0.0.0-20240529005216-23cca8864a10/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= From 3ad0a69c86d8e52701f0bd4b19891ed4d3fd19ec Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 29 May 2024 18:26:22 +0530 Subject: [PATCH 47/60] Stop probe on probe controller reset (#2744) --- pkg/sfu/streamallocator/probe_controller.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pkg/sfu/streamallocator/probe_controller.go b/pkg/sfu/streamallocator/probe_controller.go index 7426683c5..0a49af5de 100644 --- a/pkg/sfu/streamallocator/probe_controller.go +++ b/pkg/sfu/streamallocator/probe_controller.go @@ -65,6 +65,7 @@ func (p *ProbeController) Reset() { p.resetProbeIntervalLocked() p.resetProbeDurationLocked() + p.StopProbe() p.clearProbeLocked() } @@ -73,7 +74,7 @@ func (p *ProbeController) ProbeClusterDone(info ProbeClusterInfo) { defer p.lock.Unlock() if p.probeClusterId != info.Id { - p.params.Logger.Infow("not expected probe cluster", "probeClusterId", p.probeClusterId, "resetProbeClusterId", info.Id) + p.params.Logger.Debugw("not expected probe cluster", "probeClusterId", p.probeClusterId, "resetProbeClusterId", info.Id) return } @@ -138,7 +139,9 @@ func (p *ProbeController) MaybeFinalizeProbe( return true, true, true } - if (isComplete || p.abortedProbeClusterId != ProbeClusterIdInvalid) && p.probeEndTime.IsZero() && p.doneProbeClusterInfo.Id != ProbeClusterIdInvalid && p.doneProbeClusterInfo.Id == p.probeClusterId { + if (isComplete || p.abortedProbeClusterId != ProbeClusterIdInvalid) && + p.probeEndTime.IsZero() && + p.doneProbeClusterInfo.Id != ProbeClusterIdInvalid && p.doneProbeClusterInfo.Id == p.probeClusterId { // ensure any queueing due to probing is flushed // STREAM-ALLOCATOR-TODO: CongestionControlProbeConfig.SettleWait should actually be a certain number of RTTs. expectedDuration := float64(9.0) @@ -165,7 +168,7 @@ func (p *ProbeController) MaybeFinalizeProbe( } if !p.probeEndTime.IsZero() && time.Now().After(p.probeEndTime) { - // finalisze aborted or non-failing but non-goal-reached probe cluster + // finalize aborted or non-failing but non-goal-reached probe cluster return true, p.finalizeProbeLocked(trend), false } From 526d3a017e3c535f416393d2fea568c4d894b9f2 Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Fri, 31 May 2024 10:23:37 +0800 Subject: [PATCH 48/60] ignore abnormal forward delay (#2748) --- pkg/sfu/forwardstats.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/sfu/forwardstats.go b/pkg/sfu/forwardstats.go index 62544e3a8..47c83f0d5 100644 --- a/pkg/sfu/forwardstats.go +++ b/pkg/sfu/forwardstats.go @@ -27,13 +27,18 @@ func NewForwardStats(latencyUpdateInterval, reportInterval, latencyWindowLength } func (s *ForwardStats) Update(arrival, left time.Time) { + transit := left.Sub(arrival) + + // ignore if transit is too large or negative, this could happen if system time is adjusted + if transit < 0 || transit > 5*time.Second { + return + } leftMs := left.UnixMilli() lastMs := s.lastLeftMs.Load() if leftMs < lastMs || !s.lastLeftMs.CompareAndSwap(lastMs, leftMs) { return } - transit := left.Sub(arrival) s.lock.Lock() defer s.lock.Unlock() s.latency.Update(time.Duration(arrival.UnixNano()), float64(transit)) From b99650aaf6ea1747055be4f4890061ec674cf91e Mon Sep 17 00:00:00 2001 From: David Zhao Date: Sun, 2 Jun 2024 09:09:55 -0700 Subject: [PATCH 49/60] Send NodeID with analytics events (#2749) --- go.mod | 2 +- go.sum | 4 ++-- pkg/telemetry/analyticsservice.go | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e62ed307a..a1412367f 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20240501132628-6105557bbb9a - github.com/livekit/protocol v1.16.1-0.20240529013258-ec704e0cf263 + github.com/livekit/protocol v1.17.0 github.com/livekit/psrpc v0.5.3-0.20240526192918-fbdaf10e6aa5 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 diff --git a/go.sum b/go.sum index 0c9273992..92a6e5895 100644 --- a/go.sum +++ b/go.sum @@ -120,8 +120,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20240501132628-6105557bbb9a h1:ATbv0x7G5tW2HgiouQ57csFE/G4gekl2oV1cxb2Dy24= github.com/livekit/mediatransportutil v0.0.0-20240501132628-6105557bbb9a/go.mod h1:jwKUCmObuiEDH0iiuJHaGMXwRs3RjrB4G6qqgkr/5oE= -github.com/livekit/protocol v1.16.1-0.20240529013258-ec704e0cf263 h1:5Iat/a2LYG4HG+4voZrpnNxhfiVd1G2M8dOVnK/uRIA= -github.com/livekit/protocol v1.16.1-0.20240529013258-ec704e0cf263/go.mod h1:cN8WmGQR+kWz1+UWcAQdFFUcbW76PnfZDdkLAbYIqd4= +github.com/livekit/protocol v1.17.0 h1:E8t9kssblb1MVEYTPF0R65lFqN4pEr4SOcnlbcI3ehc= +github.com/livekit/protocol v1.17.0/go.mod h1:cN8WmGQR+kWz1+UWcAQdFFUcbW76PnfZDdkLAbYIqd4= github.com/livekit/psrpc v0.5.3-0.20240526192918-fbdaf10e6aa5 h1:mTZyrjk5WEWMsvaYtJ42pG7DuxysKj21DKPINpGSIto= github.com/livekit/psrpc v0.5.3-0.20240526192918-fbdaf10e6aa5/go.mod h1:CQUBSPfYYAaevg1TNCc6/aYsa8DJH4jSRFdCeSZk5u0= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= diff --git a/pkg/telemetry/analyticsservice.go b/pkg/telemetry/analyticsservice.go index 3f1955873..aca94a22e 100644 --- a/pkg/telemetry/analyticsservice.go +++ b/pkg/telemetry/analyticsservice.go @@ -70,6 +70,7 @@ func (a *analyticsService) SendEvent(_ context.Context, event *livekit.Analytics return } + event.NodeId = a.nodeID event.AnalyticsKey = a.analyticsKey if err := a.events.Send(&livekit.AnalyticsEvents{ Events: []*livekit.AnalyticsEvent{event}, From e483cee662a10c4485e58ac48ad532d7d4a2eb01 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 3 Jun 2024 11:55:59 +0530 Subject: [PATCH 50/60] Prevent tracker override. (#2750) When relaying buffers are stopped and restarted. On a restart, the buffer adds a tracker. But, the tracker is not destroyed till the end. So, the old tracker and new tracker for the same layer stomp on each other and declare layer unavailable (the old tracker is not getting any packets). Fix by not creating a new tracker if one exists already. --- pkg/sfu/streamtrackermanager.go | 48 ++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/pkg/sfu/streamtrackermanager.go b/pkg/sfu/streamtrackermanager.go index 0158c55c7..5f12e85c2 100644 --- a/pkg/sfu/streamtrackermanager.go +++ b/pkg/sfu/streamtrackermanager.go @@ -168,35 +168,39 @@ func (s *StreamTrackerManager) AddDependencyDescriptorTrackers() { } func (s *StreamTrackerManager) AddTracker(layer int32) streamtracker.StreamTrackerWorker { + var tracker streamtracker.StreamTrackerWorker + s.lock.Lock() + if s.ddTracker != nil { + tracker = s.ddTracker.LayeredTracker(layer) + } else { + tracker = s.trackers[layer] + } + s.lock.Unlock() + if tracker != nil { + return tracker + } + bitrateInterval, ok := s.trackerConfig.BitrateReportInterval[layer] if !ok { return nil } - var tracker streamtracker.StreamTrackerWorker - s.lock.Lock() - if s.ddTracker != nil { - tracker = s.ddTracker.LayeredTracker(layer) + var trackerImpl streamtracker.StreamTrackerImpl + switch s.trackerConfig.StreamTrackerType { + case config.StreamTrackerTypePacket: + trackerImpl = s.createStreamTrackerPacket(layer) + case config.StreamTrackerTypeFrame: + trackerImpl = s.createStreamTrackerFrame(layer) + } + if trackerImpl == nil { + return nil } - s.lock.Unlock() - if tracker == nil { - var trackerImpl streamtracker.StreamTrackerImpl - switch s.trackerConfig.StreamTrackerType { - case config.StreamTrackerTypePacket: - trackerImpl = s.createStreamTrackerPacket(layer) - case config.StreamTrackerTypeFrame: - trackerImpl = s.createStreamTrackerFrame(layer) - } - if trackerImpl == nil { - return nil - } - tracker = streamtracker.NewStreamTracker(streamtracker.StreamTrackerParams{ - StreamTrackerImpl: trackerImpl, - BitrateReportInterval: bitrateInterval, - Logger: s.logger.WithValues("layer", layer), - }) - } + tracker = streamtracker.NewStreamTracker(streamtracker.StreamTrackerParams{ + StreamTrackerImpl: trackerImpl, + BitrateReportInterval: bitrateInterval, + Logger: s.logger.WithValues("layer", layer), + }) s.logger.Debugw("stream tracker add track", "layer", layer) tracker.OnStatusChanged(func(status streamtracker.StreamStatus) { From b7ea733492a2fd33285875dfb539f9c3cc0ecdc0 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 3 Jun 2024 12:14:42 +0530 Subject: [PATCH 51/60] Fix DD tracker addition. (#2751) DD uses a wrapped tracker. Check only stream tracker manager field for existing tracker. --- pkg/sfu/streamtrackermanager.go | 45 ++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/pkg/sfu/streamtrackermanager.go b/pkg/sfu/streamtrackermanager.go index 5f12e85c2..d4bf08f44 100644 --- a/pkg/sfu/streamtrackermanager.go +++ b/pkg/sfu/streamtrackermanager.go @@ -170,37 +170,40 @@ func (s *StreamTrackerManager) AddDependencyDescriptorTrackers() { func (s *StreamTrackerManager) AddTracker(layer int32) streamtracker.StreamTrackerWorker { var tracker streamtracker.StreamTrackerWorker s.lock.Lock() - if s.ddTracker != nil { - tracker = s.ddTracker.LayeredTracker(layer) - } else { - tracker = s.trackers[layer] - } - s.lock.Unlock() + tracker = s.trackers[layer] if tracker != nil { + s.lock.Unlock() return tracker } + if s.ddTracker != nil { + tracker = s.ddTracker.LayeredTracker(layer) + } + s.lock.Unlock() + bitrateInterval, ok := s.trackerConfig.BitrateReportInterval[layer] if !ok { return nil } - var trackerImpl streamtracker.StreamTrackerImpl - switch s.trackerConfig.StreamTrackerType { - case config.StreamTrackerTypePacket: - trackerImpl = s.createStreamTrackerPacket(layer) - case config.StreamTrackerTypeFrame: - trackerImpl = s.createStreamTrackerFrame(layer) - } - if trackerImpl == nil { - return nil - } + if tracker == nil { + var trackerImpl streamtracker.StreamTrackerImpl + switch s.trackerConfig.StreamTrackerType { + case config.StreamTrackerTypePacket: + trackerImpl = s.createStreamTrackerPacket(layer) + case config.StreamTrackerTypeFrame: + trackerImpl = s.createStreamTrackerFrame(layer) + } + if trackerImpl == nil { + return nil + } - tracker = streamtracker.NewStreamTracker(streamtracker.StreamTrackerParams{ - StreamTrackerImpl: trackerImpl, - BitrateReportInterval: bitrateInterval, - Logger: s.logger.WithValues("layer", layer), - }) + tracker = streamtracker.NewStreamTracker(streamtracker.StreamTrackerParams{ + StreamTrackerImpl: trackerImpl, + BitrateReportInterval: bitrateInterval, + Logger: s.logger.WithValues("layer", layer), + }) + } s.logger.Debugw("stream tracker add track", "layer", layer) tracker.OnStatusChanged(func(status streamtracker.StreamStatus) { From b90ade81da4a797cff5fb912e9c75d99eddea4c1 Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Mon, 3 Jun 2024 15:12:53 +0800 Subject: [PATCH 52/60] add log for unexpected jitter (#2752) --- pkg/sfu/forwardstats.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/pkg/sfu/forwardstats.go b/pkg/sfu/forwardstats.go index 47c83f0d5..0924e199a 100644 --- a/pkg/sfu/forwardstats.go +++ b/pkg/sfu/forwardstats.go @@ -1,11 +1,13 @@ package sfu import ( + "fmt" "sync" "sync/atomic" "time" "github.com/livekit/livekit-server/pkg/telemetry/prometheus" + "github.com/livekit/protocol/logger" "github.com/livekit/protocol/utils" ) @@ -46,9 +48,17 @@ func (s *ForwardStats) Update(arrival, left time.Time) { func (s *ForwardStats) GetStats() (latency, jitter time.Duration) { s.lock.Lock() - defer s.lock.Unlock() w := s.latency.Summarize() - return time.Duration(w.Mean()), time.Duration(w.StdDev()) + s.lock.Unlock() + latency, jitter = time.Duration(w.Mean()), time.Duration(w.StdDev()) + // TODO: remove this check after debugging unexpected jitter issue + if jitter > 10*time.Second { + logger.Infow("unexpected forward jitter", + "jitter", jitter, + "stats", fmt.Sprintf("count %.2f, mean %.2f, stdDev %.2f", w.Count(), w.Mean(), w.StdDev()), + ) + } + return } func (s *ForwardStats) GetLastStats(duration time.Duration) (latency, jitter time.Duration) { From 1e52b09e249acd2f85e218b95a45843f542606be Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 3 Jun 2024 13:58:58 +0530 Subject: [PATCH 53/60] Reset tracker on expected layer increase. (#2753) Was hitting the edge case mentioned in the (now deleted in this PR) comments. It is fine to reset and let it declare available again. Available layer handler will ignore repeats. --- pkg/sfu/streamtrackermanager.go | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/pkg/sfu/streamtrackermanager.go b/pkg/sfu/streamtrackermanager.go index d4bf08f44..e2091b61f 100644 --- a/pkg/sfu/streamtrackermanager.go +++ b/pkg/sfu/streamtrackermanager.go @@ -325,20 +325,11 @@ func (s *StreamTrackerManager) SetMaxExpectedSpatialLayer(layer int32) int32 { // // Some higher layer is expected to start. // If the layer was not detected as stopped (i.e. it is still in available layers), - // don't need to do anything. If not, reset the stream tracker so that - // the layer is declared available on the first packet. - // - // NOTE: There may be a race between checking if a layer is available and - // resetting the tracker, i.e. the track may stop just after checking. - // But, those conditions should be rare. In those cases, the restart will - // take longer. + // resetting tracker will declare layer available afresh. That's fine as it will be + // a no-op in available layers handling. // var trackersToReset []streamtracker.StreamTrackerWorker for l := s.maxExpectedLayer + 1; l <= layer; l++ { - if s.hasSpatialLayerLocked(l) { - continue - } - if s.trackers[l] != nil { trackersToReset = append(trackersToReset, s.trackers[l]) } From 03bb468472d5e0d23f30b70ec217dc9a7a2140dd Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 4 Jun 2024 08:00:26 +0530 Subject: [PATCH 54/60] Log range map for debugging. (#2754) * Log range map for debugging. * log details on errors * log details --- pkg/sfu/buffer/buffer.go | 36 ++++++++++++++++++++++++++++++++---- pkg/sfu/utils/rangemap.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 4 deletions(-) diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go index 2b82f3760..e48692ba2 100644 --- a/pkg/sfu/buffer/buffer.go +++ b/pkg/sfu/buffer/buffer.go @@ -551,7 +551,13 @@ func (b *Buffer) calc(rawPkt []byte, rtpPacket *rtp.Packet, arrivalTime time.Tim // 44 - padding only - out-of-order + duplicate - dropped as duplicate // if err := b.snRangeMap.ExcludeRange(flowState.ExtSequenceNumber, flowState.ExtSequenceNumber+1); err != nil { - b.logger.Errorw("could not exclude range", err, "sn", rtpPacket.SequenceNumber, "esn", flowState.ExtSequenceNumber) + b.logger.Errorw( + "could not exclude range", err, + "sn", rtpPacket.SequenceNumber, + "esn", flowState.ExtSequenceNumber, + "rtpStats", b.rtpStats, + "snRangeMap", b.snRangeMap, + ) } } return @@ -560,7 +566,14 @@ func (b *Buffer) calc(rawPkt []byte, rtpPacket *rtp.Packet, arrivalTime time.Tim // add to RTX buffer using sequence number after accounting for dropped padding only packets snAdjustment, err := b.snRangeMap.GetValue(flowState.ExtSequenceNumber) if err != nil { - b.logger.Errorw("could not get sequence number adjustment", err, "sn", flowState.ExtSequenceNumber, "payloadSize", len(rtpPacket.Payload)) + b.logger.Errorw( + "could not get sequence number adjustment", err, + "sn", rtpPacket.SequenceNumber, + "esn", flowState.ExtSequenceNumber, + "payloadSize", len(rtpPacket.Payload), + "rtpStats", b.rtpStats, + "snRangeMap", b.snRangeMap, + ) return } flowState.ExtSequenceNumber -= snAdjustment @@ -577,10 +590,18 @@ func (b *Buffer) calc(rawPkt []byte, rtpPacket *rtp.Packet, arrivalTime time.Tim "snAdjustment", snAdjustment, "incomingSequenceNumber", flowState.ExtSequenceNumber+snAdjustment, "rtpStats", b.rtpStats, + "snRangeMap", b.snRangeMap, ) } } else if err != bucket.ErrRTXPacket { - b.logger.Warnw("could not add packet to bucket", err) + b.logger.Warnw( + "could not add packet to bucket", err, + "flowState", &flowState, + "snAdjustment", snAdjustment, + "incomingSequenceNumber", flowState.ExtSequenceNumber+snAdjustment, + "rtpStats", b.rtpStats, + "snRangeMap", b.snRangeMap, + ) } return } @@ -605,7 +626,14 @@ func (b *Buffer) patchExtPacket(ep *ExtPacket, buf []byte) *ExtPacket { if err != nil { packetNotFoundCount := b.packetNotFoundCount.Inc() if (packetNotFoundCount-1)%20 == 0 { - b.logger.Warnw("could not get packet from bucket", err, "sn", ep.Packet.SequenceNumber, "headSN", b.bucket.HeadSequenceNumber(), "count", packetNotFoundCount) + b.logger.Warnw( + "could not get packet from bucket", err, + "sn", ep.Packet.SequenceNumber, + "headSN", b.bucket.HeadSequenceNumber(), + "count", packetNotFoundCount, + "rtpStats", b.rtpStats, + "snRangeMap", b.snRangeMap, + ) } return nil } diff --git a/pkg/sfu/utils/rangemap.go b/pkg/sfu/utils/rangemap.go index fe5480832..6ca2332eb 100644 --- a/pkg/sfu/utils/rangemap.go +++ b/pkg/sfu/utils/rangemap.go @@ -19,6 +19,8 @@ import ( "fmt" "math" "unsafe" + + "go.uber.org/zap/zapcore" ) const ( @@ -40,12 +42,23 @@ type valueType interface { uint32 | uint64 } +// --------------------------------------------------- + type rangeVal[RT rangeType, VT valueType] struct { start RT end RT value VT } +func (r rangeVal[RT, VT]) MarshalLogObject(e zapcore.ObjectEncoder) error { + e.AddUint64("start", uint64(r.start)) + e.AddUint64("end", uint64(r.end)) + e.AddUint64("value", uint64(r.value)) + return nil +} + +// --------------------------------------------------- + type RangeMap[RT rangeType, VT valueType] struct { halfRange RT @@ -63,6 +76,21 @@ func NewRangeMap[RT rangeType, VT valueType](size int) *RangeMap[RT, VT] { return r } +func (r *RangeMap[RT, VT]) MarshalLogObject(e zapcore.ObjectEncoder) error { + e.AddInt("numRanges", len(r.ranges)) + + // just the last 10 ranges max + startIdx := len(r.ranges) - 10 + if startIdx < 0 { + startIdx = 0 + } + for i := startIdx; i < len(r.ranges); i++ { + e.AddObject(fmt.Sprintf("range[%d]", i), r.ranges[i]) + } + + return nil +} + func (r *RangeMap[RT, VT]) ClearAndResetValue(start RT, val VT) { r.initRanges(start, val) } From 49e1848d1d4b17cf2f0f52bc32bba21a66262de4 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 4 Jun 2024 22:38:39 +0530 Subject: [PATCH 55/60] Move resync to TrackSender interface so that it can be called directly from up track (#2755) --- pkg/rtc/mediatracksubscriptions.go | 8 -------- pkg/sfu/downtrack.go | 1 + 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/pkg/rtc/mediatracksubscriptions.go b/pkg/rtc/mediatracksubscriptions.go index 78c045e88..aaf5b02a1 100644 --- a/pkg/rtc/mediatracksubscriptions.go +++ b/pkg/rtc/mediatracksubscriptions.go @@ -330,14 +330,6 @@ func (t *MediaTrackSubscriptions) closeSubscribedTrack(subTrack types.Subscribed } } -func (t *MediaTrackSubscriptions) ResyncAllSubscribers() { - t.params.Logger.Debugw("resyncing all subscribers") - - for _, subTrack := range t.getAllSubscribedTracks() { - subTrack.DownTrack().Resync() - } -} - func (t *MediaTrackSubscriptions) GetAllSubscribers() []livekit.ParticipantID { t.subscribedTracksMu.RLock() defer t.subscribedTracksMu.RUnlock() diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 5f3dd893f..f9d1f1358 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -63,6 +63,7 @@ type TrackSender interface { layer int32, publisherSRData *buffer.RTCPSenderReportData, ) error + Resync() } // ------------------------------------------------------------------- From 857c3ab63e588b09bb61b4da0bb256301510d803 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 4 Jun 2024 21:59:17 -0700 Subject: [PATCH 56/60] chore(deps): update goreleaser/goreleaser-action action to v6 (#2756) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index ef0960712..84f9ef1fb 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -38,7 +38,7 @@ jobs: go-version: '>=1.21' - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v5 + uses: goreleaser/goreleaser-action@v6 with: distribution: goreleaser version: latest From fdde44d926139c4c7fc27538eb40f35fe6a76139 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 5 Jun 2024 13:02:33 +0530 Subject: [PATCH 57/60] Attach time stamp offset to publisher sender report. (#2757) There are cases where the RTP time stamp does not increment acros mute/unmute. Seems to happen fairly consistently with React Native clients. Something like the following happens - Track is progressing - Mute at `t = x`, assume RTP time stamp at the point is `y` and RTP clock rate is `z`. - Through mute, more RTCP sender reports come in from publisher and the RTP time stamp in those reports are progressing at expected rate of `z` RTP clock ticks per second. - Forwarding path uses those sender reports from publisher to build the sender report for subscribers. - Unmute happens at `t = x + a` seconds. - Ideally packets coming in after that, should have a time stamp of `y + (a * z)`, but they tend to have something a little bit more than `y`. - RTCP sender reports also have a time stamp that goes back. SFU ignores these. - Mean while the forwarding path has adjusted to the new RTP time stamp base and it has calculated a TS offset (from publisher -> subscriber). Effectively, that offset comes out close to `(a * z)`, i. e. jump corresponding to the mute interval. - When it is time to send a RTCP sender report to subscriber, the old sender report from publisher is used (as intervening ones from publisher were rejected because time stamp is moving backwards). The problem is that the old report is used with new offset. So, it looks like time stamp jumped ahead by `a` seconds. Address it by storing time stamp offset at the time of receiving the publisher side sender report. And use that while sending subscriber side sender report. There are very edge cases where this can get mismatched, but should be rare. Hopefully, this should prevent unnecessary jumps in time stamp in RTCP sender report to subscribers. --- pkg/sfu/forwarder.go | 59 ++++++++++++++++++++++++++------------------ 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index bc0e7f31c..87c6cc345 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -216,8 +216,9 @@ func (f ForwarderState) String() string { // ------------------------------------------------------------------- type refInfo struct { - senderReport *buffer.RTCPSenderReportData - tsOffset uint64 + senderReport *buffer.RTCPSenderReportData + tsOffset uint64 + isTSOffsetValid bool } type Forwarder struct { @@ -567,16 +568,39 @@ func (f *Forwarder) GetMaxSubscribedSpatial() int32 { return layer } +func (f *Forwarder) getReferenceLayer() (int32, int32) { + if f.lastSSRC == 0 { + return buffer.InvalidLayerSpatial, buffer.InvalidLayerSpatial + } + + if f.kind == webrtc.RTPCodecTypeAudio { + return 0, 0 + } + + currentLayerSpatial := f.vls.GetCurrent().Spatial + if currentLayerSpatial < 0 || currentLayerSpatial > buffer.DefaultMaxLayerSpatial { + return buffer.InvalidLayerSpatial, buffer.InvalidLayerSpatial + } + + if f.refIsSVC { + return 0, currentLayerSpatial + } + + return currentLayerSpatial, currentLayerSpatial +} + func (f *Forwarder) SetRefSenderReport(isSVC bool, layer int32, srData *buffer.RTCPSenderReportData) { f.lock.Lock() defer f.lock.Unlock() f.refIsSVC = isSVC - if isSVC { - layer = 0 - } + refLayer, _ := f.getReferenceLayer() if layer >= 0 && int(layer) < len(f.refInfos) { - f.refInfos[layer].senderReport = srData + f.refInfos[layer] = refInfo{srData, 0, false} + if layer == refLayer { + f.refInfos[layer].tsOffset = f.rtpMunger.GetTSOffset() + f.refInfos[layer].isTSOffsetValid = true + } } } @@ -611,7 +635,7 @@ func (f *Forwarder) clearRefSenderReportsLocked() { // By clearing sender report on (re)start of a stream, subscribers will wait for a fresh report // after unmute to send sender report. for layer := int32(0); layer < buffer.DefaultMaxLayerSpatial+1; layer++ { - f.refInfos[layer] = refInfo{nil, 0} + f.refInfos[layer] = refInfo{nil, 0, false} } } @@ -619,23 +643,12 @@ func (f *Forwarder) GetSenderReportParams() (int32, uint64, *buffer.RTCPSenderRe f.lock.RLock() defer f.lock.RUnlock() - if f.kind == webrtc.RTPCodecTypeAudio { - return 0, f.refInfos[0].tsOffset, f.refInfos[0].senderReport + refLayer, currentLayerSpatial := f.getReferenceLayer() + if refLayer == buffer.InvalidLayerSpatial || !f.refInfos[refLayer].isTSOffsetValid { + return buffer.InvalidLayerSpatial, 0, nil } - currentLayerSpatial := f.vls.GetCurrent().Spatial - if currentLayerSpatial < 0 || currentLayerSpatial > buffer.DefaultMaxLayerSpatial { - return currentLayerSpatial, 0, nil - } - - refSenderReport := f.refInfos[currentLayerSpatial].senderReport - tsOffset := f.refInfos[currentLayerSpatial].tsOffset - if f.refIsSVC { - refSenderReport = f.refInfos[0].senderReport - tsOffset = f.refInfos[0].tsOffset - } - - return currentLayerSpatial, tsOffset, refSenderReport + return currentLayerSpatial, f.refInfos[refLayer].tsOffset, f.refInfos[refLayer].senderReport } func (f *Forwarder) isDeficientLocked() bool { @@ -1588,7 +1601,6 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e f.codecMunger.SetLast(extPkt) f.clearRefSenderReportsLocked() - f.refInfos[layer].tsOffset = f.rtpMunger.GetTSOffset() f.logger.Debugw( "starting forwarding", @@ -1827,7 +1839,6 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e } f.rtpMunger.UpdateSnTsOffsets(extPkt, 1, extNextTS-extLastTS) - f.refInfos[layer].tsOffset = f.rtpMunger.GetTSOffset() f.codecMunger.UpdateOffsets(extPkt) return nil } From cfd3777f47423345e9065207671e72466b80e411 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 5 Jun 2024 21:54:22 +0530 Subject: [PATCH 58/60] Use a safety net OnClose to remove track from peer connection. (#2758) --- pkg/rtc/mediatracksubscriptions.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pkg/rtc/mediatracksubscriptions.go b/pkg/rtc/mediatracksubscriptions.go index aaf5b02a1..e2f4d5b00 100644 --- a/pkg/rtc/mediatracksubscriptions.go +++ b/pkg/rtc/mediatracksubscriptions.go @@ -289,6 +289,20 @@ func (t *MediaTrackSubscriptions) AddSubscriber(sub types.LocalParticipant, wr * // negotiation isn't required if we've replaced track subTrack.SetNeedsNegotiation(!replacedTrack) subTrack.SetRTPSender(sender) + // it is possible that subscribed track is closed before subscription manager sets + // the `OnClose` callback. That handler in subscription manager removes the track + // from the peer connection. + // + // But, the subscription could be removed early if the published track is closed + // while adding subscription. In those cases, subscription manager would not have set + // the `OnClose` callback. So, set it here to handle cases of early close. + subTrack.OnClose(func(willBeResumed bool) { + if !willBeResumed { + if err := sub.RemoveTrackFromSubscriber(sender); err != nil { + t.params.Logger.Warnw("could not remove track from peer connection", err) + } + } + }) downTrack.SetTransceiver(transceiver) From d3f9e8b8e5691c6f332b9fcb1bc963332a49bec5 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 5 Jun 2024 23:06:22 +0530 Subject: [PATCH 59/60] Do not add tracker for invalid layers. (#2759) Previously, the bit rate interval config was checked first. That would have returned `!ok` for invalid layers. A recent change to prevent duplicate tracker addition re-arranged the code and the tracker array was accessed out-of-bounds. Unclear why an invalid layer is passed in. Need to investigate that. --- pkg/sfu/streamtrackermanager.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/sfu/streamtrackermanager.go b/pkg/sfu/streamtrackermanager.go index e2091b61f..890b70257 100644 --- a/pkg/sfu/streamtrackermanager.go +++ b/pkg/sfu/streamtrackermanager.go @@ -168,6 +168,10 @@ func (s *StreamTrackerManager) AddDependencyDescriptorTrackers() { } func (s *StreamTrackerManager) AddTracker(layer int32) streamtracker.StreamTrackerWorker { + if layer < 0 || int(layer) >= len(s.trackers) { + return nil + } + var tracker streamtracker.StreamTrackerWorker s.lock.Lock() tracker = s.trackers[layer] From f1886ece4278a4c63fada9a48ac7fe2b5595d656 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Wed, 5 Jun 2024 19:46:34 -0700 Subject: [PATCH 60/60] update protocol (#2760) * update protocol * deps --- cmd/server/commands.go | 3 ++- go.mod | 2 +- go.sum | 4 ++-- pkg/agent/client.go | 3 ++- pkg/agent/worker.go | 3 ++- pkg/routing/node.go | 3 ++- pkg/routing/selector/regionaware_test.go | 3 ++- pkg/routing/signal.go | 4 ++-- pkg/rtc/participant.go | 3 ++- pkg/rtc/participant_internal_test.go | 3 ++- pkg/rtc/testutils.go | 5 +++-- pkg/service/clients.go | 3 ++- pkg/service/ingress.go | 5 +++-- pkg/service/redisstore.go | 4 ++-- pkg/service/redisstore_test.go | 9 +++++---- pkg/service/roomallocator.go | 3 ++- pkg/service/roommanager.go | 3 ++- pkg/service/sip.go | 5 +++-- pkg/telemetry/analyticsservice.go | 7 ++++--- pkg/telemetry/events.go | 4 ++-- test/integration_helpers.go | 8 ++++---- test/webhook_test.go | 4 ++-- 22 files changed, 53 insertions(+), 38 deletions(-) diff --git a/cmd/server/commands.go b/cmd/server/commands.go index 624af6345..147352427 100644 --- a/cmd/server/commands.go +++ b/cmd/server/commands.go @@ -28,6 +28,7 @@ import ( "github.com/livekit/protocol/auth" "github.com/livekit/protocol/utils" + "github.com/livekit/protocol/utils/guid" "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/routing" @@ -35,7 +36,7 @@ import ( ) func generateKeys(_ *cli.Context) error { - apiKey := utils.NewGuid(utils.APIKeyPrefix) + apiKey := guid.New(utils.APIKeyPrefix) secret := utils.RandomSecret() fmt.Println("API Key: ", apiKey) fmt.Println("API Secret: ", secret) diff --git a/go.mod b/go.mod index a1412367f..78874e7ed 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20240501132628-6105557bbb9a - github.com/livekit/protocol v1.17.0 + github.com/livekit/protocol v1.17.1-0.20240606023900-429fec77a69b github.com/livekit/psrpc v0.5.3-0.20240526192918-fbdaf10e6aa5 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 diff --git a/go.sum b/go.sum index 92a6e5895..9beab0153 100644 --- a/go.sum +++ b/go.sum @@ -120,8 +120,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20240501132628-6105557bbb9a h1:ATbv0x7G5tW2HgiouQ57csFE/G4gekl2oV1cxb2Dy24= github.com/livekit/mediatransportutil v0.0.0-20240501132628-6105557bbb9a/go.mod h1:jwKUCmObuiEDH0iiuJHaGMXwRs3RjrB4G6qqgkr/5oE= -github.com/livekit/protocol v1.17.0 h1:E8t9kssblb1MVEYTPF0R65lFqN4pEr4SOcnlbcI3ehc= -github.com/livekit/protocol v1.17.0/go.mod h1:cN8WmGQR+kWz1+UWcAQdFFUcbW76PnfZDdkLAbYIqd4= +github.com/livekit/protocol v1.17.1-0.20240606023900-429fec77a69b h1:VZMvqc23x/dXRpJQLc6CIkCuLUjev0HDLFO9NCEqfOk= +github.com/livekit/protocol v1.17.1-0.20240606023900-429fec77a69b/go.mod h1:cN8WmGQR+kWz1+UWcAQdFFUcbW76PnfZDdkLAbYIqd4= github.com/livekit/psrpc v0.5.3-0.20240526192918-fbdaf10e6aa5 h1:mTZyrjk5WEWMsvaYtJ42pG7DuxysKj21DKPINpGSIto= github.com/livekit/psrpc v0.5.3-0.20240526192918-fbdaf10e6aa5/go.mod h1:CQUBSPfYYAaevg1TNCc6/aYsa8DJH4jSRFdCeSZk5u0= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= diff --git a/pkg/agent/client.go b/pkg/agent/client.go index 94eeed949..d2fdabf91 100644 --- a/pkg/agent/client.go +++ b/pkg/agent/client.go @@ -27,6 +27,7 @@ import ( "github.com/livekit/protocol/logger" "github.com/livekit/protocol/rpc" "github.com/livekit/protocol/utils" + "github.com/livekit/protocol/utils/guid" "github.com/livekit/psrpc" ) @@ -121,7 +122,7 @@ func (c *agentClient) LaunchJob(ctx context.Context, desc *JobDescription) { target.ForEach(func(ns string) { c.workers.Submit(func() { _, err := c.client.JobRequest(ctx, ns, jobTypeTopic, &livekit.Job{ - Id: utils.NewGuid(utils.AgentJobPrefix), + Id: guid.New(utils.AgentJobPrefix), Type: desc.JobType, Room: desc.Room, Participant: desc.Participant, diff --git a/pkg/agent/worker.go b/pkg/agent/worker.go index 666681b39..bff334988 100644 --- a/pkg/agent/worker.go +++ b/pkg/agent/worker.go @@ -28,6 +28,7 @@ import ( "github.com/livekit/protocol/logger" "github.com/livekit/protocol/utils" putil "github.com/livekit/protocol/utils" + "github.com/livekit/protocol/utils/guid" ) type WorkerProtocolVersion int @@ -353,7 +354,7 @@ func (w *Worker) handleSimulateJob(simulate *livekit.SimulateJobRequest) { } job := &livekit.Job{ - Id: utils.NewGuid(utils.AgentJobPrefix), + Id: guid.New(utils.AgentJobPrefix), Type: jobType, Room: simulate.Room, Participant: simulate.Participant, diff --git a/pkg/routing/node.go b/pkg/routing/node.go index 022d740ae..ff8f8d3ab 100644 --- a/pkg/routing/node.go +++ b/pkg/routing/node.go @@ -20,6 +20,7 @@ import ( "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/utils" + "github.com/livekit/protocol/utils/guid" "github.com/livekit/livekit-server/pkg/config" ) @@ -27,7 +28,7 @@ import ( type LocalNode *livekit.Node func NewLocalNode(conf *config.Config) (LocalNode, error) { - nodeID := utils.NewGuid(utils.NodePrefix) + nodeID := guid.New(utils.NodePrefix) if conf.RTC.NodeIP == "" { return nil, ErrIPNotSet } diff --git a/pkg/routing/selector/regionaware_test.go b/pkg/routing/selector/regionaware_test.go index 75a5bef31..bc4b2036f 100644 --- a/pkg/routing/selector/regionaware_test.go +++ b/pkg/routing/selector/regionaware_test.go @@ -22,6 +22,7 @@ import ( "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/utils" + "github.com/livekit/protocol/utils/guid" "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/routing/selector" @@ -152,7 +153,7 @@ func newTestNodeInRegion(region string, available bool) *livekit.Node { load = 1.0 } return &livekit.Node{ - Id: utils.NewGuid(utils.NodePrefix), + Id: guid.New(utils.NodePrefix), Region: region, State: livekit.NodeState_SERVING, Stats: &livekit.NodeStats{ diff --git a/pkg/routing/signal.go b/pkg/routing/signal.go index c4f05b781..858fab7c1 100644 --- a/pkg/routing/signal.go +++ b/pkg/routing/signal.go @@ -28,7 +28,7 @@ import ( "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/rpc" - "github.com/livekit/protocol/utils" + "github.com/livekit/protocol/utils/guid" "github.com/livekit/psrpc" "github.com/livekit/psrpc/pkg/middleware" ) @@ -84,7 +84,7 @@ func (r *signalClient) StartParticipantSignal( resSource MessageSource, err error, ) { - connectionID = livekit.ConnectionID(utils.NewGuid("CO_")) + connectionID = livekit.ConnectionID(guid.New("CO_")) ss, err := pi.ToStartSession(roomName, connectionID) if err != nil { return diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 05a2f27b2..1238fcca1 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -38,6 +38,7 @@ import ( "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/utils" + "github.com/livekit/protocol/utils/guid" "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/routing" @@ -2297,7 +2298,7 @@ func (p *ParticipantImpl) setStableTrackID(cid string, info *livekit.TrackInfo) case livekit.TrackSource_SCREEN_SHARE_AUDIO: trackPrefix += "s" } - trackID = utils.NewGuid(trackPrefix) + trackID = guid.New(trackPrefix) } info.Sid = trackID } diff --git a/pkg/rtc/participant_internal_test.go b/pkg/rtc/participant_internal_test.go index a7bd9b4e3..217213029 100644 --- a/pkg/rtc/participant_internal_test.go +++ b/pkg/rtc/participant_internal_test.go @@ -31,6 +31,7 @@ import ( "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/utils" + "github.com/livekit/protocol/utils/guid" "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/routing" @@ -749,7 +750,7 @@ func newParticipantForTestWithOpts(identity livekit.ParticipantIdentity, opts *p FmtpLine: c.FmtpLine, }) } - sid := livekit.ParticipantID(utils.NewGuid(utils.ParticipantPrefix)) + sid := livekit.ParticipantID(guid.New(utils.ParticipantPrefix)) p, _ := NewParticipant(ParticipantParams{ SID: sid, Identity: identity, diff --git a/pkg/rtc/testutils.go b/pkg/rtc/testutils.go index e233fa5f3..8231f02ff 100644 --- a/pkg/rtc/testutils.go +++ b/pkg/rtc/testutils.go @@ -18,6 +18,7 @@ import ( "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/utils" + "github.com/livekit/protocol/utils/guid" "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/rtc/types/typesfakes" @@ -25,7 +26,7 @@ import ( func NewMockParticipant(identity livekit.ParticipantIdentity, protocol types.ProtocolVersion, hidden bool, publisher bool) *typesfakes.FakeLocalParticipant { p := &typesfakes.FakeLocalParticipant{} - sid := utils.NewGuid(utils.ParticipantPrefix) + sid := guid.New(utils.ParticipantPrefix) p.IDReturns(livekit.ParticipantID(sid)) p.IdentityReturns(identity) p.StateReturns(livekit.ParticipantInfo_JOINED) @@ -80,7 +81,7 @@ func NewMockParticipant(identity livekit.ParticipantIdentity, protocol types.Pro func NewMockTrack(kind livekit.TrackType, name string) *typesfakes.FakeMediaTrack { t := &typesfakes.FakeMediaTrack{} - t.IDReturns(livekit.TrackID(utils.NewGuid(utils.TrackPrefix))) + t.IDReturns(livekit.TrackID(guid.New(utils.TrackPrefix))) t.KindReturns(kind) t.NameReturns(name) t.ToProtoReturns(&livekit.TrackInfo{ diff --git a/pkg/service/clients.go b/pkg/service/clients.go index 89faa691a..05c05ca7a 100644 --- a/pkg/service/clients.go +++ b/pkg/service/clients.go @@ -24,6 +24,7 @@ import ( "github.com/livekit/protocol/logger" "github.com/livekit/protocol/rpc" "github.com/livekit/protocol/utils" + "github.com/livekit/protocol/utils/guid" ) //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate @@ -73,7 +74,7 @@ func (s *egressLauncher) StartEgressWithClusterId(ctx context.Context, clusterId // Ensure we have an Egress ID if req.EgressId == "" { - req.EgressId = utils.NewGuid(utils.EgressPrefix) + req.EgressId = guid.New(utils.EgressPrefix) } return s.client.StartEgress(ctx, clusterId, req) diff --git a/pkg/service/ingress.go b/pkg/service/ingress.go index d5b87df81..e0a08b720 100644 --- a/pkg/service/ingress.go +++ b/pkg/service/ingress.go @@ -26,6 +26,7 @@ import ( "github.com/livekit/protocol/logger" "github.com/livekit/protocol/rpc" "github.com/livekit/protocol/utils" + "github.com/livekit/protocol/utils/guid" "github.com/livekit/psrpc" ) @@ -145,11 +146,11 @@ func (s *IngressService) CreateIngressWithUrl(ctx context.Context, urlStr string var sk string if req.InputType != livekit.IngressInput_URL_INPUT { - sk = utils.NewGuid("") + sk = guid.New("") } info := &livekit.IngressInfo{ - IngressId: utils.NewGuid(utils.IngressPrefix), + IngressId: guid.New(utils.IngressPrefix), Name: req.Name, StreamKey: sk, Url: urlStr, diff --git a/pkg/service/redisstore.go b/pkg/service/redisstore.go index 67a573874..52dfcaa62 100644 --- a/pkg/service/redisstore.go +++ b/pkg/service/redisstore.go @@ -29,7 +29,7 @@ import ( "github.com/livekit/protocol/ingress" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" - "github.com/livekit/protocol/utils" + "github.com/livekit/protocol/utils/guid" "github.com/livekit/livekit-server/version" ) @@ -240,7 +240,7 @@ func (s *RedisStore) DeleteRoom(ctx context.Context, roomName livekit.RoomName) } func (s *RedisStore) LockRoom(_ context.Context, roomName livekit.RoomName, duration time.Duration) (string, error) { - token := utils.NewGuid("LOCK") + token := guid.New("LOCK") key := RoomLockPrefix + string(roomName) startTime := time.Now() diff --git a/pkg/service/redisstore_test.go b/pkg/service/redisstore_test.go index 32e550378..5f62a71f1 100644 --- a/pkg/service/redisstore_test.go +++ b/pkg/service/redisstore_test.go @@ -26,6 +26,7 @@ import ( "github.com/livekit/protocol/ingress" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/utils" + "github.com/livekit/protocol/utils/guid" "github.com/livekit/livekit-server/pkg/service" ) @@ -164,8 +165,8 @@ func TestEgressStore(t *testing.T) { // store egress info info := &livekit.EgressInfo{ - EgressId: utils.NewGuid(utils.EgressPrefix), - RoomId: utils.NewGuid(utils.RoomPrefix), + EgressId: guid.New(utils.EgressPrefix), + RoomId: guid.New(utils.RoomPrefix), RoomName: roomName, Status: livekit.EgressStatus_EGRESS_STARTING, Request: &livekit.EgressInfo_RoomComposite{ @@ -184,8 +185,8 @@ func TestEgressStore(t *testing.T) { // store another info2 := &livekit.EgressInfo{ - EgressId: utils.NewGuid(utils.EgressPrefix), - RoomId: utils.NewGuid(utils.RoomPrefix), + EgressId: guid.New(utils.EgressPrefix), + RoomId: guid.New(utils.RoomPrefix), RoomName: "another-egress-test", Status: livekit.EgressStatus_EGRESS_STARTING, Request: &livekit.EgressInfo_RoomComposite{ diff --git a/pkg/service/roomallocator.go b/pkg/service/roomallocator.go index 54043e6a6..c8556e645 100644 --- a/pkg/service/roomallocator.go +++ b/pkg/service/roomallocator.go @@ -22,6 +22,7 @@ import ( "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/utils" + "github.com/livekit/protocol/utils/guid" "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/routing" @@ -66,7 +67,7 @@ func (r *StandardRoomAllocator) CreateRoom(ctx context.Context, req *livekit.Cre if errors.Is(err, ErrRoomNotFound) { created = true rm = &livekit.Room{ - Sid: utils.NewGuid(utils.RoomPrefix), + Sid: guid.New(utils.RoomPrefix), Name: req.Name, CreationTime: time.Now().Unix(), TurnPassword: utils.RandomSecret(), diff --git a/pkg/service/roommanager.go b/pkg/service/roommanager.go index 16437ff38..b4d180788 100644 --- a/pkg/service/roommanager.go +++ b/pkg/service/roommanager.go @@ -33,6 +33,7 @@ import ( "github.com/livekit/protocol/logger" "github.com/livekit/protocol/rpc" "github.com/livekit/protocol/utils" + "github.com/livekit/protocol/utils/guid" "github.com/livekit/protocol/utils/must" "github.com/livekit/psrpc" @@ -378,7 +379,7 @@ func (r *RoomManager) StartSession( pv := types.ProtocolVersion(pi.Client.Protocol) rtcConf := *r.rtcConfig rtcConf.SetBufferFactory(room.GetBufferFactory()) - sid := livekit.ParticipantID(utils.NewGuid(utils.ParticipantPrefix)) + sid := livekit.ParticipantID(guid.New(utils.ParticipantPrefix)) pLogger := rtc.LoggerWithParticipant( rtc.LoggerWithRoom(logger.GetLogger(), room.Name(), room.ID()), pi.Identity, diff --git a/pkg/service/sip.go b/pkg/service/sip.go index 417457c8f..7421a5bd8 100644 --- a/pkg/service/sip.go +++ b/pkg/service/sip.go @@ -24,6 +24,7 @@ import ( "github.com/livekit/protocol/rpc" "github.com/livekit/protocol/sip" "github.com/livekit/protocol/utils" + "github.com/livekit/protocol/utils/guid" "github.com/livekit/psrpc" "github.com/livekit/livekit-server/pkg/config" @@ -91,7 +92,7 @@ func (s *SIPService) CreateSIPTrunk(ctx context.Context, req *livekit.CreateSIPT } // Now we can generate ID and store. - info.SipTrunkId = utils.NewGuid(utils.SIPTrunkPrefix) + info.SipTrunkId = guid.New(utils.SIPTrunkPrefix) if err := s.store.StoreSIPTrunk(ctx, info); err != nil { return nil, err } @@ -153,7 +154,7 @@ func (s *SIPService) CreateSIPDispatchRule(ctx context.Context, req *livekit.Cre } // Now we can generate ID and store. - info.SipDispatchRuleId = utils.NewGuid(utils.SIPDispatchRulePrefix) + info.SipDispatchRuleId = guid.New(utils.SIPDispatchRulePrefix) if err := s.store.StoreSIPDispatchRule(ctx, info); err != nil { return nil, err } diff --git a/pkg/telemetry/analyticsservice.go b/pkg/telemetry/analyticsservice.go index aca94a22e..7e2707550 100644 --- a/pkg/telemetry/analyticsservice.go +++ b/pkg/telemetry/analyticsservice.go @@ -21,6 +21,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/livekit/rpc" "github.com/livekit/protocol/logger" "github.com/livekit/livekit-server/pkg/config" @@ -39,9 +40,9 @@ type analyticsService struct { nodeID string sequenceNumber atomic.Uint64 - events livekit.AnalyticsRecorderService_IngestEventsClient - stats livekit.AnalyticsRecorderService_IngestStatsClient - nodeRooms livekit.AnalyticsRecorderService_IngestNodeRoomStatesClient + events rpc.AnalyticsRecorderService_IngestEventsClient + stats rpc.AnalyticsRecorderService_IngestStatsClient + nodeRooms rpc.AnalyticsRecorderService_IngestNodeRoomStatesClient } func NewAnalyticsService(_ *config.Config, currentNode routing.LocalNode) AnalyticsService { diff --git a/pkg/telemetry/events.go b/pkg/telemetry/events.go index e16f5fadd..a43a086a4 100644 --- a/pkg/telemetry/events.go +++ b/pkg/telemetry/events.go @@ -23,7 +23,7 @@ import ( "github.com/livekit/livekit-server/pkg/telemetry/prometheus" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" - "github.com/livekit/protocol/utils" + "github.com/livekit/protocol/utils/guid" "github.com/livekit/protocol/webhook" ) @@ -33,7 +33,7 @@ func (t *telemetryService) NotifyEvent(ctx context.Context, event *livekit.Webho } event.CreatedAt = time.Now().Unix() - event.Id = utils.NewGuid("EV_") + event.Id = guid.New("EV_") if err := t.notifier.QueueNotify(ctx, event); err != nil { logger.Warnw("failed to notify webhook", err, "event", event.Event) diff --git a/test/integration_helpers.go b/test/integration_helpers.go index 81db35ed4..8fffe6b81 100644 --- a/test/integration_helpers.go +++ b/test/integration_helpers.go @@ -35,7 +35,7 @@ import ( "github.com/livekit/protocol/auth" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" - "github.com/livekit/protocol/utils" + "github.com/livekit/protocol/utils/guid" ) const ( @@ -80,8 +80,8 @@ func setupSingleNodeTest(name string) (*service.LivekitServer, func()) { func setupMultiNodeTest(name string) (*service.LivekitServer, *service.LivekitServer, func()) { logger.Infow("----------------STARTING TEST----------------", "test", name) - s1 := createMultiNodeServer(utils.NewGuid(nodeID1), defaultServerPort) - s2 := createMultiNodeServer(utils.NewGuid(nodeID2), secondServerPort) + s1 := createMultiNodeServer(guid.New(nodeID1), defaultServerPort) + s2 := createMultiNodeServer(guid.New(nodeID2), secondServerPort) go s1.Start() go s2.Start() @@ -161,7 +161,7 @@ func createSingleNodeServer(configUpdater func(*config.Config)) *service.Livekit if err != nil { panic(fmt.Sprintf("could not create local node: %v", err)) } - currentNode.Id = utils.NewGuid(nodeID1) + currentNode.Id = guid.New(nodeID1) s, err := service.InitializeServer(conf, currentNode) if err != nil { diff --git a/test/webhook_test.go b/test/webhook_test.go index 0463eea1a..e572f7c6b 100644 --- a/test/webhook_test.go +++ b/test/webhook_test.go @@ -30,7 +30,7 @@ import ( "github.com/livekit/protocol/auth" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" - "github.com/livekit/protocol/utils" + "github.com/livekit/protocol/utils/guid" "github.com/livekit/protocol/webhook" "github.com/livekit/livekit-server/pkg/config" @@ -137,7 +137,7 @@ func setupServerWithWebhook() (server *service.LivekitServer, testServer *webhoo if err != nil { return } - currentNode.Id = utils.NewGuid(nodeID1) + currentNode.Id = guid.New(nodeID1) server, err = service.InitializeServer(conf, currentNode) if err != nil {