From ffb831aa8c748755412d0ce838653011be8f3eda Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 22 Mar 2024 11:56:50 +0530 Subject: [PATCH 01/78] Cache transceiver before closing subscribed track. (#2594) On migration, when subscription moved from remote -> local, transceiver caching was racing. Although a very small possibility, it could happen like so 1. down track close 2. down track close callback fires go routine to close subscribed track 3. subscribed track close handler in subscription manager tries to reconcile 4. reconcile adds subscribed track again 5. cannot find cached transceiver as caching happens after down track close finishes in stap 1 above. Although there are a couple of gortouine jumps (step 2 fires a goroutine to close subscribed track and step 4 will reconcile in a goroutine too), it is theoretically possible that the step 1 has not finished and hence transceiver is not cached. Fix is to move caching to before closing subscribed track. --- pkg/rtc/mediatracksubscriptions.go | 17 ++++++----- pkg/rtc/participant.go | 7 +++-- pkg/sfu/downtrack.go | 33 ++++++++++++---------- pkg/sfu/streamallocator/streamallocator.go | 8 +++++- 4 files changed, 40 insertions(+), 25 deletions(-) diff --git a/pkg/rtc/mediatracksubscriptions.go b/pkg/rtc/mediatracksubscriptions.go index ee67987e4..3501babe6 100644 --- a/pkg/rtc/mediatracksubscriptions.go +++ b/pkg/rtc/mediatracksubscriptions.go @@ -322,13 +322,6 @@ func (t *MediaTrackSubscriptions) closeSubscribedTrack(subTrack types.Subscribed if willBeResumed { dt.CloseWithFlush(false) - - // cache transceiver for potential re-use on resume - tr := dt.GetTransceiver() - if tr != nil { - sub := subTrack.Subscriber() - sub.CacheDownTrack(subTrack.ID(), tr, dt.GetState()) - } } else { // flushing blocks, avoid blocking when publisher removes all its subscribers go dt.CloseWithFlush(true) @@ -426,6 +419,16 @@ func (t *MediaTrackSubscriptions) downTrackClosed( t.subscribedTracksMu.Unlock() if subTrack != nil { + // cache transceiver for potential re-use on resume + if willBeResumed { + dt := subTrack.DownTrack() + tr := dt.GetTransceiver() + if tr != nil { + sub := subTrack.Subscriber() + sub.CacheDownTrack(subTrack.ID(), tr, dt.GetState()) + } + } + subTrack.Close(willBeResumed) } } diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 5f8f7e60e..ccd67ab97 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -2362,6 +2362,7 @@ func (p *ParticipantImpl) CacheDownTrack(trackID livekit.TrackID, rtpTransceiver p.subLogger.Infow("cached transceiver changed", "trackID", trackID) } p.cachedDownTracks[trackID] = &downTrackState{transceiver: rtpTransceiver, downTrack: downTrack} + p.subLogger.Debugw("caching downtrack", "trackID", trackID) p.lock.Unlock() } @@ -2369,6 +2370,9 @@ func (p *ParticipantImpl) UncacheDownTrack(rtpTransceiver *webrtc.RTPTransceiver p.lock.Lock() for trackID, dts := range p.cachedDownTracks { if dts.transceiver == rtpTransceiver { + if dts := p.cachedDownTracks[trackID]; dts != nil { + p.subLogger.Debugw("uncaching downtrack", "trackID", trackID) + } delete(p.cachedDownTracks, trackID) break } @@ -2380,8 +2384,7 @@ func (p *ParticipantImpl) GetCachedDownTrack(trackID livekit.TrackID) (*webrtc.R p.lock.RLock() defer p.lock.RUnlock() - dts := p.cachedDownTracks[trackID] - if dts != nil { + if dts := p.cachedDownTracks[trackID]; dts != nil { return dts.transceiver, dts.downTrack } diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 41e6eb67f..8312aeafd 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -533,13 +533,9 @@ func (d *DownTrack) SubscriberID() livekit.ParticipantID { // Sets RTP header extensions for this track func (d *DownTrack) SetRTPHeaderExtensions(rtpHeaderExtensions []webrtc.RTPHeaderExtensionParameter) { - d.streamAllocatorLock.RLock() - listener := d.streamAllocatorListener - d.streamAllocatorLock.RUnlock() - isBWEEnabled := true - if listener != nil { - isBWEEnabled = listener.IsBWEEnabled(d) + if sal := d.getStreamAllocatorListener(); sal != nil { + isBWEEnabled = sal.IsBWEEnabled(d) } for _, ext := range rtpHeaderExtensions { switch ext.URI { @@ -669,13 +665,23 @@ func (d *DownTrack) maxLayerNotifierWorker() { d.params.Logger.Debugw("max subscribed layer processed", "layer", maxLayerSpatial, "event", event) if onMaxSubscribedLayerChanged := d.getOnMaxLayerChanged(); onMaxSubscribedLayerChanged != nil { - d.params.Logger.Debugw("notifying max subscribed layer", "layer", maxLayerSpatial, "event", event) + d.params.Logger.Debugw( + "notifying max subscribed layer", + "layer", maxLayerSpatial, + "event", event, + "subscriberID", d.SubscriberID(), + ) onMaxSubscribedLayerChanged(d, maxLayerSpatial) } } if onMaxSubscribedLayerChanged := d.getOnMaxLayerChanged(); onMaxSubscribedLayerChanged != nil { - d.params.Logger.Debugw("notifying max subscribed layer", "layer", buffer.InvalidLayerSpatial, "event", "close") + d.params.Logger.Debugw( + "notifying max subscribed layer", + "layer", buffer.InvalidLayerSpatial, + "event", "close", + "subscriberID", d.SubscriberID(), + ) onMaxSubscribedLayerChanged(d, buffer.InvalidLayerSpatial) } } @@ -865,13 +871,9 @@ func (d *DownTrack) WritePaddingRTP(bytesToSend int, paddingOnMute bool, forceMa // Mute enables or disables media forwarding - subscriber triggered func (d *DownTrack) Mute(muted bool) { - d.streamAllocatorLock.RLock() - listener := d.streamAllocatorListener - d.streamAllocatorLock.RUnlock() - isSubscribeMutable := true - if listener != nil { - isSubscribeMutable = listener.IsSubscribeMutable(d) + if sal := d.getStreamAllocatorListener(); sal != nil { + isSubscribeMutable = sal.IsSubscribeMutable(d) } changed := d.forwarder.Mute(muted, isSubscribeMutable) d.handleMute(muted, changed) @@ -987,9 +989,10 @@ func (d *DownTrack) CloseWithFlush(flush bool) { d.rtcpReader.Close() d.rtcpReader.OnPacket(nil) } - d.bindLock.Unlock() + d.connectionStats.Close() + d.rtpStats.Stop() d.params.Logger.Debugw("rtp stats", "direction", "downstream", diff --git a/pkg/sfu/streamallocator/streamallocator.go b/pkg/sfu/streamallocator/streamallocator.go index 352b50c2e..3abc28a6b 100644 --- a/pkg/sfu/streamallocator/streamallocator.go +++ b/pkg/sfu/streamallocator/streamallocator.go @@ -241,10 +241,16 @@ func (s *StreamAllocator) AddTrack(downTrack *sfu.DownTrack, params AddTrackPara track := NewTrack(downTrack, params.Source, params.IsSimulcast, params.PublisherID, s.params.Logger) track.SetPriority(params.Priority) + trackID := livekit.TrackID(downTrack.ID()) s.videoTracksMu.Lock() - s.videoTracks[livekit.TrackID(downTrack.ID())] = track + oldTrack := s.videoTracks[trackID] + s.videoTracks[trackID] = track s.videoTracksMu.Unlock() + if oldTrack != nil { + oldTrack.DownTrack().SetStreamAllocatorListener(nil) + } + downTrack.SetStreamAllocatorListener(s) if s.prober.IsRunning() { // STREAM-ALLOCATOR-TODO: this can be changed to adapt to probe rate From 95f5c94b4d01a9bce8d80310fd409312fe57e367 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 22 Mar 2024 23:22:20 +0530 Subject: [PATCH 02/78] Notify initial permissions (#2595) * Notify initial permissions NOTE: This does add an initial subscription permission notification which should be fine, but something to watch for. A stress test combining - mute/unmute on publisher side. - allowing/revoking permission for subscriber from publisher side. - subscribing/unsubscribing from subscriber side. results in a scenario where a subscription permission update of `not_allowed` being sent and on a re-subscribe, an `allowed` update does not happen. It happens like so - Subscription revoke cloes the down track of subscriber. - The subscription is still desired. - So, a subscription reconcile runs and sees `permission: false`. This sends subscription permission of `not_allowed`. - Unsubscribe request comes in and sets `desired: false`. - Reconsiler runs again and sees `desired: false` and `subscribedTrack: nil`. This cleans up the subscription. - Publisher grants permission for the subscriber. - Subscriber subscribes to the track again. A new subscription is created. - Reconciler runs and sees `permission: true`, but there is no permission change as it is a new subscription object. So, `allowed` subscription permission update is not sent and the client is stuck at `not_allowed`. Fix, maintain if permission has been initialized. Has the effect of sending an initial update which should be fine. * clean up comment * no default --- pkg/rtc/subscriptionmanager.go | 35 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/pkg/rtc/subscriptionmanager.go b/pkg/rtc/subscriptionmanager.go index 8232c7527..a7dfa6608 100644 --- a/pkg/rtc/subscriptionmanager.go +++ b/pkg/rtc/subscriptionmanager.go @@ -497,8 +497,6 @@ func (m *SubscriptionManager) subscribe(s *trackSubscription) error { s.setPublisher(res.PublisherIdentity, res.PublisherID) - // since hasPermission defaults to true, we will want to send a message to the client the first time - // that we discover permissions were denied permChanged := s.setHasPermission(res.HasPermission) if permChanged { m.params.Participant.SubscriptionPermissionUpdate(s.getPublisherID(), trackID, res.HasPermission) @@ -722,19 +720,20 @@ type trackSubscription struct { trackID livekit.TrackID logger logger.Logger - lock sync.RWMutex - desired bool - publisherID livekit.ParticipantID - publisherIdentity livekit.ParticipantIdentity - settings *livekit.UpdateTrackSettings - changedNotifier types.ChangeNotifier - removedNotifier types.ChangeNotifier - hasPermission bool - subscribedTrack types.SubscribedTrack - eventSent atomic.Bool - numAttempts atomic.Int32 - bound bool - kind atomic.Pointer[livekit.TrackType] + lock sync.RWMutex + desired bool + publisherID livekit.ParticipantID + publisherIdentity livekit.ParticipantIdentity + settings *livekit.UpdateTrackSettings + changedNotifier types.ChangeNotifier + removedNotifier types.ChangeNotifier + hasPermissionInitialized bool + hasPermission bool + subscribedTrack types.SubscribedTrack + eventSent atomic.Bool + numAttempts atomic.Int32 + bound bool + kind atomic.Pointer[livekit.TrackType] // the later of when subscription was requested OR when the first failure was encountered OR when permission is granted // this timestamp determines when failures are reported @@ -746,8 +745,6 @@ func newTrackSubscription(subscriberID livekit.ParticipantID, trackID livekit.Tr subscriberID: subscriberID, trackID: trackID, logger: l, - // default allow - hasPermission: true, } } @@ -796,9 +793,11 @@ func (s *trackSubscription) setDesired(desired bool) bool { func (s *trackSubscription) setHasPermission(perm bool) bool { s.lock.Lock() defer s.lock.Unlock() - if s.hasPermission == perm { + if s.hasPermissionInitialized && s.hasPermission == perm { return false } + + s.hasPermissionInitialized = true s.hasPermission = perm if s.hasPermission { // when permission is granted, reset the timer so it has sufficient time to reconcile From 2dba3b2d2e1dd12aec85f63b00cce2f3ec87e1a4 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 25 Mar 2024 15:07:29 +0530 Subject: [PATCH 03/78] Protect duplicate subscription (another try). (#2596) Another case of duplciate tracks in SDP. During migration (if both publisher and subscriber migrate), subscriber could attach the remote track of the publisher. But, while that is happening, publisher could migrate into the node and close the remote media track. This was causing subscriber to switch from attaching to remote media track -> attaching to local media track. But, as remote media track was closed while add subscription was happening, the subscriber is removed without subscription manager being aware of it. So, the subscription manager's reconcile and the remove subscriber is racing and when subscription manager re-subscribes, caching has not run yet and that creates a duplicate. Delay removing subscribed track till after caching is done. That means, even if the reconciler runs, it will get an `errAlreadySubscribed` error and it will force it to reconcile again. By the time the subscribed track is deleted from the subscriptions map, caching is done. --- pkg/rtc/mediatracksubscriptions.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/pkg/rtc/mediatracksubscriptions.go b/pkg/rtc/mediatracksubscriptions.go index 3501babe6..20d4ea7d7 100644 --- a/pkg/rtc/mediatracksubscriptions.go +++ b/pkg/rtc/mediatracksubscriptions.go @@ -413,13 +413,14 @@ func (t *MediaTrackSubscriptions) downTrackClosed( willBeResumed bool, ) { subscriberID := sub.ID() - t.subscribedTracksMu.Lock() + t.subscribedTracksMu.RLock() subTrack := t.subscribedTracks[subscriberID] - delete(t.subscribedTracks, subscriberID) - t.subscribedTracksMu.Unlock() + t.subscribedTracksMu.RUnlock() if subTrack != nil { - // cache transceiver for potential re-use on resume + // Cache transceiver for potential re-use on resume. + // To ensure subscription manager does not re-subscribe before caching, + // delete the subscribed track only after caching. if willBeResumed { dt := subTrack.DownTrack() tr := dt.GetTransceiver() @@ -429,6 +430,10 @@ func (t *MediaTrackSubscriptions) downTrackClosed( } } + t.subscribedTracksMu.Lock() + delete(t.subscribedTracks, subscriberID) + t.subscribedTracksMu.Unlock() + subTrack.Close(willBeResumed) } } From d8226abf00b06ba4e9a7857ab73681f337c5cfd9 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 25 Mar 2024 14:48:50 -0700 Subject: [PATCH 04/78] Update golang.org/x/exp digest to a685a6e (#2597) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b15f7bf03..663630fcf 100644 --- a/go.mod +++ b/go.mod @@ -48,7 +48,7 @@ require ( github.com/urfave/negroni/v3 v3.1.0 go.uber.org/atomic v1.11.0 go.uber.org/zap v1.27.0 - golang.org/x/exp v0.0.0-20240318143956-a85f2c67cd81 + golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 golang.org/x/sync v0.6.0 google.golang.org/protobuf v1.33.0 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index 670d00671..c2eb9bb89 100644 --- a/go.sum +++ b/go.sum @@ -309,8 +309,8 @@ golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1m golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= -golang.org/x/exp v0.0.0-20240318143956-a85f2c67cd81 h1:6R2FC06FonbXQ8pK11/PDFY6N6LWlf9KlzibaCapmqc= -golang.org/x/exp v0.0.0-20240318143956-a85f2c67cd81/go.mod h1:CQ1k9gNrJ50XIzaKCRR2hssIjF07kZFEiieALBM/ARQ= +golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 h1:aAcj0Da7eBAtrTp03QXWvm88pSyOt+UgdZw2BFZ+lEw= +golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8/go.mod h1:CQ1k9gNrJ50XIzaKCRR2hssIjF07kZFEiieALBM/ARQ= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= From 0480f99a8326a99e2ec6172cc49ae220cf5f1e76 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 26 Mar 2024 17:33:24 +0530 Subject: [PATCH 05/78] Tweak adaptation to increase in propagation delay. (#2598) * Tweak adaptation to increase in propagation delay. A couple of issues - RTCP Sender Reports rate will vary based on underying track bitrate. (at least in theory, not all entities will do it though, for example SFU does standard rate of one per three seconds irrespective of track bit rate). So, adapt the long term estimate of propagation delay delta based on spacing of reports. - Re-init of propagation delay to adapt to path change was taking the last value before the switch. But, that one value could have been an outlier and accepting it is not great. So, adapt spike time propagation delay in a smoother fashion to ensure that all values during spike contribute to the final value. * clean up --- pkg/sfu/buffer/rtpstats_receiver.go | 31 +++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/pkg/sfu/buffer/rtpstats_receiver.go b/pkg/sfu/buffer/rtpstats_receiver.go index 0a1b61a28..fcce8e411 100644 --- a/pkg/sfu/buffer/rtpstats_receiver.go +++ b/pkg/sfu/buffer/rtpstats_receiver.go @@ -41,6 +41,8 @@ const ( cPropagationDelayFallFactor = float64(0.95) cPropagationDelayRiseFactor = float64(0.05) + cPropagationDelaySpikeAdaptationFactor = float64(0.5) + // do not adapt to small OR large (outlier) changes cPropagationDelayDeltaThresholdMin = 5 * time.Millisecond cPropagationDelayDeltaThresholdMaxFactor = 2 @@ -51,7 +53,7 @@ const ( // A long term version of delta of propagation delay is maintained and delta propagation delay exceeding // a factor of the long term version is considered a sharp increase. That will trigger the start of the // path change condition and if it persists, propagation delay will be reset. - cPropagationDelayDeltaAdaptationFactor = float64(0.05) + cPropagationDelayDeltaMaxInterval = 10 * time.Second cPropagationDelayDeltaHighResetNumReports = 3 cPropagationDelayDeltaHighResetWait = 10 * time.Second ) @@ -83,6 +85,7 @@ type RTPStatsReceiver struct { longTermDeltaPropagationDelay time.Duration propagationDelayDeltaHighCount int propagationDelayDeltaHighStartTime time.Time + propagationDelaySpike time.Duration clockSkewCount int outOfOrderSsenderReportCount int @@ -370,11 +373,17 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *RTCPSenderReportData) "current", &srDataCopy, } } - initPropagationDelay := func(pd time.Duration) { - r.propagationDelay = pd - r.longTermDeltaPropagationDelay = 0 + resetDelta := func() { r.propagationDelayDeltaHighCount = 0 r.propagationDelayDeltaHighStartTime = time.Time{} + r.propagationDelaySpike = 0 + } + initPropagationDelay := func(pd time.Duration) { + r.propagationDelay = pd + + r.longTermDeltaPropagationDelay = 0 + + resetDelta() } ntpTime := srDataCopy.NTPTimestamp.Time() @@ -392,14 +401,18 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *RTCPSenderReportData) if r.propagationDelayDeltaHighStartTime.IsZero() { r.propagationDelayDeltaHighStartTime = time.Now() } + if r.propagationDelaySpike == 0 { + r.propagationDelaySpike = propagationDelay + } else { + r.propagationDelaySpike += time.Duration(cPropagationDelaySpikeAdaptationFactor * float64(propagationDelay-r.propagationDelaySpike)) + } if r.propagationDelayDeltaHighCount >= cPropagationDelayDeltaHighResetNumReports && time.Since(r.propagationDelayDeltaHighStartTime) >= cPropagationDelayDeltaHighResetWait { r.logger.Debugw("re-initializing propagation delay", append(getPropagationFields(), "newPropagationDelay", propagationDelay.String())...) - initPropagationDelay(propagationDelay) + initPropagationDelay(r.propagationDelaySpike) } } else { - r.propagationDelayDeltaHighCount = 0 - r.propagationDelayDeltaHighStartTime = time.Time{} + resetDelta() if deltaPropagationDelay.Abs() > cPropagationDelayDeltaThresholdMin { factor := cPropagationDelayFallFactor @@ -421,7 +434,9 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *RTCPSenderReportData) if r.longTermDeltaPropagationDelay == 0 { r.longTermDeltaPropagationDelay = deltaPropagationDelay } else { - r.longTermDeltaPropagationDelay += time.Duration(cPropagationDelayDeltaAdaptationFactor * float64(deltaPropagationDelay-r.longTermDeltaPropagationDelay)) + sinceLastReport := srDataCopy.NTPTimestamp.Time().Sub(r.srNewest.NTPTimestamp.Time()) + adaptationFactor := min(1.0, float64(sinceLastReport)/float64(cPropagationDelayDeltaMaxInterval)) + r.longTermDeltaPropagationDelay += time.Duration(adaptationFactor * float64(deltaPropagationDelay-r.longTermDeltaPropagationDelay)) } } // adjust receive time to estimated propagation delay From 45581433ccf9500cfb2a672defbc878fa3453528 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 27 Mar 2024 18:45:53 +0530 Subject: [PATCH 06/78] Add option to enable bitrate based scoring (#2600) --- pkg/sfu/connectionquality/connectionstats.go | 26 ++++--- .../connectionquality/connectionstats_test.go | 73 +++++++++++++------ pkg/sfu/connectionquality/scorer.go | 15 ++-- pkg/sfu/downtrack.go | 3 +- pkg/sfu/receiver.go | 24 ++++++ 5 files changed, 98 insertions(+), 43 deletions(-) diff --git a/pkg/sfu/connectionquality/connectionstats.go b/pkg/sfu/connectionquality/connectionstats.go index 16ac76d1d..c221ea130 100644 --- a/pkg/sfu/connectionquality/connectionstats.go +++ b/pkg/sfu/connectionquality/connectionstats.go @@ -45,14 +45,15 @@ type ConnectionStatsSenderProvider interface { } type ConnectionStatsParams struct { - UpdateInterval time.Duration - MimeType string - IsFECEnabled bool - IncludeRTT bool - IncludeJitter bool - ReceiverProvider ConnectionStatsReceiverProvider - SenderProvider ConnectionStatsSenderProvider - Logger logger.Logger + UpdateInterval time.Duration + MimeType string + IsFECEnabled bool + IncludeRTT bool + IncludeJitter bool + EnableBitrateScore bool + ReceiverProvider ConnectionStatsReceiverProvider + SenderProvider ConnectionStatsSenderProvider + Logger logger.Logger } type ConnectionStats struct { @@ -76,10 +77,11 @@ func NewConnectionStats(params ConnectionStatsParams) *ConnectionStats { return &ConnectionStats{ params: params, scorer: newQualityScorer(qualityScorerParams{ - PacketLossWeight: getPacketLossWeight(params.MimeType, params.IsFECEnabled), // LK-TODO: have to notify codec change? - IncludeRTT: params.IncludeRTT, - IncludeJitter: params.IncludeJitter, - Logger: params.Logger, + PacketLossWeight: getPacketLossWeight(params.MimeType, params.IsFECEnabled), // LK-TODO: have to notify codec change? + IncludeRTT: params.IncludeRTT, + IncludeJitter: params.IncludeJitter, + EnableBitrateScore: params.EnableBitrateScore, + Logger: params.Logger, }), } } diff --git a/pkg/sfu/connectionquality/connectionstats_test.go b/pkg/sfu/connectionquality/connectionstats_test.go index 7b07ba8bc..569238a40 100644 --- a/pkg/sfu/connectionquality/connectionstats_test.go +++ b/pkg/sfu/connectionquality/connectionstats_test.go @@ -26,23 +26,6 @@ import ( "github.com/livekit/protocol/logger" ) -func newConnectionStats( - mimeType string, - isFECEnabled bool, - includeRTT bool, - includeJitter bool, - receiverProvider ConnectionStatsReceiverProvider, -) *ConnectionStats { - return NewConnectionStats(ConnectionStatsParams{ - MimeType: mimeType, - IsFECEnabled: isFECEnabled, - IncludeRTT: includeRTT, - IncludeJitter: includeJitter, - ReceiverProvider: receiverProvider, - Logger: logger.GetLogger(), - }) -} - // ----------------------------------------------- type testReceiverProvider struct { @@ -66,7 +49,15 @@ func (trp *testReceiverProvider) GetDeltaStats() map[uint32]*buffer.StreamStatsW func TestConnectionQuality(t *testing.T) { trp := newTestReceiverProvider() t.Run("quality scorer operation", func(t *testing.T) { - cs := newConnectionStats("audio/opus", false, true, true, trp) + cs := NewConnectionStats(ConnectionStatsParams{ + MimeType: "audio/opus", + IsFECEnabled: false, + IncludeRTT: true, + IncludeJitter: true, + EnableBitrateScore: true, + ReceiverProvider: trp, + Logger: logger.GetLogger(), + }) duration := 5 * time.Second now := time.Now() @@ -441,7 +432,14 @@ func TestConnectionQuality(t *testing.T) { }) t.Run("quality scorer dependent rtt", func(t *testing.T) { - cs := newConnectionStats("audio/opus", false, false, true, trp) + cs := NewConnectionStats(ConnectionStatsParams{ + MimeType: "audio/opus", + IsFECEnabled: false, + IncludeRTT: false, + IncludeJitter: true, + ReceiverProvider: trp, + Logger: logger.GetLogger(), + }) duration := 5 * time.Second now := time.Now() @@ -469,7 +467,14 @@ func TestConnectionQuality(t *testing.T) { }) t.Run("quality scorer dependent jitter", func(t *testing.T) { - cs := newConnectionStats("audio/opus", false, true, false, trp) + cs := NewConnectionStats(ConnectionStatsParams{ + MimeType: "audio/opus", + IsFECEnabled: false, + IncludeRTT: true, + IncludeJitter: false, + ReceiverProvider: trp, + Logger: logger.GetLogger(), + }) duration := 5 * time.Second now := time.Now() @@ -634,7 +639,14 @@ func TestConnectionQuality(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - cs := newConnectionStats(tc.mimeType, tc.isFECEnabled, true, true, trp) + cs := NewConnectionStats(ConnectionStatsParams{ + MimeType: tc.mimeType, + IsFECEnabled: tc.isFECEnabled, + IncludeRTT: true, + IncludeJitter: true, + ReceiverProvider: trp, + Logger: logger.GetLogger(), + }) duration := 5 * time.Second now := time.Now() @@ -727,7 +739,15 @@ func TestConnectionQuality(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - cs := newConnectionStats("video/vp8", false, true, true, trp) + cs := NewConnectionStats(ConnectionStatsParams{ + MimeType: "video/vp8", + IsFECEnabled: false, + IncludeRTT: true, + IncludeJitter: true, + EnableBitrateScore: true, + ReceiverProvider: trp, + Logger: logger.GetLogger(), + }) duration := 5 * time.Second now := time.Now() @@ -814,7 +834,14 @@ func TestConnectionQuality(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - cs := newConnectionStats("video/vp8", false, true, true, trp) + cs := NewConnectionStats(ConnectionStatsParams{ + MimeType: "video/vp8", + IsFECEnabled: false, + IncludeRTT: true, + IncludeJitter: true, + ReceiverProvider: trp, + Logger: logger.GetLogger(), + }) duration := 5 * time.Second now := time.Now() diff --git a/pkg/sfu/connectionquality/scorer.go b/pkg/sfu/connectionquality/scorer.go index 33421fcb9..4ed30faa3 100644 --- a/pkg/sfu/connectionquality/scorer.go +++ b/pkg/sfu/connectionquality/scorer.go @@ -123,8 +123,8 @@ func (w *windowStat) calculatePacketScore(plw float64, includeRTT bool, includeJ return score } -func (w *windowStat) calculateBitrateScore(expectedBitrate int64) float64 { - if expectedBitrate == 0 { +func (w *windowStat) calculateBitrateScore(expectedBitrate int64, isEnabled bool) float64 { + if expectedBitrate == 0 || !isEnabled { // unsupported mode OR all layers stopped return cMaxScore } @@ -180,10 +180,11 @@ func (w *windowStat) MarshalLogObject(e zapcore.ObjectEncoder) error { // ------------------------------------------ type qualityScorerParams struct { - PacketLossWeight float64 - IncludeRTT bool - IncludeJitter bool - Logger logger.Logger + PacketLossWeight float64 + IncludeRTT bool + IncludeJitter bool + EnableBitrateScore bool + Logger logger.Logger } type qualityScorer struct { @@ -396,7 +397,7 @@ func (q *qualityScorer) updateAtLocked(stat *windowStat, at time.Time) { score = qualityTransitionScore[livekit.ConnectionQuality_LOST] } else { packetScore := stat.calculatePacketScore(plw, q.params.IncludeRTT, q.params.IncludeJitter) - bitrateScore := stat.calculateBitrateScore(expectedBitrate) + bitrateScore := stat.calculateBitrateScore(expectedBitrate, q.params.EnableBitrateScore) layerScore := math.Max(math.Min(cMaxScore, cMaxScore-(expectedDistance*distanceWeight)), 0.0) minScore := math.Min(packetScore, bitrateScore) diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 8312aeafd..8c0a38494 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -1090,7 +1090,7 @@ func (d *DownTrack) UpTrackMaxTemporalLayerSeenChange(maxTemporalLayerSeen int32 } } -func (d *DownTrack) maybeAddTransition(_ int64, distance float64, pauseReason VideoPauseReason) { +func (d *DownTrack) maybeAddTransition(bitrate int64, distance float64, pauseReason VideoPauseReason) { if d.kind == webrtc.RTPCodecTypeAudio { return } @@ -1100,6 +1100,7 @@ func (d *DownTrack) maybeAddTransition(_ int64, distance float64, pauseReason Vi } else { d.connectionStats.UpdatePause(false) d.connectionStats.AddLayerTransition(distance) + d.connectionStats.AddBitrateTransition(bitrate) } } diff --git a/pkg/sfu/receiver.go b/pkg/sfu/receiver.go index 3028e98e3..2192c6a66 100644 --- a/pkg/sfu/receiver.go +++ b/pkg/sfu/receiver.go @@ -430,8 +430,31 @@ func (w *WebRTCReceiver) AddDownTrack(track TrackSender) error { return nil } +func (w *WebRTCReceiver) notifyMaxExpectedLayer(layer int32) { + ti := w.TrackInfo() + if ti == nil { + return + } + + if w.Kind() == webrtc.RTPCodecTypeAudio || ti.Source == livekit.TrackSource_SCREEN_SHARE { + // screen share tracks have highly variable bitrate, do not use bit rate based quality for those + return + } + + expectedBitrate := int64(0) + for _, vl := range ti.Layers { + l := buffer.VideoQualityToSpatialLayer(vl.Quality, ti) + if l <= layer { + expectedBitrate += int64(vl.Bitrate) + } + } + + w.connectionStats.AddBitrateTransition(expectedBitrate) +} + func (w *WebRTCReceiver) SetMaxExpectedSpatialLayer(layer int32) { w.streamTrackerManager.SetMaxExpectedSpatialLayer(layer) + w.notifyMaxExpectedLayer(layer) if layer == buffer.InvalidLayerSpatial { w.connectionStats.UpdateLayerMute(true) @@ -463,6 +486,7 @@ func (w *WebRTCReceiver) OnMaxPublishedLayerChanged(maxPublishedLayer int32) { dt.UpTrackMaxPublishedLayerChange(maxPublishedLayer) }) + w.notifyMaxExpectedLayer(maxPublishedLayer) w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired()) } From bc5fc17bdce8f755d20a8b0b6e411b5a0feaa767 Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Thu, 28 Mar 2024 15:59:03 +0800 Subject: [PATCH 07/78] Log high jitter case (#2602) --- pkg/sfu/playoutdelay.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/sfu/playoutdelay.go b/pkg/sfu/playoutdelay.go index 2dc756d39..47eeffb18 100644 --- a/pkg/sfu/playoutdelay.go +++ b/pkg/sfu/playoutdelay.go @@ -59,6 +59,8 @@ type PlayoutDelayController struct { logger logger.Logger rtpStats *buffer.RTPStatsSender snapshotID uint32 + + highDelayCount atomic.Uint32 } func NewPlayoutDelayController(minDelay, maxDelay uint32, logger logger.Logger, rtpStats *buffer.RTPStatsSender) (*PlayoutDelayController, error) { @@ -113,7 +115,9 @@ func (c *PlayoutDelayController) SetJitter(jitter uint32) { return } if targetDelay > targetDelayLogThreshold { - c.logger.Debugw("high playout delay", "target", targetDelay, "jitter", jitter, "nackPercent", nackPercent, "current", c.currentDelay) + if c.highDelayCount.Add(1)%100 == 1 { + c.logger.Infow("high playout delay", "target", targetDelay, "jitter", jitter, "nackPercent", nackPercent, "current", c.currentDelay) + } } c.currentDelay = targetDelay c.lock.Unlock() From 95df9737a6455475f7dec93eeca2f5a701e43e2c Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Fri, 29 Mar 2024 09:05:53 +0800 Subject: [PATCH 08/78] Fix twcc has chance to miss for firefox simulcast rtx (#2601) --- pkg/rtc/transport.go | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index 1e0cd1a93..ea4d7420a 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -1689,7 +1689,7 @@ func (t *PCTransport) handleRemoteOfferReceived(sd *webrtc.SessionDescription) e if err := t.setRemoteDescription(*sd); err != nil { return err } - rtxRepairs := rtxRepairsFromSDP(parsed, t.params.Logger) + rtxRepairs := nonSimulcastRTXRepairsFromSDP(parsed, t.params.Logger) if len(rtxRepairs) > 0 { t.params.Logger.Debugw("rtx pairs found from sdp", "ssrcs", rtxRepairs) for repair, base := range rtxRepairs { @@ -1820,11 +1820,19 @@ func configureAudioTransceiver(tr *webrtc.RTPTransceiver, stereo bool, nack bool tr.SetCodecPreferences(configCodecs) } -func rtxRepairsFromSDP(s *sdp.SessionDescription, logger logger.Logger) map[uint32]uint32 { +func nonSimulcastRTXRepairsFromSDP(s *sdp.SessionDescription, logger logger.Logger) map[uint32]uint32 { rtxRepairFlows := map[uint32]uint32{} for _, media := range s.MediaDescriptions { + // extract rtx repair flows from the media section for non-simulcast stream, + // pion will handle simulcast streams by rid probe, don't need handle it here. + var ridFound bool + rtxPairs := make(map[uint32]uint32) + findRTX: for _, attr := range media.Attributes { switch attr.Key { + case "rid": + ridFound = true + break findRTX case sdp.AttrKeySSRCGroup: split := strings.Split(attr.Value, " ") if split[0] == sdp.SemanticTokenFlowIdentification { @@ -1842,11 +1850,16 @@ func rtxRepairsFromSDP(s *sdp.SessionDescription, logger logger.Logger) map[uint logger.Warnw("Failed to parse SSRC", err, "ssrc", split[2]) continue } - rtxRepairFlows[uint32(rtxRepairFlow)] = uint32(baseSsrc) + rtxPairs[uint32(rtxRepairFlow)] = uint32(baseSsrc) } } } } + if !ridFound { + for rtx, base := range rtxPairs { + rtxRepairFlows[rtx] = base + } + } } return rtxRepairFlows From 0a35e59ebdaf4f1893989f48780a09883007aad9 Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Fri, 29 Mar 2024 17:24:31 +0800 Subject: [PATCH 09/78] Replace sleep with sync.Cond to reduce jitter (#2603) --- pkg/sfu/buffer/buffer.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go index 123472ad6..b533318da 100644 --- a/pkg/sfu/buffer/buffer.go +++ b/pkg/sfu/buffer/buffer.go @@ -69,6 +69,7 @@ type ExtPacket struct { // Buffer contains all packets type Buffer struct { sync.RWMutex + readCond *sync.Cond bucket *bucket.Bucket nacker *nack.NackQueue maxVideoPkts int @@ -144,6 +145,7 @@ func NewBuffer(ssrc uint32, maxVideoPkts, maxAudioPkts int) *Buffer { pliThrottle: int64(500 * time.Millisecond), logger: l.WithComponent(sutils.ComponentPub).WithComponent(sutils.ComponentSFU), } + b.readCond = sync.NewCond(&b.RWMutex) b.extPackets.SetMinCapacity(7) return b } @@ -324,6 +326,7 @@ func (b *Buffer) Write(pkt []byte) (n int, err error) { b.payloadType = rtpPacket.PayloadType b.calc(pkt, &rtpPacket, time.Now(), false) b.Unlock() + b.readCond.Signal() return } @@ -398,24 +401,23 @@ func (b *Buffer) Read(buff []byte) (n int, err error) { } func (b *Buffer) ReadExtended(buf []byte) (*ExtPacket, error) { + b.Lock() for { if b.closed.Load() { + b.Unlock() return nil, io.EOF } - b.Lock() if b.extPackets.Len() > 0 { ep := b.extPackets.PopFront() ep = b.patchExtPacket(ep, buf) if ep == nil { - b.Unlock() continue } b.Unlock() return ep, nil } - b.Unlock() - time.Sleep(10 * time.Millisecond) + b.readCond.Wait() } } @@ -437,6 +439,7 @@ func (b *Buffer) Close() error { } } + b.readCond.Broadcast() if b.onClose != nil { b.onClose() } From f1c991c5477760cc4696759f8efc41764cc1fdfc Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Fri, 29 Mar 2024 06:30:12 -0700 Subject: [PATCH 10/78] skip logging retry message when ws disconnections before signal finishes (#2604) --- pkg/service/rtcservice.go | 6 +----- pkg/telemetry/signalanddatastats.go | 12 +++++++++--- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/pkg/service/rtcservice.go b/pkg/service/rtcservice.go index dba8a1896..60100729f 100644 --- a/pkg/service/rtcservice.go +++ b/pkg/service/rtcservice.go @@ -219,14 +219,10 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { var cr connectionResult var initialResponse *livekit.SignalResponse for i := 0; i < 3; i++ { - if err = r.Context().Err(); err != nil { - break - } - connectionTimeout := 3 * time.Second * time.Duration(i+1) ctx := utils.ContextWithAttempt(r.Context(), i) cr, initialResponse, err = s.startConnection(ctx, roomName, pi, connectionTimeout) - if err == nil { + if err == nil || errors.Is(err, context.Canceled) { break } if i < 2 { diff --git a/pkg/telemetry/signalanddatastats.go b/pkg/telemetry/signalanddatastats.go index 086192532..1ec931563 100644 --- a/pkg/telemetry/signalanddatastats.go +++ b/pkg/telemetry/signalanddatastats.go @@ -162,7 +162,10 @@ func (s *BytesSignalStats) ResolveRoom(ri *livekit.Room) { s.mu.Lock() defer s.mu.Unlock() if s.ri == nil && ri.GetSid() != "" { - s.ri = ri + s.ri = &livekit.Room{ + Sid: ri.Sid, + Name: ri.Name, + } s.maybeStart() } } @@ -170,8 +173,11 @@ func (s *BytesSignalStats) ResolveRoom(ri *livekit.Room) { func (s *BytesSignalStats) ResolveParticipant(pi *livekit.ParticipantInfo) { s.mu.Lock() defer s.mu.Unlock() - if s.pi == nil { - s.pi = pi + if s.pi == nil && pi != nil { + s.pi = &livekit.ParticipantInfo{ + Sid: pi.Sid, + Identity: pi.Identity, + } s.maybeStart() } } From b5de6460735d00bc89015c363abf28ad8afa36e6 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sat, 30 Mar 2024 00:31:26 +0530 Subject: [PATCH 11/78] Remove redundant check. (#2605) * Remove redundant check. That check is already at the ouside check. * print string * space --- pkg/sfu/buffer/rtpstats_receiver.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/pkg/sfu/buffer/rtpstats_receiver.go b/pkg/sfu/buffer/rtpstats_receiver.go index fcce8e411..4c3476f78 100644 --- a/pkg/sfu/buffer/rtpstats_receiver.go +++ b/pkg/sfu/buffer/rtpstats_receiver.go @@ -414,23 +414,23 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *RTCPSenderReportData) } else { resetDelta() - if deltaPropagationDelay.Abs() > cPropagationDelayDeltaThresholdMin { - factor := cPropagationDelayFallFactor - if propagationDelay > r.propagationDelay { - factor = cPropagationDelayRiseFactor - } - fields := append( - getPropagationFields(), - "adjustedPropagationDelay", r.propagationDelay+time.Duration(factor*float64(propagationDelay-r.propagationDelay)), - ) // TODO-REMOVE - r.logger.Debugw("adapting propagation delay", fields...) // TODO-REMOVE - r.propagationDelay += time.Duration(factor * float64(propagationDelay-r.propagationDelay)) + factor := cPropagationDelayFallFactor + if propagationDelay > r.propagationDelay { + factor = cPropagationDelayRiseFactor } + adjustedPropagationDelay := r.propagationDelay + time.Duration(factor*float64(propagationDelay-r.propagationDelay)) // TODO-REMOVE + fields := append( + getPropagationFields(), + "adjustedPropagationDelay", adjustedPropagationDelay.String(), + ) // TODO-REMOVE + r.logger.Debugw("adapting propagation delay", fields...) // TODO-REMOVE + r.propagationDelay += time.Duration(factor * float64(propagationDelay-r.propagationDelay)) } } else { r.propagationDelayDeltaHighCount = 0 r.propagationDelayDeltaHighStartTime = time.Time{} } + if r.longTermDeltaPropagationDelay == 0 { r.longTermDeltaPropagationDelay = deltaPropagationDelay } else { From 4c9e59dc25a6cb4abbf4126cb42996bbb0b2e111 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sat, 30 Mar 2024 21:53:18 +0530 Subject: [PATCH 12/78] Small tweaks to propagation delay adaptation. (#2607) --- pkg/sfu/buffer/rtpstats_receiver.go | 46 ++++++++++++++--------------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/pkg/sfu/buffer/rtpstats_receiver.go b/pkg/sfu/buffer/rtpstats_receiver.go index 4c3476f78..ecdc0be8c 100644 --- a/pkg/sfu/buffer/rtpstats_receiver.go +++ b/pkg/sfu/buffer/rtpstats_receiver.go @@ -38,23 +38,24 @@ const ( // lower value as that could be the real propagation delay. If it rises, adapt slowly // as it might be a temporary change or slow drift. See below for handling of high deltas // which could be a result of a path change. - cPropagationDelayFallFactor = float64(0.95) - cPropagationDelayRiseFactor = float64(0.05) + cPropagationDelayFallFactor = float64(0.9) + cPropagationDelayRiseFactor = float64(0.1) cPropagationDelaySpikeAdaptationFactor = float64(0.5) - // do not adapt to small OR large (outlier) changes - cPropagationDelayDeltaThresholdMin = 5 * time.Millisecond - cPropagationDelayDeltaThresholdMaxFactor = 2 + cPropagationDelayDeltaMaxInterval = 10 * time.Second // To account for path changes mid-stream, if the delta of the propagation delay is consistently higher, reset. // Reset at whichever of the below happens later. + // 1. 10 seconds of persistent high delta. + // 2. at least 2 reports with high delta. // // A long term version of delta of propagation delay is maintained and delta propagation delay exceeding // a factor of the long term version is considered a sharp increase. That will trigger the start of the // path change condition and if it persists, propagation delay will be reset. - cPropagationDelayDeltaMaxInterval = 10 * time.Second - cPropagationDelayDeltaHighResetNumReports = 3 + cPropagationDelayDeltaThresholdMin = 10 * time.Millisecond + cPropagationDelayDeltaThresholdMaxFactor = 2 + cPropagationDelayDeltaHighResetNumReports = 2 cPropagationDelayDeltaHighResetWait = 10 * time.Second ) @@ -394,7 +395,7 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *RTCPSenderReportData) r.logger.Debugw("initializing propagation delay", getPropagationFields()...) } else { deltaPropagationDelay = propagationDelay - r.propagationDelay - if deltaPropagationDelay.Abs() > cPropagationDelayDeltaThresholdMin { // ignore small changes + if deltaPropagationDelay.Abs() > cPropagationDelayDeltaThresholdMin { // ignore small changes for path change consideration if r.longTermDeltaPropagationDelay != 0 && deltaPropagationDelay > 0 && deltaPropagationDelay > r.longTermDeltaPropagationDelay*time.Duration(cPropagationDelayDeltaThresholdMaxFactor) { r.logger.Debugw("sharp increase in propagation delay, skipping", getPropagationFields()...) // TODO-REMOVE r.propagationDelayDeltaHighCount++ @@ -411,24 +412,21 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *RTCPSenderReportData) r.logger.Debugw("re-initializing propagation delay", append(getPropagationFields(), "newPropagationDelay", propagationDelay.String())...) initPropagationDelay(r.propagationDelaySpike) } - } else { - resetDelta() - - factor := cPropagationDelayFallFactor - if propagationDelay > r.propagationDelay { - factor = cPropagationDelayRiseFactor - } - adjustedPropagationDelay := r.propagationDelay + time.Duration(factor*float64(propagationDelay-r.propagationDelay)) // TODO-REMOVE - fields := append( - getPropagationFields(), - "adjustedPropagationDelay", adjustedPropagationDelay.String(), - ) // TODO-REMOVE - r.logger.Debugw("adapting propagation delay", fields...) // TODO-REMOVE - r.propagationDelay += time.Duration(factor * float64(propagationDelay-r.propagationDelay)) } } else { - r.propagationDelayDeltaHighCount = 0 - r.propagationDelayDeltaHighStartTime = time.Time{} + resetDelta() + + factor := cPropagationDelayFallFactor + if propagationDelay > r.propagationDelay { + factor = cPropagationDelayRiseFactor + } + adjustedPropagationDelay := r.propagationDelay + time.Duration(factor*float64(propagationDelay-r.propagationDelay)) // TODO-REMOVE + fields := append( + getPropagationFields(), + "adjustedPropagationDelay", adjustedPropagationDelay.String(), + ) // TODO-REMOVE + r.logger.Debugw("adapting propagation delay", fields...) // TODO-REMOVE + r.propagationDelay += time.Duration(factor * float64(propagationDelay-r.propagationDelay)) } if r.longTermDeltaPropagationDelay == 0 { From 278ae72f706606146d13144360d0fe948fe3d517 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sun, 31 Mar 2024 11:15:50 +0530 Subject: [PATCH 13/78] Avoid duplicate receivers on migration. (#2608) * Avoid duplicate receivers on migration. When migrating, post migration call to set up could add duplicate receivers. * don't need to check upgraded --- pkg/rtc/mediatrackreceiver.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/rtc/mediatrackreceiver.go b/pkg/rtc/mediatrackreceiver.go index 0283488a0..19f4483f5 100644 --- a/pkg/rtc/mediatrackreceiver.go +++ b/pkg/rtc/mediatrackreceiver.go @@ -161,17 +161,17 @@ func (t *MediaTrackReceiver) SetupReceiver(receiver sfu.TrackReceiver, priority receivers := slices.Clone(t.receivers) // codec position maybe taken by DummyReceiver, check and upgrade to WebRTCReceiver - var upgradeReceiver bool + var existingReceiver bool for _, r := range receivers { if strings.EqualFold(r.Codec().MimeType, receiver.Codec().MimeType) { + existingReceiver = true if d, ok := r.TrackReceiver.(*DummyReceiver); ok { d.Upgrade(receiver) - upgradeReceiver = true - break } + break } } - if !upgradeReceiver { + if !existingReceiver { receivers = append(receivers, &simulcastReceiver{TrackReceiver: receiver, priority: priority}) } From b1a4d00fa98bf8cdf945f2bb8debfda38bad74d2 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 1 Apr 2024 16:14:30 +0530 Subject: [PATCH 14/78] Replace receiver when there is an existing one. (#2611) The receiver should not change, but code wise, the option of replacing receiver object makes more sense, i.e. otherwise, it could look like we are leaving the stale object in there without replacing with new receiver of same type. --- pkg/rtc/mediatrackreceiver.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/pkg/rtc/mediatrackreceiver.go b/pkg/rtc/mediatrackreceiver.go index 19f4483f5..6c40b778e 100644 --- a/pkg/rtc/mediatrackreceiver.go +++ b/pkg/rtc/mediatrackreceiver.go @@ -161,19 +161,21 @@ func (t *MediaTrackReceiver) SetupReceiver(receiver sfu.TrackReceiver, priority receivers := slices.Clone(t.receivers) // codec position maybe taken by DummyReceiver, check and upgrade to WebRTCReceiver - var existingReceiver bool - for _, r := range receivers { + idx := -1 + for i, r := range receivers { if strings.EqualFold(r.Codec().MimeType, receiver.Codec().MimeType) { - existingReceiver = true - if d, ok := r.TrackReceiver.(*DummyReceiver); ok { - d.Upgrade(receiver) - } + idx = i break } } - if !existingReceiver { - receivers = append(receivers, &simulcastReceiver{TrackReceiver: receiver, priority: priority}) + if idx != -1 { + if d, ok := receivers[idx].TrackReceiver.(*DummyReceiver); ok { + d.Upgrade(receiver) + } + // replace receiver + receivers = slices.Delete(receivers, idx, idx+1) } + receivers = append(receivers, &simulcastReceiver{TrackReceiver: receiver, priority: priority}) sort.Slice(receivers, func(i, j int) bool { return receivers[i].Priority() < receivers[j].Priority() From 3ed0527ef59b9f7b9b333cc261dd317554881648 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 1 Apr 2024 18:53:54 +0530 Subject: [PATCH 15/78] Reduce chances of missing layer post migration. (#2612) When migrating out, it is possible that a published layer is not notified to the migrating in node. Reduce chances of that layer not getting published to the new node by curbing RTCP during migration. It could still happen if stars line up, but this should reduce the window to a much smaller one. --- pkg/rtc/participant.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index ccd67ab97..56ab3ea7e 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -2341,6 +2341,25 @@ func (p *ParticipantImpl) DebugInfo() map[string]interface{} { } func (p *ParticipantImpl) postRtcp(pkts []rtcp.Packet) { + p.lock.RLock() + migrationTimer := p.migrationTimer + p.lock.RUnlock() + + // Once migration out is active, layers getting added would not be communicated to + // where the publisher is migrating to. Without SSRC, `UnhandleSimulcastInterceptor` + // cannot be set up on the migrating in node. Without that interceptor, simulcast + // probing will fail. + // + // Clients usually send `rid` RTP header extension till they get an RTCP Receiver Report + // from the remote side. So, by curbing RTCP when migration is active, even if a new layer + // get published to this node, client should continue to send `rid` to the new node + // post migration and the new node can do regular simulcast probing (without the + // `UnhandleSimulcastInterceptor`) to fire `OnTrack` on that layer. And when the new node + // sends RTCP Receiver Report back to the client, client will stop `rid`. + if migrationTimer != nil { + return + } + p.pubRTCPQueue.Enqueue(func() { if err := p.TransportManager.WritePublisherRTCP(pkts); err != nil && !IsEOF(err) { p.pubLogger.Errorw("could not write RTCP to participant", err) From 19326a716261d577d96f84c1f8f94dfc79007712 Mon Sep 17 00:00:00 2001 From: Denys Smirnov Date: Mon, 1 Apr 2024 19:20:51 +0300 Subject: [PATCH 16/78] Pass ringtone flag for SIP outbound. (#2613) --- go.mod | 3 ++- go.sum | 6 ++++-- pkg/service/sip.go | 1 + 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 663630fcf..9afa36623 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20240302142739-1c3dd691a1b8 - github.com/livekit/protocol v1.12.1-0.20240321094538-0d9caadf760e + github.com/livekit/protocol v1.12.1-0.20240331203140-766ababa37ae github.com/livekit/psrpc v0.5.3-0.20240312110212-61ab09477c30 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 @@ -97,6 +97,7 @@ require ( github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap/exp v0.2.0 // indirect golang.org/x/crypto v0.21.0 // indirect golang.org/x/mod v0.16.0 // indirect golang.org/x/net v0.22.0 // indirect diff --git a/go.sum b/go.sum index c2eb9bb89..1c6c09cb8 100644 --- a/go.sum +++ b/go.sum @@ -132,8 +132,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20240302142739-1c3dd691a1b8 h1:xawydPEACNO5Ncs2LgioTjWghXQ0eUN1q1RnVUUyVnI= github.com/livekit/mediatransportutil v0.0.0-20240302142739-1c3dd691a1b8/go.mod h1:jwKUCmObuiEDH0iiuJHaGMXwRs3RjrB4G6qqgkr/5oE= -github.com/livekit/protocol v1.12.1-0.20240321094538-0d9caadf760e h1:XR7vPLN7c/R6R87UARoBW2csVKd7RuTXwG+XsjczbT0= -github.com/livekit/protocol v1.12.1-0.20240321094538-0d9caadf760e/go.mod h1:G7Pa985GhZv2MCC3UnUocBhZfi3DsWA6WmlSkkpQYTM= +github.com/livekit/protocol v1.12.1-0.20240331203140-766ababa37ae h1:uUc+ou3R6xXkrIRNMYHraGTYhCSs6D4p9++Vwq8CK0E= +github.com/livekit/protocol v1.12.1-0.20240331203140-766ababa37ae/go.mod h1:mcv7L2DWB6iRckI++egmwFU7YEy3W+aLHnusNhioqi0= github.com/livekit/psrpc v0.5.3-0.20240312110212-61ab09477c30 h1:3GEU6vP+KLTTOEqsFKW+PgIUp+i+s0jaUqogQc/hb7M= github.com/livekit/psrpc v0.5.3-0.20240312110212-61ab09477c30/go.mod h1:CQUBSPfYYAaevg1TNCc6/aYsa8DJH4jSRFdCeSZk5u0= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= @@ -297,6 +297,8 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.uber.org/zap/exp v0.2.0 h1:FtGenNNeCATRB3CmB/yEUnjEFeJWpB/pMcy7e2bKPYs= +go.uber.org/zap/exp v0.2.0/go.mod h1:t0gqAIdh1MfKv9EwN/dLwfZnJxe9ITAZN78HEWPFWDQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= diff --git a/pkg/service/sip.go b/pkg/service/sip.go index c3989e854..1bcf8f603 100644 --- a/pkg/service/sip.go +++ b/pkg/service/sip.go @@ -197,6 +197,7 @@ func (s *SIPService) CreateSIPParticipantWithToken(ctx context.Context, req *liv RoomName: req.RoomName, ParticipantIdentity: req.ParticipantIdentity, Dtmf: req.Dtmf, + PlayRingtone: req.PlayRingtone, WsUrl: wsUrl, Token: token, } From f4ead066012811dd4df226732bfcf64371f7a0bf Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 22:17:20 -0700 Subject: [PATCH 17/78] Update pion deps (#2587) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 10 +++++----- go.sum | 19 +++++++++++-------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index 9afa36623..256bd38f0 100644 --- a/go.mod +++ b/go.mod @@ -28,14 +28,14 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/pion/dtls/v2 v2.2.10 github.com/pion/ice/v2 v2.3.14 - github.com/pion/interceptor v0.1.25 + github.com/pion/interceptor v0.1.27 github.com/pion/rtcp v1.2.14 - github.com/pion/rtp v1.8.3 - github.com/pion/sctp v1.8.12 - github.com/pion/sdp/v3 v3.0.8 + github.com/pion/rtp v1.8.5 + github.com/pion/sctp v1.8.13 + github.com/pion/sdp/v3 v3.0.9 github.com/pion/transport/v2 v2.2.4 github.com/pion/turn/v2 v2.1.5 - github.com/pion/webrtc/v3 v3.2.29 + github.com/pion/webrtc/v3 v3.2.32 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.19.0 github.com/redis/go-redis/v9 v9.5.1 diff --git a/go.sum b/go.sum index 1c6c09cb8..d1b66010a 100644 --- a/go.sum +++ b/go.sum @@ -192,8 +192,9 @@ github.com/pion/dtls/v2 v2.2.10/go.mod h1:d9SYc9fch0CqK90mRk1dC7AkzzpwJj6u2GU3u+ github.com/pion/ice/v2 v2.3.13/go.mod h1:KXJJcZK7E8WzrBEYnV4UtqEZsGeWfHxsNqhVcVvgjxw= github.com/pion/ice/v2 v2.3.14 h1:A7UaEmalw12Fko8YO0qguUbWyE69BnN4mDEqT7cLWQI= github.com/pion/ice/v2 v2.3.14/go.mod h1:KXJJcZK7E8WzrBEYnV4UtqEZsGeWfHxsNqhVcVvgjxw= -github.com/pion/interceptor v0.1.25 h1:pwY9r7P6ToQ3+IF0bajN0xmk/fNw/suTgaTdlwTDmhc= github.com/pion/interceptor v0.1.25/go.mod h1:wkbPYAak5zKsfpVDYMtEfWEy8D4zL+rpxCxPImLOg3Y= +github.com/pion/interceptor v0.1.27 h1:mZ01OiGiukwRxezmDGzYjjokCVlDOk4T6BfaL5qrtGo= +github.com/pion/interceptor v0.1.27/go.mod h1:/vVaqLwDjGv4GRbgmChIKZIT5EXFDijwmj4WmIYy9bI= github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= github.com/pion/mdns v0.0.12 h1:CiMYlY+O0azojWDmxdNr7ADGrnZ+V6Ilfner+6mSVK8= @@ -205,13 +206,15 @@ github.com/pion/rtcp v1.2.12/go.mod h1:sn6qjxvnwyAkkPzPULIbVqSKI5Dv54Rv7VG0kNxh9 github.com/pion/rtcp v1.2.14 h1:KCkGV3vJ+4DAJmvP0vaQShsb0xkRfWkO540Gy102KyE= github.com/pion/rtcp v1.2.14/go.mod h1:sn6qjxvnwyAkkPzPULIbVqSKI5Dv54Rv7VG0kNxh9L4= github.com/pion/rtp v1.8.2/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= -github.com/pion/rtp v1.8.3 h1:VEHxqzSVQxCkKDSHro5/4IUUG1ea+MFdqR2R3xSpNU8= github.com/pion/rtp v1.8.3/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= +github.com/pion/rtp v1.8.4/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= +github.com/pion/rtp v1.8.5 h1:uYzINfaK+9yWs7r537z/Rc1SvT8ILjBcmDOpJcTB+OU= +github.com/pion/rtp v1.8.5/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= github.com/pion/sctp v1.8.5/go.mod h1:SUFFfDpViyKejTAdwD1d/HQsCu+V/40cCs2nZIvC3s0= -github.com/pion/sctp v1.8.12 h1:2VX50pedElH+is6FI+OKyRTeN5oy4mrk2HjnGa3UCmY= -github.com/pion/sctp v1.8.12/go.mod h1:cMLT45jqw3+jiJCrtHVwfQLnfR0MGZ4rgOJwUOIqLkI= -github.com/pion/sdp/v3 v3.0.8 h1:yd/wkrS0nzXEAb+uwv1TL3SG/gzsTiXHVOtXtD7EKl0= -github.com/pion/sdp/v3 v3.0.8/go.mod h1:B5xmvENq5IXJimIO4zfp6LAe1fD9N+kFv+V/1lOdz8M= +github.com/pion/sctp v1.8.13 h1:YUJR44pWM2FPUhkl8l+vDyF2EDE3aTWtr3c+LDhCRcQ= +github.com/pion/sctp v1.8.13/go.mod h1:YKSgO/bO/6aOMP9LCie1DuD7m+GamiK2yIiPM6vH+GA= +github.com/pion/sdp/v3 v3.0.9 h1:pX++dCHoHUwq43kuwf3PyJfHlwIj4hXA7Vrifiq0IJY= +github.com/pion/sdp/v3 v3.0.9/go.mod h1:B5xmvENq5IXJimIO4zfp6LAe1fD9N+kFv+V/1lOdz8M= github.com/pion/srtp/v2 v2.0.18 h1:vKpAXfawO9RtTRKZJbG4y0v1b11NZxQnxRl85kGuUlo= github.com/pion/srtp/v2 v2.0.18/go.mod h1:0KJQjA99A6/a0DOVTu1PhDSw0CXF2jTkqOoMg3ODqdA= github.com/pion/stun v0.6.1 h1:8lp6YejULeHBF8NmV8e2787BogQhduZugh5PdhDyyN4= @@ -228,8 +231,8 @@ github.com/pion/transport/v3 v3.0.1/go.mod h1:UY7kiITrlMv7/IKgd5eTUcaahZx5oUN3l9 github.com/pion/turn/v2 v2.1.3/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= github.com/pion/turn/v2 v2.1.5 h1:tTyy7TM3DCoX9IxTt/yHc/bThiRLyXK3T1YbNcgx9k4= github.com/pion/turn/v2 v2.1.5/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= -github.com/pion/webrtc/v3 v3.2.29 h1:flXjxjlqpp3FjkpSSBKwv7UOfbUvan9+gFY6A5ZaAn4= -github.com/pion/webrtc/v3 v3.2.29/go.mod h1:M+5YSvBDPAkHHRwGXlplIFBQI5mXm6Y4byns1OpiX68= +github.com/pion/webrtc/v3 v3.2.32 h1:AGARHapcj1L9py06mbf0V/zgQ5cPqboSoNxr9Ck/TZU= +github.com/pion/webrtc/v3 v3.2.32/go.mod h1:8nsXAU16J1wabqt93u1PwwhkCbGrA/B143sRepFzJ1Y= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= From 3c8980b443a29e77657dcb756cddee36b6457a81 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 2 Apr 2024 11:03:50 +0530 Subject: [PATCH 18/78] Fix media track close check. (#2614) With the change in https://github.com/livekit/livekit/pull/2611, the dummy receiver was replaced with real receiver. But, the close check was using the dummy receiver. Doing two things - Use the dummy receiver post upgrade also (NOTE: this is not needed, but just keeping old behaviour) - Fix the close check to count number of open receivers. --- pkg/rtc/mediatrackreceiver.go | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/pkg/rtc/mediatrackreceiver.go b/pkg/rtc/mediatrackreceiver.go index 6c40b778e..a29b2a630 100644 --- a/pkg/rtc/mediatrackreceiver.go +++ b/pkg/rtc/mediatrackreceiver.go @@ -161,6 +161,7 @@ func (t *MediaTrackReceiver) SetupReceiver(receiver sfu.TrackReceiver, priority receivers := slices.Clone(t.receivers) // codec position maybe taken by DummyReceiver, check and upgrade to WebRTCReceiver + receiverToAdd := receiver idx := -1 for i, r := range receivers { if strings.EqualFold(r.Codec().MimeType, receiver.Codec().MimeType) { @@ -171,11 +172,12 @@ func (t *MediaTrackReceiver) SetupReceiver(receiver sfu.TrackReceiver, priority if idx != -1 { if d, ok := receivers[idx].TrackReceiver.(*DummyReceiver); ok { d.Upgrade(receiver) + receiverToAdd = d } // replace receiver receivers = slices.Delete(receivers, idx, idx+1) } - receivers = append(receivers, &simulcastReceiver{TrackReceiver: receiver, priority: priority}) + receivers = append(receivers, &simulcastReceiver{TrackReceiver: receiverToAdd, priority: priority}) sort.Slice(receivers, func(i, j int) bool { return receivers[i].Priority() < receivers[j].Priority() @@ -321,15 +323,21 @@ func (t *MediaTrackReceiver) TryClose() bool { return true } + numActiveReceivers := 0 for _, receiver := range t.receivers { - if dr, _ := receiver.TrackReceiver.(*DummyReceiver); dr != nil && dr.Receiver() != nil { - t.lock.RUnlock() - return false + dr, ok := receiver.TrackReceiver.(*DummyReceiver) + if !ok || dr.Receiver() != nil { + // !ok means real receiver OR + // dummy receiver with a regular receiver attached + numActiveReceivers++ } } t.lock.RUnlock() - t.Close() + if numActiveReceivers != 0 { + return false + } + t.Close() return true } From 860702e9dcf93701931e7caee9ec3cd78409a757 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 2 Apr 2024 14:21:20 +0530 Subject: [PATCH 19/78] Prevent large spikes in propagation delay (#2615) * Prevent large spikes in propagation delay A few tweaks - Large spike in propagation delay due to congested channel results in long term estimate getting high value. Ignore outliers in long term estimate. - Introduce a new field for adjusted arrival time as adjusting the arrival time in place meant it got applied again across the relay and that caused different propagation delay on remote nodes. - Reset path change counters as long as there is any sample that is not higher than the multiple of long term. There was a case of o Sample with high value that triggered path change start. o Then some samples with high enough delta, but did not meet the criteria for increasing counter further. o Some time later, another sample met the threshold and that triggered a path change re-init. * do not adapt to large delta --- pkg/sfu/buffer/rtpstats_base.go | 6 +++-- pkg/sfu/buffer/rtpstats_receiver.go | 40 +++++++++++++++++------------ pkg/sfu/buffer/rtpstats_sender.go | 5 ++-- 3 files changed, 31 insertions(+), 20 deletions(-) diff --git a/pkg/sfu/buffer/rtpstats_base.go b/pkg/sfu/buffer/rtpstats_base.go index 11c9517c6..6d6203c17 100644 --- a/pkg/sfu/buffer/rtpstats_base.go +++ b/pkg/sfu/buffer/rtpstats_base.go @@ -114,6 +114,7 @@ type RTCPSenderReportData struct { RTPTimestampExt uint64 NTPTimestamp mediatransportutil.NtpTime At time.Time + AtAdjusted time.Time } func (r *RTCPSenderReportData) ToString() string { @@ -121,7 +122,7 @@ func (r *RTCPSenderReportData) ToString() string { return "" } - return fmt.Sprintf("ntp: %s, rtp: %d, extRtp: %d, at: %s", r.NTPTimestamp.Time().String(), r.RTPTimestamp, r.RTPTimestampExt, r.At.String()) + return fmt.Sprintf("ntp: %s, rtp: %d, extRtp: %d, at: %s, atAdj: %s", r.NTPTimestamp.Time().String(), r.RTPTimestamp, r.RTPTimestampExt, r.At.String(), r.AtAdjusted.String()) } func (r *RTCPSenderReportData) MarshalLogObject(e zapcore.ObjectEncoder) error { @@ -133,6 +134,7 @@ func (r *RTCPSenderReportData) MarshalLogObject(e zapcore.ObjectEncoder) error { e.AddUint32("RTPTimestamp", r.RTPTimestamp) e.AddUint64("RTPTimestampExt", r.RTPTimestampExt) e.AddTime("At", r.At) + e.AddTime("AtAdjusted", r.AtAdjusted) return nil } @@ -495,7 +497,7 @@ func (r *rtpStatsBase) maybeAdjustFirstPacketTime(srData *RTCPSenderReportData, // abnormal delay (maybe due to pacing or maybe due to queuing // in some network element along the way), push back first time // to an earlier instance. - timeSinceReceive := time.Since(srData.At) + timeSinceReceive := time.Since(srData.AtAdjusted) extNowTS := srData.RTPTimestampExt - tsOffset + uint64(timeSinceReceive.Nanoseconds()*int64(r.params.ClockRate)/1e9) samplesDiff := int64(extNowTS - extStartTS) if samplesDiff < 0 { diff --git a/pkg/sfu/buffer/rtpstats_receiver.go b/pkg/sfu/buffer/rtpstats_receiver.go index ecdc0be8c..706be886f 100644 --- a/pkg/sfu/buffer/rtpstats_receiver.go +++ b/pkg/sfu/buffer/rtpstats_receiver.go @@ -43,20 +43,19 @@ const ( cPropagationDelaySpikeAdaptationFactor = float64(0.5) - cPropagationDelayDeltaMaxInterval = 10 * time.Second - // To account for path changes mid-stream, if the delta of the propagation delay is consistently higher, reset. // Reset at whichever of the below happens later. // 1. 10 seconds of persistent high delta. - // 2. at least 2 reports with high delta. + // 2. at least 2 consecutive reports with high delta. // - // A long term version of delta of propagation delay is maintained and delta propagation delay exceeding - // a factor of the long term version is considered a sharp increase. That will trigger the start of the + // A long term estimate of delta of propagation delay is maintained and delta propagation delay exceeding + // a factor of the long term estimate is considered a sharp increase. That will trigger the start of the // path change condition and if it persists, propagation delay will be reset. - cPropagationDelayDeltaThresholdMin = 10 * time.Millisecond - cPropagationDelayDeltaThresholdMaxFactor = 2 - cPropagationDelayDeltaHighResetNumReports = 2 - cPropagationDelayDeltaHighResetWait = 10 * time.Second + cPropagationDelayDeltaThresholdMin = 10 * time.Millisecond + cPropagationDelayDeltaThresholdMaxFactor = 2 + cPropagationDelayDeltaHighResetNumReports = 2 + cPropagationDelayDeltaHighResetWait = 10 * time.Second + cPropagationDelayDeltaLongTermAdaptationThreshold = 50 * time.Millisecond ) type RTPFlowState struct { @@ -369,6 +368,7 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *RTCPSenderReportData) "receivedDeltaPropagationDelay", deltaPropagationDelay.String(), "deltaHighCount", r.propagationDelayDeltaHighCount, "sinceDeltaHighStart", time.Since(r.propagationDelayDeltaHighStartTime).String(), + "propagationDelaySpike", r.propagationDelaySpike.String(), "first", r.srFirst, "last", r.srNewest, "current", &srDataCopy, @@ -395,9 +395,11 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *RTCPSenderReportData) r.logger.Debugw("initializing propagation delay", getPropagationFields()...) } else { deltaPropagationDelay = propagationDelay - r.propagationDelay - if deltaPropagationDelay.Abs() > cPropagationDelayDeltaThresholdMin { // ignore small changes for path change consideration - if r.longTermDeltaPropagationDelay != 0 && deltaPropagationDelay > 0 && deltaPropagationDelay > r.longTermDeltaPropagationDelay*time.Duration(cPropagationDelayDeltaThresholdMaxFactor) { - r.logger.Debugw("sharp increase in propagation delay, skipping", getPropagationFields()...) // TODO-REMOVE + if deltaPropagationDelay > cPropagationDelayDeltaThresholdMin { // ignore small changes for path change consideration + if r.longTermDeltaPropagationDelay != 0 && + deltaPropagationDelay > 0 && + deltaPropagationDelay > r.longTermDeltaPropagationDelay*time.Duration(cPropagationDelayDeltaThresholdMaxFactor) { + r.logger.Debugw("sharp increase in propagation delay", getPropagationFields()...) // TODO-REMOVE r.propagationDelayDeltaHighCount++ if r.propagationDelayDeltaHighStartTime.IsZero() { r.propagationDelayDeltaHighStartTime = time.Now() @@ -412,6 +414,8 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *RTCPSenderReportData) r.logger.Debugw("re-initializing propagation delay", append(getPropagationFields(), "newPropagationDelay", propagationDelay.String())...) initPropagationDelay(r.propagationDelaySpike) } + } else { + resetDelta() } } else { resetDelta() @@ -432,13 +436,17 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *RTCPSenderReportData) if r.longTermDeltaPropagationDelay == 0 { r.longTermDeltaPropagationDelay = deltaPropagationDelay } else { - sinceLastReport := srDataCopy.NTPTimestamp.Time().Sub(r.srNewest.NTPTimestamp.Time()) - adaptationFactor := min(1.0, float64(sinceLastReport)/float64(cPropagationDelayDeltaMaxInterval)) - r.longTermDeltaPropagationDelay += time.Duration(adaptationFactor * float64(deltaPropagationDelay-r.longTermDeltaPropagationDelay)) + if deltaPropagationDelay < cPropagationDelayDeltaLongTermAdaptationThreshold { + // do not adapt to large +ve spikes, can happen when channel is congested and reports are delivered very late + // if the spike is in fact a path change, it will persist and handled by path change detection above + sinceLastReport := srDataCopy.NTPTimestamp.Time().Sub(r.srNewest.NTPTimestamp.Time()) + adaptationFactor := min(1.0, float64(sinceLastReport)/float64(cPropagationDelayDeltaHighResetWait)) + r.longTermDeltaPropagationDelay += time.Duration(adaptationFactor * float64(deltaPropagationDelay-r.longTermDeltaPropagationDelay)) + } } } // adjust receive time to estimated propagation delay - srDataCopy.At = ntpTime.Add(r.propagationDelay) + srDataCopy.AtAdjusted = ntpTime.Add(r.propagationDelay) r.srNewest = &srDataCopy r.maybeAdjustFirstPacketTime(r.srNewest, 0, r.timestamp.GetExtendedStart()) diff --git a/pkg/sfu/buffer/rtpstats_sender.go b/pkg/sfu/buffer/rtpstats_sender.go index 4fe39c0c5..b7cc94c06 100644 --- a/pkg/sfu/buffer/rtpstats_sender.go +++ b/pkg/sfu/buffer/rtpstats_sender.go @@ -633,8 +633,8 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, publisherSRData *RTCPS return nil } - timeSincePublisherSR := time.Since(publisherSRData.At) - now := publisherSRData.At.Add(timeSincePublisherSR) + timeSincePublisherSR := time.Since(publisherSRData.AtAdjusted) + now := publisherSRData.AtAdjusted.Add(timeSincePublisherSR) nowNTP := mediatransportutil.ToNtpTime(now) nowRTPExt := publisherSRData.RTPTimestampExt - tsOffset + uint64(timeSincePublisherSR.Nanoseconds()*int64(r.params.ClockRate)/1e9) @@ -643,6 +643,7 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, publisherSRData *RTCPS RTPTimestamp: uint32(nowRTPExt), RTPTimestampExt: nowRTPExt, At: now, + AtAdjusted: now, } getFields := func() []interface{} { From 1caa6ff6d049a2575403c3b99c93fd49aa0d56a6 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 2 Apr 2024 22:29:04 +0530 Subject: [PATCH 20/78] Revert "Update pion deps (#2587)" (#2616) This reverts commit f4ead066012811dd4df226732bfcf64371f7a0bf. --- go.mod | 10 +++++----- go.sum | 19 ++++++++----------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index 256bd38f0..9afa36623 100644 --- a/go.mod +++ b/go.mod @@ -28,14 +28,14 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/pion/dtls/v2 v2.2.10 github.com/pion/ice/v2 v2.3.14 - github.com/pion/interceptor v0.1.27 + github.com/pion/interceptor v0.1.25 github.com/pion/rtcp v1.2.14 - github.com/pion/rtp v1.8.5 - github.com/pion/sctp v1.8.13 - github.com/pion/sdp/v3 v3.0.9 + github.com/pion/rtp v1.8.3 + github.com/pion/sctp v1.8.12 + github.com/pion/sdp/v3 v3.0.8 github.com/pion/transport/v2 v2.2.4 github.com/pion/turn/v2 v2.1.5 - github.com/pion/webrtc/v3 v3.2.32 + github.com/pion/webrtc/v3 v3.2.29 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.19.0 github.com/redis/go-redis/v9 v9.5.1 diff --git a/go.sum b/go.sum index d1b66010a..1c6c09cb8 100644 --- a/go.sum +++ b/go.sum @@ -192,9 +192,8 @@ github.com/pion/dtls/v2 v2.2.10/go.mod h1:d9SYc9fch0CqK90mRk1dC7AkzzpwJj6u2GU3u+ github.com/pion/ice/v2 v2.3.13/go.mod h1:KXJJcZK7E8WzrBEYnV4UtqEZsGeWfHxsNqhVcVvgjxw= github.com/pion/ice/v2 v2.3.14 h1:A7UaEmalw12Fko8YO0qguUbWyE69BnN4mDEqT7cLWQI= github.com/pion/ice/v2 v2.3.14/go.mod h1:KXJJcZK7E8WzrBEYnV4UtqEZsGeWfHxsNqhVcVvgjxw= +github.com/pion/interceptor v0.1.25 h1:pwY9r7P6ToQ3+IF0bajN0xmk/fNw/suTgaTdlwTDmhc= github.com/pion/interceptor v0.1.25/go.mod h1:wkbPYAak5zKsfpVDYMtEfWEy8D4zL+rpxCxPImLOg3Y= -github.com/pion/interceptor v0.1.27 h1:mZ01OiGiukwRxezmDGzYjjokCVlDOk4T6BfaL5qrtGo= -github.com/pion/interceptor v0.1.27/go.mod h1:/vVaqLwDjGv4GRbgmChIKZIT5EXFDijwmj4WmIYy9bI= github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= github.com/pion/mdns v0.0.12 h1:CiMYlY+O0azojWDmxdNr7ADGrnZ+V6Ilfner+6mSVK8= @@ -206,15 +205,13 @@ github.com/pion/rtcp v1.2.12/go.mod h1:sn6qjxvnwyAkkPzPULIbVqSKI5Dv54Rv7VG0kNxh9 github.com/pion/rtcp v1.2.14 h1:KCkGV3vJ+4DAJmvP0vaQShsb0xkRfWkO540Gy102KyE= github.com/pion/rtcp v1.2.14/go.mod h1:sn6qjxvnwyAkkPzPULIbVqSKI5Dv54Rv7VG0kNxh9L4= github.com/pion/rtp v1.8.2/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= +github.com/pion/rtp v1.8.3 h1:VEHxqzSVQxCkKDSHro5/4IUUG1ea+MFdqR2R3xSpNU8= github.com/pion/rtp v1.8.3/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= -github.com/pion/rtp v1.8.4/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= -github.com/pion/rtp v1.8.5 h1:uYzINfaK+9yWs7r537z/Rc1SvT8ILjBcmDOpJcTB+OU= -github.com/pion/rtp v1.8.5/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= github.com/pion/sctp v1.8.5/go.mod h1:SUFFfDpViyKejTAdwD1d/HQsCu+V/40cCs2nZIvC3s0= -github.com/pion/sctp v1.8.13 h1:YUJR44pWM2FPUhkl8l+vDyF2EDE3aTWtr3c+LDhCRcQ= -github.com/pion/sctp v1.8.13/go.mod h1:YKSgO/bO/6aOMP9LCie1DuD7m+GamiK2yIiPM6vH+GA= -github.com/pion/sdp/v3 v3.0.9 h1:pX++dCHoHUwq43kuwf3PyJfHlwIj4hXA7Vrifiq0IJY= -github.com/pion/sdp/v3 v3.0.9/go.mod h1:B5xmvENq5IXJimIO4zfp6LAe1fD9N+kFv+V/1lOdz8M= +github.com/pion/sctp v1.8.12 h1:2VX50pedElH+is6FI+OKyRTeN5oy4mrk2HjnGa3UCmY= +github.com/pion/sctp v1.8.12/go.mod h1:cMLT45jqw3+jiJCrtHVwfQLnfR0MGZ4rgOJwUOIqLkI= +github.com/pion/sdp/v3 v3.0.8 h1:yd/wkrS0nzXEAb+uwv1TL3SG/gzsTiXHVOtXtD7EKl0= +github.com/pion/sdp/v3 v3.0.8/go.mod h1:B5xmvENq5IXJimIO4zfp6LAe1fD9N+kFv+V/1lOdz8M= github.com/pion/srtp/v2 v2.0.18 h1:vKpAXfawO9RtTRKZJbG4y0v1b11NZxQnxRl85kGuUlo= github.com/pion/srtp/v2 v2.0.18/go.mod h1:0KJQjA99A6/a0DOVTu1PhDSw0CXF2jTkqOoMg3ODqdA= github.com/pion/stun v0.6.1 h1:8lp6YejULeHBF8NmV8e2787BogQhduZugh5PdhDyyN4= @@ -231,8 +228,8 @@ github.com/pion/transport/v3 v3.0.1/go.mod h1:UY7kiITrlMv7/IKgd5eTUcaahZx5oUN3l9 github.com/pion/turn/v2 v2.1.3/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= github.com/pion/turn/v2 v2.1.5 h1:tTyy7TM3DCoX9IxTt/yHc/bThiRLyXK3T1YbNcgx9k4= github.com/pion/turn/v2 v2.1.5/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= -github.com/pion/webrtc/v3 v3.2.32 h1:AGARHapcj1L9py06mbf0V/zgQ5cPqboSoNxr9Ck/TZU= -github.com/pion/webrtc/v3 v3.2.32/go.mod h1:8nsXAU16J1wabqt93u1PwwhkCbGrA/B143sRepFzJ1Y= +github.com/pion/webrtc/v3 v3.2.29 h1:flXjxjlqpp3FjkpSSBKwv7UOfbUvan9+gFY6A5ZaAn4= +github.com/pion/webrtc/v3 v3.2.29/go.mod h1:M+5YSvBDPAkHHRwGXlplIFBQI5mXm6Y4byns1OpiX68= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= From 63b1fba08266452e8ba4e49ecba46c5f3148e5f7 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 3 Apr 2024 12:23:18 +0530 Subject: [PATCH 21/78] Add start/end time to AnalyticsStream. (#2618) * Add start/end time to AnalyticsStream. * fix test --- go.mod | 8 ++-- go.sum | 16 +++---- pkg/sfu/buffer/buffer.go | 18 +++---- pkg/sfu/buffer/rtpstats_base.go | 13 +++-- pkg/sfu/buffer/rtpstats_sender.go | 2 +- pkg/sfu/connectionquality/connectionstats.go | 6 ++- .../connectionquality/connectionstats_test.go | 48 +++++++++---------- pkg/sfu/connectionquality/scorer.go | 2 +- pkg/telemetry/statsworker.go | 14 ++++++ 9 files changed, 72 insertions(+), 55 deletions(-) diff --git a/go.mod b/go.mod index 9afa36623..9ca9921c1 100644 --- a/go.mod +++ b/go.mod @@ -19,8 +19,8 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20240302142739-1c3dd691a1b8 - github.com/livekit/protocol v1.12.1-0.20240331203140-766ababa37ae - github.com/livekit/psrpc v0.5.3-0.20240312110212-61ab09477c30 + github.com/livekit/protocol v1.12.1-0.20240403063258-fc68e8d82b26 + github.com/livekit/psrpc v0.5.3-0.20240327035954-cec3a0e614be github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1 @@ -79,7 +79,7 @@ require ( github.com/mattn/go-runewidth v0.0.9 // indirect github.com/mdlayher/netlink v1.7.1 // indirect github.com/mdlayher/socket v0.4.0 // indirect - github.com/nats-io/nats.go v1.33.1 // indirect + github.com/nats-io/nats.go v1.34.0 // indirect github.com/nats-io/nkeys v0.4.7 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/pion/datachannel v1.5.5 // indirect @@ -104,7 +104,7 @@ require ( golang.org/x/sys v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.19.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect google.golang.org/grpc v1.62.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 1c6c09cb8..fb236c8f8 100644 --- a/go.sum +++ b/go.sum @@ -132,10 +132,10 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20240302142739-1c3dd691a1b8 h1:xawydPEACNO5Ncs2LgioTjWghXQ0eUN1q1RnVUUyVnI= github.com/livekit/mediatransportutil v0.0.0-20240302142739-1c3dd691a1b8/go.mod h1:jwKUCmObuiEDH0iiuJHaGMXwRs3RjrB4G6qqgkr/5oE= -github.com/livekit/protocol v1.12.1-0.20240331203140-766ababa37ae h1:uUc+ou3R6xXkrIRNMYHraGTYhCSs6D4p9++Vwq8CK0E= -github.com/livekit/protocol v1.12.1-0.20240331203140-766ababa37ae/go.mod h1:mcv7L2DWB6iRckI++egmwFU7YEy3W+aLHnusNhioqi0= -github.com/livekit/psrpc v0.5.3-0.20240312110212-61ab09477c30 h1:3GEU6vP+KLTTOEqsFKW+PgIUp+i+s0jaUqogQc/hb7M= -github.com/livekit/psrpc v0.5.3-0.20240312110212-61ab09477c30/go.mod h1:CQUBSPfYYAaevg1TNCc6/aYsa8DJH4jSRFdCeSZk5u0= +github.com/livekit/protocol v1.12.1-0.20240403063258-fc68e8d82b26 h1:8Oog+IH4NVbq7PEXYU7bIW8Kw4rqd0HgtWaOh++9ZcE= +github.com/livekit/protocol v1.12.1-0.20240403063258-fc68e8d82b26/go.mod h1:mcv7L2DWB6iRckI++egmwFU7YEy3W+aLHnusNhioqi0= +github.com/livekit/psrpc v0.5.3-0.20240327035954-cec3a0e614be h1:W1nCFZ19rYAORMBNX82NeVPHjADN0UyORr6refbUXpU= +github.com/livekit/psrpc v0.5.3-0.20240327035954-cec3a0e614be/go.mod h1:CQUBSPfYYAaevg1TNCc6/aYsa8DJH4jSRFdCeSZk5u0= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= github.com/mackerelio/go-osstat v0.2.4/go.mod h1:Zy+qzGdZs3A9cuIqmgbJvwbmLQH9dJvtio5ZjJTbdlQ= github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= @@ -165,8 +165,8 @@ github.com/mdlayher/socket v0.4.0 h1:280wsy40IC9M9q1uPGcLBwXpcTQDtoGwVt+BNoITxIw github.com/mdlayher/socket v0.4.0/go.mod h1:xxFqz5GRCUN3UEOm9CZqEJsAbe1C8OwSK46NlmWuVoc= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/nats-io/nats.go v1.33.1 h1:8TxLZZ/seeEfR97qV0/Bl939tpDnt2Z2fK3HkPypj70= -github.com/nats-io/nats.go v1.33.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8= +github.com/nats-io/nats.go v1.34.0 h1:fnxnPCNiwIG5w08rlMcEKTUw4AV/nKyGCOJE8TdhSPk= +github.com/nats-io/nats.go v1.34.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8= github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI= github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= @@ -440,8 +440,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7 h1:8EeVk1VKMD+GD/neyEHGmz7pFblqPjHoi+PGQIlLx2s= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda h1:LI5DOvAxUPMv/50agcLLoo+AdWc1irS9Rzz4vPuD1V4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go index b533318da..8cfa7e663 100644 --- a/pkg/sfu/buffer/buffer.go +++ b/pkg/sfu/buffer/buffer.go @@ -793,14 +793,16 @@ func (b *Buffer) mayGrowBucket() { return } oldCap := cap - deltaInfo := b.rtpStats.DeltaInfo(b.ppsSnapshotId) - if deltaInfo != nil && deltaInfo.Duration > 500*time.Millisecond { - pps := int(time.Duration(deltaInfo.Packets) * time.Second / deltaInfo.Duration) - for pps > cap && cap < maxPkts { - cap = b.bucket.Grow() - } - if cap > oldCap { - b.logger.Debugw("grow bucket", "from", oldCap, "to", cap, "pps", pps) + if deltaInfo := b.rtpStats.DeltaInfo(b.ppsSnapshotId); deltaInfo != nil { + duration := deltaInfo.EndTime.Sub(deltaInfo.StartTime) + if duration > 500*time.Millisecond { + pps := int(time.Duration(deltaInfo.Packets) * time.Second / duration) + for pps > cap && cap < maxPkts { + cap = b.bucket.Grow() + } + if cap > oldCap { + b.logger.Debugw("grow bucket", "from", oldCap, "to", cap, "pps", pps) + } } } } diff --git a/pkg/sfu/buffer/rtpstats_base.go b/pkg/sfu/buffer/rtpstats_base.go index 6d6203c17..77586d38b 100644 --- a/pkg/sfu/buffer/rtpstats_base.go +++ b/pkg/sfu/buffer/rtpstats_base.go @@ -55,7 +55,7 @@ func RTPDriftToString(r *livekit.RTPDrift) string { type RTPDeltaInfo struct { StartTime time.Time - Duration time.Duration + EndTime time.Time Packets uint32 Bytes uint64 HeaderBytes uint64 @@ -572,7 +572,7 @@ func (r *rtpStatsBase) deltaInfo(snapshotID uint32, extStartSN uint64, extHighes if packetsExpected == 0 { return &RTPDeltaInfo{ StartTime: startTime, - Duration: endTime.Sub(startTime), + EndTime: endTime, } } @@ -592,7 +592,7 @@ func (r *rtpStatsBase) deltaInfo(snapshotID uint32, extStartSN uint64, extHighes return &RTPDeltaInfo{ StartTime: startTime, - Duration: endTime.Sub(startTime), + EndTime: endTime, Packets: uint32(packetsExpected), Bytes: now.bytes - then.bytes, HeaderBytes: now.headerBytes - then.headerBytes, @@ -997,9 +997,8 @@ func AggregateRTPDeltaInfo(deltaInfoList []*RTPDeltaInfo) *RTPDeltaInfo { startTime = deltaInfo.StartTime } - endedAt := deltaInfo.StartTime.Add(deltaInfo.Duration) - if endTime.IsZero() || endTime.Before(endedAt) { - endTime = endedAt + if endTime.IsZero() || endTime.Before(deltaInfo.EndTime) { + endTime = deltaInfo.EndTime } packets += deltaInfo.Packets @@ -1038,7 +1037,7 @@ func AggregateRTPDeltaInfo(deltaInfoList []*RTPDeltaInfo) *RTPDeltaInfo { return &RTPDeltaInfo{ StartTime: startTime, - Duration: endTime.Sub(startTime), + EndTime: endTime, Packets: packets, Bytes: bytes, HeaderBytes: headerBytes, diff --git a/pkg/sfu/buffer/rtpstats_sender.go b/pkg/sfu/buffer/rtpstats_sender.go index b7cc94c06..b85efb829 100644 --- a/pkg/sfu/buffer/rtpstats_sender.go +++ b/pkg/sfu/buffer/rtpstats_sender.go @@ -776,7 +776,7 @@ func (r *RTPStatsSender) DeltaInfoSender(senderSnapshotID uint32) *RTPDeltaInfo return &RTPDeltaInfo{ StartTime: startTime, - Duration: endTime.Sub(startTime), + EndTime: endTime, Packets: packetsExpected - uint32(now.packetsPadding-then.packetsPadding), Bytes: now.bytes - then.bytes, HeaderBytes: now.headerBytes - then.headerBytes, diff --git a/pkg/sfu/connectionquality/connectionstats.go b/pkg/sfu/connectionquality/connectionstats.go index c221ea130..6c955dc71 100644 --- a/pkg/sfu/connectionquality/connectionstats.go +++ b/pkg/sfu/connectionquality/connectionstats.go @@ -22,6 +22,7 @@ import ( "github.com/frostbyte73/core" "github.com/pion/webrtc/v3" "go.uber.org/atomic" + "google.golang.org/protobuf/types/known/timestamppb" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" @@ -205,7 +206,7 @@ func (cs *ConnectionStats) updateScoreWithAggregate(agg *buffer.RTPDeltaInfo, at var stat windowStat if agg != nil { stat.startedAt = agg.StartTime - stat.duration = agg.Duration + stat.duration = agg.EndTime.Sub(agg.StartTime) stat.packetsExpected = agg.Packets + agg.PacketsPadding stat.packetsLost = agg.PacketsLost stat.packetsMissing = agg.PacketsMissing @@ -263,7 +264,6 @@ func (cs *ConnectionStats) updateScoreFromReceiverReport(at time.Time) (float32, } if streamingStartedAt.After(agg.StartTime) { - agg.Duration = agg.StartTime.Add(agg.Duration).Sub(streamingStartedAt) agg.StartTime = streamingStartedAt } return cs.updateScoreWithAggregate(agg, at), streams @@ -428,6 +428,8 @@ func toAnalyticsStream(ssrc uint32, deltaStats *buffer.RTPDeltaInfo) *livekit.An packetsLost -= deltaStats.PacketsMissing } return &livekit.AnalyticsStream{ + StartTime: timestamppb.New(deltaStats.StartTime), + EndTime: timestamppb.New(deltaStats.EndTime), Ssrc: ssrc, PrimaryPackets: deltaStats.Packets, PrimaryBytes: deltaStats.Bytes, diff --git a/pkg/sfu/connectionquality/connectionstats_test.go b/pkg/sfu/connectionquality/connectionstats_test.go index 569238a40..c34a6d2ce 100644 --- a/pkg/sfu/connectionquality/connectionstats_test.go +++ b/pkg/sfu/connectionquality/connectionstats_test.go @@ -75,7 +75,7 @@ func TestConnectionQuality(t *testing.T) { 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 250, }, }, @@ -91,7 +91,7 @@ func TestConnectionQuality(t *testing.T) { 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 120, PacketsLost: 30, }, @@ -99,7 +99,7 @@ func TestConnectionQuality(t *testing.T) { 2: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 130, PacketsLost: 0, }, @@ -118,7 +118,7 @@ func TestConnectionQuality(t *testing.T) { 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 250, }, }, @@ -134,7 +134,7 @@ func TestConnectionQuality(t *testing.T) { 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 250, }, }, @@ -150,7 +150,7 @@ func TestConnectionQuality(t *testing.T) { 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 250, }, }, @@ -166,7 +166,7 @@ func TestConnectionQuality(t *testing.T) { 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 250, PacketsLost: 13, }, @@ -183,7 +183,7 @@ func TestConnectionQuality(t *testing.T) { 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 250, }, }, @@ -199,7 +199,7 @@ func TestConnectionQuality(t *testing.T) { 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 250, }, }, @@ -215,7 +215,7 @@ func TestConnectionQuality(t *testing.T) { 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 250, PacketsLost: 30, }, @@ -240,7 +240,7 @@ func TestConnectionQuality(t *testing.T) { 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 0, }, }, @@ -256,7 +256,7 @@ func TestConnectionQuality(t *testing.T) { 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 0, }, }, @@ -281,7 +281,7 @@ func TestConnectionQuality(t *testing.T) { 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 250, PacketsLost: 0, }, @@ -301,7 +301,7 @@ func TestConnectionQuality(t *testing.T) { 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 50, PacketsLost: 5, }, @@ -323,7 +323,7 @@ func TestConnectionQuality(t *testing.T) { 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 250, PacketsLost: 5, RttMax: 400, @@ -349,7 +349,7 @@ func TestConnectionQuality(t *testing.T) { 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 250, Bytes: 8_000_000 / 8 / 5, }, @@ -368,7 +368,7 @@ func TestConnectionQuality(t *testing.T) { 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 250, Bytes: 8_000_000 / 8 / 5, }, @@ -392,7 +392,7 @@ func TestConnectionQuality(t *testing.T) { 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 250, Bytes: 8_000_000 / 8 / 5, }, @@ -419,7 +419,7 @@ func TestConnectionQuality(t *testing.T) { 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 250, Bytes: 8_000_000 / 8 / 5, }, @@ -453,7 +453,7 @@ func TestConnectionQuality(t *testing.T) { 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 250, PacketsLost: 5, RttMax: 700, @@ -488,7 +488,7 @@ func TestConnectionQuality(t *testing.T) { 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 250, PacketsLost: 5, JitterMax: 200, @@ -657,7 +657,7 @@ func TestConnectionQuality(t *testing.T) { 123: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: tc.packetsExpected, PacketsLost: uint32(math.Ceil(eq.packetLossPercentage * float64(tc.packetsExpected) / 100.0)), }, @@ -761,7 +761,7 @@ func TestConnectionQuality(t *testing.T) { 123: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 100, Bytes: tc.bytes, }, @@ -855,7 +855,7 @@ func TestConnectionQuality(t *testing.T) { 123: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, - Duration: duration, + EndTime: now.Add(duration), Packets: 200, }, }, diff --git a/pkg/sfu/connectionquality/scorer.go b/pkg/sfu/connectionquality/scorer.go index 4ed30faa3..b38355c06 100644 --- a/pkg/sfu/connectionquality/scorer.go +++ b/pkg/sfu/connectionquality/scorer.go @@ -505,7 +505,7 @@ func (q *qualityScorer) isPaused() bool { } func (q *qualityScorer) getPacketLossWeight(stat *windowStat) float64 { - if stat == nil || stat.duration == 0 { + if stat == nil || stat.duration <= 0 { return q.params.PacketLossWeight } diff --git a/pkg/telemetry/statsworker.go b/pkg/telemetry/statsworker.go index d48a6d6d5..18bde9e55 100644 --- a/pkg/telemetry/statsworker.go +++ b/pkg/telemetry/statsworker.go @@ -158,6 +158,8 @@ func coalesce(stats []*livekit.AnalyticsStat) *livekit.AnalyticsStat { } // find aggregates across streams + startTime := time.Time{} + endTime := time.Time{} scoreSum := float32(0.0) // used for average minScore := float32(0.0) // min score in batched stats var scores []float32 // used for median @@ -183,6 +185,16 @@ func coalesce(stats []*livekit.AnalyticsStat) *livekit.AnalyticsStat { } for _, analyticsStream := range stat.Streams { + start := analyticsStream.StartTime.AsTime() + if startTime.IsZero() || startTime.After(start) { + startTime = start + } + + end := analyticsStream.EndTime.AsTime() + if endTime.IsZero() || endTime.Before(end) { + endTime = end + } + if analyticsStream.Rtt > maxRtt { maxRtt = analyticsStream.Rtt } @@ -216,6 +228,8 @@ func coalesce(stats []*livekit.AnalyticsStat) *livekit.AnalyticsStat { } } } + coalescedStream.StartTime = timestamppb.New(startTime) + coalescedStream.EndTime = timestamppb.New(endTime) coalescedStream.Rtt = maxRtt coalescedStream.Jitter = maxJitter From dc67f505a53bab27b28c4b617268e855624064ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Thu, 4 Apr 2024 00:25:42 +0200 Subject: [PATCH 22/78] agent service: new protocol & namespaces (#2545) * initial worker impl * fix test * fix build * TestAgentNamespaces * log err * nit cmt * TestAgentMultiNode * Update pkg/agent/worker.go Co-authored-by: David Zhao * retry on worker selection & fix review comments * Update roommanager.go * license * use testutils.WIthTimeout * abstract namespace/enabled logic into agent.Client, incrementally dispatch * typos and dates * lock * timeout is now optional * pass in topics instead of fixed * handler handles connections * onIdle, numConnections * fix WithGrants * update protocol * check agent client * broadcast after unlock * fix data race * remove ReadChan, fix dispatcher --------- Co-authored-by: David Zhao Co-authored-by: David Colburn --- go.mod | 2 +- go.sum | 4 +- pkg/agent/client.go | 185 ++++++++ pkg/agent/job.go | 86 ++++ pkg/agent/worker.go | 385 ++++++++++++++++ pkg/rtc/agentclient.go | 92 ---- pkg/rtc/room.go | 42 +- pkg/rtc/types/interfaces.go | 1 + pkg/service/agentservice.go | 577 ++++++++++-------------- pkg/service/auth.go | 30 +- pkg/service/roommanager.go | 16 +- pkg/service/roomservice.go | 28 +- pkg/service/roomservice_test.go | 6 +- pkg/service/rtcservice.go | 23 +- pkg/service/server.go | 1 + pkg/service/wire.go | 4 +- pkg/service/wire_gen.go | 14 +- pkg/testutils/timeout.go | 10 +- pkg/utils/incrementaldispatcher.go | 79 ++++ pkg/utils/incrementaldispatcher_test.go | 96 ++++ test/agent.go | 51 ++- test/agent_test.go | 154 ++++++- 22 files changed, 1326 insertions(+), 560 deletions(-) create mode 100644 pkg/agent/client.go create mode 100644 pkg/agent/job.go create mode 100644 pkg/agent/worker.go delete mode 100644 pkg/rtc/agentclient.go create mode 100644 pkg/utils/incrementaldispatcher.go create mode 100644 pkg/utils/incrementaldispatcher_test.go diff --git a/go.mod b/go.mod index 9ca9921c1..6c02284ac 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20240302142739-1c3dd691a1b8 - github.com/livekit/protocol v1.12.1-0.20240403063258-fc68e8d82b26 + github.com/livekit/protocol v1.12.1-0.20240403204952-bc6c7ffd71f0 github.com/livekit/psrpc v0.5.3-0.20240327035954-cec3a0e614be github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 diff --git a/go.sum b/go.sum index fb236c8f8..ed8db6b06 100644 --- a/go.sum +++ b/go.sum @@ -132,8 +132,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20240302142739-1c3dd691a1b8 h1:xawydPEACNO5Ncs2LgioTjWghXQ0eUN1q1RnVUUyVnI= github.com/livekit/mediatransportutil v0.0.0-20240302142739-1c3dd691a1b8/go.mod h1:jwKUCmObuiEDH0iiuJHaGMXwRs3RjrB4G6qqgkr/5oE= -github.com/livekit/protocol v1.12.1-0.20240403063258-fc68e8d82b26 h1:8Oog+IH4NVbq7PEXYU7bIW8Kw4rqd0HgtWaOh++9ZcE= -github.com/livekit/protocol v1.12.1-0.20240403063258-fc68e8d82b26/go.mod h1:mcv7L2DWB6iRckI++egmwFU7YEy3W+aLHnusNhioqi0= +github.com/livekit/protocol v1.12.1-0.20240403204952-bc6c7ffd71f0 h1:BfQN4k6YG+XTdfbXnA2XZLAXinRNlJrGdTLAjyOW2Rg= +github.com/livekit/protocol v1.12.1-0.20240403204952-bc6c7ffd71f0/go.mod h1:mcv7L2DWB6iRckI++egmwFU7YEy3W+aLHnusNhioqi0= github.com/livekit/psrpc v0.5.3-0.20240327035954-cec3a0e614be h1:W1nCFZ19rYAORMBNX82NeVPHjADN0UyORr6refbUXpU= github.com/livekit/psrpc v0.5.3-0.20240327035954-cec3a0e614be/go.mod h1:CQUBSPfYYAaevg1TNCc6/aYsa8DJH4jSRFdCeSZk5u0= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= diff --git a/pkg/agent/client.go b/pkg/agent/client.go new file mode 100644 index 000000000..94eeed949 --- /dev/null +++ b/pkg/agent/client.go @@ -0,0 +1,185 @@ +// Copyright 2024 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package agent + +import ( + "context" + "sync" + "time" + + "github.com/gammazero/workerpool" + "google.golang.org/protobuf/types/known/emptypb" + + serverutils "github.com/livekit/livekit-server/pkg/utils" + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/rpc" + "github.com/livekit/protocol/utils" + "github.com/livekit/psrpc" +) + +const ( + EnabledCacheTTL = 1 * time.Minute + RoomAgentTopic = "room" + PublisherAgentTopic = "publisher" + DefaultHandlerNamespace = "" + + CheckEnabledTimeout = 5 * time.Second +) + +type Client interface { + // LaunchJob starts a room or participant job on an agent. + // it will launch a job once for each worker in each namespace + LaunchJob(ctx context.Context, desc *JobDescription) + Stop() error +} + +type JobDescription struct { + JobType livekit.JobType + Room *livekit.Room + // only set for participant jobs + Participant *livekit.ParticipantInfo +} + +type agentClient struct { + client rpc.AgentInternalClient + + mu sync.RWMutex + + // cache response to avoid constantly checking with controllers + // cache is invalidated with AgentRegistered updates + roomNamespaces *serverutils.IncrementalDispatcher[string] + publisherNamespaces *serverutils.IncrementalDispatcher[string] + enabledExpiresAt time.Time + + workers *workerpool.WorkerPool + + invalidateSub psrpc.Subscription[*emptypb.Empty] + subDone chan struct{} +} + +func NewAgentClient(bus psrpc.MessageBus) (Client, error) { + client, err := rpc.NewAgentInternalClient(bus) + if err != nil { + return nil, err + } + + c := &agentClient{ + client: client, + workers: workerpool.New(50), + subDone: make(chan struct{}), + } + + sub, err := c.client.SubscribeWorkerRegistered(context.Background(), DefaultHandlerNamespace) + if err != nil { + return nil, err + } + + c.invalidateSub = sub + + go func() { + // invalidate cache + for range sub.Channel() { + c.mu.Lock() + c.roomNamespaces = nil + c.publisherNamespaces = nil + c.mu.Unlock() + } + + c.subDone <- struct{}{} + }() + + return c, nil +} + +func (c *agentClient) LaunchJob(ctx context.Context, desc *JobDescription) { + roomNamespaces, publisherNamespaces, needsRefresh := c.getOrCreateDispatchers() + + if needsRefresh { + go c.checkEnabled(ctx, roomNamespaces, publisherNamespaces) + } + + target := roomNamespaces + jobTypeTopic := RoomAgentTopic + if desc.JobType == livekit.JobType_JT_PUBLISHER { + target = publisherNamespaces + jobTypeTopic = PublisherAgentTopic + } + + target.ForEach(func(ns string) { + c.workers.Submit(func() { + _, err := c.client.JobRequest(ctx, ns, jobTypeTopic, &livekit.Job{ + Id: utils.NewGuid(utils.AgentJobPrefix), + Type: desc.JobType, + Room: desc.Room, + Participant: desc.Participant, + Namespace: ns, + }) + if err != nil { + logger.Errorw("failed to send job request", err, "namespace", ns, "jobType", jobTypeTopic) + } + }) + }) +} + +func (c *agentClient) getOrCreateDispatchers() (*serverutils.IncrementalDispatcher[string], *serverutils.IncrementalDispatcher[string], bool) { + c.mu.Lock() + defer c.mu.Unlock() + + if time.Since(c.enabledExpiresAt) > EnabledCacheTTL || c.roomNamespaces == nil || c.publisherNamespaces == nil { + c.roomNamespaces = serverutils.NewIncrementalDispatcher[string]() + c.publisherNamespaces = serverutils.NewIncrementalDispatcher[string]() + return c.roomNamespaces, c.publisherNamespaces, true + } + return c.roomNamespaces, c.publisherNamespaces, false +} + +func (c *agentClient) checkEnabled(ctx context.Context, roomNamespaces, publisherNamespaces *serverutils.IncrementalDispatcher[string]) { + defer roomNamespaces.Done() + defer publisherNamespaces.Done() + resChan, err := c.client.CheckEnabled(ctx, &rpc.CheckEnabledRequest{}, psrpc.WithRequestTimeout(CheckEnabledTimeout)) + if err != nil { + logger.Errorw("failed to check enabled", err) + return + } + + roomNSMap := make(map[string]bool) + publisherNSMap := make(map[string]bool) + + for r := range resChan { + if r.Result.GetRoomEnabled() { + for _, ns := range r.Result.GetNamespaces() { + if _, ok := roomNSMap[ns]; !ok { + roomNamespaces.Add(ns) + roomNSMap[ns] = true + } + } + } + if r.Result.GetPublisherEnabled() { + for _, ns := range r.Result.GetNamespaces() { + if _, ok := publisherNSMap[ns]; !ok { + publisherNamespaces.Add(ns) + publisherNSMap[ns] = true + } + } + } + } +} + +func (c *agentClient) Stop() error { + _ = c.invalidateSub.Close() + <-c.subDone + return nil +} diff --git a/pkg/agent/job.go b/pkg/agent/job.go new file mode 100644 index 000000000..6edfe4a27 --- /dev/null +++ b/pkg/agent/job.go @@ -0,0 +1,86 @@ +// Copyright 2024 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package agent + +import ( + "sync" + + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" +) + +// Represents a job that is being executed by a worker +type Job struct { + id string + jobType livekit.JobType + status livekit.JobStatus + namespace string + + mu sync.Mutex + load float32 + + Logger logger.Logger +} + +func NewJob(id, namespace string, jobType livekit.JobType) *Job { + return &Job{ + id: id, + status: livekit.JobStatus_JS_UNKNOWN, + jobType: jobType, + namespace: namespace, + } +} + +func (j *Job) ID() string { + return j.id +} + +func (j *Job) Namespace() string { + return j.namespace +} + +func (j *Job) Type() livekit.JobType { + return j.jobType +} + +func (j *Job) WorkerLoad() float32 { + // Current load that this job is taking on its worker + j.mu.Lock() + defer j.mu.Unlock() + return j.load +} + +func (j *Job) UpdateStatus(req *livekit.UpdateJobStatus) { + j.mu.Lock() + + if req.Status != nil { + j.status = *req.Status // End of the job, SUCCESS or FAILURE + + if j.status == livekit.JobStatus_JS_FAILED { + j.Logger.Errorw("job failed", nil, "id", j.id, "type", j.jobType, "error", req.Error) + } + } + + j.load = req.Load + j.mu.Unlock() + + if req.Metadata != nil { + j.UpdateMetadata(req.GetMetadata()) + } +} + +func (j *Job) UpdateMetadata(metadata string) { + j.Logger.Debugw("job metadata", nil, "id", j.id, "metadata", metadata) +} diff --git a/pkg/agent/worker.go b/pkg/agent/worker.go new file mode 100644 index 000000000..b6f69fcfe --- /dev/null +++ b/pkg/agent/worker.go @@ -0,0 +1,385 @@ +// Copyright 2024 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package agent + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "time" + + "github.com/gorilla/websocket" + + pagent "github.com/livekit/protocol/agent" + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/utils" + putil "github.com/livekit/protocol/utils" +) + +type WorkerProtocolVersion int + +const CurrentProtocol = 1 + +const ( + registerTimeout = 10 * time.Second + assignJobTimeout = 10 * time.Second + pingFrequency = 10 * time.Second +) + +var ( + ErrWorkerClosed = errors.New("worker closed") + ErrWorkerNotAvailable = errors.New("worker not available") + ErrAvailabilityTimeout = errors.New("agent worker availability timeout") +) + +type sigConn interface { + WriteServerMessage(msg *livekit.ServerMessage) (int, error) +} + +type Worker struct { + id string + jobType livekit.JobType + version string + name string + namespace string + load float32 + permissions *livekit.ParticipantPermission + apiKey string + apiSecret string + serverInfo *livekit.ServerInfo + mu sync.Mutex + + protocolVersion WorkerProtocolVersion + registered atomic.Bool + status livekit.WorkerStatus + runningJobs map[string]*Job + + onWorkerRegistered func(w *Worker) + + conn *websocket.Conn + sigConn sigConn + closed chan struct{} + + availability map[string]chan *livekit.AvailabilityResponse + + ctx context.Context + cancel context.CancelFunc + + Logger logger.Logger +} + +func NewWorker( + protocolVersion WorkerProtocolVersion, + apiKey string, + apiSecret string, + serverInfo *livekit.ServerInfo, + conn *websocket.Conn, + sigConn sigConn, + logger logger.Logger, +) *Worker { + ctx, cancel := context.WithCancel(context.Background()) + + w := &Worker{ + id: putil.NewGuid(utils.AgentWorkerPrefix), + protocolVersion: protocolVersion, + apiKey: apiKey, + apiSecret: apiSecret, + serverInfo: serverInfo, + closed: make(chan struct{}), + runningJobs: make(map[string]*Job), + availability: make(map[string]chan *livekit.AvailabilityResponse), + conn: conn, + sigConn: sigConn, + ctx: ctx, + cancel: cancel, + Logger: logger, + } + + go func() { + <-time.After(registerTimeout) + if !w.registered.Load() && !w.IsClosed() { + w.Logger.Warnw("worker did not register in time", nil, "id", w.id) + w.Close() + } + }() + + return w +} + +func (w *Worker) sendRequest(req *livekit.ServerMessage) { + if _, err := w.sigConn.WriteServerMessage(req); err != nil { + w.Logger.Errorw("error writing to websocket", err) + } +} + +func (w *Worker) ID() string { + return w.id +} + +func (w *Worker) JobType() livekit.JobType { + w.mu.Lock() + defer w.mu.Unlock() + return w.jobType +} + +func (w *Worker) Namespace() string { + w.mu.Lock() + defer w.mu.Unlock() + return w.namespace +} + +func (w *Worker) Status() livekit.WorkerStatus { + w.mu.Lock() + defer w.mu.Unlock() + return w.status +} + +func (w *Worker) Load() float32 { + w.mu.Lock() + defer w.mu.Unlock() + return w.load +} + +func (w *Worker) OnWorkerRegistered(f func(w *Worker)) { + w.mu.Lock() + defer w.mu.Unlock() + w.onWorkerRegistered = f +} + +func (w *Worker) Registered() bool { + return w.registered.Load() +} + +func (w *Worker) RunningJobs() map[string]*Job { + jobs := make(map[string]*Job, len(w.runningJobs)) + w.mu.Lock() + defer w.mu.Unlock() + for k, v := range w.runningJobs { + jobs[k] = v + } + return jobs +} + +func (w *Worker) AssignJob(ctx context.Context, job *livekit.Job) error { + availCh := make(chan *livekit.AvailabilityResponse, 1) + + w.mu.Lock() + w.availability[job.Id] = availCh + w.mu.Unlock() + + w.sendRequest(&livekit.ServerMessage{Message: &livekit.ServerMessage_Availability{ + Availability: &livekit.AvailabilityRequest{Job: job}, + }}) + + // See handleAvailability for the response + select { + case res := <-availCh: + if !res.Available { + return ErrWorkerNotAvailable + } + + token, err := pagent.BuildAgentToken(w.apiKey, w.apiSecret, job.Room.Name, res.ParticipantIdentity, res.ParticipantName, res.ParticipantMetadata, w.permissions) + if err != nil { + w.Logger.Errorw("failed to build agent token", err) + return err + } + + // In OSS, Url is nil, and the used API Key is the same as the one used to connect the worker + w.sendRequest(&livekit.ServerMessage{Message: &livekit.ServerMessage_Assignment{ + Assignment: &livekit.JobAssignment{Job: job, Url: nil, Token: token}, + }}) + + // TODO(theomonnom): Check if an agent was successfully connected to the room before returning + return nil + case <-time.After(assignJobTimeout): + return ErrAvailabilityTimeout + case <-w.ctx.Done(): + return ErrWorkerClosed + case <-ctx.Done(): + return ctx.Err() + } +} + +func (w *Worker) UpdateStatus(status *livekit.UpdateWorkerStatus) { + w.mu.Lock() + if status.Status != nil { + w.status = status.GetStatus() + } + w.load = status.GetLoad() + w.mu.Unlock() + + if status.Metadata != nil { + w.UpdateMetadata(status.GetMetadata()) + } +} + +func (w *Worker) UpdateMetadata(metadata string) { + w.Logger.Debugw("worker metadata updated", nil, "metadata", metadata) +} + +func (w *Worker) IsClosed() bool { + select { + case <-w.closed: + return true + default: + return false + } +} + +func (w *Worker) Close() { + w.mu.Lock() + if w.IsClosed() { + w.mu.Unlock() + return + } + + w.Logger.Infow("closing worker") + + close(w.closed) + w.cancel() + _ = w.conn.Close() + w.mu.Unlock() +} + +func (w *Worker) HandleMessage(req *livekit.WorkerMessage) { + switch m := req.Message.(type) { + case *livekit.WorkerMessage_Register: + go w.handleRegister(m.Register) + case *livekit.WorkerMessage_Availability: + go w.handleAvailability(m.Availability) + case *livekit.WorkerMessage_UpdateJob: + go w.handleJobUpdate(m.UpdateJob) + case *livekit.WorkerMessage_SimulateJob: + go w.handleSimulateJob(m.SimulateJob) + case *livekit.WorkerMessage_Ping: + go w.handleWorkerPing(m.Ping) + case *livekit.WorkerMessage_UpdateWorker: + go w.handleWorkerStatus(m.UpdateWorker) + case *livekit.WorkerMessage_MigrateJob: + go w.handleMigrateJob(m.MigrateJob) + } +} + +func (w *Worker) handleRegister(req *livekit.RegisterWorkerRequest) { + if w.registered.Load() { + w.Logger.Warnw("worker already registered", nil, "id", w.id) + return + } + + w.mu.Lock() + onWorkerRegistered := w.onWorkerRegistered + w.jobType = req.Type + w.version = req.Version + w.name = req.Name + w.namespace = req.GetNamespace() + + if req.AllowedPermissions != nil { + w.permissions = req.AllowedPermissions + } else { + // Use default agent permissions + w.permissions = &livekit.ParticipantPermission{ + CanSubscribe: true, + CanPublish: true, + CanPublishData: true, + CanUpdateMetadata: true, + } + } + + w.status = livekit.WorkerStatus_WS_AVAILABLE + w.registered.Store(true) + w.mu.Unlock() + + w.sendRequest(&livekit.ServerMessage{ + Message: &livekit.ServerMessage_Register{ + Register: &livekit.RegisterWorkerResponse{ + WorkerId: w.ID(), + ServerInfo: w.serverInfo, + }, + }, + }) + + if onWorkerRegistered != nil { + onWorkerRegistered(w) + } +} + +func (w *Worker) handleAvailability(res *livekit.AvailabilityResponse) { + w.mu.Lock() + defer w.mu.Unlock() + + availCh, ok := w.availability[res.JobId] + if !ok { + w.Logger.Warnw("received availability response for unknown job", nil, "jobId", res.JobId) + return + } + + availCh <- res + delete(w.availability, res.JobId) +} + +func (w *Worker) handleJobUpdate(update *livekit.UpdateJobStatus) { + w.mu.Lock() + job, ok := w.runningJobs[update.JobId] + w.mu.Unlock() + + if !ok { + w.Logger.Warnw("received job update for unknown job", nil, "jobId", update.JobId) + return + } + + job.UpdateStatus(update) +} + +func (w *Worker) handleSimulateJob(simulate *livekit.SimulateJobRequest) { + jobType := livekit.JobType_JT_ROOM + if simulate.Participant != nil { + jobType = livekit.JobType_JT_PUBLISHER + } + + job := &livekit.Job{ + Id: utils.NewGuid(utils.AgentJobPrefix), + Type: jobType, + Room: simulate.Room, + Participant: simulate.Participant, + Namespace: w.Namespace(), + } + + ctx := context.Background() + err := w.AssignJob(ctx, job) + if err != nil { + w.Logger.Errorw("failed to simulate job, assignment failed", err, "jobId", job.Id) + } + +} + +func (w *Worker) handleWorkerPing(ping *livekit.WorkerPing) { + w.sendRequest(&livekit.ServerMessage{Message: &livekit.ServerMessage_Pong{ + Pong: &livekit.WorkerPong{ + LastTimestamp: ping.Timestamp, + Timestamp: time.Now().UnixMilli(), + }, + }}) +} + +func (w *Worker) handleWorkerStatus(update *livekit.UpdateWorkerStatus) { + w.UpdateStatus(update) +} + +func (w *Worker) handleMigrateJob(migrate *livekit.MigrateJobRequest) { + // TODO(theomonnom): On OSS this is not implemented + // We could maybe just move a specific job to another worker +} diff --git a/pkg/rtc/agentclient.go b/pkg/rtc/agentclient.go deleted file mode 100644 index 65be3ba7c..000000000 --- a/pkg/rtc/agentclient.go +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright 2023 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package rtc - -import ( - "context" - "time" - - "github.com/livekit/protocol/livekit" - "github.com/livekit/protocol/logger" - "github.com/livekit/protocol/rpc" - "github.com/livekit/psrpc" -) - -const ( - RoomAgentTopic = "room" - PublisherAgentTopic = "publisher" -) - -type AgentClient interface { - CheckEnabled(ctx context.Context, req *rpc.CheckEnabledRequest) *rpc.CheckEnabledResponse - JobRequest(ctx context.Context, job *livekit.Job) -} - -type agentClient struct { - client rpc.AgentInternalClient -} - -func NewAgentClient(bus psrpc.MessageBus) (AgentClient, error) { - client, err := rpc.NewAgentInternalClient(bus) - if err != nil { - return nil, err - } - return &agentClient{client: client}, nil -} - -func (c *agentClient) CheckEnabled(ctx context.Context, req *rpc.CheckEnabledRequest) *rpc.CheckEnabledResponse { - res := &rpc.CheckEnabledResponse{} - resChan, err := c.client.CheckEnabled(ctx, req, psrpc.WithRequestTimeout(time.Second)) - if err != nil { - return res - } - - for r := range resChan { - if r.Err != nil { - continue - } - if r.Result.RoomEnabled { - res.RoomEnabled = true - if res.PublisherEnabled { - return res - } - } - if r.Result.PublisherEnabled { - res.PublisherEnabled = true - if res.RoomEnabled { - return res - } - } - } - - return res -} - -func (c *agentClient) JobRequest(ctx context.Context, job *livekit.Job) { - var topic string - var logError bool - switch job.Type { - case livekit.JobType_JT_ROOM: - topic = RoomAgentTopic - case livekit.JobType_JT_PUBLISHER: - topic = PublisherAgentTopic - logError = true - } - - _, err := c.client.JobRequest(ctx, topic, job) - if err != nil && logError { - logger.Warnw("agent job request failed", err) - } -} diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index e50e84b83..900c415f8 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -34,9 +34,9 @@ import ( "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" - "github.com/livekit/protocol/rpc" "github.com/livekit/protocol/utils" + "github.com/livekit/livekit-server/pkg/agent" "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/routing" "github.com/livekit/livekit-server/pkg/rtc/types" @@ -103,8 +103,7 @@ type Room struct { trackManager *RoomTrackManager // agents - agentClient AgentClient - publisherAgentsEnabled bool + agentClient agent.Client // map of identity -> Participant participants map[livekit.ParticipantIdentity]types.LocalParticipant @@ -142,7 +141,7 @@ func NewRoom( audioConfig *config.AudioConfig, serverInfo *livekit.ServerInfo, telemetry telemetry.TelemetryService, - agentClient AgentClient, + agentClient agent.Client, egressLauncher EgressLauncher, ) *Room { r := &Room{ @@ -183,21 +182,6 @@ func NewRoom( } r.protoProxy = utils.NewProtoProxy[*livekit.Room](roomUpdateInterval, r.updateProto) - if agentClient != nil { - go func() { - res := r.agentClient.CheckEnabled(context.Background(), &rpc.CheckEnabledRequest{}) - if res.PublisherEnabled { - r.lock.Lock() - r.publisherAgentsEnabled = true - // if there are already published tracks, start the agents - for identity := range r.hasPublished { - r.launchPublisherAgent(r.participants[identity]) - } - r.lock.Unlock() - } - }() - } - go r.audioUpdateWorker() go r.connectionQualityWorker() go r.changeUpdateWorker() @@ -1013,13 +997,10 @@ func (r *Room) onTrackPublished(participant types.LocalParticipant, track types. r.lock.Lock() hasPublished := r.hasPublished[participant.Identity()] r.hasPublished[participant.Identity()] = true - publisherAgentsEnabled := r.publisherAgentsEnabled r.lock.Unlock() if !hasPublished { - if publisherAgentsEnabled { - r.launchPublisherAgent(participant) - } + r.launchPublisherAgent(participant) if r.internal != nil && r.internal.ParticipantEgress != nil { go func() { if err := StartParticipantEgress( @@ -1429,18 +1410,15 @@ func (r *Room) simulationCleanupWorker() { } func (r *Room) launchPublisherAgent(p types.Participant) { - if p == nil || p.IsRecorder() || p.IsAgent() { + if p == nil || p.IsRecorder() || p.IsAgent() || r.agentClient == nil { return } - go func() { - r.agentClient.JobRequest(context.Background(), &livekit.Job{ - Id: utils.NewGuid("JP_"), - Type: livekit.JobType_JT_PUBLISHER, - Room: r.ToProto(), - Participant: p.ToProto(), - }) - }() + go r.agentClient.LaunchJob(context.Background(), &agent.JobDescription{ + JobType: livekit.JobType_JT_PUBLISHER, + Room: r.ToProto(), + Participant: p.ToProto(), + }) } func (r *Room) DebugInfo() map[string]interface{} { diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index b0943d4f4..967e63960 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -586,3 +586,4 @@ type OperationMonitor interface { Check() error IsIdle() bool } + diff --git a/pkg/service/agentservice.go b/pkg/service/agentservice.go index 20636bc60..e8f391cb4 100644 --- a/pkg/service/agentservice.go +++ b/pkg/service/agentservice.go @@ -1,4 +1,4 @@ -// Copyright 2023 LiveKit, Inc. +// Copyright 2024 LiveKit, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ import ( "io" "math/rand" "net/http" + "strconv" "strings" "sync" "time" @@ -27,16 +28,19 @@ import ( "github.com/gorilla/websocket" "google.golang.org/protobuf/types/known/emptypb" + "github.com/livekit/livekit-server/pkg/agent" + "github.com/livekit/livekit-server/pkg/config" + "github.com/livekit/livekit-server/pkg/routing" "github.com/livekit/livekit-server/pkg/rtc" + "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/utils" + "github.com/livekit/livekit-server/version" + "github.com/livekit/protocol/auth" "github.com/livekit/protocol/livekit" - "github.com/livekit/protocol/logger" "github.com/livekit/protocol/rpc" "github.com/livekit/psrpc" ) -const AgentServiceVersion = "0.1.0" - type AgentService struct { upgrader websocket.Upgrader @@ -44,38 +48,30 @@ type AgentService struct { } type AgentHandler struct { - agentServer rpc.AgentInternalServer - roomTopic string - publisherTopic string + agentServer rpc.AgentInternalServer + mu sync.Mutex - mu sync.Mutex - availability map[string]chan *availability - unregistered map[*websocket.Conn]*worker - roomRegistered bool - roomWorkers map[string]*worker - publisherRegistered bool - publisherWorkers map[string]*worker - onWorkerRegistered func(handler *AgentHandler) + serverInfo *livekit.ServerInfo + workers map[string]*agent.Worker + keyProvider auth.KeyProvider + + namespaces map[string]*namespaceInfo + publisherEnabled bool + roomEnabled bool + roomTopic string + publisherTopic string } -type worker struct { - mu sync.Mutex - conn *websocket.Conn - sigConn *WSSignalConnection - - id string - jobType livekit.JobType - status livekit.WorkerStatus - activeJobs int - logger logger.Logger +type namespaceInfo struct { + numPublishers int32 + numRooms int32 } -type availability struct { - workerID string - available bool -} - -func NewAgentService(bus psrpc.MessageBus) (*AgentService, error) { +func NewAgentService(conf *config.Config, + currentNode routing.LocalNode, + bus psrpc.MessageBus, + keyProvider auth.KeyProvider, +) (*AgentService, error) { s := &AgentService{ upgrader: websocket.Upgrader{}, } @@ -86,12 +82,26 @@ func NewAgentService(bus psrpc.MessageBus) (*AgentService, error) { return true } + serverInfo := &livekit.ServerInfo{ + Edition: livekit.ServerInfo_Standard, + Version: version.Version, + Protocol: types.CurrentProtocol, + AgentProtocol: agent.CurrentProtocol, + Region: conf.Region, + NodeId: currentNode.Id, + } + agentServer, err := rpc.NewAgentInternalServer(s, bus) if err != nil { return nil, err } - s.AgentHandler = NewAgentHandler(agentServer, rtc.RoomAgentTopic, rtc.PublisherAgentTopic) - + s.AgentHandler = NewAgentHandler( + agentServer, + keyProvider, + serverInfo, + agent.RoomAgentTopic, + agent.PublisherAgentTopic, + ) return s, nil } @@ -116,64 +126,64 @@ func (s *AgentService) ServeHTTP(writer http.ResponseWriter, r *http.Request) { return } - s.HandleConnection(r.Context(), conn) + s.HandleConnection(r, conn, nil) } -func NewAgentHandler(agentServer rpc.AgentInternalServer, roomTopic, publisherTopic string) *AgentHandler { +func NewAgentHandler( + agentServer rpc.AgentInternalServer, + keyProvider auth.KeyProvider, + serverInfo *livekit.ServerInfo, + roomTopic string, + publisherTopic string, +) *AgentHandler { return &AgentHandler{ - agentServer: agentServer, - roomTopic: roomTopic, - publisherTopic: publisherTopic, - availability: make(map[string]chan *availability), - unregistered: make(map[*websocket.Conn]*worker), - roomWorkers: make(map[string]*worker), - publisherWorkers: make(map[string]*worker), + agentServer: agentServer, + workers: make(map[string]*agent.Worker), + namespaces: make(map[string]*namespaceInfo), + serverInfo: serverInfo, + keyProvider: keyProvider, + roomTopic: roomTopic, + publisherTopic: publisherTopic, } } -// OnWorkerRegistered registers a callback to be called when the first worker of each type is registered -func (s *AgentHandler) OnWorkerRegistered(handler func(handler *AgentHandler)) { - s.mu.Lock() - defer s.mu.Unlock() - s.onWorkerRegistered = handler -} +func (h *AgentHandler) HandleConnection(r *http.Request, conn *websocket.Conn, onIdle func()) { + var protocol agent.WorkerProtocolVersion + if pv, err := strconv.Atoi(r.FormValue("protocol")); err == nil { + protocol = agent.WorkerProtocolVersion(pv) + } -func (s *AgentHandler) HandleConnection(ctx context.Context, conn *websocket.Conn) { sigConn := NewWSSignalConnection(conn) - w := &worker{ - conn: conn, - sigConn: sigConn, - logger: utils.GetLogger(ctx), - } - s.mu.Lock() - s.unregistered[conn] = w - s.mu.Unlock() + logger := utils.GetLogger(r.Context()) + + apiKey := GetAPIKey(r.Context()) + apiSecret := h.keyProvider.GetSecret(apiKey) + + worker := agent.NewWorker(protocol, apiKey, apiSecret, h.serverInfo, conn, sigConn, logger) + worker.OnWorkerRegistered(h.registerWorkerTopic) + + h.mu.Lock() + h.workers[worker.ID()] = worker + h.mu.Unlock() defer func() { - s.mu.Lock() - if w.id == "" { - delete(s.unregistered, conn) - } else { - switch w.jobType { - case livekit.JobType_JT_ROOM: - delete(s.roomWorkers, w.id) - if s.roomRegistered && !s.roomAvailableLocked() { - s.roomRegistered = false - s.agentServer.DeregisterJobRequestTopic(s.roomTopic) - } - case livekit.JobType_JT_PUBLISHER: - delete(s.publisherWorkers, w.id) - if s.publisherRegistered && !s.publisherAvailableLocked() { - s.publisherRegistered = false - s.agentServer.DeregisterJobRequestTopic(s.publisherTopic) - } - } + worker.Close() + + h.mu.Lock() + delete(h.workers, worker.ID()) + numWorkers := len(h.workers) + h.mu.Unlock() + + if worker.Registered() { + h.deregisterWorkerTopic(worker) + } + + if numWorkers == 0 && onIdle != nil { + onIdle() } - s.mu.Unlock() }() - // handle incoming requests from websocket for { req, _, err := sigConn.ReadWorkerMessage() if err != nil { @@ -188,321 +198,220 @@ func (s *AgentHandler) HandleConnection(ctx context.Context, conn *websocket.Con websocket.CloseNormalClosure, websocket.CloseNoStatusReceived, ) { - w.logger.Infow("Agent worker closed WS connection", "wsError", err) + worker.Logger.Infow("worker closed WS connection", "wsError", err) } else { - w.logger.Errorw("error reading from websocket", err) + worker.Logger.Errorw("error reading from websocket", err) } return } - switch m := req.Message.(type) { - case *livekit.WorkerMessage_Register: - go s.handleRegister(w, m.Register) - case *livekit.WorkerMessage_Availability: - go s.handleAvailability(w, m.Availability) - case *livekit.WorkerMessage_JobUpdate: - go s.handleJobUpdate(w, m.JobUpdate) - case *livekit.WorkerMessage_Status: - go s.handleStatus(w, m.Status) - } + worker.HandleMessage(req) } } -func (s *AgentHandler) handleRegister(worker *worker, msg *livekit.RegisterWorkerRequest) { - if err := s.doHandleRegister(worker, msg); err != nil { - worker.logger.Errorw("failed to register worker", err, "workerID", msg.WorkerId, "jobType", msg.Type) - worker.conn.Close() - } -} - -func (s *AgentHandler) doHandleRegister(worker *worker, msg *livekit.RegisterWorkerRequest) error { - if msg.WorkerId == "" { - return errors.New("invalid worker id") - } - - s.mu.Lock() - if worker.id != "" { - s.mu.Unlock() - return errors.New("worker already registered") - } - - onRegistered := s.onWorkerRegistered - firstWorker := false - - switch msg.Type { - case livekit.JobType_JT_ROOM: - worker.id = msg.WorkerId - worker.jobType = msg.Type - delete(s.unregistered, worker.conn) - s.roomWorkers[worker.id] = worker - - if !s.roomRegistered { - err := s.agentServer.RegisterJobRequestTopic(s.roomTopic) - if err != nil { - worker.logger.Errorw("failed to register room agents", err) - } else { - s.roomRegistered = true - firstWorker = true - } - } - - case livekit.JobType_JT_PUBLISHER: - worker.id = msg.WorkerId - worker.jobType = msg.Type - delete(s.unregistered, worker.conn) - s.publisherWorkers[worker.id] = worker - - if !s.publisherRegistered { - err := s.agentServer.RegisterJobRequestTopic(s.publisherTopic) - if err != nil { - worker.logger.Errorw("failed to register publisher agents", err) - } else { - s.publisherRegistered = true - firstWorker = true - } - } - default: - s.mu.Unlock() - return errors.New("invalid job type") - } - s.mu.Unlock() - - _, err := worker.sigConn.WriteServerMessage(&livekit.ServerMessage{ - Message: &livekit.ServerMessage_Register{ - Register: &livekit.RegisterWorkerResponse{ - WorkerId: worker.id, - ServerVersion: AgentServiceVersion, - }, - }, - }) - if err != nil { - worker.logger.Errorw("failed to write server message", err) - } - - if firstWorker && onRegistered != nil { - onRegistered(s) - } - return nil -} - -func (s *AgentHandler) handleAvailability(w *worker, msg *livekit.AvailabilityResponse) { - s.mu.Lock() - availabilityChan, ok := s.availability[msg.JobId] - s.mu.Unlock() +func (h *AgentHandler) registerWorkerTopic(w *agent.Worker) { + h.mu.Lock() + info, ok := h.namespaces[w.Namespace()] + numPublishers := int32(0) + numRooms := int32(0) if ok { - availabilityChan <- &availability{ - workerID: w.id, - available: msg.Available, + numPublishers = info.numPublishers + numRooms = info.numRooms + } + + var err error + if w.JobType() == livekit.JobType_JT_PUBLISHER { + numPublishers++ + if numPublishers == 1 { + err = h.agentServer.RegisterJobRequestTopic(w.Namespace(), h.publisherTopic) + } + + } else if w.JobType() == livekit.JobType_JT_ROOM { + numRooms++ + if numRooms == 1 { + err = h.agentServer.RegisterJobRequestTopic(w.Namespace(), h.roomTopic) } } -} -func (s *AgentHandler) handleJobUpdate(w *worker, msg *livekit.JobStatusUpdate) { - switch msg.Status { - case livekit.JobStatus_JS_SUCCESS: - w.logger.Debugw("job complete", "jobID", msg.JobId) - case livekit.JobStatus_JS_FAILED: - w.logger.Warnw("job failed", errors.New(msg.Error), "jobID", msg.JobId) + if err != nil { + w.Logger.Errorw("failed to register job request topic", err) + h.mu.Unlock() + w.Close() // Close the worker + return } - w.mu.Lock() - w.activeJobs-- - w.mu.Unlock() + h.namespaces[w.Namespace()] = &namespaceInfo{ + numPublishers: numPublishers, + numRooms: numRooms, + } + + h.roomEnabled = h.roomAvailableLocked() + h.publisherEnabled = h.publisherAvailableLocked() + h.mu.Unlock() + + err = h.agentServer.PublishWorkerRegistered(context.Background(), agent.DefaultHandlerNamespace, &emptypb.Empty{}) + if err != nil { + w.Logger.Errorw("failed to publish worker registered", err) + } } -func (s *AgentHandler) handleStatus(w *worker, msg *livekit.UpdateWorkerStatus) { - s.mu.Lock() - defer s.mu.Unlock() +func (h *AgentHandler) deregisterWorkerTopic(worker *agent.Worker) { + h.mu.Lock() + defer h.mu.Unlock() - w.mu.Lock() - w.status = msg.Status - w.mu.Unlock() + info, ok := h.namespaces[worker.Namespace()] + if !ok { + return + } - switch w.jobType { - case livekit.JobType_JT_ROOM: - if s.roomRegistered && !s.roomAvailableLocked() { - s.roomRegistered = false - s.agentServer.DeregisterJobRequestTopic(s.roomTopic) - } else if !s.roomRegistered && s.roomAvailableLocked() { - if err := s.agentServer.RegisterJobRequestTopic(s.roomTopic); err != nil { - w.logger.Errorw("failed to register room agents", err) - } else { - s.roomRegistered = true - } + if worker.JobType() == livekit.JobType_JT_PUBLISHER { + info.numPublishers-- + if info.numPublishers == 0 { + h.agentServer.DeregisterJobRequestTopic(worker.Namespace(), h.publisherTopic) } - case livekit.JobType_JT_PUBLISHER: - if s.publisherRegistered && !s.publisherAvailableLocked() { - s.publisherRegistered = false - s.agentServer.DeregisterJobRequestTopic(s.publisherTopic) - } else if !s.publisherRegistered && s.publisherAvailableLocked() { - if err := s.agentServer.RegisterJobRequestTopic(s.publisherTopic); err != nil { - w.logger.Errorw("failed to register publisher agents", err) - } else { - s.publisherRegistered = true - } + } else if worker.JobType() == livekit.JobType_JT_ROOM { + info.numRooms-- + if info.numRooms == 0 { + h.agentServer.DeregisterJobRequestTopic(worker.Namespace(), h.roomTopic) } } -} -func (s *AgentHandler) CheckEnabled(_ context.Context, _ *rpc.CheckEnabledRequest) (*rpc.CheckEnabledResponse, error) { - s.mu.Lock() - res := &rpc.CheckEnabledResponse{ - RoomEnabled: len(s.roomWorkers) > 0, - PublisherEnabled: len(s.publisherWorkers) > 0, - } - s.mu.Unlock() - return res, nil -} - -func (s *AgentHandler) JobRequest(ctx context.Context, job *livekit.Job) (*emptypb.Empty, error) { - s.mu.Lock() - ac := make(chan *availability, 100) - s.availability[job.Id] = ac - s.mu.Unlock() - - defer func() { - s.mu.Lock() - delete(s.availability, job.Id) - s.mu.Unlock() - }() - - var pool map[string]*worker - switch job.Type { - case livekit.JobType_JT_ROOM: - pool = s.roomWorkers - case livekit.JobType_JT_PUBLISHER: - pool = s.publisherWorkers + if info.numPublishers == 0 && info.numRooms == 0 { + delete(h.namespaces, worker.Namespace()) } + h.roomEnabled = h.roomAvailableLocked() + h.publisherEnabled = h.publisherAvailableLocked() +} + +func (h *AgentHandler) roomAvailableLocked() bool { + for _, w := range h.workers { + if w.JobType() == livekit.JobType_JT_ROOM { + return true + } + } + return false + +} + +func (h *AgentHandler) publisherAvailableLocked() bool { + for _, w := range h.workers { + if w.JobType() == livekit.JobType_JT_PUBLISHER { + return true + } + } + + return false +} + +func (h *AgentHandler) JobRequest(ctx context.Context, job *livekit.Job) (*emptypb.Empty, error) { attempted := make(map[string]bool) for { - select { - case <-ctx.Done(): - return nil, psrpc.NewErrorf(psrpc.DeadlineExceeded, "request timed out") - default: - s.mu.Lock() - var selected *worker - for _, w := range pool { - if attempted[w.id] { - continue - } - if w.status == livekit.WorkerStatus_WS_AVAILABLE { - if w.activeJobs > 0 { - selected = w - break - } else if selected == nil { - selected = w - } - } - } - s.mu.Unlock() - - if selected == nil { - return nil, psrpc.NewErrorf(psrpc.Unavailable, "no workers available") + h.mu.Lock() + var selected *agent.Worker + var maxLoad float32 + for _, w := range h.workers { + if w.Namespace() != job.Namespace || w.JobType() != job.Type { + continue } - attempted[selected.id] = true - _, err := selected.sigConn.WriteServerMessage(&livekit.ServerMessage{Message: &livekit.ServerMessage_Availability{ - Availability: &livekit.AvailabilityRequest{Job: job}, - }}) - if err != nil { - selected.logger.Errorw("failed to send availability request", err, "workerID", selected.id) + _, ok := attempted[w.ID()] + if ok { + continue } - select { - case <-ctx.Done(): - return nil, psrpc.NewErrorf(psrpc.DeadlineExceeded, "request timed out") - case res := <-ac: - if res.available { - _, err = selected.sigConn.WriteServerMessage(&livekit.ServerMessage{Message: &livekit.ServerMessage_Assignment{ - Assignment: &livekit.JobAssignment{Job: job}, - }}) - if err != nil { - selected.logger.Errorw("failed to assign job", err, "workerID", selected.id) - } else { - selected.mu.Lock() - selected.activeJobs++ - selected.mu.Unlock() - return &emptypb.Empty{}, nil - } + if w.Status() == livekit.WorkerStatus_WS_AVAILABLE { + load := w.Load() + if len(w.RunningJobs()) > 0 && load > maxLoad { + maxLoad = load + selected = w + } else if selected == nil { + selected = w } } + } + h.mu.Unlock() + + if selected == nil { + return nil, psrpc.NewErrorf(psrpc.DeadlineExceeded, "no workers available") + } + + attempted[selected.ID()] = true + + err := selected.AssignJob(ctx, job) + if err != nil { + if errors.Is(err, agent.ErrWorkerNotAvailable) { + continue // Try another worker + } + return nil, err + } + + return &emptypb.Empty{}, nil } + } -func (s *AgentHandler) JobRequestAffinity(ctx context.Context, job *livekit.Job) float32 { - s.mu.Lock() - defer s.mu.Unlock() - - var pool map[string]*worker - switch job.Type { - case livekit.JobType_JT_ROOM: - pool = s.roomWorkers - case livekit.JobType_JT_PUBLISHER: - pool = s.publisherWorkers - } +func (h *AgentHandler) JobRequestAffinity(ctx context.Context, job *livekit.Job) float32 { + h.mu.Lock() + defer h.mu.Unlock() var affinity float32 - for _, w := range pool { - if w.status == livekit.WorkerStatus_WS_AVAILABLE { - if w.activeJobs > 0 { - return 1 + var maxLoad float32 + for _, w := range h.workers { + if w.Namespace() != job.Namespace || w.JobType() != job.Type { + continue + } + + if w.Status() == livekit.WorkerStatus_WS_AVAILABLE { + load := w.Load() + if len(w.RunningJobs()) > 0 && load > maxLoad { + maxLoad = load + affinity = 0.5 + load/2 } else { affinity = 0.5 } } + } return affinity } -func (s *AgentHandler) NumConnections() int { - s.mu.Lock() - defer s.mu.Unlock() +func (h *AgentHandler) CheckEnabled(ctx context.Context, req *rpc.CheckEnabledRequest) (*rpc.CheckEnabledResponse, error) { + h.mu.Lock() + defer h.mu.Unlock() + namespaces := make([]string, 0, len(h.namespaces)) + for ns := range h.namespaces { + namespaces = append(namespaces, ns) + } - return len(s.unregistered) + len(s.roomWorkers) + len(s.publisherWorkers) + return &rpc.CheckEnabledResponse{ + Namespaces: namespaces, + RoomEnabled: h.roomEnabled, + PublisherEnabled: h.publisherEnabled, + }, nil } -func (s *AgentHandler) DrainConnections(interval time.Duration) { +func (h *AgentHandler) NumConnections() int { + h.mu.Lock() + defer h.mu.Unlock() + return len(h.workers) +} + +func (h *AgentHandler) DrainConnections(interval time.Duration) { // jitter drain start time.Sleep(time.Duration(rand.Int63n(int64(interval)))) t := time.NewTicker(interval) defer t.Stop() - s.mu.Lock() - defer s.mu.Unlock() + h.mu.Lock() + defer h.mu.Unlock() - for conn := range s.unregistered { - _ = conn.Close() - <-t.C - } - for _, w := range s.roomWorkers { - _ = w.conn.Close() - <-t.C - } - for _, w := range s.publisherWorkers { - _ = w.conn.Close() + for _, w := range h.workers { + w.Close() <-t.C } } - -func (s *AgentHandler) roomAvailableLocked() bool { - for _, w := range s.roomWorkers { - if w.status == livekit.WorkerStatus_WS_AVAILABLE { - return true - } - } - return false -} - -func (s *AgentHandler) publisherAvailableLocked() bool { - for _, w := range s.publisherWorkers { - if w.status == livekit.WorkerStatus_WS_AVAILABLE { - return true - } - } - return false -} diff --git a/pkg/service/auth.go b/pkg/service/auth.go index 5633b2889..c319eceff 100644 --- a/pkg/service/auth.go +++ b/pkg/service/auth.go @@ -34,6 +34,11 @@ const ( type grantsKey struct{} +type grantsValue struct { + claims *auth.ClaimGrants + apiKey string +} + var ( ErrPermissionDenied = errors.New("permissions denied") ErrMissingAuthorization = errors.New("invalid authorization header. Must start with " + bearerPrefix) @@ -93,7 +98,10 @@ func (m *APIKeyAuthMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request, // set grants in context ctx := r.Context() - r = r.WithContext(context.WithValue(ctx, grantsKey{}, grants)) + r = r.WithContext(context.WithValue(ctx, grantsKey{}, &grantsValue{ + claims: grants, + apiKey: v.APIKey(), + })) } next.ServeHTTP(w, r) @@ -101,15 +109,27 @@ func (m *APIKeyAuthMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request, func GetGrants(ctx context.Context) *auth.ClaimGrants { val := ctx.Value(grantsKey{}) - claims, ok := val.(*auth.ClaimGrants) + v, ok := val.(*grantsValue) if !ok { return nil } - return claims + return v.claims } -func WithGrants(ctx context.Context, grants *auth.ClaimGrants) context.Context { - return context.WithValue(ctx, grantsKey{}, grants) +func GetAPIKey(ctx context.Context) string { + val := ctx.Value(grantsKey{}) + v, ok := val.(*grantsValue) + if !ok { + return "" + } + return v.apiKey +} + +func WithGrants(ctx context.Context, grants *auth.ClaimGrants, apiKey string) context.Context { + return context.WithValue(ctx, grantsKey{}, &grantsValue{ + claims: grants, + apiKey: apiKey, + }) } func SetAuthorizationToken(r *http.Request, token string) { diff --git a/pkg/service/roommanager.go b/pkg/service/roommanager.go index 96dd7036c..3d3b8dd13 100644 --- a/pkg/service/roommanager.go +++ b/pkg/service/roommanager.go @@ -24,6 +24,7 @@ import ( "github.com/pkg/errors" "golang.org/x/exp/maps" + "github.com/livekit/livekit-server/pkg/agent" "github.com/livekit/mediatransportutil/pkg/rtcconfig" "github.com/livekit/protocol/auth" "github.com/livekit/protocol/livekit" @@ -70,7 +71,7 @@ type RoomManager struct { roomStore ObjectStore telemetry telemetry.TelemetryService clientConfManager clientconfiguration.ClientConfigurationManager - agentClient rtc.AgentClient + agentClient agent.Client egressLauncher rtc.EgressLauncher versionGenerator utils.TimedVersionGenerator turnAuthHandler *TURNAuthHandler @@ -91,7 +92,7 @@ func NewLocalRoomManager( router routing.Router, telemetry telemetry.TelemetryService, clientConfManager clientconfiguration.ClientConfigurationManager, - agentClient rtc.AgentClient, + agentClient agent.Client, egressLauncher rtc.EgressLauncher, versionGenerator utils.TimedVersionGenerator, turnAuthHandler *TURNAuthHandler, @@ -121,11 +122,12 @@ func NewLocalRoomManager( iceConfigCache: make(map[livekit.ParticipantIdentity]*iceConfigCacheEntry), serverInfo: &livekit.ServerInfo{ - Edition: livekit.ServerInfo_Standard, - Version: version.Version, - Protocol: types.CurrentProtocol, - Region: conf.Region, - NodeId: currentNode.Id, + Edition: livekit.ServerInfo_Standard, + Version: version.Version, + Protocol: types.CurrentProtocol, + AgentProtocol: agent.CurrentProtocol, + Region: conf.Region, + NodeId: currentNode.Id, }, }, nil } diff --git a/pkg/service/roomservice.go b/pkg/service/roomservice.go index 3da38851c..5227d141e 100644 --- a/pkg/service/roomservice.go +++ b/pkg/service/roomservice.go @@ -22,12 +22,12 @@ import ( "github.com/pkg/errors" "github.com/twitchtv/twirp" + "github.com/livekit/livekit-server/pkg/agent" "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/routing" "github.com/livekit/livekit-server/pkg/rtc" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/rpc" - "github.com/livekit/protocol/utils" "github.com/livekit/psrpc" ) @@ -39,7 +39,7 @@ type RoomService struct { router routing.MessageRouter roomAllocator RoomAllocator roomStore ServiceStore - agentClient rtc.AgentClient + agentClient agent.Client egressLauncher rtc.EgressLauncher topicFormatter rpc.TopicFormatter roomClient rpc.TypedRoomClient @@ -53,7 +53,7 @@ func NewRoomService( router routing.MessageRouter, roomAllocator RoomAllocator, serviceStore ServiceStore, - agentClient rtc.AgentClient, + agentClient agent.Client, egressLauncher rtc.EgressLauncher, topicFormatter rpc.TopicFormatter, roomClient rpc.TypedRoomClient, @@ -101,13 +101,10 @@ func (s *RoomService) CreateRoom(ctx context.Context, req *livekit.CreateRoomReq defer res.ResponseSource.Close() if created { - go func() { - s.agentClient.JobRequest(ctx, &livekit.Job{ - Id: utils.NewGuid("JR_"), - Type: livekit.JobType_JT_ROOM, - Room: rm, - }) - }() + go s.agentClient.LaunchJob(ctx, &agent.JobDescription{ + JobType: livekit.JobType_JT_ROOM, + Room: rm, + }) if req.Egress != nil && req.Egress.Room != nil { _, err = s.egressLauncher.StartEgress(ctx, &rpc.StartEgressRequest{ @@ -301,13 +298,10 @@ func (s *RoomService) UpdateRoomMetadata(ctx context.Context, req *livekit.Updat } if created { - go func() { - s.agentClient.JobRequest(ctx, &livekit.Job{ - Id: utils.NewGuid("JR_"), - Type: livekit.JobType_JT_ROOM, - Room: room, - }) - }() + go s.agentClient.LaunchJob(ctx, &agent.JobDescription{ + JobType: livekit.JobType_JT_ROOM, + Room: room, + }) } return room, nil diff --git a/pkg/service/roomservice_test.go b/pkg/service/roomservice_test.go index 0237117f4..aa10a6399 100644 --- a/pkg/service/roomservice_test.go +++ b/pkg/service/roomservice_test.go @@ -38,7 +38,7 @@ func TestDeleteRoom(t *testing.T) { grant := &auth.ClaimGrants{ Video: &auth.VideoGrant{}, } - ctx := service.WithGrants(context.Background(), grant) + ctx := service.WithGrants(context.Background(), grant, "") _, err := svc.DeleteRoom(ctx, &livekit.DeleteRoomRequest{ Room: "testroom", }) @@ -52,7 +52,7 @@ func TestMetaDataLimits(t *testing.T) { grant := &auth.ClaimGrants{ Video: &auth.VideoGrant{}, } - ctx := service.WithGrants(context.Background(), grant) + ctx := service.WithGrants(context.Background(), grant, "") _, err := svc.UpdateParticipant(ctx, &livekit.UpdateParticipantRequest{ Room: "testroom", Identity: "123", @@ -82,7 +82,7 @@ func TestMetaDataLimits(t *testing.T) { grant := &auth.ClaimGrants{ Video: &auth.VideoGrant{}, } - ctx := service.WithGrants(context.Background(), grant) + ctx := service.WithGrants(context.Background(), grant, "") _, err := svc.UpdateParticipant(ctx, &livekit.UpdateParticipantRequest{ Room: "testroom", Identity: "123", diff --git a/pkg/service/rtcservice.go b/pkg/service/rtcservice.go index 60100729f..efe314c5a 100644 --- a/pkg/service/rtcservice.go +++ b/pkg/service/rtcservice.go @@ -32,6 +32,7 @@ import ( "go.uber.org/atomic" "golang.org/x/exp/maps" + "github.com/livekit/livekit-server/pkg/agent" "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/routing" "github.com/livekit/livekit-server/pkg/routing/selector" @@ -40,7 +41,6 @@ import ( "github.com/livekit/livekit-server/pkg/telemetry/prometheus" "github.com/livekit/livekit-server/pkg/utils" "github.com/livekit/protocol/livekit" - putil "github.com/livekit/protocol/utils" "github.com/livekit/psrpc" ) @@ -54,7 +54,7 @@ type RTCService struct { isDev bool limits config.LimitConfig parser *uaparser.Parser - agentClient rtc.AgentClient + agentClient agent.Client telemetry telemetry.TelemetryService mu sync.Mutex @@ -67,7 +67,7 @@ func NewRTCService( store ServiceStore, router routing.MessageRouter, currentNode routing.LocalNode, - agentClient rtc.AgentClient, + agentClient agent.Client, telemetry telemetry.TelemetryService, ) *RTCService { s := &RTCService{ @@ -524,6 +524,13 @@ func (s *RTCService) startConnection( return cr, nil, err } + if created && s.agentClient != nil { + go s.agentClient.LaunchJob(ctx, &agent.JobDescription{ + JobType: livekit.JobType_JT_ROOM, + Room: cr.Room, + }) + } + // this needs to be started first *before* using router functions on this node cr.StartParticipantSignalResults, err = s.router.StartParticipantSignal(ctx, roomName, pi) if err != nil { @@ -541,16 +548,6 @@ func (s *RTCService) startConnection( return cr, nil, err } - if created && s.agentClient != nil { - go func() { - s.agentClient.JobRequest(ctx, &livekit.Job{ - Id: putil.NewGuid("JR_"), - Type: livekit.JobType_JT_ROOM, - Room: cr.Room, - }) - }() - } - return cr, initialResponse, nil } diff --git a/pkg/service/server.go b/pkg/service/server.go index fcb09a648..5dc123fc0 100644 --- a/pkg/service/server.go +++ b/pkg/service/server.go @@ -125,6 +125,7 @@ func NewLivekitServer(conf *config.Config, mux.HandleFunc("/debug/goroutine", s.debugGoroutines) mux.HandleFunc("/debug/rooms", s.debugInfo) } + mux.Handle(roomServer.PathPrefix(), roomServer) mux.Handle(egressServer.PathPrefix(), egressServer) mux.Handle(ingressServer.PathPrefix(), ingressServer) diff --git a/pkg/service/wire.go b/pkg/service/wire.go index 654f26345..3eb4a64c6 100644 --- a/pkg/service/wire.go +++ b/pkg/service/wire.go @@ -30,7 +30,7 @@ import ( "github.com/livekit/livekit-server/pkg/clientconfiguration" "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/routing" - "github.com/livekit/livekit-server/pkg/rtc" + "github.com/livekit/livekit-server/pkg/agent" "github.com/livekit/livekit-server/pkg/telemetry" "github.com/livekit/protocol/auth" "github.com/livekit/protocol/livekit" @@ -77,7 +77,7 @@ func InitializeServer(conf *config.Config, currentNode routing.LocalNode) (*Live NewRoomService, NewRTCService, NewAgentService, - rtc.NewAgentClient, + agent.NewAgentClient, getSignalRelayConfig, NewDefaultSignalServer, routing.NewSignalClient, diff --git a/pkg/service/wire_gen.go b/pkg/service/wire_gen.go index 82d232e55..80b8bd8c0 100644 --- a/pkg/service/wire_gen.go +++ b/pkg/service/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run -mod=mod github.com/google/wire/cmd/wire +//go:generate go run github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject @@ -8,10 +8,10 @@ package service import ( "fmt" + "github.com/livekit/livekit-server/pkg/agent" "github.com/livekit/livekit-server/pkg/clientconfiguration" "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/routing" - "github.com/livekit/livekit-server/pkg/rtc" "github.com/livekit/livekit-server/pkg/telemetry" "github.com/livekit/protocol/auth" "github.com/livekit/protocol/livekit" @@ -60,7 +60,7 @@ func InitializeServer(conf *config.Config, currentNode routing.LocalNode) (*Live if err != nil { return nil, err } - agentClient, err := rtc.NewAgentClient(messageBus) + client, err := agent.NewAgentClient(messageBus) if err != nil { return nil, err } @@ -95,7 +95,7 @@ func InitializeServer(conf *config.Config, currentNode routing.LocalNode) (*Live if err != nil { return nil, err } - roomService, err := NewRoomService(roomConfig, apiConfig, psrpcConfig, router, roomAllocator, objectStore, agentClient, rtcEgressLauncher, topicFormatter, roomClient, participantClient) + roomService, err := NewRoomService(roomConfig, apiConfig, psrpcConfig, router, roomAllocator, objectStore, client, rtcEgressLauncher, topicFormatter, roomClient, participantClient) if err != nil { return nil, err } @@ -112,15 +112,15 @@ func InitializeServer(conf *config.Config, currentNode routing.LocalNode) (*Live return nil, err } sipService := NewSIPService(sipConfig, nodeID, messageBus, sipClient, sipStore, roomService, telemetryService) - rtcService := NewRTCService(conf, roomAllocator, objectStore, router, currentNode, agentClient, telemetryService) - agentService, err := NewAgentService(messageBus) + rtcService := NewRTCService(conf, roomAllocator, objectStore, router, currentNode, client, telemetryService) + agentService, err := NewAgentService(conf, currentNode, messageBus, keyProvider) if err != nil { return nil, err } clientConfigurationManager := createClientConfiguration() timedVersionGenerator := utils.NewDefaultTimedVersionGenerator() turnAuthHandler := NewTURNAuthHandler(keyProvider) - roomManager, err := NewLocalRoomManager(conf, objectStore, currentNode, router, telemetryService, clientConfigurationManager, agentClient, rtcEgressLauncher, timedVersionGenerator, turnAuthHandler, messageBus) + roomManager, err := NewLocalRoomManager(conf, objectStore, currentNode, router, telemetryService, clientConfigurationManager, client, rtcEgressLauncher, timedVersionGenerator, turnAuthHandler, messageBus) if err != nil { return nil, err } diff --git a/pkg/testutils/timeout.go b/pkg/testutils/timeout.go index 11debef92..7745b2edc 100644 --- a/pkg/testutils/timeout.go +++ b/pkg/testutils/timeout.go @@ -24,15 +24,19 @@ var ( ConnectTimeout = 30 * time.Second ) -func WithTimeout(t *testing.T, f func() string) { - ctx, cancel := context.WithTimeout(context.Background(), ConnectTimeout) +func WithTimeout(t *testing.T, f func() string, timeouts ...time.Duration) { + timeout := ConnectTimeout + if len(timeouts) > 0 { + timeout = timeouts[0] + } + ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() lastErr := "" for { select { case <-ctx.Done(): if lastErr != "" { - t.Fatalf("did not reach expected state after %v: %s", ConnectTimeout, lastErr) + t.Fatalf("did not reach expected state after %v: %s", timeout, lastErr) } case <-time.After(10 * time.Millisecond): lastErr = f() diff --git a/pkg/utils/incrementaldispatcher.go b/pkg/utils/incrementaldispatcher.go new file mode 100644 index 000000000..dcc23d55c --- /dev/null +++ b/pkg/utils/incrementaldispatcher.go @@ -0,0 +1,79 @@ +/* + * Copyright 2024 LiveKit, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "sync" + + "github.com/frostbyte73/core" +) + +// IncrementalDispatcher is a dispatcher that allows multiple consumers to consume items as they become +// available, while producers can add items at anytime. +type IncrementalDispatcher[T any] struct { + done core.Fuse + lock sync.RWMutex + cond *sync.Cond + items []T +} + +func NewIncrementalDispatcher[T any]() *IncrementalDispatcher[T] { + p := &IncrementalDispatcher[T]{} + p.cond = sync.NewCond(&p.lock) + return p +} + +func (d *IncrementalDispatcher[T]) Add(item T) { + if d.done.IsBroken() { + return + } + d.lock.Lock() + d.items = append(d.items, item) + d.lock.Unlock() + d.cond.Broadcast() +} + +func (d *IncrementalDispatcher[T]) Done() { + d.done.Break() + d.cond.Broadcast() +} + +func (d *IncrementalDispatcher[T]) ForEach(fn func(T)) { + idx := 0 + dispatchFromIdx := func() { + var itemsToDispatch []T + d.lock.RLock() + for idx < len(d.items) { + itemsToDispatch = append(itemsToDispatch, d.items[idx]) + idx++ + } + d.lock.RUnlock() + for _, item := range itemsToDispatch { + fn(item) + } + } + for !d.done.IsBroken() { + dispatchFromIdx() + d.lock.Lock() + if idx == len(d.items) { + d.cond.Wait() + } + d.lock.Unlock() + } + + dispatchFromIdx() +} diff --git a/pkg/utils/incrementaldispatcher_test.go b/pkg/utils/incrementaldispatcher_test.go new file mode 100644 index 000000000..da1393c64 --- /dev/null +++ b/pkg/utils/incrementaldispatcher_test.go @@ -0,0 +1,96 @@ +/* + * Copyright 2024 LiveKit, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils_test + +import ( + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/livekit/livekit-server/pkg/testutils" + "github.com/livekit/livekit-server/pkg/utils" +) + +func TestForEach(t *testing.T) { + producer := utils.NewIncrementalDispatcher[int]() + go func() { + defer producer.Done() + producer.Add(1) + producer.Add(2) + producer.Add(3) + }() + + sum := 0 + producer.ForEach(func(item int) { + sum += item + }) + + require.Equal(t, 6, sum) +} + +func TestConcurrentConsumption(t *testing.T) { + producer := utils.NewIncrementalDispatcher[int]() + numConsumers := 5 + sums := make([]atomic.Int32, numConsumers) + var wg sync.WaitGroup + + for i := 0; i < numConsumers; i++ { + wg.Add(1) + i := i + go func() { + defer wg.Done() + producer.ForEach(func(item int) { + sums[i].Add(int32(item)) + }) + }() + } + + // Add items + expectedSum := 0 + for i := 0; i < 20; i++ { + expectedSum += i + producer.Add(i) + } + + for i := 0; i < numConsumers; i++ { + testutils.WithTimeout(t, func() string { + if sums[i].Load() != int32(expectedSum) { + return fmt.Sprintf("consumer %d did not consume all the items. expected %d, actual: %d", + i, expectedSum, sums[i].Load()) + } + return "" + }, time.Second) + } + + // keep adding and ensure it's consumed + for i := 20; i < 30; i++ { + expectedSum += i + producer.Add(i) + } + + // wait for all consumers to finish + producer.Done() + wg.Wait() + + for i := 0; i < numConsumers; i++ { + require.Equal(t, int32(expectedSum), sums[i].Load(), "consumer %d did not match", i) + } +} diff --git a/test/agent.go b/test/agent.go index cc04cad84..244084ffb 100644 --- a/test/agent.go +++ b/test/agent.go @@ -25,7 +25,6 @@ import ( "google.golang.org/protobuf/proto" "github.com/livekit/protocol/livekit" - "github.com/livekit/protocol/utils" ) type agentClient struct { @@ -38,11 +37,13 @@ type agentClient struct { participantAvailability atomic.Int32 participantJobs atomic.Int32 + requestedJobs chan *livekit.Job + done chan struct{} } -func newAgentClient(token string) (*agentClient, error) { - host := fmt.Sprintf("ws://localhost:%d", defaultServerPort) +func newAgentClient(token string, port uint32) (*agentClient, error) { + host := fmt.Sprintf("ws://localhost:%d", port) u, err := url.Parse(host + "/agent") if err != nil { return nil, err @@ -57,25 +58,24 @@ func newAgentClient(token string) (*agentClient, error) { } return &agentClient{ - conn: conn, - done: make(chan struct{}), + conn: conn, + requestedJobs: make(chan *livekit.Job, 100), + done: make(chan struct{}), }, nil } -func (c *agentClient) Run(jobType livekit.JobType) (err error) { +func (c *agentClient) Run(jobType livekit.JobType, namespace string) (err error) { go c.read() - workerID := utils.NewGuid("W_") - switch jobType { case livekit.JobType_JT_ROOM: err = c.write(&livekit.WorkerMessage{ Message: &livekit.WorkerMessage_Register{ Register: &livekit.RegisterWorkerRequest{ - Type: livekit.JobType_JT_ROOM, - WorkerId: workerID, - Version: "version", - Name: "name", + Type: livekit.JobType_JT_ROOM, + Version: "version", + Namespace: &namespace, + Name: "name", }, }, }) @@ -84,10 +84,10 @@ func (c *agentClient) Run(jobType livekit.JobType) (err error) { err = c.write(&livekit.WorkerMessage{ Message: &livekit.WorkerMessage_Register{ Register: &livekit.RegisterWorkerRequest{ - Type: livekit.JobType_JT_PUBLISHER, - WorkerId: workerID, - Version: "version", - Name: "name", + Type: livekit.JobType_JT_PUBLISHER, + Version: "version", + Namespace: &namespace, + Name: "name", }, }, }) @@ -139,6 +139,8 @@ func (c *agentClient) handleAvailability(req *livekit.AvailabilityRequest) { c.participantAvailability.Inc() } + c.requestedJobs <- req.Job + c.write(&livekit.WorkerMessage{ Message: &livekit.WorkerMessage_Availability{ Availability: &livekit.AvailabilityResponse{ @@ -157,16 +159,23 @@ func (c *agentClient) write(msg *livekit.WorkerMessage) error { c.mu.Lock() defer c.mu.Unlock() - b, err := proto.Marshal(msg) - if err != nil { - return err - } + select { + case <-c.done: + return nil + default: + b, err := proto.Marshal(msg) + if err != nil { + return err + } - return c.conn.WriteMessage(websocket.BinaryMessage, b) + return c.conn.WriteMessage(websocket.BinaryMessage, b) + } } func (c *agentClient) close() { + c.mu.Lock() close(c.done) _ = c.conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")) _ = c.conn.Close() + c.mu.Unlock() } diff --git a/test/agent_test.go b/test/agent_test.go index a8825577b..88e44b19c 100644 --- a/test/agent_test.go +++ b/test/agent_test.go @@ -15,72 +15,184 @@ package test import ( + "fmt" "testing" "time" "github.com/stretchr/testify/require" + "github.com/livekit/livekit-server/pkg/testutils" "github.com/livekit/protocol/auth" "github.com/livekit/protocol/livekit" ) +var ( + RegisterTimeout = 2 * time.Second + AssignJobTimeout = 3 * time.Second +) + func TestAgents(t *testing.T) { _, finish := setupSingleNodeTest("TestAgents") defer finish() - ac1, err := newAgentClient(agentToken()) + + ac1, err := newAgentClient(agentToken(), defaultServerPort) require.NoError(t, err) - ac2, err := newAgentClient(agentToken()) + ac2, err := newAgentClient(agentToken(), defaultServerPort) require.NoError(t, err) - ac3, err := newAgentClient(agentToken()) + ac3, err := newAgentClient(agentToken(), defaultServerPort) require.NoError(t, err) - ac4, err := newAgentClient(agentToken()) + ac4, err := newAgentClient(agentToken(), defaultServerPort) require.NoError(t, err) defer ac1.close() defer ac2.close() defer ac3.close() defer ac4.close() - ac1.Run(livekit.JobType_JT_ROOM) - ac2.Run(livekit.JobType_JT_ROOM) - ac3.Run(livekit.JobType_JT_PUBLISHER) - ac4.Run(livekit.JobType_JT_PUBLISHER) + ac1.Run(livekit.JobType_JT_ROOM, "namespace") + ac2.Run(livekit.JobType_JT_ROOM, "namespace") + ac3.Run(livekit.JobType_JT_PUBLISHER, "namespace") + ac4.Run(livekit.JobType_JT_PUBLISHER, "namespace") - time.Sleep(time.Second * 3) + testutils.WithTimeout(t, func() string { + if ac1.registered.Load() != 1 || ac2.registered.Load() != 1 || ac3.registered.Load() != 1 || ac4.registered.Load() != 1 { + return "worker not registered" + } - require.Equal(t, int32(1), ac1.registered.Load()) - require.Equal(t, int32(1), ac2.registered.Load()) - require.Equal(t, int32(1), ac3.registered.Load()) - require.Equal(t, int32(1), ac4.registered.Load()) + return "" + }, RegisterTimeout) c1 := createRTCClient("c1", defaultServerPort, nil) c2 := createRTCClient("c2", defaultServerPort, nil) waitUntilConnected(t, c1, c2) // publish 2 tracks - t1, err := c1.AddStaticTrack("audio/opus", "audio", "webcam") + t1, err := c1.AddStaticTrack("audio/opus", "audio", "micro") require.NoError(t, err) defer t1.Stop() t2, err := c1.AddStaticTrack("video/vp8", "video", "webcam") require.NoError(t, err) defer t2.Stop() - time.Sleep(time.Second * 3) + testutils.WithTimeout(t, func() string { + if ac1.roomJobs.Load()+ac2.roomJobs.Load() != 1 { + return "room job not assigned" + } - require.Equal(t, int32(1), ac1.roomJobs.Load()+ac2.roomJobs.Load()) - require.Equal(t, int32(1), ac3.participantJobs.Load()+ac4.participantJobs.Load()) + if ac3.participantJobs.Load()+ac4.participantJobs.Load() != 1 { + return fmt.Sprintf("participant jobs not assigned, ac3: %d, ac4: %d", ac3.participantJobs.Load(), ac4.participantJobs.Load()) + } + + return "" + }, 6 * time.Second) // publish 2 tracks - t3, err := c2.AddStaticTrack("audio/opus", "audio", "webcam") + t3, err := c2.AddStaticTrack("audio/opus", "audio", "micro") require.NoError(t, err) defer t3.Stop() t4, err := c2.AddStaticTrack("video/vp8", "video", "webcam") require.NoError(t, err) defer t4.Stop() - time.Sleep(time.Second * 3) + testutils.WithTimeout(t, func() string { + if ac1.roomJobs.Load()+ac2.roomJobs.Load() != 1 { + return "room job must be assigned 1 time" + } - require.Equal(t, int32(1), ac1.roomJobs.Load()+ac2.roomJobs.Load()) - require.Equal(t, int32(2), ac3.participantJobs.Load()+ac4.participantJobs.Load()) + if ac3.participantJobs.Load()+ac4.participantJobs.Load() != 2 { + return "2 publisher jobs must assigned" + } + + return "" + }, AssignJobTimeout) +} + +func TestAgentNamespaces(t *testing.T) { + _, finish := setupSingleNodeTest("TestAgentNamespaces") + defer finish() + + ac1, err := newAgentClient(agentToken(), defaultServerPort) + require.NoError(t, err) + ac2, err := newAgentClient(agentToken(), defaultServerPort) + require.NoError(t, err) + defer ac1.close() + defer ac2.close() + ac1.Run(livekit.JobType_JT_ROOM, "namespace") + ac2.Run(livekit.JobType_JT_ROOM, "namespace2") + + testutils.WithTimeout(t, func() string { + if ac1.registered.Load() != 1 || ac2.registered.Load() != 1 { + return "worker not registered" + } + return "" + }, RegisterTimeout) + + c1 := createRTCClient("c1", defaultServerPort, nil) + waitUntilConnected(t, c1) + + testutils.WithTimeout(t, func() string { + if ac1.roomJobs.Load() != 1 || ac2.roomJobs.Load() != 1 { + return "room job not assigned" + } + + job1 := <-ac1.requestedJobs + job2 := <-ac2.requestedJobs + + if job1.Namespace != "namespace" { + return "namespace is not 'namespace'" + } + + if job2.Namespace != "namespace2" { + return "namespace is not 'namespace2'" + } + + if job1.Id == job2.Id { + return "job ids are the same" + } + + return "" + }, AssignJobTimeout) + +} + +func TestAgentMultiNode(t *testing.T) { + _, _, finish := setupMultiNodeTest("TestAgentMultiNode") + defer finish() + + ac1, err := newAgentClient(agentToken(), defaultServerPort) + require.NoError(t, err) + ac2, err := newAgentClient(agentToken(), defaultServerPort) + defer ac1.close() + defer ac2.close() + ac1.Run(livekit.JobType_JT_ROOM, "namespace") + ac2.Run(livekit.JobType_JT_PUBLISHER, "namespace") + + testutils.WithTimeout(t, func() string { + if ac1.registered.Load() != 1 || ac2.registered.Load() != 1 { + return "worker not registered" + } + return "" + }, RegisterTimeout) + + c1 := createRTCClient("c1", secondServerPort, nil) // Create a room on the second node + waitUntilConnected(t, c1) + + t1, err := c1.AddStaticTrack("audio/opus", "audio", "micro") + require.NoError(t, err) + defer t1.Stop() + + time.Sleep(time.Second * 10) + + testutils.WithTimeout(t, func() string { + if ac1.roomJobs.Load() != 1 { + return "room job not assigned" + } + + if ac2.participantJobs.Load() != 1 { + return "participant job not assigned" + } + + return "" + }, AssignJobTimeout) } func agentToken() string { From 41b70ef555e3f501300b413d6fc5c1a414fdb7a1 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Wed, 3 Apr 2024 17:02:55 -0700 Subject: [PATCH 23/78] Update Pion & SCTP to handle ZeroChecksum interop (#2619) Details: https://github.com/pion/sctp/pull/327 --- go.mod | 8 ++++---- go.sum | 20 ++++++++++++-------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 6c02284ac..0df8c9f6f 100644 --- a/go.mod +++ b/go.mod @@ -30,12 +30,12 @@ require ( github.com/pion/ice/v2 v2.3.14 github.com/pion/interceptor v0.1.25 github.com/pion/rtcp v1.2.14 - github.com/pion/rtp v1.8.3 - github.com/pion/sctp v1.8.12 - github.com/pion/sdp/v3 v3.0.8 + github.com/pion/rtp v1.8.5 + github.com/pion/sctp v1.8.14 + github.com/pion/sdp/v3 v3.0.9 github.com/pion/transport/v2 v2.2.4 github.com/pion/turn/v2 v2.1.5 - github.com/pion/webrtc/v3 v3.2.29 + github.com/pion/webrtc/v3 v3.2.34 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.19.0 github.com/redis/go-redis/v9 v9.5.1 diff --git a/go.sum b/go.sum index ed8db6b06..e5e0dba7b 100644 --- a/go.sum +++ b/go.sum @@ -205,13 +205,14 @@ github.com/pion/rtcp v1.2.12/go.mod h1:sn6qjxvnwyAkkPzPULIbVqSKI5Dv54Rv7VG0kNxh9 github.com/pion/rtcp v1.2.14 h1:KCkGV3vJ+4DAJmvP0vaQShsb0xkRfWkO540Gy102KyE= github.com/pion/rtcp v1.2.14/go.mod h1:sn6qjxvnwyAkkPzPULIbVqSKI5Dv54Rv7VG0kNxh9L4= github.com/pion/rtp v1.8.2/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= -github.com/pion/rtp v1.8.3 h1:VEHxqzSVQxCkKDSHro5/4IUUG1ea+MFdqR2R3xSpNU8= github.com/pion/rtp v1.8.3/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= +github.com/pion/rtp v1.8.5 h1:uYzINfaK+9yWs7r537z/Rc1SvT8ILjBcmDOpJcTB+OU= +github.com/pion/rtp v1.8.5/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= github.com/pion/sctp v1.8.5/go.mod h1:SUFFfDpViyKejTAdwD1d/HQsCu+V/40cCs2nZIvC3s0= -github.com/pion/sctp v1.8.12 h1:2VX50pedElH+is6FI+OKyRTeN5oy4mrk2HjnGa3UCmY= -github.com/pion/sctp v1.8.12/go.mod h1:cMLT45jqw3+jiJCrtHVwfQLnfR0MGZ4rgOJwUOIqLkI= -github.com/pion/sdp/v3 v3.0.8 h1:yd/wkrS0nzXEAb+uwv1TL3SG/gzsTiXHVOtXtD7EKl0= -github.com/pion/sdp/v3 v3.0.8/go.mod h1:B5xmvENq5IXJimIO4zfp6LAe1fD9N+kFv+V/1lOdz8M= +github.com/pion/sctp v1.8.14 h1:NzwwDrtpvbdqMMWV9Q6NYGbHE/FQmjI+GEQLyeJahu4= +github.com/pion/sctp v1.8.14/go.mod h1:P6PbDVA++OJMrVNg2AL3XtYHV4uD6dvfyOovCgMs0PE= +github.com/pion/sdp/v3 v3.0.9 h1:pX++dCHoHUwq43kuwf3PyJfHlwIj4hXA7Vrifiq0IJY= +github.com/pion/sdp/v3 v3.0.9/go.mod h1:B5xmvENq5IXJimIO4zfp6LAe1fD9N+kFv+V/1lOdz8M= github.com/pion/srtp/v2 v2.0.18 h1:vKpAXfawO9RtTRKZJbG4y0v1b11NZxQnxRl85kGuUlo= github.com/pion/srtp/v2 v2.0.18/go.mod h1:0KJQjA99A6/a0DOVTu1PhDSw0CXF2jTkqOoMg3ODqdA= github.com/pion/stun v0.6.1 h1:8lp6YejULeHBF8NmV8e2787BogQhduZugh5PdhDyyN4= @@ -223,13 +224,14 @@ github.com/pion/transport/v2 v2.2.2/go.mod h1:OJg3ojoBJopjEeECq2yJdXH9YVrUJ1uQ++ github.com/pion/transport/v2 v2.2.3/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= github.com/pion/transport/v2 v2.2.4 h1:41JJK6DZQYSeVLxILA2+F4ZkKb4Xd/tFJZRFZQ9QAlo= github.com/pion/transport/v2 v2.2.4/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= -github.com/pion/transport/v3 v3.0.1 h1:gDTlPJwROfSfz6QfSi0ZmeCSkFcnWWiiR9ES0ouANiM= github.com/pion/transport/v3 v3.0.1/go.mod h1:UY7kiITrlMv7/IKgd5eTUcaahZx5oUN3l9SzK5f5xE0= +github.com/pion/transport/v3 v3.0.2 h1:r+40RJR25S9w3jbA6/5uEPTzcdn7ncyU44RWCbHkLg4= +github.com/pion/transport/v3 v3.0.2/go.mod h1:nIToODoOlb5If2jF9y2Igfx3PFYWfuXi37m0IlWa/D0= github.com/pion/turn/v2 v2.1.3/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= github.com/pion/turn/v2 v2.1.5 h1:tTyy7TM3DCoX9IxTt/yHc/bThiRLyXK3T1YbNcgx9k4= github.com/pion/turn/v2 v2.1.5/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= -github.com/pion/webrtc/v3 v3.2.29 h1:flXjxjlqpp3FjkpSSBKwv7UOfbUvan9+gFY6A5ZaAn4= -github.com/pion/webrtc/v3 v3.2.29/go.mod h1:M+5YSvBDPAkHHRwGXlplIFBQI5mXm6Y4byns1OpiX68= +github.com/pion/webrtc/v3 v3.2.34 h1:wcKWYlVdfw+Zpdzx9csz/ou87ru9RGrl0yDJ2vKY+70= +github.com/pion/webrtc/v3 v3.2.34/go.mod h1:0vW+VYQwUumq9R/dWjRE1IT+jeHh3MtiYDszdNrLjxo= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -348,6 +350,7 @@ golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -415,6 +418,7 @@ golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= From 5cfcbc0ca6c12995d0b47118a7c580c0ab3dc467 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 4 Apr 2024 18:23:30 +0530 Subject: [PATCH 24/78] Move caching of publisher sender report to subscriber side. (#2622) * Move caching of publisher sender report to subscriber side. Please see inline for descriptive comments on why. Basically, pause/unpause using replaceTrack(null)/replaceTrack(actualTrack) can cause time stamp in sender report sent to subscribers jump ahead. This prevents that. With the caching on subscriber side, cleaning up the caching on publisher side. * fix compile, test still failing, need to debug * skip reference TS for testing --- pkg/rtc/wrappedreceiver.go | 15 --- pkg/sfu/downtrack.go | 16 ++- pkg/sfu/forwarder.go | 177 +++++++++++++++++++++++--------- pkg/sfu/forwarder_test.go | 2 +- pkg/sfu/receiver.go | 13 --- pkg/sfu/streamtrackermanager.go | 114 -------------------- 6 files changed, 136 insertions(+), 201 deletions(-) diff --git a/pkg/rtc/wrappedreceiver.go b/pkg/rtc/wrappedreceiver.go index 442a8df83..42fe9d707 100644 --- a/pkg/rtc/wrappedreceiver.go +++ b/pkg/rtc/wrappedreceiver.go @@ -26,7 +26,6 @@ import ( "github.com/livekit/protocol/logger" "github.com/livekit/livekit-server/pkg/sfu" - "github.com/livekit/livekit-server/pkg/sfu/buffer" ) // wrapper around WebRTC receiver, overriding its ID @@ -317,20 +316,6 @@ func (d *DummyReceiver) GetRedReceiver() sfu.TrackReceiver { return d } -func (d *DummyReceiver) GetReferenceLayerRTPTimestamp(ts uint32, layer int32, referenceLayer int32) (uint32, error) { - if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { - return r.GetReferenceLayerRTPTimestamp(ts, layer, referenceLayer) - } - return 0, errors.New("receiver not available") -} - -func (d *DummyReceiver) GetRTCPSenderReportData(layer int32) *buffer.RTCPSenderReportData { - if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { - return r.GetRTCPSenderReportData(layer) - } - return nil -} - func (d *DummyReceiver) GetTrackStats() *livekit.RTPStats { if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { return r.GetTrackStats() diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 8c0a38494..d02e9de4d 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -317,7 +317,7 @@ func NewDownTrack(params DowntrackParams) (*DownTrack, error) { d.forwarder = NewForwarder( d.kind, params.Logger, - d.params.Receiver.GetReferenceLayerRTPTimestamp, + false, d.getExpectedRTPTimestamp, ) @@ -1306,8 +1306,8 @@ func (d *DownTrack) CreateSenderReport() *rtcp.SenderReport { return nil } - layer, tsOffset := d.forwarder.GetCurrentSpatialAndTSOffset() - return d.rtpStats.GetRtcpSenderReport(d.ssrc, d.params.Receiver.GetRTCPSenderReportData(layer), tsOffset) + _, tsOffset, refSenderReport := d.forwarder.GetSenderReportParams() + return d.rtpStats.GetRtcpSenderReport(d.ssrc, refSenderReport, tsOffset) } func (d *DownTrack) writeBlankFrameRTP(duration float32, generation uint32) chan struct{} { @@ -1950,9 +1950,11 @@ func (d *DownTrack) HandleRTCPSenderReportData( layer int32, publisherSRData *buffer.RTCPSenderReportData, ) error { - currentLayer, tsOffset := d.forwarder.GetCurrentSpatialAndTSOffset() + d.forwarder.SetRefSenderReport(isSVC, layer, publisherSRData) + + currentLayer, tsOffset, refSenderReport := d.forwarder.GetSenderReportParams() if layer == currentLayer || (layer == 0 && isSVC) { - d.handleRTCPSenderReportData(publisherSRData, tsOffset) + d.handleRTCPSenderReportData(refSenderReport, tsOffset) } return nil } @@ -2010,10 +2012,6 @@ func (d *DownTrack) sendingPacket(hdr *rtp.Header, payloadSize int, spmd *sendPa } if spmd.tp.isResuming { - // adjust first packet time on a resumption so that subsequent switches get a more accurate expected time stamp - currentLayer, tsOffset := d.forwarder.GetCurrentSpatialAndTSOffset() - d.handleRTCPSenderReportData(d.params.Receiver.GetRTCPSenderReportData(currentLayer), tsOffset) - if sal := d.getStreamAllocatorListener(); sal != nil { sal.OnResume(d) } diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index de50a829b..3dcf0e4f6 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -44,7 +44,7 @@ const ( TransitionCostSpatial = 10 ResumeBehindThresholdSeconds = float64(0.2) // 200ms - ResumeBehindHighTresholdSeconds = float64(2.0) // 2 seconds + ResumeBehindHighThresholdSeconds = float64(2.0) // 2 seconds LayerSwitchBehindThresholdSeconds = float64(0.05) // 50ms SwitchAheadThresholdSeconds = float64(0.025) // 25ms ) @@ -185,13 +185,12 @@ type TranslationParams struct { // ------------------------------------------------------------------- type ForwarderState struct { - Started bool - ReferenceLayerSpatial int32 - PreStartTime time.Time - ExtFirstTS uint64 - RefTSOffset uint64 - RTP RTPMungerState - Codec interface{} + Started bool + PreStartTime time.Time + ExtFirstTS uint64 + RefTSOffset uint64 + RTP RTPMungerState + Codec interface{} } func (f ForwarderState) String() string { @@ -200,9 +199,8 @@ func (f ForwarderState) String() string { case codecmunger.VP8State: codecString = codecState.String() } - return fmt.Sprintf("ForwarderState{started: %v, referenceLayerSpatial: %d, preStartTime: %s, extFirstTS: %d, refTSOffset: %d, rtp: %s, codec: %s}", + return fmt.Sprintf("ForwarderState{started: %v, preStartTime: %s, extFirstTS: %d, refTSOffset: %d, rtp: %s, codec: %s}", f.Started, - f.ReferenceLayerSpatial, f.PreStartTime.String(), f.ExtFirstTS, f.RefTSOffset, @@ -214,12 +212,12 @@ func (f ForwarderState) String() string { // ------------------------------------------------------------------- type Forwarder struct { - lock sync.RWMutex - codec webrtc.RTPCodecCapability - kind webrtc.RTPCodecType - logger logger.Logger - getReferenceLayerRTPTimestamp func(ts uint32, layer int32, referenceLayer int32) (uint32, error) - getExpectedRTPTimestamp func(at time.Time) (uint64, error) + lock sync.RWMutex + codec webrtc.RTPCodecCapability + kind webrtc.RTPCodecType + logger logger.Logger + skipReferenceTS bool + getExpectedRTPTimestamp func(at time.Time) (uint64, error) muted bool pubMuted bool @@ -231,6 +229,8 @@ type Forwarder struct { lastSSRC uint32 referenceLayerSpatial int32 refTSOffset uint64 + refSenderReports [buffer.DefaultMaxLayerSpatial + 1]*buffer.RTCPSenderReportData + refIsSVC bool provisional *VideoAllocationProvisional @@ -246,19 +246,19 @@ type Forwarder struct { func NewForwarder( kind webrtc.RTPCodecType, logger logger.Logger, - getReferenceLayerRTPTimestamp func(ts uint32, layer int32, referenceLayer int32) (uint32, error), + skipReferenceTS bool, getExpectedRTPTimestamp func(at time.Time) (uint64, error), ) *Forwarder { f := &Forwarder{ - kind: kind, - logger: logger, - getReferenceLayerRTPTimestamp: getReferenceLayerRTPTimestamp, - getExpectedRTPTimestamp: getExpectedRTPTimestamp, - referenceLayerSpatial: buffer.InvalidLayerSpatial, - lastAllocation: VideoAllocationDefault, - rtpMunger: NewRTPMunger(logger), - vls: videolayerselector.NewNull(logger), - codecMunger: codecmunger.NewNull(logger), + kind: kind, + logger: logger, + skipReferenceTS: skipReferenceTS, + getExpectedRTPTimestamp: getExpectedRTPTimestamp, + referenceLayerSpatial: buffer.InvalidLayerSpatial, + lastAllocation: VideoAllocationDefault, + rtpMunger: NewRTPMunger(logger), + vls: videolayerselector.NewNull(logger), + codecMunger: codecmunger.NewNull(logger), } if f.kind == webrtc.RTPCodecTypeVideo { @@ -375,13 +375,12 @@ func (f *Forwarder) GetState() ForwarderState { } return ForwarderState{ - Started: f.started, - ReferenceLayerSpatial: f.referenceLayerSpatial, - PreStartTime: f.preStartTime, - ExtFirstTS: f.extFirstTS, - RefTSOffset: f.refTSOffset, - RTP: f.rtpMunger.GetLast(), - Codec: f.codecMunger.GetState(), + Started: f.started, + PreStartTime: f.preStartTime, + ExtFirstTS: f.extFirstTS, + RefTSOffset: f.refTSOffset, + RTP: f.rtpMunger.GetLast(), + Codec: f.codecMunger.GetState(), } } @@ -397,7 +396,6 @@ func (f *Forwarder) SeedState(state ForwarderState) { f.codecMunger.SeedState(state.Codec) f.started = true - f.referenceLayerSpatial = state.ReferenceLayerSpatial f.preStartTime = state.PreStartTime f.extFirstTS = state.ExtFirstTS f.refTSOffset = state.RefTSOffset @@ -556,15 +554,72 @@ func (f *Forwarder) GetMaxSubscribedSpatial() int32 { return layer } -func (f *Forwarder) GetCurrentSpatialAndTSOffset() (int32, uint64) { +func (f *Forwarder) SetRefSenderReport(isSVC bool, layer int32, srData *buffer.RTCPSenderReportData) { + f.lock.Lock() + defer f.lock.Unlock() + + f.refIsSVC = isSVC + if isSVC { + layer = 0 + } + if layer >= 0 && int(layer) < len(f.refSenderReports) { + f.refSenderReports[layer] = srData + } +} + +func (f *Forwarder) clearRefSenderReportsLocked() { + // On (re)start of fowarding, clear any old publisher sender reports. + // This is done to prevent use of potentially stale publisher sender reports. + // + // It is possible to implement mute using pause/unpause + // which can implemented using a replaceTrack(null)/replaceTrack(track). + // In those cases, the RTP time stamp may not jump across + // the mute/pause valley (for the time it is replaced with null track). + // So, relying on a report that happened before unmute/unpause + // could result in incorrect RTCP sender report on subscriber side. + // + // It could happen like this + // 1. Normal operation: publisher sending sender reports and + // suscribers use reports from publisher to calculate and send + // RTCP sender report. + // 2. Publisher pauses: there are no more reports. + // 3. When paused, subscriber can still use the publisher side sender + // report to send reports. Although the time since last publisher + // sender report is increasing, the reports are correct though. + // 4. Publisher unpauses after 20 seconds. But, it may not have advanced + // RTP Timestamp by that much. Let us say, it advances only by 5 seconds. + // 5. When subscriber starts forwarding packets, it will calculate + // a new time stamp offset to adjust to the new time stamp of publisher. + // 6. But, when that same offset is used on an old publisher sender report + // (i. e. a report from before the pause), the subscriber side sender + // reports jumps ahead in time by 15 seconds. + // + // By clearing sender report on (re)start of a stream, subscribers will wait for a fresh report + // after unmute to send sender report. + for layer := int32(0); layer < buffer.DefaultMaxLayerSpatial+1; layer++ { + f.refSenderReports[layer] = nil + } +} + +func (f *Forwarder) GetSenderReportParams() (int32, uint64, *buffer.RTCPSenderReportData) { f.lock.RLock() defer f.lock.RUnlock() if f.kind == webrtc.RTPCodecTypeAudio { - return 0, f.rtpMunger.GetPinnedTSOffset() + return 0, f.rtpMunger.GetPinnedTSOffset(), f.refSenderReports[0] } - return f.vls.GetCurrent().Spatial, f.rtpMunger.GetPinnedTSOffset() + currentLayerSpatial := f.vls.GetCurrent().Spatial + if currentLayerSpatial < 0 || currentLayerSpatial > buffer.DefaultMaxLayerSpatial { + return currentLayerSpatial, f.rtpMunger.GetPinnedTSOffset(), nil + } + + refSenderReport := f.refSenderReports[currentLayerSpatial] + if f.refIsSVC { + refSenderReport = f.refSenderReports[0] + } + + return currentLayerSpatial, f.rtpMunger.GetPinnedTSOffset(), refSenderReport } func (f *Forwarder) isDeficientLocked() bool { @@ -1420,6 +1475,7 @@ func (f *Forwarder) resyncLocked() { if f.pubMuted { f.resumeBehindThreshold = ResumeBehindThresholdSeconds } + f.referenceLayerSpatial = buffer.InvalidLayerSpatial } func (f *Forwarder) CheckSync() (bool, int32) { @@ -1481,12 +1537,32 @@ func (f *Forwarder) GetTranslationParams(extPkt *buffer.ExtPacket, layer int32) }, ErrUnknownKind } +func (f *Forwarder) getReferenceLayerRTPTimestamp(ts uint32, refLayer, targetLayer int32) (uint32, error) { + srRef := f.refSenderReports[refLayer] + srTarget := f.refSenderReports[targetLayer] + if srRef == nil || srRef.NTPTimestamp == 0 || srTarget == nil || srTarget.NTPTimestamp == 0 { + return 0, fmt.Errorf("invalid layer(s), refLayer: %d, targetLayer: %d", refLayer, targetLayer) + } + + ntpDiff := srRef.NTPTimestamp.Time().Sub(srTarget.NTPTimestamp.Time()) + rtpDiff := ntpDiff.Nanoseconds() * int64(f.codec.ClockRate) / 1e9 + + // calculate other layer's time stamp at the same time as ref layer's NTP time + normalizedOtherTS := srTarget.RTPTimestamp + uint32(rtpDiff) + + // now both layers' time stamp refer to the same NTP time and the diff is the offset between the layers + offset := srRef.RTPTimestamp - normalizedOtherTS + + return ts + offset, nil +} + func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) error { if !f.started { f.started = true f.referenceLayerSpatial = layer f.rtpMunger.SetLastSnTs(extPkt) f.codecMunger.SetLast(extPkt) + f.clearRefSenderReportsLocked() f.logger.Debugw( "starting forwarding", "sequenceNumber", extPkt.Packet.SequenceNumber, @@ -1535,27 +1611,29 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e extLastTS := rtpMungerState.ExtLastTS extExpectedTS := extLastTS extRefTS := extExpectedTS + refTS := uint32(extRefTS) switchingAt := time.Now() - if f.getReferenceLayerRTPTimestamp != nil { - ts, err := f.getReferenceLayerRTPTimestamp(extPkt.Packet.Timestamp, layer, f.referenceLayerSpatial) + if !f.skipReferenceTS { + var err error + refTS, err = f.getReferenceLayerRTPTimestamp(extPkt.Packet.Timestamp, layer, f.referenceLayerSpatial) if err != nil { - // error out if extRefTS is not available. It can happen when there is no sender report + // error out if refTS is not available. It can happen when there is no sender report // for the layer being switched to. Can especially happen at the start of the track when layer switches are // potentially happening very quickly. Erroring out and waiting for a layer for which a sender report has been // received will calculate a better offset, but may result in initial adaptation to take a bit longer depending // on how often publisher/remote side sends RTCP sender report. return err } + } - extRefTS = (extRefTS & 0xFFFF_FFFF_0000_0000) + uint64(ts) + extRefTS = (extRefTS & 0xFFFF_FFFF_0000_0000) + uint64(refTS) - expectedTS32 := uint32(extExpectedTS) - if (ts-expectedTS32) < 1<<31 && ts < expectedTS32 { - extRefTS += (1 << 32) - } - if (expectedTS32-ts) < 1<<31 && expectedTS32 < ts && extRefTS >= 1<<32 { - extRefTS -= (1 << 32) - } + expectedTS := uint32(extExpectedTS) + if (refTS-expectedTS) < 1<<31 && refTS < expectedTS { + extRefTS += (1 << 32) + } + if (expectedTS-refTS) < 1<<31 && expectedTS < refTS && extRefTS >= 1<<32 { + extRefTS -= (1 << 32) } if f.getExpectedRTPTimestamp != nil { @@ -1611,7 +1689,7 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e if f.resumeBehindThreshold > 0 && diffSeconds > f.resumeBehindThreshold { logTransition("resume, reference too far behind", extExpectedTS, extRefTS, extLastTS, diffSeconds) extNextTS = extExpectedTS - } else if diffSeconds > ResumeBehindHighTresholdSeconds { + } else if diffSeconds > ResumeBehindHighThresholdSeconds { // could be due to incorrect reference calculation logTransition("resume, reference very far behind", extExpectedTS, extRefTS, extLastTS, diffSeconds) extNextTS = extExpectedTS @@ -1625,6 +1703,7 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e extNextTS = extRefTS } f.resumeBehindThreshold = 0.0 + f.clearRefSenderReportsLocked() } else { // switching between layers, check if extRefTS is too far behind the last sent diffSeconds := float64(int64(extRefTS-extLastTS)) / float64(f.codec.ClockRate) diff --git a/pkg/sfu/forwarder_test.go b/pkg/sfu/forwarder_test.go index aaecb03b8..9279bc736 100644 --- a/pkg/sfu/forwarder_test.go +++ b/pkg/sfu/forwarder_test.go @@ -32,7 +32,7 @@ func disable(f *Forwarder) { } func newForwarder(codec webrtc.RTPCodecCapability, kind webrtc.RTPCodecType) *Forwarder { - f := NewForwarder(kind, logger.GetLogger(), nil, nil) + f := NewForwarder(kind, logger.GetLogger(), true, nil) f.DetermineCodec(codec, nil) return f } diff --git a/pkg/sfu/receiver.go b/pkg/sfu/receiver.go index 2192c6a66..e1bebcd21 100644 --- a/pkg/sfu/receiver.go +++ b/pkg/sfu/receiver.go @@ -83,9 +83,6 @@ type TrackReceiver interface { GetTemporalLayerFpsForSpatial(layer int32) []float32 - GetReferenceLayerRTPTimestamp(ts uint32, layer int32, referenceLayer int32) (uint32, error) - GetRTCPSenderReportData(layer int32) *buffer.RTCPSenderReportData - GetTrackStats() *livekit.RTPStats } @@ -350,8 +347,6 @@ func (w *WebRTCReceiver) AddUpTrack(track *webrtc.TrackRemote, buff *buffer.Buff buff.OnRtcpFeedback(w.sendRTCP) buff.OnRtcpSenderReport(func() { srData := buff.GetSenderReportData() - w.streamTrackerManager.SetRTCPSenderReportData(layer, srData) - w.downTrackSpreader.Broadcast(func(dt TrackSender) { _ = dt.HandleRTCPSenderReportData(w.codec.PayloadType, w.isSVC, layer, srData) }) @@ -806,14 +801,6 @@ func (w *WebRTCReceiver) GetTemporalLayerFpsForSpatial(layer int32) []float32 { return b.GetTemporalLayerFpsForSpatial(layer) } -func (w *WebRTCReceiver) GetReferenceLayerRTPTimestamp(ts uint32, layer int32, referenceLayer int32) (uint32, error) { - return w.streamTrackerManager.GetReferenceLayerRTPTimestamp(ts, layer, referenceLayer) -} - -func (w *WebRTCReceiver) GetRTCPSenderReportData(layer int32) *buffer.RTCPSenderReportData { - return w.streamTrackerManager.GetRTCPSenderReportData(layer) -} - // closes all track senders in parallel, returns when all are closed func closeTrackSenders(senders []TrackSender) { wg := sync.WaitGroup{} diff --git a/pkg/sfu/streamtrackermanager.go b/pkg/sfu/streamtrackermanager.go index b5f36dc97..0158c55c7 100644 --- a/pkg/sfu/streamtrackermanager.go +++ b/pkg/sfu/streamtrackermanager.go @@ -15,8 +15,6 @@ package sfu import ( - "fmt" - "math" "sort" "sync" "time" @@ -33,10 +31,6 @@ import ( "github.com/livekit/livekit-server/pkg/sfu/streamtracker" ) -const ( - senderReportThresholdSeconds = float64(60.0) -) - // --------------------------------------------------- type StreamTrackerManagerListener interface { @@ -69,10 +63,6 @@ type StreamTrackerManager struct { maxExpectedLayer int32 paused bool - senderReportMu sync.RWMutex - senderReports [buffer.DefaultMaxLayerSpatial + 1]*buffer.RTCPSenderReportData - layerOffsets [buffer.DefaultMaxLayerSpatial + 1][buffer.DefaultMaxLayerSpatial + 1]uint32 - closed core.Fuse listener StreamTrackerManagerListener @@ -550,110 +540,6 @@ func (s *StreamTrackerManager) maxExpectedLayerFromTrackInfo() { } } -func (s *StreamTrackerManager) updateLayerOffsetLocked(ref, other int32) { - srRef := s.senderReports[ref] - srOther := s.senderReports[other] - if srRef == nil || srRef.NTPTimestamp == 0 || srOther == nil || srOther.NTPTimestamp == 0 { - return - } - - ntpDiff := srRef.NTPTimestamp.Time().Sub(srOther.NTPTimestamp.Time()) - if math.Abs(ntpDiff.Seconds()) > senderReportThresholdSeconds { - // offset is updated only if the layers' sender reports are close enough. - // - // Rationale: higher layers could be paused for extended periods of time - // due to adaptive stream/dynacast or publisher constraints like CPU/bandwidth. - // The check is to avoid using very old reports. - return - } - rtpDiff := ntpDiff.Nanoseconds() * int64(s.clockRate) / 1e9 - - // calculate other layer's time stamp at the same time as ref layer's NTP time - normalizedOtherTS := srOther.RTPTimestamp + uint32(rtpDiff) - - // now both layers' time stamp refer to the same NTP time and the diff is the offset between the layers - offset := srRef.RTPTimestamp - normalizedOtherTS - - // use minimal offset to indicate value availability in the extremely unlikely case of - // both layers using the same timestamp - if offset == 0 { - s.logger.Debugw( - "using default offset", - "ref", ref, - "refNTP", srRef.NTPTimestamp.Time().String(), - "refRTP", srRef.RTPTimestamp, - "other", other, - "otherNTP", srOther.NTPTimestamp.Time().String(), - "otherRTP", srOther.RTPTimestamp, - ) - offset = 1 - } - - s.layerOffsets[ref][other] = offset -} - -func (s *StreamTrackerManager) SetRTCPSenderReportData(layer int32, srData *buffer.RTCPSenderReportData) { - s.senderReportMu.Lock() - defer s.senderReportMu.Unlock() - - if layer < 0 || int(layer) >= len(s.senderReports) { - return - } - - s.senderReports[layer] = srData - - // (re)fill offsets as necessary for received layer. - for i := int32(0); i < buffer.DefaultMaxLayerSpatial+1; i++ { - if i == layer { - continue - } - - // treating layer for which report was received as reference layer - s.updateLayerOffsetLocked(layer, i) - - // and the other way - s.updateLayerOffsetLocked(i, layer) - } -} - -func (s *StreamTrackerManager) GetRTCPSenderReportData(layer int32) *buffer.RTCPSenderReportData { - s.senderReportMu.Lock() - defer s.senderReportMu.Unlock() - - if layer < 0 || int(layer) >= len(s.senderReports) { - return nil - } - - // SVC-TODO: better SVC detection - if s.isSVC { - // there is only one stream in SVC - layer = 0 - } - - return s.senderReports[layer] -} - -func (s *StreamTrackerManager) GetReferenceLayerRTPTimestamp(ts uint32, layer int32, referenceLayer int32) (uint32, error) { - s.senderReportMu.RLock() - defer s.senderReportMu.RUnlock() - - if layer < 0 || int(layer) >= len(s.layerOffsets[0]) || referenceLayer < 0 || int(referenceLayer) >= len(s.layerOffsets) { - return 0, fmt.Errorf("invalid layer, target: %d, reference: %d", layer, referenceLayer) - } - - // SVC-TODO: better SVC detection - if s.isSVC { - // there is only one stream in SVC - return ts, nil - } - - if layer != referenceLayer && s.layerOffsets[referenceLayer][layer] == 0 { - return 0, fmt.Errorf("offset unavailable, target: %d, reference: %d", layer, referenceLayer) - } - - return ts + s.layerOffsets[referenceLayer][layer], nil -} - func (s *StreamTrackerManager) GetMaxTemporalLayerSeen() int32 { s.lock.RLock() defer s.lock.RUnlock() From d3f0436d25fd835fd9af52c0b3b3da6600bee18c Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 4 Apr 2024 21:09:11 +0530 Subject: [PATCH 25/78] Fix reference time stamp layer. (#2623) - Had arguments reversed. - Also, cannot take away reference layer from state as a new layer as reference could have a time stamp that is widely different from expected. So, put that back. --- pkg/sfu/forwarder.go | 49 +++++++++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 3dcf0e4f6..721dd6c9e 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -185,12 +185,13 @@ type TranslationParams struct { // ------------------------------------------------------------------- type ForwarderState struct { - Started bool - PreStartTime time.Time - ExtFirstTS uint64 - RefTSOffset uint64 - RTP RTPMungerState - Codec interface{} + Started bool + ReferenceLayerSpatial int32 + PreStartTime time.Time + ExtFirstTS uint64 + RefTSOffset uint64 + RTP RTPMungerState + Codec interface{} } func (f ForwarderState) String() string { @@ -199,8 +200,9 @@ func (f ForwarderState) String() string { case codecmunger.VP8State: codecString = codecState.String() } - return fmt.Sprintf("ForwarderState{started: %v, preStartTime: %s, extFirstTS: %d, refTSOffset: %d, rtp: %s, codec: %s}", + return fmt.Sprintf("ForwarderState{started: %v, referenceLayerSpatial: %d, preStartTime: %s, extFirstTS: %d, refTSOffset: %d, rtp: %s, codec: %s}", f.Started, + f.ReferenceLayerSpatial, f.PreStartTime.String(), f.ExtFirstTS, f.RefTSOffset, @@ -375,12 +377,13 @@ func (f *Forwarder) GetState() ForwarderState { } return ForwarderState{ - Started: f.started, - PreStartTime: f.preStartTime, - ExtFirstTS: f.extFirstTS, - RefTSOffset: f.refTSOffset, - RTP: f.rtpMunger.GetLast(), - Codec: f.codecMunger.GetState(), + Started: f.started, + ReferenceLayerSpatial: f.referenceLayerSpatial, + PreStartTime: f.preStartTime, + ExtFirstTS: f.extFirstTS, + RefTSOffset: f.refTSOffset, + RTP: f.rtpMunger.GetLast(), + Codec: f.codecMunger.GetState(), } } @@ -396,6 +399,7 @@ func (f *Forwarder) SeedState(state ForwarderState) { f.codecMunger.SeedState(state.Codec) f.started = true + f.referenceLayerSpatial = state.ReferenceLayerSpatial f.preStartTime = state.PreStartTime f.extFirstTS = state.ExtFirstTS f.refTSOffset = state.RefTSOffset @@ -1475,7 +1479,6 @@ func (f *Forwarder) resyncLocked() { if f.pubMuted { f.resumeBehindThreshold = ResumeBehindThresholdSeconds } - f.referenceLayerSpatial = buffer.InvalidLayerSpatial } func (f *Forwarder) CheckSync() (bool, int32) { @@ -1538,10 +1541,18 @@ func (f *Forwarder) GetTranslationParams(extPkt *buffer.ExtPacket, layer int32) } func (f *Forwarder) getReferenceLayerRTPTimestamp(ts uint32, refLayer, targetLayer int32) (uint32, error) { + if refLayer < 0 || int(refLayer) > len(f.refSenderReports) || targetLayer < 0 || int(targetLayer) > len(f.refSenderReports) { + return 0, fmt.Errorf("invalid layer(s), refLayer: %d, targetLayer: %d", refLayer, targetLayer) + } + + if refLayer == targetLayer || f.refIsSVC { + return ts, nil + } + srRef := f.refSenderReports[refLayer] srTarget := f.refSenderReports[targetLayer] if srRef == nil || srRef.NTPTimestamp == 0 || srTarget == nil || srTarget.NTPTimestamp == 0 { - return 0, fmt.Errorf("invalid layer(s), refLayer: %d, targetLayer: %d", refLayer, targetLayer) + return 0, fmt.Errorf("unavailable layer(s), refLayer: %d, targetLayer: %d", refLayer, targetLayer) } ntpDiff := srRef.NTPTimestamp.Time().Sub(srTarget.NTPTimestamp.Time()) @@ -1615,7 +1626,7 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e switchingAt := time.Now() if !f.skipReferenceTS { var err error - refTS, err = f.getReferenceLayerRTPTimestamp(extPkt.Packet.Timestamp, layer, f.referenceLayerSpatial) + refTS, err = f.getReferenceLayerRTPTimestamp(extPkt.Packet.Timestamp, f.referenceLayerSpatial, layer) if err != nil { // error out if refTS is not available. It can happen when there is no sender report // for the layer being switched to. Can especially happen at the start of the track when layer switches are @@ -1703,6 +1714,12 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e extNextTS = extRefTS } f.resumeBehindThreshold = 0.0 + + // sender reports are cleared after calculating switch time stamp + // as relative differences between layers should remain the same. + // TODO: If the relative difference changes a lot, probably have to + // abandon the checks above and just use the expected timestamp + // as the next time stamp. f.clearRefSenderReportsLocked() } else { // switching between layers, check if extRefTS is too far behind the last sent From e93611eafa016edaa5cc9d1b2a3c88ccfb6ac8b0 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 5 Apr 2024 18:21:38 +0530 Subject: [PATCH 26/78] Log sender reports. (#2625) --- pkg/sfu/buffer/rtpstats_sender.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pkg/sfu/buffer/rtpstats_sender.go b/pkg/sfu/buffer/rtpstats_sender.go index b85efb829..af917cd9e 100644 --- a/pkg/sfu/buffer/rtpstats_sender.go +++ b/pkg/sfu/buffer/rtpstats_sender.go @@ -306,6 +306,8 @@ func (r *RTPStatsSender) Update( "hdrSize", hdrSize, "payloadSize", payloadSize, "paddingSize", paddingSize, + "firstSR", r.srFirst, + "lastSR", r.srNewest, ) } @@ -372,6 +374,8 @@ func (r *RTPStatsSender) Update( "hdrSize", hdrSize, "payloadSize", payloadSize, "paddingSize", paddingSize, + "firstSR", r.srFirst, + "lastSR", r.srNewest, ) } @@ -466,6 +470,8 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt "lastRR", r.lastRR, "sinceLastRR", time.Since(r.lastRRTime).String(), "receivedRR", rr, + "firstSR", r.srFirst, + "lastSR", r.srNewest, ) return } @@ -545,6 +551,8 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt "packetsInInterval", extReceivedRRSN-s.extLastRRSN, "extHighestSNFromRR", r.extHighestSNFromRR, "packetsLostFromRR", r.packetsLostFromRR, + "firstSR", r.srFirst, + "lastSR", r.srNewest, ) continue } @@ -580,6 +588,8 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt "extHighestSNFromRR", r.extHighestSNFromRR, "packetsLostFromRR", r.packetsLostFromRR, "count", r.metadataCacheOverflowCount, + "firstSR", r.srFirst, + "lastSR", r.srNewest, ) } r.metadataCacheOverflowCount++ @@ -736,6 +746,8 @@ func (r *RTPStatsSender) DeltaInfoSender(senderSnapshotID uint32) *RTPDeltaInfo "startTime", startTime.String(), "endTime", endTime.String(), "duration", endTime.Sub(startTime).String(), + "firstSR", r.srFirst, + "lastSR", r.srNewest, ) return nil } From fff937a89c1db6333219fe7f7d5accc0200d1908 Mon Sep 17 00:00:00 2001 From: David Colburn Date: Fri, 5 Apr 2024 12:59:06 -0700 Subject: [PATCH 27/78] participant kinds (#2626) --- pkg/rtc/participant.go | 33 ++- pkg/rtc/room.go | 16 +- pkg/rtc/types/interfaces.go | 5 +- .../typesfakes/fake_local_participant.go | 260 +++++++++--------- pkg/rtc/types/typesfakes/fake_participant.go | 150 +++++----- pkg/service/wire_gen.go | 2 +- 6 files changed, 235 insertions(+), 231 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 56ab3ea7e..090930875 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -325,6 +325,25 @@ func (p *ParticipantImpl) State() livekit.ParticipantInfo_State { return p.state.Load().(livekit.ParticipantInfo_State) } +func (p *ParticipantImpl) Kind() livekit.ParticipantInfo_Kind { + p.lock.RLock() + defer p.lock.RUnlock() + + return p.grants.GetParticipantKind() +} + +func (p *ParticipantImpl) IsDependent() bool { + p.lock.RLock() + defer p.lock.RUnlock() + + switch p.grants.GetParticipantKind() { + case livekit.ParticipantInfo_AGENT, livekit.ParticipantInfo_EGRESS: + return true + default: + return false + } +} + func (p *ParticipantImpl) ProtocolVersion() types.ProtocolVersion { return p.params.ProtocolVersion } @@ -1090,20 +1109,6 @@ func (p *ParticipantImpl) Hidden() bool { return p.hidden.Load() } -func (p *ParticipantImpl) IsRecorder() bool { - p.lock.RLock() - defer p.lock.RUnlock() - - return p.grants.Video.Recorder -} - -func (p *ParticipantImpl) IsAgent() bool { - p.lock.RLock() - defer p.lock.RUnlock() - - return p.grants.Video.Agent -} - func (p *ParticipantImpl) VerifySubscribeParticipantInfo(pID livekit.ParticipantID, version uint32) { if !p.IsReady() { // we have not sent a JoinResponse yet. metadata would be covered in JoinResponse diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index 900c415f8..ad3fc5d18 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -318,10 +318,10 @@ func (r *Room) Join(participant types.LocalParticipant, requestSource routing.Me if r.participants[participant.Identity()] != nil { return ErrAlreadyJoined } - if r.protoRoom.MaxParticipants > 0 && !participant.IsRecorder() { + if r.protoRoom.MaxParticipants > 0 && !participant.IsDependent() { numParticipants := uint32(0) for _, p := range r.participants { - if !p.IsRecorder() { + if !p.IsDependent() { numParticipants++ } } @@ -417,7 +417,7 @@ func (r *Room) Join(participant types.LocalParticipant, requestSource routing.Me "numParticipants", len(r.participants), ) - if participant.IsRecorder() && !r.protoRoom.ActiveRecording { + if participant.Kind() == livekit.ParticipantInfo_EGRESS && !r.protoRoom.ActiveRecording { r.protoRoom.ActiveRecording = true r.protoProxy.MarkDirty(true) } else { @@ -559,10 +559,10 @@ func (r *Room) RemoveParticipant(identity livekit.ParticipantIdentity, pID livek } immediateChange := false - if p.IsRecorder() { + if p.Kind() == livekit.ParticipantInfo_EGRESS { activeRecording := false for _, op := range r.participants { - if op.IsRecorder() { + if op.Kind() == livekit.ParticipantInfo_EGRESS { activeRecording = true break } @@ -750,7 +750,7 @@ func (r *Room) CloseIfEmpty() { } for _, p := range r.participants { - if !p.IsRecorder() { + if !p.IsDependent() { r.lock.Unlock() return } @@ -1226,7 +1226,7 @@ func (r *Room) updateProto() *livekit.Room { room.NumPublishers = 0 room.NumParticipants = 0 for _, p := range r.GetParticipants() { - if !p.IsRecorder() { + if !p.IsDependent() { room.NumParticipants++ } if p.IsPublisher() { @@ -1410,7 +1410,7 @@ func (r *Room) simulationCleanupWorker() { } func (r *Room) launchPublisherAgent(p types.Participant) { - if p == nil || p.IsRecorder() || p.IsAgent() || r.agentClient == nil { + if p == nil || p.IsDependent() || r.agentClient == nil { return } diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index 967e63960..ebbfe9620 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -245,6 +245,8 @@ type Participant interface { Identity() livekit.ParticipantIdentity State() livekit.ParticipantInfo_State CloseReason() ParticipantCloseReason + Kind() livekit.ParticipantInfo_Kind + IsDependent() bool CanSkipBroadcast() bool ToProto() *livekit.ParticipantInfo @@ -265,8 +267,6 @@ type Participant interface { // permissions Hidden() bool - IsRecorder() bool - IsAgent() bool Close(sendLeave bool, reason ParticipantCloseReason, isExpectedToResume bool) error @@ -586,4 +586,3 @@ type OperationMonitor interface { Check() error IsIdle() bool } - diff --git a/pkg/rtc/types/typesfakes/fake_local_participant.go b/pkg/rtc/types/typesfakes/fake_local_participant.go index 37c03ef10..21f433338 100644 --- a/pkg/rtc/types/typesfakes/fake_local_participant.go +++ b/pkg/rtc/types/typesfakes/fake_local_participant.go @@ -444,16 +444,6 @@ type FakeLocalParticipant struct { identityReturnsOnCall map[int]struct { result1 livekit.ParticipantIdentity } - IsAgentStub func() bool - isAgentMutex sync.RWMutex - isAgentArgsForCall []struct { - } - isAgentReturns struct { - result1 bool - } - isAgentReturnsOnCall map[int]struct { - result1 bool - } IsClosedStub func() bool isClosedMutex sync.RWMutex isClosedArgsForCall []struct { @@ -464,6 +454,16 @@ type FakeLocalParticipant struct { isClosedReturnsOnCall map[int]struct { result1 bool } + IsDependentStub func() bool + isDependentMutex sync.RWMutex + isDependentArgsForCall []struct { + } + isDependentReturns struct { + result1 bool + } + isDependentReturnsOnCall map[int]struct { + result1 bool + } IsDisconnectedStub func() bool isDisconnectedMutex sync.RWMutex isDisconnectedArgsForCall []struct { @@ -504,16 +504,6 @@ type FakeLocalParticipant struct { isReadyReturnsOnCall map[int]struct { result1 bool } - IsRecorderStub func() bool - isRecorderMutex sync.RWMutex - isRecorderArgsForCall []struct { - } - isRecorderReturns struct { - result1 bool - } - isRecorderReturnsOnCall map[int]struct { - result1 bool - } IsSubscribedToStub func(livekit.ParticipantID) bool isSubscribedToMutex sync.RWMutex isSubscribedToArgsForCall []struct { @@ -530,6 +520,16 @@ type FakeLocalParticipant struct { issueFullReconnectArgsForCall []struct { arg1 types.ParticipantCloseReason } + KindStub func() livekit.ParticipantInfo_Kind + kindMutex sync.RWMutex + kindArgsForCall []struct { + } + kindReturns struct { + result1 livekit.ParticipantInfo_Kind + } + kindReturnsOnCall map[int]struct { + result1 livekit.ParticipantInfo_Kind + } MaybeStartMigrationStub func(bool, func()) bool maybeStartMigrationMutex sync.RWMutex maybeStartMigrationArgsForCall []struct { @@ -3233,59 +3233,6 @@ func (fake *FakeLocalParticipant) IdentityReturnsOnCall(i int, result1 livekit.P }{result1} } -func (fake *FakeLocalParticipant) IsAgent() bool { - fake.isAgentMutex.Lock() - ret, specificReturn := fake.isAgentReturnsOnCall[len(fake.isAgentArgsForCall)] - fake.isAgentArgsForCall = append(fake.isAgentArgsForCall, struct { - }{}) - stub := fake.IsAgentStub - fakeReturns := fake.isAgentReturns - fake.recordInvocation("IsAgent", []interface{}{}) - fake.isAgentMutex.Unlock() - if stub != nil { - return stub() - } - if specificReturn { - return ret.result1 - } - return fakeReturns.result1 -} - -func (fake *FakeLocalParticipant) IsAgentCallCount() int { - fake.isAgentMutex.RLock() - defer fake.isAgentMutex.RUnlock() - return len(fake.isAgentArgsForCall) -} - -func (fake *FakeLocalParticipant) IsAgentCalls(stub func() bool) { - fake.isAgentMutex.Lock() - defer fake.isAgentMutex.Unlock() - fake.IsAgentStub = stub -} - -func (fake *FakeLocalParticipant) IsAgentReturns(result1 bool) { - fake.isAgentMutex.Lock() - defer fake.isAgentMutex.Unlock() - fake.IsAgentStub = nil - fake.isAgentReturns = struct { - result1 bool - }{result1} -} - -func (fake *FakeLocalParticipant) IsAgentReturnsOnCall(i int, result1 bool) { - fake.isAgentMutex.Lock() - defer fake.isAgentMutex.Unlock() - fake.IsAgentStub = nil - if fake.isAgentReturnsOnCall == nil { - fake.isAgentReturnsOnCall = make(map[int]struct { - result1 bool - }) - } - fake.isAgentReturnsOnCall[i] = struct { - result1 bool - }{result1} -} - func (fake *FakeLocalParticipant) IsClosed() bool { fake.isClosedMutex.Lock() ret, specificReturn := fake.isClosedReturnsOnCall[len(fake.isClosedArgsForCall)] @@ -3339,6 +3286,59 @@ func (fake *FakeLocalParticipant) IsClosedReturnsOnCall(i int, result1 bool) { }{result1} } +func (fake *FakeLocalParticipant) IsDependent() bool { + fake.isDependentMutex.Lock() + ret, specificReturn := fake.isDependentReturnsOnCall[len(fake.isDependentArgsForCall)] + fake.isDependentArgsForCall = append(fake.isDependentArgsForCall, struct { + }{}) + stub := fake.IsDependentStub + fakeReturns := fake.isDependentReturns + fake.recordInvocation("IsDependent", []interface{}{}) + fake.isDependentMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) IsDependentCallCount() int { + fake.isDependentMutex.RLock() + defer fake.isDependentMutex.RUnlock() + return len(fake.isDependentArgsForCall) +} + +func (fake *FakeLocalParticipant) IsDependentCalls(stub func() bool) { + fake.isDependentMutex.Lock() + defer fake.isDependentMutex.Unlock() + fake.IsDependentStub = stub +} + +func (fake *FakeLocalParticipant) IsDependentReturns(result1 bool) { + fake.isDependentMutex.Lock() + defer fake.isDependentMutex.Unlock() + fake.IsDependentStub = nil + fake.isDependentReturns = struct { + result1 bool + }{result1} +} + +func (fake *FakeLocalParticipant) IsDependentReturnsOnCall(i int, result1 bool) { + fake.isDependentMutex.Lock() + defer fake.isDependentMutex.Unlock() + fake.IsDependentStub = nil + if fake.isDependentReturnsOnCall == nil { + fake.isDependentReturnsOnCall = make(map[int]struct { + result1 bool + }) + } + fake.isDependentReturnsOnCall[i] = struct { + result1 bool + }{result1} +} + func (fake *FakeLocalParticipant) IsDisconnected() bool { fake.isDisconnectedMutex.Lock() ret, specificReturn := fake.isDisconnectedReturnsOnCall[len(fake.isDisconnectedArgsForCall)] @@ -3551,59 +3551,6 @@ func (fake *FakeLocalParticipant) IsReadyReturnsOnCall(i int, result1 bool) { }{result1} } -func (fake *FakeLocalParticipant) IsRecorder() bool { - fake.isRecorderMutex.Lock() - ret, specificReturn := fake.isRecorderReturnsOnCall[len(fake.isRecorderArgsForCall)] - fake.isRecorderArgsForCall = append(fake.isRecorderArgsForCall, struct { - }{}) - stub := fake.IsRecorderStub - fakeReturns := fake.isRecorderReturns - fake.recordInvocation("IsRecorder", []interface{}{}) - fake.isRecorderMutex.Unlock() - if stub != nil { - return stub() - } - if specificReturn { - return ret.result1 - } - return fakeReturns.result1 -} - -func (fake *FakeLocalParticipant) IsRecorderCallCount() int { - fake.isRecorderMutex.RLock() - defer fake.isRecorderMutex.RUnlock() - return len(fake.isRecorderArgsForCall) -} - -func (fake *FakeLocalParticipant) IsRecorderCalls(stub func() bool) { - fake.isRecorderMutex.Lock() - defer fake.isRecorderMutex.Unlock() - fake.IsRecorderStub = stub -} - -func (fake *FakeLocalParticipant) IsRecorderReturns(result1 bool) { - fake.isRecorderMutex.Lock() - defer fake.isRecorderMutex.Unlock() - fake.IsRecorderStub = nil - fake.isRecorderReturns = struct { - result1 bool - }{result1} -} - -func (fake *FakeLocalParticipant) IsRecorderReturnsOnCall(i int, result1 bool) { - fake.isRecorderMutex.Lock() - defer fake.isRecorderMutex.Unlock() - fake.IsRecorderStub = nil - if fake.isRecorderReturnsOnCall == nil { - fake.isRecorderReturnsOnCall = make(map[int]struct { - result1 bool - }) - } - fake.isRecorderReturnsOnCall[i] = struct { - result1 bool - }{result1} -} - func (fake *FakeLocalParticipant) IsSubscribedTo(arg1 livekit.ParticipantID) bool { fake.isSubscribedToMutex.Lock() ret, specificReturn := fake.isSubscribedToReturnsOnCall[len(fake.isSubscribedToArgsForCall)] @@ -3697,6 +3644,59 @@ func (fake *FakeLocalParticipant) IssueFullReconnectArgsForCall(i int) types.Par return argsForCall.arg1 } +func (fake *FakeLocalParticipant) Kind() livekit.ParticipantInfo_Kind { + fake.kindMutex.Lock() + ret, specificReturn := fake.kindReturnsOnCall[len(fake.kindArgsForCall)] + fake.kindArgsForCall = append(fake.kindArgsForCall, struct { + }{}) + stub := fake.KindStub + fakeReturns := fake.kindReturns + fake.recordInvocation("Kind", []interface{}{}) + fake.kindMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) KindCallCount() int { + fake.kindMutex.RLock() + defer fake.kindMutex.RUnlock() + return len(fake.kindArgsForCall) +} + +func (fake *FakeLocalParticipant) KindCalls(stub func() livekit.ParticipantInfo_Kind) { + fake.kindMutex.Lock() + defer fake.kindMutex.Unlock() + fake.KindStub = stub +} + +func (fake *FakeLocalParticipant) KindReturns(result1 livekit.ParticipantInfo_Kind) { + fake.kindMutex.Lock() + defer fake.kindMutex.Unlock() + fake.KindStub = nil + fake.kindReturns = struct { + result1 livekit.ParticipantInfo_Kind + }{result1} +} + +func (fake *FakeLocalParticipant) KindReturnsOnCall(i int, result1 livekit.ParticipantInfo_Kind) { + fake.kindMutex.Lock() + defer fake.kindMutex.Unlock() + fake.KindStub = nil + if fake.kindReturnsOnCall == nil { + fake.kindReturnsOnCall = make(map[int]struct { + result1 livekit.ParticipantInfo_Kind + }) + } + fake.kindReturnsOnCall[i] = struct { + result1 livekit.ParticipantInfo_Kind + }{result1} +} + func (fake *FakeLocalParticipant) MaybeStartMigration(arg1 bool, arg2 func()) bool { fake.maybeStartMigrationMutex.Lock() ret, specificReturn := fake.maybeStartMigrationReturnsOnCall[len(fake.maybeStartMigrationArgsForCall)] @@ -6403,10 +6403,10 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.iDMutex.RUnlock() fake.identityMutex.RLock() defer fake.identityMutex.RUnlock() - fake.isAgentMutex.RLock() - defer fake.isAgentMutex.RUnlock() fake.isClosedMutex.RLock() defer fake.isClosedMutex.RUnlock() + fake.isDependentMutex.RLock() + defer fake.isDependentMutex.RUnlock() fake.isDisconnectedMutex.RLock() defer fake.isDisconnectedMutex.RUnlock() fake.isIdleMutex.RLock() @@ -6415,12 +6415,12 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.isPublisherMutex.RUnlock() fake.isReadyMutex.RLock() defer fake.isReadyMutex.RUnlock() - fake.isRecorderMutex.RLock() - defer fake.isRecorderMutex.RUnlock() fake.isSubscribedToMutex.RLock() defer fake.isSubscribedToMutex.RUnlock() fake.issueFullReconnectMutex.RLock() defer fake.issueFullReconnectMutex.RUnlock() + fake.kindMutex.RLock() + defer fake.kindMutex.RUnlock() fake.maybeStartMigrationMutex.RLock() defer fake.maybeStartMigrationMutex.RUnlock() fake.migrateStateMutex.RLock() diff --git a/pkg/rtc/types/typesfakes/fake_participant.go b/pkg/rtc/types/typesfakes/fake_participant.go index 9f6f97e14..f20a4cefe 100644 --- a/pkg/rtc/types/typesfakes/fake_participant.go +++ b/pkg/rtc/types/typesfakes/fake_participant.go @@ -128,14 +128,14 @@ type FakeParticipant struct { identityReturnsOnCall map[int]struct { result1 livekit.ParticipantIdentity } - IsAgentStub func() bool - isAgentMutex sync.RWMutex - isAgentArgsForCall []struct { + IsDependentStub func() bool + isDependentMutex sync.RWMutex + isDependentArgsForCall []struct { } - isAgentReturns struct { + isDependentReturns struct { result1 bool } - isAgentReturnsOnCall map[int]struct { + isDependentReturnsOnCall map[int]struct { result1 bool } IsPublisherStub func() bool @@ -148,15 +148,15 @@ type FakeParticipant struct { isPublisherReturnsOnCall map[int]struct { result1 bool } - IsRecorderStub func() bool - isRecorderMutex sync.RWMutex - isRecorderArgsForCall []struct { + KindStub func() livekit.ParticipantInfo_Kind + kindMutex sync.RWMutex + kindArgsForCall []struct { } - isRecorderReturns struct { - result1 bool + kindReturns struct { + result1 livekit.ParticipantInfo_Kind } - isRecorderReturnsOnCall map[int]struct { - result1 bool + kindReturnsOnCall map[int]struct { + result1 livekit.ParticipantInfo_Kind } RemovePublishedTrackStub func(types.MediaTrack, bool, bool) removePublishedTrackMutex sync.RWMutex @@ -848,15 +848,15 @@ func (fake *FakeParticipant) IdentityReturnsOnCall(i int, result1 livekit.Partic }{result1} } -func (fake *FakeParticipant) IsAgent() bool { - fake.isAgentMutex.Lock() - ret, specificReturn := fake.isAgentReturnsOnCall[len(fake.isAgentArgsForCall)] - fake.isAgentArgsForCall = append(fake.isAgentArgsForCall, struct { +func (fake *FakeParticipant) IsDependent() bool { + fake.isDependentMutex.Lock() + ret, specificReturn := fake.isDependentReturnsOnCall[len(fake.isDependentArgsForCall)] + fake.isDependentArgsForCall = append(fake.isDependentArgsForCall, struct { }{}) - stub := fake.IsAgentStub - fakeReturns := fake.isAgentReturns - fake.recordInvocation("IsAgent", []interface{}{}) - fake.isAgentMutex.Unlock() + stub := fake.IsDependentStub + fakeReturns := fake.isDependentReturns + fake.recordInvocation("IsDependent", []interface{}{}) + fake.isDependentMutex.Unlock() if stub != nil { return stub() } @@ -866,37 +866,37 @@ func (fake *FakeParticipant) IsAgent() bool { return fakeReturns.result1 } -func (fake *FakeParticipant) IsAgentCallCount() int { - fake.isAgentMutex.RLock() - defer fake.isAgentMutex.RUnlock() - return len(fake.isAgentArgsForCall) +func (fake *FakeParticipant) IsDependentCallCount() int { + fake.isDependentMutex.RLock() + defer fake.isDependentMutex.RUnlock() + return len(fake.isDependentArgsForCall) } -func (fake *FakeParticipant) IsAgentCalls(stub func() bool) { - fake.isAgentMutex.Lock() - defer fake.isAgentMutex.Unlock() - fake.IsAgentStub = stub +func (fake *FakeParticipant) IsDependentCalls(stub func() bool) { + fake.isDependentMutex.Lock() + defer fake.isDependentMutex.Unlock() + fake.IsDependentStub = stub } -func (fake *FakeParticipant) IsAgentReturns(result1 bool) { - fake.isAgentMutex.Lock() - defer fake.isAgentMutex.Unlock() - fake.IsAgentStub = nil - fake.isAgentReturns = struct { +func (fake *FakeParticipant) IsDependentReturns(result1 bool) { + fake.isDependentMutex.Lock() + defer fake.isDependentMutex.Unlock() + fake.IsDependentStub = nil + fake.isDependentReturns = struct { result1 bool }{result1} } -func (fake *FakeParticipant) IsAgentReturnsOnCall(i int, result1 bool) { - fake.isAgentMutex.Lock() - defer fake.isAgentMutex.Unlock() - fake.IsAgentStub = nil - if fake.isAgentReturnsOnCall == nil { - fake.isAgentReturnsOnCall = make(map[int]struct { +func (fake *FakeParticipant) IsDependentReturnsOnCall(i int, result1 bool) { + fake.isDependentMutex.Lock() + defer fake.isDependentMutex.Unlock() + fake.IsDependentStub = nil + if fake.isDependentReturnsOnCall == nil { + fake.isDependentReturnsOnCall = make(map[int]struct { result1 bool }) } - fake.isAgentReturnsOnCall[i] = struct { + fake.isDependentReturnsOnCall[i] = struct { result1 bool }{result1} } @@ -954,15 +954,15 @@ func (fake *FakeParticipant) IsPublisherReturnsOnCall(i int, result1 bool) { }{result1} } -func (fake *FakeParticipant) IsRecorder() bool { - fake.isRecorderMutex.Lock() - ret, specificReturn := fake.isRecorderReturnsOnCall[len(fake.isRecorderArgsForCall)] - fake.isRecorderArgsForCall = append(fake.isRecorderArgsForCall, struct { +func (fake *FakeParticipant) Kind() livekit.ParticipantInfo_Kind { + fake.kindMutex.Lock() + ret, specificReturn := fake.kindReturnsOnCall[len(fake.kindArgsForCall)] + fake.kindArgsForCall = append(fake.kindArgsForCall, struct { }{}) - stub := fake.IsRecorderStub - fakeReturns := fake.isRecorderReturns - fake.recordInvocation("IsRecorder", []interface{}{}) - fake.isRecorderMutex.Unlock() + stub := fake.KindStub + fakeReturns := fake.kindReturns + fake.recordInvocation("Kind", []interface{}{}) + fake.kindMutex.Unlock() if stub != nil { return stub() } @@ -972,38 +972,38 @@ func (fake *FakeParticipant) IsRecorder() bool { return fakeReturns.result1 } -func (fake *FakeParticipant) IsRecorderCallCount() int { - fake.isRecorderMutex.RLock() - defer fake.isRecorderMutex.RUnlock() - return len(fake.isRecorderArgsForCall) +func (fake *FakeParticipant) KindCallCount() int { + fake.kindMutex.RLock() + defer fake.kindMutex.RUnlock() + return len(fake.kindArgsForCall) } -func (fake *FakeParticipant) IsRecorderCalls(stub func() bool) { - fake.isRecorderMutex.Lock() - defer fake.isRecorderMutex.Unlock() - fake.IsRecorderStub = stub +func (fake *FakeParticipant) KindCalls(stub func() livekit.ParticipantInfo_Kind) { + fake.kindMutex.Lock() + defer fake.kindMutex.Unlock() + fake.KindStub = stub } -func (fake *FakeParticipant) IsRecorderReturns(result1 bool) { - fake.isRecorderMutex.Lock() - defer fake.isRecorderMutex.Unlock() - fake.IsRecorderStub = nil - fake.isRecorderReturns = struct { - result1 bool +func (fake *FakeParticipant) KindReturns(result1 livekit.ParticipantInfo_Kind) { + fake.kindMutex.Lock() + defer fake.kindMutex.Unlock() + fake.KindStub = nil + fake.kindReturns = struct { + result1 livekit.ParticipantInfo_Kind }{result1} } -func (fake *FakeParticipant) IsRecorderReturnsOnCall(i int, result1 bool) { - fake.isRecorderMutex.Lock() - defer fake.isRecorderMutex.Unlock() - fake.IsRecorderStub = nil - if fake.isRecorderReturnsOnCall == nil { - fake.isRecorderReturnsOnCall = make(map[int]struct { - result1 bool +func (fake *FakeParticipant) KindReturnsOnCall(i int, result1 livekit.ParticipantInfo_Kind) { + fake.kindMutex.Lock() + defer fake.kindMutex.Unlock() + fake.KindStub = nil + if fake.kindReturnsOnCall == nil { + fake.kindReturnsOnCall = make(map[int]struct { + result1 livekit.ParticipantInfo_Kind }) } - fake.isRecorderReturnsOnCall[i] = struct { - result1 bool + fake.kindReturnsOnCall[i] = struct { + result1 livekit.ParticipantInfo_Kind }{result1} } @@ -1416,12 +1416,12 @@ func (fake *FakeParticipant) Invocations() map[string][][]interface{} { defer fake.iDMutex.RUnlock() fake.identityMutex.RLock() defer fake.identityMutex.RUnlock() - fake.isAgentMutex.RLock() - defer fake.isAgentMutex.RUnlock() + fake.isDependentMutex.RLock() + defer fake.isDependentMutex.RUnlock() fake.isPublisherMutex.RLock() defer fake.isPublisherMutex.RUnlock() - fake.isRecorderMutex.RLock() - defer fake.isRecorderMutex.RUnlock() + fake.kindMutex.RLock() + defer fake.kindMutex.RUnlock() fake.removePublishedTrackMutex.RLock() defer fake.removePublishedTrackMutex.RUnlock() fake.setMetadataMutex.RLock() diff --git a/pkg/service/wire_gen.go b/pkg/service/wire_gen.go index 80b8bd8c0..08d493b13 100644 --- a/pkg/service/wire_gen.go +++ b/pkg/service/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run github.com/google/wire/cmd/wire +//go:generate go run -mod=mod github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject From 4603b5c05363155a0c3960724bed282ff3ce7960 Mon Sep 17 00:00:00 2001 From: David Colburn Date: Fri, 5 Apr 2024 14:24:34 -0700 Subject: [PATCH 28/78] make IsDependent backwards compatible (#2627) --- pkg/rtc/participant.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 090930875..b74088629 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -340,7 +340,7 @@ func (p *ParticipantImpl) IsDependent() bool { case livekit.ParticipantInfo_AGENT, livekit.ParticipantInfo_EGRESS: return true default: - return false + return p.grants.Video.Agent || p.grants.Video.Recorder } } From f4314686d136d8fac140832e64a39b07abcab4eb Mon Sep 17 00:00:00 2001 From: David Zhao Date: Fri, 5 Apr 2024 20:32:29 -0700 Subject: [PATCH 29/78] Improve Agent logging (#2628) --- pkg/agent/worker.go | 3 +++ pkg/service/agentservice.go | 43 +++++++++++++++++++++++++++---------- 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/pkg/agent/worker.go b/pkg/agent/worker.go index b6f69fcfe..666681b39 100644 --- a/pkg/agent/worker.go +++ b/pkg/agent/worker.go @@ -303,6 +303,8 @@ func (w *Worker) handleRegister(req *livekit.RegisterWorkerRequest) { w.registered.Store(true) w.mu.Unlock() + w.Logger.Debugw("worker registered", "request", req) + w.sendRequest(&livekit.ServerMessage{ Message: &livekit.ServerMessage_Register{ Register: &livekit.RegisterWorkerResponse{ @@ -376,6 +378,7 @@ func (w *Worker) handleWorkerPing(ping *livekit.WorkerPing) { } func (w *Worker) handleWorkerStatus(update *livekit.UpdateWorkerStatus) { + w.Logger.Debugw("worker status update", "status", update.Status, "load", update.Load) w.UpdateStatus(update) } diff --git a/pkg/service/agentservice.go b/pkg/service/agentservice.go index e8f391cb4..e3153347c 100644 --- a/pkg/service/agentservice.go +++ b/pkg/service/agentservice.go @@ -33,10 +33,10 @@ import ( "github.com/livekit/livekit-server/pkg/routing" "github.com/livekit/livekit-server/pkg/rtc" "github.com/livekit/livekit-server/pkg/rtc/types" - "github.com/livekit/livekit-server/pkg/utils" "github.com/livekit/livekit-server/version" "github.com/livekit/protocol/auth" "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" "github.com/livekit/protocol/rpc" "github.com/livekit/psrpc" ) @@ -50,6 +50,7 @@ type AgentService struct { type AgentHandler struct { agentServer rpc.AgentInternalServer mu sync.Mutex + logger logger.Logger serverInfo *livekit.ServerInfo workers map[string]*agent.Worker @@ -98,6 +99,7 @@ func NewAgentService(conf *config.Config, s.AgentHandler = NewAgentHandler( agentServer, keyProvider, + logger.GetLogger(), serverInfo, agent.RoomAgentTopic, agent.PublisherAgentTopic, @@ -132,12 +134,14 @@ func (s *AgentService) ServeHTTP(writer http.ResponseWriter, r *http.Request) { func NewAgentHandler( agentServer rpc.AgentInternalServer, keyProvider auth.KeyProvider, + logger logger.Logger, serverInfo *livekit.ServerInfo, roomTopic string, publisherTopic string, ) *AgentHandler { return &AgentHandler{ agentServer: agentServer, + logger: logger, workers: make(map[string]*agent.Worker), namespaces: make(map[string]*namespaceInfo), serverInfo: serverInfo, @@ -155,13 +159,11 @@ func (h *AgentHandler) HandleConnection(r *http.Request, conn *websocket.Conn, o sigConn := NewWSSignalConnection(conn) - logger := utils.GetLogger(r.Context()) - apiKey := GetAPIKey(r.Context()) apiSecret := h.keyProvider.GetSecret(apiKey) - worker := agent.NewWorker(protocol, apiKey, apiSecret, h.serverInfo, conn, sigConn, logger) - worker.OnWorkerRegistered(h.registerWorkerTopic) + worker := agent.NewWorker(protocol, apiKey, apiSecret, h.serverInfo, conn, sigConn, h.logger) + worker.OnWorkerRegistered(h.handleWorkerRegister) h.mu.Lock() h.workers[worker.ID()] = worker @@ -176,7 +178,7 @@ func (h *AgentHandler) HandleConnection(r *http.Request, conn *websocket.Conn, o h.mu.Unlock() if worker.Registered() { - h.deregisterWorkerTopic(worker) + h.handleWorkerDeregister(worker) } if numWorkers == 0 && onIdle != nil { @@ -209,7 +211,7 @@ func (h *AgentHandler) HandleConnection(r *http.Request, conn *websocket.Conn, o } } -func (h *AgentHandler) registerWorkerTopic(w *agent.Worker) { +func (h *AgentHandler) handleWorkerRegister(w *agent.Worker) { h.mu.Lock() info, ok := h.namespaces[w.Namespace()] @@ -220,16 +222,19 @@ func (h *AgentHandler) registerWorkerTopic(w *agent.Worker) { numRooms = info.numRooms } + shouldNotify := false var err error if w.JobType() == livekit.JobType_JT_PUBLISHER { numPublishers++ if numPublishers == 1 { + shouldNotify = true err = h.agentServer.RegisterJobRequestTopic(w.Namespace(), h.publisherTopic) } } else if w.JobType() == livekit.JobType_JT_ROOM { numRooms++ if numRooms == 1 { + shouldNotify = true err = h.agentServer.RegisterJobRequestTopic(w.Namespace(), h.roomTopic) } } @@ -250,13 +255,16 @@ func (h *AgentHandler) registerWorkerTopic(w *agent.Worker) { h.publisherEnabled = h.publisherAvailableLocked() h.mu.Unlock() - err = h.agentServer.PublishWorkerRegistered(context.Background(), agent.DefaultHandlerNamespace, &emptypb.Empty{}) - if err != nil { - w.Logger.Errorw("failed to publish worker registered", err) + if shouldNotify { + h.logger.Infow("initial worker registered", "namespace", w.Namespace(), "jobType", w.JobType()) + err = h.agentServer.PublishWorkerRegistered(context.Background(), agent.DefaultHandlerNamespace, &emptypb.Empty{}) + if err != nil { + w.Logger.Errorw("failed to publish worker registered", err) + } } } -func (h *AgentHandler) deregisterWorkerTopic(worker *agent.Worker) { +func (h *AgentHandler) handleWorkerDeregister(worker *agent.Worker) { h.mu.Lock() defer h.mu.Unlock() @@ -278,6 +286,7 @@ func (h *AgentHandler) deregisterWorkerTopic(worker *agent.Worker) { } if info.numPublishers == 0 && info.numRooms == 0 { + h.logger.Debugw("last worker deregistered") delete(h.namespaces, worker.Namespace()) } @@ -340,6 +349,18 @@ func (h *AgentHandler) JobRequest(ctx context.Context, job *livekit.Job) (*empty attempted[selected.ID()] = true + values := []interface{}{ + "jobID", job.Id, + "namespace", job.Namespace, + "workerID", selected.ID(), + } + if job.Room != nil { + values = append(values, "room", job.Room.Name, "roomID", job.Room.Sid) + } + if job.Participant != nil { + values = append(values, "participant", job.Participant.Identity) + } + logger.Debugw("assigning job", values...) err := selected.AssignJob(ctx, job) if err != nil { if errors.Is(err, agent.ErrWorkerNotAvailable) { From 8852d71a8a4258b3d5a17158681afd08108c3b5a Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sat, 6 Apr 2024 11:28:04 +0530 Subject: [PATCH 30/78] Disable audio loss proxying. (#2629) * Disable audio loss proxying. Added a config which is off by default. With audio NACKs, that is the preferred repair mechanism. With RED, repair is built in via packet redundancy to recover from isolated losses. So, proxying is not required. But, leaving it in there with a config that is disabled by default. * fix test --- pkg/config/config.go | 2 ++ pkg/rtc/mediatrack.go | 1 - pkg/sfu/buffer/buffer.go | 21 ++++++++++++++++---- pkg/sfu/buffer/buffer_test.go | 37 ++++++++++++++++++++++++++++++++++- pkg/sfu/receiver.go | 1 + 5 files changed, 56 insertions(+), 6 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 82f338f9a..241bb8f0d 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -191,6 +191,8 @@ type AudioConfig struct { SmoothIntervals uint32 `yaml:"smooth_intervals,omitempty"` // enable red encoding downtrack for opus only audio up track ActiveREDEncoding bool `yaml:"active_red_encoding,omitempty"` + // enable proxying weakest subscriber loss to publisher in RTCP Receiver Report + EnableLossProxying bool `yaml:"enable_loss_proxying,omitempty"` } type StreamTrackerPacketConfig struct { diff --git a/pkg/rtc/mediatrack.go b/pkg/rtc/mediatrack.go index c28c563c3..f575b4e13 100644 --- a/pkg/rtc/mediatrack.go +++ b/pkg/rtc/mediatrack.go @@ -96,7 +96,6 @@ func NewMediaTrack(params MediaTrackParams, ti *livekit.TrackInfo) *MediaTrack { }) t.MediaLossProxy.OnMediaLossUpdate(func(fractionalLoss uint8) { if t.buffer != nil { - // ok to access buffer since receivers are added before subscribers t.buffer.SetLastFractionLostReport(fractionalLoss) } }) diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go index 8cfa7e663..70f9940b3 100644 --- a/pkg/sfu/buffer/buffer.go +++ b/pkg/sfu/buffer/buffer.go @@ -93,9 +93,10 @@ type Buffer struct { latestTSForAudioLevelInitialized bool latestTSForAudioLevel uint32 - twcc *twcc.Responder - audioLevelParams audio.AudioLevelParams - audioLevel *audio.AudioLevel + twcc *twcc.Responder + audioLevelParams audio.AudioLevelParams + audioLevel *audio.AudioLevel + enableAudioLossProxying bool lastPacketRead int @@ -182,6 +183,13 @@ func (b *Buffer) SetAudioLevelParams(audioLevelParams audio.AudioLevelParams) { b.audioLevelParams = audioLevelParams } +func (b *Buffer) SetAudioLossProxying(enable bool) { + b.Lock() + defer b.Unlock() + + b.enableAudioLossProxying = enable +} + func (b *Buffer) Bind(params webrtc.RTPParameters, codec webrtc.RTPCodecCapability) { b.Lock() defer b.Unlock() @@ -824,7 +832,12 @@ func (b *Buffer) buildReceptionReport() *rtcp.ReceptionReport { return nil } - return b.rtpStats.GetRtcpReceptionReport(b.mediaSSRC, b.lastFractionLostToReport, b.rrSnapshotId) + proxyLoss := b.lastFractionLostToReport + if b.codecType == webrtc.RTPCodecTypeAudio && !b.enableAudioLossProxying { + proxyLoss = 0 + } + + return b.rtpStats.GetRtcpReceptionReport(b.mediaSSRC, proxyLoss, b.rrSnapshotId) } func (b *Buffer) SetSenderReportData(rtpTime uint32, ntpTime uint64) { diff --git a/pkg/sfu/buffer/buffer_test.go b/pkg/sfu/buffer/buffer_test.go index 2125ca907..d27ef3fe2 100644 --- a/pkg/sfu/buffer/buffer_test.go +++ b/pkg/sfu/buffer/buffer_test.go @@ -208,9 +208,12 @@ func TestNewBuffer(t *testing.T) { func TestFractionLostReport(t *testing.T) { buff := NewBuffer(123, 1, 1) require.NotNil(t, buff) - buff.codecType = webrtc.RTPCodecTypeVideo + var wg sync.WaitGroup + + // with loss proxying wg.Add(1) + buff.SetAudioLossProxying(true) buff.SetLastFractionLostReport(55) buff.OnRtcpFeedback(func(fb []rtcp.Packet) { for _, pkt := range fb { @@ -241,6 +244,38 @@ func TestFractionLostReport(t *testing.T) { require.NoError(t, err) } wg.Wait() + + wg.Add(1) + buff.SetAudioLossProxying(false) + buff.OnRtcpFeedback(func(fb []rtcp.Packet) { + for _, pkt := range fb { + switch p := pkt.(type) { + case *rtcp.ReceiverReport: + for _, v := range p.Reports { + require.EqualValues(t, 0, v.FractionLost) + } + wg.Done() + } + } + }) + buff.Bind(webrtc.RTPParameters{ + HeaderExtensions: nil, + Codecs: []webrtc.RTPCodecParameters{opusCodec}, + }, opusCodec.RTPCodecCapability) + for i := 0; i < 15; i++ { + pkt := rtp.Packet{ + Header: rtp.Header{SequenceNumber: uint16(i), Timestamp: uint32(i)}, + Payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x9f, 0x94, 0x1}, + } + b, err := pkt.Marshal() + require.NoError(t, err) + if i == 1 { + time.Sleep(1 * time.Second) + } + _, err = buff.Write(b) + require.NoError(t, err) + } + wg.Wait() } func BenchmarkMemcpu(b *testing.B) { diff --git a/pkg/sfu/receiver.go b/pkg/sfu/receiver.go index e1bebcd21..e6e9c54ae 100644 --- a/pkg/sfu/receiver.go +++ b/pkg/sfu/receiver.go @@ -344,6 +344,7 @@ func (w *WebRTCReceiver) AddUpTrack(track *webrtc.TrackRemote, buff *buffer.Buff ObserveDuration: w.audioConfig.UpdateInterval, SmoothIntervals: w.audioConfig.SmoothIntervals, }) + buff.SetAudioLossProxying(w.audioConfig.EnableLossProxying) buff.OnRtcpFeedback(w.sendRTCP) buff.OnRtcpSenderReport(func() { srData := buff.GetSenderReportData() From 8334149034243bb9013ccf511a30c5f9c5689ad1 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sat, 6 Apr 2024 12:13:21 +0530 Subject: [PATCH 31/78] update mediatransportutil (#2630) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 0df8c9f6f..0cb8c9a77 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 - github.com/livekit/mediatransportutil v0.0.0-20240302142739-1c3dd691a1b8 + github.com/livekit/mediatransportutil v0.0.0-20240406063423-a67d961689df github.com/livekit/protocol v1.12.1-0.20240403204952-bc6c7ffd71f0 github.com/livekit/psrpc v0.5.3-0.20240327035954-cec3a0e614be github.com/mackerelio/go-osstat v0.2.4 diff --git a/go.sum b/go.sum index e5e0dba7b..d616ba9e6 100644 --- a/go.sum +++ b/go.sum @@ -130,8 +130,8 @@ github.com/lithammer/shortuuid/v4 v4.0.0 h1:QRbbVkfgNippHOS8PXDkti4NaWeyYfcBTHtw github.com/lithammer/shortuuid/v4 v4.0.0/go.mod h1:Zs8puNcrvf2rV9rTH51ZLLcj7ZXqQI3lv67aw4KiB1Y= github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkDaKb5iXdynYrzB84ErPPO4LbRASk58= github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= -github.com/livekit/mediatransportutil v0.0.0-20240302142739-1c3dd691a1b8 h1:xawydPEACNO5Ncs2LgioTjWghXQ0eUN1q1RnVUUyVnI= -github.com/livekit/mediatransportutil v0.0.0-20240302142739-1c3dd691a1b8/go.mod h1:jwKUCmObuiEDH0iiuJHaGMXwRs3RjrB4G6qqgkr/5oE= +github.com/livekit/mediatransportutil v0.0.0-20240406063423-a67d961689df h1:DVhJRlF6/CtiyxJVy3QsbS9bf7GyUMuRZONMwZxIWpY= +github.com/livekit/mediatransportutil v0.0.0-20240406063423-a67d961689df/go.mod h1:jwKUCmObuiEDH0iiuJHaGMXwRs3RjrB4G6qqgkr/5oE= github.com/livekit/protocol v1.12.1-0.20240403204952-bc6c7ffd71f0 h1:BfQN4k6YG+XTdfbXnA2XZLAXinRNlJrGdTLAjyOW2Rg= github.com/livekit/protocol v1.12.1-0.20240403204952-bc6c7ffd71f0/go.mod h1:mcv7L2DWB6iRckI++egmwFU7YEy3W+aLHnusNhioqi0= github.com/livekit/psrpc v0.5.3-0.20240327035954-cec3a0e614be h1:W1nCFZ19rYAORMBNX82NeVPHjADN0UyORr6refbUXpU= From 0712719c2fdb33fb8a5895601f39226df85e2c69 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Sat, 6 Apr 2024 10:36:02 -0700 Subject: [PATCH 32/78] Fix deadlock in IncrementalDispatcher (#2632) --- pkg/utils/incrementaldispatcher.go | 7 +++++++ pkg/utils/incrementaldispatcher_test.go | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/utils/incrementaldispatcher.go b/pkg/utils/incrementaldispatcher.go index dcc23d55c..2aaaf9ffd 100644 --- a/pkg/utils/incrementaldispatcher.go +++ b/pkg/utils/incrementaldispatcher.go @@ -48,8 +48,10 @@ func (d *IncrementalDispatcher[T]) Add(item T) { } func (d *IncrementalDispatcher[T]) Done() { + d.lock.Lock() d.done.Break() d.cond.Broadcast() + d.lock.Unlock() } func (d *IncrementalDispatcher[T]) ForEach(fn func(T)) { @@ -69,6 +71,11 @@ func (d *IncrementalDispatcher[T]) ForEach(fn func(T)) { for !d.done.IsBroken() { dispatchFromIdx() d.lock.Lock() + // need to check again because Done may have been called while dispatching + if d.done.IsBroken() { + d.lock.Unlock() + break + } if idx == len(d.items) { d.cond.Wait() } diff --git a/pkg/utils/incrementaldispatcher_test.go b/pkg/utils/incrementaldispatcher_test.go index da1393c64..828019559 100644 --- a/pkg/utils/incrementaldispatcher_test.go +++ b/pkg/utils/incrementaldispatcher_test.go @@ -48,7 +48,7 @@ func TestForEach(t *testing.T) { func TestConcurrentConsumption(t *testing.T) { producer := utils.NewIncrementalDispatcher[int]() - numConsumers := 5 + numConsumers := 100 sums := make([]atomic.Int32, numConsumers) var wg sync.WaitGroup From ddece1fbb0ea6c3fa2ec7437d98faea82d651ca9 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 8 Apr 2024 11:29:55 +0530 Subject: [PATCH 33/78] Use aarival time in cached packets. (#2633) --- pkg/sfu/buffer/buffer.go | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go index 70f9940b3..8b7678671 100644 --- a/pkg/sfu/buffer/buffer.go +++ b/pkg/sfu/buffer/buffer.go @@ -301,9 +301,10 @@ func (b *Buffer) Write(pkt []byte) (n int, err error) { return } + now := time.Now() if b.twcc != nil && b.twccExt != 0 && !b.closed.Load() { if ext := rtpPacket.GetExtension(b.twccExt); ext != nil { - b.twcc.Push(rtpPacket.SSRC, binary.BigEndian.Uint16(ext[0:2]), time.Now().UnixNano(), rtpPacket.Marker) + b.twcc.Push(rtpPacket.SSRC, binary.BigEndian.Uint16(ext[0:2]), now.UnixNano(), rtpPacket.Marker) } } @@ -316,7 +317,7 @@ func (b *Buffer) Write(pkt []byte) (n int, err error) { return } - pb.writeRTX(&rtpPacket) + pb.writeRTX(&rtpPacket, now) return } @@ -325,7 +326,7 @@ func (b *Buffer) Write(pkt []byte) (n int, err error) { copy(packet, pkt) b.pPackets = append(b.pPackets, pendingPacket{ packet: packet, - arrivalTime: time.Now(), + arrivalTime: now, }) b.Unlock() return @@ -353,11 +354,11 @@ func (b *Buffer) SetPrimaryBufferForRTX(primaryBuffer *Buffer) { if rtpPacket.Padding && len(rtpPacket.Payload) == 0 { continue } - primaryBuffer.writeRTX(&rtpPacket) + primaryBuffer.writeRTX(&rtpPacket, pp.arrivalTime) } } -func (b *Buffer) writeRTX(rtxPkt *rtp.Packet) (n int, err error) { +func (b *Buffer) writeRTX(rtxPkt *rtp.Packet, arrivalTime time.Time) (n int, err error) { b.Lock() defer b.Unlock() if !b.bound { @@ -368,19 +369,18 @@ func (b *Buffer) writeRTX(rtxPkt *rtp.Packet) (n int, err error) { b.rtxPktBuf = make([]byte, bucket.MaxPktSize) } - videoPkt := *rtxPkt - videoPkt.PayloadType = b.payloadType - videoPkt.SequenceNumber = binary.BigEndian.Uint16(rtxPkt.Payload[:2]) - videoPkt.SSRC = b.mediaSSRC - videoPkt.Payload = rtxPkt.Payload[2:] - n, err = videoPkt.MarshalTo(b.rtxPktBuf) - + repairedPkt := *rtxPkt + repairedPkt.PayloadType = b.payloadType + repairedPkt.SequenceNumber = binary.BigEndian.Uint16(rtxPkt.Payload[:2]) + repairedPkt.SSRC = b.mediaSSRC + repairedPkt.Payload = rtxPkt.Payload[2:] + n, err = repairedPkt.MarshalTo(b.rtxPktBuf) if err != nil { - b.logger.Errorw("could not marshal repaired packet", err, "ssrc", b.mediaSSRC, "sn", videoPkt.SequenceNumber) + b.logger.Errorw("could not marshal repaired packet", err, "ssrc", b.mediaSSRC, "sn", repairedPkt.SequenceNumber) return } - b.calc(b.rtxPktBuf[:n], &videoPkt, time.Now(), true) + b.calc(b.rtxPktBuf[:n], &repairedPkt, arrivalTime, true) return } @@ -664,7 +664,6 @@ func (b *Buffer) updateStreamState(p *rtp.Packet, arrivalTime time.Time) RTPFlow } func (b *Buffer) processHeaderExtensions(p *rtp.Packet, arrivalTime time.Time, isRTX bool) { - if b.audioLevelExt != 0 && !isRTX { if !b.latestTSForAudioLevelInitialized { b.latestTSForAudioLevelInitialized = true From 196fed605e33ec40c4417544a69a659e0e632874 Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Mon, 8 Apr 2024 21:50:50 +0800 Subject: [PATCH 34/78] Limit playout delay change for high jitter (#2635) * Limit playout delay change for high jitter * fix test --- pkg/sfu/playoutdelay.go | 32 ++++++++++++++++++-------------- pkg/sfu/playoutdelay_test.go | 16 ++++++++++------ 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/pkg/sfu/playoutdelay.go b/pkg/sfu/playoutdelay.go index 47eeffb18..991f1401b 100644 --- a/pkg/sfu/playoutdelay.go +++ b/pkg/sfu/playoutdelay.go @@ -17,6 +17,7 @@ package sfu import ( "sync" "sync/atomic" + "time" "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/livekit-server/pkg/sfu/rtpextension" @@ -30,11 +31,11 @@ const ( PlayoutDelaySending PlayoutDelayAcked - jitterLowMultiToDelay = 10 - jitterHighMultiToDelay = 15 - jitterHighThreshold = 15 - + jitterMultiToDelay = 10 targetDelayLogThreshold = 500 + + // limit max delay change to make it smoother for a/v sync + maxDelayChangePerSec = 80 ) func (s PlayoutDelayState) String() string { @@ -56,6 +57,7 @@ type PlayoutDelayController struct { currentDelay uint32 extBytes atomic.Value //[]byte sendingAtSeq uint16 + sendingAtTime time.Time logger logger.Logger rtpStats *buffer.RTPStatsSender snapshotID uint32 @@ -89,20 +91,21 @@ func (c *PlayoutDelayController) SetJitter(jitter uint32) { } c.lock.Lock() - multi := jitterLowMultiToDelay - if jitter >= jitterHighThreshold { - multi = jitterHighMultiToDelay - } - targetDelay := jitter * uint32(multi) + targetDelay := jitter * jitterMultiToDelay if nackPercent > 60 { targetDelay += (nackPercent - 60) * 2 } - // increase delay quickly, decrease slowly to make fps more stable - if targetDelay > c.currentDelay { - targetDelay = (targetDelay-c.currentDelay)*3/4 + c.currentDelay - } else { - targetDelay = c.currentDelay - (c.currentDelay-targetDelay)/5 + elapsed := time.Since(c.sendingAtTime) + delayChangeLimit := uint32(maxDelayChangePerSec * elapsed.Seconds()) + if delayChangeLimit > maxDelayChangePerSec { + delayChangeLimit = maxDelayChangePerSec + } + + if targetDelay > c.currentDelay+delayChangeLimit { + targetDelay = c.currentDelay + delayChangeLimit + } else if c.currentDelay > targetDelay+delayChangeLimit { + targetDelay = c.currentDelay - delayChangeLimit } if targetDelay < c.minDelay { targetDelay = c.minDelay @@ -138,6 +141,7 @@ func (c *PlayoutDelayController) GetDelayExtension(seq uint16) []byte { c.lock.Lock() c.state.Store(int32(PlayoutDelaySending)) c.sendingAtSeq = seq + c.sendingAtTime = time.Now() c.lock.Unlock() return c.extBytes.Load().([]byte) case PlayoutDelaySending: diff --git a/pkg/sfu/playoutdelay_test.go b/pkg/sfu/playoutdelay_test.go index 05a88f5d0..cc24d2848 100644 --- a/pkg/sfu/playoutdelay_test.go +++ b/pkg/sfu/playoutdelay_test.go @@ -16,6 +16,7 @@ package sfu import ( "testing" + "time" "github.com/stretchr/testify/require" @@ -26,23 +27,23 @@ import ( func TestPlayoutDelay(t *testing.T) { stats := buffer.NewRTPStatsSender(buffer.RTPStatsParams{ClockRate: 900000, Logger: logger.GetLogger()}) - c, err := NewPlayoutDelayController(100, 1000, logger.GetLogger(), stats) + c, err := NewPlayoutDelayController(100, 120, logger.GetLogger(), stats) require.NoError(t, err) ext := c.GetDelayExtension(100) - playoutDelayEqual(t, ext, 100, 1000) + playoutDelayEqual(t, ext, 100, 120) ext = c.GetDelayExtension(105) - playoutDelayEqual(t, ext, 100, 1000) + playoutDelayEqual(t, ext, 100, 120) // seq acked before delay changed c.OnSeqAcked(65534) ext = c.GetDelayExtension(105) - playoutDelayEqual(t, ext, 100, 1000) + playoutDelayEqual(t, ext, 100, 120) c.OnSeqAcked(90) ext = c.GetDelayExtension(105) - playoutDelayEqual(t, ext, 100, 1000) + playoutDelayEqual(t, ext, 100, 120) // seq acked, no extension sent for new packet c.OnSeqAcked(103) @@ -55,16 +56,19 @@ func TestPlayoutDelay(t *testing.T) { require.Nil(t, ext) // delay changed, generate new extension to send + time.Sleep(200 * time.Millisecond) c.SetJitter(50) + t.Log(c.currentDelay, c.state.Load()) ext = c.GetDelayExtension(108) var delay rtpextension.PlayOutDelay require.NoError(t, delay.Unmarshal(ext)) require.Greater(t, delay.Min, uint16(100)) // can't go above max + time.Sleep(200 * time.Millisecond) c.SetJitter(10000) ext = c.GetDelayExtension(109) - playoutDelayEqual(t, ext, 1000, 1000) + playoutDelayEqual(t, ext, 120, 120) } func playoutDelayEqual(t *testing.T, data []byte, min, max uint16) { From 10c8582a6b01c37fb2db7e343b5204c713fbb7cd Mon Sep 17 00:00:00 2001 From: Mathew Kamkar <578302+matkam@users.noreply.github.com> Date: Mon, 8 Apr 2024 21:15:17 -0700 Subject: [PATCH 35/78] get cpu stats from cgroup, remove env (#2636) * get cpu stats from cgroup, remove env * undo rand seed removal * tests --- cmd/server/main.go | 4 ++- pkg/config/config.go | 5 ---- pkg/rtc/room_test.go | 2 +- pkg/service/signal_test.go | 2 +- pkg/service/wire_gen.go | 2 +- pkg/telemetry/prometheus/node.go | 44 ++++++++++++++++++----------- pkg/telemetry/prometheus/packets.go | 24 ++++++++-------- pkg/telemetry/prometheus/quality.go | 8 +++--- pkg/telemetry/prometheus/rooms.go | 18 ++++++------ pkg/telemetry/stats_test.go | 2 +- test/integration_helpers.go | 2 +- 11 files changed, 61 insertions(+), 52 deletions(-) diff --git a/cmd/server/main.go b/cmd/server/main.go index 3e8e3344d..427702954 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -272,7 +272,9 @@ func startServer(c *cli.Context) error { return err } - prometheus.Init(currentNode.Id, currentNode.Type, conf.Environment) + if err := prometheus.Init(currentNode.Id, currentNode.Type); err != nil { + return err + } server, err := service.InitializeServer(conf, currentNode) if err != nil { diff --git a/pkg/config/config.go b/pkg/config/config.go index 241bb8f0d..78cd0f1e1 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -61,7 +61,6 @@ type Config struct { Port uint32 `yaml:"port,omitempty"` BindAddresses []string `yaml:"bind_addresses,omitempty"` PrometheusPort uint32 `yaml:"prometheus_port,omitempty"` - Environment string `yaml:"environment,omitempty"` RTC RTCConfig `yaml:"rtc,omitempty"` Redis redisLiveKit.RedisConfig `yaml:"redis,omitempty"` Audio AudioConfig `yaml:"audio,omitempty"` @@ -574,10 +573,6 @@ func NewConfig(confString string, strictMode bool, c *cli.Context, baseFlags []c conf.Logging.ComponentLevels["pion"] = conf.Logging.PionLevel } - if conf.Development { - conf.Environment = "dev" - } - return &conf, nil } diff --git a/pkg/rtc/room_test.go b/pkg/rtc/room_test.go index c73b5fa5c..40a8b30ad 100644 --- a/pkg/rtc/room_test.go +++ b/pkg/rtc/room_test.go @@ -38,7 +38,7 @@ import ( ) func init() { - prometheus.Init("test", livekit.NodeType_SERVER, "test") + prometheus.Init("test", livekit.NodeType_SERVER) } const ( diff --git a/pkg/service/signal_test.go b/pkg/service/signal_test.go index 951837176..9da50679f 100644 --- a/pkg/service/signal_test.go +++ b/pkg/service/signal_test.go @@ -35,7 +35,7 @@ import ( ) func init() { - prometheus.Init("node", livekit.NodeType_CONTROLLER, "test") + prometheus.Init("node", livekit.NodeType_CONTROLLER) } func TestSignal(t *testing.T) { diff --git a/pkg/service/wire_gen.go b/pkg/service/wire_gen.go index 08d493b13..80b8bd8c0 100644 --- a/pkg/service/wire_gen.go +++ b/pkg/service/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run -mod=mod github.com/google/wire/cmd/wire +//go:generate go run github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject diff --git a/pkg/telemetry/prometheus/node.go b/pkg/telemetry/prometheus/node.go index ba6dd4507..703615693 100644 --- a/pkg/telemetry/prometheus/node.go +++ b/pkg/telemetry/prometheus/node.go @@ -24,6 +24,7 @@ import ( "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/rpc" + "github.com/livekit/protocol/utils/hwstats" ) const ( @@ -41,11 +42,13 @@ var ( sysDroppedPacketsStart uint32 promSysPacketGauge *prometheus.GaugeVec promSysDroppedPacketPctGauge prometheus.Gauge + + cpuStats *hwstats.CPUStats ) -func Init(nodeID string, nodeType livekit.NodeType, env string) { +func Init(nodeID string, nodeType livekit.NodeType) error { if initialized.Swap(true) { - return + return nil } MessageCounter = prometheus.NewCounterVec( @@ -53,7 +56,7 @@ func Init(nodeID string, nodeType livekit.NodeType, env string) { Namespace: livekitNamespace, Subsystem: "node", Name: "messages", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, []string{"type", "status"}, ) @@ -63,7 +66,7 @@ func Init(nodeID string, nodeType livekit.NodeType, env string) { Namespace: livekitNamespace, Subsystem: "node", Name: "service_operation", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, []string{"type", "status", "error_type"}, ) @@ -73,7 +76,7 @@ func Init(nodeID string, nodeType livekit.NodeType, env string) { Namespace: livekitNamespace, Subsystem: "node", Name: "twirp_request_status", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, []string{"service", "method", "status", "code"}, ) @@ -83,7 +86,7 @@ func Init(nodeID string, nodeType livekit.NodeType, env string) { Namespace: livekitNamespace, Subsystem: "node", Name: "packet_total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, Help: "System level packet count. Count starts at 0 when service is first started.", }, []string{"type"}, @@ -94,7 +97,7 @@ func Init(nodeID string, nodeType livekit.NodeType, env string) { Namespace: livekitNamespace, Subsystem: "node", Name: "dropped_packets", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, Help: "System level dropped outgoing packet percentage.", }, ) @@ -107,10 +110,18 @@ func Init(nodeID string, nodeType livekit.NodeType, env string) { sysPacketsStart, sysDroppedPacketsStart, _ = getTCStats() - initPacketStats(nodeID, nodeType, env) - initRoomStats(nodeID, nodeType, env) - rpc.InitPSRPCStats(prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}) - initQualityStats(nodeID, nodeType, env) + initPacketStats(nodeID, nodeType) + initRoomStats(nodeID, nodeType) + rpc.InitPSRPCStats(prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}) + initQualityStats(nodeID, nodeType) + + var err error + cpuStats, err = hwstats.NewCPUStats(nil) + if err != nil { + return err + } + + return nil } func GetUpdatedNodeStats(prev *livekit.NodeStats, prevAverage *livekit.NodeStats) (*livekit.NodeStats, bool, error) { @@ -119,9 +130,10 @@ func GetUpdatedNodeStats(prev *livekit.NodeStats, prevAverage *livekit.NodeStats return nil, false, err } - cpuLoad, numCPUs, err := getCPUStats() - if err != nil { - return nil, false, err + var cpuLoad float64 + cpuIdle := cpuStats.GetCPUIdle() + if cpuIdle > 0 { + cpuLoad = 1 - (cpuIdle / cpuStats.NumCPU()) } // On MacOS, get "\"vm_stat\": executable file not found in $PATH" although it is in /usr/bin @@ -198,8 +210,8 @@ func GetUpdatedNodeStats(prev *livekit.NodeStats, prevAverage *livekit.NodeStats ParticipantSignalConnectedPerSec: prevAverage.ParticipantSignalConnectedPerSec, ParticipantRtcInitPerSec: prevAverage.ParticipantRtcInitPerSec, ParticipantRtcConnectedPerSec: prevAverage.ParticipantRtcConnectedPerSec, - NumCpus: numCPUs, - CpuLoad: cpuLoad, + NumCpus: uint32(cpuStats.NumCPU()), // this will round down to the nearest integer + CpuLoad: float32(cpuLoad), MemoryTotal: memTotal, MemoryUsed: memUsed, LoadAvgLast1Min: float32(loadAvg.Loadavg1), diff --git a/pkg/telemetry/prometheus/packets.go b/pkg/telemetry/prometheus/packets.go index 6b643e903..3efd1eb28 100644 --- a/pkg/telemetry/prometheus/packets.go +++ b/pkg/telemetry/prometheus/packets.go @@ -67,55 +67,55 @@ var ( promPacketBytesOutgoingRetransmit prometheus.Counter ) -func initPacketStats(nodeID string, nodeType livekit.NodeType, env string) { +func initPacketStats(nodeID string, nodeType livekit.NodeType) { promPacketTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "packet", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, promPacketLabels) promPacketBytes = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "packet", Name: "bytes", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, promPacketLabels) promNackTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "nack", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, promRTCPLabels) promPliTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "pli", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, promRTCPLabels) promFirTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "fir", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, promRTCPLabels) promPacketLossTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "packet_loss", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, promStreamLabels) promPacketLoss = prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: livekitNamespace, Subsystem: "packet_loss", Name: "percent", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, Buckets: []float64{0.0, 0.1, 0.3, 0.5, 0.7, 1, 5, 10, 40, 100}, }, promStreamLabels) promJitter = prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: livekitNamespace, Subsystem: "jitter", Name: "us", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, // 1ms, 10ms, 30ms, 50ms, 70ms, 100ms, 300ms, 600ms, 1s Buckets: []float64{1000, 10000, 30000, 50000, 70000, 100000, 300000, 600000, 1000000}, @@ -124,20 +124,20 @@ func initPacketStats(nodeID string, nodeType livekit.NodeType, env string) { Namespace: livekitNamespace, Subsystem: "rtt", Name: "ms", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, Buckets: []float64{50, 100, 150, 200, 250, 500, 750, 1000, 5000, 10000}, }, promStreamLabels) promParticipantJoin = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "participant_join", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, []string{"state"}) promConnections = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: livekitNamespace, Subsystem: "connection", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, []string{"kind"}) prometheus.MustRegister(promPacketTotal) diff --git a/pkg/telemetry/prometheus/quality.go b/pkg/telemetry/prometheus/quality.go index 481b7558f..c9eaf6b65 100644 --- a/pkg/telemetry/prometheus/quality.go +++ b/pkg/telemetry/prometheus/quality.go @@ -26,26 +26,26 @@ var ( qualityDrop *prometheus.CounterVec ) -func initQualityStats(nodeID string, nodeType livekit.NodeType, env string) { +func initQualityStats(nodeID string, nodeType livekit.NodeType) { qualityRating = prometheus.NewHistogram(prometheus.HistogramOpts{ Namespace: livekitNamespace, Subsystem: "quality", Name: "rating", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, Buckets: []float64{0, 1, 2}, }) qualityScore = prometheus.NewHistogram(prometheus.HistogramOpts{ Namespace: livekitNamespace, Subsystem: "quality", Name: "score", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, Buckets: []float64{1.0, 2.0, 2.5, 3.0, 3.25, 3.5, 3.75, 4.0, 4.25, 4.5}, }) qualityDrop = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "quality", Name: "drop", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, []string{"direction"}) prometheus.MustRegister(qualityRating) diff --git a/pkg/telemetry/prometheus/rooms.go b/pkg/telemetry/prometheus/rooms.go index 91c7cc2ba..6c7540d78 100644 --- a/pkg/telemetry/prometheus/rooms.go +++ b/pkg/telemetry/prometheus/rooms.go @@ -47,18 +47,18 @@ var ( promSessionStartTime *prometheus.HistogramVec ) -func initRoomStats(nodeID string, nodeType livekit.NodeType, env string) { +func initRoomStats(nodeID string, nodeType livekit.NodeType) { promRoomCurrent = prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: livekitNamespace, Subsystem: "room", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }) promRoomDuration = prometheus.NewHistogram(prometheus.HistogramOpts{ Namespace: livekitNamespace, Subsystem: "room", Name: "duration_seconds", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, Buckets: []float64{ 5, 10, 60, 5 * 60, 10 * 60, 30 * 60, 60 * 60, 2 * 60 * 60, 5 * 60 * 60, 10 * 60 * 60, }, @@ -67,37 +67,37 @@ func initRoomStats(nodeID string, nodeType livekit.NodeType, env string) { Namespace: livekitNamespace, Subsystem: "participant", Name: "total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }) promTrackPublishedCurrent = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: livekitNamespace, Subsystem: "track", Name: "published_total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, []string{"kind"}) promTrackSubscribedCurrent = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: livekitNamespace, Subsystem: "track", Name: "subscribed_total", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, []string{"kind"}) promTrackPublishCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "track", Name: "publish_counter", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, []string{"kind", "state"}) promTrackSubscribeCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "track", Name: "subscribe_counter", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, []string{"state", "error"}) promSessionStartTime = prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: livekitNamespace, Subsystem: "session", Name: "start_time_ms", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, Buckets: prometheus.ExponentialBucketsRange(100, 10000, 15), }, []string{"protocol_version"}) diff --git a/pkg/telemetry/stats_test.go b/pkg/telemetry/stats_test.go index 1758c8b45..89051ee92 100644 --- a/pkg/telemetry/stats_test.go +++ b/pkg/telemetry/stats_test.go @@ -29,7 +29,7 @@ import ( ) func init() { - prometheus.Init("test", livekit.NodeType_SERVER, "test") + prometheus.Init("test", livekit.NodeType_SERVER) } type telemetryServiceFixture struct { diff --git a/test/integration_helpers.go b/test/integration_helpers.go index 0ce079c23..81db35ed4 100644 --- a/test/integration_helpers.go +++ b/test/integration_helpers.go @@ -58,7 +58,7 @@ var roomClient livekit.RoomService func init() { config.InitLoggerFromConfig(&config.DefaultConfig.Logging) - prometheus.Init("test", livekit.NodeType_SERVER, "test") + prometheus.Init("test", livekit.NodeType_SERVER) } func setupSingleNodeTest(name string) (*service.LivekitServer, func()) { From 6b0f7403efaca07bb37cf1bf3770b8e437adc4d4 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 9 Apr 2024 11:19:41 +0530 Subject: [PATCH 36/78] Log fix. (#2637) Else, it was logging something like `candidateError: json: unsupported type: func() interface {}` --- pkg/rtc/transport.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index ea4d7420a..257820320 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -1336,10 +1336,7 @@ func (t *PCTransport) handleLocalICECandidate(e *event) error { filtered := false if c != nil { if t.preferTCP.Load() && c.Protocol != webrtc.ICEProtocolTCP { - t.params.Logger.Debugw("filtering out local candidate", - "candidate", func() interface{} { - return c.String() - }) + t.params.Logger.Debugw("filtering out local candidate", "candidate", c.String()) filtered = true } t.connectionDetails.AddLocalCandidate(c, filtered) From 4b7e5dc1cc5dc417f26420c854e9019a15cfe62f Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Tue, 9 Apr 2024 23:14:15 -0700 Subject: [PATCH 37/78] reduce gc from stream allocator rate monitor (#2638) * reduce gc from stream allocator rate monitor * deps * comment out rate monitor --- go.mod | 3 +- go.sum | 6 +- pkg/rtc/signalhandler.go | 7 -- pkg/sfu/streamallocator/ratemonitor.go | 88 +++++++++++----------- pkg/sfu/streamallocator/streamallocator.go | 10 +-- 5 files changed, 55 insertions(+), 59 deletions(-) diff --git a/go.mod b/go.mod index 0cb8c9a77..2b54b57af 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20240406063423-a67d961689df - github.com/livekit/protocol v1.12.1-0.20240403204952-bc6c7ffd71f0 + github.com/livekit/protocol v1.12.1-0.20240410060226-b6a979d8cfce github.com/livekit/psrpc v0.5.3-0.20240327035954-cec3a0e614be github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 @@ -55,6 +55,7 @@ require ( ) require ( + github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect diff --git a/go.sum b/go.sum index d616ba9e6..c105929f9 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/avast/retry-go/v4 v4.5.1 h1:AxIx0HGi4VZ3I02jr78j5lZ3M6x1E0Ivxa6b0pUUh7o= github.com/avast/retry-go/v4 v4.5.1/go.mod h1:/sipNsvNB3RRuT5iNcb6h73nw3IBmXJ/H3XrCQYSOpc= +github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= +github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bep/debounce v1.2.1 h1:v67fRdBA9UQu2NhLFXrSg0Brw7CexQekrBwDMM8bzeY= @@ -132,8 +134,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20240406063423-a67d961689df h1:DVhJRlF6/CtiyxJVy3QsbS9bf7GyUMuRZONMwZxIWpY= github.com/livekit/mediatransportutil v0.0.0-20240406063423-a67d961689df/go.mod h1:jwKUCmObuiEDH0iiuJHaGMXwRs3RjrB4G6qqgkr/5oE= -github.com/livekit/protocol v1.12.1-0.20240403204952-bc6c7ffd71f0 h1:BfQN4k6YG+XTdfbXnA2XZLAXinRNlJrGdTLAjyOW2Rg= -github.com/livekit/protocol v1.12.1-0.20240403204952-bc6c7ffd71f0/go.mod h1:mcv7L2DWB6iRckI++egmwFU7YEy3W+aLHnusNhioqi0= +github.com/livekit/protocol v1.12.1-0.20240410060226-b6a979d8cfce h1:cbuw8FQ5S1vX6avOmlj6f8IwliXsOGjOa3UR0YDucX8= +github.com/livekit/protocol v1.12.1-0.20240410060226-b6a979d8cfce/go.mod h1:jB6PWwf4tdMAwy+jxexqaVWuQiiklAtO4F5zZzWkTII= github.com/livekit/psrpc v0.5.3-0.20240327035954-cec3a0e614be h1:W1nCFZ19rYAORMBNX82NeVPHjADN0UyORr6refbUXpU= github.com/livekit/psrpc v0.5.3-0.20240327035954-cec3a0e614be/go.mod h1:CQUBSPfYYAaevg1TNCc6/aYsa8DJH4jSRFdCeSZk5u0= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= diff --git a/pkg/rtc/signalhandler.go b/pkg/rtc/signalhandler.go index e835cf6ba..cab96e23e 100644 --- a/pkg/rtc/signalhandler.go +++ b/pkg/rtc/signalhandler.go @@ -57,13 +57,6 @@ func HandleParticipantSignal(room types.Room, participant types.LocalParticipant case *livekit.SignalRequest_Leave: pLogger.Debugw("client leaving room") room.RemoveParticipant(participant.Identity(), participant.ID(), types.ParticipantCloseReasonClientRequestLeave) - case *livekit.SignalRequest_UpdateLayers: - err := room.UpdateVideoLayers(participant, msg.UpdateLayers) - if err != nil { - pLogger.Warnw("could not update video layers", err, - "update", msg.UpdateLayers) - return nil - } case *livekit.SignalRequest_SubscriptionPermission: err := room.UpdateSubscriptionPermission(participant, msg.SubscriptionPermission) if err != nil { diff --git a/pkg/sfu/streamallocator/ratemonitor.go b/pkg/sfu/streamallocator/ratemonitor.go index 9c2ba243e..f2275822f 100644 --- a/pkg/sfu/streamallocator/ratemonitor.go +++ b/pkg/sfu/streamallocator/ratemonitor.go @@ -16,6 +16,7 @@ package streamallocator import ( "fmt" + "sync" "time" "github.com/livekit/protocol/utils/timeseries" @@ -31,6 +32,7 @@ const ( // ------------------------------------------------ type RateMonitor struct { + mu sync.Mutex bitrateEstimate *timeseries.TimeSeries[int64] managedBytesSent *timeseries.TimeSeries[uint32] managedBytesRetransmitted *timeseries.TimeSeries[uint32] @@ -67,6 +69,9 @@ func NewRateMonitor() *RateMonitor { } func (r *RateMonitor) Update(estimate int64, managedBytesSent uint32, managedBytesRetransmitted uint32, unmanagedBytesSent uint32, unmanagedBytesRetransmitted uint32) { + r.mu.Lock() + defer r.mu.Unlock() + now := time.Now() r.bitrateEstimate.AddSampleAt(estimate, now) r.managedBytesSent.AddSampleAt(managedBytesSent, now) @@ -82,36 +87,36 @@ func (r *RateMonitor) Update(estimate int64, managedBytesSent uint32, managedByt // Reason is that the estimate could be higher than the actual rate by a significant amount. // So, updating periodically to flush out samples that will not contribute to queueing would be good. func (r *RateMonitor) GetQueuingGuess() float64 { - _, _, _, _, _, qd := r.getRates(queueMonitorWindow) - return qd + _, _, _, _, _, queuingDelay := r.getRates(queueMonitorWindow) + return queuingDelay } -func (r *RateMonitor) getRates(monitorDuration time.Duration) (float64, float64, float64, float64, float64, float64) { - threshold := time.Now().Add(-monitorDuration) - bitrateEstimateSamples := r.bitrateEstimate.GetSamplesAfter(threshold) - managedBytesSentSamples := r.managedBytesSent.GetSamplesAfter(threshold) - managedBytesRetransmittedSamples := r.managedBytesRetransmitted.GetSamplesAfter(threshold) - unmanagedBytesSentSamples := r.unmanagedBytesSent.GetSamplesAfter(threshold) - unmanagedBytesRetransmittedSamples := r.unmanagedBytesRetransmitted.GetSamplesAfter(threshold) +func (r *RateMonitor) getRates(monitorDuration time.Duration) (totalBitrateEstimate, totalManagedSent, totalManagedRetransmitted, totalUnmanagedSent, totalUnmanagedRetransmitted, queuingDelay float64) { + r.mu.Lock() + defer r.mu.Unlock() - if len(bitrateEstimateSamples) == 0 || (len(managedBytesSentSamples)+len(managedBytesRetransmittedSamples)+len(unmanagedBytesSentSamples)+len(unmanagedBytesRetransmittedSamples)) == 0 { - return 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 + threshold := time.Now().Add(-monitorDuration) + if !r.bitrateEstimate.HasSamplesAfter(threshold) || + !(r.managedBytesSent.HasSamplesAfter(threshold) || + r.managedBytesRetransmitted.HasSamplesAfter(threshold) || + r.unmanagedBytesSent.HasSamplesAfter(threshold) || + r.unmanagedBytesRetransmitted.HasSamplesAfter(threshold)) { + return } - totalBitrateEstimate := getTimeWeightedSum(bitrateEstimateSamples) - totalManagedSent := getRate(managedBytesSentSamples) * 8 - totalManagedRetransmitted := getRate(managedBytesRetransmittedSamples) * 8 - totalUnmanagedSent := getRate(unmanagedBytesSentSamples) * 8 - totalUnmanagedRetransmitted := getRate(unmanagedBytesRetransmittedSamples) * 8 + totalBitrateEstimate = getTimeWeightedSum(r.bitrateEstimate.ReverseIterateSamplesAfter(threshold)) + totalManagedSent = getRate(r.managedBytesSent.ReverseIterateSamplesAfter(threshold)) * 8 + totalManagedRetransmitted = getRate(r.managedBytesRetransmitted.ReverseIterateSamplesAfter(threshold)) * 8 + totalUnmanagedSent = getRate(r.unmanagedBytesSent.ReverseIterateSamplesAfter(threshold)) * 8 + totalUnmanagedRetransmitted = getRate(r.unmanagedBytesRetransmitted.ReverseIterateSamplesAfter(threshold)) * 8 totalBits := totalManagedSent + totalManagedRetransmitted + totalUnmanagedSent + totalUnmanagedRetransmitted - queuingDelay := float64(0.0) if totalBits > totalBitrateEstimate { - latestBitrateEstimate := bitrateEstimateSamples[len(bitrateEstimateSamples)-1].Value + latestBitrateEstimate := r.bitrateEstimate.Back().Value excessBits := totalBits - totalBitrateEstimate queuingDelay = excessBits / float64(latestBitrateEstimate) } - return totalBitrateEstimate, totalManagedSent, totalManagedRetransmitted, totalUnmanagedSent, totalUnmanagedRetransmitted, queuingDelay + return } func (r *RateMonitor) updateHistory() { @@ -124,10 +129,12 @@ func (r *RateMonitor) updateHistory() { return } + r.mu.Lock() r.history = append( r.history, fmt.Sprintf("t: %+v, e: %.2f, m: %.2f/%.2f, um: %.2f/%.2f, qd: %.2f", time.Now().UnixMilli(), e, m, mr, um, umr, qd), ) + r.mu.Unlock() } func (r *RateMonitor) GetHistory() []string { @@ -136,37 +143,30 @@ func (r *RateMonitor) GetHistory() []string { // ------------------------------------------------ -func getTimeWeightedSum[T int64 | uint32](samples []timeseries.TimeSeriesSample[T]) float64 { - if len(samples) < 2 { - return 0.0 - } - +func getTimeWeightedSum[T int64 | uint32](it timeseries.ReverseIterator[T]) float64 { sum := 0.0 - for i := 1; i < len(samples); i++ { - diff := samples[i].At.Sub(samples[i-1].At).Seconds() - sum += diff * float64(samples[i-1].Value) + next := time.Now() + for it.Next() { + diff := next.Sub(it.Value().At).Seconds() + sum += diff * float64(it.Value().Value) + next = it.Value().At } - - diff := time.Now().Sub(samples[len(samples)-1].At).Seconds() - sum += diff * float64(samples[len(samples)-1].Value) return sum } -func getRate[T int64 | uint32](samples []timeseries.TimeSeriesSample[T]) float64 { - if len(samples) < 2 { - return 0.0 +func getRate[T int64 | uint32](it timeseries.ReverseIterator[T]) float64 { + var sum float64 + var first, last time.Time + for it.Next() { + if last.IsZero() { + last = it.Value().At + } + first = it.Value().At + sum += float64(it.Value().Value) } - sum := 0.0 - // start at 1 as the first sample duration is not available - for i := 1; i < len(samples); i++ { - sum += float64(samples[i].Value) + if duration := last.Sub(first); duration > 0 { + return sum / duration.Seconds() } - - duration := samples[len(samples)-1].At.Sub(samples[0].At) - if duration == 0 { - return 0.0 - } - - return sum / duration.Seconds() + return 0 } diff --git a/pkg/sfu/streamallocator/streamallocator.go b/pkg/sfu/streamallocator/streamallocator.go index 3abc28a6b..08ecd3421 100644 --- a/pkg/sfu/streamallocator/streamallocator.go +++ b/pkg/sfu/streamallocator/streamallocator.go @@ -157,7 +157,7 @@ type StreamAllocator struct { prober *Prober channelObserver *ChannelObserver - rateMonitor *RateMonitor + // rateMonitor *RateMonitor videoTracksMu sync.RWMutex videoTracks map[livekit.TrackID]*Track @@ -178,7 +178,7 @@ func NewStreamAllocator(params StreamAllocatorParams) *StreamAllocator { prober: NewProber(ProberParams{ Logger: params.Logger, }), - rateMonitor: NewRateMonitor(), + // rateMonitor: NewRateMonitor(), videoTracks: make(map[livekit.TrackID]*Track), eventsQueue: utils.NewOpsQueue(utils.OpsQueueParams{ Name: "stream-allocator", @@ -825,8 +825,8 @@ func (s *StreamAllocator) handleNewEstimateInNonProbe() { ) s.params.Logger.Debugw( fmt.Sprintf("stream allocator: channel congestion detected, %s channel capacity: experimental", action), - "rateHistory", s.rateMonitor.GetHistory(), - "expectedQueuing", s.rateMonitor.GetQueuingGuess(), + // "rateHistory", s.rateMonitor.GetHistory(), + // "expectedQueuing", s.rateMonitor.GetQueuingGuess(), "nackHistory", s.channelObserver.GetNackHistory(), "trackHistory", s.getTracksHistory(), ) @@ -1431,7 +1431,7 @@ func (s *StreamAllocator) monitorRate(estimate int64) { } } - s.rateMonitor.Update(estimate, managedBytesSent, managedBytesRetransmitted, unmanagedBytesSent, unmanagedBytesRetransmitted) + // s.rateMonitor.Update(estimate, managedBytesSent, managedBytesRetransmitted, unmanagedBytesSent, unmanagedBytesRetransmitted) } func (s *StreamAllocator) updateTracksHistory() { From c6ee34d083e80591be3914c1928194e2a65b1226 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 10 Apr 2024 13:31:25 +0530 Subject: [PATCH 38/78] Cleaning up stream allocator data. (#2639) * Cleaning up stream allocator data. Marking it with STREAM-ALLOCATOR-DATA for easier use later if needed. * clean up a bit more * wire_gen * wire_gen --- pkg/service/wire_gen.go | 2 +- pkg/sfu/downtrack.go | 18 ++++++++- pkg/sfu/streamallocator/channelobserver.go | 2 + pkg/sfu/streamallocator/nacktracker.go | 10 +++-- pkg/sfu/streamallocator/streamallocator.go | 46 ++++++++++++++-------- pkg/sfu/streamallocator/track.go | 24 +++++------ 6 files changed, 68 insertions(+), 34 deletions(-) diff --git a/pkg/service/wire_gen.go b/pkg/service/wire_gen.go index 80b8bd8c0..08d493b13 100644 --- a/pkg/service/wire_gen.go +++ b/pkg/service/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run github.com/google/wire/cmd/wire +//go:generate go run -mod=mod github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index d02e9de4d..7e9081901 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -144,11 +144,13 @@ func (d DownTrackState) String() string { // ------------------------------------------------------------------- +/* STREAM-ALLOCATOR-DATA type NackInfo struct { Timestamp uint32 SequenceNumber uint16 Attempts uint8 } +*/ type DownTrackStreamAllocatorListener interface { // RTCP received @@ -179,11 +181,13 @@ type DownTrackStreamAllocatorListener interface { // packet(s) sent OnPacketsSent(dt *DownTrack, size int) + /* STREAM-ALLOCATOR-DATA // NACKs received OnNACK(dt *DownTrack, nackInfos []NackInfo) // RTCP Receiver Report received OnRTCPReceiverReport(dt *DownTrack, rr rtcp.ReceptionReport) + */ // check if track should participate in BWE IsBWEEnabled(dt *DownTrack) bool @@ -266,8 +270,10 @@ type DownTrack struct { streamAllocatorListener DownTrackStreamAllocatorListener streamAllocatorReportGeneration int streamAllocatorBytesCounter atomic.Uint32 + /* STREAM-ALLOCATOR-DATA bytesSent atomic.Uint32 bytesRetransmitted atomic.Uint32 + */ playoutDelay *PlayoutDelayController @@ -1538,9 +1544,11 @@ func (d *DownTrack) handleRTCP(bytes []byte) { rttToReport = rtt } + /* STREAM-ALLOCATOR-DATA if sal := d.getStreamAllocatorListener(); sal != nil { sal.OnRTCPReceiverReport(d, r) } + */ if d.playoutDelay != nil { jitterMs := uint64(r.Jitter*1e3) / uint64(d.codec.ClockRate) @@ -1655,18 +1663,20 @@ func (d *DownTrack) retransmitPackets(nacks []uint16) { nackAcks := uint32(0) nackMisses := uint32(0) numRepeatedNACKs := uint32(0) - nackInfos := make([]NackInfo, 0, len(filtered)) + // STREAM-ALLOCATOR-DATA nackInfos := make([]NackInfo, 0, len(filtered)) for _, epm := range d.sequencer.getExtPacketMetas(filtered) { if disallowedLayers[epm.layer] { continue } nackAcks++ + /* STREAM-ALLOCATOR-DATA nackInfos = append(nackInfos, NackInfo{ SequenceNumber: epm.targetSeqNo, Timestamp: epm.timestamp, Attempts: epm.nacked, }) + */ pktBuff := *src n, err := d.params.Receiver.ReadRTP(pktBuff, uint8(epm.layer), epm.sourceSeqNo) @@ -1738,6 +1748,7 @@ func (d *DownTrack) retransmitPackets(nacks []uint16) { d.totalRepeatedNACKs.Add(numRepeatedNACKs) d.rtpStats.UpdateNackProcessed(nackAcks, nackMisses, numRepeatedNACKs) + /* STREAM-ALLOCATOR-DATA // STREAM-ALLOCATOR-EXPERIMENTAL-TODO-START // Need to check on the following // - get all NACKs from sequencer even if SFU is not acknowledging, @@ -1753,6 +1764,7 @@ func (d *DownTrack) retransmitPackets(nacks []uint16) { if sal := d.getStreamAllocatorListener(); sal != nil && len(nackInfos) != 0 { sal.OnNACK(d, nackInfos) } + */ } func (d *DownTrack) getTranslatedRTPHeader(extPkt *buffer.ExtPacket, tp *TranslationParams) (*rtp.Header, error) { @@ -1841,9 +1853,11 @@ func (d *DownTrack) GetNackStats() (totalPackets uint32, totalRepeatedNACKs uint return } +/* STREAM-ALLOCATOR-DATA func (d *DownTrack) GetAndResetBytesSent() (uint32, uint32) { return d.bytesSent.Swap(0), d.bytesRetransmitted.Swap(0) } +*/ func (d *DownTrack) onBindAndConnectedChange() { d.writable.Store(d.connected.Load() && d.bound.Load()) @@ -1981,11 +1995,13 @@ func (d *DownTrack) sendingPacket(hdr *rtp.Header, payloadSize int, spmd *sendPa // STREAM-ALLOCATOR-TODO: remove this stream allocator bytes counter once stream allocator changes fully to pull bytes counter size := uint32(hdrSize + payloadSize) d.streamAllocatorBytesCounter.Add(size) + /* STREAM-ALLOCATOR-DATA if spmd.isRTX { d.bytesRetransmitted.Add(size) } else { d.bytesSent.Add(size) } + */ } // update RTPStats diff --git a/pkg/sfu/streamallocator/channelobserver.go b/pkg/sfu/streamallocator/channelobserver.go index 9afab9c49..585484408 100644 --- a/pkg/sfu/streamallocator/channelobserver.go +++ b/pkg/sfu/streamallocator/channelobserver.go @@ -134,9 +134,11 @@ func (c *ChannelObserver) GetNackRatio() float64 { return c.nackTracker.GetRatio() } +/* STREAM-ALLOCATOR-DATA func (c *ChannelObserver) GetNackHistory() []string { return c.nackTracker.GetHistory() } +*/ func (c *ChannelObserver) GetTrend() (ChannelTrend, ChannelCongestionReason) { estimateDirection := c.estimateTrend.GetDirection() diff --git a/pkg/sfu/streamallocator/nacktracker.go b/pkg/sfu/streamallocator/nacktracker.go index b353781e5..c7131a01b 100644 --- a/pkg/sfu/streamallocator/nacktracker.go +++ b/pkg/sfu/streamallocator/nacktracker.go @@ -38,20 +38,22 @@ type NackTracker struct { packets uint32 repeatedNacks uint32 + /* STREAM-ALLOCATOR-DATA // STREAM-ALLOCATOR-EXPERIMENTAL-TODO: remove when cleaning up experimental stuff history []string + */ } func NewNackTracker(params NackTrackerParams) *NackTracker { return &NackTracker{ - params: params, - history: make([]string, 0, 10), + params: params, + // STREAM-ALLOCATOR-DATA history: make([]string, 0, 10), } } func (n *NackTracker) Add(packets uint32, repeatedNacks uint32) { if n.params.WindowMaxDuration != 0 && !n.windowStartTime.IsZero() && time.Since(n.windowStartTime) > n.params.WindowMaxDuration { - n.updateHistory() + // STREAM-ALLOCATOR-DATA n.updateHistory() n.windowStartTime = time.Time{} n.packets = 0 @@ -104,6 +106,7 @@ func (n *NackTracker) ToString() string { return fmt.Sprintf("n: %s, %s, p: %d, rn: %d, rn/p: %.2f", n.params.Name, window, n.packets, n.repeatedNacks, n.GetRatio()) } +/* STREAM-ALLOCATOR-DATA func (n *NackTracker) GetHistory() []string { return n.history } @@ -115,5 +118,6 @@ func (n *NackTracker) updateHistory() { n.history = append(n.history, n.ToString()) } +*/ // ------------------------------------------------ diff --git a/pkg/sfu/streamallocator/streamallocator.go b/pkg/sfu/streamallocator/streamallocator.go index 08ecd3421..3bff2aca5 100644 --- a/pkg/sfu/streamallocator/streamallocator.go +++ b/pkg/sfu/streamallocator/streamallocator.go @@ -85,8 +85,8 @@ const ( streamAllocatorSignalResume streamAllocatorSignalSetAllowPause streamAllocatorSignalSetChannelCapacity - streamAllocatorSignalNACK - streamAllocatorSignalRTCPReceiverReport + // STREAM-ALLOCATOR-DATA streamAllocatorSignalNACK + // STREAM-ALLOCATOR-DATA streamAllocatorSignalRTCPReceiverReport ) func (s streamAllocatorSignal) String() string { @@ -111,10 +111,12 @@ func (s streamAllocatorSignal) String() string { return "SET_ALLOW_PAUSE" case streamAllocatorSignalSetChannelCapacity: return "SET_CHANNEL_CAPACITY" - case streamAllocatorSignalNACK: - return "NACK" - case streamAllocatorSignalRTCPReceiverReport: - return "RTCP_RECEIVER_REPORT" + /* STREAM-ALLOCATOR-DATA + case streamAllocatorSignalNACK: + return "NACK" + case streamAllocatorSignalRTCPReceiverReport: + return "RTCP_RECEIVER_REPORT" + */ default: return fmt.Sprintf("%d", int(s)) } @@ -157,7 +159,7 @@ type StreamAllocator struct { prober *Prober channelObserver *ChannelObserver - // rateMonitor *RateMonitor + // STREAM-ALLOCATOR-DATA rateMonitor *RateMonitor videoTracksMu sync.RWMutex videoTracks map[livekit.TrackID]*Track @@ -178,7 +180,7 @@ func NewStreamAllocator(params StreamAllocatorParams) *StreamAllocator { prober: NewProber(ProberParams{ Logger: params.Logger, }), - // rateMonitor: NewRateMonitor(), + // STREAM-ALLOCATOR-DATA rateMonitor: NewRateMonitor(), videoTracks: make(map[livekit.TrackID]*Track), eventsQueue: utils.NewOpsQueue(utils.OpsQueueParams{ Name: "stream-allocator", @@ -461,6 +463,7 @@ func (s *StreamAllocator) OnPacketsSent(downTrack *sfu.DownTrack, size int) { s.prober.PacketsSent(size) } +/* STREAM-ALLOCATOR-DATA // called by a video DownTrack when it processes NACKs func (s *StreamAllocator) OnNACK(downTrack *sfu.DownTrack, nackInfos []sfu.NackInfo) { s.postEvent(Event{ @@ -479,6 +482,7 @@ func (s *StreamAllocator) OnRTCPReceiverReport(downTrack *sfu.DownTrack, rr rtcp Data: rr, }) } +*/ // called when prober wants to send packet(s) func (s *StreamAllocator) OnSendProbe(bytesToSend int) { @@ -598,10 +602,12 @@ func (s *StreamAllocator) handleEvent(event *Event) { s.handleSignalSetAllowPause(event) case streamAllocatorSignalSetChannelCapacity: s.handleSignalSetChannelCapacity(event) - case streamAllocatorSignalNACK: - s.handleSignalNACK(event) - case streamAllocatorSignalRTCPReceiverReport: - s.handleSignalRTCPReceiverReport(event) + /* STREAM-ALLOCATOR-DATA + case streamAllocatorSignalNACK: + s.handleSignalNACK(event) + case streamAllocatorSignalRTCPReceiverReport: + s.handleSignalRTCPReceiverReport(event) + */ } } @@ -635,7 +641,7 @@ func (s *StreamAllocator) handleSignalAdjustState(event *Event) { func (s *StreamAllocator) handleSignalEstimate(event *Event) { receivedEstimate, _ := event.Data.(int64) s.lastReceivedEstimate = receivedEstimate - s.monitorRate(receivedEstimate) + // s.monitorRate(receivedEstimate) // while probing, maintain estimate separately to enable keeping current committed estimate if probe fails if s.probeController.IsInProbe() { @@ -662,7 +668,7 @@ func (s *StreamAllocator) handleSignalPeriodicPing(event *Event) { s.maybeProbe() } - s.updateTracksHistory() + // s.updateTracksHistory() } func (s *StreamAllocator) handleSignalSendProbe(event *Event) { @@ -719,6 +725,7 @@ func (s *StreamAllocator) handleSignalSetChannelCapacity(event *Event) { } } +/* STREAM-ALLOCATOR-DATA func (s *StreamAllocator) handleSignalNACK(event *Event) { nackInfos := event.Data.([]sfu.NackInfo) @@ -742,6 +749,7 @@ func (s *StreamAllocator) handleSignalRTCPReceiverReport(event *Event) { track.ProcessRTCPReceiverReport(rr) } } +*/ func (s *StreamAllocator) setState(state streamAllocatorState) { if s.state == state { @@ -823,13 +831,15 @@ func (s *StreamAllocator) handleNewEstimateInNonProbe() { "commitThreshold(bps)", commitThreshold, "channel", s.channelObserver.ToString(), ) + /* STREAM-ALLOCATOR-DATA s.params.Logger.Debugw( fmt.Sprintf("stream allocator: channel congestion detected, %s channel capacity: experimental", action), - // "rateHistory", s.rateMonitor.GetHistory(), - // "expectedQueuing", s.rateMonitor.GetQueuingGuess(), + "rateHistory", s.rateMonitor.GetHistory(), + "expectedQueuing", s.rateMonitor.GetQueuingGuess(), "nackHistory", s.channelObserver.GetNackHistory(), "trackHistory", s.getTracksHistory(), ) + */ if estimateToCommit > commitThreshold { // estimate to commit is either higher or within tolerance of expected uage, skip committing and re-allocating return @@ -1407,6 +1417,7 @@ func (s *StreamAllocator) getMaxDistanceSortedDeficient() MaxDistanceSorter { return maxDistanceSorter } +/* STREAM-ALLOCATOR-DATA // STREAM-ALLOCATOR-EXPERIMENTAL-TODO // Monitor sent rate vs estimate to figure out queuing on congestion. // Idea here is to pause all managed tracks on congestion detection immediately till queue drains. @@ -1431,7 +1442,7 @@ func (s *StreamAllocator) monitorRate(estimate int64) { } } - // s.rateMonitor.Update(estimate, managedBytesSent, managedBytesRetransmitted, unmanagedBytesSent, unmanagedBytesRetransmitted) + s.rateMonitor.Update(estimate, managedBytesSent, managedBytesRetransmitted, unmanagedBytesSent, unmanagedBytesRetransmitted) } func (s *StreamAllocator) updateTracksHistory() { @@ -1449,6 +1460,7 @@ func (s *StreamAllocator) getTracksHistory() map[livekit.TrackID]string { return history } +*/ // ------------------------------------------------ diff --git a/pkg/sfu/streamallocator/track.go b/pkg/sfu/streamallocator/track.go index be93b9e4b..1a0669585 100644 --- a/pkg/sfu/streamallocator/track.go +++ b/pkg/sfu/streamallocator/track.go @@ -15,14 +15,8 @@ package streamallocator import ( - "fmt" - "sort" - "time" - - "github.com/livekit/mediatransportutil" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" - "github.com/pion/rtcp" "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/livekit-server/pkg/sfu/buffer" @@ -41,6 +35,7 @@ type Track struct { totalPackets uint32 totalRepeatedNacks uint32 + /* STREAM-ALLOCATOR-DATA nackInfos map[uint16]sfu.NackInfo // STREAM-ALLOCATOR-EXPERIMENTAL-TODO: remove after experimental nackHistory []string @@ -53,6 +48,7 @@ type Track struct { maxRTT uint32 // STREAM-ALLOCATOR-EXPERIMENTAL-TODO: remove after experimental receiverReportHistory []string + */ isDirty bool @@ -67,15 +63,17 @@ func NewTrack( logger logger.Logger, ) *Track { t := &Track{ - downTrack: downTrack, - source: source, - isSimulcast: isSimulcast, - publisherID: publisherID, - logger: logger, + downTrack: downTrack, + source: source, + isSimulcast: isSimulcast, + publisherID: publisherID, + logger: logger, + /* STREAM-ALLOCATOR-DATA nackInfos: make(map[uint16]sfu.NackInfo), nackHistory: make([]string, 0, 10), receiverReportHistory: make([]string, 0, 10), - streamState: StreamStateInactive, + */ + streamState: StreamStateInactive, } t.SetPriority(0) t.SetMaxLayer(downTrack.MaxLayer()) @@ -220,6 +218,7 @@ func (t *Track) GetNackDelta() (uint32, uint32) { return packetDelta, nackDelta } +/* STREAM-ALLOCATOR-DATA func (t *Track) UpdateNack(nackInfos []sfu.NackInfo) { for _, ni := range nackInfos { t.nackInfos[ni.SequenceNumber] = ni @@ -363,6 +362,7 @@ func (t *Track) updateReceiverReportHistory() { fmt.Sprintf("t: %+v, l: %d, p: %d, rtt: %d", time.Now().Format(time.UnixDate), dl, dp, maxRTT), ) } +*/ // ------------------------------------------------ From e1b68012a1e8c3ff5cd795830b8892245b7d2d4b Mon Sep 17 00:00:00 2001 From: wanshuangcheng <166296003+wanshuangcheng@users.noreply.github.com> Date: Thu, 11 Apr 2024 00:27:48 +0800 Subject: [PATCH 39/78] chore: fix typos in comment (#2634) Signed-off-by: wanshuangcheng --- pkg/sfu/codecmunger/vp8.go | 2 +- pkg/sfu/connectionquality/scorer.go | 2 +- pkg/sfu/redreceiver_test.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/sfu/codecmunger/vp8.go b/pkg/sfu/codecmunger/vp8.go index c26ea0e3c..ca352efb2 100644 --- a/pkg/sfu/codecmunger/vp8.go +++ b/pkg/sfu/codecmunger/vp8.go @@ -240,7 +240,7 @@ func (v *VP8) UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap // if there is a gap, packet is forwarded irrespective of temporal layer as it cannot be determined // which layer the missing packets belong to. A layer could have multiple packets. So, keep track - // of pictures that are forwarded even though they will be filterd out based on temporal layer + // of pictures that are forwarded even though they will be filtered out based on temporal layer // requirements. That allows forwarding of the complete picture. if vp8.T && vp8.TID > uint8(maxTemporalLayer) { v.exemptedPictureIds.Set(extPictureId, true) diff --git a/pkg/sfu/connectionquality/scorer.go b/pkg/sfu/connectionquality/scorer.go index b38355c06..61019d013 100644 --- a/pkg/sfu/connectionquality/scorer.go +++ b/pkg/sfu/connectionquality/scorer.go @@ -382,7 +382,7 @@ func (q *qualityScorer) updateAtLocked(stat *windowStat, at time.Time) { // considered (as long as enough time has passed since unmute). // // Similarly, when paused (possibly due to congestion), score is immediately - // set to cMinScore for responsiveness. The layer transision is reest. + // set to cMinScore for responsiveness. The layer transition is reset. // On a resume, quality climbs back up using normal operation. if q.isMuted() || !q.isUnmutedEnough(at) || q.isLayerMuted() || q.isPaused() { q.lastUpdateAt = at diff --git a/pkg/sfu/redreceiver_test.go b/pkg/sfu/redreceiver_test.go index 72261f7db..d19181446 100644 --- a/pkg/sfu/redreceiver_test.go +++ b/pkg/sfu/redreceiver_test.go @@ -204,7 +204,7 @@ func TestRedReceiver(t *testing.T) { verifyRedEncodings(t, dt.lastReceivedPkt, expectPkt) } - // and then a few packets with a large timestmap jump, should contain only primary + // and then a few packets with a large timestamp jump, should contain only primary for _, pkt := range generatePkts(header, 4, 40*tsStep) { red.ForwardRTP(&buffer.ExtPacket{ Packet: pkt, From 407614b28ef130959e50d5d29acde355396bacf0 Mon Sep 17 00:00:00 2001 From: David Colburn Date: Wed, 10 Apr 2024 12:44:46 -0700 Subject: [PATCH 40/78] fix jobRequestAffinity (#2641) --- pkg/service/agentservice.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/service/agentservice.go b/pkg/service/agentservice.go index e3153347c..9a7106438 100644 --- a/pkg/service/agentservice.go +++ b/pkg/service/agentservice.go @@ -390,7 +390,7 @@ func (h *AgentHandler) JobRequestAffinity(ctx context.Context, job *livekit.Job) if len(w.RunningJobs()) > 0 && load > maxLoad { maxLoad = load affinity = 0.5 + load/2 - } else { + } else if affinity == 0 { affinity = 0.5 } } From 21fbda3470b672adc5986d62226a0c4b05eaf9c8 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 11 Apr 2024 10:58:19 +0530 Subject: [PATCH 41/78] Silence some noisy debug logs (#2643) --- pkg/sfu/buffer/rtpstats_receiver.go | 8 +------- pkg/sfu/buffer/rtpstats_sender.go | 2 +- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/pkg/sfu/buffer/rtpstats_receiver.go b/pkg/sfu/buffer/rtpstats_receiver.go index 706be886f..3fc6c19a1 100644 --- a/pkg/sfu/buffer/rtpstats_receiver.go +++ b/pkg/sfu/buffer/rtpstats_receiver.go @@ -399,7 +399,7 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *RTCPSenderReportData) if r.longTermDeltaPropagationDelay != 0 && deltaPropagationDelay > 0 && deltaPropagationDelay > r.longTermDeltaPropagationDelay*time.Duration(cPropagationDelayDeltaThresholdMaxFactor) { - r.logger.Debugw("sharp increase in propagation delay", getPropagationFields()...) // TODO-REMOVE + r.logger.Debugw("sharp increase in propagation delay", getPropagationFields()...) r.propagationDelayDeltaHighCount++ if r.propagationDelayDeltaHighStartTime.IsZero() { r.propagationDelayDeltaHighStartTime = time.Now() @@ -424,12 +424,6 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *RTCPSenderReportData) if propagationDelay > r.propagationDelay { factor = cPropagationDelayRiseFactor } - adjustedPropagationDelay := r.propagationDelay + time.Duration(factor*float64(propagationDelay-r.propagationDelay)) // TODO-REMOVE - fields := append( - getPropagationFields(), - "adjustedPropagationDelay", adjustedPropagationDelay.String(), - ) // TODO-REMOVE - r.logger.Debugw("adapting propagation delay", fields...) // TODO-REMOVE r.propagationDelay += time.Duration(factor * float64(propagationDelay-r.propagationDelay)) } diff --git a/pkg/sfu/buffer/rtpstats_sender.go b/pkg/sfu/buffer/rtpstats_sender.go index af917cd9e..be3f12990 100644 --- a/pkg/sfu/buffer/rtpstats_sender.go +++ b/pkg/sfu/buffer/rtpstats_sender.go @@ -679,7 +679,7 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, publisherSRData *RTCPS rtpDiffSinceLastReport := nowRTPExt - r.srNewest.RTPTimestampExt windowClockRate := float64(rtpDiffSinceLastReport) / timeSinceLastReport.Seconds() if timeSinceLastReport.Seconds() > 0.2 && math.Abs(float64(r.params.ClockRate)-windowClockRate) > 0.2*float64(r.params.ClockRate) { - if r.clockSkewCount%10 == 0 { + if r.clockSkewCount%100 == 0 { fields := append( getFields(), "timeSinceLastReport", timeSinceLastReport.String(), From eaaf44d2a28e896a0b33185f24ab6d4e00700d53 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Wed, 10 Apr 2024 22:44:40 -0700 Subject: [PATCH 42/78] v1.6.0 (#2644) --- CHANGELOG | 40 ++++++++++++++++++++++++++++++++++++++++ version/version.go | 2 +- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index d8408a4c5..362fb85ce 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,6 +2,46 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.6.0] - 2024-04-10 + +### Added +- Support for Participant.Kind. (#2505 #2626) +- Support XR request/response for rtt calculation (#2536) +- Added support for departureTimeout to keep the room open after participant depart (#2549) +- Added support for Egress Proxy (#2570) +- Added support for SIP DTMF data messages. (#2559) +- Add option to enable bitrate based scoring (#2600) +- Agent service: support for orchestration v2 & namespaces (#2545 #2641) +- Ability to disable audio loss proxying. (#2629) + +### Fixed +- Prevent multiple debounce of quality downgrade. (#2499) +- fix pli throttle locking (#2521) +- Use the correct snapshot id for PPS. (#2528) +- Validate SIP trunks and rules when creating new ones. (#2535) +- Remove subscriber if track closed while adding subscriber. (#2537) +- fix #2539, do not kill the keepaliveWorker task when the ping timeout occurs (#2555) +- Improved A/V sync, proper RTCP report past mute. (#2588) +- Protect duplicate subscription. (#2596) +- Fix twcc has chance to miss for firefox simulcast rtx (#2601) +- Limit playout delay change for high jitter (#2635) + +### Changed +- Replace reflect.Equal with generic sliceEqual (#2494) +- Some optimisations in the forwarding path. (#2035) +- Reduce heap for dependency descriptor in forwarding path. (#2496) +- Separate buffer size config for video and audio. (#2498) +- update pion/ice for tcpmux memory improvement (#2500) +- Close published track always. (#2508) +- use dynamic bucket size (#2524) +- Refactoring channel handling (#2532) +- Forward publisher sender report instead of generating. (#2572) +- Notify initial permissions (#2595) +- Replace sleep with sync.Cond to reduce jitter (#2603) +- Prevent large spikes in propagation delay (#2615) +- reduce gc from stream allocator rate monitor (#2638) + + ## [1.5.3] - 2024-02-17 ### Added diff --git a/version/version.go b/version/version.go index 17d23954c..96df3705f 100644 --- a/version/version.go +++ b/version/version.go @@ -14,4 +14,4 @@ package version -const Version = "1.5.3" +const Version = "1.6.0" From ad1f508680eb4c0a32bb5378556130e81aa727a2 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 11 Apr 2024 15:25:10 +0530 Subject: [PATCH 43/78] Add support for "abs-capture-time" extension. (#2640) * Add support for "abs-capture-time" extension. Currently, it is just passed through from publisher -> subscriber side. TODO: Need to store in sequencer and restore for retransmission. * abs-capture-time in retransmissions * clean up * fix test * more test fixes * more test fixes * more test fixes * log only when size is non-zero * log on both sides for debugging * add marshal/unmarshal * normalize abs capture time to SFU clock * comment out adding abs-capture-time from registered extensions --- pkg/rtc/config.go | 12 +- pkg/rtc/mediaengine.go | 80 +++++++++--- pkg/rtc/mediatrackreceiver.go | 2 +- pkg/rtc/participant_sdp.go | 2 +- pkg/rtc/transport.go | 5 +- pkg/sfu/buffer/buffer.go | 75 +++++++----- pkg/sfu/buffer/dependencydescriptorparser.go | 2 +- pkg/sfu/buffer/fps_test.go | 6 +- pkg/sfu/buffer/frameintegrity.go | 2 +- pkg/sfu/buffer/frameintegrity_test.go | 24 ++-- pkg/sfu/downtrack.go | 92 ++++++++++++-- pkg/sfu/forwarder.go | 2 +- pkg/sfu/playoutdelay.go | 10 +- pkg/sfu/playoutdelay_test.go | 6 +- pkg/sfu/receiver.go | 2 +- .../abscapturetime/abscapturetime.go | 114 ++++++++++++++++++ .../dependencydescriptor/bitstreamreader.go | 0 .../dependencydescriptor/bitstreamwriter.go | 0 .../dependencydescriptorextension.go | 0 .../dependencydescriptorextension_test.go | 0 .../dependencydescriptorreader.go | 0 .../dependencydescriptorwriter.go | 0 .../{ => playoutdelay}/playoutdelay.go | 2 +- .../{ => playoutdelay}/playoutdelay_test.go | 2 +- pkg/sfu/sequencer.go | 6 + pkg/sfu/sequencer_test.go | 28 ++++- pkg/sfu/streamtracker/streamtracker_dd.go | 2 +- .../streamtracker/streamtracker_dd_test.go | 2 +- pkg/sfu/videolayerselector/decodetarget.go | 2 +- .../dependencydescriptor.go | 2 +- .../dependencydescriptor_test.go | 2 +- pkg/sfu/videolayerselector/framechain.go | 2 +- 32 files changed, 383 insertions(+), 103 deletions(-) create mode 100644 pkg/sfu/rtpextension/abscapturetime/abscapturetime.go rename pkg/sfu/{ => rtpextension}/dependencydescriptor/bitstreamreader.go (100%) rename pkg/sfu/{ => rtpextension}/dependencydescriptor/bitstreamwriter.go (100%) rename pkg/sfu/{ => rtpextension}/dependencydescriptor/dependencydescriptorextension.go (100%) rename pkg/sfu/{ => rtpextension}/dependencydescriptor/dependencydescriptorextension_test.go (100%) rename pkg/sfu/{ => rtpextension}/dependencydescriptor/dependencydescriptorreader.go (100%) rename pkg/sfu/{ => rtpextension}/dependencydescriptor/dependencydescriptorwriter.go (100%) rename pkg/sfu/rtpextension/{ => playoutdelay}/playoutdelay.go (99%) rename pkg/sfu/rtpextension/{ => playoutdelay}/playoutdelay_test.go (98%) diff --git a/pkg/rtc/config.go b/pkg/rtc/config.go index f8a8054be..5c608d1f2 100644 --- a/pkg/rtc/config.go +++ b/pkg/rtc/config.go @@ -20,7 +20,7 @@ import ( "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/sfu/buffer" - dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" "github.com/livekit/mediatransportutil/pkg/rtcconfig" ) @@ -88,6 +88,7 @@ func NewWebRTCConfig(conf *config.Config) (*WebRTCConfig, error) { sdp.SDESMidURI, sdp.SDESRTPStreamIDURI, sdp.AudioLevelURI, + //act.AbsCaptureTimeURI, }, Video: []string{ sdp.SDESMidURI, @@ -96,6 +97,7 @@ func NewWebRTCConfig(conf *config.Config) (*WebRTCConfig, error) { frameMarking, dd.ExtensionURI, repairedRTPStreamID, + //act.AbsCaptureTimeURI, }, }, RTCPFeedback: RTCPFeedbackConfig{ @@ -115,7 +117,13 @@ func NewWebRTCConfig(conf *config.Config) (*WebRTCConfig, error) { subscriberConfig := DirectionConfig{ StrictACKs: conf.RTC.StrictACKs, RTPHeaderExtension: RTPHeaderExtensionConfig{ - Video: []string{dd.ExtensionURI}, + Video: []string{ + dd.ExtensionURI, + //act.AbsCaptureTimeURI, + }, + Audio: []string{ + //act.AbsCaptureTimeURI, + }, }, RTCPFeedback: RTCPFeedbackConfig{ Video: []webrtc.RTCPFeedback{ diff --git a/pkg/rtc/mediaengine.go b/pkg/rtc/mediaengine.go index 55836b472..a9b1d623b 100644 --- a/pkg/rtc/mediaengine.go +++ b/pkg/rtc/mediaengine.go @@ -29,9 +29,22 @@ const ( videoRTXMimeType = "video/rtx" ) -var opusCodecCapability = webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus, ClockRate: 48000, Channels: 2, SDPFmtpLine: "minptime=10;useinbandfec=1"} -var redCodecCapability = webrtc.RTPCodecCapability{MimeType: sfu.MimeTypeAudioRed, ClockRate: 48000, Channels: 2, SDPFmtpLine: "111/111"} -var videoRTX = webrtc.RTPCodecCapability{MimeType: videoRTXMimeType, ClockRate: 90000} +var opusCodecCapability = webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeOpus, + ClockRate: 48000, + Channels: 2, + SDPFmtpLine: "minptime=10;useinbandfec=1", +} +var redCodecCapability = webrtc.RTPCodecCapability{ + MimeType: sfu.MimeTypeAudioRed, + ClockRate: 48000, + Channels: 2, + SDPFmtpLine: "111/111", +} +var videoRTX = webrtc.RTPCodecCapability{ + MimeType: videoRTXMimeType, + ClockRate: 90000, +} func registerCodecs(me *webrtc.MediaEngine, codecs []*livekit.Codec, rtcpFeedback RTCPFeedbackConfig, filterOutH264HighProfile bool) error { opusCodec := opusCodecCapability @@ -61,32 +74,65 @@ func registerCodecs(me *webrtc.MediaEngine, codecs []*livekit.Codec, rtcpFeedbac h264HighProfileFmtp := "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640032" for _, codec := range []webrtc.RTPCodecParameters{ { - RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP8, ClockRate: 90000, RTCPFeedback: rtcpFeedback.Video}, - PayloadType: 96, + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeVP8, + ClockRate: 90000, + RTCPFeedback: rtcpFeedback.Video, + }, + PayloadType: 96, }, { - RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP9, ClockRate: 90000, SDPFmtpLine: "profile-id=0", RTCPFeedback: rtcpFeedback.Video}, - PayloadType: 98, + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeVP9, + ClockRate: 90000, + SDPFmtpLine: "profile-id=0", + RTCPFeedback: rtcpFeedback.Video, + }, + PayloadType: 98, }, { - RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP9, ClockRate: 90000, SDPFmtpLine: "profile-id=1", RTCPFeedback: rtcpFeedback.Video}, - PayloadType: 100, + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeVP9, + ClockRate: 90000, + SDPFmtpLine: "profile-id=1", + RTCPFeedback: rtcpFeedback.Video, + }, + PayloadType: 100, }, { - RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264, ClockRate: 90000, SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f", RTCPFeedback: rtcpFeedback.Video}, - PayloadType: 125, + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeH264, + ClockRate: 90000, + SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f", + RTCPFeedback: rtcpFeedback.Video, + }, + PayloadType: 125, }, { - RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264, ClockRate: 90000, SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f", RTCPFeedback: rtcpFeedback.Video}, - PayloadType: 108, + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeH264, + ClockRate: 90000, + SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f", + RTCPFeedback: rtcpFeedback.Video, + }, + PayloadType: 108, }, { - RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264, ClockRate: 90000, SDPFmtpLine: h264HighProfileFmtp, RTCPFeedback: rtcpFeedback.Video}, - PayloadType: 123, + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeH264, + ClockRate: 90000, + SDPFmtpLine: h264HighProfileFmtp, + RTCPFeedback: rtcpFeedback.Video, + }, + PayloadType: 123, }, { - RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeAV1, ClockRate: 90000, RTCPFeedback: rtcpFeedback.Video}, - PayloadType: 35, + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeAV1, + ClockRate: 90000, + RTCPFeedback: rtcpFeedback.Video, + }, + PayloadType: 35, }, } { if filterOutH264HighProfile && codec.RTPCodecCapability.SDPFmtpLine == h264HighProfileFmtp { diff --git a/pkg/rtc/mediatrackreceiver.go b/pkg/rtc/mediatrackreceiver.go index a29b2a630..1644ca4ca 100644 --- a/pkg/rtc/mediatrackreceiver.go +++ b/pkg/rtc/mediatrackreceiver.go @@ -34,7 +34,7 @@ import ( "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/livekit-server/pkg/sfu/buffer" - "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" "github.com/livekit/livekit-server/pkg/telemetry" ) diff --git a/pkg/rtc/participant_sdp.go b/pkg/rtc/participant_sdp.go index 2516291d3..d3fae1577 100644 --- a/pkg/rtc/participant_sdp.go +++ b/pkg/rtc/participant_sdp.go @@ -22,7 +22,7 @@ import ( "github.com/pion/sdp/v3" "github.com/pion/webrtc/v3" - dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" "github.com/livekit/protocol/livekit" lksdp "github.com/livekit/protocol/sdp" ) diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index 257820320..74117c8f1 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -47,7 +47,7 @@ import ( "github.com/livekit/livekit-server/pkg/rtc/types" sfuinterceptor "github.com/livekit/livekit-server/pkg/sfu/interceptor" "github.com/livekit/livekit-server/pkg/sfu/pacer" - "github.com/livekit/livekit-server/pkg/sfu/rtpextension" + pd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/playoutdelay" "github.com/livekit/livekit-server/pkg/sfu/streamallocator" sfuutils "github.com/livekit/livekit-server/pkg/sfu/utils" "github.com/livekit/livekit-server/pkg/telemetry/prometheus" @@ -222,9 +222,8 @@ type TransportParams struct { func newPeerConnection(params TransportParams, onBandwidthEstimator func(estimator cc.BandwidthEstimator)) (*webrtc.PeerConnection, *webrtc.MediaEngine, error) { directionConfig := params.DirectionConfig - if params.AllowPlayoutDelay { - directionConfig.RTPHeaderExtension.Video = append(directionConfig.RTPHeaderExtension.Video, rtpextension.PlayoutDelayURI) + directionConfig.RTPHeaderExtension.Video = append(directionConfig.RTPHeaderExtension.Video, pd.PlayoutDelayURI) } // Some of the browser clients do not handle H.264 High Profile in signalling properly. diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go index 8b7678671..04e468d7e 100644 --- a/pkg/sfu/buffer/buffer.go +++ b/pkg/sfu/buffer/buffer.go @@ -31,7 +31,8 @@ import ( "go.uber.org/atomic" "github.com/livekit/livekit-server/pkg/sfu/audio" - dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + act "github.com/livekit/livekit-server/pkg/sfu/rtpextension/abscapturetime" + dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" "github.com/livekit/livekit-server/pkg/sfu/utils" sutils "github.com/livekit/livekit-server/pkg/utils" "github.com/livekit/mediatransportutil" @@ -64,29 +65,30 @@ type ExtPacket struct { KeyFrame bool RawPacket []byte DependencyDescriptor *ExtDependencyDescriptor + AbsCaptureTimeExt *act.AbsCaptureTime } // Buffer contains all packets type Buffer struct { sync.RWMutex - readCond *sync.Cond - bucket *bucket.Bucket - nacker *nack.NackQueue - maxVideoPkts int - maxAudioPkts int - codecType webrtc.RTPCodecType - payloadType uint8 - extPackets deque.Deque[*ExtPacket] - pPackets []pendingPacket - closeOnce sync.Once - mediaSSRC uint32 - clockRate uint32 - lastReport time.Time - twccExt uint8 - audioLevelExt uint8 - bound bool - closed atomic.Bool - mime string + readCond *sync.Cond + bucket *bucket.Bucket + nacker *nack.NackQueue + maxVideoPkts int + maxAudioPkts int + codecType webrtc.RTPCodecType + payloadType uint8 + extPackets deque.Deque[*ExtPacket] + pPackets []pendingPacket + closeOnce sync.Once + mediaSSRC uint32 + clockRate uint32 + lastReport time.Time + twccExtID uint8 + audioLevelExtID uint8 + bound bool + closed atomic.Bool + mime string snRangeMap *utils.RangeMap[uint64, uint64] @@ -120,7 +122,7 @@ type Buffer struct { logger logger.Logger // dependency descriptor - ddExt uint8 + ddExtID uint8 ddParser *DependencyDescriptorParser paused bool @@ -133,6 +135,8 @@ type Buffer struct { primaryBufferForRTX *Buffer rtxPktBuf []byte + + absCaptureTimeExtID uint8 } // NewBuffer constructs a new Buffer @@ -173,7 +177,7 @@ func (b *Buffer) SetTWCCAndExtID(twcc *twcc.Responder, extID uint8) { defer b.Unlock() b.twcc = twcc - b.twccExt = extID + b.twccExtID = extID } func (b *Buffer) SetAudioLevelParams(audioLevelParams audio.AudioLevelParams) { @@ -223,18 +227,21 @@ func (b *Buffer) Bind(params webrtc.RTPParameters, codec webrtc.RTPCodecCapabili for _, ext := range params.HeaderExtensions { switch ext.URI { case dd.ExtensionURI: - b.ddExt = uint8(ext.ID) + b.ddExtID = uint8(ext.ID) frc := NewFrameRateCalculatorDD(b.clockRate, b.logger) for i := range b.frameRateCalculator { b.frameRateCalculator[i] = frc.GetFrameRateCalculatorForSpatial(int32(i)) } - b.ddParser = NewDependencyDescriptorParser(b.ddExt, b.logger, func(spatial, temporal int32) { + b.ddParser = NewDependencyDescriptorParser(b.ddExtID, b.logger, func(spatial, temporal int32) { frc.SetMaxLayer(spatial, temporal) }) case sdp.AudioLevelURI: - b.audioLevelExt = uint8(ext.ID) + b.audioLevelExtID = uint8(ext.ID) b.audioLevel = audio.NewAudioLevel(b.audioLevelParams) + + case act.AbsCaptureTimeURI: + b.absCaptureTimeExtID = uint8(ext.ID) } } @@ -302,8 +309,8 @@ func (b *Buffer) Write(pkt []byte) (n int, err error) { } now := time.Now() - if b.twcc != nil && b.twccExt != 0 && !b.closed.Load() { - if ext := rtpPacket.GetExtension(b.twccExt); ext != nil { + if b.twcc != nil && b.twccExtID != 0 && !b.closed.Load() { + if ext := rtpPacket.GetExtension(b.twccExtID); ext != nil { b.twcc.Push(rtpPacket.SSRC, binary.BigEndian.Uint16(ext[0:2]), now.UnixNano(), rtpPacket.Marker) } } @@ -664,12 +671,12 @@ func (b *Buffer) updateStreamState(p *rtp.Packet, arrivalTime time.Time) RTPFlow } func (b *Buffer) processHeaderExtensions(p *rtp.Packet, arrivalTime time.Time, isRTX bool) { - if b.audioLevelExt != 0 && !isRTX { + if b.audioLevelExtID != 0 && !isRTX { if !b.latestTSForAudioLevelInitialized { b.latestTSForAudioLevelInitialized = true b.latestTSForAudioLevel = p.Timestamp } - if e := p.GetExtension(b.audioLevelExt); e != nil { + if e := p.GetExtension(b.audioLevelExtID); e != nil { ext := rtp.AudioLevelExtension{} if err := ext.Unmarshal(e); err == nil { if (p.Timestamp - b.latestTSForAudioLevel) < (1 << 31) { @@ -729,6 +736,7 @@ func (b *Buffer) getExtPacket(rtpPacket *rtp.Packet, arrivalTime time.Time, flow ep.Spatial = InvalidLayerSpatial // vp8 don't have spatial scalability, reset to invalid } ep.Payload = vp8Packet + case "video/vp9": if ep.DependencyDescriptor == nil { var vp9Packet codecs.VP9Packet @@ -744,8 +752,10 @@ func (b *Buffer) getExtPacket(rtpPacket *rtp.Packet, arrivalTime time.Time, flow ep.Payload = vp9Packet } ep.KeyFrame = IsVP9KeyFrame(rtpPacket.Payload) + case "video/h264": ep.KeyFrame = IsH264KeyFrame(rtpPacket.Payload) + case "video/av1": ep.KeyFrame = IsAV1KeyFrame(rtpPacket.Payload) } @@ -756,6 +766,15 @@ func (b *Buffer) getExtPacket(rtpPacket *rtp.Packet, arrivalTime time.Time, flow } } + if b.absCaptureTimeExtID != 0 { + extData := rtpPacket.GetExtension(b.absCaptureTimeExtID) + + var actExt act.AbsCaptureTime + if err := actExt.Unmarshal(extData); err == nil { + ep.AbsCaptureTimeExt = &actExt + } + } + return ep } diff --git a/pkg/sfu/buffer/dependencydescriptorparser.go b/pkg/sfu/buffer/dependencydescriptorparser.go index da79c301b..30851442c 100644 --- a/pkg/sfu/buffer/dependencydescriptorparser.go +++ b/pkg/sfu/buffer/dependencydescriptorparser.go @@ -21,7 +21,7 @@ import ( "github.com/pion/rtp" "go.uber.org/atomic" - dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" "github.com/livekit/livekit-server/pkg/sfu/utils" "github.com/livekit/protocol/logger" diff --git a/pkg/sfu/buffer/fps_test.go b/pkg/sfu/buffer/fps_test.go index 4a85d2808..39dd93df3 100644 --- a/pkg/sfu/buffer/fps_test.go +++ b/pkg/sfu/buffer/fps_test.go @@ -20,7 +20,7 @@ import ( "github.com/pion/rtp" "github.com/stretchr/testify/require" - "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" "github.com/livekit/protocol/logger" ) @@ -46,9 +46,9 @@ func (f *testFrameInfo) toDD() *ExtPacket { return &ExtPacket{ Packet: &rtp.Packet{Header: f.header}, DependencyDescriptor: &ExtDependencyDescriptor{ - Descriptor: &dependencydescriptor.DependencyDescriptor{ + Descriptor: &dd.DependencyDescriptor{ FrameNumber: f.framenumber, - FrameDependencies: &dependencydescriptor.FrameDependencyTemplate{ + FrameDependencies: &dd.FrameDependencyTemplate{ FrameDiffs: f.frameDiff, }, }, diff --git a/pkg/sfu/buffer/frameintegrity.go b/pkg/sfu/buffer/frameintegrity.go index 536f793cf..935263b7d 100644 --- a/pkg/sfu/buffer/frameintegrity.go +++ b/pkg/sfu/buffer/frameintegrity.go @@ -15,7 +15,7 @@ package buffer import ( - dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" ) type FrameEntity struct { diff --git a/pkg/sfu/buffer/frameintegrity_test.go b/pkg/sfu/buffer/frameintegrity_test.go index 144a9f0f2..2815cf5e9 100644 --- a/pkg/sfu/buffer/frameintegrity_test.go +++ b/pkg/sfu/buffer/frameintegrity_test.go @@ -20,47 +20,47 @@ import ( "github.com/stretchr/testify/require" - "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" ) func TestFrameIntegrityChecker(t *testing.T) { fc := NewFrameIntegrityChecker(100, 1000) // first frame out of order - fc.AddPacket(10, 10, &dependencydescriptor.DependencyDescriptor{}) + fc.AddPacket(10, 10, &dd.DependencyDescriptor{}) require.False(t, fc.FrameIntegrity(10)) - fc.AddPacket(9, 10, &dependencydescriptor.DependencyDescriptor{FirstPacketInFrame: true}) + fc.AddPacket(9, 10, &dd.DependencyDescriptor{FirstPacketInFrame: true}) require.False(t, fc.FrameIntegrity(10)) - fc.AddPacket(11, 10, &dependencydescriptor.DependencyDescriptor{LastPacketInFrame: true}) + fc.AddPacket(11, 10, &dd.DependencyDescriptor{LastPacketInFrame: true}) require.True(t, fc.FrameIntegrity(10)) // single packet frame - fc.AddPacket(100, 100, &dependencydescriptor.DependencyDescriptor{FirstPacketInFrame: true, LastPacketInFrame: true}) + fc.AddPacket(100, 100, &dd.DependencyDescriptor{FirstPacketInFrame: true, LastPacketInFrame: true}) require.True(t, fc.FrameIntegrity(100)) require.False(t, fc.FrameIntegrity(101)) require.False(t, fc.FrameIntegrity(99)) // frame too old than first frame - fc.AddPacket(99, 99, &dependencydescriptor.DependencyDescriptor{FirstPacketInFrame: true, LastPacketInFrame: true}) + fc.AddPacket(99, 99, &dd.DependencyDescriptor{FirstPacketInFrame: true, LastPacketInFrame: true}) // multiple packet frame, out of order - fc.AddPacket(2001, 2001, &dependencydescriptor.DependencyDescriptor{}) + fc.AddPacket(2001, 2001, &dd.DependencyDescriptor{}) require.False(t, fc.FrameIntegrity(2001)) require.False(t, fc.FrameIntegrity(1999)) // out of frame count(100) require.False(t, fc.FrameIntegrity(100)) require.False(t, fc.FrameIntegrity(1900)) - fc.AddPacket(2000, 2001, &dependencydescriptor.DependencyDescriptor{FirstPacketInFrame: true}) + fc.AddPacket(2000, 2001, &dd.DependencyDescriptor{FirstPacketInFrame: true}) require.False(t, fc.FrameIntegrity(2001)) - fc.AddPacket(2002, 2001, &dependencydescriptor.DependencyDescriptor{LastPacketInFrame: true}) + fc.AddPacket(2002, 2001, &dd.DependencyDescriptor{LastPacketInFrame: true}) require.True(t, fc.FrameIntegrity(2001)) // duplicate packet - fc.AddPacket(2001, 2001, &dependencydescriptor.DependencyDescriptor{}) + fc.AddPacket(2001, 2001, &dd.DependencyDescriptor{}) require.True(t, fc.FrameIntegrity(2001)) // frame too old - fc.AddPacket(900, 1900, &dependencydescriptor.DependencyDescriptor{FirstPacketInFrame: true, LastPacketInFrame: true}) + fc.AddPacket(900, 1900, &dd.DependencyDescriptor{FirstPacketInFrame: true, LastPacketInFrame: true}) require.False(t, fc.FrameIntegrity(1900)) for frame := uint64(2002); frame < 2102; frame++ { @@ -75,7 +75,7 @@ func TestFrameIntegrityChecker(t *testing.T) { rand.Seed(int64(frame)) rand.Shuffle(len(frames), func(i, j int) { frames[i], frames[j] = frames[j], frames[i] }) for i, f := range frames { - fc.AddPacket(f, frame, &dependencydescriptor.DependencyDescriptor{ + fc.AddPacket(f, frame, &dd.DependencyDescriptor{ FirstPacketInFrame: f == firstFrame, LastPacketInFrame: f == lastFrame, }) diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 7e9081901..878fcf19a 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -35,9 +35,10 @@ import ( "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/livekit-server/pkg/sfu/connectionquality" - dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" "github.com/livekit/livekit-server/pkg/sfu/pacer" - "github.com/livekit/livekit-server/pkg/sfu/rtpextension" + act "github.com/livekit/livekit-server/pkg/sfu/rtpextension/abscapturetime" + dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" + pd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/playoutdelay" "github.com/livekit/livekit-server/pkg/sfu/utils" ) @@ -237,6 +238,7 @@ type DownTrack struct { transportWideExtID int dependencyDescriptorExtID int playoutDelayExtID int + absCaptureTimeExtID int transceiver atomic.Pointer[webrtc.RTPTransceiver] writeStream webrtc.TrackLocalWriter rtcpReader *buffer.RTCPReader @@ -553,7 +555,7 @@ func (d *DownTrack) SetRTPHeaderExtensions(rtpHeaderExtensions []webrtc.RTPHeade } case dd.ExtensionURI: d.dependencyDescriptorExtID = ext.ID - case rtpextension.PlayoutDelayURI: + case pd.PlayoutDelayURI: d.playoutDelayExtID = ext.ID case sdp.TransportCCURI: if isBWEEnabled { @@ -561,6 +563,8 @@ func (d *DownTrack) SetRTPHeaderExtensions(rtpHeaderExtensions []webrtc.RTPHeade } else { d.transportWideExtID = 0 } + case act.AbsCaptureTimeURI: + d.absCaptureTimeExtID = ext.ID } } } @@ -730,13 +734,59 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { var extensions []pacer.ExtensionData if tp.ddBytes != nil { - extensions = []pacer.ExtensionData{{ID: uint8(d.dependencyDescriptorExtID), Payload: tp.ddBytes}} + extensions = append( + extensions, + pacer.ExtensionData{ + ID: uint8(d.dependencyDescriptorExtID), + Payload: tp.ddBytes, + }, + ) } if d.playoutDelayExtID != 0 && d.playoutDelay != nil { if val := d.playoutDelay.GetDelayExtension(hdr.SequenceNumber); val != nil { - extensions = append(extensions, pacer.ExtensionData{ID: uint8(d.playoutDelayExtID), Payload: val}) + extensions = append( + extensions, + pacer.ExtensionData{ + ID: uint8(d.playoutDelayExtID), + Payload: val, + }, + ) + + // NOTE: play out delay extension is not cached in sequencer, + // i. e. they will not be added to retransmitted packet. + // But, it is okay as the extension is added till a RTCP Receiver Report for + // the corresponding sequence number is received. + // The extreme case is all packets containing the play out delay are lost and + // all of them retransmitted and an RTCP Receiver Report received for those + // retransmited sequence numbers. But, that is highly improbable, if not impossible. } } + var actBytes []byte + if extPkt.AbsCaptureTimeExt != nil && d.absCaptureTimeExtID != 0 { + // normalize capture time to SFU clock. + // NOTE: even if there is estimated offset populated, just re-map the + // absolute capture time stamp as it should be the same RTCP sender report + // clock domain of publisher. SFU is normalising sender reports of publisher + // to SFU clock before sending to subscribers. So, capture time should be + // normalized to the same clock. Clear out any offset. + _, _, refSenderReport := d.forwarder.GetSenderReportParams() + if refSenderReport != nil { + actExtCopy := *extPkt.AbsCaptureTimeExt + if err = actExtCopy.Rewrite(refSenderReport.AtAdjusted.Sub(refSenderReport.NTPTimestamp.Time())); err == nil { + actBytes, err = actExtCopy.Marshal() + if err == nil { + extensions = append( + extensions, + pacer.ExtensionData{ + ID: uint8(d.absCaptureTimeExtID), + Payload: actBytes, + }, + ) + } + } + } + } + if d.sequencer != nil { d.sequencer.push( extPkt.Arrival, @@ -748,6 +798,7 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { payload[:outgoingHeaderSize], incomingHeaderSize, tp.ddBytes, + actBytes, ) } @@ -1715,11 +1766,30 @@ func (d *DownTrack) retransmitPackets(nacks []uint16) { payload = payload[:int(epm.numCodecBytesOut)+len(pkt.Payload)-int(epm.numCodecBytesIn)] } - var ddBytes []byte - if len(epm.ddBytesSlice) != 0 { - ddBytes = epm.ddBytesSlice - } else { - ddBytes = epm.ddBytes[:epm.ddBytesSize] + var extensions []pacer.ExtensionData + if d.dependencyDescriptorExtID != 0 { + var ddBytes []byte + if len(epm.ddBytesSlice) != 0 { + ddBytes = epm.ddBytesSlice + } else { + ddBytes = epm.ddBytes[:epm.ddBytesSize] + } + extensions = append( + extensions, + pacer.ExtensionData{ + ID: uint8(d.dependencyDescriptorExtID), + Payload: ddBytes, + }, + ) + } + if d.absCaptureTimeExtID != 0 && len(epm.actBytes) != 0 { + extensions = append( + extensions, + pacer.ExtensionData{ + ID: uint8(d.absCaptureTimeExtID), + Payload: epm.actBytes, + }, + ) } d.sendingPacket( @@ -1735,7 +1805,7 @@ func (d *DownTrack) retransmitPackets(nacks []uint16) { ) d.pacer.Enqueue(pacer.Packet{ Header: &pkt.Header, - Extensions: []pacer.ExtensionData{{ID: uint8(d.dependencyDescriptorExtID), Payload: ddBytes}}, + Extensions: extensions, Payload: payload, AbsSendTimeExtID: uint8(d.absSendTimeExtID), TransportWideExtID: uint8(d.transportWideExtID), diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 721dd6c9e..af610e34c 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -31,7 +31,7 @@ import ( "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/livekit-server/pkg/sfu/codecmunger" - dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" "github.com/livekit/livekit-server/pkg/sfu/videolayerselector" "github.com/livekit/livekit-server/pkg/sfu/videolayerselector/temporallayerselector" ) diff --git a/pkg/sfu/playoutdelay.go b/pkg/sfu/playoutdelay.go index 991f1401b..c0ed985b0 100644 --- a/pkg/sfu/playoutdelay.go +++ b/pkg/sfu/playoutdelay.go @@ -20,7 +20,7 @@ import ( "time" "github.com/livekit/livekit-server/pkg/sfu/buffer" - "github.com/livekit/livekit-server/pkg/sfu/rtpextension" + pd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/playoutdelay" "github.com/livekit/protocol/logger" ) @@ -67,10 +67,10 @@ type PlayoutDelayController struct { func NewPlayoutDelayController(minDelay, maxDelay uint32, logger logger.Logger, rtpStats *buffer.RTPStatsSender) (*PlayoutDelayController, error) { if maxDelay == 0 && minDelay > 0 { - maxDelay = rtpextension.MaxPlayoutDelayDefault + maxDelay = pd.MaxPlayoutDelayDefault } - if maxDelay > rtpextension.PlayoutDelayMaxValue { - maxDelay = rtpextension.PlayoutDelayMaxValue + if maxDelay > pd.PlayoutDelayMaxValue { + maxDelay = pd.PlayoutDelayMaxValue } c := &PlayoutDelayController{ currentDelay: minDelay, @@ -153,7 +153,7 @@ func (c *PlayoutDelayController) GetDelayExtension(seq uint16) []byte { } func (c *PlayoutDelayController) createExtData() error { - delay := rtpextension.PlayoutDelayFromValue( + delay := pd.PlayoutDelayFromValue( uint16(c.currentDelay), uint16(c.maxDelay), ) diff --git a/pkg/sfu/playoutdelay_test.go b/pkg/sfu/playoutdelay_test.go index cc24d2848..338b4d64b 100644 --- a/pkg/sfu/playoutdelay_test.go +++ b/pkg/sfu/playoutdelay_test.go @@ -21,7 +21,7 @@ import ( "github.com/stretchr/testify/require" "github.com/livekit/livekit-server/pkg/sfu/buffer" - "github.com/livekit/livekit-server/pkg/sfu/rtpextension" + pd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/playoutdelay" "github.com/livekit/protocol/logger" ) @@ -60,7 +60,7 @@ func TestPlayoutDelay(t *testing.T) { c.SetJitter(50) t.Log(c.currentDelay, c.state.Load()) ext = c.GetDelayExtension(108) - var delay rtpextension.PlayOutDelay + var delay pd.PlayOutDelay require.NoError(t, delay.Unmarshal(ext)) require.Greater(t, delay.Min, uint16(100)) @@ -72,7 +72,7 @@ func TestPlayoutDelay(t *testing.T) { } func playoutDelayEqual(t *testing.T, data []byte, min, max uint16) { - var delay rtpextension.PlayOutDelay + var delay pd.PlayOutDelay require.NoError(t, delay.Unmarshal(data)) require.Equal(t, min, delay.Min) require.Equal(t, max, delay.Max) diff --git a/pkg/sfu/receiver.go b/pkg/sfu/receiver.go index e6e9c54ae..78d49fb2b 100644 --- a/pkg/sfu/receiver.go +++ b/pkg/sfu/receiver.go @@ -35,7 +35,7 @@ import ( "github.com/livekit/livekit-server/pkg/sfu/audio" "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/livekit-server/pkg/sfu/connectionquality" - dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" ) var ( diff --git a/pkg/sfu/rtpextension/abscapturetime/abscapturetime.go b/pkg/sfu/rtpextension/abscapturetime/abscapturetime.go new file mode 100644 index 000000000..1eb3f8d2a --- /dev/null +++ b/pkg/sfu/rtpextension/abscapturetime/abscapturetime.go @@ -0,0 +1,114 @@ +// Copyright 2024 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package abscapturetime + +import ( + "encoding/binary" + "errors" + "time" + + "github.com/livekit/mediatransportutil" +) + +const ( + AbsCaptureTimeURI = "http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time" +) + +var ( + errInvalidData = errors.New("invalid data") + errTooSmall = errors.New("buffer too small") +) + +// Reference: https://webrtc.googlesource.com/src/+/refs/heads/main/docs/native-code/rtp-hdrext/abs-capture-time/ +// +// Data layout of the shortened version of abs-capture-time with a 1-byte header + 8 bytes of data: +// +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | ID | len=7 | absolute capture timestamp (bit 0-23) | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | absolute capture timestamp (bit 24-55) | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | ... (56-63) | +// +-+-+-+-+-+-+-+-+ +// +//Data layout of the extended version of abs-capture-time with a 1-byte header + 16 bytes of data: +// +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | ID | len=15| absolute capture timestamp (bit 0-23) | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | absolute capture timestamp (bit 24-55) | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | ... (56-63) | estimated capture clock offset (bit 0-23) | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | estimated capture clock offset (bit 24-55) | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | ... (56-63) | +// +-+-+-+-+-+-+-+-+ + +type AbsCaptureTime struct { + absoluteCaptureTimestamp mediatransportutil.NtpTime + estimatedCaptureClockOffset int64 +} + +func AbsCaptureTimeFromValue(absoluteCaptureTimestamp uint64, estimatedCaptureClockOffset int64) *AbsCaptureTime { + return &AbsCaptureTime{ + absoluteCaptureTimestamp: mediatransportutil.NtpTime(absoluteCaptureTimestamp), + estimatedCaptureClockOffset: estimatedCaptureClockOffset, + } +} + +func (a *AbsCaptureTime) Rewrite(offset time.Duration) error { + if a.absoluteCaptureTimestamp == 0 { + return errInvalidData + } + + capturedAt := a.absoluteCaptureTimestamp.Time().Add(offset) + a.absoluteCaptureTimestamp = mediatransportutil.ToNtpTime(capturedAt) + a.estimatedCaptureClockOffset = 0 + return nil +} + +func (a *AbsCaptureTime) Marshal() ([]byte, error) { + if a.absoluteCaptureTimestamp == 0 { + return nil, errInvalidData + } + + size := 8 + if a.estimatedCaptureClockOffset != 0 { + size += 8 + } + marshalled := make([]byte, size) + binary.BigEndian.PutUint64(marshalled, uint64(a.absoluteCaptureTimestamp)) + if a.estimatedCaptureClockOffset != 0 { + binary.BigEndian.PutUint64(marshalled[8:], uint64(a.estimatedCaptureClockOffset)) + } + return marshalled, nil +} + +func (a *AbsCaptureTime) Unmarshal(marshalled []byte) error { + if len(marshalled) < 8 { + return errTooSmall + } + + a.absoluteCaptureTimestamp = mediatransportutil.NtpTime(binary.BigEndian.Uint64(marshalled)) + if len(marshalled) >= 16 { + a.estimatedCaptureClockOffset = int64(binary.BigEndian.Uint64(marshalled[8:])) + } + return nil +} diff --git a/pkg/sfu/dependencydescriptor/bitstreamreader.go b/pkg/sfu/rtpextension/dependencydescriptor/bitstreamreader.go similarity index 100% rename from pkg/sfu/dependencydescriptor/bitstreamreader.go rename to pkg/sfu/rtpextension/dependencydescriptor/bitstreamreader.go diff --git a/pkg/sfu/dependencydescriptor/bitstreamwriter.go b/pkg/sfu/rtpextension/dependencydescriptor/bitstreamwriter.go similarity index 100% rename from pkg/sfu/dependencydescriptor/bitstreamwriter.go rename to pkg/sfu/rtpextension/dependencydescriptor/bitstreamwriter.go diff --git a/pkg/sfu/dependencydescriptor/dependencydescriptorextension.go b/pkg/sfu/rtpextension/dependencydescriptor/dependencydescriptorextension.go similarity index 100% rename from pkg/sfu/dependencydescriptor/dependencydescriptorextension.go rename to pkg/sfu/rtpextension/dependencydescriptor/dependencydescriptorextension.go diff --git a/pkg/sfu/dependencydescriptor/dependencydescriptorextension_test.go b/pkg/sfu/rtpextension/dependencydescriptor/dependencydescriptorextension_test.go similarity index 100% rename from pkg/sfu/dependencydescriptor/dependencydescriptorextension_test.go rename to pkg/sfu/rtpextension/dependencydescriptor/dependencydescriptorextension_test.go diff --git a/pkg/sfu/dependencydescriptor/dependencydescriptorreader.go b/pkg/sfu/rtpextension/dependencydescriptor/dependencydescriptorreader.go similarity index 100% rename from pkg/sfu/dependencydescriptor/dependencydescriptorreader.go rename to pkg/sfu/rtpextension/dependencydescriptor/dependencydescriptorreader.go diff --git a/pkg/sfu/dependencydescriptor/dependencydescriptorwriter.go b/pkg/sfu/rtpextension/dependencydescriptor/dependencydescriptorwriter.go similarity index 100% rename from pkg/sfu/dependencydescriptor/dependencydescriptorwriter.go rename to pkg/sfu/rtpextension/dependencydescriptor/dependencydescriptorwriter.go diff --git a/pkg/sfu/rtpextension/playoutdelay.go b/pkg/sfu/rtpextension/playoutdelay/playoutdelay.go similarity index 99% rename from pkg/sfu/rtpextension/playoutdelay.go rename to pkg/sfu/rtpextension/playoutdelay/playoutdelay.go index 2e311a621..d1017846e 100644 --- a/pkg/sfu/rtpextension/playoutdelay.go +++ b/pkg/sfu/rtpextension/playoutdelay/playoutdelay.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package rtpextension +package playoutdelay import ( "encoding/binary" diff --git a/pkg/sfu/rtpextension/playoutdelay_test.go b/pkg/sfu/rtpextension/playoutdelay/playoutdelay_test.go similarity index 98% rename from pkg/sfu/rtpextension/playoutdelay_test.go rename to pkg/sfu/rtpextension/playoutdelay/playoutdelay_test.go index 7c07ed26c..5a2dc078b 100644 --- a/pkg/sfu/rtpextension/playoutdelay_test.go +++ b/pkg/sfu/rtpextension/playoutdelay/playoutdelay_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package rtpextension +package playoutdelay import ( "testing" diff --git a/pkg/sfu/sequencer.go b/pkg/sfu/sequencer.go index 0fafcfc81..a78b23eb6 100644 --- a/pkg/sfu/sequencer.go +++ b/pkg/sfu/sequencer.go @@ -75,6 +75,8 @@ type packetMeta struct { ddBytes [8]byte ddBytesSize uint8 ddBytesSlice []byte + // abs-capture-time of packet + actBytes []byte } type extPacketMeta struct { @@ -134,6 +136,7 @@ func (s *sequencer) push( codecBytes []byte, numCodecBytesIn int, ddBytes []byte, + actBytes []byte, ) { s.Lock() defer s.Unlock() @@ -220,6 +223,8 @@ func (s *sequencer) push( copy(pm.ddBytes[:pm.ddBytesSize], ddBytes) } + pm.actBytes = append([]byte{}, actBytes...) + if extModifiedSN > s.extHighestSN { s.extHighestSN = extModifiedSN } @@ -344,6 +349,7 @@ func (s *sequencer) getExtPacketMetas(seqNo []uint16) []extPacketMeta { } epm.codecBytesSlice = append([]byte{}, meta.codecBytesSlice...) epm.ddBytesSlice = append([]byte{}, meta.ddBytesSlice...) + epm.actBytes = append([]byte{}, meta.actBytes...) extPacketMetas = append(extPacketMetas, epm) } } diff --git a/pkg/sfu/sequencer_test.go b/pkg/sfu/sequencer_test.go index 39dcec906..f8eabe045 100644 --- a/pkg/sfu/sequencer_test.go +++ b/pkg/sfu/sequencer_test.go @@ -29,11 +29,11 @@ func Test_sequencer(t *testing.T) { off := uint16(15) for i := uint64(1); i < 518; i++ { - seq.push(time.Now(), i, i+uint64(off), 123, true, 2, nil, 0, nil) + seq.push(time.Now(), i, i+uint64(off), 123, true, 2, nil, 0, nil, nil) } // send the last two out-of-order - seq.push(time.Now(), 519, 519+uint64(off), 123, false, 2, nil, 0, nil) - seq.push(time.Now(), 518, 518+uint64(off), 123, true, 2, nil, 0, nil) + seq.push(time.Now(), 519, 519+uint64(off), 123, false, 2, nil, 0, nil, nil) + seq.push(time.Now(), 518, 518+uint64(off), 123, true, 2, nil, 0, nil, nil) req := []uint16{57, 58, 62, 63, 513, 514, 515, 516, 517} res := seq.getExtPacketMetas(req) @@ -63,14 +63,14 @@ func Test_sequencer(t *testing.T) { require.Equal(t, val.extTimestamp, uint64(123)) } - seq.push(time.Now(), 521, 521+uint64(off), 123, true, 1, nil, 0, nil) + seq.push(time.Now(), 521, 521+uint64(off), 123, true, 1, nil, 0, nil, nil) m := seq.getExtPacketMetas([]uint16{521 + off}) require.Equal(t, 0, len(m)) time.Sleep((ignoreRetransmission + 10) * time.Millisecond) m = seq.getExtPacketMetas([]uint16{521 + off}) require.Equal(t, 1, len(m)) - seq.push(time.Now(), 505, 505+uint64(off), 123, false, 1, nil, 0, nil) + seq.push(time.Now(), 505, 505+uint64(off), 123, false, 1, nil, 0, nil, nil) m = seq.getExtPacketMetas([]uint16{505 + off}) require.Equal(t, 0, len(m)) time.Sleep((ignoreRetransmission + 10) * time.Millisecond) @@ -99,6 +99,8 @@ func Test_sequencer_getNACKSeqNo_exclusion(t *testing.T) { ddBytesOdd []byte ddBytesEven []byte ddBytesOversized []byte + actBytesOdd []byte + actBytesEven []byte } tests := []struct { @@ -132,6 +134,8 @@ func Test_sequencer_getNACKSeqNo_exclusion(t *testing.T) { ddBytesOdd: []byte{8, 9, 10}, ddBytesEven: []byte{11, 12}, ddBytesOversized: []byte{11, 12, 13, 14, 15, 16, 17, 18, 19}, + actBytesOdd: []byte{0, 1, 2, 3, 4, 5, 6, 7}, + actBytesEven: []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, }, args: args{ seqNo: []uint16{65526 + 5, 65527 + 5, 65530 + 5, 0 /* 65531 input */, 1 /* 65532 input */, 2 /* 65533 input */, 3 /* 65534 input */}, @@ -162,6 +166,7 @@ func Test_sequencer_getNACKSeqNo_exclusion(t *testing.T) { tt.fields.codecBytesOversized, len(tt.fields.codecBytesOversized), tt.fields.ddBytesOversized, + tt.fields.actBytesOdd, ) } else { if i.seqNo%2 == 0 { @@ -175,6 +180,7 @@ func Test_sequencer_getNACKSeqNo_exclusion(t *testing.T) { tt.fields.codecBytesEven, tt.fields.numCodecBytesInEven, tt.fields.ddBytesEven, + tt.fields.actBytesEven, ) } else { n.push( @@ -187,6 +193,7 @@ func Test_sequencer_getNACKSeqNo_exclusion(t *testing.T) { tt.fields.codecBytesOdd, tt.fields.numCodecBytesInOdd, tt.fields.ddBytesOdd, + tt.fields.actBytesOdd, ) } } @@ -204,6 +211,7 @@ func Test_sequencer_getNACKSeqNo_exclusion(t *testing.T) { require.Equal(t, uint8(len(tt.fields.codecBytesOversized)), sn.numCodecBytesIn) require.Equal(t, tt.fields.ddBytesOversized, sn.ddBytesSlice) require.Equal(t, uint8(len(tt.fields.codecBytesOversized)), sn.ddBytesSize) + require.Equal(t, tt.fields.actBytesOdd, sn.actBytes) } else { if sn.sourceSeqNo%2 == 0 { require.Equal(t, tt.fields.markerEven, sn.marker) @@ -211,12 +219,14 @@ func Test_sequencer_getNACKSeqNo_exclusion(t *testing.T) { require.Equal(t, uint8(tt.fields.numCodecBytesInEven), sn.numCodecBytesIn) require.Equal(t, tt.fields.ddBytesEven, sn.ddBytes[:sn.ddBytesSize]) require.Equal(t, uint8(len(tt.fields.ddBytesEven)), sn.ddBytesSize) + require.Equal(t, tt.fields.actBytesEven, sn.actBytes) } else { require.Equal(t, tt.fields.markerOdd, sn.marker) require.Equal(t, tt.fields.codecBytesOdd, sn.codecBytes[:sn.numCodecBytesOut]) require.Equal(t, uint8(tt.fields.numCodecBytesInOdd), sn.numCodecBytesIn) require.Equal(t, tt.fields.ddBytesOdd, sn.ddBytes[:sn.ddBytesSize]) require.Equal(t, uint8(len(tt.fields.ddBytesOdd)), sn.ddBytesSize) + require.Equal(t, tt.fields.actBytesOdd, sn.actBytes) } } } @@ -246,6 +256,8 @@ func Test_sequencer_getNACKSeqNo_no_exclusion(t *testing.T) { numCodecBytesInEven int ddBytesOdd []byte ddBytesEven []byte + actBytesOdd []byte + actBytesEven []byte } tests := []struct { @@ -278,6 +290,8 @@ func Test_sequencer_getNACKSeqNo_no_exclusion(t *testing.T) { numCodecBytesInEven: 4, ddBytesOdd: []byte{8, 9, 10}, ddBytesEven: []byte{11, 12}, + actBytesOdd: []byte{8, 9, 10}, + actBytesEven: []byte{11, 12}, }, args: args{ seqNo: []uint16{4 + 5, 5 + 5, 8 + 5, 9 + 5, 10 + 5, 11 + 5, 12 + 5}, @@ -306,6 +320,7 @@ func Test_sequencer_getNACKSeqNo_no_exclusion(t *testing.T) { tt.fields.codecBytesEven, tt.fields.numCodecBytesInEven, tt.fields.ddBytesEven, + tt.fields.actBytesEven, ) } else { n.push( @@ -318,6 +333,7 @@ func Test_sequencer_getNACKSeqNo_no_exclusion(t *testing.T) { tt.fields.codecBytesOdd, tt.fields.numCodecBytesInOdd, tt.fields.ddBytesOdd, + tt.fields.actBytesOdd, ) } } @@ -334,12 +350,14 @@ func Test_sequencer_getNACKSeqNo_no_exclusion(t *testing.T) { require.Equal(t, uint8(tt.fields.numCodecBytesInEven), sn.numCodecBytesIn) require.Equal(t, tt.fields.ddBytesEven, sn.ddBytes[:sn.ddBytesSize]) require.Equal(t, uint8(len(tt.fields.ddBytesEven)), sn.ddBytesSize) + require.Equal(t, tt.fields.actBytesEven, sn.actBytes) } else { require.Equal(t, tt.fields.markerOdd, sn.marker) require.Equal(t, tt.fields.codecBytesOdd, sn.codecBytes[:sn.numCodecBytesOut]) require.Equal(t, uint8(tt.fields.numCodecBytesInOdd), sn.numCodecBytesIn) require.Equal(t, tt.fields.ddBytesOdd, sn.ddBytes[:sn.ddBytesSize]) require.Equal(t, uint8(len(tt.fields.ddBytesOdd)), sn.ddBytesSize) + require.Equal(t, tt.fields.actBytesOdd, sn.actBytes) } } if !reflect.DeepEqual(got, tt.want) { diff --git a/pkg/sfu/streamtracker/streamtracker_dd.go b/pkg/sfu/streamtracker/streamtracker_dd.go index be8009eb1..29b876c70 100644 --- a/pkg/sfu/streamtracker/streamtracker_dd.go +++ b/pkg/sfu/streamtracker/streamtracker_dd.go @@ -21,7 +21,7 @@ import ( "go.uber.org/atomic" "github.com/livekit/livekit-server/pkg/sfu/buffer" - dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" ) type StreamTrackerDependencyDescriptor struct { diff --git a/pkg/sfu/streamtracker/streamtracker_dd_test.go b/pkg/sfu/streamtracker/streamtracker_dd_test.go index f638e4e2d..8e3f4001c 100644 --- a/pkg/sfu/streamtracker/streamtracker_dd_test.go +++ b/pkg/sfu/streamtracker/streamtracker_dd_test.go @@ -21,7 +21,7 @@ import ( "github.com/stretchr/testify/require" "github.com/livekit/livekit-server/pkg/sfu/buffer" - dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" "github.com/livekit/protocol/logger" ) diff --git a/pkg/sfu/videolayerselector/decodetarget.go b/pkg/sfu/videolayerselector/decodetarget.go index 7204b329f..70114f21f 100644 --- a/pkg/sfu/videolayerselector/decodetarget.go +++ b/pkg/sfu/videolayerselector/decodetarget.go @@ -18,7 +18,7 @@ import ( "fmt" "github.com/livekit/livekit-server/pkg/sfu/buffer" - dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" ) type DecodeTarget struct { diff --git a/pkg/sfu/videolayerselector/dependencydescriptor.go b/pkg/sfu/videolayerselector/dependencydescriptor.go index 15d9df1eb..acfe5b85c 100644 --- a/pkg/sfu/videolayerselector/dependencydescriptor.go +++ b/pkg/sfu/videolayerselector/dependencydescriptor.go @@ -20,7 +20,7 @@ import ( "sync" "github.com/livekit/livekit-server/pkg/sfu/buffer" - dede "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + dede "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" "github.com/livekit/protocol/logger" ) diff --git a/pkg/sfu/videolayerselector/dependencydescriptor_test.go b/pkg/sfu/videolayerselector/dependencydescriptor_test.go index 2b346dedb..a562988f9 100644 --- a/pkg/sfu/videolayerselector/dependencydescriptor_test.go +++ b/pkg/sfu/videolayerselector/dependencydescriptor_test.go @@ -22,7 +22,7 @@ import ( "github.com/stretchr/testify/require" "github.com/livekit/livekit-server/pkg/sfu/buffer" - dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" "github.com/livekit/protocol/logger" ) diff --git a/pkg/sfu/videolayerselector/framechain.go b/pkg/sfu/videolayerselector/framechain.go index 7b226ed0c..836783864 100644 --- a/pkg/sfu/videolayerselector/framechain.go +++ b/pkg/sfu/videolayerselector/framechain.go @@ -15,7 +15,7 @@ package videolayerselector import ( - dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" "github.com/livekit/protocol/logger" ) From d55948f7615978362878478af7838ab04b0d257a Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 11 Apr 2024 20:00:13 +0530 Subject: [PATCH 44/78] Add PropagationDelay API to sender report data (#2646) --- pkg/sfu/buffer/rtpstats_base.go | 11 ++++++++++- pkg/sfu/downtrack.go | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/pkg/sfu/buffer/rtpstats_base.go b/pkg/sfu/buffer/rtpstats_base.go index 77586d38b..83c667dab 100644 --- a/pkg/sfu/buffer/rtpstats_base.go +++ b/pkg/sfu/buffer/rtpstats_base.go @@ -117,12 +117,21 @@ type RTCPSenderReportData struct { AtAdjusted time.Time } +func (r *RTCPSenderReportData) PropagationDelay() time.Duration { + return r.AtAdjusted.Sub(r.NTPTimestamp.Time()) +} + func (r *RTCPSenderReportData) ToString() string { if r == nil { return "" } - return fmt.Sprintf("ntp: %s, rtp: %d, extRtp: %d, at: %s, atAdj: %s", r.NTPTimestamp.Time().String(), r.RTPTimestamp, r.RTPTimestampExt, r.At.String(), r.AtAdjusted.String()) + return fmt.Sprintf("ntp: %s, rtp: %d, extRtp: %d, at: %s, atAdj: %s", + r.NTPTimestamp.Time().String(), + r.RTPTimestamp, + r.RTPTimestampExt, + r.At.String(), r.AtAdjusted.String(), + ) } func (r *RTCPSenderReportData) MarshalLogObject(e zapcore.ObjectEncoder) error { diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 878fcf19a..035472df3 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -772,7 +772,7 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { _, _, refSenderReport := d.forwarder.GetSenderReportParams() if refSenderReport != nil { actExtCopy := *extPkt.AbsCaptureTimeExt - if err = actExtCopy.Rewrite(refSenderReport.AtAdjusted.Sub(refSenderReport.NTPTimestamp.Time())); err == nil { + if err = actExtCopy.Rewrite(refSenderReport.PropagationDelay()); err == nil { actBytes, err = actExtCopy.Marshal() if err == nil { extensions = append( From 990cf6877bfb7d645df666c6085e6e0ca731a452 Mon Sep 17 00:00:00 2001 From: David Colburn Date: Thu, 11 Apr 2024 11:14:15 -0700 Subject: [PATCH 45/78] backwards compatability for IsRecorder (#2647) * backwards compatability for IsRecorder * regenerate fakes --- pkg/rtc/participant.go | 7 ++ pkg/rtc/room.go | 6 +- pkg/rtc/types/interfaces.go | 1 + .../typesfakes/fake_local_participant.go | 65 +++++++++++++++++++ pkg/rtc/types/typesfakes/fake_participant.go | 65 +++++++++++++++++++ 5 files changed, 141 insertions(+), 3 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index b74088629..e49e57f3a 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -332,6 +332,13 @@ func (p *ParticipantImpl) Kind() livekit.ParticipantInfo_Kind { return p.grants.GetParticipantKind() } +func (p *ParticipantImpl) IsRecorder() bool { + p.lock.RLock() + defer p.lock.RUnlock() + + return p.grants.GetParticipantKind() == livekit.ParticipantInfo_EGRESS || p.grants.Video.Recorder +} + func (p *ParticipantImpl) IsDependent() bool { p.lock.RLock() defer p.lock.RUnlock() diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index ad3fc5d18..cc8645a9a 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -417,7 +417,7 @@ func (r *Room) Join(participant types.LocalParticipant, requestSource routing.Me "numParticipants", len(r.participants), ) - if participant.Kind() == livekit.ParticipantInfo_EGRESS && !r.protoRoom.ActiveRecording { + if participant.IsRecorder() && !r.protoRoom.ActiveRecording { r.protoRoom.ActiveRecording = true r.protoProxy.MarkDirty(true) } else { @@ -559,10 +559,10 @@ func (r *Room) RemoveParticipant(identity livekit.ParticipantIdentity, pID livek } immediateChange := false - if p.Kind() == livekit.ParticipantInfo_EGRESS { + if p.IsRecorder() { activeRecording := false for _, op := range r.participants { - if op.Kind() == livekit.ParticipantInfo_EGRESS { + if op.IsRecorder() { activeRecording = true break } diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index ebbfe9620..57d6f3922 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -246,6 +246,7 @@ type Participant interface { State() livekit.ParticipantInfo_State CloseReason() ParticipantCloseReason Kind() livekit.ParticipantInfo_Kind + IsRecorder() bool IsDependent() bool CanSkipBroadcast() bool diff --git a/pkg/rtc/types/typesfakes/fake_local_participant.go b/pkg/rtc/types/typesfakes/fake_local_participant.go index 21f433338..351697226 100644 --- a/pkg/rtc/types/typesfakes/fake_local_participant.go +++ b/pkg/rtc/types/typesfakes/fake_local_participant.go @@ -504,6 +504,16 @@ type FakeLocalParticipant struct { isReadyReturnsOnCall map[int]struct { result1 bool } + IsRecorderStub func() bool + isRecorderMutex sync.RWMutex + isRecorderArgsForCall []struct { + } + isRecorderReturns struct { + result1 bool + } + isRecorderReturnsOnCall map[int]struct { + result1 bool + } IsSubscribedToStub func(livekit.ParticipantID) bool isSubscribedToMutex sync.RWMutex isSubscribedToArgsForCall []struct { @@ -3551,6 +3561,59 @@ func (fake *FakeLocalParticipant) IsReadyReturnsOnCall(i int, result1 bool) { }{result1} } +func (fake *FakeLocalParticipant) IsRecorder() bool { + fake.isRecorderMutex.Lock() + ret, specificReturn := fake.isRecorderReturnsOnCall[len(fake.isRecorderArgsForCall)] + fake.isRecorderArgsForCall = append(fake.isRecorderArgsForCall, struct { + }{}) + stub := fake.IsRecorderStub + fakeReturns := fake.isRecorderReturns + fake.recordInvocation("IsRecorder", []interface{}{}) + fake.isRecorderMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) IsRecorderCallCount() int { + fake.isRecorderMutex.RLock() + defer fake.isRecorderMutex.RUnlock() + return len(fake.isRecorderArgsForCall) +} + +func (fake *FakeLocalParticipant) IsRecorderCalls(stub func() bool) { + fake.isRecorderMutex.Lock() + defer fake.isRecorderMutex.Unlock() + fake.IsRecorderStub = stub +} + +func (fake *FakeLocalParticipant) IsRecorderReturns(result1 bool) { + fake.isRecorderMutex.Lock() + defer fake.isRecorderMutex.Unlock() + fake.IsRecorderStub = nil + fake.isRecorderReturns = struct { + result1 bool + }{result1} +} + +func (fake *FakeLocalParticipant) IsRecorderReturnsOnCall(i int, result1 bool) { + fake.isRecorderMutex.Lock() + defer fake.isRecorderMutex.Unlock() + fake.IsRecorderStub = nil + if fake.isRecorderReturnsOnCall == nil { + fake.isRecorderReturnsOnCall = make(map[int]struct { + result1 bool + }) + } + fake.isRecorderReturnsOnCall[i] = struct { + result1 bool + }{result1} +} + func (fake *FakeLocalParticipant) IsSubscribedTo(arg1 livekit.ParticipantID) bool { fake.isSubscribedToMutex.Lock() ret, specificReturn := fake.isSubscribedToReturnsOnCall[len(fake.isSubscribedToArgsForCall)] @@ -6415,6 +6478,8 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.isPublisherMutex.RUnlock() fake.isReadyMutex.RLock() defer fake.isReadyMutex.RUnlock() + fake.isRecorderMutex.RLock() + defer fake.isRecorderMutex.RUnlock() fake.isSubscribedToMutex.RLock() defer fake.isSubscribedToMutex.RUnlock() fake.issueFullReconnectMutex.RLock() diff --git a/pkg/rtc/types/typesfakes/fake_participant.go b/pkg/rtc/types/typesfakes/fake_participant.go index f20a4cefe..3c06229a1 100644 --- a/pkg/rtc/types/typesfakes/fake_participant.go +++ b/pkg/rtc/types/typesfakes/fake_participant.go @@ -148,6 +148,16 @@ type FakeParticipant struct { isPublisherReturnsOnCall map[int]struct { result1 bool } + IsRecorderStub func() bool + isRecorderMutex sync.RWMutex + isRecorderArgsForCall []struct { + } + isRecorderReturns struct { + result1 bool + } + isRecorderReturnsOnCall map[int]struct { + result1 bool + } KindStub func() livekit.ParticipantInfo_Kind kindMutex sync.RWMutex kindArgsForCall []struct { @@ -954,6 +964,59 @@ func (fake *FakeParticipant) IsPublisherReturnsOnCall(i int, result1 bool) { }{result1} } +func (fake *FakeParticipant) IsRecorder() bool { + fake.isRecorderMutex.Lock() + ret, specificReturn := fake.isRecorderReturnsOnCall[len(fake.isRecorderArgsForCall)] + fake.isRecorderArgsForCall = append(fake.isRecorderArgsForCall, struct { + }{}) + stub := fake.IsRecorderStub + fakeReturns := fake.isRecorderReturns + fake.recordInvocation("IsRecorder", []interface{}{}) + fake.isRecorderMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeParticipant) IsRecorderCallCount() int { + fake.isRecorderMutex.RLock() + defer fake.isRecorderMutex.RUnlock() + return len(fake.isRecorderArgsForCall) +} + +func (fake *FakeParticipant) IsRecorderCalls(stub func() bool) { + fake.isRecorderMutex.Lock() + defer fake.isRecorderMutex.Unlock() + fake.IsRecorderStub = stub +} + +func (fake *FakeParticipant) IsRecorderReturns(result1 bool) { + fake.isRecorderMutex.Lock() + defer fake.isRecorderMutex.Unlock() + fake.IsRecorderStub = nil + fake.isRecorderReturns = struct { + result1 bool + }{result1} +} + +func (fake *FakeParticipant) IsRecorderReturnsOnCall(i int, result1 bool) { + fake.isRecorderMutex.Lock() + defer fake.isRecorderMutex.Unlock() + fake.IsRecorderStub = nil + if fake.isRecorderReturnsOnCall == nil { + fake.isRecorderReturnsOnCall = make(map[int]struct { + result1 bool + }) + } + fake.isRecorderReturnsOnCall[i] = struct { + result1 bool + }{result1} +} + func (fake *FakeParticipant) Kind() livekit.ParticipantInfo_Kind { fake.kindMutex.Lock() ret, specificReturn := fake.kindReturnsOnCall[len(fake.kindArgsForCall)] @@ -1420,6 +1483,8 @@ func (fake *FakeParticipant) Invocations() map[string][][]interface{} { defer fake.isDependentMutex.RUnlock() fake.isPublisherMutex.RLock() defer fake.isPublisherMutex.RUnlock() + fake.isRecorderMutex.RLock() + defer fake.isRecorderMutex.RUnlock() fake.kindMutex.RLock() defer fake.kindMutex.RUnlock() fake.removePublishedTrackMutex.RLock() From ec41d20f81fdad361206862efaed1f61f4c95e81 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 12 Apr 2024 20:39:53 +0530 Subject: [PATCH 46/78] Reduce RED weight in half. (#2648) --- pkg/sfu/connectionquality/connectionstats.go | 8 ++++---- .../connectionquality/connectionstats_test.go | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/pkg/sfu/connectionquality/connectionstats.go b/pkg/sfu/connectionquality/connectionstats.go index 6c955dc71..8f6035a06 100644 --- a/pkg/sfu/connectionquality/connectionstats.go +++ b/pkg/sfu/connectionquality/connectionstats.go @@ -379,7 +379,7 @@ func (cs *ConnectionStats) updateStatsWorker() { // For audio: // // o Opus without FEC or RED suffers the most through packet loss, hence has the highest weight -// o RED with two packet redundancy can absorb two out of every three packets lost, so packet loss is not as detrimental and therefore lower weight +// o RED with two packet redundancy can absorb one out of every two packets lost, so packet loss is not as detrimental and therefore lower weight // // For video: // @@ -396,10 +396,10 @@ func getPacketLossWeight(mimeType string, isFecEnabled bool) float64 { } case strings.EqualFold(mimeType, "audio/red"): - // 10%: fall to GOOD, 30.0%: fall to POOR - plw = 2.0 + // 5%: fall to GOOD, 15.0%: fall to POOR + plw = 4.0 if isFecEnabled { - // 15%: fall to GOOD, 45.0%: fall to POOR + // 7.5%: fall to GOOD, 22.5%: fall to POOR plw /= 1.5 } diff --git a/pkg/sfu/connectionquality/connectionstats_test.go b/pkg/sfu/connectionquality/connectionstats_test.go index c34a6d2ce..0feedac7f 100644 --- a/pkg/sfu/connectionquality/connectionstats_test.go +++ b/pkg/sfu/connectionquality/connectionstats_test.go @@ -563,7 +563,7 @@ func TestConnectionQuality(t *testing.T) { }, }, }, - // "audio/red" - no fec - 0 <= loss < 10%: EXCELLENT, 10% <= loss < 30%: GOOD, >= 30%: POOR + // "audio/red" - no fec - 0 <= loss < 5%: EXCELLENT, 5% <= loss < 15%: GOOD, >= 15%: POOR { name: "audio/red - no fec", mimeType: "audio/red", @@ -571,23 +571,23 @@ func TestConnectionQuality(t *testing.T) { packetsExpected: 200, expectedQualities: []expectedQuality{ { - packetLossPercentage: 8.0, + packetLossPercentage: 4.0, expectedMOS: 4.6, expectedQuality: livekit.ConnectionQuality_EXCELLENT, }, { - packetLossPercentage: 12.0, + packetLossPercentage: 6.0, expectedMOS: 4.1, expectedQuality: livekit.ConnectionQuality_GOOD, }, { - packetLossPercentage: 39.0, + packetLossPercentage: 19.5, expectedMOS: 2.1, expectedQuality: livekit.ConnectionQuality_POOR, }, }, }, - // "audio/red" - fec - 0 <= loss < 15%: EXCELLENT, 15% <= loss < 45%: GOOD, >= 45%: POOR + // "audio/red" - fec - 0 <= loss < 7.5%: EXCELLENT, 7.5% <= loss < 22.5%: GOOD, >= 22.5%: POOR { name: "audio/red - fec", mimeType: "audio/red", @@ -595,17 +595,17 @@ func TestConnectionQuality(t *testing.T) { packetsExpected: 200, expectedQualities: []expectedQuality{ { - packetLossPercentage: 12.0, + packetLossPercentage: 6.0, expectedMOS: 4.6, expectedQuality: livekit.ConnectionQuality_EXCELLENT, }, { - packetLossPercentage: 20.0, + packetLossPercentage: 10.0, expectedMOS: 4.1, expectedQuality: livekit.ConnectionQuality_GOOD, }, { - packetLossPercentage: 60.0, + packetLossPercentage: 30.0, expectedMOS: 2.1, expectedQuality: livekit.ConnectionQuality_POOR, }, From 7d5c991d8d82c01ec2321062e2e17316c1c3ff7a Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Sun, 14 Apr 2024 12:49:13 -0700 Subject: [PATCH 47/78] add disconnected chan to participant (#2650) --- pkg/rtc/participant.go | 11 +++- pkg/rtc/types/interfaces.go | 1 + .../typesfakes/fake_local_participant.go | 65 +++++++++++++++++++ pkg/service/roommanager.go | 9 +-- 4 files changed, 77 insertions(+), 9 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index e49e57f3a..fe55b7bc1 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -147,7 +147,8 @@ type ParticipantImpl struct { isClosed atomic.Bool closeReason atomic.Value // types.ParticipantCloseReason - state atomic.Value // livekit.ParticipantInfo_State + state atomic.Value // livekit.ParticipantInfo_State + disconnected chan struct{} resSinkMu sync.Mutex resSink routing.MessageSink @@ -241,7 +242,8 @@ func NewParticipant(params ParticipantParams) (*ParticipantImpl, error) { return nil, ErrMissingGrants } p := &ParticipantImpl{ - params: params, + params: params, + disconnected: make(chan struct{}), pubRTCPQueue: sutils.NewOpsQueue(sutils.OpsQueueParams{ Name: "pub-rtcp", MinSize: 64, @@ -372,6 +374,10 @@ func (p *ParticipantImpl) IsDisconnected() bool { return p.State() == livekit.ParticipantInfo_DISCONNECTED } +func (p *ParticipantImpl) Disconnected() <-chan struct{} { + return p.disconnected +} + func (p *ParticipantImpl) IsIdle() bool { // check if there are any published tracks that are subscribed for _, t := range p.GetPublishedTracks() { @@ -841,6 +847,7 @@ func (p *ParticipantImpl) Close(sendLeave bool, reason types.ParticipantCloseRea p.UpTrackManager.Close(isExpectedToResume) p.updateState(livekit.ParticipantInfo_DISCONNECTED) + close(p.disconnected) // ensure this is synchronized p.CloseSignalConnection(types.SignallingCloseReasonParticipantClose) diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index 57d6f3922..e072d1935 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -308,6 +308,7 @@ type LocalParticipant interface { IsClosed() bool IsReady() bool IsDisconnected() bool + Disconnected() <-chan struct{} IsIdle() bool SubscriberAsPrimary() bool GetClientInfo() *livekit.ClientInfo diff --git a/pkg/rtc/types/typesfakes/fake_local_participant.go b/pkg/rtc/types/typesfakes/fake_local_participant.go index 351697226..b0843f428 100644 --- a/pkg/rtc/types/typesfakes/fake_local_participant.go +++ b/pkg/rtc/types/typesfakes/fake_local_participant.go @@ -168,6 +168,16 @@ type FakeLocalParticipant struct { debugInfoReturnsOnCall map[int]struct { result1 map[string]interface{} } + DisconnectedStub func() <-chan struct{} + disconnectedMutex sync.RWMutex + disconnectedArgsForCall []struct { + } + disconnectedReturns struct { + result1 <-chan struct{} + } + disconnectedReturnsOnCall map[int]struct { + result1 <-chan struct{} + } GetAdaptiveStreamStub func() bool getAdaptiveStreamMutex sync.RWMutex getAdaptiveStreamArgsForCall []struct { @@ -1770,6 +1780,59 @@ func (fake *FakeLocalParticipant) DebugInfoReturnsOnCall(i int, result1 map[stri }{result1} } +func (fake *FakeLocalParticipant) Disconnected() <-chan struct{} { + fake.disconnectedMutex.Lock() + ret, specificReturn := fake.disconnectedReturnsOnCall[len(fake.disconnectedArgsForCall)] + fake.disconnectedArgsForCall = append(fake.disconnectedArgsForCall, struct { + }{}) + stub := fake.DisconnectedStub + fakeReturns := fake.disconnectedReturns + fake.recordInvocation("Disconnected", []interface{}{}) + fake.disconnectedMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) DisconnectedCallCount() int { + fake.disconnectedMutex.RLock() + defer fake.disconnectedMutex.RUnlock() + return len(fake.disconnectedArgsForCall) +} + +func (fake *FakeLocalParticipant) DisconnectedCalls(stub func() <-chan struct{}) { + fake.disconnectedMutex.Lock() + defer fake.disconnectedMutex.Unlock() + fake.DisconnectedStub = stub +} + +func (fake *FakeLocalParticipant) DisconnectedReturns(result1 <-chan struct{}) { + fake.disconnectedMutex.Lock() + defer fake.disconnectedMutex.Unlock() + fake.DisconnectedStub = nil + fake.disconnectedReturns = struct { + result1 <-chan struct{} + }{result1} +} + +func (fake *FakeLocalParticipant) DisconnectedReturnsOnCall(i int, result1 <-chan struct{}) { + fake.disconnectedMutex.Lock() + defer fake.disconnectedMutex.Unlock() + fake.DisconnectedStub = nil + if fake.disconnectedReturnsOnCall == nil { + fake.disconnectedReturnsOnCall = make(map[int]struct { + result1 <-chan struct{} + }) + } + fake.disconnectedReturnsOnCall[i] = struct { + result1 <-chan struct{} + }{result1} +} + func (fake *FakeLocalParticipant) GetAdaptiveStream() bool { fake.getAdaptiveStreamMutex.Lock() ret, specificReturn := fake.getAdaptiveStreamReturnsOnCall[len(fake.getAdaptiveStreamArgsForCall)] @@ -6408,6 +6471,8 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.connectedAtMutex.RUnlock() fake.debugInfoMutex.RLock() defer fake.debugInfoMutex.RUnlock() + fake.disconnectedMutex.RLock() + defer fake.disconnectedMutex.RUnlock() fake.getAdaptiveStreamMutex.RLock() defer fake.getAdaptiveStreamMutex.RUnlock() fake.getAudioLevelMutex.RLock() diff --git a/pkg/service/roommanager.go b/pkg/service/roommanager.go index 3d3b8dd13..9855e6b64 100644 --- a/pkg/service/roommanager.go +++ b/pkg/service/roommanager.go @@ -620,15 +620,10 @@ func (r *RoomManager) rtcSessionWorker(room *rtc.Room, participant types.LocalPa _ = r.refreshToken(participant) tokenTicker := time.NewTicker(tokenRefreshInterval) defer tokenTicker.Stop() - stateCheckTicker := time.NewTicker(time.Millisecond * 500) - defer stateCheckTicker.Stop() for { select { - case <-stateCheckTicker.C: - // periodic check to ensure participant didn't become disconnected - if participant.IsDisconnected() { - return - } + case <-participant.Disconnected(): + return case <-tokenTicker.C: // refresh token with the first API Key/secret pair if err := r.refreshToken(participant); err != nil { From 04d193e0b2756bcd089b62845200ae171fb03ee9 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 16 Apr 2024 10:10:06 +0530 Subject: [PATCH 48/78] Update mediatransportutil. (#2652) Also, use adjusted time of sender report for drift logging. --- go.mod | 2 +- go.sum | 4 ++-- pkg/sfu/buffer/rtpstats_base.go | 13 +++++++------ 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 2b54b57af..4b9584ee5 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 - github.com/livekit/mediatransportutil v0.0.0-20240406063423-a67d961689df + github.com/livekit/mediatransportutil v0.0.0-20240416023643-881d3dc5423e github.com/livekit/protocol v1.12.1-0.20240410060226-b6a979d8cfce github.com/livekit/psrpc v0.5.3-0.20240327035954-cec3a0e614be github.com/mackerelio/go-osstat v0.2.4 diff --git a/go.sum b/go.sum index c105929f9..39191d6b6 100644 --- a/go.sum +++ b/go.sum @@ -132,8 +132,8 @@ github.com/lithammer/shortuuid/v4 v4.0.0 h1:QRbbVkfgNippHOS8PXDkti4NaWeyYfcBTHtw github.com/lithammer/shortuuid/v4 v4.0.0/go.mod h1:Zs8puNcrvf2rV9rTH51ZLLcj7ZXqQI3lv67aw4KiB1Y= github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkDaKb5iXdynYrzB84ErPPO4LbRASk58= github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= -github.com/livekit/mediatransportutil v0.0.0-20240406063423-a67d961689df h1:DVhJRlF6/CtiyxJVy3QsbS9bf7GyUMuRZONMwZxIWpY= -github.com/livekit/mediatransportutil v0.0.0-20240406063423-a67d961689df/go.mod h1:jwKUCmObuiEDH0iiuJHaGMXwRs3RjrB4G6qqgkr/5oE= +github.com/livekit/mediatransportutil v0.0.0-20240416023643-881d3dc5423e h1:ss4VwrouYiDpuNJ9BUTH+WsW+GDdJS70iZp8ii3/0Lc= +github.com/livekit/mediatransportutil v0.0.0-20240416023643-881d3dc5423e/go.mod h1:jwKUCmObuiEDH0iiuJHaGMXwRs3RjrB4G6qqgkr/5oE= github.com/livekit/protocol v1.12.1-0.20240410060226-b6a979d8cfce h1:cbuw8FQ5S1vX6avOmlj6f8IwliXsOGjOa3UR0YDucX8= github.com/livekit/protocol v1.12.1-0.20240410060226-b6a979d8cfce/go.mod h1:jB6PWwf4tdMAwy+jxexqaVWuQiiklAtO4F5zZzWkTII= github.com/livekit/psrpc v0.5.3-0.20240327035954-cec3a0e614be h1:W1nCFZ19rYAORMBNX82NeVPHjADN0UyORr6refbUXpU= diff --git a/pkg/sfu/buffer/rtpstats_base.go b/pkg/sfu/buffer/rtpstats_base.go index 83c667dab..04028465d 100644 --- a/pkg/sfu/buffer/rtpstats_base.go +++ b/pkg/sfu/buffer/rtpstats_base.go @@ -130,7 +130,8 @@ func (r *RTCPSenderReportData) ToString() string { r.NTPTimestamp.Time().String(), r.RTPTimestamp, r.RTPTimestampExt, - r.At.String(), r.AtAdjusted.String(), + r.At.String(), + r.AtAdjusted.String(), ) } @@ -879,8 +880,8 @@ func (r *rtpStatsBase) getDrift(extStartTS, extHighestTS uint64) (packetDrift *l rtpClockTicks := r.srNewest.RTPTimestampExt - r.srFirst.RTPTimestampExt elapsed := r.srNewest.NTPTimestamp.Time().Sub(r.srFirst.NTPTimestamp.Time()) - driftSamples := int64(rtpClockTicks - uint64(elapsed.Nanoseconds()*int64(r.params.ClockRate)/1e9)) if elapsed.Seconds() > 0.0 { + driftSamples := int64(rtpClockTicks - uint64(elapsed.Nanoseconds()*int64(r.params.ClockRate)/1e9)) ntpReportDrift = &livekit.RTPDrift{ StartTime: timestamppb.New(r.srFirst.NTPTimestamp.Time()), EndTime: timestamppb.New(r.srNewest.NTPTimestamp.Time()), @@ -894,12 +895,12 @@ func (r *rtpStatsBase) getDrift(extStartTS, extHighestTS uint64) (packetDrift *l } } - elapsed = r.srNewest.At.Sub(r.srFirst.At) - driftSamples = int64(rtpClockTicks - uint64(elapsed.Nanoseconds()*int64(r.params.ClockRate)/1e9)) + elapsed = r.srNewest.AtAdjusted.Sub(r.srFirst.AtAdjusted) if elapsed.Seconds() > 0.0 { + driftSamples := int64(rtpClockTicks - uint64(elapsed.Nanoseconds()*int64(r.params.ClockRate)/1e9)) rebasedReportDrift = &livekit.RTPDrift{ - StartTime: timestamppb.New(r.srFirst.At), - EndTime: timestamppb.New(r.srNewest.At), + StartTime: timestamppb.New(r.srFirst.AtAdjusted), + EndTime: timestamppb.New(r.srNewest.AtAdjusted), Duration: elapsed.Seconds(), StartTimestamp: r.srFirst.RTPTimestampExt, EndTimestamp: r.srNewest.RTPTimestampExt, From 844cecaffc89190a7915723797b181e77fe08d24 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Mon, 15 Apr 2024 22:55:24 -0700 Subject: [PATCH 49/78] update pion deps (#2653) * update pion deps * update webrtc --- go.mod | 6 ++--- go.sum | 69 +++++----------------------------------------------------- 2 files changed, 9 insertions(+), 66 deletions(-) diff --git a/go.mod b/go.mod index 4b9584ee5..071cda10a 100644 --- a/go.mod +++ b/go.mod @@ -27,15 +27,15 @@ require ( github.com/mitchellh/go-homedir v1.1.0 github.com/olekukonko/tablewriter v0.0.5 github.com/pion/dtls/v2 v2.2.10 - github.com/pion/ice/v2 v2.3.14 + github.com/pion/ice/v2 v2.3.15 github.com/pion/interceptor v0.1.25 github.com/pion/rtcp v1.2.14 github.com/pion/rtp v1.8.5 - github.com/pion/sctp v1.8.14 + github.com/pion/sctp v1.8.16 github.com/pion/sdp/v3 v3.0.9 github.com/pion/transport/v2 v2.2.4 github.com/pion/turn/v2 v2.1.5 - github.com/pion/webrtc/v3 v3.2.34 + github.com/pion/webrtc/v3 v3.2.38 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.19.0 github.com/redis/go-redis/v9 v9.5.1 diff --git a/go.sum b/go.sum index 39191d6b6..4ef72eab7 100644 --- a/go.sum +++ b/go.sum @@ -40,8 +40,6 @@ github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= github.com/frostbyte73/core v0.0.10 h1:D4DQXdPb8ICayz0n75rs4UYTXrUSdxzUfeleuNJORsU= github.com/frostbyte73/core v0.0.10/go.mod h1:XsOGqrqe/VEV7+8vJ+3a8qnCIXNbKsoEiu/czs7nrcU= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gammazero/deque v0.2.1 h1:qSdsbG6pgp6nL7A0+K/B7s12mcCY/5l5SIUpMOl+dC0= @@ -52,20 +50,9 @@ github.com/go-jose/go-jose/v3 v3.0.3 h1:fFKWeig/irsp7XD2zBxvnmA/XaRWp5V3CBsZXJF7 github.com/go-jose/go-jose/v3 v3.0.3/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -98,7 +85,6 @@ github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+l github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/josharian/native v0.0.0-20200817173448-b6b71def0850/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= github.com/josharian/native v1.0.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA= @@ -173,17 +159,8 @@ github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI= github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= github.com/pion/datachannel v1.5.5 h1:10ef4kwdjije+M9d7Xm9im2Y3O6A6ccQb0zcqZcJew8= @@ -191,9 +168,8 @@ github.com/pion/datachannel v1.5.5/go.mod h1:iMz+lECmfdCMqFRhXhcA/219B0SQlbpoR2V github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= github.com/pion/dtls/v2 v2.2.10 h1:u2Axk+FyIR1VFTPurktB+1zoEPGIW3bmyj3LEFrXjAA= github.com/pion/dtls/v2 v2.2.10/go.mod h1:d9SYc9fch0CqK90mRk1dC7AkzzpwJj6u2GU3u+9pqFE= -github.com/pion/ice/v2 v2.3.13/go.mod h1:KXJJcZK7E8WzrBEYnV4UtqEZsGeWfHxsNqhVcVvgjxw= -github.com/pion/ice/v2 v2.3.14 h1:A7UaEmalw12Fko8YO0qguUbWyE69BnN4mDEqT7cLWQI= -github.com/pion/ice/v2 v2.3.14/go.mod h1:KXJJcZK7E8WzrBEYnV4UtqEZsGeWfHxsNqhVcVvgjxw= +github.com/pion/ice/v2 v2.3.15 h1:oCGVqnd6OWmJr4I6eQwSWn8VJDF45wIXFTjV3tyyris= +github.com/pion/ice/v2 v2.3.15/go.mod h1:KXJJcZK7E8WzrBEYnV4UtqEZsGeWfHxsNqhVcVvgjxw= github.com/pion/interceptor v0.1.25 h1:pwY9r7P6ToQ3+IF0bajN0xmk/fNw/suTgaTdlwTDmhc= github.com/pion/interceptor v0.1.25/go.mod h1:wkbPYAak5zKsfpVDYMtEfWEy8D4zL+rpxCxPImLOg3Y= github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= @@ -211,8 +187,8 @@ github.com/pion/rtp v1.8.3/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU github.com/pion/rtp v1.8.5 h1:uYzINfaK+9yWs7r537z/Rc1SvT8ILjBcmDOpJcTB+OU= github.com/pion/rtp v1.8.5/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= github.com/pion/sctp v1.8.5/go.mod h1:SUFFfDpViyKejTAdwD1d/HQsCu+V/40cCs2nZIvC3s0= -github.com/pion/sctp v1.8.14 h1:NzwwDrtpvbdqMMWV9Q6NYGbHE/FQmjI+GEQLyeJahu4= -github.com/pion/sctp v1.8.14/go.mod h1:P6PbDVA++OJMrVNg2AL3XtYHV4uD6dvfyOovCgMs0PE= +github.com/pion/sctp v1.8.16 h1:PKrMs+o9EMLRvFfXq59WFsC+V8mN1wnKzqrv+3D/gYY= +github.com/pion/sctp v1.8.16/go.mod h1:P6PbDVA++OJMrVNg2AL3XtYHV4uD6dvfyOovCgMs0PE= github.com/pion/sdp/v3 v3.0.9 h1:pX++dCHoHUwq43kuwf3PyJfHlwIj4hXA7Vrifiq0IJY= github.com/pion/sdp/v3 v3.0.9/go.mod h1:B5xmvENq5IXJimIO4zfp6LAe1fD9N+kFv+V/1lOdz8M= github.com/pion/srtp/v2 v2.0.18 h1:vKpAXfawO9RtTRKZJbG4y0v1b11NZxQnxRl85kGuUlo= @@ -232,8 +208,8 @@ github.com/pion/transport/v3 v3.0.2/go.mod h1:nIToODoOlb5If2jF9y2Igfx3PFYWfuXi37 github.com/pion/turn/v2 v2.1.3/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= github.com/pion/turn/v2 v2.1.5 h1:tTyy7TM3DCoX9IxTt/yHc/bThiRLyXK3T1YbNcgx9k4= github.com/pion/turn/v2 v2.1.5/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= -github.com/pion/webrtc/v3 v3.2.34 h1:wcKWYlVdfw+Zpdzx9csz/ou87ru9RGrl0yDJ2vKY+70= -github.com/pion/webrtc/v3 v3.2.34/go.mod h1:0vW+VYQwUumq9R/dWjRE1IT+jeHh3MtiYDszdNrLjxo= +github.com/pion/webrtc/v3 v3.2.38 h1:oA52VJAJhOjSi1JpKjf0CM+cCiZ3b7jBxvsoOiajeDU= +github.com/pion/webrtc/v3 v3.2.38/go.mod h1:AQ8p56OLbm3MjhYovYdgPuyX6oc+JcKx/HFoCGFcYzA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -257,7 +233,6 @@ github.com/rs/cors v1.10.1 h1:L0uuZVXIKlI1SShY2nhFfo44TYvDPQ1w4oFkUJNfhyo= github.com/rs/cors v1.10.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/sclevine/spec v1.4.0 h1:z/Q9idDcay5m5irkZ28M7PtQM4aOISzOpj4bUPkDee8= github.com/sclevine/spec v1.4.0/go.mod h1:LvpgJaFyvQzRvc1kaDs0bulYwzC70PbiYjC4QnFHkOM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -266,7 +241,6 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -287,7 +261,6 @@ github.com/urfave/negroni/v3 v3.1.0 h1:lzmuxGSpnJCT/ujgIAjkU3+LW3NX8alCglO/L6KjI github.com/urfave/negroni/v3 v3.1.0/go.mod h1:jWvnX03kcSjDBl/ShB0iHvx5uOs7mAzZXW+JvJ5XYAs= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= @@ -304,7 +277,6 @@ go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.uber.org/zap/exp v0.2.0 h1:FtGenNNeCATRB3CmB/yEUnjEFeJWpB/pMcy7e2bKPYs= go.uber.org/zap/exp v0.2.0/go.mod h1:t0gqAIdh1MfKv9EwN/dLwfZnJxe9ITAZN78HEWPFWDQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= @@ -317,29 +289,24 @@ golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 h1:aAcj0Da7eBAtrTp03QXWvm88pSyOt+UgdZw2BFZ+lEw= golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8/go.mod h1:CQ1k9gNrJ50XIzaKCRR2hssIjF07kZFEiieALBM/ARQ= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201216054612-986b41b23924/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -352,37 +319,28 @@ golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201118182958-a01c418693c7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201218084310-7d0127a74742/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210110051926-789bb1bd4061/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210123111255-9b0068b26619/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210216163648-f7da38b97c65/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -420,7 +378,6 @@ golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -435,7 +392,6 @@ golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= @@ -443,21 +399,12 @@ golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda h1:LI5DOvAxUPMv/50agcLLoo+AdWc1irS9Rzz4vPuD1V4= google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -466,12 +413,8 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 5c38d58987ed126a0d4932672d9a0d29b0a26fa9 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Tue, 16 Apr 2024 02:51:49 -0700 Subject: [PATCH 50/78] add typed ops queue (#2655) * add typed ops queue * tidy --- pkg/sfu/streamallocator/streamallocator.go | 34 ++++++------ pkg/utils/opsqueue.go | 63 ++++++++++++++++++---- 2 files changed, 69 insertions(+), 28 deletions(-) diff --git a/pkg/sfu/streamallocator/streamallocator.go b/pkg/sfu/streamallocator/streamallocator.go index 3bff2aca5..acb3ed477 100644 --- a/pkg/sfu/streamallocator/streamallocator.go +++ b/pkg/sfu/streamallocator/streamallocator.go @@ -168,7 +168,7 @@ type StreamAllocator struct { state streamAllocatorState - eventsQueue *utils.OpsQueue + eventsQueue *utils.TypedOpsQueue[Event] isStopped atomic.Bool } @@ -182,7 +182,7 @@ func NewStreamAllocator(params StreamAllocatorParams) *StreamAllocator { }), // STREAM-ALLOCATOR-DATA rateMonitor: NewRateMonitor(), videoTracks: make(map[livekit.TrackID]*Track), - eventsQueue: utils.NewOpsQueue(utils.OpsQueueParams{ + eventsQueue: utils.NewTypedOpsQueue[Event](utils.OpsQueueParams{ Name: "stream-allocator", MinSize: 64, Logger: params.Logger, @@ -559,9 +559,7 @@ func (s *StreamAllocator) maybePostEventAllocateTrack(downTrack *sfu.DownTrack) } func (s *StreamAllocator) postEvent(event Event) { - s.eventsQueue.Enqueue(func() { - s.handleEvent(&event) - }) + s.eventsQueue.Enqueue(s.handleEvent, event) } func (s *StreamAllocator) ping() { @@ -580,7 +578,7 @@ func (s *StreamAllocator) ping() { } } -func (s *StreamAllocator) handleEvent(event *Event) { +func (s *StreamAllocator) handleEvent(event Event) { switch event.Signal { case streamAllocatorSignalAllocateTrack: s.handleSignalAllocateTrack(event) @@ -611,7 +609,7 @@ func (s *StreamAllocator) handleEvent(event *Event) { } } -func (s *StreamAllocator) handleSignalAllocateTrack(event *Event) { +func (s *StreamAllocator) handleSignalAllocateTrack(event Event) { s.videoTracksMu.Lock() track := s.videoTracks[event.TrackID] if track != nil { @@ -624,7 +622,7 @@ func (s *StreamAllocator) handleSignalAllocateTrack(event *Event) { } } -func (s *StreamAllocator) handleSignalAllocateAllTracks(event *Event) { +func (s *StreamAllocator) handleSignalAllocateAllTracks(Event) { s.videoTracksMu.Lock() s.isAllocateAllPending = false s.videoTracksMu.Unlock() @@ -634,11 +632,11 @@ func (s *StreamAllocator) handleSignalAllocateAllTracks(event *Event) { } } -func (s *StreamAllocator) handleSignalAdjustState(event *Event) { +func (s *StreamAllocator) handleSignalAdjustState(Event) { s.adjustState() } -func (s *StreamAllocator) handleSignalEstimate(event *Event) { +func (s *StreamAllocator) handleSignalEstimate(event Event) { receivedEstimate, _ := event.Data.(int64) s.lastReceivedEstimate = receivedEstimate // s.monitorRate(receivedEstimate) @@ -651,7 +649,7 @@ func (s *StreamAllocator) handleSignalEstimate(event *Event) { } } -func (s *StreamAllocator) handleSignalPeriodicPing(event *Event) { +func (s *StreamAllocator) handleSignalPeriodicPing(Event) { // finalize probe if necessary trend, _ := s.channelObserver.GetTrend() isHandled, isNotFailing, isGoalReached := s.probeController.MaybeFinalizeProbe( @@ -671,7 +669,7 @@ func (s *StreamAllocator) handleSignalPeriodicPing(event *Event) { // s.updateTracksHistory() } -func (s *StreamAllocator) handleSignalSendProbe(event *Event) { +func (s *StreamAllocator) handleSignalSendProbe(event Event) { bytesToSend := event.Data.(int) if bytesToSend <= 0 { return @@ -692,12 +690,12 @@ func (s *StreamAllocator) handleSignalSendProbe(event *Event) { } } -func (s *StreamAllocator) handleSignalProbeClusterDone(event *Event) { +func (s *StreamAllocator) handleSignalProbeClusterDone(event Event) { info, _ := event.Data.(ProbeClusterInfo) s.probeController.ProbeClusterDone(info) } -func (s *StreamAllocator) handleSignalResume(event *Event) { +func (s *StreamAllocator) handleSignalResume(event Event) { s.videoTracksMu.Lock() track := s.videoTracks[event.TrackID] s.videoTracksMu.Unlock() @@ -711,11 +709,11 @@ func (s *StreamAllocator) handleSignalResume(event *Event) { } } -func (s *StreamAllocator) handleSignalSetAllowPause(event *Event) { +func (s *StreamAllocator) handleSignalSetAllowPause(event Event) { s.allowPause = event.Data.(bool) } -func (s *StreamAllocator) handleSignalSetChannelCapacity(event *Event) { +func (s *StreamAllocator) handleSignalSetChannelCapacity(event Event) { s.overriddenChannelCapacity = event.Data.(int64) if s.overriddenChannelCapacity > 0 { s.params.Logger.Infow("allocating on override channel capacity", "override", s.overriddenChannelCapacity) @@ -726,7 +724,7 @@ func (s *StreamAllocator) handleSignalSetChannelCapacity(event *Event) { } /* STREAM-ALLOCATOR-DATA -func (s *StreamAllocator) handleSignalNACK(event *Event) { +func (s *StreamAllocator) handleSignalNACK(event Event) { nackInfos := event.Data.([]sfu.NackInfo) s.videoTracksMu.Lock() @@ -738,7 +736,7 @@ func (s *StreamAllocator) handleSignalNACK(event *Event) { } } -func (s *StreamAllocator) handleSignalRTCPReceiverReport(event *Event) { +func (s *StreamAllocator) handleSignalRTCPReceiverReport(event Event) { rr := event.Data.(rtcp.ReceptionReport) s.videoTracksMu.Lock() diff --git a/pkg/utils/opsqueue.go b/pkg/utils/opsqueue.go index bda4ceea1..9f6b2cb78 100644 --- a/pkg/utils/opsqueue.go +++ b/pkg/utils/opsqueue.go @@ -19,8 +19,8 @@ import ( "sync" "github.com/gammazero/deque" + "github.com/livekit/protocol/logger" - "github.com/livekit/protocol/utils" ) type OpsQueueParams struct { @@ -30,28 +30,71 @@ type OpsQueueParams struct { Logger logger.Logger } +type untypedOpsQueueOp func() + +func (it untypedOpsQueueOp) run() { + it() +} + type OpsQueue struct { + OpsQueueBase[untypedOpsQueueOp] +} + +func NewOpsQueue(params OpsQueueParams) *OpsQueue { + return &OpsQueue{ + OpsQueueBase: *newOpsQueueBase[untypedOpsQueueOp](params), + } +} + +type typedOpsQueueOp[T any] struct { + fn func(T) + arg T +} + +func (it typedOpsQueueOp[T]) run() { + it.fn(it.arg) +} + +type TypedOpsQueue[T any] struct { + OpsQueueBase[typedOpsQueueOp[T]] +} + +func NewTypedOpsQueue[T any](params OpsQueueParams) *TypedOpsQueue[T] { + return &TypedOpsQueue[T]{ + OpsQueueBase: *newOpsQueueBase[typedOpsQueueOp[T]](params), + } +} + +func (oq *TypedOpsQueue[T]) Enqueue(fn func(T), arg T) { + oq.OpsQueueBase.Enqueue(typedOpsQueueOp[T]{fn, arg}) +} + +type opsQueueItem interface { + run() +} + +type OpsQueueBase[T opsQueueItem] struct { params OpsQueueParams lock sync.Mutex - ops deque.Deque[func()] + ops deque.Deque[T] wake chan struct{} isStarted bool doneChan chan struct{} isStopped bool } -func NewOpsQueue(params OpsQueueParams) *OpsQueue { - oq := &OpsQueue{ +func newOpsQueueBase[T opsQueueItem](params OpsQueueParams) *OpsQueueBase[T] { + oq := &OpsQueueBase[T]{ params: params, wake: make(chan struct{}, 1), doneChan: make(chan struct{}), } - oq.ops.SetMinCapacity(uint(utils.Min(bits.Len64(uint64(oq.params.MinSize-1)), 7))) + oq.ops.SetMinCapacity(uint(min(bits.Len64(uint64(oq.params.MinSize-1)), 7))) return oq } -func (oq *OpsQueue) Start() { +func (oq *OpsQueueBase[T]) Start() { oq.lock.Lock() if oq.isStarted { oq.lock.Unlock() @@ -64,7 +107,7 @@ func (oq *OpsQueue) Start() { go oq.process() } -func (oq *OpsQueue) Stop() <-chan struct{} { +func (oq *OpsQueueBase[T]) Stop() <-chan struct{} { oq.lock.Lock() if oq.isStopped { oq.lock.Unlock() @@ -77,7 +120,7 @@ func (oq *OpsQueue) Stop() <-chan struct{} { return oq.doneChan } -func (oq *OpsQueue) Enqueue(op func()) { +func (oq *OpsQueueBase[T]) Enqueue(op T) { oq.lock.Lock() defer oq.lock.Unlock() @@ -94,7 +137,7 @@ func (oq *OpsQueue) Enqueue(op func()) { } } -func (oq *OpsQueue) process() { +func (oq *OpsQueueBase[T]) process() { defer close(oq.doneChan) for { @@ -113,7 +156,7 @@ func (oq *OpsQueue) process() { op := oq.ops.PopFront() oq.lock.Unlock() - op() + op.run() } } } From 1ab0879d28170592f24ec08eed958c0368ef585f Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 16 Apr 2024 15:49:45 +0530 Subject: [PATCH 51/78] ICE config cache module. (#2654) * ICE config cache module. * generic key type * no ICEConfig in StartSession * clean up --- pkg/rtc/room.go | 11 ++++- pkg/rtc/transportmanager.go | 8 ++-- pkg/service/roommanager.go | 47 +++++++-------------- pkg/utils/ice_config_cache.go | 78 +++++++++++++++++++++++++++++++++++ 4 files changed, 107 insertions(+), 37 deletions(-) create mode 100644 pkg/utils/ice_config_cache.go diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index cc8645a9a..7a8da9a7e 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -479,7 +479,14 @@ func (r *Room) GetParticipantRequestSource(identity livekit.ParticipantIdentity) return r.participantRequestSources[identity] } -func (r *Room) ResumeParticipant(p types.LocalParticipant, requestSource routing.MessageSource, responseSink routing.MessageSink, iceServers []*livekit.ICEServer, reason livekit.ReconnectReason) error { +func (r *Room) ResumeParticipant( + p types.LocalParticipant, + requestSource routing.MessageSource, + responseSink routing.MessageSink, + iceConfig *livekit.ICEConfig, + iceServers []*livekit.ICEServer, + reason livekit.ReconnectReason, +) error { r.ReplaceParticipantRequestSource(p.Identity(), requestSource) // close previous sink, and link to new one p.CloseSignalConnection(types.SignallingCloseReasonResume) @@ -521,7 +528,7 @@ func (r *Room) ResumeParticipant(p types.LocalParticipant, requestSource routing } _ = p.SendRoomUpdate(r.ToProto()) - p.ICERestart(nil) + p.ICERestart(iceConfig) // check for simulated signal disconnect on resume r.simulationLock.Lock() diff --git a/pkg/rtc/transportmanager.go b/pkg/rtc/transportmanager.go index 049b76a5e..3bc666b57 100644 --- a/pkg/rtc/transportmanager.go +++ b/pkg/rtc/transportmanager.go @@ -431,9 +431,7 @@ func (t *TransportManager) HandleClientReconnect(reason livekit.ReconnectReason) } func (t *TransportManager) ICERestart(iceConfig *livekit.ICEConfig) error { - if iceConfig != nil { - t.SetICEConfig(iceConfig) - } + t.SetICEConfig(iceConfig) return t.subscriber.ICERestart() } @@ -445,7 +443,9 @@ func (t *TransportManager) OnICEConfigChanged(f func(iceConfig *livekit.ICEConfi } func (t *TransportManager) SetICEConfig(iceConfig *livekit.ICEConfig) { - t.configureICE(iceConfig, true) + if iceConfig != nil { + t.configureICE(iceConfig, true) + } } func (t *TransportManager) resetTransportConfigureLocked(reconfigured bool) { diff --git a/pkg/service/roommanager.go b/pkg/service/roommanager.go index 9855e6b64..409a6e34b 100644 --- a/pkg/service/roommanager.go +++ b/pkg/service/roommanager.go @@ -25,6 +25,7 @@ import ( "golang.org/x/exp/maps" "github.com/livekit/livekit-server/pkg/agent" + sutils "github.com/livekit/livekit-server/pkg/utils" "github.com/livekit/mediatransportutil/pkg/rtcconfig" "github.com/livekit/protocol/auth" "github.com/livekit/protocol/livekit" @@ -48,14 +49,13 @@ const ( roomPurgeSeconds = 24 * 60 * 60 tokenRefreshInterval = 5 * time.Minute tokenDefaultTTL = 10 * time.Minute - iceConfigTTL = 5 * time.Minute ) var affinityEpoch = time.Date(2000, 0, 0, 0, 0, 0, 0, time.UTC) -type iceConfigCacheEntry struct { - iceConfig *livekit.ICEConfig - modifiedAt time.Time +type iceConfigCacheKey struct { + roomName livekit.RoomName + participantIdentity livekit.ParticipantIdentity } // RoomManager manages rooms and its interaction with participants. @@ -82,7 +82,7 @@ type RoomManager struct { roomServers utils.MultitonService[rpc.RoomTopic] participantServers utils.MultitonService[rpc.ParticipantTopic] - iceConfigCache map[livekit.ParticipantIdentity]*iceConfigCacheEntry + iceConfigCache *sutils.IceConfigCache[iceConfigCacheKey] } func NewLocalRoomManager( @@ -119,7 +119,7 @@ func NewLocalRoomManager( rooms: make(map[livekit.RoomName]*rtc.Room), - iceConfigCache: make(map[livekit.ParticipantIdentity]*iceConfigCacheEntry), + iceConfigCache: sutils.NewIceConfigCache[iceConfigCacheKey](), serverInfo: &livekit.ServerInfo{ Edition: livekit.ServerInfo_Standard, @@ -230,6 +230,8 @@ func (r *RoomManager) Stop() { _ = r.rtcConfig.TCPMuxListener.Close() } } + + r.iceConfigCache.Stop() } // StartSession starts WebRTC session when a new participant is connected, takes place on RTC node @@ -303,14 +305,12 @@ func (r *RoomManager) StartSession( "reason", pi.ReconnectReason, "numParticipants", room.GetParticipantCount(), ) - iceConfig := r.getIceConfig(participant) - if iceConfig == nil { - iceConfig = &livekit.ICEConfig{} - } + iceConfig := r.getIceConfig(roomName, participant) if err = room.ResumeParticipant( participant, requestSource, responseSink, + iceConfig, r.iceServersForParticipant( apiKey, participant, @@ -444,7 +444,7 @@ func (r *RoomManager) StartSession( if err != nil { return err } - iceConfig := r.setIceConfig(participant) + iceConfig := r.setIceConfig(roomName, participant) // join room opts := rtc.ParticipantOptions{ @@ -504,12 +504,7 @@ func (r *RoomManager) StartSession( } }) participant.OnICEConfigChanged(func(participant types.LocalParticipant, iceConfig *livekit.ICEConfig) { - r.lock.Lock() - r.iceConfigCache[participant.Identity()] = &iceConfigCacheEntry{ - iceConfig: iceConfig, - modifiedAt: time.Now(), - } - r.lock.Unlock() + r.iceConfigCache.Put(iceConfigCacheKey{roomName, participant.Identity()}, iceConfig, time.Now()) }) go r.rtcSessionWorker(room, participant, requestSource) @@ -871,24 +866,14 @@ func (r *RoomManager) refreshToken(participant types.LocalParticipant) error { return nil } -func (r *RoomManager) setIceConfig(participant types.LocalParticipant) *livekit.ICEConfig { - iceConfig := r.getIceConfig(participant) - if iceConfig == nil { - return &livekit.ICEConfig{} - } +func (r *RoomManager) setIceConfig(roomName livekit.RoomName, participant types.LocalParticipant) *livekit.ICEConfig { + iceConfig := r.getIceConfig(roomName, participant) participant.SetICEConfig(iceConfig) return iceConfig } -func (r *RoomManager) getIceConfig(participant types.LocalParticipant) *livekit.ICEConfig { - r.lock.Lock() - defer r.lock.Unlock() - iceConfigCacheEntry, ok := r.iceConfigCache[participant.Identity()] - if !ok || time.Since(iceConfigCacheEntry.modifiedAt) > iceConfigTTL { - delete(r.iceConfigCache, participant.Identity()) - return nil - } - return iceConfigCacheEntry.iceConfig +func (r *RoomManager) getIceConfig(roomName livekit.RoomName, participant types.LocalParticipant) *livekit.ICEConfig { + return r.iceConfigCache.Get(iceConfigCacheKey{roomName, participant.Identity()}) } func (r *RoomManager) getFirstKeyPair() (string, string, error) { diff --git a/pkg/utils/ice_config_cache.go b/pkg/utils/ice_config_cache.go new file mode 100644 index 000000000..2fb4f49e7 --- /dev/null +++ b/pkg/utils/ice_config_cache.go @@ -0,0 +1,78 @@ +package utils + +import ( + "sync" + "time" + + "github.com/livekit/protocol/livekit" + "go.uber.org/atomic" +) + +const ( + iceConfigTTL = 5 * time.Minute +) + +type iceConfigCacheEntry struct { + iceConfig *livekit.ICEConfig + modifiedAt time.Time +} + +type IceConfigCache[T comparable] struct { + lock sync.Mutex + entries map[T]*iceConfigCacheEntry + + stopped atomic.Bool +} + +func NewIceConfigCache[T comparable]() *IceConfigCache[T] { + icc := &IceConfigCache[T]{ + entries: make(map[T]*iceConfigCacheEntry), + } + + go icc.pruneWorker() + return icc +} + +func (icc *IceConfigCache[T]) Stop() { + icc.stopped.Store(true) +} + +func (icc *IceConfigCache[T]) Put(key T, iceConfig *livekit.ICEConfig, at time.Time) { + icc.lock.Lock() + defer icc.lock.Unlock() + + icc.entries[key] = &iceConfigCacheEntry{ + iceConfig: iceConfig, + modifiedAt: at, + } +} + +func (icc *IceConfigCache[T]) Get(key T) *livekit.ICEConfig { + icc.lock.Lock() + defer icc.lock.Unlock() + + entry, ok := icc.entries[key] + if !ok || time.Since(entry.modifiedAt) > iceConfigTTL { + delete(icc.entries, key) + return &livekit.ICEConfig{} + } + + return entry.iceConfig +} + +func (icc *IceConfigCache[T]) pruneWorker() { + ticker := time.NewTicker(iceConfigTTL / 2) + defer ticker.Stop() + + for !icc.stopped.Load() { + <-ticker.C + + icc.lock.Lock() + for key, entry := range icc.entries { + if time.Since(entry.modifiedAt) > iceConfigTTL { + delete(icc.entries, key) + } + } + icc.lock.Unlock() + } +} From b77f0256c7f8159fdbf6c086dfd337ceb146c4b8 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Tue, 16 Apr 2024 03:23:08 -0700 Subject: [PATCH 52/78] use typed ops queue in pctransport (#2656) --- pkg/rtc/participant.go | 16 ++++++++------ pkg/rtc/transport.go | 49 ++++++++++++++++++++---------------------- 2 files changed, 32 insertions(+), 33 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index fe55b7bc1..0e889078d 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -164,7 +164,7 @@ type ParticipantImpl struct { disconnectTimer *time.Timer migrationTimer *time.Timer - pubRTCPQueue *sutils.OpsQueue + pubRTCPQueue *sutils.TypedOpsQueue[[]rtcp.Packet] // hold reference for MediaTrack twcc *twcc.Responder @@ -244,7 +244,7 @@ func NewParticipant(params ParticipantParams) (*ParticipantImpl, error) { p := &ParticipantImpl{ params: params, disconnected: make(chan struct{}), - pubRTCPQueue: sutils.NewOpsQueue(sutils.OpsQueueParams{ + pubRTCPQueue: sutils.NewTypedOpsQueue[[]rtcp.Packet](sutils.OpsQueueParams{ Name: "pub-rtcp", MinSize: 64, Logger: params.Logger, @@ -2379,11 +2379,13 @@ func (p *ParticipantImpl) postRtcp(pkts []rtcp.Packet) { return } - p.pubRTCPQueue.Enqueue(func() { - if err := p.TransportManager.WritePublisherRTCP(pkts); err != nil && !IsEOF(err) { - p.pubLogger.Errorw("could not write RTCP to participant", err) - } - }) + p.pubRTCPQueue.Enqueue(p.writePublisherRTCP, pkts) +} + +func (p *ParticipantImpl) writePublisherRTCP(pkts []rtcp.Packet) { + if err := p.TransportManager.WritePublisherRTCP(pkts); err != nil && !IsEOF(err) { + p.pubLogger.Errorw("could not write RTCP to participant", err) + } } func (p *ParticipantImpl) setDowntracksConnected() { diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index 74117c8f1..c4e496be5 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -183,7 +183,7 @@ type PCTransport struct { preferTCP atomic.Bool isClosed atomic.Bool - eventsQueue *sutils.OpsQueue + eventsQueue *sutils.TypedOpsQueue[event] // the following should be accessed only in event processing go routine cacheLocalCandidates bool @@ -388,7 +388,7 @@ func NewPCTransport(params TransportParams) (*PCTransport, error) { params: params, debouncedNegotiate: debounce.New(negotiationFrequency), negotiationState: transport.NegotiationStateNone, - eventsQueue: sutils.NewOpsQueue(utils.OpsQueueParams{ + eventsQueue: sutils.NewTypedOpsQueue[event](utils.OpsQueueParams{ Name: "transport", MinSize: 64, Logger: params.Logger, @@ -1241,37 +1241,34 @@ func (t *PCTransport) parseTrackMid(offer webrtc.SessionDescription, senders map } func (t *PCTransport) postEvent(event event) { - t.eventsQueue.Enqueue(func() { - err := t.handleEvent(&event) - if err != nil { - if !t.isClosed.Load() { - t.params.Logger.Warnw("error handling event", err, "event", event.String()) - t.params.Handler.OnNegotiationFailed() - } - } - }) + t.eventsQueue.Enqueue(t.handleEvent, event) } -func (t *PCTransport) handleEvent(e *event) error { +func (t *PCTransport) handleEvent(e event) { + var err error switch e.signal { case signalICEGatheringComplete: - return t.handleICEGatheringComplete(e) + err = t.handleICEGatheringComplete(e) case signalLocalICECandidate: - return t.handleLocalICECandidate(e) + err = t.handleLocalICECandidate(e) case signalRemoteICECandidate: - return t.handleRemoteICECandidate(e) + err = t.handleRemoteICECandidate(e) case signalSendOffer: - return t.handleSendOffer(e) + err = t.handleSendOffer(e) case signalRemoteDescriptionReceived: - return t.handleRemoteDescriptionReceived(e) + err = t.handleRemoteDescriptionReceived(e) case signalICERestart: - return t.handleICERestart(e) + err = t.handleICERestart(e) + } + if err != nil { + if !t.isClosed.Load() { + t.params.Logger.Warnw("error handling event", err, "event", e.String()) + t.params.Handler.OnNegotiationFailed() + } } - - return nil } -func (t *PCTransport) handleICEGatheringComplete(_ *event) error { +func (t *PCTransport) handleICEGatheringComplete(_ event) error { if t.params.IsOfferer { return t.handleICEGatheringCompleteOfferer() } else { @@ -1329,7 +1326,7 @@ func (t *PCTransport) clearLocalDescriptionSent() { t.connectionDetails.Clear() } -func (t *PCTransport) handleLocalICECandidate(e *event) error { +func (t *PCTransport) handleLocalICECandidate(e event) error { c := e.data.(*webrtc.ICECandidate) filtered := false @@ -1353,7 +1350,7 @@ func (t *PCTransport) handleLocalICECandidate(e *event) error { return t.params.Handler.OnICECandidate(c, t.params.Transport) } -func (t *PCTransport) handleRemoteICECandidate(e *event) error { +func (t *PCTransport) handleRemoteICECandidate(e event) error { c := e.data.(*webrtc.ICECandidateInit) filtered := false @@ -1549,11 +1546,11 @@ func (t *PCTransport) createAndSendOffer(options *webrtc.OfferOptions) error { return t.localDescriptionSent() } -func (t *PCTransport) handleSendOffer(_ *event) error { +func (t *PCTransport) handleSendOffer(_ event) error { return t.createAndSendOffer(nil) } -func (t *PCTransport) handleRemoteDescriptionReceived(e *event) error { +func (t *PCTransport) handleRemoteDescriptionReceived(e event) error { sd := e.data.(*webrtc.SessionDescription) if sd.Type == webrtc.SDPTypeOffer { return t.handleRemoteOfferReceived(sd) @@ -1778,7 +1775,7 @@ func (t *PCTransport) doICERestart() error { } } -func (t *PCTransport) handleICERestart(_ *event) error { +func (t *PCTransport) handleICERestart(_ event) error { return t.doICERestart() } From 6afa63ded377f71591050076e6d4732ded26b0df Mon Sep 17 00:00:00 2001 From: Benjamin Pracht Date: Tue, 16 Apr 2024 18:10:14 +0200 Subject: [PATCH 53/78] Use the ingress state updated_at field to ensure that out of order RPC do not overwrite state (#2657) --- go.mod | 4 ++-- go.sum | 8 ++++---- pkg/service/redisstore.go | 8 ++++++++ 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 071cda10a..9a19b25fc 100644 --- a/go.mod +++ b/go.mod @@ -18,8 +18,8 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 - github.com/livekit/mediatransportutil v0.0.0-20240416023643-881d3dc5423e - github.com/livekit/protocol v1.12.1-0.20240410060226-b6a979d8cfce + github.com/livekit/mediatransportutil v0.0.0-20240406063423-a67d961689df + github.com/livekit/protocol v1.12.1-0.20240416154343-2d633a51d825 github.com/livekit/psrpc v0.5.3-0.20240327035954-cec3a0e614be github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 diff --git a/go.sum b/go.sum index 4ef72eab7..dc97b17ca 100644 --- a/go.sum +++ b/go.sum @@ -118,10 +118,10 @@ github.com/lithammer/shortuuid/v4 v4.0.0 h1:QRbbVkfgNippHOS8PXDkti4NaWeyYfcBTHtw github.com/lithammer/shortuuid/v4 v4.0.0/go.mod h1:Zs8puNcrvf2rV9rTH51ZLLcj7ZXqQI3lv67aw4KiB1Y= github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkDaKb5iXdynYrzB84ErPPO4LbRASk58= github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= -github.com/livekit/mediatransportutil v0.0.0-20240416023643-881d3dc5423e h1:ss4VwrouYiDpuNJ9BUTH+WsW+GDdJS70iZp8ii3/0Lc= -github.com/livekit/mediatransportutil v0.0.0-20240416023643-881d3dc5423e/go.mod h1:jwKUCmObuiEDH0iiuJHaGMXwRs3RjrB4G6qqgkr/5oE= -github.com/livekit/protocol v1.12.1-0.20240410060226-b6a979d8cfce h1:cbuw8FQ5S1vX6avOmlj6f8IwliXsOGjOa3UR0YDucX8= -github.com/livekit/protocol v1.12.1-0.20240410060226-b6a979d8cfce/go.mod h1:jB6PWwf4tdMAwy+jxexqaVWuQiiklAtO4F5zZzWkTII= +github.com/livekit/mediatransportutil v0.0.0-20240406063423-a67d961689df h1:DVhJRlF6/CtiyxJVy3QsbS9bf7GyUMuRZONMwZxIWpY= +github.com/livekit/mediatransportutil v0.0.0-20240406063423-a67d961689df/go.mod h1:jwKUCmObuiEDH0iiuJHaGMXwRs3RjrB4G6qqgkr/5oE= +github.com/livekit/protocol v1.12.1-0.20240416154343-2d633a51d825 h1:HJWtEtuiRoKE+/cRjLY0UHKnB6iMOAp1/17CXG6e5W0= +github.com/livekit/protocol v1.12.1-0.20240416154343-2d633a51d825/go.mod h1:jB6PWwf4tdMAwy+jxexqaVWuQiiklAtO4F5zZzWkTII= github.com/livekit/psrpc v0.5.3-0.20240327035954-cec3a0e614be h1:W1nCFZ19rYAORMBNX82NeVPHjADN0UyORr6refbUXpU= github.com/livekit/psrpc v0.5.3-0.20240327035954-cec3a0e614be/go.mod h1:CQUBSPfYYAaevg1TNCc6/aYsa8DJH4jSRFdCeSZk5u0= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= diff --git a/pkg/service/redisstore.go b/pkg/service/redisstore.go index 91b437f40..67a573874 100644 --- a/pkg/service/redisstore.go +++ b/pkg/service/redisstore.go @@ -608,6 +608,7 @@ func (s *RedisStore) storeIngressState(_ context.Context, ingressId string, stat // Use a "transaction" to remove the old room association if it changed txf := func(tx *redis.Tx) error { var oldStartedAt int64 + var oldUpdatedAt int64 oldState, err := s.loadIngressState(tx, ingressId) switch err { @@ -615,6 +616,7 @@ func (s *RedisStore) storeIngressState(_ context.Context, ingressId string, stat // Ingress state doesn't exist yet case nil: oldStartedAt = oldState.StartedAt + oldUpdatedAt = oldState.UpdatedAt default: return err } @@ -625,6 +627,12 @@ func (s *RedisStore) storeIngressState(_ context.Context, ingressId string, stat return ingress.ErrIngressOutOfDate } + if state.StartedAt == oldStartedAt && state.UpdatedAt < oldUpdatedAt { + // Do not overwrite with an old state in case RPCs were delivered out of order. + // All RPCs come from the same ingress server and should thus be on the same clock. + return nil + } + p.Set(s.ctx, IngressStatePrefix+ingressId, data, 0) return nil From 14b934a78073117aa525aecb5e7fa00ece6d2f4e Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 17 Apr 2024 09:45:10 +0530 Subject: [PATCH 54/78] Log ICE candidates to debug TCP connection issues. (#2658) --- pkg/rtc/participant_signal.go | 4 ++++ pkg/rtc/transport.go | 24 +++++++++++++++++++++++- pkg/rtc/transportmanager.go | 15 --------------- 3 files changed, 27 insertions(+), 16 deletions(-) diff --git a/pkg/rtc/participant_signal.go b/pkg/rtc/participant_signal.go index d4dbafa35..dc0a2405a 100644 --- a/pkg/rtc/participant_signal.go +++ b/pkg/rtc/participant_signal.go @@ -22,6 +22,7 @@ import ( "github.com/pion/webrtc/v3" "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" "github.com/livekit/psrpc" "github.com/livekit/livekit-server/pkg/routing" @@ -254,6 +255,9 @@ func (p *ParticipantImpl) sendDisconnectUpdatesForReconnect() error { func (p *ParticipantImpl) sendICECandidate(c *webrtc.ICECandidate, target livekit.SignalTarget) error { trickle := ToProtoTrickle(c.ToJSON()) trickle.Target = target + + p.params.Logger.Debugw("sending ICE candidate", "transport", target, "trickle", logger.Proto(trickle)) + return p.writeMessage(&livekit.SignalResponse{ Message: &livekit.SignalResponse_Trickle{ Trickle: trickle, diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index c4e496be5..b4cc64b0b 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -24,6 +24,7 @@ import ( "github.com/bep/debounce" "github.com/pion/dtls/v2/pkg/crypto/elliptic" + "github.com/pion/ice/v2" "github.com/pion/interceptor" "github.com/pion/interceptor/pkg/cc" "github.com/pion/interceptor/pkg/gcc" @@ -702,6 +703,19 @@ func (t *PCTransport) SetPreferTCP(preferTCP bool) { } func (t *PCTransport) AddICECandidate(candidate webrtc.ICECandidateInit) { + if !t.params.Config.UseMDNS { + candidateValue := strings.TrimPrefix(candidate.Candidate, "candidate:") + if candidateValue != "" { + candidate, err := ice.UnmarshalCandidate(candidateValue) + if err != nil { + t.params.Logger.Errorw("failed to parse ice candidate", err) + } else if strings.HasSuffix(candidate.Address(), ".local") { + t.params.Logger.Debugw("ignoring mDNS candidate", "candidate", candidateValue) + return + } + } + } + t.postEvent(event{ signal: signalRemoteICECandidate, data: &candidate, @@ -1314,6 +1328,7 @@ func (t *PCTransport) localDescriptionSent() error { for _, c := range cachedLocalCandidates { if err := t.params.Handler.OnICECandidate(c, t.params.Transport); err != nil { + t.params.Logger.Warnw("failed to send cached ICE candidate", err, "candidate", c) return err } } @@ -1347,7 +1362,12 @@ func (t *PCTransport) handleLocalICECandidate(e event) error { return nil } - return t.params.Handler.OnICECandidate(c, t.params.Transport) + if err := t.params.Handler.OnICECandidate(c, t.params.Transport); err != nil { + t.params.Logger.Warnw("failed to send ICE candidate", err, "candidate", c) + return err + } + + return nil } func (t *PCTransport) handleRemoteICECandidate(e event) error { @@ -1370,6 +1390,7 @@ func (t *PCTransport) handleRemoteICECandidate(e event) error { } if err := t.pc.AddICECandidate(*c); err != nil { + t.params.Logger.Warnw("failed to add cached ICE candidate", err, "candidate", c) return errors.Wrap(err, "add ice candidate failed") } @@ -1605,6 +1626,7 @@ func (t *PCTransport) setRemoteDescription(sd webrtc.SessionDescription) error { for _, c := range t.pendingRemoteCandidates { if err := t.pc.AddICECandidate(*c); err != nil { + t.params.Logger.Warnw("failed to add cached ICE candidate", err, "candidate", c) return errors.Wrap(err, "add ice candidate failed") } } diff --git a/pkg/rtc/transportmanager.go b/pkg/rtc/transportmanager.go index 3bc666b57..64569f196 100644 --- a/pkg/rtc/transportmanager.go +++ b/pkg/rtc/transportmanager.go @@ -16,11 +16,9 @@ package rtc import ( "math/bits" - "strings" "sync" "time" - "github.com/pion/ice/v2" "github.com/pion/rtcp" "github.com/pion/sdp/v3" "github.com/pion/webrtc/v3" @@ -372,19 +370,6 @@ func (t *TransportManager) HandleAnswer(answer webrtc.SessionDescription) { // AddICECandidate adds candidates for remote peer func (t *TransportManager) AddICECandidate(candidate webrtc.ICECandidateInit, target livekit.SignalTarget) { - if !t.params.Config.UseMDNS { - candidateValue := strings.TrimPrefix(candidate.Candidate, "candidate:") - if candidateValue != "" { - candidate, err := ice.UnmarshalCandidate(candidateValue) - if err != nil { - t.params.Logger.Errorw("failed to parse ice candidate", err) - } else if strings.HasSuffix(candidate.Address(), ".local") { - t.params.Logger.Debugw("ignoring mDNS candidate", "candidate", candidateValue, "target", target) - return - } - } - } - switch target { case livekit.SignalTarget_PUBLISHER: t.publisher.AddICECandidate(candidate) From e96e8de7251ecb887298b60e7f120ad2de228beb Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 17 Apr 2024 11:14:40 +0530 Subject: [PATCH 55/78] Debug logging addition of ICE candidate (#2659) --- pkg/rtc/transport.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index b4cc64b0b..6a975ac28 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -1392,6 +1392,8 @@ func (t *PCTransport) handleRemoteICECandidate(e event) error { if err := t.pc.AddICECandidate(*c); err != nil { t.params.Logger.Warnw("failed to add cached ICE candidate", err, "candidate", c) return errors.Wrap(err, "add ice candidate failed") + } else { + t.params.Logger.Debugw("added cached ICE candidate", "candidate", c) } return nil @@ -1628,6 +1630,8 @@ func (t *PCTransport) setRemoteDescription(sd webrtc.SessionDescription) error { if err := t.pc.AddICECandidate(*c); err != nil { t.params.Logger.Warnw("failed to add cached ICE candidate", err, "candidate", c) return errors.Wrap(err, "add ice candidate failed") + } else { + t.params.Logger.Debugw("added cached ICE candidate", "candidate", c) } } t.pendingRemoteCandidates = nil From 8f8825cb88a6bb55ea057643e0a012b1e2a919fd Mon Sep 17 00:00:00 2001 From: David Colburn Date: Wed, 17 Apr 2024 13:12:57 -0700 Subject: [PATCH 56/78] fix participant, ensure room name matches (#2660) --- pkg/service/roomallocator.go | 14 ++++++++++---- pkg/service/roomservice.go | 2 ++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/pkg/service/roomallocator.go b/pkg/service/roomallocator.go index 04543a59d..54043e6a6 100644 --- a/pkg/service/roomallocator.go +++ b/pkg/service/roomallocator.go @@ -16,6 +16,7 @@ package service import ( "context" + "errors" "time" "github.com/livekit/protocol/livekit" @@ -62,7 +63,7 @@ func (r *StandardRoomAllocator) CreateRoom(ctx context.Context, req *livekit.Cre // find existing room and update it var created bool rm, internal, err := r.roomStore.LoadRoom(ctx, livekit.RoomName(req.Name), true) - if err == ErrRoomNotFound { + if errors.Is(err, ErrRoomNotFound) { created = true rm = &livekit.Room{ Sid: utils.NewGuid(utils.RoomPrefix), @@ -88,8 +89,13 @@ func (r *StandardRoomAllocator) CreateRoom(ctx context.Context, req *livekit.Cre if req.Metadata != "" { rm.Metadata = req.Metadata } - if req.Egress != nil && req.Egress.Tracks != nil { - internal.TrackEgress = req.Egress.Tracks + if req.Egress != nil { + if req.Egress.Participant != nil { + internal.ParticipantEgress = req.Egress.Participant + } + if req.Egress.Tracks != nil { + internal.TrackEgress = req.Egress.Tracks + } } if req.MinPlayoutDelay > 0 || req.MaxPlayoutDelay > 0 { internal.PlayoutDelay = &livekit.PlayoutDelay{ @@ -108,7 +114,7 @@ func (r *StandardRoomAllocator) CreateRoom(ctx context.Context, req *livekit.Cre // check if room already assigned existing, err := r.router.GetNodeForRoom(ctx, livekit.RoomName(rm.Name)) - if err != routing.ErrNotFound && err != nil { + if !errors.Is(err, routing.ErrNotFound) && err != nil { return nil, false, err } diff --git a/pkg/service/roomservice.go b/pkg/service/roomservice.go index 5227d141e..319e06cb5 100644 --- a/pkg/service/roomservice.go +++ b/pkg/service/roomservice.go @@ -107,6 +107,8 @@ func (s *RoomService) CreateRoom(ctx context.Context, req *livekit.CreateRoomReq }) if req.Egress != nil && req.Egress.Room != nil { + // ensure room name matches + req.Egress.Room.RoomName = req.Name _, err = s.egressLauncher.StartEgress(ctx, &rpc.StartEgressRequest{ Request: &rpc.StartEgressRequest_RoomComposite{ RoomComposite: req.Egress.Room, From 0974f3d961ccd2ba57412c3afc577265b2cf4652 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Wed, 17 Apr 2024 22:14:17 -0700 Subject: [PATCH 57/78] replace keyframe ticker with timer (#2661) --- pkg/sfu/downtrack.go | 17 ++++++++--------- pkg/utils/opsqueue.go | 36 ++++++++++++++++-------------------- 2 files changed, 24 insertions(+), 29 deletions(-) diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 035472df3..f2cce209d 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" "io" + "math" "strings" "sync" "time" @@ -625,21 +626,21 @@ func (d *DownTrack) keyFrameRequester() { return time.Duration(interval) * time.Millisecond } - interval := getInterval() - ticker := time.NewTicker(interval) - defer ticker.Stop() + timer := time.NewTimer(math.MaxInt64) + defer timer.Stop() - for { - if d.IsClosed() { - return + for !d.IsClosed() { + if !timer.Stop() { + <-timer.C } + timer.Reset(getInterval()) select { case _, more := <-d.keyFrameRequesterCh: if !more { return } - case <-ticker.C: + case <-timer.C: } locked, layer := d.forwarder.CheckSync() @@ -648,8 +649,6 @@ func (d *DownTrack) keyFrameRequester() { d.params.Receiver.SendPLI(layer, false) d.rtpStats.UpdateLayerLockPliAndTime(1) } - - ticker.Reset(getInterval()) } } diff --git a/pkg/utils/opsqueue.go b/pkg/utils/opsqueue.go index 9f6b2cb78..2513f0b8c 100644 --- a/pkg/utils/opsqueue.go +++ b/pkg/utils/opsqueue.go @@ -30,50 +30,46 @@ type OpsQueueParams struct { Logger logger.Logger } -type untypedOpsQueueOp func() +type UntypedQueueOp func() -func (it untypedOpsQueueOp) run() { +func (it UntypedQueueOp) run() { it() } type OpsQueue struct { - OpsQueueBase[untypedOpsQueueOp] + opsQueueBase[UntypedQueueOp] } func NewOpsQueue(params OpsQueueParams) *OpsQueue { - return &OpsQueue{ - OpsQueueBase: *newOpsQueueBase[untypedOpsQueueOp](params), - } + return &OpsQueue{*newOpsQueueBase[UntypedQueueOp](params)} } -type typedOpsQueueOp[T any] struct { +type typedQueueOp[T any] struct { fn func(T) arg T } -func (it typedOpsQueueOp[T]) run() { +func (it typedQueueOp[T]) run() { it.fn(it.arg) } type TypedOpsQueue[T any] struct { - OpsQueueBase[typedOpsQueueOp[T]] + opsQueueBase[typedQueueOp[T]] } func NewTypedOpsQueue[T any](params OpsQueueParams) *TypedOpsQueue[T] { - return &TypedOpsQueue[T]{ - OpsQueueBase: *newOpsQueueBase[typedOpsQueueOp[T]](params), - } + return &TypedOpsQueue[T]{*newOpsQueueBase[typedQueueOp[T]](params)} } func (oq *TypedOpsQueue[T]) Enqueue(fn func(T), arg T) { - oq.OpsQueueBase.Enqueue(typedOpsQueueOp[T]{fn, arg}) + oq.opsQueueBase.Enqueue(typedQueueOp[T]{fn, arg}) } type opsQueueItem interface { run() } -type OpsQueueBase[T opsQueueItem] struct { +type opsQueueBase[T opsQueueItem] struct { params OpsQueueParams lock sync.Mutex @@ -84,8 +80,8 @@ type OpsQueueBase[T opsQueueItem] struct { isStopped bool } -func newOpsQueueBase[T opsQueueItem](params OpsQueueParams) *OpsQueueBase[T] { - oq := &OpsQueueBase[T]{ +func newOpsQueueBase[T opsQueueItem](params OpsQueueParams) *opsQueueBase[T] { + oq := &opsQueueBase[T]{ params: params, wake: make(chan struct{}, 1), doneChan: make(chan struct{}), @@ -94,7 +90,7 @@ func newOpsQueueBase[T opsQueueItem](params OpsQueueParams) *OpsQueueBase[T] { return oq } -func (oq *OpsQueueBase[T]) Start() { +func (oq *opsQueueBase[T]) Start() { oq.lock.Lock() if oq.isStarted { oq.lock.Unlock() @@ -107,7 +103,7 @@ func (oq *OpsQueueBase[T]) Start() { go oq.process() } -func (oq *OpsQueueBase[T]) Stop() <-chan struct{} { +func (oq *opsQueueBase[T]) Stop() <-chan struct{} { oq.lock.Lock() if oq.isStopped { oq.lock.Unlock() @@ -120,7 +116,7 @@ func (oq *OpsQueueBase[T]) Stop() <-chan struct{} { return oq.doneChan } -func (oq *OpsQueueBase[T]) Enqueue(op T) { +func (oq *opsQueueBase[T]) Enqueue(op T) { oq.lock.Lock() defer oq.lock.Unlock() @@ -137,7 +133,7 @@ func (oq *OpsQueueBase[T]) Enqueue(op T) { } } -func (oq *OpsQueueBase[T]) process() { +func (oq *opsQueueBase[T]) process() { defer close(oq.doneChan) for { From 483aafcfc02157bc18a2e13f98431b17bab2fab1 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Thu, 18 Apr 2024 01:24:45 -0700 Subject: [PATCH 58/78] fix key frame timer (#2662) --- pkg/sfu/downtrack.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index f2cce209d..3ee2d7dc3 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -627,12 +627,11 @@ func (d *DownTrack) keyFrameRequester() { } timer := time.NewTimer(math.MaxInt64) + timer.Stop() + defer timer.Stop() for !d.IsClosed() { - if !timer.Stop() { - <-timer.C - } timer.Reset(getInterval()) select { @@ -640,6 +639,9 @@ func (d *DownTrack) keyFrameRequester() { if !more { return } + if !timer.Stop() { + <-timer.C + } case <-timer.C: } From 0b3587b10dce0bfffa2dd39b9067b7527815a315 Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Fri, 19 Apr 2024 15:25:48 +0800 Subject: [PATCH 59/78] Disable dynamic playout delay for screenshare track (#2663) Screenshare video has inaccuracy jitter due to its low frame rate and bursty traffic --- pkg/rtc/mediatracksubscriptions.go | 1 + pkg/sfu/downtrack.go | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/pkg/rtc/mediatracksubscriptions.go b/pkg/rtc/mediatracksubscriptions.go index 20d4ea7d7..158cafe10 100644 --- a/pkg/rtc/mediatracksubscriptions.go +++ b/pkg/rtc/mediatracksubscriptions.go @@ -129,6 +129,7 @@ func (t *MediaTrackSubscriptions) AddSubscriber(sub types.LocalParticipant, wr * downTrack, err := sfu.NewDownTrack(sfu.DowntrackParams{ Codecs: codecs, + Source: t.params.MediaTrack.Source(), Receiver: wr, BufferFactory: sub.GetBufferFactory(), SubID: subscriberID, diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 3ee2d7dc3..7c77c8122 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -202,6 +202,7 @@ type ReceiverReportListener func(dt *DownTrack, report *rtcp.ReceiverReport) type DowntrackParams struct { Codecs []webrtc.RTPCodecParameters + Source livekit.TrackSource Receiver TrackReceiver BufferFactory *buffer.Factory SubID livekit.ParticipantID @@ -1603,9 +1604,12 @@ func (d *DownTrack) handleRTCP(bytes []byte) { */ if d.playoutDelay != nil { - jitterMs := uint64(r.Jitter*1e3) / uint64(d.codec.ClockRate) d.playoutDelay.OnSeqAcked(uint16(r.LastSequenceNumber)) - d.playoutDelay.SetJitter(uint32(jitterMs)) + // screen share track has inaccuracy jitter due to its low frame rate and bursty traffic + if d.params.Source != livekit.TrackSource_SCREEN_SHARE { + jitterMs := uint64(r.Jitter*1e3) / uint64(d.codec.ClockRate) + d.playoutDelay.SetJitter(uint32(jitterMs)) + } } } if len(rr.Reports) > 0 { From 8eb86f10770f8391051dfc9226a9d3cf2a4c63a1 Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Fri, 19 Apr 2024 16:48:36 +0800 Subject: [PATCH 60/78] Don't log dd invalid template index (#2664) If the first packet of keyframe has template structure is lost then subsequent packets rely on it will report invalid tempalte error which is expected. --- pkg/sfu/buffer/dependencydescriptorparser.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/sfu/buffer/dependencydescriptorparser.go b/pkg/sfu/buffer/dependencydescriptorparser.go index 30851442c..1effe38b0 100644 --- a/pkg/sfu/buffer/dependencydescriptorparser.go +++ b/pkg/sfu/buffer/dependencydescriptorparser.go @@ -90,7 +90,7 @@ func (r *DependencyDescriptorParser) Parse(pkt *rtp.Packet) (*ExtDependencyDescr } _, err := ext.Unmarshal(ddBuf) if err != nil { - if err != dd.ErrDDReaderNoStructure { + if err != dd.ErrDDReaderNoStructure && err != dd.ErrDDReaderInvalidTemplateIndex { r.logger.Infow("failed to parse generic dependency descriptor", err, "payload", pkt.PayloadType, "ddbufLen", len(ddBuf)) } return nil, videoLayer, err From f4745b843798f37a6221a54328760d68b37ad145 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 19 Apr 2024 22:32:13 +0530 Subject: [PATCH 61/78] Do codec munging when munging RTP header. (#2665) * Do codec munging when munging RTP header. It was possible for probe packets to get in between RTP munging and codec munging and throw off sequence number while dropping packets. Affected only VP8 as it does codec munging. * do not pass in buffer as it is created anyway * flip fields * flip order * fix test * call translate for all tracks * simplify --- pkg/sfu/codecmunger/codecmunger.go | 4 +- pkg/sfu/codecmunger/null.go | 8 +- pkg/sfu/codecmunger/vp8.go | 24 ++--- pkg/sfu/codecmunger/vp8_test.go | 46 ++++---- pkg/sfu/downtrack.go | 29 +++-- pkg/sfu/forwarder.go | 70 ++++++------ pkg/sfu/forwarder_test.go | 165 ++++++++++++----------------- 7 files changed, 153 insertions(+), 193 deletions(-) diff --git a/pkg/sfu/codecmunger/codecmunger.go b/pkg/sfu/codecmunger/codecmunger.go index 94543ea38..8f2253850 100644 --- a/pkg/sfu/codecmunger/codecmunger.go +++ b/pkg/sfu/codecmunger/codecmunger.go @@ -33,7 +33,7 @@ type CodecMunger interface { SetLast(extPkt *buffer.ExtPacket) UpdateOffsets(extPkt *buffer.ExtPacket) - UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap bool, maxTemporal int32, outputHeader []byte) (int, int, error) + UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap bool, maxTemporal int32) (int, []byte, error) - UpdateAndGetPadding(newPicture bool, outputHeader []byte) (int, error) + UpdateAndGetPadding(newPicture bool) ([]byte, error) } diff --git a/pkg/sfu/codecmunger/null.go b/pkg/sfu/codecmunger/null.go index c845b4ce1..9d5757011 100644 --- a/pkg/sfu/codecmunger/null.go +++ b/pkg/sfu/codecmunger/null.go @@ -45,10 +45,10 @@ func (n *Null) SetLast(_extPkt *buffer.ExtPacket) { func (n *Null) UpdateOffsets(_extPkt *buffer.ExtPacket) { } -func (n *Null) UpdateAndGet(_extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap bool, maxTemporal int32, outputHeader []byte) (int, int, error) { - return 0, 0, nil +func (n *Null) UpdateAndGet(_extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap bool, maxTemporal int32) (int, []byte, error) { + return 0, nil, nil } -func (n *Null) UpdateAndGetPadding(newPicture bool, outputHeader []byte) (int, error) { - return 0, nil +func (n *Null) UpdateAndGetPadding(newPicture bool) ([]byte, error) { + return nil, nil } diff --git a/pkg/sfu/codecmunger/vp8.go b/pkg/sfu/codecmunger/vp8.go index ca352efb2..97c37bc4c 100644 --- a/pkg/sfu/codecmunger/vp8.go +++ b/pkg/sfu/codecmunger/vp8.go @@ -158,10 +158,10 @@ func (v *VP8) UpdateOffsets(extPkt *buffer.ExtPacket) { v.exemptedPictureIds = orderedmap.NewOrderedMap[int32, bool]() } -func (v *VP8) UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap bool, maxTemporalLayer int32, outputHeader []byte) (int, int, error) { +func (v *VP8) UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap bool, maxTemporalLayer int32) (int, []byte, error) { vp8, ok := extPkt.Payload.(buffer.VP8) if !ok { - return 0, 0, ErrNotVP8 + return 0, nil, ErrNotVP8 } extPictureId := v.pictureIdWrapHandler.Unwrap(vp8.PictureID, vp8.M) @@ -170,7 +170,7 @@ func (v *VP8) UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap if snOutOfOrder { pictureIdOffset, ok := v.missingPictureIds.Get(extPictureId) if !ok { - return 0, 0, ErrOutOfOrderVP8PictureIdCacheMiss + return 0, nil, ErrOutOfOrderVP8PictureIdCacheMiss } // the out-of-order picture id cannot be deleted from the cache @@ -195,11 +195,11 @@ func (v *VP8) UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap IsKeyFrame: vp8.IsKeyFrame, HeaderSize: vp8.HeaderSize + buffer.VPxPictureIdSizeDiff(mungedPictureId > 127, vp8.M), } - n, err := vp8Packet.MarshalTo(outputHeader) + vp8HeaderBytes, err := vp8Packet.Marshal() if err != nil { - return 0, 0, err + return 0, nil, err } - return vp8.HeaderSize, n, nil + return vp8.HeaderSize, vp8HeaderBytes, nil } prevMaxPictureId := v.pictureIdWrapHandler.MaxPictureId() @@ -267,7 +267,7 @@ func (v *VP8) UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap v.pictureIdOffset += 1 } - return 0, 0, ErrFilteredVP8TemporalLayer + return 0, nil, ErrFilteredVP8TemporalLayer } } } @@ -302,14 +302,14 @@ func (v *VP8) UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap IsKeyFrame: vp8.IsKeyFrame, HeaderSize: vp8.HeaderSize + buffer.VPxPictureIdSizeDiff(mungedPictureId > 127, vp8.M), } - n, err := vp8Packet.MarshalTo(outputHeader) + vp8HeaderBytes, err := vp8Packet.Marshal() if err != nil { - return 0, 0, err + return 0, nil, err } - return vp8.HeaderSize, n, nil + return vp8.HeaderSize, vp8HeaderBytes, nil } -func (v *VP8) UpdateAndGetPadding(newPicture bool, outputHeader []byte) (int, error) { +func (v *VP8) UpdateAndGetPadding(newPicture bool) ([]byte, error) { offset := 0 if newPicture { offset = 1 @@ -367,7 +367,7 @@ func (v *VP8) UpdateAndGetPadding(newPicture bool, outputHeader []byte) (int, er IsKeyFrame: true, HeaderSize: headerSize, } - return vp8Packet.MarshalTo(outputHeader) + return vp8Packet.Marshal() } // for testing only diff --git a/pkg/sfu/codecmunger/vp8_test.go b/pkg/sfu/codecmunger/vp8_test.go index 155c7249c..31d33dd65 100644 --- a/pkg/sfu/codecmunger/vp8_test.go +++ b/pkg/sfu/codecmunger/vp8_test.go @@ -166,7 +166,6 @@ func TestUpdateOffsets(t *testing.T) { func TestOutOfOrderPictureId(t *testing.T) { v := newVP8() - buf := make([]byte, 100) params := &testutils.TestExtPacketParams{ SequenceNumber: 23333, @@ -190,17 +189,17 @@ func TestOutOfOrderPictureId(t *testing.T) { } extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8) v.SetLast(extPkt) - v.UpdateAndGet(extPkt, false, false, 2, buf) + v.UpdateAndGet(extPkt, false, false, 2) // out-of-order sequence number not in the missing picture id cache vp8.PictureID = 13466 extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) - nIn, nOut, err := v.UpdateAndGet(extPkt, true, false, 2, buf) + nIn, buf, err := v.UpdateAndGet(extPkt, true, false, 2) require.Error(t, err) require.ErrorIs(t, err, ErrOutOfOrderVP8PictureIdCacheMiss) require.Equal(t, 0, nIn) - require.Equal(t, 0, nOut) + require.Nil(t, buf) // create a hole in picture id vp8.PictureID = 13469 @@ -223,10 +222,10 @@ func TestOutOfOrderPictureId(t *testing.T) { } marshalledVP8, err := expectedVP8.Marshal() require.NoError(t, err) - nIn, nOut, err = v.UpdateAndGet(extPkt, false, true, 2, buf) + nIn, buf, err = v.UpdateAndGet(extPkt, false, true, 2) require.NoError(t, err) require.Equal(t, 6, nIn) - require.Equal(t, marshalledVP8, buf[:nOut]) + require.Equal(t, marshalledVP8, buf) // all three, the last, the current and the in-between should have been added to missing picture id cache value, ok := v.PictureIdOffset(13467) @@ -262,15 +261,14 @@ func TestOutOfOrderPictureId(t *testing.T) { } marshalledVP8, err = expectedVP8.Marshal() require.NoError(t, err) - nIn, nOut, err = v.UpdateAndGet(extPkt, true, false, 2, buf) + nIn, buf, err = v.UpdateAndGet(extPkt, true, false, 2) require.NoError(t, err) require.Equal(t, 6, nIn) - require.Equal(t, marshalledVP8, buf[:nOut]) + require.Equal(t, marshalledVP8, buf) } func TestTemporalLayerFiltering(t *testing.T) { v := newVP8() - buf := make([]byte, 100) params := &testutils.TestExtPacketParams{ SequenceNumber: 23333, @@ -296,11 +294,11 @@ func TestTemporalLayerFiltering(t *testing.T) { v.SetLast(extPkt) // translate - nIn, nOut, err := v.UpdateAndGet(extPkt, false, false, 0, buf) + nIn, buf, err := v.UpdateAndGet(extPkt, false, false, 0) require.Error(t, err) require.ErrorIs(t, err, ErrFilteredVP8TemporalLayer) require.Equal(t, 0, nIn) - require.Equal(t, 0, nOut) + require.Nil(t, buf) dropped, _ := v.droppedPictureIds.Get(13467) require.True(t, dropped) require.EqualValues(t, 1, v.pictureIdOffset) @@ -310,11 +308,11 @@ func TestTemporalLayerFiltering(t *testing.T) { params.SequenceNumber = 23334 extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) - nIn, nOut, err = v.UpdateAndGet(extPkt, false, false, 0, buf) + nIn, buf, err = v.UpdateAndGet(extPkt, false, false, 0) require.Error(t, err) require.ErrorIs(t, err, ErrFilteredVP8TemporalLayer) require.Equal(t, 0, nIn) - require.Equal(t, 0, nOut) + require.Nil(t, buf) dropped, _ = v.droppedPictureIds.Get(13467) require.True(t, dropped) require.EqualValues(t, 1, v.pictureIdOffset) @@ -324,11 +322,11 @@ func TestTemporalLayerFiltering(t *testing.T) { params.SequenceNumber = 23337 extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) - nIn, nOut, err = v.UpdateAndGet(extPkt, false, false, 0, buf) + nIn, buf, err = v.UpdateAndGet(extPkt, false, false, 0) require.Error(t, err) require.ErrorIs(t, err, ErrFilteredVP8TemporalLayer) require.Equal(t, 0, nIn) - require.Equal(t, 0, nOut) + require.Nil(t, buf) dropped, _ = v.droppedPictureIds.Get(13467) require.True(t, dropped) require.EqualValues(t, 1, v.pictureIdOffset) @@ -336,7 +334,6 @@ func TestTemporalLayerFiltering(t *testing.T) { func TestGapInSequenceNumberSamePicture(t *testing.T) { v := newVP8() - buf := make([]byte, 100) params := &testutils.TestExtPacketParams{ SequenceNumber: 65533, @@ -379,10 +376,10 @@ func TestGapInSequenceNumberSamePicture(t *testing.T) { } marshalledVP8, err := expectedVP8.Marshal() require.NoError(t, err) - nIn, nOut, err := v.UpdateAndGet(extPkt, false, false, 2, buf) + nIn, buf, err := v.UpdateAndGet(extPkt, false, false, 2) require.NoError(t, err) require.Equal(t, 6, nIn) - require.Equal(t, marshalledVP8, buf[:nOut]) + require.Equal(t, marshalledVP8, buf) // telling there is a gap in sequence number will add pictures to missing picture cache expectedVP8 = &buffer.VP8{ @@ -402,10 +399,10 @@ func TestGapInSequenceNumberSamePicture(t *testing.T) { } marshalledVP8, err = expectedVP8.Marshal() require.NoError(t, err) - nIn, nOut, err = v.UpdateAndGet(extPkt, false, true, 2, buf) + nIn, buf, err = v.UpdateAndGet(extPkt, false, true, 2) require.NoError(t, err) require.Equal(t, 6, nIn) - require.Equal(t, marshalledVP8, buf[:nOut]) + require.Equal(t, marshalledVP8, buf) value, ok := v.PictureIdOffset(13467) require.True(t, ok) @@ -414,7 +411,6 @@ func TestGapInSequenceNumberSamePicture(t *testing.T) { func TestUpdateAndGetPadding(t *testing.T) { v := newVP8() - buf := make([]byte, 100) params := &testutils.TestExtPacketParams{ SequenceNumber: 23333, @@ -442,7 +438,7 @@ func TestUpdateAndGetPadding(t *testing.T) { v.SetLast(extPkt) // getting padding with repeat of last picture - n, err := v.UpdateAndGetPadding(false, buf) + buf, err := v.UpdateAndGetPadding(false) require.NoError(t, err) expectedVP8 := buffer.VP8{ FirstByte: 16, @@ -461,10 +457,10 @@ func TestUpdateAndGetPadding(t *testing.T) { } marshalledVP8, err := expectedVP8.Marshal() require.NoError(t, err) - require.Equal(t, marshalledVP8, buf[:n]) + require.Equal(t, marshalledVP8, buf) // getting padding with new picture - n, err = v.UpdateAndGetPadding(true, buf) + buf, err = v.UpdateAndGetPadding(true) require.NoError(t, err) expectedVP8 = buffer.VP8{ FirstByte: 16, @@ -483,7 +479,7 @@ func TestUpdateAndGetPadding(t *testing.T) { } marshalledVP8, err = expectedVP8.Marshal() require.NoError(t, err) - require.Equal(t, marshalledVP8, buf[:n]) + require.Equal(t, marshalledVP8, buf) } func TestVP8PictureIdWrapHandler(t *testing.T) { diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 7c77c8122..5f3dd893f 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -714,18 +714,14 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { poolEntity := PacketFactory.Get().(*[]byte) payload := *poolEntity - shouldForward, incomingHeaderSize, outgoingHeaderSize, err := d.forwarder.TranslateCodecHeader(extPkt, &tp.rtp, payload) - if !shouldForward { - PacketFactory.Put(poolEntity) - return err - } - n := copy(payload[outgoingHeaderSize:], extPkt.Packet.Payload[incomingHeaderSize:]) - if n != len(extPkt.Packet.Payload[incomingHeaderSize:]) { - d.params.Logger.Errorw("payload overflow", nil, "want", len(extPkt.Packet.Payload[incomingHeaderSize:]), "have", n) + copy(payload, tp.codecBytes) + n := copy(payload[len(tp.codecBytes):], extPkt.Packet.Payload[tp.incomingHeaderSize:]) + if n != len(extPkt.Packet.Payload[tp.incomingHeaderSize:]) { + d.params.Logger.Errorw("payload overflow", nil, "want", len(extPkt.Packet.Payload[tp.incomingHeaderSize:]), "have", n) PacketFactory.Put(poolEntity) return ErrPayloadOverflow } - payload = payload[:outgoingHeaderSize+n] + payload = payload[:len(tp.codecBytes)+n] hdr, err := d.getTranslatedRTPHeader(extPkt, &tp) if err != nil { @@ -797,8 +793,8 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { tp.rtp.extTimestamp, hdr.Marker, int8(layer), - payload[:outgoingHeaderSize], - incomingHeaderSize, + payload[:len(tp.codecBytes)], + tp.incomingHeaderSize, tp.ddBytes, actBytes, ) @@ -1506,15 +1502,16 @@ func (d *DownTrack) getVP8BlankFrame(frameEndNeeded bool) ([]byte, error) { // Used even when closing out a previous frame. Looks like receivers // do not care about content (it will probably end up being an undecodable // frame, but that should be okay as there are key frames following) - payload := make([]byte, 1000) - n, err := d.forwarder.GetPadding(frameEndNeeded, payload) + header, err := d.forwarder.GetPadding(frameEndNeeded) if err != nil { return nil, err } - copy(payload[n:], VP8KeyFrame8x8) - trailerLen := d.maybeAddTrailer(payload[n+len(VP8KeyFrame8x8):]) - return payload[:n+len(VP8KeyFrame8x8)+trailerLen], nil + payload := make([]byte, 1000) + copy(payload, header) + copy(payload[len(header):], VP8KeyFrame8x8) + trailerLen := d.maybeAddTrailer(payload[len(header)+len(VP8KeyFrame8x8):]) + return payload[:len(header)+len(VP8KeyFrame8x8)+trailerLen], nil } func (d *DownTrack) getH264BlankFrame(_frameEndNeeded bool) ([]byte, error) { diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index af610e34c..92c354c3b 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -174,12 +174,14 @@ func (v *VideoTransition) MarshalLogObject(e zapcore.ObjectEncoder) error { // ------------------------------------------------------------------- type TranslationParams struct { - shouldDrop bool - isResuming bool - isSwitching bool - rtp TranslationParamsRTP - ddBytes []byte - marker bool + shouldDrop bool + isResuming bool + isSwitching bool + rtp TranslationParamsRTP + ddBytes []byte + incomingHeaderSize int + codecBytes []byte + marker bool } // ------------------------------------------------------------------- @@ -1771,11 +1773,11 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e } // should be called with lock held -func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer int32, tp *TranslationParams) error { +func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer int32, tp *TranslationParams) (bool, error) { if f.lastSSRC != extPkt.Packet.SSRC { if err := f.processSourceSwitch(extPkt, layer); err != nil { tp.shouldDrop = true - return nil + return false, nil } f.logger.Debugw("switching feed", "from", f.lastSSRC, "to", extPkt.Packet.SSRC) f.lastSSRC = extPkt.Packet.SSRC @@ -1785,19 +1787,24 @@ func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer i if err != nil { tp.shouldDrop = true if err == ErrPaddingOnlyPacket || err == ErrDuplicatePacket || err == ErrOutOfOrderSequenceNumberCacheMiss { - return nil + return false, nil } - return err + return false, err } tp.rtp = tpRTP - return nil + + if len(extPkt.Packet.Payload) > 0 { + return f.translateCodecHeader(extPkt, tp) + } + + return false, nil } // should be called with lock held func (f *Forwarder) getTranslationParamsAudio(extPkt *buffer.ExtPacket, layer int32) (TranslationParams, error) { tp := TranslationParams{} - if err := f.getTranslationParamsCommon(extPkt, layer, &tp); err != nil { + if _, err := f.getTranslationParamsCommon(extPkt, layer, &tp); err != nil { tp.shouldDrop = true return tp, err } @@ -1861,49 +1868,40 @@ func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer in return tp, nil } - err := f.getTranslationParamsCommon(extPkt, layer, &tp) + isTemporalSwitching, err := f.getTranslationParamsCommon(extPkt, layer, &tp) if tp.shouldDrop { - maybeRollback(result.IsSwitching) + maybeRollback(result.IsSwitching || isTemporalSwitching) return tp, err } - return tp, nil + return tp, err } -func (f *Forwarder) TranslateCodecHeader(extPkt *buffer.ExtPacket, tpr *TranslationParamsRTP, outputBuffer []byte) (bool, int, int, error) { - f.lock.Lock() - defer f.lock.Unlock() - - maybeRollback := func(isSwitching bool) { - if isSwitching { - f.vls.Rollback() - } - } - +func (f *Forwarder) translateCodecHeader(extPkt *buffer.ExtPacket, tp *TranslationParams) (bool, error) { // codec specific forwarding check and any needed packet munging tl, isSwitching := f.vls.SelectTemporal(extPkt) - inputSize, outputSize, err := f.codecMunger.UpdateAndGet( + inputSize, codecBytes, err := f.codecMunger.UpdateAndGet( extPkt, - tpr.snOrdering == SequenceNumberOrderingOutOfOrder, - tpr.snOrdering == SequenceNumberOrderingGap, + tp.rtp.snOrdering == SequenceNumberOrderingOutOfOrder, + tp.rtp.snOrdering == SequenceNumberOrderingGap, tl, - outputBuffer, ) if err != nil { + tp.shouldDrop = true if err == codecmunger.ErrFilteredVP8TemporalLayer || err == codecmunger.ErrOutOfOrderVP8PictureIdCacheMiss { if err == codecmunger.ErrFilteredVP8TemporalLayer { // filtered temporal layer, update sequence number offset to prevent holes f.rtpMunger.PacketDropped(extPkt) } - maybeRollback(isSwitching) - return false, 0, 0, nil + return isSwitching, nil } - maybeRollback(isSwitching) - return false, 0, 0, err + return isSwitching, err } + tp.incomingHeaderSize = inputSize + tp.codecBytes = codecBytes - return true, inputSize, outputSize, nil + return isSwitching, nil } func (f *Forwarder) maybeStart() { @@ -1980,11 +1978,11 @@ func (f *Forwarder) GetSnTsForBlankFrames(frameRate uint32, numPackets int) ([]S return snts, frameEndNeeded, err } -func (f *Forwarder) GetPadding(frameEndNeeded bool, outputBuffer []byte) (int, error) { +func (f *Forwarder) GetPadding(frameEndNeeded bool) ([]byte, error) { f.lock.Lock() defer f.lock.Unlock() - return f.codecMunger.UpdateAndGetPadding(!frameEndNeeded, outputBuffer) + return f.codecMunger.UpdateAndGetPadding(!frameEndNeeded) } func (f *Forwarder) RTPMungerDebugInfo() map[string]interface{} { diff --git a/pkg/sfu/forwarder_test.go b/pkg/sfu/forwarder_test.go index 9279bc736..608ab9155 100644 --- a/pkg/sfu/forwarder_test.go +++ b/pkg/sfu/forwarder_test.go @@ -1368,7 +1368,6 @@ func TestForwarderGetTranslationParamsAudio(t *testing.T) { } func TestForwarderGetTranslationParamsVideo(t *testing.T) { - buf := make([]byte, 100) f := newForwarder(testutils.TestVP8Codec, webrtc.RTPCodecTypeVideo) params := &testutils.TestExtPacketParams{ @@ -1432,22 +1431,6 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { IsKeyFrame: true, } extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) - expectedTP = TranslationParams{ - isSwitching: true, - isResuming: true, - rtp: TranslationParamsRTP{ - snOrdering: SequenceNumberOrderingContiguous, - extSequenceNumber: 23333, - extTimestamp: 0xabcdef, - }, - marker: true, - } - actualTP, err = f.GetTranslationParams(extPkt, 0) - require.NoError(t, err) - require.Equal(t, expectedTP, actualTP) - require.True(t, f.started) - require.Equal(t, f.lastSSRC, params.SSRC) - expectedVP8 := &buffer.VP8{ FirstByte: 25, I: true, @@ -1464,13 +1447,23 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { IsKeyFrame: true, } marshalledVP8, err := expectedVP8.Marshal() + expectedTP = TranslationParams{ + isSwitching: true, + isResuming: true, + rtp: TranslationParamsRTP{ + snOrdering: SequenceNumberOrderingContiguous, + extSequenceNumber: 23333, + extTimestamp: 0xabcdef, + }, + incomingHeaderSize: 6, + codecBytes: marshalledVP8, + marker: true, + } + actualTP, err = f.GetTranslationParams(extPkt, 0) require.NoError(t, err) - shouldForward, incomingHeaderSize, outgoingHeaderSize, err := f.TranslateCodecHeader(extPkt, &actualTP.rtp, buf) - require.NoError(t, err) - require.True(t, shouldForward) - require.Equal(t, 6, incomingHeaderSize) - require.Equal(t, 6, outgoingHeaderSize) - require.Equal(t, marshalledVP8, buf[:outgoingHeaderSize]) + require.Equal(t, expectedTP, actualTP) + require.True(t, f.started) + require.Equal(t, f.lastSSRC, params.SSRC) // send a duplicate, should be dropped expectedTP = TranslationParams{ @@ -1518,17 +1511,6 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { PayloadSize: 20, } extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) - expectedTP = TranslationParams{ - rtp: TranslationParamsRTP{ - snOrdering: SequenceNumberOrderingContiguous, - extSequenceNumber: 23334, - extTimestamp: 0xabcdef, - }, - } - actualTP, err = f.GetTranslationParams(extPkt, 0) - require.NoError(t, err) - require.Equal(t, expectedTP, actualTP) - expectedVP8 = &buffer.VP8{ FirstByte: 25, I: true, @@ -1546,12 +1528,18 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { } marshalledVP8, err = expectedVP8.Marshal() require.NoError(t, err) - shouldForward, incomingHeaderSize, outgoingHeaderSize, err = f.TranslateCodecHeader(extPkt, &actualTP.rtp, buf) + expectedTP = TranslationParams{ + rtp: TranslationParamsRTP{ + snOrdering: SequenceNumberOrderingContiguous, + extSequenceNumber: 23334, + extTimestamp: 0xabcdef, + }, + incomingHeaderSize: 6, + codecBytes: marshalledVP8, + } + actualTP, err = f.GetTranslationParams(extPkt, 0) require.NoError(t, err) - require.True(t, shouldForward) - require.Equal(t, 6, incomingHeaderSize) - require.Equal(t, 6, outgoingHeaderSize) - require.Equal(t, marshalledVP8, buf[:outgoingHeaderSize]) + require.Equal(t, expectedTP, actualTP) // temporal layer matching target, should be forwarded params = &testutils.TestExtPacketParams{ @@ -1577,17 +1565,6 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { IsKeyFrame: true, } extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) - expectedTP = TranslationParams{ - rtp: TranslationParamsRTP{ - snOrdering: SequenceNumberOrderingContiguous, - extSequenceNumber: 23335, - extTimestamp: 0xabcdef, - }, - } - actualTP, err = f.GetTranslationParams(extPkt, 0) - require.NoError(t, err) - require.Equal(t, expectedTP, actualTP) - expectedVP8 = &buffer.VP8{ FirstByte: 25, I: true, @@ -1605,12 +1582,18 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { } marshalledVP8, err = expectedVP8.Marshal() require.NoError(t, err) - shouldForward, incomingHeaderSize, outgoingHeaderSize, err = f.TranslateCodecHeader(extPkt, &actualTP.rtp, buf) + expectedTP = TranslationParams{ + rtp: TranslationParamsRTP{ + snOrdering: SequenceNumberOrderingContiguous, + extSequenceNumber: 23335, + extTimestamp: 0xabcdef, + }, + incomingHeaderSize: 6, + codecBytes: marshalledVP8, + } + actualTP, err = f.GetTranslationParams(extPkt, 0) require.NoError(t, err) - require.True(t, shouldForward) - require.Equal(t, 6, incomingHeaderSize) - require.Equal(t, 6, outgoingHeaderSize) - require.Equal(t, marshalledVP8, buf[:outgoingHeaderSize]) + require.Equal(t, expectedTP, actualTP) // temporal layer higher than target, should be dropped params = &testutils.TestExtPacketParams{ @@ -1636,6 +1619,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { } extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) expectedTP = TranslationParams{ + shouldDrop: true, rtp: TranslationParamsRTP{ snOrdering: SequenceNumberOrderingContiguous, extSequenceNumber: 23336, @@ -1646,10 +1630,6 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { require.NoError(t, err) require.Equal(t, expectedTP, actualTP) - shouldForward, incomingHeaderSize, outgoingHeaderSize, err = f.TranslateCodecHeader(extPkt, &actualTP.rtp, buf) - require.NoError(t, err) - require.False(t, shouldForward) - // RTP sequence number and VP8 picture id should be contiguous after dropping higher temporal layer picture params = &testutils.TestExtPacketParams{ SequenceNumber: 23338, @@ -1673,17 +1653,6 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { IsKeyFrame: false, } extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) - expectedTP = TranslationParams{ - rtp: TranslationParamsRTP{ - snOrdering: SequenceNumberOrderingContiguous, - extSequenceNumber: 23336, - extTimestamp: 0xabcdef, - }, - } - actualTP, err = f.GetTranslationParams(extPkt, 0) - require.NoError(t, err) - require.Equal(t, expectedTP, actualTP) - expectedVP8 = &buffer.VP8{ FirstByte: 25, I: true, @@ -1701,12 +1670,18 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { } marshalledVP8, err = expectedVP8.Marshal() require.NoError(t, err) - shouldForward, incomingHeaderSize, outgoingHeaderSize, err = f.TranslateCodecHeader(extPkt, &actualTP.rtp, buf) + expectedTP = TranslationParams{ + rtp: TranslationParamsRTP{ + snOrdering: SequenceNumberOrderingContiguous, + extSequenceNumber: 23336, + extTimestamp: 0xabcdef, + }, + incomingHeaderSize: 6, + codecBytes: marshalledVP8, + } + actualTP, err = f.GetTranslationParams(extPkt, 0) require.NoError(t, err) - require.True(t, shouldForward) - require.Equal(t, 6, incomingHeaderSize) - require.Equal(t, 6, outgoingHeaderSize) - require.Equal(t, marshalledVP8, buf[:outgoingHeaderSize]) + require.Equal(t, expectedTP, actualTP) // padding only packet after a gap should be forwarded params = &testutils.TestExtPacketParams{ @@ -1776,19 +1751,6 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { } extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) - expectedTP = TranslationParams{ - isSwitching: true, - rtp: TranslationParamsRTP{ - snOrdering: SequenceNumberOrderingContiguous, - extSequenceNumber: 23339, - extTimestamp: 0xabcdf0, - }, - } - actualTP, err = f.GetTranslationParams(extPkt, 1) - require.NoError(t, err) - require.Equal(t, expectedTP, actualTP) - require.Equal(t, f.lastSSRC, params.SSRC) - expectedVP8 = &buffer.VP8{ FirstByte: 25, I: true, @@ -1806,12 +1768,20 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { } marshalledVP8, err = expectedVP8.Marshal() require.NoError(t, err) - shouldForward, incomingHeaderSize, outgoingHeaderSize, err = f.TranslateCodecHeader(extPkt, &actualTP.rtp, buf) + expectedTP = TranslationParams{ + isSwitching: true, + rtp: TranslationParamsRTP{ + snOrdering: SequenceNumberOrderingContiguous, + extSequenceNumber: 23339, + extTimestamp: 0xabcdf0, + }, + incomingHeaderSize: 5, + codecBytes: marshalledVP8, + } + actualTP, err = f.GetTranslationParams(extPkt, 1) require.NoError(t, err) - require.True(t, shouldForward) - require.Equal(t, 5, incomingHeaderSize) - require.Equal(t, 6, outgoingHeaderSize) - require.Equal(t, marshalledVP8, buf[:outgoingHeaderSize]) + require.Equal(t, expectedTP, actualTP) + require.Equal(t, f.lastSSRC, params.SSRC) } func TestForwarderGetSnTsForPadding(t *testing.T) { @@ -1959,7 +1929,6 @@ func TestForwarderGetSnTsForBlankFrames(t *testing.T) { } func TestForwarderGetPaddingVP8(t *testing.T) { - buf := make([]byte, 100) f := newForwarder(testutils.TestVP8Codec, webrtc.RTPCodecTypeVideo) params := &testutils.TestExtPacketParams{ @@ -2010,11 +1979,11 @@ func TestForwarderGetPaddingVP8(t *testing.T) { HeaderSize: 6, IsKeyFrame: true, } - n, err := f.GetPadding(true, buf) + buf, err := f.GetPadding(true) require.NoError(t, err) marshalledVP8, err := expectedVP8.Marshal() require.NoError(t, err) - require.Equal(t, marshalledVP8, buf[:n]) + require.Equal(t, marshalledVP8, buf) // getting padding with no frame end needed, should get next picture id expectedVP8 = buffer.VP8{ @@ -2032,9 +2001,9 @@ func TestForwarderGetPaddingVP8(t *testing.T) { HeaderSize: 6, IsKeyFrame: true, } - n, err = f.GetPadding(false, buf) + buf, err = f.GetPadding(false) require.NoError(t, err) marshalledVP8, err = expectedVP8.Marshal() require.NoError(t, err) - require.Equal(t, marshalledVP8, buf[:n]) + require.Equal(t, marshalledVP8, buf) } From 680a07e896913329c3abadf1241fe22ba679f2bc Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Fri, 19 Apr 2024 19:21:55 -0700 Subject: [PATCH 62/78] update pion deps (#2667) * update pion deps * deps --- go.mod | 4 ++-- go.sum | 8 ++++---- pkg/sfu/downtrackspreader.go | 4 +--- pkg/utils/opsqueue.go | 8 ++++---- 4 files changed, 11 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index 9a19b25fc..5535f6473 100644 --- a/go.mod +++ b/go.mod @@ -27,14 +27,14 @@ require ( github.com/mitchellh/go-homedir v1.1.0 github.com/olekukonko/tablewriter v0.0.5 github.com/pion/dtls/v2 v2.2.10 - github.com/pion/ice/v2 v2.3.15 + github.com/pion/ice/v2 v2.3.16 github.com/pion/interceptor v0.1.25 github.com/pion/rtcp v1.2.14 github.com/pion/rtp v1.8.5 github.com/pion/sctp v1.8.16 github.com/pion/sdp/v3 v3.0.9 github.com/pion/transport/v2 v2.2.4 - github.com/pion/turn/v2 v2.1.5 + github.com/pion/turn/v2 v2.1.6 github.com/pion/webrtc/v3 v3.2.38 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.19.0 diff --git a/go.sum b/go.sum index dc97b17ca..decbb3e53 100644 --- a/go.sum +++ b/go.sum @@ -168,8 +168,8 @@ github.com/pion/datachannel v1.5.5/go.mod h1:iMz+lECmfdCMqFRhXhcA/219B0SQlbpoR2V github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= github.com/pion/dtls/v2 v2.2.10 h1:u2Axk+FyIR1VFTPurktB+1zoEPGIW3bmyj3LEFrXjAA= github.com/pion/dtls/v2 v2.2.10/go.mod h1:d9SYc9fch0CqK90mRk1dC7AkzzpwJj6u2GU3u+9pqFE= -github.com/pion/ice/v2 v2.3.15 h1:oCGVqnd6OWmJr4I6eQwSWn8VJDF45wIXFTjV3tyyris= -github.com/pion/ice/v2 v2.3.15/go.mod h1:KXJJcZK7E8WzrBEYnV4UtqEZsGeWfHxsNqhVcVvgjxw= +github.com/pion/ice/v2 v2.3.16 h1:ajKSorlci0Vos36tFpfYZaTTNcmZrh6giN0YSuRjSQ8= +github.com/pion/ice/v2 v2.3.16/go.mod h1:KXJJcZK7E8WzrBEYnV4UtqEZsGeWfHxsNqhVcVvgjxw= github.com/pion/interceptor v0.1.25 h1:pwY9r7P6ToQ3+IF0bajN0xmk/fNw/suTgaTdlwTDmhc= github.com/pion/interceptor v0.1.25/go.mod h1:wkbPYAak5zKsfpVDYMtEfWEy8D4zL+rpxCxPImLOg3Y= github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= @@ -206,8 +206,8 @@ github.com/pion/transport/v3 v3.0.1/go.mod h1:UY7kiITrlMv7/IKgd5eTUcaahZx5oUN3l9 github.com/pion/transport/v3 v3.0.2 h1:r+40RJR25S9w3jbA6/5uEPTzcdn7ncyU44RWCbHkLg4= github.com/pion/transport/v3 v3.0.2/go.mod h1:nIToODoOlb5If2jF9y2Igfx3PFYWfuXi37m0IlWa/D0= github.com/pion/turn/v2 v2.1.3/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= -github.com/pion/turn/v2 v2.1.5 h1:tTyy7TM3DCoX9IxTt/yHc/bThiRLyXK3T1YbNcgx9k4= -github.com/pion/turn/v2 v2.1.5/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= +github.com/pion/turn/v2 v2.1.6 h1:Xr2niVsiPTB0FPtt+yAWKFUkU1eotQbGgpTIld4x1Gc= +github.com/pion/turn/v2 v2.1.6/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= github.com/pion/webrtc/v3 v3.2.38 h1:oA52VJAJhOjSi1JpKjf0CM+cCiZ3b7jBxvsoOiajeDU= github.com/pion/webrtc/v3 v3.2.38/go.mod h1:AQ8p56OLbm3MjhYovYdgPuyX6oc+JcKx/HFoCGFcYzA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= diff --git a/pkg/sfu/downtrackspreader.go b/pkg/sfu/downtrackspreader.go index 3768592f3..68fc7b411 100644 --- a/pkg/sfu/downtrackspreader.go +++ b/pkg/sfu/downtrackspreader.go @@ -96,9 +96,7 @@ func (d *DownTrackSpreader) Broadcast(writer func(TrackSender)) { // 100µs is enough to amortize the overhead and provide sufficient load balancing. // WriteRTP takes about 50µs on average, so we write to 2 down tracks per loop. step := uint64(2) - utils.ParallelExec(downTracks, threshold, step, func(dt TrackSender) { - writer(dt) - }) + utils.ParallelExec(downTracks, threshold, step, writer) } func (d *DownTrackSpreader) DownTrackCount() int { diff --git a/pkg/utils/opsqueue.go b/pkg/utils/opsqueue.go index 2513f0b8c..30f9f4b39 100644 --- a/pkg/utils/opsqueue.go +++ b/pkg/utils/opsqueue.go @@ -32,8 +32,8 @@ type OpsQueueParams struct { type UntypedQueueOp func() -func (it UntypedQueueOp) run() { - it() +func (op UntypedQueueOp) run() { + op() } type OpsQueue struct { @@ -49,8 +49,8 @@ type typedQueueOp[T any] struct { arg T } -func (it typedQueueOp[T]) run() { - it.fn(it.arg) +func (op typedQueueOp[T]) run() { + op.fn(op.arg) } type TypedOpsQueue[T any] struct { From 365b6f91e2008a9798c1cc5edf9146722d970cf0 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Fri, 19 Apr 2024 21:58:14 -0700 Subject: [PATCH 63/78] update pion/ice (#2668) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 5535f6473..5bf90ef26 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/mitchellh/go-homedir v1.1.0 github.com/olekukonko/tablewriter v0.0.5 github.com/pion/dtls/v2 v2.2.10 - github.com/pion/ice/v2 v2.3.16 + github.com/pion/ice/v2 v2.3.17 github.com/pion/interceptor v0.1.25 github.com/pion/rtcp v1.2.14 github.com/pion/rtp v1.8.5 diff --git a/go.sum b/go.sum index decbb3e53..2d4bcb001 100644 --- a/go.sum +++ b/go.sum @@ -168,8 +168,8 @@ github.com/pion/datachannel v1.5.5/go.mod h1:iMz+lECmfdCMqFRhXhcA/219B0SQlbpoR2V github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= github.com/pion/dtls/v2 v2.2.10 h1:u2Axk+FyIR1VFTPurktB+1zoEPGIW3bmyj3LEFrXjAA= github.com/pion/dtls/v2 v2.2.10/go.mod h1:d9SYc9fch0CqK90mRk1dC7AkzzpwJj6u2GU3u+9pqFE= -github.com/pion/ice/v2 v2.3.16 h1:ajKSorlci0Vos36tFpfYZaTTNcmZrh6giN0YSuRjSQ8= -github.com/pion/ice/v2 v2.3.16/go.mod h1:KXJJcZK7E8WzrBEYnV4UtqEZsGeWfHxsNqhVcVvgjxw= +github.com/pion/ice/v2 v2.3.17 h1:o3wz2w3BpXdQyYcKpLyrcPufFA/zcgOrV71JIhIPt8U= +github.com/pion/ice/v2 v2.3.17/go.mod h1:KXJJcZK7E8WzrBEYnV4UtqEZsGeWfHxsNqhVcVvgjxw= github.com/pion/interceptor v0.1.25 h1:pwY9r7P6ToQ3+IF0bajN0xmk/fNw/suTgaTdlwTDmhc= github.com/pion/interceptor v0.1.25/go.mod h1:wkbPYAak5zKsfpVDYMtEfWEy8D4zL+rpxCxPImLOg3Y= github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= From c195dd4a1932549cd9e49c63b4ab178f6038d64a Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Sun, 21 Apr 2024 00:44:52 -0700 Subject: [PATCH 64/78] update pion/ice (#2669) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 5bf90ef26..7a6787451 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/mitchellh/go-homedir v1.1.0 github.com/olekukonko/tablewriter v0.0.5 github.com/pion/dtls/v2 v2.2.10 - github.com/pion/ice/v2 v2.3.17 + github.com/pion/ice/v2 v2.3.18 github.com/pion/interceptor v0.1.25 github.com/pion/rtcp v1.2.14 github.com/pion/rtp v1.8.5 diff --git a/go.sum b/go.sum index 2d4bcb001..f5741fe46 100644 --- a/go.sum +++ b/go.sum @@ -168,8 +168,8 @@ github.com/pion/datachannel v1.5.5/go.mod h1:iMz+lECmfdCMqFRhXhcA/219B0SQlbpoR2V github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= github.com/pion/dtls/v2 v2.2.10 h1:u2Axk+FyIR1VFTPurktB+1zoEPGIW3bmyj3LEFrXjAA= github.com/pion/dtls/v2 v2.2.10/go.mod h1:d9SYc9fch0CqK90mRk1dC7AkzzpwJj6u2GU3u+9pqFE= -github.com/pion/ice/v2 v2.3.17 h1:o3wz2w3BpXdQyYcKpLyrcPufFA/zcgOrV71JIhIPt8U= -github.com/pion/ice/v2 v2.3.17/go.mod h1:KXJJcZK7E8WzrBEYnV4UtqEZsGeWfHxsNqhVcVvgjxw= +github.com/pion/ice/v2 v2.3.18 h1:8Q1WsWhWlyHyb5TbLE9bv5AQviaiq0Airetgae9mAH0= +github.com/pion/ice/v2 v2.3.18/go.mod h1:KXJJcZK7E8WzrBEYnV4UtqEZsGeWfHxsNqhVcVvgjxw= github.com/pion/interceptor v0.1.25 h1:pwY9r7P6ToQ3+IF0bajN0xmk/fNw/suTgaTdlwTDmhc= github.com/pion/interceptor v0.1.25/go.mod h1:wkbPYAak5zKsfpVDYMtEfWEy8D4zL+rpxCxPImLOg3Y= github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= From af0b0c4734b460e0da8e9efe39c7103de7c51cc2 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sun, 21 Apr 2024 23:35:24 +0530 Subject: [PATCH 65/78] Connection quality LOST only if RTCP is also not available. (#2670) * Connection quality LOST only if RTCP is also not available. It is possible that sender stops all layers of video due to some constraint (CPU or bandwidth). Packet reception going dry due to that should not trigger `LOST` quality. Add last received RTCP time also to distinguish the case of real `LOST` and sender stopping traffic. Some bits to watch for - With audio, RTCP reports could be more than 5 seconds apart (5 seconds is the default interval for connection quality scorer), but audio senders usually send silence packets even when there is no input. So audio completely stopping can be considered `LOST`. - With video, have to observe if all clients continue to send RTCP even if all layers are stopped. - RTCP bandwidth is not supposed to exceed the primary stream bandwidth. libwebrtc calculates that and spaces out RTCP reports accordingly. That is the reason why audio reports are that far apart. If a video stream is encoded at a very low bit rate, it could also be sending RTCP rarely. So, there is the case of LOST being indistinguishable from sender stopping all layers. But, this should be a rare case. * typo --- pkg/sfu/buffer/buffer.go | 11 ++++ pkg/sfu/buffer/rtpstats_receiver.go | 11 ++++ pkg/sfu/connectionquality/connectionstats.go | 11 ++-- .../connectionquality/connectionstats_test.go | 51 +++++++++++++++++-- pkg/sfu/connectionquality/scorer.go | 14 +++-- pkg/sfu/receiver.go | 19 +++++++ 6 files changed, 106 insertions(+), 11 deletions(-) diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go index 04e468d7e..85a9f07eb 100644 --- a/pkg/sfu/buffer/buffer.go +++ b/pkg/sfu/buffer/buffer.go @@ -976,6 +976,17 @@ func (b *Buffer) GetDeltaStats() *StreamStatsWithLayers { } } +func (b *Buffer) GetLastSenderReportTime() time.Time { + b.RLock() + defer b.RUnlock() + + if b.rtpStats == nil { + return time.Time{} + } + + return b.rtpStats.LastSenderReportTime() +} + func (b *Buffer) GetAudioLevel() (float64, bool) { b.RLock() defer b.RUnlock() diff --git a/pkg/sfu/buffer/rtpstats_receiver.go b/pkg/sfu/buffer/rtpstats_receiver.go index 3fc6c19a1..6fa9969af 100644 --- a/pkg/sfu/buffer/rtpstats_receiver.go +++ b/pkg/sfu/buffer/rtpstats_receiver.go @@ -458,6 +458,17 @@ func (r *RTPStatsReceiver) GetRtcpSenderReportData() *RTCPSenderReportData { return &srNewestCopy } +func (r *RTPStatsReceiver) LastSenderReportTime() time.Time { + r.lock.RLock() + defer r.lock.RUnlock() + + if r.srNewest != nil { + return r.srNewest.At + } + + return time.Time{} +} + func (r *RTPStatsReceiver) GetRtcpReceptionReport(ssrc uint32, proxyFracLost uint8, snapshotID uint32) *rtcp.ReceptionReport { r.lock.Lock() defer r.lock.Unlock() diff --git a/pkg/sfu/connectionquality/connectionstats.go b/pkg/sfu/connectionquality/connectionstats.go index 8f6035a06..f124edb5c 100644 --- a/pkg/sfu/connectionquality/connectionstats.go +++ b/pkg/sfu/connectionquality/connectionstats.go @@ -37,6 +37,7 @@ const ( type ConnectionStatsReceiverProvider interface { GetDeltaStats() map[uint32]*buffer.StreamStatsWithLayers + GetLastSenderReportTime() time.Time } type ConnectionStatsSenderProvider interface { @@ -202,7 +203,7 @@ func (cs *ConnectionStats) GetScoreAndQuality() (float32, livekit.ConnectionQual return cs.scorer.GetMOSAndQuality() } -func (cs *ConnectionStats) updateScoreWithAggregate(agg *buffer.RTPDeltaInfo, at time.Time) float32 { +func (cs *ConnectionStats) updateScoreWithAggregate(agg *buffer.RTPDeltaInfo, lastRTCPAt time.Time, at time.Time) float32 { var stat windowStat if agg != nil { stat.startedAt = agg.StartTime @@ -214,6 +215,8 @@ func (cs *ConnectionStats) updateScoreWithAggregate(agg *buffer.RTPDeltaInfo, at stat.bytes = agg.Bytes - agg.HeaderBytes // only use media payload size stat.rttMax = agg.RttMax stat.jitterMax = agg.JitterMax + + stat.lastRTCPAt = lastRTCPAt } if at.IsZero() { cs.scorer.Update(&stat) @@ -246,7 +249,7 @@ func (cs *ConnectionStats) updateScoreFromReceiverReport(at time.Time) (float32, } if time.Since(marker) > noReceiverReportTooLongThreshold { // have not received receiver report for a long time when streaming, run with nil stat - return cs.updateScoreWithAggregate(nil, at), nil + return cs.updateScoreWithAggregate(nil, time.Time{}, at), nil } // wait for receiver report, return current score @@ -266,7 +269,7 @@ func (cs *ConnectionStats) updateScoreFromReceiverReport(at time.Time) (float32, if streamingStartedAt.After(agg.StartTime) { agg.StartTime = streamingStartedAt } - return cs.updateScoreWithAggregate(agg, at), streams + return cs.updateScoreWithAggregate(agg, time.Time{}, at), streams } func (cs *ConnectionStats) updateScoreAt(at time.Time) (float32, map[uint32]*buffer.StreamStatsWithLayers) { @@ -290,7 +293,7 @@ func (cs *ConnectionStats) updateScoreAt(at time.Time) (float32, map[uint32]*buf deltaInfoList = append(deltaInfoList, s.RTPStats) } agg := buffer.AggregateRTPDeltaInfo(deltaInfoList) - return cs.updateScoreWithAggregate(agg, at), streams + return cs.updateScoreWithAggregate(agg, cs.params.ReceiverProvider.GetLastSenderReportTime(), at), streams } func (cs *ConnectionStats) updateStreamingStart(at time.Time) time.Time { diff --git a/pkg/sfu/connectionquality/connectionstats_test.go b/pkg/sfu/connectionquality/connectionstats_test.go index 0feedac7f..78e217130 100644 --- a/pkg/sfu/connectionquality/connectionstats_test.go +++ b/pkg/sfu/connectionquality/connectionstats_test.go @@ -29,7 +29,8 @@ import ( // ----------------------------------------------- type testReceiverProvider struct { - streams map[uint32]*buffer.StreamStatsWithLayers + streams map[uint32]*buffer.StreamStatsWithLayers + lastSenderReportTime time.Time } func newTestReceiverProvider() *testReceiverProvider { @@ -44,6 +45,14 @@ func (trp *testReceiverProvider) GetDeltaStats() map[uint32]*buffer.StreamStatsW return trp.streams } +func (trp *testReceiverProvider) setLastSenderReportTime(at time.Time) { + trp.lastSenderReportTime = at +} + +func (trp *testReceiverProvider) GetLastSenderReportTime() time.Time { + return trp.lastSenderReportTime +} + // ----------------------------------------------- func TestConnectionQuality(t *testing.T) { @@ -232,7 +241,7 @@ func TestConnectionQuality(t *testing.T) { require.Greater(t, float32(4.6), mos) require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality) - // unmute at time so that next window does not satisfy the unmute time threshold. + // unmute at specific time to ensure next window does not satisfy the unmute time threshold. // that means even if the next update has 0 packets, it should hold state and stay at EXCELLENT quality cs.UpdateMuteAt(false, now.Add(3*time.Second)) @@ -250,7 +259,8 @@ func TestConnectionQuality(t *testing.T) { require.Greater(t, float32(4.6), mos) require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality) - // next update with no packets should knock quality down to LOST + // next update with no packets, + // but last RTCP is not set, should knock quality down to POOR now = now.Add(duration) trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{ 1: { @@ -264,13 +274,46 @@ func TestConnectionQuality(t *testing.T) { cs.updateScoreAt(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(2.1), mos) + require.Equal(t, livekit.ConnectionQuality_POOR, quality) + + // another dry spell, but last RTCP is not stale, should keep quality at POOR + now = now.Add(duration) + trp.setLastSenderReportTime(now.Add(time.Second)) + trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{ + 1: { + RTPStats: &buffer.RTPDeltaInfo{ + StartTime: now, + EndTime: now.Add(duration), + Packets: 0, + }, + }, + }) + cs.updateScoreAt(now.Add(duration)) + mos, quality = cs.GetScoreAndQuality() + require.Greater(t, float32(2.1), mos) + require.Equal(t, livekit.ConnectionQuality_POOR, quality) + + // yet another dry spell, but last RTCP is stale, should knock down quality at LOST + now = now.Add(duration) + trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{ + 1: { + RTPStats: &buffer.RTPDeltaInfo{ + StartTime: now, + EndTime: now.Add(duration), + Packets: 0, + }, + }, + }) + cs.updateScoreAt(now.Add(duration)) + mos, quality = cs.GetScoreAndQuality() + require.Greater(t, float32(1.3), mos) require.Equal(t, livekit.ConnectionQuality_LOST, quality) // mute when LOST should not bump up score/quality now = now.Add(duration) cs.UpdateMuteAt(true, now.Add(1*time.Second)) mos, quality = cs.GetScoreAndQuality() - require.Greater(t, float32(2.1), mos) + require.Greater(t, float32(1.3), mos) require.Equal(t, livekit.ConnectionQuality_LOST, quality) // unmute and send packets to bring quality back up diff --git a/pkg/sfu/connectionquality/scorer.go b/pkg/sfu/connectionquality/scorer.go index 61019d013..facef770a 100644 --- a/pkg/sfu/connectionquality/scorer.go +++ b/pkg/sfu/connectionquality/scorer.go @@ -61,6 +61,7 @@ type windowStat struct { bytes uint64 rttMax uint32 jitterMax float64 + lastRTCPAt time.Time } func (w *windowStat) calculatePacketScore(plw float64, includeRTT bool, includeJitter bool) float64 { @@ -147,7 +148,7 @@ func (w *windowStat) calculateBitrateScore(expectedBitrate int64, isEnabled bool } func (w *windowStat) String() string { - return fmt.Sprintf("start: %+v, dur: %+v, pe: %d, pl: %d, pm: %d, pooo: %d, b: %d, rtt: %d, jitter: %0.2f", + return fmt.Sprintf("start: %+v, dur: %+v, pe: %d, pl: %d, pm: %d, pooo: %d, b: %d, rtt: %d, jitter: %0.2f, lastRTCP: %+v", w.startedAt, w.duration, w.packetsExpected, @@ -157,6 +158,7 @@ func (w *windowStat) String() string { w.bytes, w.rttMax, w.jitterMax, + w.lastRTCPAt, ) } @@ -174,6 +176,7 @@ func (w *windowStat) MarshalLogObject(e zapcore.ObjectEncoder) error { e.AddUint64("bytes", w.bytes) e.AddUint32("rttMax", w.rttMax) e.AddFloat64("jitterMax", w.jitterMax) + e.AddTime("lastRTCPAt", w.lastRTCPAt) return nil } @@ -393,8 +396,13 @@ func (q *qualityScorer) updateAtLocked(stat *windowStat, at time.Time) { reason := "none" var score float64 if stat.packetsExpected == 0 { - reason = "dry" - score = qualityTransitionScore[livekit.ConnectionQuality_LOST] + if !stat.lastRTCPAt.IsZero() && at.Sub(stat.lastRTCPAt) > stat.duration { + reason = "dry" + score = qualityTransitionScore[livekit.ConnectionQuality_LOST] + } else { + reason = "rtcp" + score = qualityTransitionScore[livekit.ConnectionQuality_POOR] + } } else { packetScore := stat.calculatePacketScore(plw, q.params.IncludeRTT, q.params.IncludeJitter) bitrateScore := stat.calculateBitrateScore(expectedBitrate, q.params.EnableBitrateScore) diff --git a/pkg/sfu/receiver.go b/pkg/sfu/receiver.go index 78d49fb2b..0cb210273 100644 --- a/pkg/sfu/receiver.go +++ b/pkg/sfu/receiver.go @@ -647,6 +647,25 @@ func (w *WebRTCReceiver) GetDeltaStats() map[uint32]*buffer.StreamStatsWithLayer return deltaStats } +func (w *WebRTCReceiver) GetLastSenderReportTime() time.Time { + w.bufferMu.RLock() + defer w.bufferMu.RUnlock() + + latestSRTime := time.Time{} + for _, buff := range w.buffers { + if buff == nil { + continue + } + + srAt := buff.GetLastSenderReportTime() + if srAt.After(latestSRTime) { + latestSRTime = srAt + } + } + + return latestSRTime +} + func (w *WebRTCReceiver) forwardRTP(layer int32) { pktBuf := make([]byte, bucket.MaxPktSize) tracker := w.streamTrackerManager.GetTracker(layer) From 47801f8b7910d70aeb6585445485ea4dc11330af Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 21 Apr 2024 11:05:29 -0700 Subject: [PATCH 66/78] Update livekit deps (#2415) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 7a6787451..5697900b2 100644 --- a/go.mod +++ b/go.mod @@ -18,9 +18,9 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 - github.com/livekit/mediatransportutil v0.0.0-20240406063423-a67d961689df - github.com/livekit/protocol v1.12.1-0.20240416154343-2d633a51d825 - github.com/livekit/psrpc v0.5.3-0.20240327035954-cec3a0e614be + github.com/livekit/mediatransportutil v0.0.0-20240416023643-881d3dc5423e + github.com/livekit/protocol v1.12.1-0.20240420195247-e418881086ea + github.com/livekit/psrpc v0.5.3-0.20240403150641-811331b106d9 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1 diff --git a/go.sum b/go.sum index f5741fe46..415e8a155 100644 --- a/go.sum +++ b/go.sum @@ -118,12 +118,12 @@ github.com/lithammer/shortuuid/v4 v4.0.0 h1:QRbbVkfgNippHOS8PXDkti4NaWeyYfcBTHtw github.com/lithammer/shortuuid/v4 v4.0.0/go.mod h1:Zs8puNcrvf2rV9rTH51ZLLcj7ZXqQI3lv67aw4KiB1Y= github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkDaKb5iXdynYrzB84ErPPO4LbRASk58= github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= -github.com/livekit/mediatransportutil v0.0.0-20240406063423-a67d961689df h1:DVhJRlF6/CtiyxJVy3QsbS9bf7GyUMuRZONMwZxIWpY= -github.com/livekit/mediatransportutil v0.0.0-20240406063423-a67d961689df/go.mod h1:jwKUCmObuiEDH0iiuJHaGMXwRs3RjrB4G6qqgkr/5oE= -github.com/livekit/protocol v1.12.1-0.20240416154343-2d633a51d825 h1:HJWtEtuiRoKE+/cRjLY0UHKnB6iMOAp1/17CXG6e5W0= -github.com/livekit/protocol v1.12.1-0.20240416154343-2d633a51d825/go.mod h1:jB6PWwf4tdMAwy+jxexqaVWuQiiklAtO4F5zZzWkTII= -github.com/livekit/psrpc v0.5.3-0.20240327035954-cec3a0e614be h1:W1nCFZ19rYAORMBNX82NeVPHjADN0UyORr6refbUXpU= -github.com/livekit/psrpc v0.5.3-0.20240327035954-cec3a0e614be/go.mod h1:CQUBSPfYYAaevg1TNCc6/aYsa8DJH4jSRFdCeSZk5u0= +github.com/livekit/mediatransportutil v0.0.0-20240416023643-881d3dc5423e h1:ss4VwrouYiDpuNJ9BUTH+WsW+GDdJS70iZp8ii3/0Lc= +github.com/livekit/mediatransportutil v0.0.0-20240416023643-881d3dc5423e/go.mod h1:jwKUCmObuiEDH0iiuJHaGMXwRs3RjrB4G6qqgkr/5oE= +github.com/livekit/protocol v1.12.1-0.20240420195247-e418881086ea h1:eXYVST3KMoIedAAub79A78K1B4qAV6visG/1aoG1LCI= +github.com/livekit/protocol v1.12.1-0.20240420195247-e418881086ea/go.mod h1:jB6PWwf4tdMAwy+jxexqaVWuQiiklAtO4F5zZzWkTII= +github.com/livekit/psrpc v0.5.3-0.20240403150641-811331b106d9 h1:4CngtPIJ58WcQ1sUDGdxJDkTndQpN6M/T8jXvRAd7Oc= +github.com/livekit/psrpc v0.5.3-0.20240403150641-811331b106d9/go.mod h1:CQUBSPfYYAaevg1TNCc6/aYsa8DJH4jSRFdCeSZk5u0= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= github.com/mackerelio/go-osstat v0.2.4/go.mod h1:Zy+qzGdZs3A9cuIqmgbJvwbmLQH9dJvtio5ZjJTbdlQ= github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= From 2ad0efc28f003316a6fc356a8f35b43ea86cf84f Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 22 Apr 2024 23:04:56 +0530 Subject: [PATCH 67/78] Handle large jumps in RTCP sender report timestamp. (#2674) * Handle large jumps in RTCP sender report timestamp. Seeing cases of RTCP Sender Report spaced apart by more than half the RTP Timestamp range. Maybe a case of laptop going to sleep and waking up. Handle it using time diff from last report and calculating expected timestamp. * try go 1.22 --- .github/workflows/buildtest.yaml | 6 ++--- pkg/sfu/buffer/rtpstats_receiver.go | 41 ++++++++++++++++++++++++----- 2 files changed, 37 insertions(+), 10 deletions(-) diff --git a/.github/workflows/buildtest.yaml b/.github/workflows/buildtest.yaml index 34a13ab6c..a9c8a5aa7 100644 --- a/.github/workflows/buildtest.yaml +++ b/.github/workflows/buildtest.yaml @@ -17,9 +17,9 @@ name: Test on: workflow_dispatch: push: - branches: [ master ] + branches: [master] pull_request: - branches: [ master ] + branches: [master] jobs: test: @@ -35,7 +35,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: "1.21" + go-version: "1.22" - name: Set up gotestfmt run: go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@v2.4.1 diff --git a/pkg/sfu/buffer/rtpstats_receiver.go b/pkg/sfu/buffer/rtpstats_receiver.go index 6fa9969af..bd1b16341 100644 --- a/pkg/sfu/buffer/rtpstats_receiver.go +++ b/pkg/sfu/buffer/rtpstats_receiver.go @@ -56,6 +56,9 @@ const ( cPropagationDelayDeltaHighResetNumReports = 2 cPropagationDelayDeltaHighResetWait = 10 * time.Second cPropagationDelayDeltaLongTermAdaptationThreshold = 50 * time.Millisecond + + // number of seconds the current report RTP timestamp can be off from expected RTP timestamp + cReportSlack = float64(60.0) ) type RTPFlowState struct { @@ -295,14 +298,38 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *RTCPSenderReportData) tsCycles := uint64(0) if r.srNewest != nil { - tsCycles = r.srNewest.RTPTimestampExt & 0xFFFF_FFFF_0000_0000 - if (srData.RTPTimestamp-r.srNewest.RTPTimestamp) < (1<<31) && srData.RTPTimestamp < r.srNewest.RTPTimestamp { - tsCycles += (1 << 32) - } + // use time since last sender report to ensure long gaps where the time stamp might + // jump more than half the range + timeSinceLastReport := srData.NTPTimestamp.Time().Sub(r.srNewest.NTPTimestamp.Time()) + expectedRTPTimestampExt := r.srNewest.RTPTimestampExt + uint64(timeSinceLastReport.Nanoseconds()*int64(r.params.ClockRate)/1e9) + lbound := expectedRTPTimestampExt - uint64(cReportSlack*float64(r.params.ClockRate)) + ubound := expectedRTPTimestampExt + uint64(cReportSlack*float64(r.params.ClockRate)) + isInRange := (srData.RTPTimestamp-uint32(lbound) < (1 << 31)) && (uint32(ubound)-srData.RTPTimestamp < (1 << 31)) + if isInRange { + lbTSCycles := lbound & 0xFFFF_FFFF_0000_0000 + ubTSCycles := ubound & 0xFFFF_FFFF_0000_0000 + if lbTSCycles == ubTSCycles { + tsCycles = lbTSCycles + } else { + if srData.RTPTimestamp < (1 << 31) { + // rolled over + tsCycles = ubTSCycles + } else { + tsCycles = lbTSCycles + } + } + } else { + // ideally this method should not be required, but there are clients + // negotiating one clock rate, but actually send media at a different rate. + tsCycles = r.srNewest.RTPTimestampExt & 0xFFFF_FFFF_0000_0000 + if (srData.RTPTimestamp-r.srNewest.RTPTimestamp) < (1<<31) && srData.RTPTimestamp < r.srNewest.RTPTimestamp { + tsCycles += (1 << 32) + } - if tsCycles >= (1 << 32) { - if (srData.RTPTimestamp-r.srNewest.RTPTimestamp) >= (1<<31) && srData.RTPTimestamp > r.srNewest.RTPTimestamp { - tsCycles -= (1 << 32) + if tsCycles >= (1 << 32) { + if (srData.RTPTimestamp-r.srNewest.RTPTimestamp) >= (1<<31) && srData.RTPTimestamp > r.srNewest.RTPTimestamp { + tsCycles -= (1 << 32) + } } } } From 8117f92465ae2afdd7249819eb48311c42361a58 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Apr 2024 11:01:57 -0700 Subject: [PATCH 68/78] Bump golang.org/x/net from 0.22.0 to 0.23.0 (#2673) * Bump golang.org/x/net from 0.22.0 to 0.23.0 Bumps [golang.org/x/net](https://github.com/golang/net) from 0.22.0 to 0.23.0. - [Commits](https://github.com/golang/net/compare/v0.22.0...v0.23.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: indirect ... Signed-off-by: dependabot[bot] * go 1.22 --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Mathew Kamkar <578302+matkam@users.noreply.github.com> --- .github/workflows/buildtest.yaml | 2 +- go.mod | 4 ++-- go.sum | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/buildtest.yaml b/.github/workflows/buildtest.yaml index a9c8a5aa7..a012f3f68 100644 --- a/.github/workflows/buildtest.yaml +++ b/.github/workflows/buildtest.yaml @@ -35,7 +35,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: "1.22" + go-version-file: "go.mod" - name: Set up gotestfmt run: go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@v2.4.1 diff --git a/go.mod b/go.mod index 5697900b2..ce5f488e1 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/livekit/livekit-server -go 1.21 +go 1.22 require ( github.com/avast/retry-go/v4 v4.5.1 @@ -101,7 +101,7 @@ require ( go.uber.org/zap/exp v0.2.0 // indirect golang.org/x/crypto v0.21.0 // indirect golang.org/x/mod v0.16.0 // indirect - golang.org/x/net v0.22.0 // indirect + golang.org/x/net v0.23.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.19.0 // indirect diff --git a/go.sum b/go.sum index 415e8a155..f571264f5 100644 --- a/go.sum +++ b/go.sum @@ -319,8 +319,8 @@ golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= -golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= -golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From c4354575eca918048808b4a8503ffc1cc8afcd33 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Mon, 22 Apr 2024 11:33:28 -0700 Subject: [PATCH 69/78] do not capture pointers in ops queue closures (#2675) --- pkg/rtc/participant.go | 21 ++++--- pkg/rtc/transport.go | 50 ++++++++-------- pkg/sfu/streamallocator/streamallocator.go | 66 +++++++++++----------- pkg/utils/opsqueue.go | 5 +- 4 files changed, 72 insertions(+), 70 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 0e889078d..cb8bbb207 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -75,6 +75,11 @@ type downTrackState struct { downTrack sfu.DownTrackState } +type postRtcpOp struct { + *ParticipantImpl + pkts []rtcp.Packet +} + // --------------------------------------------------------------- type participantUpdateInfo struct { @@ -164,7 +169,7 @@ type ParticipantImpl struct { disconnectTimer *time.Timer migrationTimer *time.Timer - pubRTCPQueue *sutils.TypedOpsQueue[[]rtcp.Packet] + pubRTCPQueue *sutils.TypedOpsQueue[postRtcpOp] // hold reference for MediaTrack twcc *twcc.Responder @@ -244,7 +249,7 @@ func NewParticipant(params ParticipantParams) (*ParticipantImpl, error) { p := &ParticipantImpl{ params: params, disconnected: make(chan struct{}), - pubRTCPQueue: sutils.NewTypedOpsQueue[[]rtcp.Packet](sutils.OpsQueueParams{ + pubRTCPQueue: sutils.NewTypedOpsQueue[postRtcpOp](sutils.OpsQueueParams{ Name: "pub-rtcp", MinSize: 64, Logger: params.Logger, @@ -2379,13 +2384,11 @@ func (p *ParticipantImpl) postRtcp(pkts []rtcp.Packet) { return } - p.pubRTCPQueue.Enqueue(p.writePublisherRTCP, pkts) -} - -func (p *ParticipantImpl) writePublisherRTCP(pkts []rtcp.Packet) { - if err := p.TransportManager.WritePublisherRTCP(pkts); err != nil && !IsEOF(err) { - p.pubLogger.Errorw("could not write RTCP to participant", err) - } + p.pubRTCPQueue.Enqueue(func(op postRtcpOp) { + if err := op.TransportManager.WritePublisherRTCP(op.pkts); err != nil && !IsEOF(err) { + op.pubLogger.Errorw("could not write RTCP to participant", err) + } + }, postRtcpOp{p, pkts}) } func (p *ParticipantImpl) setDowntracksConnected() { diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index 6a975ac28..53e650e94 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -123,6 +123,7 @@ func (s signal) String() string { // ------------------------------------------------------- type event struct { + *PCTransport signal signal data interface{} } @@ -1254,32 +1255,31 @@ func (t *PCTransport) parseTrackMid(offer webrtc.SessionDescription, senders map return nil } -func (t *PCTransport) postEvent(event event) { - t.eventsQueue.Enqueue(t.handleEvent, event) -} - -func (t *PCTransport) handleEvent(e event) { - var err error - switch e.signal { - case signalICEGatheringComplete: - err = t.handleICEGatheringComplete(e) - case signalLocalICECandidate: - err = t.handleLocalICECandidate(e) - case signalRemoteICECandidate: - err = t.handleRemoteICECandidate(e) - case signalSendOffer: - err = t.handleSendOffer(e) - case signalRemoteDescriptionReceived: - err = t.handleRemoteDescriptionReceived(e) - case signalICERestart: - err = t.handleICERestart(e) - } - if err != nil { - if !t.isClosed.Load() { - t.params.Logger.Warnw("error handling event", err, "event", e.String()) - t.params.Handler.OnNegotiationFailed() +func (t *PCTransport) postEvent(e event) { + e.PCTransport = t + t.eventsQueue.Enqueue(func(e event) { + var err error + switch e.signal { + case signalICEGatheringComplete: + err = e.handleICEGatheringComplete(e) + case signalLocalICECandidate: + err = e.handleLocalICECandidate(e) + case signalRemoteICECandidate: + err = e.handleRemoteICECandidate(e) + case signalSendOffer: + err = e.handleSendOffer(e) + case signalRemoteDescriptionReceived: + err = e.handleRemoteDescriptionReceived(e) + case signalICERestart: + err = e.handleICERestart(e) } - } + if err != nil { + if !e.isClosed.Load() { + e.params.Logger.Warnw("error handling event", err, "event", e.String()) + e.params.Handler.OnNegotiationFailed() + } + } + }, e) } func (t *PCTransport) handleICEGatheringComplete(_ event) error { diff --git a/pkg/sfu/streamallocator/streamallocator.go b/pkg/sfu/streamallocator/streamallocator.go index acb3ed477..e68d51717 100644 --- a/pkg/sfu/streamallocator/streamallocator.go +++ b/pkg/sfu/streamallocator/streamallocator.go @@ -125,6 +125,7 @@ func (s streamAllocatorSignal) String() string { // --------------------------------------------------------------------------- type Event struct { + *StreamAllocator Signal streamAllocatorSignal TrackID livekit.TrackID Data interface{} @@ -558,10 +559,6 @@ func (s *StreamAllocator) maybePostEventAllocateTrack(downTrack *sfu.DownTrack) } } -func (s *StreamAllocator) postEvent(event Event) { - s.eventsQueue.Enqueue(s.handleEvent, event) -} - func (s *StreamAllocator) ping() { ticker := time.NewTicker(100 * time.Millisecond) defer ticker.Stop() @@ -578,35 +575,38 @@ func (s *StreamAllocator) ping() { } } -func (s *StreamAllocator) handleEvent(event Event) { - switch event.Signal { - case streamAllocatorSignalAllocateTrack: - s.handleSignalAllocateTrack(event) - case streamAllocatorSignalAllocateAllTracks: - s.handleSignalAllocateAllTracks(event) - case streamAllocatorSignalAdjustState: - s.handleSignalAdjustState(event) - case streamAllocatorSignalEstimate: - s.handleSignalEstimate(event) - case streamAllocatorSignalPeriodicPing: - s.handleSignalPeriodicPing(event) - case streamAllocatorSignalSendProbe: - s.handleSignalSendProbe(event) - case streamAllocatorSignalProbeClusterDone: - s.handleSignalProbeClusterDone(event) - case streamAllocatorSignalResume: - s.handleSignalResume(event) - case streamAllocatorSignalSetAllowPause: - s.handleSignalSetAllowPause(event) - case streamAllocatorSignalSetChannelCapacity: - s.handleSignalSetChannelCapacity(event) - /* STREAM-ALLOCATOR-DATA - case streamAllocatorSignalNACK: - s.handleSignalNACK(event) - case streamAllocatorSignalRTCPReceiverReport: - s.handleSignalRTCPReceiverReport(event) - */ - } +func (s *StreamAllocator) postEvent(event Event) { + event.StreamAllocator = s + s.eventsQueue.Enqueue(func(event Event) { + switch event.Signal { + case streamAllocatorSignalAllocateTrack: + event.handleSignalAllocateTrack(event) + case streamAllocatorSignalAllocateAllTracks: + event.handleSignalAllocateAllTracks(event) + case streamAllocatorSignalAdjustState: + event.handleSignalAdjustState(event) + case streamAllocatorSignalEstimate: + event.handleSignalEstimate(event) + case streamAllocatorSignalPeriodicPing: + event.handleSignalPeriodicPing(event) + case streamAllocatorSignalSendProbe: + event.handleSignalSendProbe(event) + case streamAllocatorSignalProbeClusterDone: + event.handleSignalProbeClusterDone(event) + case streamAllocatorSignalResume: + event.handleSignalResume(event) + case streamAllocatorSignalSetAllowPause: + event.handleSignalSetAllowPause(event) + case streamAllocatorSignalSetChannelCapacity: + event.handleSignalSetChannelCapacity(event) + /* STREAM-ALLOCATOR-DATA + case streamAllocatorSignalNACK: + event.s.handleSignalNACK(event) + case streamAllocatorSignalRTCPReceiverReport: + event.s.handleSignalRTCPReceiverReport(event) + */ + } + }, event) } func (s *StreamAllocator) handleSignalAllocateTrack(event Event) { diff --git a/pkg/utils/opsqueue.go b/pkg/utils/opsqueue.go index 30f9f4b39..c4e00e4ec 100644 --- a/pkg/utils/opsqueue.go +++ b/pkg/utils/opsqueue.go @@ -81,13 +81,12 @@ type opsQueueBase[T opsQueueItem] struct { } func newOpsQueueBase[T opsQueueItem](params OpsQueueParams) *opsQueueBase[T] { - oq := &opsQueueBase[T]{ + return &opsQueueBase[T]{ params: params, + ops: *deque.New[T](min(bits.Len64(uint64(params.MinSize-1)), 7)), wake: make(chan struct{}, 1), doneChan: make(chan struct{}), } - oq.ops.SetMinCapacity(uint(min(bits.Len64(uint64(oq.params.MinSize-1)), 7))) - return oq } func (oq *opsQueueBase[T]) Start() { From 8f053851260ffd4e6b536713a126b8c547174e1a Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 23 Apr 2024 10:49:55 +0530 Subject: [PATCH 70/78] TTL param for ICE config cache (#2676) * TTL param for ICE config cache * rename to min --- pkg/service/roommanager.go | 2 +- pkg/utils/ice_config_cache.go | 17 ++++++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/pkg/service/roommanager.go b/pkg/service/roommanager.go index 409a6e34b..3aafad3fd 100644 --- a/pkg/service/roommanager.go +++ b/pkg/service/roommanager.go @@ -119,7 +119,7 @@ func NewLocalRoomManager( rooms: make(map[livekit.RoomName]*rtc.Room), - iceConfigCache: sutils.NewIceConfigCache[iceConfigCacheKey](), + iceConfigCache: sutils.NewIceConfigCache[iceConfigCacheKey](0), serverInfo: &livekit.ServerInfo{ Edition: livekit.ServerInfo_Standard, diff --git a/pkg/utils/ice_config_cache.go b/pkg/utils/ice_config_cache.go index 2fb4f49e7..a3ff2a22a 100644 --- a/pkg/utils/ice_config_cache.go +++ b/pkg/utils/ice_config_cache.go @@ -9,7 +9,7 @@ import ( ) const ( - iceConfigTTL = 5 * time.Minute + iceConfigTTLMin = 5 * time.Minute ) type iceConfigCacheEntry struct { @@ -19,16 +19,23 @@ type iceConfigCacheEntry struct { type IceConfigCache[T comparable] struct { lock sync.Mutex + ttl time.Duration entries map[T]*iceConfigCacheEntry stopped atomic.Bool } -func NewIceConfigCache[T comparable]() *IceConfigCache[T] { +func NewIceConfigCache[T comparable](ttl time.Duration) *IceConfigCache[T] { icc := &IceConfigCache[T]{ entries: make(map[T]*iceConfigCacheEntry), } + if ttl < iceConfigTTLMin { + icc.ttl = iceConfigTTLMin + } else { + icc.ttl = ttl + } + go icc.pruneWorker() return icc } @@ -52,7 +59,7 @@ func (icc *IceConfigCache[T]) Get(key T) *livekit.ICEConfig { defer icc.lock.Unlock() entry, ok := icc.entries[key] - if !ok || time.Since(entry.modifiedAt) > iceConfigTTL { + if !ok || time.Since(entry.modifiedAt) > icc.ttl { delete(icc.entries, key) return &livekit.ICEConfig{} } @@ -61,7 +68,7 @@ func (icc *IceConfigCache[T]) Get(key T) *livekit.ICEConfig { } func (icc *IceConfigCache[T]) pruneWorker() { - ticker := time.NewTicker(iceConfigTTL / 2) + ticker := time.NewTicker(icc.ttl / 2) defer ticker.Stop() for !icc.stopped.Load() { @@ -69,7 +76,7 @@ func (icc *IceConfigCache[T]) pruneWorker() { icc.lock.Lock() for key, entry := range icc.entries { - if time.Since(entry.modifiedAt) > iceConfigTTL { + if time.Since(entry.modifiedAt) > icc.ttl { delete(icc.entries, key) } } From f239f8bff1b45c1f8a9dae3293d8626d454ad77e Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Tue, 23 Apr 2024 16:09:02 +0800 Subject: [PATCH 71/78] Fix SubParticipant twice when paticipant left (#2672) --- pkg/telemetry/events.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/pkg/telemetry/events.go b/pkg/telemetry/events.go index 1d0a9c546..1a5d22853 100644 --- a/pkg/telemetry/events.go +++ b/pkg/telemetry/events.go @@ -161,18 +161,14 @@ func (t *telemetryService) ParticipantLeft(ctx context.Context, ) { t.enqueue(func() { isConnected := false - hasWorker := false if worker, ok := t.getWorker(livekit.ParticipantID(participant.Sid)); ok { - hasWorker = true isConnected = worker.IsConnected() + if worker.ClosedAt().IsZero() { + prometheus.SubParticipant() + } worker.Close() } - if hasWorker { - // signifies we had incremented participant count - prometheus.SubParticipant() - } - if isConnected && shouldSendEvent { t.NotifyEvent(ctx, &livekit.WebhookEvent{ Event: webhook.EventParticipantLeft, From c51b3e3fe25f9fd03ed3ac4d07778f7cbecd1e8a Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Tue, 23 Apr 2024 02:20:54 -0700 Subject: [PATCH 72/78] use ttlcache (#2677) * use ttlcache * go * test --- go.mod | 1 + go.sum | 2 + pkg/service/roommanager.go | 2 +- pkg/utils/ice_config_cache.go | 85 -------------------------------- pkg/utils/iceconfigcache.go | 42 ++++++++++++++++ pkg/utils/iceconfigcache_test.go | 18 +++++++ 6 files changed, 64 insertions(+), 86 deletions(-) delete mode 100644 pkg/utils/ice_config_cache.go create mode 100644 pkg/utils/iceconfigcache.go create mode 100644 pkg/utils/iceconfigcache_test.go diff --git a/go.mod b/go.mod index ce5f488e1..7b117582b 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/hashicorp/go-version v1.6.0 github.com/hashicorp/golang-lru/v2 v2.0.7 + github.com/jellydator/ttlcache/v3 v3.2.0 github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20240416023643-881d3dc5423e diff --git a/go.sum b/go.sum index f571264f5..d5c92f6fe 100644 --- a/go.sum +++ b/go.sum @@ -85,6 +85,8 @@ github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+l github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/jellydator/ttlcache/v3 v3.2.0 h1:6lqVJ8X3ZaUwvzENqPAobDsXNExfUJd61u++uW8a3LE= +github.com/jellydator/ttlcache/v3 v3.2.0/go.mod h1:hi7MGFdMAwZna5n2tuvh63DvFLzVKySzCVW6+0gA2n4= github.com/josharian/native v0.0.0-20200817173448-b6b71def0850/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= github.com/josharian/native v1.0.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA= diff --git a/pkg/service/roommanager.go b/pkg/service/roommanager.go index 3aafad3fd..3dfa146d5 100644 --- a/pkg/service/roommanager.go +++ b/pkg/service/roommanager.go @@ -504,7 +504,7 @@ func (r *RoomManager) StartSession( } }) participant.OnICEConfigChanged(func(participant types.LocalParticipant, iceConfig *livekit.ICEConfig) { - r.iceConfigCache.Put(iceConfigCacheKey{roomName, participant.Identity()}, iceConfig, time.Now()) + r.iceConfigCache.Put(iceConfigCacheKey{roomName, participant.Identity()}, iceConfig) }) go r.rtcSessionWorker(room, participant, requestSource) diff --git a/pkg/utils/ice_config_cache.go b/pkg/utils/ice_config_cache.go deleted file mode 100644 index a3ff2a22a..000000000 --- a/pkg/utils/ice_config_cache.go +++ /dev/null @@ -1,85 +0,0 @@ -package utils - -import ( - "sync" - "time" - - "github.com/livekit/protocol/livekit" - "go.uber.org/atomic" -) - -const ( - iceConfigTTLMin = 5 * time.Minute -) - -type iceConfigCacheEntry struct { - iceConfig *livekit.ICEConfig - modifiedAt time.Time -} - -type IceConfigCache[T comparable] struct { - lock sync.Mutex - ttl time.Duration - entries map[T]*iceConfigCacheEntry - - stopped atomic.Bool -} - -func NewIceConfigCache[T comparable](ttl time.Duration) *IceConfigCache[T] { - icc := &IceConfigCache[T]{ - entries: make(map[T]*iceConfigCacheEntry), - } - - if ttl < iceConfigTTLMin { - icc.ttl = iceConfigTTLMin - } else { - icc.ttl = ttl - } - - go icc.pruneWorker() - return icc -} - -func (icc *IceConfigCache[T]) Stop() { - icc.stopped.Store(true) -} - -func (icc *IceConfigCache[T]) Put(key T, iceConfig *livekit.ICEConfig, at time.Time) { - icc.lock.Lock() - defer icc.lock.Unlock() - - icc.entries[key] = &iceConfigCacheEntry{ - iceConfig: iceConfig, - modifiedAt: at, - } -} - -func (icc *IceConfigCache[T]) Get(key T) *livekit.ICEConfig { - icc.lock.Lock() - defer icc.lock.Unlock() - - entry, ok := icc.entries[key] - if !ok || time.Since(entry.modifiedAt) > icc.ttl { - delete(icc.entries, key) - return &livekit.ICEConfig{} - } - - return entry.iceConfig -} - -func (icc *IceConfigCache[T]) pruneWorker() { - ticker := time.NewTicker(icc.ttl / 2) - defer ticker.Stop() - - for !icc.stopped.Load() { - <-ticker.C - - icc.lock.Lock() - for key, entry := range icc.entries { - if time.Since(entry.modifiedAt) > icc.ttl { - delete(icc.entries, key) - } - } - icc.lock.Unlock() - } -} diff --git a/pkg/utils/iceconfigcache.go b/pkg/utils/iceconfigcache.go new file mode 100644 index 000000000..e42ed2bb6 --- /dev/null +++ b/pkg/utils/iceconfigcache.go @@ -0,0 +1,42 @@ +package utils + +import ( + "time" + + "github.com/jellydator/ttlcache/v3" + + "github.com/livekit/protocol/livekit" +) + +const ( + iceConfigTTLMin = 5 * time.Minute +) + +type IceConfigCache[T comparable] struct { + c *ttlcache.Cache[T, *livekit.ICEConfig] +} + +func NewIceConfigCache[T comparable](ttl time.Duration) *IceConfigCache[T] { + cache := ttlcache.New( + ttlcache.WithTTL[T, *livekit.ICEConfig](max(ttl, iceConfigTTLMin)), + ttlcache.WithDisableTouchOnHit[T, *livekit.ICEConfig](), + ) + go cache.Start() + + return &IceConfigCache[T]{cache} +} + +func (icc *IceConfigCache[T]) Stop() { + icc.c.Stop() +} + +func (icc *IceConfigCache[T]) Put(key T, iceConfig *livekit.ICEConfig) { + icc.c.Set(key, iceConfig, ttlcache.DefaultTTL) +} + +func (icc *IceConfigCache[T]) Get(key T) *livekit.ICEConfig { + if it := icc.c.Get(key); it != nil { + return it.Value() + } + return &livekit.ICEConfig{} +} diff --git a/pkg/utils/iceconfigcache_test.go b/pkg/utils/iceconfigcache_test.go new file mode 100644 index 000000000..78feebb0e --- /dev/null +++ b/pkg/utils/iceconfigcache_test.go @@ -0,0 +1,18 @@ +package utils + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/livekit/protocol/livekit" +) + +func TestIceConfigCache(t *testing.T) { + cache := NewIceConfigCache[string](10 * time.Second) + t.Cleanup(cache.Stop) + + cache.Put("test", &livekit.ICEConfig{}) + require.NotNil(t, cache) +} From dfadb3f0870fc27caf59e1dc3de21826716a0ecc Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 23 Apr 2024 22:48:47 -0700 Subject: [PATCH 73/78] Update pion deps (#2617) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 6 +++--- go.sum | 14 ++++++-------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 7b117582b..5b26ccaac 100644 --- a/go.mod +++ b/go.mod @@ -28,10 +28,10 @@ require ( github.com/mitchellh/go-homedir v1.1.0 github.com/olekukonko/tablewriter v0.0.5 github.com/pion/dtls/v2 v2.2.10 - github.com/pion/ice/v2 v2.3.18 - github.com/pion/interceptor v0.1.25 + github.com/pion/ice/v2 v2.3.19 + github.com/pion/interceptor v0.1.29 github.com/pion/rtcp v1.2.14 - github.com/pion/rtp v1.8.5 + github.com/pion/rtp v1.8.6 github.com/pion/sctp v1.8.16 github.com/pion/sdp/v3 v3.0.9 github.com/pion/transport/v2 v2.2.4 diff --git a/go.sum b/go.sum index d5c92f6fe..cb35e4bc6 100644 --- a/go.sum +++ b/go.sum @@ -170,24 +170,22 @@ github.com/pion/datachannel v1.5.5/go.mod h1:iMz+lECmfdCMqFRhXhcA/219B0SQlbpoR2V github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= github.com/pion/dtls/v2 v2.2.10 h1:u2Axk+FyIR1VFTPurktB+1zoEPGIW3bmyj3LEFrXjAA= github.com/pion/dtls/v2 v2.2.10/go.mod h1:d9SYc9fch0CqK90mRk1dC7AkzzpwJj6u2GU3u+9pqFE= -github.com/pion/ice/v2 v2.3.18 h1:8Q1WsWhWlyHyb5TbLE9bv5AQviaiq0Airetgae9mAH0= -github.com/pion/ice/v2 v2.3.18/go.mod h1:KXJJcZK7E8WzrBEYnV4UtqEZsGeWfHxsNqhVcVvgjxw= -github.com/pion/interceptor v0.1.25 h1:pwY9r7P6ToQ3+IF0bajN0xmk/fNw/suTgaTdlwTDmhc= -github.com/pion/interceptor v0.1.25/go.mod h1:wkbPYAak5zKsfpVDYMtEfWEy8D4zL+rpxCxPImLOg3Y= +github.com/pion/ice/v2 v2.3.19 h1:1GoMRTMnB6bCP4aGy2MjxK3w4laDkk+m7svJb/eqybc= +github.com/pion/ice/v2 v2.3.19/go.mod h1:KXJJcZK7E8WzrBEYnV4UtqEZsGeWfHxsNqhVcVvgjxw= +github.com/pion/interceptor v0.1.29 h1:39fsnlP1U8gw2JzOFWdfCU82vHvhW9o0rZnZF56wF+M= +github.com/pion/interceptor v0.1.29/go.mod h1:ri+LGNjRUc5xUNtDEPzfdkmSqISixVTBF/z/Zms/6T4= github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= github.com/pion/mdns v0.0.12 h1:CiMYlY+O0azojWDmxdNr7ADGrnZ+V6Ilfner+6mSVK8= github.com/pion/mdns v0.0.12/go.mod h1:VExJjv8to/6Wqm1FXK+Ii/Z9tsVk/F5sD/N70cnYFbk= github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= -github.com/pion/rtcp v1.2.10/go.mod h1:ztfEwXZNLGyF1oQDttz/ZKIBaeeg/oWbRYqzBM9TL1I= github.com/pion/rtcp v1.2.12/go.mod h1:sn6qjxvnwyAkkPzPULIbVqSKI5Dv54Rv7VG0kNxh9L4= github.com/pion/rtcp v1.2.14 h1:KCkGV3vJ+4DAJmvP0vaQShsb0xkRfWkO540Gy102KyE= github.com/pion/rtcp v1.2.14/go.mod h1:sn6qjxvnwyAkkPzPULIbVqSKI5Dv54Rv7VG0kNxh9L4= -github.com/pion/rtp v1.8.2/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= github.com/pion/rtp v1.8.3/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= -github.com/pion/rtp v1.8.5 h1:uYzINfaK+9yWs7r537z/Rc1SvT8ILjBcmDOpJcTB+OU= -github.com/pion/rtp v1.8.5/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= +github.com/pion/rtp v1.8.6 h1:MTmn/b0aWWsAzux2AmP8WGllusBVw4NPYPVFFd7jUPw= +github.com/pion/rtp v1.8.6/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= github.com/pion/sctp v1.8.5/go.mod h1:SUFFfDpViyKejTAdwD1d/HQsCu+V/40cCs2nZIvC3s0= github.com/pion/sctp v1.8.16 h1:PKrMs+o9EMLRvFfXq59WFsC+V8mN1wnKzqrv+3D/gYY= github.com/pion/sctp v1.8.16/go.mod h1:P6PbDVA++OJMrVNg2AL3XtYHV4uD6dvfyOovCgMs0PE= From b55038af03990217340cb242769ab9bec1a29725 Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Wed, 24 Apr 2024 16:26:07 +0800 Subject: [PATCH 74/78] Detach subscriber datachannel to save memory (#2680) Sfu don't read message from subscriber datachannel, detach it to bypass the readLoop can save 128KB memory and 2 goroutines per participant. --- pkg/rtc/transport.go | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index 53e650e94..9ceca8e72 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -53,7 +53,6 @@ import ( sfuutils "github.com/livekit/livekit-server/pkg/sfu/utils" "github.com/livekit/livekit-server/pkg/telemetry/prometheus" "github.com/livekit/livekit-server/pkg/utils" - sutils "github.com/livekit/livekit-server/pkg/utils" ) const ( @@ -75,8 +74,6 @@ const ( minConnectTimeoutAfterICE = 10 * time.Second maxConnectTimeoutAfterICE = 20 * time.Second // max duration for waiting pc to connect after ICE is connected - maxICECandidates = 20 - shortConnectionThreshold = 90 * time.Second ) @@ -185,7 +182,7 @@ type PCTransport struct { preferTCP atomic.Bool isClosed atomic.Bool - eventsQueue *sutils.TypedOpsQueue[event] + eventsQueue *utils.TypedOpsQueue[event] // the following should be accessed only in event processing go routine cacheLocalCandidates bool @@ -305,6 +302,7 @@ func newPeerConnection(params TransportParams, onBandwidthEstimator func(estimat ir := &interceptor.Registry{} if params.IsSendSide { + se.DetachDataChannels() if params.CongestionControlConfig.UseSendSideBWE { gf, err := cc.NewInterceptor(func() (cc.BandwidthEstimator, error) { return gcc.NewSendSideBWE( @@ -390,7 +388,7 @@ func NewPCTransport(params TransportParams) (*PCTransport, error) { params: params, debouncedNegotiate: debounce.New(negotiationFrequency), negotiationState: transport.NegotiationStateNone, - eventsQueue: sutils.NewTypedOpsQueue[event](utils.OpsQueueParams{ + eventsQueue: utils.NewTypedOpsQueue[event](utils.OpsQueueParams{ Name: "transport", MinSize: 64, Logger: params.Logger, @@ -402,7 +400,7 @@ func NewPCTransport(params TransportParams) (*PCTransport, error) { if params.IsSendSide { t.streamAllocator = streamallocator.NewStreamAllocator(streamallocator.StreamAllocatorParams{ Config: params.CongestionControlConfig, - Logger: params.Logger.WithComponent(sutils.ComponentCongestionControl), + Logger: params.Logger.WithComponent(utils.ComponentCongestionControl), }) t.streamAllocator.OnStreamStateChange(params.Handler.OnStreamStateChange) t.streamAllocator.Start() @@ -855,8 +853,22 @@ func (t *PCTransport) CreateDataChannel(label string, dci *webrtc.DataChannelIni defer t.lock.Unlock() *dcPtr = dc if t.params.DirectionConfig.StrictACKs { - dc.OnOpen(dcReadyHandler) + dc.OnOpen(func() { + if t.params.IsSendSide { + if _, err := dc.Detach(); err != nil { + t.params.Logger.Warnw("failed to detach data channel", err) + } + } + dcReadyHandler() + }) } else { + dc.OnOpen(func() { + if t.params.IsSendSide { + if _, err := dc.Detach(); err != nil { + t.params.Logger.Warnw("failed to detach data channel", err) + } + } + }) dc.OnDial(dcReadyHandler) } dc.OnClose(dcCloseHandler) From 90b47424b533eb7a5b6415bd6594c169daf644fe Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 26 Apr 2024 12:06:35 +0530 Subject: [PATCH 75/78] Handle UpdateLocalAudioTrack and UpdateLocalVideoTrack. (#2684) * Handle UpdateLocalAudioTrack and UpdateLocalVideoTrack. - Update the TrackInfo - NOTE: populating Stereo and DisableDtx fields although there are features now. - The audio features in UpdateLocalAudioTrack is applied as is, i. e. the update has the latest set of features. - Emits a track update which will broadcast a participant update. TODO: ----- - Telemetry event with track update? * update deps --- go.mod | 27 ++-- go.sum | 53 +++---- pkg/rtc/mediatrackreceiver.go | 37 ++++- pkg/rtc/participant.go | 6 + pkg/rtc/signalhandler.go | 21 +++ pkg/rtc/types/interfaces.go | 4 + .../typesfakes/fake_local_media_track.go | 78 +++++++++ .../typesfakes/fake_local_participant.go | 148 ++++++++++++++++++ pkg/rtc/types/typesfakes/fake_media_track.go | 78 +++++++++ pkg/rtc/types/typesfakes/fake_participant.go | 148 ++++++++++++++++++ pkg/rtc/uptrackmanager.go | 30 ++++ 11 files changed, 588 insertions(+), 42 deletions(-) diff --git a/go.mod b/go.mod index 5b26ccaac..1981bf306 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20240416023643-881d3dc5423e - github.com/livekit/protocol v1.12.1-0.20240420195247-e418881086ea + github.com/livekit/protocol v1.12.1-0.20240426044238-2d50a792225e github.com/livekit/psrpc v0.5.3-0.20240403150641-811331b106d9 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 @@ -49,8 +49,8 @@ require ( github.com/urfave/negroni/v3 v3.1.0 go.uber.org/atomic v1.11.0 go.uber.org/zap v1.27.0 - golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 - golang.org/x/sync v0.6.0 + golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f + golang.org/x/sync v0.7.0 google.golang.org/protobuf v1.33.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -58,7 +58,7 @@ require ( require ( github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect @@ -67,7 +67,6 @@ require ( github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-jose/go-jose/v3 v3.0.3 // indirect github.com/go-logr/logr v1.4.1 // indirect - github.com/golang/protobuf v1.5.4 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/subcommands v1.2.0 // indirect github.com/google/uuid v1.6.0 // indirect @@ -75,13 +74,13 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.5 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/josharian/native v1.1.0 // indirect - github.com/klauspost/compress v1.17.7 // indirect + github.com/klauspost/compress v1.17.8 // indirect github.com/klauspost/cpuid/v2 v2.2.6 // indirect github.com/lithammer/shortuuid/v4 v4.0.0 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect github.com/mdlayher/netlink v1.7.1 // indirect github.com/mdlayher/socket v0.4.0 // indirect - github.com/nats-io/nats.go v1.34.0 // indirect + github.com/nats-io/nats.go v1.34.1 // indirect github.com/nats-io/nkeys v0.4.7 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/pion/datachannel v1.5.5 // indirect @@ -100,13 +99,13 @@ require ( github.com/zeebo/xxh3 v1.0.2 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap/exp v0.2.0 // indirect - golang.org/x/crypto v0.21.0 // indirect - golang.org/x/mod v0.16.0 // indirect - golang.org/x/net v0.23.0 // indirect - golang.org/x/sys v0.18.0 // indirect + golang.org/x/crypto v0.22.0 // indirect + golang.org/x/mod v0.17.0 // indirect + golang.org/x/net v0.24.0 // indirect + golang.org/x/sys v0.19.0 // indirect golang.org/x/text v0.14.0 // indirect - golang.org/x/tools v0.19.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect - google.golang.org/grpc v1.62.1 // indirect + golang.org/x/tools v0.20.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be // indirect + google.golang.org/grpc v1.63.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index cb35e4bc6..c14b8d58b 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,8 @@ github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cilium/ebpf v0.5.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= github.com/cilium/ebpf v0.8.1 h1:bLSSEbBLqGPXxls55pGr5qWZaTqcmfDJHhou7t254ao= @@ -50,8 +50,6 @@ github.com/go-jose/go-jose/v3 v3.0.3 h1:fFKWeig/irsp7XD2zBxvnmA/XaRWp5V3CBsZXJF7 github.com/go-jose/go-jose/v3 v3.0.3/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -103,8 +101,8 @@ github.com/jsimonetti/rtnetlink v0.0.0-20211022192332-93da33804786 h1:N527AHMa79 github.com/jsimonetti/rtnetlink v0.0.0-20211022192332-93da33804786/go.mod h1:v4hqbTdfQngbVSZJVWUhGE/lbTFf9jb+ygmNUDQMuOs= github.com/jxskiss/base62 v1.1.0 h1:A5zbF8v8WXx2xixnAKD2w+abC+sIzYJX+nxmhA6HWFw= github.com/jxskiss/base62 v1.1.0/go.mod h1:HhWAlUXvxKThfOlZbcuFzsqwtF5TcqS9ru3y5GfjWAc= -github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= -github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= +github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -122,8 +120,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20240416023643-881d3dc5423e h1:ss4VwrouYiDpuNJ9BUTH+WsW+GDdJS70iZp8ii3/0Lc= github.com/livekit/mediatransportutil v0.0.0-20240416023643-881d3dc5423e/go.mod h1:jwKUCmObuiEDH0iiuJHaGMXwRs3RjrB4G6qqgkr/5oE= -github.com/livekit/protocol v1.12.1-0.20240420195247-e418881086ea h1:eXYVST3KMoIedAAub79A78K1B4qAV6visG/1aoG1LCI= -github.com/livekit/protocol v1.12.1-0.20240420195247-e418881086ea/go.mod h1:jB6PWwf4tdMAwy+jxexqaVWuQiiklAtO4F5zZzWkTII= +github.com/livekit/protocol v1.12.1-0.20240426044238-2d50a792225e h1:hepBKU/dTHNLLxq+Sc9Z8DdNLi3rjcQ6VlL3CEf9NmE= +github.com/livekit/protocol v1.12.1-0.20240426044238-2d50a792225e/go.mod h1:pnn0Dv+/0K0OFqKHX6J6SreYO1dZxl6tDuAZ1ns8L/w= github.com/livekit/psrpc v0.5.3-0.20240403150641-811331b106d9 h1:4CngtPIJ58WcQ1sUDGdxJDkTndQpN6M/T8jXvRAd7Oc= github.com/livekit/psrpc v0.5.3-0.20240403150641-811331b106d9/go.mod h1:CQUBSPfYYAaevg1TNCc6/aYsa8DJH4jSRFdCeSZk5u0= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= @@ -155,8 +153,8 @@ github.com/mdlayher/socket v0.4.0 h1:280wsy40IC9M9q1uPGcLBwXpcTQDtoGwVt+BNoITxIw github.com/mdlayher/socket v0.4.0/go.mod h1:xxFqz5GRCUN3UEOm9CZqEJsAbe1C8OwSK46NlmWuVoc= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/nats-io/nats.go v1.34.0 h1:fnxnPCNiwIG5w08rlMcEKTUw4AV/nKyGCOJE8TdhSPk= -github.com/nats-io/nats.go v1.34.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8= +github.com/nats-io/nats.go v1.34.1 h1:syWey5xaNHZgicYBemv0nohUPPmaLteiBEUT6Q5+F/4= +github.com/nats-io/nats.go v1.34.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8= github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI= github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= @@ -285,16 +283,16 @@ golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98y golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= -golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 h1:aAcj0Da7eBAtrTp03QXWvm88pSyOt+UgdZw2BFZ+lEw= -golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8/go.mod h1:CQ1k9gNrJ50XIzaKCRR2hssIjF07kZFEiieALBM/ARQ= +golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY= +golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= -golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -319,15 +317,16 @@ golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -365,8 +364,8 @@ golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -396,15 +395,15 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= -golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= -golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= +golang.org/x/tools v0.20.0 h1:hz/CVckiOxybQvFw6h7b/q80NTr9IUQb4s1IIzW7KNY= +golang.org/x/tools v0.20.0/go.mod h1:WvitBU7JJf6A4jOdg4S1tviW9bhUxkgeCui/0JHctQg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda h1:LI5DOvAxUPMv/50agcLLoo+AdWc1irS9Rzz4vPuD1V4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= -google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= -google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be h1:LG9vZxsWGOmUKieR8wPAUR3u3MpnYFQZROPIMaXh7/A= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= +google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pkg/rtc/mediatrackreceiver.go b/pkg/rtc/mediatrackreceiver.go index 1644ca4ca..008f19f27 100644 --- a/pkg/rtc/mediatrackreceiver.go +++ b/pkg/rtc/mediatrackreceiver.go @@ -649,7 +649,7 @@ func (t *MediaTrackReceiver) UpdateTrackInfo(ti *livekit.TrackInfo) { break } - // for client don't use simulcast codecs (old client version or single codec) + // for clients that don't use simulcast codecs (old client version or single codec) if i == 0 { clonedInfo.Layers = ci.Layers } @@ -667,6 +667,41 @@ func (t *MediaTrackReceiver) UpdateTrackInfo(ti *livekit.TrackInfo) { t.updateTrackInfoOfReceivers() } +func (t *MediaTrackReceiver) UpdateAudioTrack(update *livekit.UpdateLocalAudioTrack) { + if t.Kind() != livekit.TrackType_AUDIO { + return + } + + t.lock.Lock() + t.trackInfo.AudioFeatures = update.Features + t.trackInfo.Stereo = false + t.trackInfo.DisableDtx = false + for _, feature := range update.Features { + switch feature { + case livekit.AudioTrackFeature_TF_STEREO: + t.trackInfo.Stereo = true + case livekit.AudioTrackFeature_TF_NO_DTX: + t.trackInfo.DisableDtx = true + } + } + t.lock.Unlock() + + t.updateTrackInfoOfReceivers() +} + +func (t *MediaTrackReceiver) UpdateVideoTrack(update *livekit.UpdateLocalVideoTrack) { + if t.Kind() != livekit.TrackType_VIDEO { + return + } + + t.lock.Lock() + t.trackInfo.Width = update.Width + t.trackInfo.Height = update.Height + t.lock.Unlock() + + t.updateTrackInfoOfReceivers() +} + func (t *MediaTrackReceiver) UpdateVideoLayers(layers []*livekit.VideoLayer) { t.lock.Lock() // set video layer ssrc info diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index cb8bbb207..bf6931806 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -1816,6 +1816,12 @@ func (p *ParticipantImpl) addPendingTrackLocked(req *livekit.AddTrackRequest) *l Encryption: req.Encryption, Stream: req.Stream, } + if req.Stereo { + ti.AudioFeatures = append(ti.AudioFeatures, livekit.AudioTrackFeature_TF_STEREO) + } + if req.DisableDtx { + ti.AudioFeatures = append(ti.AudioFeatures, livekit.AudioTrackFeature_TF_NO_DTX) + } if ti.Stream == "" { ti.Stream = StreamFromTrackSource(ti.Source) } diff --git a/pkg/rtc/signalhandler.go b/pkg/rtc/signalhandler.go index cab96e23e..d7d9c75ad 100644 --- a/pkg/rtc/signalhandler.go +++ b/pkg/rtc/signalhandler.go @@ -27,8 +27,10 @@ func HandleParticipantSignal(room types.Room, participant types.LocalParticipant switch msg := req.GetMessage().(type) { case *livekit.SignalRequest_Offer: participant.HandleOffer(FromProtoSessionDescription(msg.Offer)) + case *livekit.SignalRequest_Answer: participant.HandleAnswer(FromProtoSessionDescription(msg.Answer)) + case *livekit.SignalRequest_Trickle: candidateInit, err := FromProtoTrickle(msg.Trickle) if err != nil { @@ -36,11 +38,14 @@ func HandleParticipantSignal(room types.Room, participant types.LocalParticipant return nil } participant.AddICECandidate(candidateInit, msg.Trickle.Target) + case *livekit.SignalRequest_AddTrack: pLogger.Debugw("add track request", "trackID", msg.AddTrack.Cid) participant.AddTrack(msg.AddTrack) + case *livekit.SignalRequest_Mute: participant.SetTrackMuted(livekit.TrackID(msg.Mute.Sid), msg.Mute.Muted, false) + case *livekit.SignalRequest_Subscription: // allow participant to indicate their interest in the subscription // permission check happens later in SubscriptionManager @@ -50,25 +55,30 @@ func HandleParticipantSignal(room types.Room, participant types.LocalParticipant msg.Subscription.ParticipantTracks, msg.Subscription.Subscribe, ) + case *livekit.SignalRequest_TrackSetting: for _, sid := range livekit.StringsAsIDs[livekit.TrackID](msg.TrackSetting.TrackSids) { participant.UpdateSubscribedTrackSettings(sid, msg.TrackSetting) } + case *livekit.SignalRequest_Leave: pLogger.Debugw("client leaving room") room.RemoveParticipant(participant.Identity(), participant.ID(), types.ParticipantCloseReasonClientRequestLeave) + case *livekit.SignalRequest_SubscriptionPermission: err := room.UpdateSubscriptionPermission(participant, msg.SubscriptionPermission) if err != nil { pLogger.Warnw("could not update subscription permission", err, "permissions", msg.SubscriptionPermission) } + case *livekit.SignalRequest_SyncState: err := room.SyncState(participant, msg.SyncState) if err != nil { pLogger.Warnw("could not sync state", err, "state", msg.SyncState) } + case *livekit.SignalRequest_Simulate: err := room.SimulateScenario(participant, msg.Simulate) if err != nil { @@ -85,6 +95,17 @@ func HandleParticipantSignal(room types.Room, participant types.LocalParticipant if participant.ClaimGrants().Video.GetCanUpdateOwnMetadata() { room.UpdateParticipantMetadata(participant, msg.UpdateMetadata.Name, msg.UpdateMetadata.Metadata) } + + case *livekit.SignalRequest_UpdateAudioTrack: + if err := participant.UpdateAudioTrack(msg.UpdateAudioTrack); err != nil { + pLogger.Warnw("could not update audio track", err, "update", msg.UpdateAudioTrack) + } + + case *livekit.SignalRequest_UpdateVideoTrack: + if err := participant.UpdateVideoTrack(msg.UpdateVideoTrack); err != nil { + pLogger.Warnw("could not update video track", err, "update", msg.UpdateVideoTrack) + } } + return nil } diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index e072d1935..75c77509f 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -280,6 +280,8 @@ type Participant interface { resolverBySid func(participantID livekit.ParticipantID) LocalParticipant, ) error UpdateVideoLayers(updateVideoLayers *livekit.UpdateVideoLayers) error + UpdateAudioTrack(update *livekit.UpdateLocalAudioTrack) error + UpdateVideoTrack(update *livekit.UpdateLocalVideoTrack) error DebugInfo() map[string]interface{} } @@ -451,6 +453,8 @@ type MediaTrack interface { Stream() string UpdateTrackInfo(ti *livekit.TrackInfo) + UpdateAudioTrack(update *livekit.UpdateLocalAudioTrack) + UpdateVideoTrack(update *livekit.UpdateLocalVideoTrack) ToProto() *livekit.TrackInfo PublisherID() livekit.ParticipantID diff --git a/pkg/rtc/types/typesfakes/fake_local_media_track.go b/pkg/rtc/types/typesfakes/fake_local_media_track.go index be86e74c4..10218aba1 100644 --- a/pkg/rtc/types/typesfakes/fake_local_media_track.go +++ b/pkg/rtc/types/typesfakes/fake_local_media_track.go @@ -332,6 +332,11 @@ type FakeLocalMediaTrack struct { toProtoReturnsOnCall map[int]struct { result1 *livekit.TrackInfo } + UpdateAudioTrackStub func(*livekit.UpdateLocalAudioTrack) + updateAudioTrackMutex sync.RWMutex + updateAudioTrackArgsForCall []struct { + arg1 *livekit.UpdateLocalAudioTrack + } UpdateTrackInfoStub func(*livekit.TrackInfo) updateTrackInfoMutex sync.RWMutex updateTrackInfoArgsForCall []struct { @@ -342,6 +347,11 @@ type FakeLocalMediaTrack struct { updateVideoLayersArgsForCall []struct { arg1 []*livekit.VideoLayer } + UpdateVideoTrackStub func(*livekit.UpdateLocalVideoTrack) + updateVideoTrackMutex sync.RWMutex + updateVideoTrackArgsForCall []struct { + arg1 *livekit.UpdateLocalVideoTrack + } invocations map[string][][]interface{} invocationsMutex sync.RWMutex } @@ -2077,6 +2087,38 @@ func (fake *FakeLocalMediaTrack) ToProtoReturnsOnCall(i int, result1 *livekit.Tr }{result1} } +func (fake *FakeLocalMediaTrack) UpdateAudioTrack(arg1 *livekit.UpdateLocalAudioTrack) { + fake.updateAudioTrackMutex.Lock() + fake.updateAudioTrackArgsForCall = append(fake.updateAudioTrackArgsForCall, struct { + arg1 *livekit.UpdateLocalAudioTrack + }{arg1}) + stub := fake.UpdateAudioTrackStub + fake.recordInvocation("UpdateAudioTrack", []interface{}{arg1}) + fake.updateAudioTrackMutex.Unlock() + if stub != nil { + fake.UpdateAudioTrackStub(arg1) + } +} + +func (fake *FakeLocalMediaTrack) UpdateAudioTrackCallCount() int { + fake.updateAudioTrackMutex.RLock() + defer fake.updateAudioTrackMutex.RUnlock() + return len(fake.updateAudioTrackArgsForCall) +} + +func (fake *FakeLocalMediaTrack) UpdateAudioTrackCalls(stub func(*livekit.UpdateLocalAudioTrack)) { + fake.updateAudioTrackMutex.Lock() + defer fake.updateAudioTrackMutex.Unlock() + fake.UpdateAudioTrackStub = stub +} + +func (fake *FakeLocalMediaTrack) UpdateAudioTrackArgsForCall(i int) *livekit.UpdateLocalAudioTrack { + fake.updateAudioTrackMutex.RLock() + defer fake.updateAudioTrackMutex.RUnlock() + argsForCall := fake.updateAudioTrackArgsForCall[i] + return argsForCall.arg1 +} + func (fake *FakeLocalMediaTrack) UpdateTrackInfo(arg1 *livekit.TrackInfo) { fake.updateTrackInfoMutex.Lock() fake.updateTrackInfoArgsForCall = append(fake.updateTrackInfoArgsForCall, struct { @@ -2146,6 +2188,38 @@ func (fake *FakeLocalMediaTrack) UpdateVideoLayersArgsForCall(i int) []*livekit. return argsForCall.arg1 } +func (fake *FakeLocalMediaTrack) UpdateVideoTrack(arg1 *livekit.UpdateLocalVideoTrack) { + fake.updateVideoTrackMutex.Lock() + fake.updateVideoTrackArgsForCall = append(fake.updateVideoTrackArgsForCall, struct { + arg1 *livekit.UpdateLocalVideoTrack + }{arg1}) + stub := fake.UpdateVideoTrackStub + fake.recordInvocation("UpdateVideoTrack", []interface{}{arg1}) + fake.updateVideoTrackMutex.Unlock() + if stub != nil { + fake.UpdateVideoTrackStub(arg1) + } +} + +func (fake *FakeLocalMediaTrack) UpdateVideoTrackCallCount() int { + fake.updateVideoTrackMutex.RLock() + defer fake.updateVideoTrackMutex.RUnlock() + return len(fake.updateVideoTrackArgsForCall) +} + +func (fake *FakeLocalMediaTrack) UpdateVideoTrackCalls(stub func(*livekit.UpdateLocalVideoTrack)) { + fake.updateVideoTrackMutex.Lock() + defer fake.updateVideoTrackMutex.Unlock() + fake.UpdateVideoTrackStub = stub +} + +func (fake *FakeLocalMediaTrack) UpdateVideoTrackArgsForCall(i int) *livekit.UpdateLocalVideoTrack { + fake.updateVideoTrackMutex.RLock() + defer fake.updateVideoTrackMutex.RUnlock() + argsForCall := fake.updateVideoTrackArgsForCall[i] + return argsForCall.arg1 +} + func (fake *FakeLocalMediaTrack) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() @@ -2219,10 +2293,14 @@ func (fake *FakeLocalMediaTrack) Invocations() map[string][][]interface{} { defer fake.streamMutex.RUnlock() fake.toProtoMutex.RLock() defer fake.toProtoMutex.RUnlock() + fake.updateAudioTrackMutex.RLock() + defer fake.updateAudioTrackMutex.RUnlock() fake.updateTrackInfoMutex.RLock() defer fake.updateTrackInfoMutex.RUnlock() fake.updateVideoLayersMutex.RLock() defer fake.updateVideoLayersMutex.RUnlock() + fake.updateVideoTrackMutex.RLock() + defer fake.updateVideoTrackMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/rtc/types/typesfakes/fake_local_participant.go b/pkg/rtc/types/typesfakes/fake_local_participant.go index b0843f428..0531bb63a 100644 --- a/pkg/rtc/types/typesfakes/fake_local_participant.go +++ b/pkg/rtc/types/typesfakes/fake_local_participant.go @@ -916,6 +916,17 @@ type FakeLocalParticipant struct { unsubscribeFromTrackArgsForCall []struct { arg1 livekit.TrackID } + UpdateAudioTrackStub func(*livekit.UpdateLocalAudioTrack) error + updateAudioTrackMutex sync.RWMutex + updateAudioTrackArgsForCall []struct { + arg1 *livekit.UpdateLocalAudioTrack + } + updateAudioTrackReturns struct { + result1 error + } + updateAudioTrackReturnsOnCall map[int]struct { + result1 error + } UpdateLastSeenSignalStub func() updateLastSeenSignalMutex sync.RWMutex updateLastSeenSignalArgsForCall []struct { @@ -986,6 +997,17 @@ type FakeLocalParticipant struct { updateVideoLayersReturnsOnCall map[int]struct { result1 error } + UpdateVideoTrackStub func(*livekit.UpdateLocalVideoTrack) error + updateVideoTrackMutex sync.RWMutex + updateVideoTrackArgsForCall []struct { + arg1 *livekit.UpdateLocalVideoTrack + } + updateVideoTrackReturns struct { + result1 error + } + updateVideoTrackReturnsOnCall map[int]struct { + result1 error + } VerifySubscribeParticipantInfoStub func(livekit.ParticipantID, uint32) verifySubscribeParticipantInfoMutex sync.RWMutex verifySubscribeParticipantInfoArgsForCall []struct { @@ -5902,6 +5924,67 @@ func (fake *FakeLocalParticipant) UnsubscribeFromTrackArgsForCall(i int) livekit return argsForCall.arg1 } +func (fake *FakeLocalParticipant) UpdateAudioTrack(arg1 *livekit.UpdateLocalAudioTrack) error { + fake.updateAudioTrackMutex.Lock() + ret, specificReturn := fake.updateAudioTrackReturnsOnCall[len(fake.updateAudioTrackArgsForCall)] + fake.updateAudioTrackArgsForCall = append(fake.updateAudioTrackArgsForCall, struct { + arg1 *livekit.UpdateLocalAudioTrack + }{arg1}) + stub := fake.UpdateAudioTrackStub + fakeReturns := fake.updateAudioTrackReturns + fake.recordInvocation("UpdateAudioTrack", []interface{}{arg1}) + fake.updateAudioTrackMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) UpdateAudioTrackCallCount() int { + fake.updateAudioTrackMutex.RLock() + defer fake.updateAudioTrackMutex.RUnlock() + return len(fake.updateAudioTrackArgsForCall) +} + +func (fake *FakeLocalParticipant) UpdateAudioTrackCalls(stub func(*livekit.UpdateLocalAudioTrack) error) { + fake.updateAudioTrackMutex.Lock() + defer fake.updateAudioTrackMutex.Unlock() + fake.UpdateAudioTrackStub = stub +} + +func (fake *FakeLocalParticipant) UpdateAudioTrackArgsForCall(i int) *livekit.UpdateLocalAudioTrack { + fake.updateAudioTrackMutex.RLock() + defer fake.updateAudioTrackMutex.RUnlock() + argsForCall := fake.updateAudioTrackArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeLocalParticipant) UpdateAudioTrackReturns(result1 error) { + fake.updateAudioTrackMutex.Lock() + defer fake.updateAudioTrackMutex.Unlock() + fake.UpdateAudioTrackStub = nil + fake.updateAudioTrackReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeLocalParticipant) UpdateAudioTrackReturnsOnCall(i int, result1 error) { + fake.updateAudioTrackMutex.Lock() + defer fake.updateAudioTrackMutex.Unlock() + fake.UpdateAudioTrackStub = nil + if fake.updateAudioTrackReturnsOnCall == nil { + fake.updateAudioTrackReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.updateAudioTrackReturnsOnCall[i] = struct { + result1 error + }{result1} +} + func (fake *FakeLocalParticipant) UpdateLastSeenSignal() { fake.updateLastSeenSignalMutex.Lock() fake.updateLastSeenSignalArgsForCall = append(fake.updateLastSeenSignalArgsForCall, struct { @@ -6278,6 +6361,67 @@ func (fake *FakeLocalParticipant) UpdateVideoLayersReturnsOnCall(i int, result1 }{result1} } +func (fake *FakeLocalParticipant) UpdateVideoTrack(arg1 *livekit.UpdateLocalVideoTrack) error { + fake.updateVideoTrackMutex.Lock() + ret, specificReturn := fake.updateVideoTrackReturnsOnCall[len(fake.updateVideoTrackArgsForCall)] + fake.updateVideoTrackArgsForCall = append(fake.updateVideoTrackArgsForCall, struct { + arg1 *livekit.UpdateLocalVideoTrack + }{arg1}) + stub := fake.UpdateVideoTrackStub + fakeReturns := fake.updateVideoTrackReturns + fake.recordInvocation("UpdateVideoTrack", []interface{}{arg1}) + fake.updateVideoTrackMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) UpdateVideoTrackCallCount() int { + fake.updateVideoTrackMutex.RLock() + defer fake.updateVideoTrackMutex.RUnlock() + return len(fake.updateVideoTrackArgsForCall) +} + +func (fake *FakeLocalParticipant) UpdateVideoTrackCalls(stub func(*livekit.UpdateLocalVideoTrack) error) { + fake.updateVideoTrackMutex.Lock() + defer fake.updateVideoTrackMutex.Unlock() + fake.UpdateVideoTrackStub = stub +} + +func (fake *FakeLocalParticipant) UpdateVideoTrackArgsForCall(i int) *livekit.UpdateLocalVideoTrack { + fake.updateVideoTrackMutex.RLock() + defer fake.updateVideoTrackMutex.RUnlock() + argsForCall := fake.updateVideoTrackArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeLocalParticipant) UpdateVideoTrackReturns(result1 error) { + fake.updateVideoTrackMutex.Lock() + defer fake.updateVideoTrackMutex.Unlock() + fake.UpdateVideoTrackStub = nil + fake.updateVideoTrackReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeLocalParticipant) UpdateVideoTrackReturnsOnCall(i int, result1 error) { + fake.updateVideoTrackMutex.Lock() + defer fake.updateVideoTrackMutex.Unlock() + fake.UpdateVideoTrackStub = nil + if fake.updateVideoTrackReturnsOnCall == nil { + fake.updateVideoTrackReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.updateVideoTrackReturnsOnCall[i] = struct { + result1 error + }{result1} +} + func (fake *FakeLocalParticipant) VerifySubscribeParticipantInfo(arg1 livekit.ParticipantID, arg2 uint32) { fake.verifySubscribeParticipantInfoMutex.Lock() fake.verifySubscribeParticipantInfoArgsForCall = append(fake.verifySubscribeParticipantInfoArgsForCall, struct { @@ -6647,6 +6791,8 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.uncacheDownTrackMutex.RUnlock() fake.unsubscribeFromTrackMutex.RLock() defer fake.unsubscribeFromTrackMutex.RUnlock() + fake.updateAudioTrackMutex.RLock() + defer fake.updateAudioTrackMutex.RUnlock() fake.updateLastSeenSignalMutex.RLock() defer fake.updateLastSeenSignalMutex.RUnlock() fake.updateMediaLossMutex.RLock() @@ -6663,6 +6809,8 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.updateSubscriptionPermissionMutex.RUnlock() fake.updateVideoLayersMutex.RLock() defer fake.updateVideoLayersMutex.RUnlock() + fake.updateVideoTrackMutex.RLock() + defer fake.updateVideoTrackMutex.RUnlock() fake.verifySubscribeParticipantInfoMutex.RLock() defer fake.verifySubscribeParticipantInfoMutex.RUnlock() fake.waitUntilSubscribedMutex.RLock() diff --git a/pkg/rtc/types/typesfakes/fake_media_track.go b/pkg/rtc/types/typesfakes/fake_media_track.go index d4bdfc17e..ddfff0c3b 100644 --- a/pkg/rtc/types/typesfakes/fake_media_track.go +++ b/pkg/rtc/types/typesfakes/fake_media_track.go @@ -268,6 +268,11 @@ type FakeMediaTrack struct { toProtoReturnsOnCall map[int]struct { result1 *livekit.TrackInfo } + UpdateAudioTrackStub func(*livekit.UpdateLocalAudioTrack) + updateAudioTrackMutex sync.RWMutex + updateAudioTrackArgsForCall []struct { + arg1 *livekit.UpdateLocalAudioTrack + } UpdateTrackInfoStub func(*livekit.TrackInfo) updateTrackInfoMutex sync.RWMutex updateTrackInfoArgsForCall []struct { @@ -278,6 +283,11 @@ type FakeMediaTrack struct { updateVideoLayersArgsForCall []struct { arg1 []*livekit.VideoLayer } + UpdateVideoTrackStub func(*livekit.UpdateLocalVideoTrack) + updateVideoTrackMutex sync.RWMutex + updateVideoTrackArgsForCall []struct { + arg1 *livekit.UpdateLocalVideoTrack + } invocations map[string][][]interface{} invocationsMutex sync.RWMutex } @@ -1663,6 +1673,38 @@ func (fake *FakeMediaTrack) ToProtoReturnsOnCall(i int, result1 *livekit.TrackIn }{result1} } +func (fake *FakeMediaTrack) UpdateAudioTrack(arg1 *livekit.UpdateLocalAudioTrack) { + fake.updateAudioTrackMutex.Lock() + fake.updateAudioTrackArgsForCall = append(fake.updateAudioTrackArgsForCall, struct { + arg1 *livekit.UpdateLocalAudioTrack + }{arg1}) + stub := fake.UpdateAudioTrackStub + fake.recordInvocation("UpdateAudioTrack", []interface{}{arg1}) + fake.updateAudioTrackMutex.Unlock() + if stub != nil { + fake.UpdateAudioTrackStub(arg1) + } +} + +func (fake *FakeMediaTrack) UpdateAudioTrackCallCount() int { + fake.updateAudioTrackMutex.RLock() + defer fake.updateAudioTrackMutex.RUnlock() + return len(fake.updateAudioTrackArgsForCall) +} + +func (fake *FakeMediaTrack) UpdateAudioTrackCalls(stub func(*livekit.UpdateLocalAudioTrack)) { + fake.updateAudioTrackMutex.Lock() + defer fake.updateAudioTrackMutex.Unlock() + fake.UpdateAudioTrackStub = stub +} + +func (fake *FakeMediaTrack) UpdateAudioTrackArgsForCall(i int) *livekit.UpdateLocalAudioTrack { + fake.updateAudioTrackMutex.RLock() + defer fake.updateAudioTrackMutex.RUnlock() + argsForCall := fake.updateAudioTrackArgsForCall[i] + return argsForCall.arg1 +} + func (fake *FakeMediaTrack) UpdateTrackInfo(arg1 *livekit.TrackInfo) { fake.updateTrackInfoMutex.Lock() fake.updateTrackInfoArgsForCall = append(fake.updateTrackInfoArgsForCall, struct { @@ -1732,6 +1774,38 @@ func (fake *FakeMediaTrack) UpdateVideoLayersArgsForCall(i int) []*livekit.Video return argsForCall.arg1 } +func (fake *FakeMediaTrack) UpdateVideoTrack(arg1 *livekit.UpdateLocalVideoTrack) { + fake.updateVideoTrackMutex.Lock() + fake.updateVideoTrackArgsForCall = append(fake.updateVideoTrackArgsForCall, struct { + arg1 *livekit.UpdateLocalVideoTrack + }{arg1}) + stub := fake.UpdateVideoTrackStub + fake.recordInvocation("UpdateVideoTrack", []interface{}{arg1}) + fake.updateVideoTrackMutex.Unlock() + if stub != nil { + fake.UpdateVideoTrackStub(arg1) + } +} + +func (fake *FakeMediaTrack) UpdateVideoTrackCallCount() int { + fake.updateVideoTrackMutex.RLock() + defer fake.updateVideoTrackMutex.RUnlock() + return len(fake.updateVideoTrackArgsForCall) +} + +func (fake *FakeMediaTrack) UpdateVideoTrackCalls(stub func(*livekit.UpdateLocalVideoTrack)) { + fake.updateVideoTrackMutex.Lock() + defer fake.updateVideoTrackMutex.Unlock() + fake.UpdateVideoTrackStub = stub +} + +func (fake *FakeMediaTrack) UpdateVideoTrackArgsForCall(i int) *livekit.UpdateLocalVideoTrack { + fake.updateVideoTrackMutex.RLock() + defer fake.updateVideoTrackMutex.RUnlock() + argsForCall := fake.updateVideoTrackArgsForCall[i] + return argsForCall.arg1 +} + func (fake *FakeMediaTrack) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() @@ -1789,10 +1863,14 @@ func (fake *FakeMediaTrack) Invocations() map[string][][]interface{} { defer fake.streamMutex.RUnlock() fake.toProtoMutex.RLock() defer fake.toProtoMutex.RUnlock() + fake.updateAudioTrackMutex.RLock() + defer fake.updateAudioTrackMutex.RUnlock() fake.updateTrackInfoMutex.RLock() defer fake.updateTrackInfoMutex.RUnlock() fake.updateVideoLayersMutex.RLock() defer fake.updateVideoLayersMutex.RUnlock() + fake.updateVideoTrackMutex.RLock() + defer fake.updateVideoTrackMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/rtc/types/typesfakes/fake_participant.go b/pkg/rtc/types/typesfakes/fake_participant.go index 3c06229a1..8a785cb38 100644 --- a/pkg/rtc/types/typesfakes/fake_participant.go +++ b/pkg/rtc/types/typesfakes/fake_participant.go @@ -217,6 +217,17 @@ type FakeParticipant struct { toProtoReturnsOnCall map[int]struct { result1 *livekit.ParticipantInfo } + UpdateAudioTrackStub func(*livekit.UpdateLocalAudioTrack) error + updateAudioTrackMutex sync.RWMutex + updateAudioTrackArgsForCall []struct { + arg1 *livekit.UpdateLocalAudioTrack + } + updateAudioTrackReturns struct { + result1 error + } + updateAudioTrackReturnsOnCall map[int]struct { + result1 error + } UpdateSubscriptionPermissionStub func(*livekit.SubscriptionPermission, utils.TimedVersion, func(participantID livekit.ParticipantID) types.LocalParticipant) error updateSubscriptionPermissionMutex sync.RWMutex updateSubscriptionPermissionArgsForCall []struct { @@ -241,6 +252,17 @@ type FakeParticipant struct { updateVideoLayersReturnsOnCall map[int]struct { result1 error } + UpdateVideoTrackStub func(*livekit.UpdateLocalVideoTrack) error + updateVideoTrackMutex sync.RWMutex + updateVideoTrackArgsForCall []struct { + arg1 *livekit.UpdateLocalVideoTrack + } + updateVideoTrackReturns struct { + result1 error + } + updateVideoTrackReturnsOnCall map[int]struct { + result1 error + } invocations map[string][][]interface{} invocationsMutex sync.RWMutex } @@ -1330,6 +1352,67 @@ func (fake *FakeParticipant) ToProtoReturnsOnCall(i int, result1 *livekit.Partic }{result1} } +func (fake *FakeParticipant) UpdateAudioTrack(arg1 *livekit.UpdateLocalAudioTrack) error { + fake.updateAudioTrackMutex.Lock() + ret, specificReturn := fake.updateAudioTrackReturnsOnCall[len(fake.updateAudioTrackArgsForCall)] + fake.updateAudioTrackArgsForCall = append(fake.updateAudioTrackArgsForCall, struct { + arg1 *livekit.UpdateLocalAudioTrack + }{arg1}) + stub := fake.UpdateAudioTrackStub + fakeReturns := fake.updateAudioTrackReturns + fake.recordInvocation("UpdateAudioTrack", []interface{}{arg1}) + fake.updateAudioTrackMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeParticipant) UpdateAudioTrackCallCount() int { + fake.updateAudioTrackMutex.RLock() + defer fake.updateAudioTrackMutex.RUnlock() + return len(fake.updateAudioTrackArgsForCall) +} + +func (fake *FakeParticipant) UpdateAudioTrackCalls(stub func(*livekit.UpdateLocalAudioTrack) error) { + fake.updateAudioTrackMutex.Lock() + defer fake.updateAudioTrackMutex.Unlock() + fake.UpdateAudioTrackStub = stub +} + +func (fake *FakeParticipant) UpdateAudioTrackArgsForCall(i int) *livekit.UpdateLocalAudioTrack { + fake.updateAudioTrackMutex.RLock() + defer fake.updateAudioTrackMutex.RUnlock() + argsForCall := fake.updateAudioTrackArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeParticipant) UpdateAudioTrackReturns(result1 error) { + fake.updateAudioTrackMutex.Lock() + defer fake.updateAudioTrackMutex.Unlock() + fake.UpdateAudioTrackStub = nil + fake.updateAudioTrackReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeParticipant) UpdateAudioTrackReturnsOnCall(i int, result1 error) { + fake.updateAudioTrackMutex.Lock() + defer fake.updateAudioTrackMutex.Unlock() + fake.UpdateAudioTrackStub = nil + if fake.updateAudioTrackReturnsOnCall == nil { + fake.updateAudioTrackReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.updateAudioTrackReturnsOnCall[i] = struct { + result1 error + }{result1} +} + func (fake *FakeParticipant) UpdateSubscriptionPermission(arg1 *livekit.SubscriptionPermission, arg2 utils.TimedVersion, arg3 func(participantID livekit.ParticipantID) types.LocalParticipant) error { fake.updateSubscriptionPermissionMutex.Lock() ret, specificReturn := fake.updateSubscriptionPermissionReturnsOnCall[len(fake.updateSubscriptionPermissionArgsForCall)] @@ -1454,6 +1537,67 @@ func (fake *FakeParticipant) UpdateVideoLayersReturnsOnCall(i int, result1 error }{result1} } +func (fake *FakeParticipant) UpdateVideoTrack(arg1 *livekit.UpdateLocalVideoTrack) error { + fake.updateVideoTrackMutex.Lock() + ret, specificReturn := fake.updateVideoTrackReturnsOnCall[len(fake.updateVideoTrackArgsForCall)] + fake.updateVideoTrackArgsForCall = append(fake.updateVideoTrackArgsForCall, struct { + arg1 *livekit.UpdateLocalVideoTrack + }{arg1}) + stub := fake.UpdateVideoTrackStub + fakeReturns := fake.updateVideoTrackReturns + fake.recordInvocation("UpdateVideoTrack", []interface{}{arg1}) + fake.updateVideoTrackMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeParticipant) UpdateVideoTrackCallCount() int { + fake.updateVideoTrackMutex.RLock() + defer fake.updateVideoTrackMutex.RUnlock() + return len(fake.updateVideoTrackArgsForCall) +} + +func (fake *FakeParticipant) UpdateVideoTrackCalls(stub func(*livekit.UpdateLocalVideoTrack) error) { + fake.updateVideoTrackMutex.Lock() + defer fake.updateVideoTrackMutex.Unlock() + fake.UpdateVideoTrackStub = stub +} + +func (fake *FakeParticipant) UpdateVideoTrackArgsForCall(i int) *livekit.UpdateLocalVideoTrack { + fake.updateVideoTrackMutex.RLock() + defer fake.updateVideoTrackMutex.RUnlock() + argsForCall := fake.updateVideoTrackArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeParticipant) UpdateVideoTrackReturns(result1 error) { + fake.updateVideoTrackMutex.Lock() + defer fake.updateVideoTrackMutex.Unlock() + fake.UpdateVideoTrackStub = nil + fake.updateVideoTrackReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeParticipant) UpdateVideoTrackReturnsOnCall(i int, result1 error) { + fake.updateVideoTrackMutex.Lock() + defer fake.updateVideoTrackMutex.Unlock() + fake.UpdateVideoTrackStub = nil + if fake.updateVideoTrackReturnsOnCall == nil { + fake.updateVideoTrackReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.updateVideoTrackReturnsOnCall[i] = struct { + result1 error + }{result1} +} + func (fake *FakeParticipant) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() @@ -1499,10 +1643,14 @@ func (fake *FakeParticipant) Invocations() map[string][][]interface{} { defer fake.subscriptionPermissionMutex.RUnlock() fake.toProtoMutex.RLock() defer fake.toProtoMutex.RUnlock() + fake.updateAudioTrackMutex.RLock() + defer fake.updateAudioTrackMutex.RUnlock() fake.updateSubscriptionPermissionMutex.RLock() defer fake.updateSubscriptionPermissionMutex.RUnlock() fake.updateVideoLayersMutex.RLock() defer fake.updateVideoLayersMutex.RUnlock() + fake.updateVideoTrackMutex.RLock() + defer fake.updateVideoTrackMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/rtc/uptrackmanager.go b/pkg/rtc/uptrackmanager.go index 57cf372ac..30d5ba83b 100644 --- a/pkg/rtc/uptrackmanager.go +++ b/pkg/rtc/uptrackmanager.go @@ -254,6 +254,36 @@ func (u *UpTrackManager) UpdateVideoLayers(updateVideoLayers *livekit.UpdateVide return nil } +func (u *UpTrackManager) UpdateAudioTrack(update *livekit.UpdateLocalAudioTrack) error { + track := u.GetPublishedTrack(livekit.TrackID(update.TrackSid)) + if track == nil { + u.params.Logger.Warnw("could not find track", nil, "trackID", livekit.TrackID(update.TrackSid)) + return errors.New("could not find published track") + } + + track.UpdateAudioTrack(update) + if u.onTrackUpdated != nil { + u.onTrackUpdated(track) + } + + return nil +} + +func (u *UpTrackManager) UpdateVideoTrack(update *livekit.UpdateLocalVideoTrack) error { + track := u.GetPublishedTrack(livekit.TrackID(update.TrackSid)) + if track == nil { + u.params.Logger.Warnw("could not find track", nil, "trackID", livekit.TrackID(update.TrackSid)) + return errors.New("could not find published track") + } + + track.UpdateVideoTrack(update) + if u.onTrackUpdated != nil { + u.onTrackUpdated(track) + } + + return nil +} + func (u *UpTrackManager) AddPublishedTrack(track types.MediaTrack) { u.lock.Lock() if _, ok := u.publishedTracks[track.ID()]; !ok { From 79f6506553760e4d2e1574faaacf04b5ea60de51 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 26 Apr 2024 12:21:40 +0530 Subject: [PATCH 76/78] Clean up UpdateVideoLayers (#2685) --- go.mod | 4 +- go.sum | 8 +- pkg/rtc/mediatrackreceiver.go | 29 ------- pkg/rtc/room.go | 4 - pkg/rtc/types/interfaces.go | 3 - .../typesfakes/fake_local_media_track.go | 44 ----------- .../typesfakes/fake_local_participant.go | 74 ------------------ pkg/rtc/types/typesfakes/fake_media_track.go | 44 ----------- pkg/rtc/types/typesfakes/fake_participant.go | 74 ------------------ pkg/rtc/types/typesfakes/fake_room.go | 76 ------------------- pkg/rtc/uptrackmanager.go | 15 ---- 11 files changed, 6 insertions(+), 369 deletions(-) diff --git a/go.mod b/go.mod index 1981bf306..ed85a6780 100644 --- a/go.mod +++ b/go.mod @@ -20,8 +20,8 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20240416023643-881d3dc5423e - github.com/livekit/protocol v1.12.1-0.20240426044238-2d50a792225e - github.com/livekit/psrpc v0.5.3-0.20240403150641-811331b106d9 + github.com/livekit/protocol v1.12.1-0.20240426063020-fd19ad24b86b + github.com/livekit/psrpc v0.5.3-0.20240426045048-8ba067a45715 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1 diff --git a/go.sum b/go.sum index c14b8d58b..28895fb95 100644 --- a/go.sum +++ b/go.sum @@ -120,10 +120,10 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20240416023643-881d3dc5423e h1:ss4VwrouYiDpuNJ9BUTH+WsW+GDdJS70iZp8ii3/0Lc= github.com/livekit/mediatransportutil v0.0.0-20240416023643-881d3dc5423e/go.mod h1:jwKUCmObuiEDH0iiuJHaGMXwRs3RjrB4G6qqgkr/5oE= -github.com/livekit/protocol v1.12.1-0.20240426044238-2d50a792225e h1:hepBKU/dTHNLLxq+Sc9Z8DdNLi3rjcQ6VlL3CEf9NmE= -github.com/livekit/protocol v1.12.1-0.20240426044238-2d50a792225e/go.mod h1:pnn0Dv+/0K0OFqKHX6J6SreYO1dZxl6tDuAZ1ns8L/w= -github.com/livekit/psrpc v0.5.3-0.20240403150641-811331b106d9 h1:4CngtPIJ58WcQ1sUDGdxJDkTndQpN6M/T8jXvRAd7Oc= -github.com/livekit/psrpc v0.5.3-0.20240403150641-811331b106d9/go.mod h1:CQUBSPfYYAaevg1TNCc6/aYsa8DJH4jSRFdCeSZk5u0= +github.com/livekit/protocol v1.12.1-0.20240426063020-fd19ad24b86b h1:hPgkp/LJzhx+U2CHOc68yxGIyfFspagsyupAaqx1Ulw= +github.com/livekit/protocol v1.12.1-0.20240426063020-fd19ad24b86b/go.mod h1:pnn0Dv+/0K0OFqKHX6J6SreYO1dZxl6tDuAZ1ns8L/w= +github.com/livekit/psrpc v0.5.3-0.20240426045048-8ba067a45715 h1:vhDMOe8fxEc/amYTFo799LySPM12Fk3vc+Nc6o4gYZQ= +github.com/livekit/psrpc v0.5.3-0.20240426045048-8ba067a45715/go.mod h1:CQUBSPfYYAaevg1TNCc6/aYsa8DJH4jSRFdCeSZk5u0= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= github.com/mackerelio/go-osstat v0.2.4/go.mod h1:Zy+qzGdZs3A9cuIqmgbJvwbmLQH9dJvtio5ZjJTbdlQ= github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= diff --git a/pkg/rtc/mediatrackreceiver.go b/pkg/rtc/mediatrackreceiver.go index 008f19f27..ec4c9adf0 100644 --- a/pkg/rtc/mediatrackreceiver.go +++ b/pkg/rtc/mediatrackreceiver.go @@ -702,35 +702,6 @@ func (t *MediaTrackReceiver) UpdateVideoTrack(update *livekit.UpdateLocalVideoTr t.updateTrackInfoOfReceivers() } -func (t *MediaTrackReceiver) UpdateVideoLayers(layers []*livekit.VideoLayer) { - t.lock.Lock() - // set video layer ssrc info - for i, ci := range t.trackInfo.Codecs { - originLayers := ci.Layers - ci.Layers = []*livekit.VideoLayer{} - for layerIdx, layer := range layers { - ci.Layers = append(ci.Layers, proto.Clone(layer).(*livekit.VideoLayer)) - for _, l := range originLayers { - if l.Quality == ci.Layers[layerIdx].Quality { - if l.Ssrc != 0 { - ci.Layers[layerIdx].Ssrc = l.Ssrc - } - break - } - } - } - - // for client don't use simulcast codecs (old client version or single codec) - if i == 0 { - t.trackInfo.Layers = ci.Layers - } - } - t.lock.Unlock() - - t.updateTrackInfoOfReceivers() - t.MediaTrackSubscriptions.UpdateVideoLayers() -} - func (t *MediaTrackReceiver) TrackInfo() *livekit.TrackInfo { t.lock.RLock() defer t.lock.RUnlock() diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index 7a8da9a7e..560dc06a1 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -710,10 +710,6 @@ func (r *Room) UpdateSubscriptionPermission(participant types.LocalParticipant, return nil } -func (r *Room) UpdateVideoLayers(participant types.Participant, updateVideoLayers *livekit.UpdateVideoLayers) error { - return participant.UpdateVideoLayers(updateVideoLayers) -} - func (r *Room) ResolveMediaTrackForSubscriber(subIdentity livekit.ParticipantIdentity, trackID livekit.TrackID) types.MediaResolverResult { res := types.MediaResolverResult{} diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index 75c77509f..3b230b0ee 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -279,7 +279,6 @@ type Participant interface { timedVersion utils.TimedVersion, resolverBySid func(participantID livekit.ParticipantID) LocalParticipant, ) error - UpdateVideoLayers(updateVideoLayers *livekit.UpdateVideoLayers) error UpdateAudioTrack(update *livekit.UpdateLocalAudioTrack) error UpdateVideoTrack(update *livekit.UpdateLocalVideoTrack) error @@ -436,7 +435,6 @@ type Room interface { UpdateSubscriptionPermission(participant LocalParticipant, permissions *livekit.SubscriptionPermission) error SyncState(participant LocalParticipant, state *livekit.SyncState) error SimulateScenario(participant LocalParticipant, scenario *livekit.SimulateScenario) error - UpdateVideoLayers(participant Participant, updateVideoLayers *livekit.UpdateVideoLayers) error ResolveMediaTrackForSubscriber(subIdentity livekit.ParticipantIdentity, trackID livekit.TrackID) MediaResolverResult GetLocalParticipants() []LocalParticipant UpdateParticipantMetadata(participant LocalParticipant, name string, metadata string) @@ -464,7 +462,6 @@ type MediaTrack interface { IsMuted() bool SetMuted(muted bool) - UpdateVideoLayers(layers []*livekit.VideoLayer) IsSimulcast() bool GetAudioLevel() (level float64, active bool) diff --git a/pkg/rtc/types/typesfakes/fake_local_media_track.go b/pkg/rtc/types/typesfakes/fake_local_media_track.go index 10218aba1..3545f9187 100644 --- a/pkg/rtc/types/typesfakes/fake_local_media_track.go +++ b/pkg/rtc/types/typesfakes/fake_local_media_track.go @@ -342,11 +342,6 @@ type FakeLocalMediaTrack struct { updateTrackInfoArgsForCall []struct { arg1 *livekit.TrackInfo } - UpdateVideoLayersStub func([]*livekit.VideoLayer) - updateVideoLayersMutex sync.RWMutex - updateVideoLayersArgsForCall []struct { - arg1 []*livekit.VideoLayer - } UpdateVideoTrackStub func(*livekit.UpdateLocalVideoTrack) updateVideoTrackMutex sync.RWMutex updateVideoTrackArgsForCall []struct { @@ -2151,43 +2146,6 @@ func (fake *FakeLocalMediaTrack) UpdateTrackInfoArgsForCall(i int) *livekit.Trac return argsForCall.arg1 } -func (fake *FakeLocalMediaTrack) UpdateVideoLayers(arg1 []*livekit.VideoLayer) { - var arg1Copy []*livekit.VideoLayer - if arg1 != nil { - arg1Copy = make([]*livekit.VideoLayer, len(arg1)) - copy(arg1Copy, arg1) - } - fake.updateVideoLayersMutex.Lock() - fake.updateVideoLayersArgsForCall = append(fake.updateVideoLayersArgsForCall, struct { - arg1 []*livekit.VideoLayer - }{arg1Copy}) - stub := fake.UpdateVideoLayersStub - fake.recordInvocation("UpdateVideoLayers", []interface{}{arg1Copy}) - fake.updateVideoLayersMutex.Unlock() - if stub != nil { - fake.UpdateVideoLayersStub(arg1) - } -} - -func (fake *FakeLocalMediaTrack) UpdateVideoLayersCallCount() int { - fake.updateVideoLayersMutex.RLock() - defer fake.updateVideoLayersMutex.RUnlock() - return len(fake.updateVideoLayersArgsForCall) -} - -func (fake *FakeLocalMediaTrack) UpdateVideoLayersCalls(stub func([]*livekit.VideoLayer)) { - fake.updateVideoLayersMutex.Lock() - defer fake.updateVideoLayersMutex.Unlock() - fake.UpdateVideoLayersStub = stub -} - -func (fake *FakeLocalMediaTrack) UpdateVideoLayersArgsForCall(i int) []*livekit.VideoLayer { - fake.updateVideoLayersMutex.RLock() - defer fake.updateVideoLayersMutex.RUnlock() - argsForCall := fake.updateVideoLayersArgsForCall[i] - return argsForCall.arg1 -} - func (fake *FakeLocalMediaTrack) UpdateVideoTrack(arg1 *livekit.UpdateLocalVideoTrack) { fake.updateVideoTrackMutex.Lock() fake.updateVideoTrackArgsForCall = append(fake.updateVideoTrackArgsForCall, struct { @@ -2297,8 +2255,6 @@ func (fake *FakeLocalMediaTrack) Invocations() map[string][][]interface{} { defer fake.updateAudioTrackMutex.RUnlock() fake.updateTrackInfoMutex.RLock() defer fake.updateTrackInfoMutex.RUnlock() - fake.updateVideoLayersMutex.RLock() - defer fake.updateVideoLayersMutex.RUnlock() fake.updateVideoTrackMutex.RLock() defer fake.updateVideoTrackMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} diff --git a/pkg/rtc/types/typesfakes/fake_local_participant.go b/pkg/rtc/types/typesfakes/fake_local_participant.go index 0531bb63a..7705cda47 100644 --- a/pkg/rtc/types/typesfakes/fake_local_participant.go +++ b/pkg/rtc/types/typesfakes/fake_local_participant.go @@ -986,17 +986,6 @@ type FakeLocalParticipant struct { updateSubscriptionPermissionReturnsOnCall map[int]struct { result1 error } - UpdateVideoLayersStub func(*livekit.UpdateVideoLayers) error - updateVideoLayersMutex sync.RWMutex - updateVideoLayersArgsForCall []struct { - arg1 *livekit.UpdateVideoLayers - } - updateVideoLayersReturns struct { - result1 error - } - updateVideoLayersReturnsOnCall map[int]struct { - result1 error - } UpdateVideoTrackStub func(*livekit.UpdateLocalVideoTrack) error updateVideoTrackMutex sync.RWMutex updateVideoTrackArgsForCall []struct { @@ -6300,67 +6289,6 @@ func (fake *FakeLocalParticipant) UpdateSubscriptionPermissionReturnsOnCall(i in }{result1} } -func (fake *FakeLocalParticipant) UpdateVideoLayers(arg1 *livekit.UpdateVideoLayers) error { - fake.updateVideoLayersMutex.Lock() - ret, specificReturn := fake.updateVideoLayersReturnsOnCall[len(fake.updateVideoLayersArgsForCall)] - fake.updateVideoLayersArgsForCall = append(fake.updateVideoLayersArgsForCall, struct { - arg1 *livekit.UpdateVideoLayers - }{arg1}) - stub := fake.UpdateVideoLayersStub - fakeReturns := fake.updateVideoLayersReturns - fake.recordInvocation("UpdateVideoLayers", []interface{}{arg1}) - fake.updateVideoLayersMutex.Unlock() - if stub != nil { - return stub(arg1) - } - if specificReturn { - return ret.result1 - } - return fakeReturns.result1 -} - -func (fake *FakeLocalParticipant) UpdateVideoLayersCallCount() int { - fake.updateVideoLayersMutex.RLock() - defer fake.updateVideoLayersMutex.RUnlock() - return len(fake.updateVideoLayersArgsForCall) -} - -func (fake *FakeLocalParticipant) UpdateVideoLayersCalls(stub func(*livekit.UpdateVideoLayers) error) { - fake.updateVideoLayersMutex.Lock() - defer fake.updateVideoLayersMutex.Unlock() - fake.UpdateVideoLayersStub = stub -} - -func (fake *FakeLocalParticipant) UpdateVideoLayersArgsForCall(i int) *livekit.UpdateVideoLayers { - fake.updateVideoLayersMutex.RLock() - defer fake.updateVideoLayersMutex.RUnlock() - argsForCall := fake.updateVideoLayersArgsForCall[i] - return argsForCall.arg1 -} - -func (fake *FakeLocalParticipant) UpdateVideoLayersReturns(result1 error) { - fake.updateVideoLayersMutex.Lock() - defer fake.updateVideoLayersMutex.Unlock() - fake.UpdateVideoLayersStub = nil - fake.updateVideoLayersReturns = struct { - result1 error - }{result1} -} - -func (fake *FakeLocalParticipant) UpdateVideoLayersReturnsOnCall(i int, result1 error) { - fake.updateVideoLayersMutex.Lock() - defer fake.updateVideoLayersMutex.Unlock() - fake.UpdateVideoLayersStub = nil - if fake.updateVideoLayersReturnsOnCall == nil { - fake.updateVideoLayersReturnsOnCall = make(map[int]struct { - result1 error - }) - } - fake.updateVideoLayersReturnsOnCall[i] = struct { - result1 error - }{result1} -} - func (fake *FakeLocalParticipant) UpdateVideoTrack(arg1 *livekit.UpdateLocalVideoTrack) error { fake.updateVideoTrackMutex.Lock() ret, specificReturn := fake.updateVideoTrackReturnsOnCall[len(fake.updateVideoTrackArgsForCall)] @@ -6807,8 +6735,6 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.updateSubscribedTrackSettingsMutex.RUnlock() fake.updateSubscriptionPermissionMutex.RLock() defer fake.updateSubscriptionPermissionMutex.RUnlock() - fake.updateVideoLayersMutex.RLock() - defer fake.updateVideoLayersMutex.RUnlock() fake.updateVideoTrackMutex.RLock() defer fake.updateVideoTrackMutex.RUnlock() fake.verifySubscribeParticipantInfoMutex.RLock() diff --git a/pkg/rtc/types/typesfakes/fake_media_track.go b/pkg/rtc/types/typesfakes/fake_media_track.go index ddfff0c3b..174d49b82 100644 --- a/pkg/rtc/types/typesfakes/fake_media_track.go +++ b/pkg/rtc/types/typesfakes/fake_media_track.go @@ -278,11 +278,6 @@ type FakeMediaTrack struct { updateTrackInfoArgsForCall []struct { arg1 *livekit.TrackInfo } - UpdateVideoLayersStub func([]*livekit.VideoLayer) - updateVideoLayersMutex sync.RWMutex - updateVideoLayersArgsForCall []struct { - arg1 []*livekit.VideoLayer - } UpdateVideoTrackStub func(*livekit.UpdateLocalVideoTrack) updateVideoTrackMutex sync.RWMutex updateVideoTrackArgsForCall []struct { @@ -1737,43 +1732,6 @@ func (fake *FakeMediaTrack) UpdateTrackInfoArgsForCall(i int) *livekit.TrackInfo return argsForCall.arg1 } -func (fake *FakeMediaTrack) UpdateVideoLayers(arg1 []*livekit.VideoLayer) { - var arg1Copy []*livekit.VideoLayer - if arg1 != nil { - arg1Copy = make([]*livekit.VideoLayer, len(arg1)) - copy(arg1Copy, arg1) - } - fake.updateVideoLayersMutex.Lock() - fake.updateVideoLayersArgsForCall = append(fake.updateVideoLayersArgsForCall, struct { - arg1 []*livekit.VideoLayer - }{arg1Copy}) - stub := fake.UpdateVideoLayersStub - fake.recordInvocation("UpdateVideoLayers", []interface{}{arg1Copy}) - fake.updateVideoLayersMutex.Unlock() - if stub != nil { - fake.UpdateVideoLayersStub(arg1) - } -} - -func (fake *FakeMediaTrack) UpdateVideoLayersCallCount() int { - fake.updateVideoLayersMutex.RLock() - defer fake.updateVideoLayersMutex.RUnlock() - return len(fake.updateVideoLayersArgsForCall) -} - -func (fake *FakeMediaTrack) UpdateVideoLayersCalls(stub func([]*livekit.VideoLayer)) { - fake.updateVideoLayersMutex.Lock() - defer fake.updateVideoLayersMutex.Unlock() - fake.UpdateVideoLayersStub = stub -} - -func (fake *FakeMediaTrack) UpdateVideoLayersArgsForCall(i int) []*livekit.VideoLayer { - fake.updateVideoLayersMutex.RLock() - defer fake.updateVideoLayersMutex.RUnlock() - argsForCall := fake.updateVideoLayersArgsForCall[i] - return argsForCall.arg1 -} - func (fake *FakeMediaTrack) UpdateVideoTrack(arg1 *livekit.UpdateLocalVideoTrack) { fake.updateVideoTrackMutex.Lock() fake.updateVideoTrackArgsForCall = append(fake.updateVideoTrackArgsForCall, struct { @@ -1867,8 +1825,6 @@ func (fake *FakeMediaTrack) Invocations() map[string][][]interface{} { defer fake.updateAudioTrackMutex.RUnlock() fake.updateTrackInfoMutex.RLock() defer fake.updateTrackInfoMutex.RUnlock() - fake.updateVideoLayersMutex.RLock() - defer fake.updateVideoLayersMutex.RUnlock() fake.updateVideoTrackMutex.RLock() defer fake.updateVideoTrackMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} diff --git a/pkg/rtc/types/typesfakes/fake_participant.go b/pkg/rtc/types/typesfakes/fake_participant.go index 8a785cb38..9494b87e6 100644 --- a/pkg/rtc/types/typesfakes/fake_participant.go +++ b/pkg/rtc/types/typesfakes/fake_participant.go @@ -241,17 +241,6 @@ type FakeParticipant struct { updateSubscriptionPermissionReturnsOnCall map[int]struct { result1 error } - UpdateVideoLayersStub func(*livekit.UpdateVideoLayers) error - updateVideoLayersMutex sync.RWMutex - updateVideoLayersArgsForCall []struct { - arg1 *livekit.UpdateVideoLayers - } - updateVideoLayersReturns struct { - result1 error - } - updateVideoLayersReturnsOnCall map[int]struct { - result1 error - } UpdateVideoTrackStub func(*livekit.UpdateLocalVideoTrack) error updateVideoTrackMutex sync.RWMutex updateVideoTrackArgsForCall []struct { @@ -1476,67 +1465,6 @@ func (fake *FakeParticipant) UpdateSubscriptionPermissionReturnsOnCall(i int, re }{result1} } -func (fake *FakeParticipant) UpdateVideoLayers(arg1 *livekit.UpdateVideoLayers) error { - fake.updateVideoLayersMutex.Lock() - ret, specificReturn := fake.updateVideoLayersReturnsOnCall[len(fake.updateVideoLayersArgsForCall)] - fake.updateVideoLayersArgsForCall = append(fake.updateVideoLayersArgsForCall, struct { - arg1 *livekit.UpdateVideoLayers - }{arg1}) - stub := fake.UpdateVideoLayersStub - fakeReturns := fake.updateVideoLayersReturns - fake.recordInvocation("UpdateVideoLayers", []interface{}{arg1}) - fake.updateVideoLayersMutex.Unlock() - if stub != nil { - return stub(arg1) - } - if specificReturn { - return ret.result1 - } - return fakeReturns.result1 -} - -func (fake *FakeParticipant) UpdateVideoLayersCallCount() int { - fake.updateVideoLayersMutex.RLock() - defer fake.updateVideoLayersMutex.RUnlock() - return len(fake.updateVideoLayersArgsForCall) -} - -func (fake *FakeParticipant) UpdateVideoLayersCalls(stub func(*livekit.UpdateVideoLayers) error) { - fake.updateVideoLayersMutex.Lock() - defer fake.updateVideoLayersMutex.Unlock() - fake.UpdateVideoLayersStub = stub -} - -func (fake *FakeParticipant) UpdateVideoLayersArgsForCall(i int) *livekit.UpdateVideoLayers { - fake.updateVideoLayersMutex.RLock() - defer fake.updateVideoLayersMutex.RUnlock() - argsForCall := fake.updateVideoLayersArgsForCall[i] - return argsForCall.arg1 -} - -func (fake *FakeParticipant) UpdateVideoLayersReturns(result1 error) { - fake.updateVideoLayersMutex.Lock() - defer fake.updateVideoLayersMutex.Unlock() - fake.UpdateVideoLayersStub = nil - fake.updateVideoLayersReturns = struct { - result1 error - }{result1} -} - -func (fake *FakeParticipant) UpdateVideoLayersReturnsOnCall(i int, result1 error) { - fake.updateVideoLayersMutex.Lock() - defer fake.updateVideoLayersMutex.Unlock() - fake.UpdateVideoLayersStub = nil - if fake.updateVideoLayersReturnsOnCall == nil { - fake.updateVideoLayersReturnsOnCall = make(map[int]struct { - result1 error - }) - } - fake.updateVideoLayersReturnsOnCall[i] = struct { - result1 error - }{result1} -} - func (fake *FakeParticipant) UpdateVideoTrack(arg1 *livekit.UpdateLocalVideoTrack) error { fake.updateVideoTrackMutex.Lock() ret, specificReturn := fake.updateVideoTrackReturnsOnCall[len(fake.updateVideoTrackArgsForCall)] @@ -1647,8 +1575,6 @@ func (fake *FakeParticipant) Invocations() map[string][][]interface{} { defer fake.updateAudioTrackMutex.RUnlock() fake.updateSubscriptionPermissionMutex.RLock() defer fake.updateSubscriptionPermissionMutex.RUnlock() - fake.updateVideoLayersMutex.RLock() - defer fake.updateVideoLayersMutex.RUnlock() fake.updateVideoTrackMutex.RLock() defer fake.updateVideoTrackMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} diff --git a/pkg/rtc/types/typesfakes/fake_room.go b/pkg/rtc/types/typesfakes/fake_room.go index 67d68e8ca..0abe91d69 100644 --- a/pkg/rtc/types/typesfakes/fake_room.go +++ b/pkg/rtc/types/typesfakes/fake_room.go @@ -109,18 +109,6 @@ type FakeRoom struct { arg3 []*livekit.ParticipantTracks arg4 bool } - UpdateVideoLayersStub func(types.Participant, *livekit.UpdateVideoLayers) error - updateVideoLayersMutex sync.RWMutex - updateVideoLayersArgsForCall []struct { - arg1 types.Participant - arg2 *livekit.UpdateVideoLayers - } - updateVideoLayersReturns struct { - result1 error - } - updateVideoLayersReturnsOnCall map[int]struct { - result1 error - } invocations map[string][][]interface{} invocationsMutex sync.RWMutex } @@ -645,68 +633,6 @@ func (fake *FakeRoom) UpdateSubscriptionsArgsForCall(i int) (types.LocalParticip return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4 } -func (fake *FakeRoom) UpdateVideoLayers(arg1 types.Participant, arg2 *livekit.UpdateVideoLayers) error { - fake.updateVideoLayersMutex.Lock() - ret, specificReturn := fake.updateVideoLayersReturnsOnCall[len(fake.updateVideoLayersArgsForCall)] - fake.updateVideoLayersArgsForCall = append(fake.updateVideoLayersArgsForCall, struct { - arg1 types.Participant - arg2 *livekit.UpdateVideoLayers - }{arg1, arg2}) - stub := fake.UpdateVideoLayersStub - fakeReturns := fake.updateVideoLayersReturns - fake.recordInvocation("UpdateVideoLayers", []interface{}{arg1, arg2}) - fake.updateVideoLayersMutex.Unlock() - if stub != nil { - return stub(arg1, arg2) - } - if specificReturn { - return ret.result1 - } - return fakeReturns.result1 -} - -func (fake *FakeRoom) UpdateVideoLayersCallCount() int { - fake.updateVideoLayersMutex.RLock() - defer fake.updateVideoLayersMutex.RUnlock() - return len(fake.updateVideoLayersArgsForCall) -} - -func (fake *FakeRoom) UpdateVideoLayersCalls(stub func(types.Participant, *livekit.UpdateVideoLayers) error) { - fake.updateVideoLayersMutex.Lock() - defer fake.updateVideoLayersMutex.Unlock() - fake.UpdateVideoLayersStub = stub -} - -func (fake *FakeRoom) UpdateVideoLayersArgsForCall(i int) (types.Participant, *livekit.UpdateVideoLayers) { - fake.updateVideoLayersMutex.RLock() - defer fake.updateVideoLayersMutex.RUnlock() - argsForCall := fake.updateVideoLayersArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 -} - -func (fake *FakeRoom) UpdateVideoLayersReturns(result1 error) { - fake.updateVideoLayersMutex.Lock() - defer fake.updateVideoLayersMutex.Unlock() - fake.UpdateVideoLayersStub = nil - fake.updateVideoLayersReturns = struct { - result1 error - }{result1} -} - -func (fake *FakeRoom) UpdateVideoLayersReturnsOnCall(i int, result1 error) { - fake.updateVideoLayersMutex.Lock() - defer fake.updateVideoLayersMutex.Unlock() - fake.UpdateVideoLayersStub = nil - if fake.updateVideoLayersReturnsOnCall == nil { - fake.updateVideoLayersReturnsOnCall = make(map[int]struct { - result1 error - }) - } - fake.updateVideoLayersReturnsOnCall[i] = struct { - result1 error - }{result1} -} - func (fake *FakeRoom) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() @@ -730,8 +656,6 @@ func (fake *FakeRoom) Invocations() map[string][][]interface{} { defer fake.updateSubscriptionPermissionMutex.RUnlock() fake.updateSubscriptionsMutex.RLock() defer fake.updateSubscriptionsMutex.RUnlock() - fake.updateVideoLayersMutex.RLock() - defer fake.updateVideoLayersMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/rtc/uptrackmanager.go b/pkg/rtc/uptrackmanager.go index 30d5ba83b..a0674324e 100644 --- a/pkg/rtc/uptrackmanager.go +++ b/pkg/rtc/uptrackmanager.go @@ -239,21 +239,6 @@ func (u *UpTrackManager) HasPermission(trackID livekit.TrackID, subIdentity live return u.hasPermissionLocked(trackID, subIdentity) } -func (u *UpTrackManager) UpdateVideoLayers(updateVideoLayers *livekit.UpdateVideoLayers) error { - track := u.GetPublishedTrack(livekit.TrackID(updateVideoLayers.TrackSid)) - if track == nil { - u.params.Logger.Warnw("could not find track", nil, "trackID", livekit.TrackID(updateVideoLayers.TrackSid)) - return errors.New("could not find published track") - } - - track.UpdateVideoLayers(updateVideoLayers.Layers) - if u.onTrackUpdated != nil { - u.onTrackUpdated(track) - } - - return nil -} - func (u *UpTrackManager) UpdateAudioTrack(update *livekit.UpdateLocalAudioTrack) error { track := u.GetPublishedTrack(livekit.TrackID(update.TrackSid)) if track == nil { From 37346774bb5fb9257aa5824cec9686a6ba0af39e Mon Sep 17 00:00:00 2001 From: David Zhao Date: Fri, 26 Apr 2024 00:00:30 -0700 Subject: [PATCH 77/78] Forward transcription data packets to the room (#2687) --- pkg/rtc/participant.go | 61 ++++++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 29 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index bf6931806..a80d3bee6 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -1516,44 +1516,47 @@ func (p *ParticipantImpl) onDataMessage(kind livekit.DataPacket_Kind, data []byt dp.ParticipantIdentity = string(p.params.Identity) } + shouldForward := false // only forward on user payloads switch payload := dp.Value.(type) { case *livekit.DataPacket_User: + u := payload.User + if p.Hidden() { + u.ParticipantSid = "" + u.ParticipantIdentity = "" + } else { + u.ParticipantSid = string(p.params.SID) + u.ParticipantIdentity = string(p.params.Identity) + } + if dp.ParticipantIdentity != "" { + u.ParticipantIdentity = dp.ParticipantIdentity + } else { + dp.ParticipantIdentity = u.ParticipantIdentity + } + if len(dp.DestinationIdentities) != 0 { + u.DestinationIdentities = dp.DestinationIdentities + } else { + dp.DestinationIdentities = u.DestinationIdentities + } + shouldForward = true + case *livekit.DataPacket_SipDtmf: + if p.Kind() == livekit.ParticipantInfo_SIP { + shouldForward = true + } + case *livekit.DataPacket_Transcription: + if p.Kind() == livekit.ParticipantInfo_AGENT { + shouldForward = true + } + default: + p.pubLogger.Warnw("received unsupported data packet", nil, "payload", payload) + } + if shouldForward { p.lock.RLock() onDataPacket := p.onDataPacket p.lock.RUnlock() if onDataPacket != nil { - u := payload.User - if p.Hidden() { - u.ParticipantSid = "" - u.ParticipantIdentity = "" - } else { - u.ParticipantSid = string(p.params.SID) - u.ParticipantIdentity = string(p.params.Identity) - } - if dp.ParticipantIdentity != "" { - u.ParticipantIdentity = dp.ParticipantIdentity - } else { - dp.ParticipantIdentity = u.ParticipantIdentity - } - if len(dp.DestinationIdentities) != 0 { - u.DestinationIdentities = dp.DestinationIdentities - } else { - dp.DestinationIdentities = u.DestinationIdentities - } onDataPacket(p, kind, dp) } - case *livekit.DataPacket_SipDtmf: - if p.grants.GetParticipantKind() == livekit.ParticipantInfo_SIP { - p.lock.RLock() - onDataPacket := p.onDataPacket - p.lock.RUnlock() - if onDataPacket != nil { - onDataPacket(p, kind, dp) - } - } - default: - p.pubLogger.Warnw("received unsupported data packet", nil, "payload", payload) } p.setIsPublisher(true) From 90de06ce020ec7c5274977ed03e7f5ea7c916e9b Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Fri, 26 Apr 2024 15:24:41 +0800 Subject: [PATCH 78/78] Make datachannel optional for publisher (#2686) --- pkg/rtc/participant.go | 1 - pkg/rtc/transport.go | 29 ++++++++++++++++++++++++----- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index a80d3bee6..0c3481449 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -987,7 +987,6 @@ func (p *ParticipantImpl) SetMigrateState(s types.MigrateState) { p.TransportManager.ProcessPendingPublisherOffer() case types.MigrateStateComplete: - p.TransportManager.ProcessPendingPublisherDataChannels() } diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index 9ceca8e72..8e062f6d9 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -149,10 +149,12 @@ type PCTransport struct { lock sync.RWMutex - reliableDC *webrtc.DataChannel - reliableDCOpened bool - lossyDC *webrtc.DataChannel - lossyDCOpened bool + firstOfferReceived bool + firstOfferNoDataChannel bool + reliableDC *webrtc.DataChannel + reliableDCOpened bool + lossyDC *webrtc.DataChannel + lossyDCOpened bool iceStartedAt time.Time iceConnectedAt time.Time @@ -694,7 +696,9 @@ func (t *PCTransport) isFullyEstablished() bool { t.lock.RLock() defer t.lock.RUnlock() - return t.reliableDCOpened && t.lossyDCOpened && !t.connectedAt.IsZero() + dataChannelReady := t.firstOfferNoDataChannel || (t.reliableDCOpened && t.lossyDCOpened) + + return dataChannelReady && !t.connectedAt.IsZero() } func (t *PCTransport) SetPreferTCP(preferTCP bool) { @@ -1698,6 +1702,21 @@ func (t *PCTransport) handleRemoteOfferReceived(sd *webrtc.SessionDescription) e if err != nil { return nil } + + t.lock.Lock() + if !t.firstOfferReceived { + t.firstOfferReceived = true + var dataChannelFound bool + for _, media := range parsed.MediaDescriptions { + if strings.EqualFold(media.MediaName.Media, "application") { + dataChannelFound = true + break + } + } + t.firstOfferNoDataChannel = !dataChannelFound + } + t.lock.Unlock() + iceCredential, offerRestartICE, err := t.isRemoteOfferRestartICE(parsed) if err != nil { return errors.Wrap(err, "check remote offer restart ice failed")