From c3913927f9da675a0fb15665b65f0726fde2a549 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 13 Mar 2023 12:23:37 -0700 Subject: [PATCH 001/324] Update module github.com/pion/webrtc/v3 to v3.1.58 (#1469) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 2f3676438..ff1b11db2 100644 --- a/go.mod +++ b/go.mod @@ -34,7 +34,7 @@ require ( github.com/pion/stun v0.4.0 github.com/pion/transport/v2 v2.0.2 github.com/pion/turn/v2 v2.1.0 - github.com/pion/webrtc/v3 v3.1.56 + github.com/pion/webrtc/v3 v3.1.58 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.14.0 github.com/redis/go-redis/v9 v9.0.2 diff --git a/go.sum b/go.sum index dbff2e09b..eb6f39ff8 100644 --- a/go.sum +++ b/go.sum @@ -332,8 +332,8 @@ github.com/pion/turn/v2 v2.1.0 h1:5wGHSgGhJhP/RpabkUb/T9PdsAjkGLS6toYz5HNzoSI= github.com/pion/turn/v2 v2.1.0/go.mod h1:yrT5XbXSGX1VFSF31A3c1kCNB5bBZgk/uu5LET162qs= github.com/pion/udp/v2 v2.0.1 h1:xP0z6WNux1zWEjhC7onRA3EwwSliXqu1ElUZAQhUP54= github.com/pion/udp/v2 v2.0.1/go.mod h1:B7uvTMP00lzWdyMr/1PVZXtV3wpPIxBRd4Wl6AksXn8= -github.com/pion/webrtc/v3 v3.1.56 h1:ScaiqKQN3liQwT+kJwOBaYP6TwSfixzdUnZmzHAo0a0= -github.com/pion/webrtc/v3 v3.1.56/go.mod h1:7VhbA6ihqJlz6R/INHjyh1b8HpiV9Ct4UQvE1OB/xoM= +github.com/pion/webrtc/v3 v3.1.58 h1:husXqiKQuk6gbOqJlPHs185OskAyxUW6iAEgHghgCrc= +github.com/pion/webrtc/v3 v3.1.58/go.mod h1:jJdqoqGBlZiE3y8Z1tg1fjSkyEDCZLL+foypUBn0Lhk= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= From 75eb0e01ecd85c52d8541ae93e60829bab97a159 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 14 Mar 2023 15:06:59 +0530 Subject: [PATCH 002/324] Missed return after adding layer transition for screen share (#1514) --- pkg/sfu/downtrack.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 7b2384d2e..57811c13a 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -958,6 +958,7 @@ func (d *DownTrack) maybeAddTransition(bitrate int64, distance float64) { if ti.Source == livekit.TrackSource_SCREEN_SHARE { d.connectionStats.AddLayerTransition(distance, time.Now()) + return } d.connectionStats.AddBitrateTransition(bitrate, time.Now()) From e0495f6cab7e4c6bf77a77d7cb3fafed8b5714bc Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 14 Mar 2023 15:19:50 +0530 Subject: [PATCH 003/324] Do not calculate distance if max layers are not valid (#1515) --- pkg/sfu/streamtrackermanager.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/sfu/streamtrackermanager.go b/pkg/sfu/streamtrackermanager.go index af4532af4..7b2257d79 100644 --- a/pkg/sfu/streamtrackermanager.go +++ b/pkg/sfu/streamtrackermanager.go @@ -306,6 +306,10 @@ done: } } + if !maxLayers.IsValid() || s.maxTemporalLayerSeen < 0 { + return 0.0 + } + distance := float64(0.0) for sp := maxLayers.Spatial; sp <= s.getMaxExpectedLayerLocked(); sp++ { for t := maxLayers.Temporal; t <= s.maxTemporalLayerSeen; t++ { @@ -313,10 +317,6 @@ done: } } - if s.maxTemporalLayerSeen < 0 { - return distance - } - return distance / float64(s.maxTemporalLayerSeen+1) } From c2335968deb61615655af00aeeb762161cddd1d0 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 14 Mar 2023 16:27:39 +0530 Subject: [PATCH 004/324] Prevent evaluation over small wkndow. (#1516) With push model (i. e. connection quality evaluation triggered by reception of RTCP receiver report), it is possible that a report is received quickly after a track is started (especially with video). Those should not trigger a quality evaluation. Set `lastStatsAt` in `Start` routine and ensure that start has been called and enough time has passed since last stats time to avoid small windows. --- pkg/sfu/connectionquality/connectionstats.go | 32 ++++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/pkg/sfu/connectionquality/connectionstats.go b/pkg/sfu/connectionquality/connectionstats.go index ac6506bd8..a1a320249 100644 --- a/pkg/sfu/connectionquality/connectionstats.go +++ b/pkg/sfu/connectionquality/connectionstats.go @@ -29,8 +29,10 @@ type ConnectionStatsParams struct { } type ConnectionStats struct { - params ConnectionStatsParams - isVideo atomic.Bool + params ConnectionStatsParams + + isStarted atomic.Bool + isVideo atomic.Bool onStatsUpdate func(cs *ConnectionStats, stat *livekit.AnalyticsStat) @@ -55,8 +57,14 @@ func NewConnectionStats(params ConnectionStatsParams) *ConnectionStats { } func (cs *ConnectionStats) Start(trackInfo *livekit.TrackInfo, at time.Time) { + if cs.isStarted.Swap(true) { + return + } + cs.isVideo.Store(trackInfo.Type == livekit.TrackType_VIDEO) + cs.updateLastStatsAt(time.Now()) // force an initial wait + cs.scorer.Start(at) go cs.updateStatsWorker() @@ -130,7 +138,7 @@ func (cs *ConnectionStats) maybeMarkInProcess() bool { interval = UpdateInterval } - if cs.lastStatsAt.IsZero() || time.Since(cs.lastStatsAt) > time.Duration(processThreshold*float64(interval)) { + if cs.isStarted.Load() && time.Since(cs.lastStatsAt) > time.Duration(processThreshold*float64(interval)) { cs.statsInProcess = true return true } @@ -138,14 +146,18 @@ func (cs *ConnectionStats) maybeMarkInProcess() bool { return false } -func (cs *ConnectionStats) updateInProcess(isAvailable bool, at time.Time) { +func (cs *ConnectionStats) updateLastStatsAt(at time.Time) { + cs.lock.Lock() + defer cs.lock.Unlock() + + cs.lastStatsAt = at +} + +func (cs *ConnectionStats) clearInProcess() { cs.lock.Lock() defer cs.lock.Unlock() cs.statsInProcess = false - if isAvailable { - cs.lastStatsAt = at - } } func (cs *ConnectionStats) getStat(at time.Time) { @@ -160,12 +172,12 @@ func (cs *ConnectionStats) getStat(at time.Time) { streams := cs.params.GetDeltaStats() if len(streams) == 0 { - cs.updateInProcess(false, at) + cs.clearInProcess() return } // stats available, update last stats time - cs.updateInProcess(true, at) + cs.updateLastStatsAt(at) score := cs.updateScore(streams, at) @@ -194,6 +206,8 @@ func (cs *ConnectionStats) getStat(at time.Time) { Mime: cs.params.MimeType, }) } + + cs.clearInProcess() } func (cs *ConnectionStats) updateStatsWorker() { From b23a0e7f390d83d1acf7ebd6177870ac1f0ef9cb Mon Sep 17 00:00:00 2001 From: David Colburn Date: Tue, 14 Mar 2023 13:07:00 -0700 Subject: [PATCH 005/324] add active filter to ListEgress (#1517) * add active filter to ListEgress * update test * missed a filter --- go.mod | 2 +- go.sum | 4 ++-- pkg/service/egress.go | 7 +++++-- pkg/service/interfaces.go | 2 +- pkg/service/redisstore.go | 14 +++++++++++--- pkg/service/redisstore_test.go | 6 +++--- pkg/service/servicefakes/fake_egress_store.go | 18 ++++++++++-------- 7 files changed, 33 insertions(+), 20 deletions(-) diff --git a/go.mod b/go.mod index ff1b11db2..df314512c 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230130133657-96cfb115473a - github.com/livekit/protocol v1.5.0 + github.com/livekit/protocol v1.5.1-0.20230314035739-6d1cd857eb3b github.com/livekit/psrpc v0.2.10-0.20230310095745-5cd63568998d github.com/mackerelio/go-osstat v0.2.3 github.com/magefile/mage v1.14.0 diff --git a/go.sum b/go.sum index eb6f39ff8..4e23b6992 100644 --- a/go.sum +++ b/go.sum @@ -233,8 +233,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20230130133657-96cfb115473a h1:5UkGQpskXp7HcBmyrCwWtO7ygDWbqtjN09Yva4l/nyE= github.com/livekit/mediatransportutil v0.0.0-20230130133657-96cfb115473a/go.mod h1:1Dlx20JPoIKGP45eo+yuj0HjeE25zmyeX/EWHiPCjFw= -github.com/livekit/protocol v1.5.0 h1:jFGSkSEv0PTjUlrW/WnmERejwxyHOSE9If4VU33PYgk= -github.com/livekit/protocol v1.5.0/go.mod h1:hkK/G0wwFiLUGp9F5kxeQxq2CQuIzkmfBwKhTsc71us= +github.com/livekit/protocol v1.5.1-0.20230314035739-6d1cd857eb3b h1:rDm6mdo22cGAFNLwgNhyGwkjxR2civBlspf5//7LIeQ= +github.com/livekit/protocol v1.5.1-0.20230314035739-6d1cd857eb3b/go.mod h1:hkK/G0wwFiLUGp9F5kxeQxq2CQuIzkmfBwKhTsc71us= github.com/livekit/psrpc v0.2.10-0.20230310095745-5cd63568998d h1:3wfbd8zi7zGQCR+xfG3r2k9m2RwXUiIzR0SN4BHewwU= github.com/livekit/psrpc v0.2.10-0.20230310095745-5cd63568998d/go.mod h1:K0j8f1PgLShR7Lx80KbmwFkDH2BvOnycXGV0OSRURKc= github.com/mackerelio/go-osstat v0.2.3 h1:jAMXD5erlDE39kdX2CU7YwCGRcxIO33u/p8+Fhe5dJw= diff --git a/pkg/service/egress.go b/pkg/service/egress.go index abafa7162..1b2c97ec3 100644 --- a/pkg/service/egress.go +++ b/pkg/service/egress.go @@ -300,10 +300,13 @@ func (s *EgressService) ListEgress(ctx context.Context, req *livekit.ListEgressR if err != nil { return nil, err } - items = []*livekit.EgressInfo{info} + + if !req.Active || int32(info.Status) < int32(livekit.EgressStatus_EGRESS_COMPLETE) { + items = []*livekit.EgressInfo{info} + } } else { var err error - items, err = s.es.ListEgress(ctx, livekit.RoomName(req.RoomName)) + items, err = s.es.ListEgress(ctx, livekit.RoomName(req.RoomName), req.Active) if err != nil { return nil, err } diff --git a/pkg/service/interfaces.go b/pkg/service/interfaces.go index 607838f4b..e3b1bcef0 100644 --- a/pkg/service/interfaces.go +++ b/pkg/service/interfaces.go @@ -42,7 +42,7 @@ type ServiceStore interface { type EgressStore interface { StoreEgress(ctx context.Context, info *livekit.EgressInfo) error LoadEgress(ctx context.Context, egressID string) (*livekit.EgressInfo, error) - ListEgress(ctx context.Context, roomName livekit.RoomName) ([]*livekit.EgressInfo, error) + ListEgress(ctx context.Context, roomName livekit.RoomName, active bool) ([]*livekit.EgressInfo, error) UpdateEgress(ctx context.Context, info *livekit.EgressInfo) error } diff --git a/pkg/service/redisstore.go b/pkg/service/redisstore.go index def52e203..2558bc4a4 100644 --- a/pkg/service/redisstore.go +++ b/pkg/service/redisstore.go @@ -350,7 +350,7 @@ func (s *RedisStore) LoadEgress(_ context.Context, egressID string) (*livekit.Eg } } -func (s *RedisStore) ListEgress(_ context.Context, roomName livekit.RoomName) ([]*livekit.EgressInfo, error) { +func (s *RedisStore) ListEgress(_ context.Context, roomName livekit.RoomName, active bool) ([]*livekit.EgressInfo, error) { var infos []*livekit.EgressInfo if roomName == "" { @@ -368,7 +368,11 @@ func (s *RedisStore) ListEgress(_ context.Context, roomName livekit.RoomName) ([ if err != nil { return nil, err } - infos = append(infos, info) + + // if active, filter status starting, active, and ending + if !active || int32(info.Status) < int32(livekit.EgressStatus_EGRESS_COMPLETE) { + infos = append(infos, info) + } } } else { egressIDs, err := s.rc.SMembers(s.ctx, RoomEgressPrefix+string(roomName)).Result() @@ -389,7 +393,11 @@ func (s *RedisStore) ListEgress(_ context.Context, roomName livekit.RoomName) ([ if err != nil { return nil, err } - infos = append(infos, info) + + // if active, filter status starting, active, and ending + if !active || int32(info.Status) < int32(livekit.EgressStatus_EGRESS_COMPLETE) { + infos = append(infos, info) + } } } diff --git a/pkg/service/redisstore_test.go b/pkg/service/redisstore_test.go index 17e8e5021..949480d28 100644 --- a/pkg/service/redisstore_test.go +++ b/pkg/service/redisstore_test.go @@ -189,12 +189,12 @@ func TestEgressStore(t *testing.T) { require.NoError(t, rs.UpdateEgress(ctx, info)) // list - list, err := rs.ListEgress(ctx, "") + list, err := rs.ListEgress(ctx, "", false) require.NoError(t, err) require.Len(t, list, 2) // list by room - list, err = rs.ListEgress(ctx, livekit.RoomName(roomName)) + list, err = rs.ListEgress(ctx, livekit.RoomName(roomName), false) require.NoError(t, err) require.Len(t, list, 1) @@ -207,7 +207,7 @@ func TestEgressStore(t *testing.T) { require.NoError(t, rs.CleanEndedEgress()) // list - list, err = rs.ListEgress(ctx, livekit.RoomName(roomName)) + list, err = rs.ListEgress(ctx, livekit.RoomName(roomName), false) require.NoError(t, err) require.Len(t, list, 0) } diff --git a/pkg/service/servicefakes/fake_egress_store.go b/pkg/service/servicefakes/fake_egress_store.go index 06ee8d9cc..37f13670f 100644 --- a/pkg/service/servicefakes/fake_egress_store.go +++ b/pkg/service/servicefakes/fake_egress_store.go @@ -10,11 +10,12 @@ import ( ) type FakeEgressStore struct { - ListEgressStub func(context.Context, livekit.RoomName) ([]*livekit.EgressInfo, error) + ListEgressStub func(context.Context, livekit.RoomName, bool) ([]*livekit.EgressInfo, error) listEgressMutex sync.RWMutex listEgressArgsForCall []struct { arg1 context.Context arg2 livekit.RoomName + arg3 bool } listEgressReturns struct { result1 []*livekit.EgressInfo @@ -66,19 +67,20 @@ type FakeEgressStore struct { invocationsMutex sync.RWMutex } -func (fake *FakeEgressStore) ListEgress(arg1 context.Context, arg2 livekit.RoomName) ([]*livekit.EgressInfo, error) { +func (fake *FakeEgressStore) ListEgress(arg1 context.Context, arg2 livekit.RoomName, arg3 bool) ([]*livekit.EgressInfo, error) { fake.listEgressMutex.Lock() ret, specificReturn := fake.listEgressReturnsOnCall[len(fake.listEgressArgsForCall)] fake.listEgressArgsForCall = append(fake.listEgressArgsForCall, struct { arg1 context.Context arg2 livekit.RoomName - }{arg1, arg2}) + arg3 bool + }{arg1, arg2, arg3}) stub := fake.ListEgressStub fakeReturns := fake.listEgressReturns - fake.recordInvocation("ListEgress", []interface{}{arg1, arg2}) + fake.recordInvocation("ListEgress", []interface{}{arg1, arg2, arg3}) fake.listEgressMutex.Unlock() if stub != nil { - return stub(arg1, arg2) + return stub(arg1, arg2, arg3) } if specificReturn { return ret.result1, ret.result2 @@ -92,17 +94,17 @@ func (fake *FakeEgressStore) ListEgressCallCount() int { return len(fake.listEgressArgsForCall) } -func (fake *FakeEgressStore) ListEgressCalls(stub func(context.Context, livekit.RoomName) ([]*livekit.EgressInfo, error)) { +func (fake *FakeEgressStore) ListEgressCalls(stub func(context.Context, livekit.RoomName, bool) ([]*livekit.EgressInfo, error)) { fake.listEgressMutex.Lock() defer fake.listEgressMutex.Unlock() fake.ListEgressStub = stub } -func (fake *FakeEgressStore) ListEgressArgsForCall(i int) (context.Context, livekit.RoomName) { +func (fake *FakeEgressStore) ListEgressArgsForCall(i int) (context.Context, livekit.RoomName, bool) { fake.listEgressMutex.RLock() defer fake.listEgressMutex.RUnlock() argsForCall := fake.listEgressArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 } func (fake *FakeEgressStore) ListEgressReturns(result1 []*livekit.EgressInfo, result2 error) { From 04150c044b1ea47e5bf456b8c7f92f809053abbc Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Tue, 14 Mar 2023 17:35:32 -0700 Subject: [PATCH 006/324] count active signal sessions (#1519) * count active signal sessions * fix * generate fake --- .../routingfakes/fake_signal_client.go | 65 +++++++++++++++++++ pkg/routing/signal.go | 10 +++ 2 files changed, 75 insertions(+) diff --git a/pkg/routing/routingfakes/fake_signal_client.go b/pkg/routing/routingfakes/fake_signal_client.go index 884dac4d5..0562b7c44 100644 --- a/pkg/routing/routingfakes/fake_signal_client.go +++ b/pkg/routing/routingfakes/fake_signal_client.go @@ -10,6 +10,16 @@ import ( ) type FakeSignalClient struct { + ActiveCountStub func() int + activeCountMutex sync.RWMutex + activeCountArgsForCall []struct { + } + activeCountReturns struct { + result1 int + } + activeCountReturnsOnCall map[int]struct { + result1 int + } StartParticipantSignalStub func(context.Context, livekit.RoomName, routing.ParticipantInit, livekit.NodeID) (livekit.ConnectionID, routing.MessageSink, routing.MessageSource, error) startParticipantSignalMutex sync.RWMutex startParticipantSignalArgsForCall []struct { @@ -34,6 +44,59 @@ type FakeSignalClient struct { invocationsMutex sync.RWMutex } +func (fake *FakeSignalClient) ActiveCount() int { + fake.activeCountMutex.Lock() + ret, specificReturn := fake.activeCountReturnsOnCall[len(fake.activeCountArgsForCall)] + fake.activeCountArgsForCall = append(fake.activeCountArgsForCall, struct { + }{}) + stub := fake.ActiveCountStub + fakeReturns := fake.activeCountReturns + fake.recordInvocation("ActiveCount", []interface{}{}) + fake.activeCountMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeSignalClient) ActiveCountCallCount() int { + fake.activeCountMutex.RLock() + defer fake.activeCountMutex.RUnlock() + return len(fake.activeCountArgsForCall) +} + +func (fake *FakeSignalClient) ActiveCountCalls(stub func() int) { + fake.activeCountMutex.Lock() + defer fake.activeCountMutex.Unlock() + fake.ActiveCountStub = stub +} + +func (fake *FakeSignalClient) ActiveCountReturns(result1 int) { + fake.activeCountMutex.Lock() + defer fake.activeCountMutex.Unlock() + fake.ActiveCountStub = nil + fake.activeCountReturns = struct { + result1 int + }{result1} +} + +func (fake *FakeSignalClient) ActiveCountReturnsOnCall(i int, result1 int) { + fake.activeCountMutex.Lock() + defer fake.activeCountMutex.Unlock() + fake.ActiveCountStub = nil + if fake.activeCountReturnsOnCall == nil { + fake.activeCountReturnsOnCall = make(map[int]struct { + result1 int + }) + } + fake.activeCountReturnsOnCall[i] = struct { + result1 int + }{result1} +} + func (fake *FakeSignalClient) StartParticipantSignal(arg1 context.Context, arg2 livekit.RoomName, arg3 routing.ParticipantInit, arg4 livekit.NodeID) (livekit.ConnectionID, routing.MessageSink, routing.MessageSource, error) { fake.startParticipantSignalMutex.Lock() ret, specificReturn := fake.startParticipantSignalReturnsOnCall[len(fake.startParticipantSignalArgsForCall)] @@ -110,6 +173,8 @@ func (fake *FakeSignalClient) StartParticipantSignalReturnsOnCall(i int, result1 func (fake *FakeSignalClient) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() + fake.activeCountMutex.RLock() + defer fake.activeCountMutex.RUnlock() fake.startParticipantSignalMutex.RLock() defer fake.startParticipantSignalMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} diff --git a/pkg/routing/signal.go b/pkg/routing/signal.go index 7bbb4df93..4d0f8b812 100644 --- a/pkg/routing/signal.go +++ b/pkg/routing/signal.go @@ -3,6 +3,7 @@ package routing import ( "context" + "go.uber.org/atomic" "google.golang.org/protobuf/proto" "github.com/livekit/livekit-server/pkg/config" @@ -18,6 +19,7 @@ import ( //counterfeiter:generate . SignalClient type SignalClient interface { + ActiveCount() int StartParticipantSignal(ctx context.Context, roomName livekit.RoomName, pi ParticipantInit, nodeID livekit.NodeID) (connectionID livekit.ConnectionID, reqSink MessageSink, resSource MessageSource, err error) } @@ -25,6 +27,7 @@ type signalClient struct { nodeID livekit.NodeID config config.SignalRelayConfig client rpc.TypedSignalClient + active atomic.Int32 } func NewSignalClient(nodeID livekit.NodeID, bus psrpc.MessageBus, config config.SignalRelayConfig) (SignalClient, error) { @@ -45,6 +48,10 @@ func NewSignalClient(nodeID livekit.NodeID, bus psrpc.MessageBus, config config. }, nil } +func (r *signalClient) ActiveCount() int { + return int(r.active.Load()) +} + func (r *signalClient) StartParticipantSignal( ctx context.Context, roomName livekit.RoomName, @@ -84,6 +91,9 @@ func (r *signalClient) StartParticipantSignal( resChan := NewDefaultMessageChannel() go func() { + r.active.Inc() + defer r.active.Dec() + var err error for msg := range stream.Channel() { if err = resChan.WriteMessage(msg.Response); err != nil { From 6c0ca1b1650ed1c3b8bd5ea6d4ae88b3f2533959 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 14 Mar 2023 17:55:10 -0700 Subject: [PATCH 007/324] Update module google.golang.org/protobuf to v1.29.1 [SECURITY] (#1518) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index df314512c..bcaa2e41f 100644 --- a/go.mod +++ b/go.mod @@ -49,7 +49,7 @@ require ( go.uber.org/atomic v1.10.0 go.uber.org/zap v1.24.0 golang.org/x/sync v0.1.0 - google.golang.org/protobuf v1.29.0 + google.golang.org/protobuf v1.29.1 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go.sum b/go.sum index 4e23b6992..212a673ca 100644 --- a/go.sum +++ b/go.sum @@ -759,8 +759,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.29.0 h1:44S3JjaKmLEE4YIkjzexaP+NzZsudE3Zin5Njn/pYX0= -google.golang.org/protobuf v1.29.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.29.1 h1:7QBf+IK2gx70Ap/hDsOmam3GE0v9HicjfEdAxE62UoM= +google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 5bef98dc2a6869c923c0b77901a06b0c515f62ac Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 15 Mar 2023 11:51:14 +0530 Subject: [PATCH 008/324] Switching to layer based quality for camera tracks also. (#1520) There are cases where the layer bit rate configuration is such that the expected bitrate difference is very high. For example, setting up layer 2 (f) layer for 1.7 Mbps and layer 1 (h) for 180 kbps. With bitrate based quality, a layer drop results in going to `POOR` quality rating. With layer based, it will drop one level only. Also, cleaning up the distance to desired calculation a bit. --- pkg/sfu/buffer/rtpstats.go | 1 - .../connectionquality/connectionstats_test.go | 2 +- pkg/sfu/connectionquality/scorer.go | 2 +- pkg/sfu/downtrack.go | 14 +--- pkg/sfu/forwarder.go | 78 +++++++++++++------ pkg/sfu/forwarder_test.go | 54 +++++++------ pkg/sfu/receiver.go | 26 +------ pkg/sfu/streamtrackermanager.go | 12 +-- 8 files changed, 94 insertions(+), 95 deletions(-) diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index 9a0ba637a..c6ea61498 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -79,7 +79,6 @@ type Snapshot struct { } type SnInfo struct { - pktTime int64 hdrSize uint16 pktSize uint16 isPaddingOnly bool diff --git a/pkg/sfu/connectionquality/connectionstats_test.go b/pkg/sfu/connectionquality/connectionstats_test.go index 355a4e270..f29d4f3e8 100644 --- a/pkg/sfu/connectionquality/connectionstats_test.go +++ b/pkg/sfu/connectionquality/connectionstats_test.go @@ -606,7 +606,7 @@ func TestConnectionQuality(t *testing.T) { distance: 2.0, }, { - distance: 2.7, + distance: 2.0, offset: 1 * time.Second, }, }, diff --git a/pkg/sfu/connectionquality/scorer.go b/pkg/sfu/connectionquality/scorer.go index 1cca32299..f094c8ecc 100644 --- a/pkg/sfu/connectionquality/scorer.go +++ b/pkg/sfu/connectionquality/scorer.go @@ -19,7 +19,7 @@ const ( increaseFactor = float64(0.4) // slow increase decreaseFactor = float64(0.8) // fast decrease - distanceWeight = float64(20.0) // each spatial layer missed drops a quality level + distanceWeight = float64(25.0) // each spatial layer missed drops a quality level unmuteTimeThreshold = float64(0.5) ) diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 57811c13a..6bd4c0ea0 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -946,22 +946,12 @@ func (d *DownTrack) UpTrackMaxTemporalLayerSeenChange(maxTemporalLayerSeen int32 d.forwarder.SetMaxTemporalLayerSeen(maxTemporalLayerSeen) } -func (d *DownTrack) maybeAddTransition(bitrate int64, distance float64) { +func (d *DownTrack) maybeAddTransition(_bitrate int64, distance float64) { if d.kind == webrtc.RTPCodecTypeAudio { return } - ti := d.receiver.TrackInfo() - if ti == nil { - return - } - - if ti.Source == livekit.TrackSource_SCREEN_SHARE { - d.connectionStats.AddLayerTransition(distance, time.Now()) - return - } - - d.connectionStats.AddBitrateTransition(bitrate, time.Now()) + d.connectionStats.AddLayerTransition(distance, time.Now()) } func (d *DownTrack) UpTrackBitrateReport(_availableLayers []int32, bitrates Bitrates) { diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 1f2277374..1b0dea4a6 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -1702,43 +1702,71 @@ func getDistanceToDesired( targetLayers VideoLayers, maxLayers VideoLayers, ) float64 { - if muted || pubMuted || maxPublishedLayer == InvalidLayerSpatial || !maxLayers.IsValid() { + if muted || pubMuted || maxPublishedLayer == InvalidLayerSpatial || maxTemporalLayerSeen == InvalidLayerTemporal || !maxLayers.IsValid() { return 0.0 } - found := false - distance := float64(0.0) + adjustedMaxLayers := maxLayers + + // max available spatial is min(subscribedMax, publishedMax, availableMax) + // subscribedMax = subscriber requested max spatial layer + // publishedMax = max spatial layer ever published + // availableMax = based on bit rate measurement, available max spatial layer + maxAvailableSpatial := InvalidLayerSpatial done: - for s := maxLayers.Spatial; s >= 0; s-- { - for t := maxLayers.Temporal; t >= 0; t-- { - if brs[s][t] == 0 { - continue - } - if s == targetLayers.Spatial && t == targetLayers.Temporal { - found = true + for s := int32(len(brs)) - 1; s >= 0; s-- { + for t := int32(len(brs[0])) - 1; t >= 0; t-- { + if brs[s][t] != 0 { + maxAvailableSpatial = s break done } - - distance++ } } + if maxAvailableSpatial < adjustedMaxLayers.Spatial { + adjustedMaxLayers.Spatial = maxAvailableSpatial + } - // maybe overshooting - if !found && targetLayers.IsValid() { - distance = 0.0 - for s := targetLayers.Spatial; s > maxLayers.Spatial; s-- { - for t := maxLayers.Temporal; t >= 0; t-- { - if targetLayers.Temporal < t || brs[s][t] == 0 { - continue - } - distance-- + if maxPublishedLayer < adjustedMaxLayers.Spatial { + adjustedMaxLayers.Spatial = maxPublishedLayer + } + + // max available temporal is min(subscribedMax, temporalLayerSeenMax, availableMax) + // subscribedMax = subscriber requested max temporal layer + // temporalLayerSeenMax = max temporal layer ever published/seen + // availableMax = based on bit rate measurement, available max temporal in the adjusted max spatial layer + maxAvailableTemporal := InvalidLayerTemporal + if adjustedMaxLayers.Spatial != InvalidLayerSpatial { + for t := int32(len(brs[0])) - 1; t >= 0; t-- { + if brs[adjustedMaxLayers.Spatial][t] != 0 { + maxAvailableTemporal = t + break } } } - - if maxTemporalLayerSeen < 0 { - maxTemporalLayerSeen = 0 + if maxAvailableTemporal < adjustedMaxLayers.Temporal { + adjustedMaxLayers.Temporal = maxAvailableTemporal } - return distance / float64(maxTemporalLayerSeen+1) + if maxTemporalLayerSeen < adjustedMaxLayers.Temporal { + adjustedMaxLayers.Temporal = maxTemporalLayerSeen + } + + if !adjustedMaxLayers.IsValid() { + adjustedMaxLayers = VideoLayers{Spatial: 0, Temporal: 0} + } + + // adjust target layers if they are invalid, i. e. not streaming + adjustedTargetLayers := targetLayers + if !targetLayers.IsValid() { + adjustedTargetLayers = VideoLayers{Spatial: 0, Temporal: 0} + } + + distance := + ((adjustedMaxLayers.Spatial - adjustedTargetLayers.Spatial) * (maxTemporalLayerSeen + 1)) + + (adjustedMaxLayers.Temporal - adjustedTargetLayers.Temporal) + if !targetLayers.IsValid() { + distance++ + } + + return float64(distance) / float64(maxTemporalLayerSeen+1) } diff --git a/pkg/sfu/forwarder_test.go b/pkg/sfu/forwarder_test.go index 2765bb0ce..5398a5a61 100644 --- a/pkg/sfu/forwarder_test.go +++ b/pkg/sfu/forwarder_test.go @@ -216,6 +216,7 @@ func TestForwarderAllocateOptimal(t *testing.T) { f.parkedLayers = InvalidLayers // when max layers changes, target is opportunistic, but requested spatial layer should be at max + f.SetMaxTemporalLayerSeen(3) f.maxLayers = VideoLayers{Spatial: 1, Temporal: 3} expectedResult = VideoAllocation{ pauseReason: VideoPauseReasonNone, @@ -252,7 +253,7 @@ func TestForwarderAllocateOptimal(t *testing.T) { targetLayers: expectedTargetLayers, requestLayerSpatial: expectedTargetLayers.Spatial, maxLayers: DefaultMaxLayers, - distanceToDesired: 0, + distanceToDesired: -0.5, } result = f.AllocateOptimal(nil, bitrates, true) require.Equal(t, expectedResult, result) @@ -275,7 +276,7 @@ func TestForwarderAllocateOptimal(t *testing.T) { targetLayers: expectedTargetLayers, requestLayerSpatial: expectedTargetLayers.Spatial, maxLayers: DefaultMaxLayers, - distanceToDesired: 0, + distanceToDesired: -0.75, } result = f.AllocateOptimal(nil, emptyBitrates, true) require.Equal(t, expectedResult, result) @@ -294,7 +295,7 @@ func TestForwarderAllocateOptimal(t *testing.T) { targetLayers: DefaultMaxLayers, requestLayerSpatial: 2, maxLayers: DefaultMaxLayers, - distanceToDesired: 0, + distanceToDesired: -0.5, } result = f.AllocateOptimal([]int32{0, 1}, bitrates, true) require.Equal(t, expectedResult, result) @@ -311,7 +312,7 @@ func TestForwarderAllocateOptimal(t *testing.T) { targetLayers: DefaultMaxLayers, requestLayerSpatial: 2, maxLayers: DefaultMaxLayers, - distanceToDesired: 0, + distanceToDesired: -2.75, } result = f.AllocateOptimal([]int32{0, 1}, emptyBitrates, false) require.Equal(t, expectedResult, result) @@ -331,7 +332,7 @@ func TestForwarderAllocateOptimal(t *testing.T) { targetLayers: expectedTargetLayers, requestLayerSpatial: 2, maxLayers: DefaultMaxLayers, - distanceToDesired: 0, + distanceToDesired: -0.5, } result = f.AllocateOptimal([]int32{0, 1}, bitrates, true) require.Equal(t, expectedResult, result) @@ -352,7 +353,7 @@ func TestForwarderAllocateOptimal(t *testing.T) { targetLayers: expectedTargetLayers, requestLayerSpatial: 1, maxLayers: DefaultMaxLayers, - distanceToDesired: 1, + distanceToDesired: 0.5, } result = f.AllocateOptimal([]int32{1}, bitrates, true) require.Equal(t, expectedResult, result) @@ -374,7 +375,7 @@ func TestForwarderAllocateOptimal(t *testing.T) { targetLayers: expectedTargetLayers, requestLayerSpatial: 0, maxLayers: f.maxLayers, - distanceToDesired: 0, + distanceToDesired: -0.25, } result = f.AllocateOptimal([]int32{0, 1}, emptyBitrates, true) require.Equal(t, expectedResult, result) @@ -396,7 +397,7 @@ func TestForwarderAllocateOptimal(t *testing.T) { targetLayers: expectedTargetLayers, requestLayerSpatial: 2, maxLayers: f.maxLayers, - distanceToDesired: 0, + distanceToDesired: -2.75, } result = f.AllocateOptimal([]int32{0, 1}, emptyBitrates, true) require.Equal(t, expectedResult, result) @@ -408,6 +409,7 @@ func TestForwarderProvisionalAllocate(t *testing.T) { f.SetMaxSpatialLayer(DefaultMaxLayerSpatial) f.SetMaxTemporalLayer(DefaultMaxLayerTemporal) f.SetMaxPublishedLayer(DefaultMaxLayerSpatial) + f.SetMaxTemporalLayerSeen(DefaultMaxLayerTemporal) bitrates := Bitrates{ {1, 2, 3, 4}, @@ -447,7 +449,7 @@ func TestForwarderProvisionalAllocate(t *testing.T) { targetLayers: expectedTargetLayers, requestLayerSpatial: expectedTargetLayers.Spatial, maxLayers: DefaultMaxLayers, - distanceToDesired: 5, + distanceToDesired: 1.25, } result := f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) @@ -474,7 +476,7 @@ func TestForwarderProvisionalAllocate(t *testing.T) { targetLayers: expectedTargetLayers, requestLayerSpatial: expectedTargetLayers.Spatial, maxLayers: DefaultMaxLayers, - distanceToDesired: 11, + distanceToDesired: 2.75, } result = f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) @@ -521,7 +523,7 @@ func TestForwarderProvisionalAllocate(t *testing.T) { targetLayers: expectedTargetLayers, requestLayerSpatial: expectedTargetLayers.Spatial, maxLayers: expectedMaxLayers, - distanceToDesired: -4, + distanceToDesired: -1.75, } result = f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) @@ -565,7 +567,7 @@ func TestForwarderProvisionalAllocate(t *testing.T) { targetLayers: expectedTargetLayers, requestLayerSpatial: expectedTargetLayers.Spatial, maxLayers: expectedMaxLayers, - distanceToDesired: 0, + distanceToDesired: 0.25, } result = f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) @@ -598,7 +600,7 @@ func TestForwarderProvisionalAllocate(t *testing.T) { targetLayers: InvalidLayers, requestLayerSpatial: InvalidLayerSpatial, maxLayers: expectedMaxLayers, - distanceToDesired: 0, + distanceToDesired: 0.25, } result = f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) @@ -649,6 +651,7 @@ func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { f.SetMaxSpatialLayer(DefaultMaxLayerSpatial) f.SetMaxTemporalLayer(DefaultMaxLayerTemporal) f.SetMaxPublishedLayer(DefaultMaxLayerSpatial) + f.SetMaxTemporalLayerSeen(DefaultMaxLayerTemporal) bitrates := Bitrates{ {1, 2, 3, 4}, @@ -678,7 +681,7 @@ func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { targetLayers: expectedLayers, requestLayerSpatial: expectedLayers.Spatial, maxLayers: DefaultMaxLayers, - distanceToDesired: 9, + distanceToDesired: 2.25, } result := f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) @@ -707,7 +710,7 @@ func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { targetLayers: expectedLayers, requestLayerSpatial: expectedLayers.Spatial, maxLayers: DefaultMaxLayers, - distanceToDesired: 0, + distanceToDesired: 0.0, } result = f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) @@ -776,7 +779,7 @@ func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { targetLayers: expectedLayers, requestLayerSpatial: expectedLayers.Spatial, maxLayers: expectedMaxLayers, - distanceToDesired: -1, + distanceToDesired: -1.0, } result = f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) @@ -815,7 +818,7 @@ func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { targetLayers: expectedLayers, requestLayerSpatial: expectedLayers.Spatial, maxLayers: expectedMaxLayers, - distanceToDesired: 0, + distanceToDesired: -0.5, } result = f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) @@ -830,7 +833,7 @@ func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { targetLayers: expectedLayers, requestLayerSpatial: expectedLayers.Spatial, maxLayers: expectedMaxLayers, - distanceToDesired: 0, + distanceToDesired: -0.5, } result = f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) @@ -883,6 +886,7 @@ func TestForwarderAllocateNextHigher(t *testing.T) { f.SetMaxSpatialLayer(DefaultMaxLayerSpatial) f.SetMaxTemporalLayer(DefaultMaxLayerTemporal) f.SetMaxPublishedLayer(DefaultMaxLayerSpatial) + f.SetMaxTemporalLayerSeen(DefaultMaxLayerTemporal) // when not in deficient state, does not boost result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, bitrates, false) @@ -918,7 +922,7 @@ func TestForwarderAllocateNextHigher(t *testing.T) { targetLayers: expectedTargetLayers, requestLayerSpatial: expectedTargetLayers.Spatial, maxLayers: DefaultMaxLayers, - distanceToDesired: 3, + distanceToDesired: 2.0, } result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, bitrates, false) require.Equal(t, expectedResult, result) @@ -946,7 +950,7 @@ func TestForwarderAllocateNextHigher(t *testing.T) { targetLayers: expectedTargetLayers, requestLayerSpatial: expectedTargetLayers.Spatial, maxLayers: DefaultMaxLayers, - distanceToDesired: 2, + distanceToDesired: 1.25, } result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, bitrates, false) require.Equal(t, expectedResult, result) @@ -970,7 +974,7 @@ func TestForwarderAllocateNextHigher(t *testing.T) { targetLayers: expectedTargetLayers, requestLayerSpatial: expectedTargetLayers.Spatial, maxLayers: DefaultMaxLayers, - distanceToDesired: 1, + distanceToDesired: 0.5, } result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, bitrates, false) require.Equal(t, expectedResult, result) @@ -992,7 +996,7 @@ func TestForwarderAllocateNextHigher(t *testing.T) { targetLayers: expectedTargetLayers, requestLayerSpatial: expectedTargetLayers.Spatial, maxLayers: DefaultMaxLayers, - distanceToDesired: 0, + distanceToDesired: 0.0, } result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, bitrates, false) require.Equal(t, expectedResult, result) @@ -1027,7 +1031,7 @@ func TestForwarderAllocateNextHigher(t *testing.T) { targetLayers: expectedTargetLayers, requestLayerSpatial: expectedTargetLayers.Spatial, maxLayers: DefaultMaxLayers, - distanceToDesired: 4, + distanceToDesired: 2.25, } result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, bitrates, false) require.Equal(t, expectedResult, result) @@ -1045,7 +1049,7 @@ func TestForwarderAllocateNextHigher(t *testing.T) { targetLayers: expectedTargetLayers, requestLayerSpatial: expectedTargetLayers.Spatial, maxLayers: DefaultMaxLayers, - distanceToDesired: 4, + distanceToDesired: 2.25, } result, boosted = f.AllocateNextHigher(0, bitrates, false) require.Equal(t, expectedResult, result) @@ -1079,7 +1083,7 @@ func TestForwarderAllocateNextHigher(t *testing.T) { targetLayers: expectedTargetLayers, requestLayerSpatial: expectedTargetLayers.Spatial, maxLayers: expectedMaxLayers, - distanceToDesired: -1, + distanceToDesired: -1.0, } // overshoot should return (1, 0) even if there is not enough capacity result, boosted = f.AllocateNextHigher(bitrates[1][0]-1, bitrates, true) diff --git a/pkg/sfu/receiver.go b/pkg/sfu/receiver.go index 3eab251e4..58a3221f8 100644 --- a/pkg/sfu/receiver.go +++ b/pkg/sfu/receiver.go @@ -385,26 +385,10 @@ func (w *WebRTCReceiver) AddDownTrack(track TrackSender) error { return nil } -func (w *WebRTCReceiver) notifyMaxExpectedLayer(layer int32) { - if w.Kind() == webrtc.RTPCodecTypeAudio || w.trackInfo.Source == livekit.TrackSource_SCREEN_SHARE { - // screen share tracks have highly variable bitrate, do not use bit rate based quality for those - return - } - - expectedBitrate := int64(0) - for _, vl := range w.trackInfo.Layers { - l := buffer.VideoQualityToSpatialLayer(vl.Quality, w.trackInfo) - if l <= layer { - expectedBitrate += int64(vl.Bitrate) - } - } - - w.connectionStats.AddBitrateTransition(expectedBitrate, time.Now()) -} - func (w *WebRTCReceiver) SetMaxExpectedSpatialLayer(layer int32) { w.streamTrackerManager.SetMaxExpectedSpatialLayer(layer) - w.notifyMaxExpectedLayer(layer) + + w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired(), time.Now()) } // StreamTrackerManagerListener.OnAvailableLayersChanged @@ -427,7 +411,7 @@ func (w *WebRTCReceiver) OnMaxPublishedLayerChanged(maxPublishedLayer int32) { dt.UpTrackMaxPublishedLayerChange(maxPublishedLayer) } - w.notifyMaxExpectedLayer(maxPublishedLayer) + w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired(), time.Now()) } // StreamTrackerManagerListener.OnMaxTemporalLayerSeenChanged @@ -436,9 +420,7 @@ func (w *WebRTCReceiver) OnMaxTemporalLayerSeenChanged(maxTemporalLayerSeen int3 dt.UpTrackMaxTemporalLayerSeenChange(maxTemporalLayerSeen) } - if w.trackInfo.Source == livekit.TrackSource_SCREEN_SHARE { - w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired(), time.Now()) - } + w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired(), time.Now()) } // StreamTrackerManagerListener.OnMaxAvailableLayerChanged diff --git a/pkg/sfu/streamtrackermanager.go b/pkg/sfu/streamtrackermanager.go index 7b2257d79..d6c6ba63b 100644 --- a/pkg/sfu/streamtrackermanager.go +++ b/pkg/sfu/streamtrackermanager.go @@ -310,14 +310,10 @@ done: return 0.0 } - distance := float64(0.0) - for sp := maxLayers.Spatial; sp <= s.getMaxExpectedLayerLocked(); sp++ { - for t := maxLayers.Temporal; t <= s.maxTemporalLayerSeen; t++ { - distance++ - } - } - - return distance / float64(s.maxTemporalLayerSeen+1) + distance := + ((s.getMaxExpectedLayerLocked() - maxLayers.Spatial) * (s.maxTemporalLayerSeen + 1)) + + (s.maxTemporalLayerSeen - maxLayers.Temporal) + return float64(distance) / float64(s.maxTemporalLayerSeen+1) } func (s *StreamTrackerManager) getMaxExpectedLayerLocked() int32 { From 582adda97cab2e891b28a4daf5202eee380bcb15 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 15 Mar 2023 13:27:27 +0530 Subject: [PATCH 009/324] Fix connection quality in constrained up stream (#1521) A few things 1. Have to use expected layer in upstream distance to desired. Using min(published, expected) means if expected is higher than published, it was not caught as a missed layer. 2. Forgot to remove layer transition update in one place. It was still constrained to screen share. This caused quality to not pick up after constraint is released. 3. Switching to max layer cannot be marked on max published. Same as point #1 above. Otherwise, dynacast would kick in and turn off highest layer. --- pkg/sfu/forwarder.go | 13 +++++++++---- pkg/sfu/receiver.go | 4 +--- pkg/sfu/streamtrackermanager.go | 15 ++++++++++----- 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 1b0dea4a6..d75c890a7 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -1452,7 +1452,7 @@ func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer in // if f.ddLayerSelector != nil { // f.ddLayerSelector.SelectLayer(f.currentLayers) // } - if f.currentLayers.Spatial >= f.maxLayers.Spatial || f.currentLayers.Spatial == f.maxPublishedLayer { + if f.currentLayers.Spatial >= f.maxLayers.Spatial { tp.isSwitchingToMaxLayer = true } } @@ -1517,10 +1517,9 @@ func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer in if found { tp.isSwitchingToTargetLayer = true f.clearParkedLayers() - if f.currentLayers.Spatial >= f.maxLayers.Spatial || f.currentLayers.Spatial == f.maxPublishedLayer { + if f.currentLayers.Spatial >= f.maxLayers.Spatial { tp.isSwitchingToMaxLayer = true - // if maximum is attained, adjust target to enable fast path layer check in per-packet path f.logger.Infow( "reached max layer", "current", f.currentLayers, @@ -1531,6 +1530,9 @@ func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer in "maxPublished", f.maxPublishedLayer, "feed", extPkt.Packet.SSRC, ) + } + + if f.currentLayers.Spatial >= f.maxLayers.Spatial || f.currentLayers.Spatial == f.maxPublishedLayer { f.targetLayers.Spatial = f.currentLayers.Spatial } } @@ -1551,8 +1553,11 @@ func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer in ) f.currentLayers.Spatial = layer - if f.currentLayers.Spatial >= f.maxLayers.Spatial || f.currentLayers.Spatial == f.maxPublishedLayer { + if f.currentLayers.Spatial >= f.maxLayers.Spatial { tp.isSwitchingToMaxLayer = true + } + + if f.currentLayers.Spatial >= f.maxLayers.Spatial || f.currentLayers.Spatial == f.maxPublishedLayer { f.targetLayers.Spatial = layer } } diff --git a/pkg/sfu/receiver.go b/pkg/sfu/receiver.go index 58a3221f8..ae7493fc5 100644 --- a/pkg/sfu/receiver.go +++ b/pkg/sfu/receiver.go @@ -440,9 +440,7 @@ func (w *WebRTCReceiver) OnBitrateReport(availableLayers []int32, bitrates Bitra dt.UpTrackBitrateReport(availableLayers, bitrates) } - if w.trackInfo.Source == livekit.TrackSource_SCREEN_SHARE { - w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired(), time.Now()) - } + w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired(), time.Now()) } func (w *WebRTCReceiver) GetLayeredBitrate() ([]int32, Bitrates) { diff --git a/pkg/sfu/streamtrackermanager.go b/pkg/sfu/streamtrackermanager.go index d6c6ba63b..6fd9585f7 100644 --- a/pkg/sfu/streamtrackermanager.go +++ b/pkg/sfu/streamtrackermanager.go @@ -286,7 +286,7 @@ func (s *StreamTrackerManager) DistanceToDesired() float64 { s.lock.RLock() defer s.lock.RUnlock() - if s.paused { + if s.paused || s.maxExpectedLayer < 0 || s.maxTemporalLayerSeen < 0 { return 0 } @@ -306,13 +306,18 @@ done: } } - if !maxLayers.IsValid() || s.maxTemporalLayerSeen < 0 { - return 0.0 + adjustedMaxLayers := maxLayers + if !maxLayers.IsValid() { + adjustedMaxLayers = VideoLayers{Spatial: 0, Temporal: 0} } distance := - ((s.getMaxExpectedLayerLocked() - maxLayers.Spatial) * (s.maxTemporalLayerSeen + 1)) + - (s.maxTemporalLayerSeen - maxLayers.Temporal) + ((s.maxExpectedLayer - adjustedMaxLayers.Spatial) * (s.maxTemporalLayerSeen + 1)) + + (s.maxTemporalLayerSeen - adjustedMaxLayers.Temporal) + if !maxLayers.IsValid() { + distance++ + } + return float64(distance) / float64(s.maxTemporalLayerSeen+1) } From ed2eaaabb2b39efa1984c906bb64bd2123dd4dd6 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 15 Mar 2023 15:24:17 +0530 Subject: [PATCH 010/324] Add layer mute notification (#1522) * Layer mute * clean up * clean up * set max temporal layer seen on down track add --- pkg/sfu/connectionquality/connectionstats.go | 4 ++ .../connectionquality/connectionstats_test.go | 44 +++++++++++++++ pkg/sfu/connectionquality/scorer.go | 56 ++++++++++++++----- pkg/sfu/receiver.go | 9 ++- pkg/sfu/streamtrackermanager.go | 16 +++--- 5 files changed, 104 insertions(+), 25 deletions(-) diff --git a/pkg/sfu/connectionquality/connectionstats.go b/pkg/sfu/connectionquality/connectionstats.go index a1a320249..d2e4e30a3 100644 --- a/pkg/sfu/connectionquality/connectionstats.go +++ b/pkg/sfu/connectionquality/connectionstats.go @@ -86,6 +86,10 @@ func (cs *ConnectionStats) AddBitrateTransition(bitrate int64, at time.Time) { cs.scorer.AddBitrateTransition(bitrate, at) } +func (cs *ConnectionStats) UpdateLayerMute(isMuted bool, at time.Time) { + cs.scorer.UpdateLayerMute(isMuted, at) +} + func (cs *ConnectionStats) AddLayerTransition(distance float64, at time.Time) { cs.scorer.AddLayerTransition(distance, at) } diff --git a/pkg/sfu/connectionquality/connectionstats_test.go b/pkg/sfu/connectionquality/connectionstats_test.go index f29d4f3e8..91667cb39 100644 --- a/pkg/sfu/connectionquality/connectionstats_test.go +++ b/pkg/sfu/connectionquality/connectionstats_test.go @@ -297,6 +297,50 @@ func TestConnectionQuality(t *testing.T) { mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(4.6), mos) require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality) + + // test layer mute via UpdateLayerMute API + cs.AddBitrateTransition(1_000_000, now) + cs.AddBitrateTransition(2_000_000, now.Add(2*time.Second)) + + streams = map[uint32]*buffer.StreamStatsWithLayers{ + 1: &buffer.StreamStatsWithLayers{ + RTPStats: &buffer.RTPDeltaInfo{ + StartTime: now, + Duration: duration, + Packets: 250, + Bytes: 8_000_000 / 8 / 4, + }, + }, + } + cs.updateScore(streams, now.Add(duration)) + mos, quality = cs.GetScoreAndQuality() + require.Greater(t, float32(4.1), mos) + require.Equal(t, livekit.ConnectionQuality_GOOD, quality) + + now = now.Add(duration) + cs.UpdateLayerMute(true, now) + mos, quality = cs.GetScoreAndQuality() + require.Greater(t, float32(4.6), mos) + require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality) + + // setting bit rate after layer mute should layer unmute automatically + cs.AddBitrateTransition(1_000_000, now) + cs.AddBitrateTransition(2_000_000, now.Add(2*time.Second)) + + streams = map[uint32]*buffer.StreamStatsWithLayers{ + 1: &buffer.StreamStatsWithLayers{ + RTPStats: &buffer.RTPDeltaInfo{ + StartTime: now, + Duration: duration, + Packets: 250, + Bytes: 8_000_000 / 8 / 4, + }, + }, + } + cs.updateScore(streams, now.Add(duration)) + mos, quality = cs.GetScoreAndQuality() + require.Greater(t, float32(4.1), mos) + require.Equal(t, livekit.ConnectionQuality_GOOD, quality) }) t.Run("codecs - packet", func(t *testing.T) { diff --git a/pkg/sfu/connectionquality/scorer.go b/pkg/sfu/connectionquality/scorer.go index f094c8ecc..9e7a1b047 100644 --- a/pkg/sfu/connectionquality/scorer.go +++ b/pkg/sfu/connectionquality/scorer.go @@ -128,8 +128,8 @@ type qualityScorer struct { mutedAt time.Time unmutedAt time.Time - bitrateMutedAt time.Time - bitrateUnmutedAt time.Time + layerMutedAt time.Time + layerUnmutedAt time.Time maxPPS float64 @@ -173,11 +173,37 @@ func (q *qualityScorer) AddBitrateTransition(bitrate int64, at time.Time) { }) if bitrate == 0 { - q.bitrateMutedAt = at - q.score = maxScore + if !q.isLayerMuted() { + q.layerMutedAt = at + q.score = maxScore + } } else { - if q.bitrateUnmutedAt.IsZero() || q.bitrateMutedAt.After(q.bitrateUnmutedAt) { - q.bitrateUnmutedAt = at + if q.isLayerMuted() { + q.layerUnmutedAt = at + } + } +} + +func (q *qualityScorer) UpdateLayerMute(isMuted bool, at time.Time) { + q.lock.Lock() + defer q.lock.Unlock() + + if isMuted { + if !q.isLayerMuted() { + q.bitrateTransitions = append(q.bitrateTransitions, bitrateTransition{ + startedAt: at, + bitrate: 0, + }) + q.layerTransitions = append(q.layerTransitions, layerTransition{ + startedAt: at, + distance: 0.0, + }) + q.layerMutedAt = at + q.score = maxScore + } + } else { + if q.isLayerMuted() { + q.layerUnmutedAt = at } } } @@ -206,7 +232,7 @@ func (q *qualityScorer) Update(stat *windowStat, at time.Time) { // to stable and quality EXCELLENT for responsiveness. On an unmute, the // entire window data is considered (as long as enough time has passed since // unmute) including the data before mute. - if q.isMuted() || !q.isUnmutedEnough(at) || q.isBitrateMuted() { + if q.isMuted() || !q.isUnmutedEnough(at) || q.isLayerMuted() { q.lastUpdateAt = at return } @@ -274,16 +300,16 @@ func (q *qualityScorer) isUnmutedEnough(at time.Time) bool { sinceUnmute = at.Sub(q.unmutedAt) } - var sinceLayersUnmute time.Duration - if q.bitrateUnmutedAt.IsZero() { - sinceLayersUnmute = at.Sub(q.lastUpdateAt) + var sinceLayerUnmute time.Duration + if q.layerUnmutedAt.IsZero() { + sinceLayerUnmute = at.Sub(q.lastUpdateAt) } else { - sinceLayersUnmute = at.Sub(q.bitrateUnmutedAt) + sinceLayerUnmute = at.Sub(q.layerUnmutedAt) } validDuration := sinceUnmute - if sinceLayersUnmute < validDuration { - validDuration = sinceLayersUnmute + if sinceLayerUnmute < validDuration { + validDuration = sinceLayerUnmute } sinceLastUpdate := at.Sub(q.lastUpdateAt) @@ -291,8 +317,8 @@ func (q *qualityScorer) isUnmutedEnough(at time.Time) bool { return validDuration.Seconds()/sinceLastUpdate.Seconds() > unmuteTimeThreshold } -func (q *qualityScorer) isBitrateMuted() bool { - return !q.bitrateMutedAt.IsZero() && (q.bitrateUnmutedAt.IsZero() || q.bitrateMutedAt.After(q.bitrateUnmutedAt)) +func (q *qualityScorer) isLayerMuted() bool { + return !q.layerMutedAt.IsZero() && (q.layerUnmutedAt.IsZero() || q.layerMutedAt.After(q.layerUnmutedAt)) } func (q *qualityScorer) getPacketLossWeight(stat *windowStat) float64 { diff --git a/pkg/sfu/receiver.go b/pkg/sfu/receiver.go index ae7493fc5..52ba65fde 100644 --- a/pkg/sfu/receiver.go +++ b/pkg/sfu/receiver.go @@ -380,6 +380,7 @@ func (w *WebRTCReceiver) AddDownTrack(track TrackSender) error { track.TrackInfoAvailable() track.UpTrackMaxPublishedLayerChange(w.streamTrackerManager.GetMaxPublishedLayer()) + track.UpTrackMaxTemporalLayerSeenChange(w.streamTrackerManager.GetMaxTemporalLayerSeen()) w.downTrackSpreader.Store(track) return nil @@ -388,7 +389,13 @@ func (w *WebRTCReceiver) AddDownTrack(track TrackSender) error { func (w *WebRTCReceiver) SetMaxExpectedSpatialLayer(layer int32) { w.streamTrackerManager.SetMaxExpectedSpatialLayer(layer) - w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired(), time.Now()) + now := time.Now() + if layer == InvalidLayerSpatial { + w.connectionStats.UpdateLayerMute(true, now) + } else { + w.connectionStats.UpdateLayerMute(false, now) + w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired(), now) + } } // StreamTrackerManagerListener.OnAvailableLayersChanged diff --git a/pkg/sfu/streamtrackermanager.go b/pkg/sfu/streamtrackermanager.go index 6fd9585f7..dee95f5a7 100644 --- a/pkg/sfu/streamtrackermanager.go +++ b/pkg/sfu/streamtrackermanager.go @@ -321,15 +321,6 @@ done: return float64(distance) / float64(s.maxTemporalLayerSeen+1) } -func (s *StreamTrackerManager) getMaxExpectedLayerLocked() int32 { - // find min of layer - maxExpectedLayer := s.maxExpectedLayer - if maxExpectedLayer > s.maxPublishedLayer { - maxExpectedLayer = s.maxPublishedLayer - } - return maxExpectedLayer -} - func (s *StreamTrackerManager) GetMaxPublishedLayer() int32 { s.lock.RLock() defer s.lock.RUnlock() @@ -538,6 +529,13 @@ func (s *StreamTrackerManager) GetReferenceLayerRTPTimestamp(ts uint32, layer in return ts + (srRef.SenderReportData.RTPTimestamp - normalizedTS), nil } +func (s *StreamTrackerManager) GetMaxTemporalLayerSeen() int32 { + s.lock.RLock() + defer s.lock.RUnlock() + + return s.maxTemporalLayerSeen +} + func (s *StreamTrackerManager) updateMaxTemporalLayerSeen(brs Bitrates) { maxTemporalLayerSeen := InvalidLayerTemporal done: From 8635b0652f4ef525dc57d5cce32ea34054c0a739 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 15 Mar 2023 17:40:09 +0530 Subject: [PATCH 011/324] Start bit rate worker only for video tracks (#1523) --- pkg/sfu/streamtrackermanager.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/sfu/streamtrackermanager.go b/pkg/sfu/streamtrackermanager.go index dee95f5a7..9e95577aa 100644 --- a/pkg/sfu/streamtrackermanager.go +++ b/pkg/sfu/streamtrackermanager.go @@ -77,7 +77,9 @@ func NewStreamTrackerManager( s.maxExpectedLayerFromTrackInfo() - go s.bitrateReporter() + if s.trackInfo.Type == livekit.TrackType_VIDEO { + go s.bitrateReporter() + } return s } From b48fc21ab6ce6e895a04fc19ea1a832ed6294aa2 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 15 Mar 2023 10:40:20 -0700 Subject: [PATCH 012/324] Update actions/setup-go action to v4 (#1524) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/buildtest.yaml | 2 +- .github/workflows/docker.yaml | 2 +- .github/workflows/release.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/buildtest.yaml b/.github/workflows/buildtest.yaml index 4025f7fbb..fcf886631 100644 --- a/.github/workflows/buildtest.yaml +++ b/.github/workflows/buildtest.yaml @@ -19,7 +19,7 @@ jobs: - run: redis-cli ping - name: Set up Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: go-version: '1.18' diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 0fa3a1749..03c4ea14d 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -25,7 +25,7 @@ jobs: type=semver,pattern=v{{major}}.{{minor}} - name: Set up Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: go-version: '>=1.18' diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index d5ed958e6..a93876991 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -19,7 +19,7 @@ jobs: run: git fetch --force --tags - name: Set up Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: go-version: '>=1.18' From 7d857c95574ebd11b27565baf0ce78454b1ce307 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sat, 18 Mar 2023 10:25:20 +0530 Subject: [PATCH 013/324] Discount upstream + processing jitter from down stream jitter. (#1527) * Discount upstream + processing jitter from down stream jitter. Jitter in RTCP Receiver Report from down stream tracks includes jitter from up stream tracks and any processing in forwarding path. As packets are forwarded without any buffering (i. e. no de-jittering) in the SFU, any up stream jitter will carry forward. While taking delta stats (which is used for connection quality and reporting to analytics), discount the up stream + processing jitter so that connection quality score of down stream track is not penalized due to up stream + processing jitter. NOTE: Not discounting it in RTP stats ToString/ToProto methods as that information is useful to have for analysis/debugging. * fix typo --- pkg/sfu/buffer/rtpstats.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index c6ea61498..8a68a1232 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -801,7 +801,7 @@ func (r *RTPStats) SnapshotRtcpReceptionReport(ssrc uint32, proxyFracLost uint8, packetsLost := uint32(0) if r.params.IsReceiverReportDriven { - // should not be set for streams that need to generate reception report + // receiver report drvien should not be set for streams that need to generate reception report, but including code here for consistency packetsLost = now.packetsLostOverridden - then.packetsLostOverridden if int32(packetsLost) < 0 { packetsLost = 0 @@ -825,7 +825,7 @@ func (r *RTPStats) SnapshotRtcpReceptionReport(ssrc uint32, proxyFracLost uint8, jitter := r.jitter if r.params.IsReceiverReportDriven { - // should not be set for streams that need to generate reception report + // receiver report drvien should not be set for streams that need to generate reception report, but including code here for consistency jitter = r.jitterOverridden } @@ -910,7 +910,11 @@ func (r *RTPStats) DeltaInfo(snapshotId uint32) *RTPDeltaInfo { maxJitter := then.maxJitter if r.params.IsReceiverReportDriven { - maxJitter = then.maxJitterOverridden + // discount jitter from publisher side + internal processing + maxJitter = then.maxJitterOverridden - maxJitter + if maxJitter < 0.0 { + maxJitter = 0.0 + } } maxJitterTime := maxJitter / float64(r.params.ClockRate) * 1e6 @@ -970,6 +974,7 @@ func (r *RTPStats) ToString() string { jitter := r.jitter maxJitter := r.maxJitter if r.params.IsReceiverReportDriven { + // NOTE: jitter includes jitter from publisher and from processing jitter = r.jitterOverridden maxJitter = r.maxJitterOverridden } @@ -1043,6 +1048,7 @@ func (r *RTPStats) ToProto() *livekit.RTPStats { jitter := r.jitter maxJitter := r.maxJitter if r.params.IsReceiverReportDriven { + // NOTE: jitter includes jitter from publisher and from processing jitter = r.jitterOverridden maxJitter = r.maxJitterOverridden } @@ -1261,7 +1267,7 @@ func (r *RTPStats) updateJitter(rtph *rtp.Header, packetTime int64) { for _, s := range r.snapshots { if r.jitter > s.maxJitter { - r.maxJitter = r.jitter + s.maxJitter = r.jitter } } } From bbba3f81687abf7d68f99610b9a76dc7e589af1e Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sun, 19 Mar 2023 18:19:15 +0530 Subject: [PATCH 014/324] With opportunistic forwarding, no need to not remove layer 0 (#1529) --- pkg/sfu/streamtrackermanager.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/sfu/streamtrackermanager.go b/pkg/sfu/streamtrackermanager.go index 9e95577aa..332c4ed9f 100644 --- a/pkg/sfu/streamtrackermanager.go +++ b/pkg/sfu/streamtrackermanager.go @@ -426,8 +426,7 @@ func (s *StreamTrackerManager) removeAvailableLayer(layer int32) { newLayers := make([]int32, 0, DefaultMaxLayerSpatial+1) for _, l := range s.availableLayers { - // do not remove layers for non-simulcast - if l != layer || len(s.trackInfo.Layers) < 2 { + if l != layer { newLayers = append(newLayers, l) } } From aeefbb080eaee4c6827739e099866c548d241ddc Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sun, 19 Mar 2023 18:34:56 +0530 Subject: [PATCH 015/324] Account for time before measurement available in connection quality. (#1528) --- pkg/sfu/connectionquality/scorer.go | 3 ++ pkg/sfu/downtrack.go | 23 +++++---- pkg/sfu/forwarder.go | 72 +++++++++++++++++++++++++---- pkg/sfu/forwarder_test.go | 60 ++++++++++++------------ pkg/sfu/receiver.go | 2 + pkg/sfu/streamtrackermanager.go | 10 +++- 6 files changed, 119 insertions(+), 51 deletions(-) diff --git a/pkg/sfu/connectionquality/scorer.go b/pkg/sfu/connectionquality/scorer.go index 9e7a1b047..a1329dfca 100644 --- a/pkg/sfu/connectionquality/scorer.go +++ b/pkg/sfu/connectionquality/scorer.go @@ -124,6 +124,7 @@ type qualityScorer struct { lastUpdateAt time.Time score float64 + stat windowStat mutedAt time.Time unmutedAt time.Time @@ -276,6 +277,7 @@ func (q *qualityScorer) Update(stat *windowStat, at time.Time) { "reason", reason, "prevScore", q.score, "prevQuality", scoreToConnectionQuality(q.score), + "prevStat", q.stat, "score", score, "quality", scoreToConnectionQuality(score), "stat", stat, @@ -285,6 +287,7 @@ func (q *qualityScorer) Update(stat *windowStat, at time.Time) { } q.score = score + q.stat = *stat q.lastUpdateAt = at } diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 6bd4c0ea0..79abe27cb 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -954,8 +954,11 @@ func (d *DownTrack) maybeAddTransition(_bitrate int64, distance float64) { d.connectionStats.AddLayerTransition(distance, time.Now()) } -func (d *DownTrack) UpTrackBitrateReport(_availableLayers []int32, bitrates Bitrates) { - d.maybeAddTransition(d.forwarder.GetOptimalBandwidthNeeded(bitrates), d.forwarder.DistanceToDesired(bitrates)) +func (d *DownTrack) UpTrackBitrateReport(availableLayers []int32, bitrates Bitrates) { + d.maybeAddTransition( + d.forwarder.GetOptimalBandwidthNeeded(bitrates), + d.forwarder.DistanceToDesired(availableLayers, bitrates), + ) } // OnCloseHandler method to be called on remote tracked removed @@ -996,8 +999,8 @@ func (d *DownTrack) BandwidthRequested() int64 { } func (d *DownTrack) DistanceToDesired() float64 { - _, brs := d.receiver.GetLayeredBitrate() - return d.forwarder.DistanceToDesired(brs) + al, brs := d.receiver.GetLayeredBitrate() + return d.forwarder.DistanceToDesired(al, brs) } func (d *DownTrack) AllocateOptimal(allowOvershoot bool) VideoAllocation { @@ -1009,8 +1012,8 @@ func (d *DownTrack) AllocateOptimal(allowOvershoot bool) VideoAllocation { } func (d *DownTrack) ProvisionalAllocatePrepare() { - _, brs := d.receiver.GetLayeredBitrate() - d.forwarder.ProvisionalAllocatePrepare(brs) + al, brs := d.receiver.GetLayeredBitrate() + d.forwarder.ProvisionalAllocatePrepare(al, brs) } func (d *DownTrack) ProvisionalAllocate(availableChannelCapacity int64, layers VideoLayers, allowPause bool, allowOvershoot bool) int64 { @@ -1037,8 +1040,8 @@ func (d *DownTrack) ProvisionalAllocateCommit() VideoAllocation { } func (d *DownTrack) AllocateNextHigher(availableChannelCapacity int64, allowOvershoot bool) (VideoAllocation, bool) { - _, brs := d.receiver.GetLayeredBitrate() - allocation, available := d.forwarder.AllocateNextHigher(availableChannelCapacity, brs, allowOvershoot) + al, brs := d.receiver.GetLayeredBitrate() + allocation, available := d.forwarder.AllocateNextHigher(availableChannelCapacity, al, brs, allowOvershoot) d.maybeStartKeyFrameRequester() d.maybeAddTransition(allocation.bandwidthNeeded, allocation.distanceToDesired) return allocation, available @@ -1052,8 +1055,8 @@ func (d *DownTrack) GetNextHigherTransition(allowOvershoot bool) (VideoTransitio } func (d *DownTrack) Pause() VideoAllocation { - _, brs := d.receiver.GetLayeredBitrate() - allocation := d.forwarder.Pause(brs) + al, brs := d.receiver.GetLayeredBitrate() + allocation := d.forwarder.Pause(al, brs) d.maybeStartKeyFrameRequester() d.maybeAddTransition(allocation.bandwidthNeeded, allocation.distanceToDesired) return allocation diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index d75c890a7..c5dd4df47 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -98,6 +98,7 @@ type VideoAllocationProvisional struct { pubMuted bool maxPublishedLayer int32 maxTemporalLayerSeen int32 + availableLayers []int32 bitrates Bitrates maxLayers VideoLayers currentLayers VideoLayers @@ -497,11 +498,20 @@ func (f *Forwarder) BandwidthRequested(brs Bitrates) int64 { return brs[f.targetLayers.Spatial][f.targetLayers.Temporal] } -func (f *Forwarder) DistanceToDesired(brs Bitrates) float64 { +func (f *Forwarder) DistanceToDesired(availableLayers []int32, brs Bitrates) float64 { f.lock.RLock() defer f.lock.RUnlock() - return getDistanceToDesired(f.muted, f.pubMuted, f.maxPublishedLayer, f.maxTemporalLayerSeen, brs, f.targetLayers, f.maxLayers) + return getDistanceToDesired( + f.muted, + f.pubMuted, + f.maxPublishedLayer, + f.maxTemporalLayerSeen, + availableLayers, + brs, + f.targetLayers, + f.maxLayers, + ) } func (f *Forwarder) GetOptimalBandwidthNeeded(brs Bitrates) int64 { @@ -620,12 +630,21 @@ func (f *Forwarder) AllocateOptimal(availableLayers []int32, brs Bitrates, allow alloc.bandwidthRequested = optimalBandwidthNeeded } alloc.bandwidthDelta = alloc.bandwidthRequested - f.lastAllocation.bandwidthRequested - alloc.distanceToDesired = getDistanceToDesired(f.muted, f.pubMuted, f.maxPublishedLayer, f.maxTemporalLayerSeen, brs, alloc.targetLayers, f.maxLayers) + alloc.distanceToDesired = getDistanceToDesired( + f.muted, + f.pubMuted, + f.maxPublishedLayer, + f.maxTemporalLayerSeen, + availableLayers, + brs, + alloc.targetLayers, + f.maxLayers, + ) return f.updateAllocation(alloc, "optimal") } -func (f *Forwarder) ProvisionalAllocatePrepare(bitrates Bitrates) { +func (f *Forwarder) ProvisionalAllocatePrepare(availableLayers []int32, bitrates Bitrates) { f.lock.Lock() defer f.lock.Unlock() @@ -640,6 +659,9 @@ func (f *Forwarder) ProvisionalAllocatePrepare(bitrates Bitrates) { currentLayers: f.currentLayers, parkedLayers: f.parkedLayers, } + + f.provisional.availableLayers = make([]int32, len(availableLayers)) + copy(f.provisional.availableLayers, availableLayers) } func (f *Forwarder) ProvisionalAllocate(availableChannelCapacity int64, layers VideoLayers, allowPause bool, allowOvershoot bool) int64 { @@ -945,6 +967,7 @@ func (f *Forwarder) ProvisionalAllocateCommit() VideoAllocation { f.provisional.pubMuted, f.provisional.maxPublishedLayer, f.provisional.maxTemporalLayerSeen, + f.provisional.availableLayers, f.provisional.bitrates, f.provisional.allocatedLayers, f.provisional.maxLayers, @@ -1002,7 +1025,7 @@ func (f *Forwarder) ProvisionalAllocateCommit() VideoAllocation { return f.updateAllocation(alloc, "cooperative") } -func (f *Forwarder) AllocateNextHigher(availableChannelCapacity int64, brs Bitrates, allowOvershoot bool) (VideoAllocation, bool) { +func (f *Forwarder) AllocateNextHigher(availableChannelCapacity int64, availableLayers []int32, brs Bitrates, allowOvershoot bool) (VideoAllocation, bool) { f.lock.Lock() defer f.lock.Unlock() @@ -1053,7 +1076,16 @@ func (f *Forwarder) AllocateNextHigher(availableChannelCapacity int64, brs Bitra targetLayers: targetLayers, requestLayerSpatial: targetLayers.Spatial, maxLayers: f.maxLayers, - distanceToDesired: getDistanceToDesired(f.muted, f.pubMuted, f.maxPublishedLayer, f.maxTemporalLayerSeen, brs, targetLayers, f.maxLayers), + distanceToDesired: getDistanceToDesired( + f.muted, + f.pubMuted, + f.maxPublishedLayer, + f.maxTemporalLayerSeen, + availableLayers, + brs, + targetLayers, + f.maxLayers, + ), } if targetLayers.GreaterThan(f.maxLayers) || bandwidthRequested >= optimalBandwidthNeeded { alloc.isDeficient = false @@ -1187,7 +1219,7 @@ func (f *Forwarder) GetNextHigherTransition(brs Bitrates, allowOvershoot bool) ( return VideoTransition{}, false } -func (f *Forwarder) Pause(brs Bitrates) VideoAllocation { +func (f *Forwarder) Pause(availableLayers []int32, brs Bitrates) VideoAllocation { f.lock.Lock() defer f.lock.Unlock() @@ -1200,7 +1232,16 @@ func (f *Forwarder) Pause(brs Bitrates) VideoAllocation { targetLayers: InvalidLayers, requestLayerSpatial: InvalidLayerSpatial, maxLayers: f.maxLayers, - distanceToDesired: getDistanceToDesired(f.muted, f.pubMuted, f.maxPublishedLayer, f.maxTemporalLayerSeen, brs, InvalidLayers, f.maxLayers), + distanceToDesired: getDistanceToDesired( + f.muted, + f.pubMuted, + f.maxPublishedLayer, + f.maxTemporalLayerSeen, + availableLayers, + brs, + InvalidLayers, + f.maxLayers, + ), } switch { @@ -1703,6 +1744,7 @@ func getDistanceToDesired( pubMuted bool, maxPublishedLayer int32, maxTemporalLayerSeen int32, + availableLayers []int32, brs Bitrates, targetLayers VideoLayers, maxLayers VideoLayers, @@ -1713,11 +1755,13 @@ func getDistanceToDesired( adjustedMaxLayers := maxLayers + maxAvailableSpatial := InvalidLayerSpatial + maxAvailableTemporal := InvalidLayerTemporal + // max available spatial is min(subscribedMax, publishedMax, availableMax) // subscribedMax = subscriber requested max spatial layer // publishedMax = max spatial layer ever published // availableMax = based on bit rate measurement, available max spatial layer - maxAvailableSpatial := InvalidLayerSpatial done: for s := int32(len(brs)) - 1; s >= 0; s-- { for t := int32(len(brs[0])) - 1; t >= 0; t-- { @@ -1727,6 +1771,15 @@ done: } } } + + // before bit rate measurement is available, stream tracker could declare layer seen, account for that + for _, layer := range availableLayers { + if layer > maxAvailableSpatial { + maxAvailableSpatial = layer + maxAvailableTemporal = maxTemporalLayerSeen // till bit rate measurement is available, assume max seen as temporal + } + } + if maxAvailableSpatial < adjustedMaxLayers.Spatial { adjustedMaxLayers.Spatial = maxAvailableSpatial } @@ -1739,7 +1792,6 @@ done: // subscribedMax = subscriber requested max temporal layer // temporalLayerSeenMax = max temporal layer ever published/seen // availableMax = based on bit rate measurement, available max temporal in the adjusted max spatial layer - maxAvailableTemporal := InvalidLayerTemporal if adjustedMaxLayers.Spatial != InvalidLayerSpatial { for t := int32(len(brs[0])) - 1; t >= 0; t-- { if brs[adjustedMaxLayers.Spatial][t] != 0 { diff --git a/pkg/sfu/forwarder_test.go b/pkg/sfu/forwarder_test.go index 5398a5a61..2c3784ea3 100644 --- a/pkg/sfu/forwarder_test.go +++ b/pkg/sfu/forwarder_test.go @@ -312,7 +312,7 @@ func TestForwarderAllocateOptimal(t *testing.T) { targetLayers: DefaultMaxLayers, requestLayerSpatial: 2, maxLayers: DefaultMaxLayers, - distanceToDesired: -2.75, + distanceToDesired: -1.0, } result = f.AllocateOptimal([]int32{0, 1}, emptyBitrates, false) require.Equal(t, expectedResult, result) @@ -375,7 +375,7 @@ func TestForwarderAllocateOptimal(t *testing.T) { targetLayers: expectedTargetLayers, requestLayerSpatial: 0, maxLayers: f.maxLayers, - distanceToDesired: -0.25, + distanceToDesired: 0.0, } result = f.AllocateOptimal([]int32{0, 1}, emptyBitrates, true) require.Equal(t, expectedResult, result) @@ -397,7 +397,7 @@ func TestForwarderAllocateOptimal(t *testing.T) { targetLayers: expectedTargetLayers, requestLayerSpatial: 2, maxLayers: f.maxLayers, - distanceToDesired: -2.75, + distanceToDesired: -1.5, } result = f.AllocateOptimal([]int32{0, 1}, emptyBitrates, true) require.Equal(t, expectedResult, result) @@ -417,7 +417,7 @@ func TestForwarderProvisionalAllocate(t *testing.T) { {9, 10, 11, 12}, } - f.ProvisionalAllocatePrepare(bitrates) + f.ProvisionalAllocatePrepare(nil, bitrates) usedBitrate := f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 0, Temporal: 0}, true, false) require.Equal(t, bitrates[0][0], usedBitrate) @@ -458,7 +458,7 @@ func TestForwarderProvisionalAllocate(t *testing.T) { // when nothing fits and pausing disallowed, should allocate (0, 0) f.targetLayers = InvalidLayers - f.ProvisionalAllocatePrepare(bitrates) + f.ProvisionalAllocatePrepare(nil, bitrates) usedBitrate = f.ProvisionalAllocate(0, VideoLayers{Spatial: 0, Temporal: 0}, false, false) require.Equal(t, int64(1), usedBitrate) @@ -494,7 +494,7 @@ func TestForwarderProvisionalAllocate(t *testing.T) { {9, 10, 11, 12}, } - f.ProvisionalAllocatePrepare(bitrates) + f.ProvisionalAllocatePrepare(nil, bitrates) usedBitrate = f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 0, Temporal: 0}, false, true) require.Equal(t, int64(0), usedBitrate) @@ -540,7 +540,7 @@ func TestForwarderProvisionalAllocate(t *testing.T) { } f.currentLayers = VideoLayers{Spatial: 0, Temporal: 2} - f.ProvisionalAllocatePrepare(bitrates) + f.ProvisionalAllocatePrepare(nil, bitrates) // all the provisional allocations should not succeed because the feed is dry usedBitrate = f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 0, Temporal: 0}, false, true) @@ -578,7 +578,7 @@ func TestForwarderProvisionalAllocate(t *testing.T) { // Same case as above, but current is above max, so target should go to invalid // f.currentLayers = VideoLayers{Spatial: 1, Temporal: 2} - f.ProvisionalAllocatePrepare(bitrates) + f.ProvisionalAllocatePrepare(nil, bitrates) // all the provisional allocations below should not succeed because the feed is dry usedBitrate = f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 0, Temporal: 0}, false, true) @@ -621,7 +621,7 @@ func TestForwarderProvisionalAllocateMute(t *testing.T) { } f.Mute(true) - f.ProvisionalAllocatePrepare(bitrates) + f.ProvisionalAllocatePrepare(nil, bitrates) usedBitrate := f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 0, Temporal: 0}, true, false) require.Equal(t, int64(0), usedBitrate) @@ -659,7 +659,7 @@ func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { {9, 10, 0, 0}, } - f.ProvisionalAllocatePrepare(bitrates) + f.ProvisionalAllocatePrepare(nil, bitrates) // from scratch (InvalidLayers) should give back layer (0, 0) expectedTransition := VideoTransition{ @@ -732,7 +732,7 @@ func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { // mute f.Mute(true) - f.ProvisionalAllocatePrepare(bitrates) + f.ProvisionalAllocatePrepare(nil, bitrates) // mute should send target to InvalidLayers expectedTransition = VideoTransition{ @@ -758,7 +758,7 @@ func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { } f.targetLayers = InvalidLayers - f.ProvisionalAllocatePrepare(bitrates) + f.ProvisionalAllocatePrepare(nil, bitrates) // from scratch (InvalidLayers) should go to a layer past maximum as overshoot is allowed expectedTransition = VideoTransition{ @@ -797,7 +797,7 @@ func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { f.currentLayers = VideoLayers{Spatial: 0, Temporal: 2} f.targetLayers = InvalidLayers - f.ProvisionalAllocatePrepare(bitrates) + f.ProvisionalAllocatePrepare(nil, bitrates) // from scratch (InvalidLayers) should go to current layer // NOTE: targetLayer is set to InvalidLayers for testing, but in practice current layers valid and target layers invalid should not happen @@ -852,7 +852,7 @@ func TestForwarderProvisionalAllocateGetBestWeightedTransition(t *testing.T) { {9, 10, 11, 12}, } - f.ProvisionalAllocatePrepare(bitrates) + f.ProvisionalAllocatePrepare(nil, bitrates) f.targetLayers = VideoLayers{Spatial: 2, Temporal: 2} f.lastAllocation.bandwidthRequested = bitrates[2][2] @@ -878,7 +878,7 @@ func TestForwarderAllocateNextHigher(t *testing.T) { {0, 7, 0, 0}, } - result, boosted := f.AllocateNextHigher(ChannelCapacityInfinity, bitrates, false) + result, boosted := f.AllocateNextHigher(ChannelCapacityInfinity, nil, bitrates, false) require.Equal(t, VideoAllocationDefault, result) // no layer for audio require.False(t, boosted) @@ -889,7 +889,7 @@ func TestForwarderAllocateNextHigher(t *testing.T) { f.SetMaxTemporalLayerSeen(DefaultMaxLayerTemporal) // when not in deficient state, does not boost - result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, bitrates, false) + result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, nil, bitrates, false) require.Equal(t, VideoAllocationDefault, result) require.False(t, boosted) @@ -898,7 +898,7 @@ func TestForwarderAllocateNextHigher(t *testing.T) { Spatial: 0, Temporal: 0, } - result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, bitrates, false) + result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, nil, bitrates, false) require.Equal(t, VideoAllocationDefault, result) require.False(t, boosted) @@ -924,14 +924,14 @@ func TestForwarderAllocateNextHigher(t *testing.T) { maxLayers: DefaultMaxLayers, distanceToDesired: 2.0, } - result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, bitrates, false) + result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, nil, bitrates, false) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) require.Equal(t, expectedTargetLayers, f.TargetLayers()) require.True(t, boosted) // empty bitrates cannot increase layer, i. e. last allocation is left unchanged - result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, emptyBitrates, false) + result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, nil, emptyBitrates, false) require.Equal(t, expectedResult, result) require.False(t, boosted) @@ -952,7 +952,7 @@ func TestForwarderAllocateNextHigher(t *testing.T) { maxLayers: DefaultMaxLayers, distanceToDesired: 1.25, } - result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, bitrates, false) + result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, nil, bitrates, false) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) require.Equal(t, expectedTargetLayers, f.TargetLayers()) @@ -976,7 +976,7 @@ func TestForwarderAllocateNextHigher(t *testing.T) { maxLayers: DefaultMaxLayers, distanceToDesired: 0.5, } - result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, bitrates, false) + result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, nil, bitrates, false) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) require.Equal(t, expectedTargetLayers, f.TargetLayers()) @@ -998,7 +998,7 @@ func TestForwarderAllocateNextHigher(t *testing.T) { maxLayers: DefaultMaxLayers, distanceToDesired: 0.0, } - result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, bitrates, false) + result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, nil, bitrates, false) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) require.Equal(t, expectedTargetLayers, f.TargetLayers()) @@ -1007,7 +1007,7 @@ func TestForwarderAllocateNextHigher(t *testing.T) { // ask again, should return not boosted as there is no room to go higher f.currentLayers.Spatial = 2 f.currentLayers.Temporal = 1 - result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, bitrates, false) + result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, nil, bitrates, false) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) require.Equal(t, expectedTargetLayers, f.TargetLayers()) @@ -1033,7 +1033,7 @@ func TestForwarderAllocateNextHigher(t *testing.T) { maxLayers: DefaultMaxLayers, distanceToDesired: 2.25, } - result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, bitrates, false) + result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, nil, bitrates, false) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) require.Equal(t, expectedTargetLayers, f.TargetLayers()) @@ -1051,7 +1051,7 @@ func TestForwarderAllocateNextHigher(t *testing.T) { maxLayers: DefaultMaxLayers, distanceToDesired: 2.25, } - result, boosted = f.AllocateNextHigher(0, bitrates, false) + result, boosted = f.AllocateNextHigher(0, nil, bitrates, false) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) require.Equal(t, expectedTargetLayers, f.TargetLayers()) @@ -1086,7 +1086,7 @@ func TestForwarderAllocateNextHigher(t *testing.T) { distanceToDesired: -1.0, } // overshoot should return (1, 0) even if there is not enough capacity - result, boosted = f.AllocateNextHigher(bitrates[1][0]-1, bitrates, true) + result, boosted = f.AllocateNextHigher(bitrates[1][0]-1, nil, bitrates, true) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) require.Equal(t, expectedTargetLayers, f.TargetLayers()) @@ -1106,7 +1106,7 @@ func TestForwarderPause(t *testing.T) { {9, 10, 11, 12}, } - f.ProvisionalAllocatePrepare(bitrates) + f.ProvisionalAllocatePrepare(nil, bitrates) f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 0, Temporal: 0}, true, false) // should have set target at (0, 0) f.ProvisionalAllocateCommit() @@ -1123,7 +1123,7 @@ func TestForwarderPause(t *testing.T) { maxLayers: DefaultMaxLayers, distanceToDesired: 3, } - result := f.Pause(bitrates) + result := f.Pause(nil, bitrates) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) require.Equal(t, InvalidLayers, f.TargetLayers()) @@ -1141,7 +1141,7 @@ func TestForwarderPauseMute(t *testing.T) { {9, 10, 11, 12}, } - f.ProvisionalAllocatePrepare(bitrates) + f.ProvisionalAllocatePrepare(nil, bitrates) f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 0, Temporal: 0}, true, true) // should have set target at (0, 0) f.ProvisionalAllocateCommit() @@ -1157,7 +1157,7 @@ func TestForwarderPauseMute(t *testing.T) { maxLayers: DefaultMaxLayers, distanceToDesired: 0, } - result := f.Pause(bitrates) + result := f.Pause(nil, bitrates) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) require.Equal(t, InvalidLayers, f.TargetLayers()) diff --git a/pkg/sfu/receiver.go b/pkg/sfu/receiver.go index 52ba65fde..bc5ea1705 100644 --- a/pkg/sfu/receiver.go +++ b/pkg/sfu/receiver.go @@ -403,6 +403,8 @@ func (w *WebRTCReceiver) OnAvailableLayersChanged() { for _, dt := range w.downTrackSpreader.GetDownTracks() { dt.UpTrackLayersChange() } + + w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired(), time.Now()) } // StreamTrackerManagerListener.OnBitrateAvailabilityChanged diff --git a/pkg/sfu/streamtrackermanager.go b/pkg/sfu/streamtrackermanager.go index 332c4ed9f..d04b03a54 100644 --- a/pkg/sfu/streamtrackermanager.go +++ b/pkg/sfu/streamtrackermanager.go @@ -292,7 +292,7 @@ func (s *StreamTrackerManager) DistanceToDesired() float64 { return 0 } - _, brs := s.getLayeredBitrateLocked() + al, brs := s.getLayeredBitrateLocked() maxLayers := InvalidLayers done: @@ -308,6 +308,14 @@ done: } } + // before bit rate measurement is available, stream tracker could declare layer seen, account for that + for _, layer := range al { + if layer > maxLayers.Spatial { + maxLayers.Spatial = layer + maxLayers.Temporal = s.maxTemporalLayerSeen // till bit rate measurement is available, assume max seen as temporal + } + } + adjustedMaxLayers := maxLayers if !maxLayers.IsValid() { adjustedMaxLayers = VideoLayers{Spatial: 0, Temporal: 0} From f770f0cb6766fa4bdb77b1f217a8f985b675fec2 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sun, 19 Mar 2023 21:57:35 +0530 Subject: [PATCH 016/324] Use pointer to struct in logging (#1530) --- pkg/sfu/connectionquality/scorer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/sfu/connectionquality/scorer.go b/pkg/sfu/connectionquality/scorer.go index a1329dfca..1a2fb6825 100644 --- a/pkg/sfu/connectionquality/scorer.go +++ b/pkg/sfu/connectionquality/scorer.go @@ -277,7 +277,7 @@ func (q *qualityScorer) Update(stat *windowStat, at time.Time) { "reason", reason, "prevScore", q.score, "prevQuality", scoreToConnectionQuality(q.score), - "prevStat", q.stat, + "prevStat", &q.stat, "score", score, "quality", scoreToConnectionQuality(score), "stat", stat, From 65ad4b2c4319cf845144f86c28e0e5d4732f7d19 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 20 Mar 2023 12:22:08 +0530 Subject: [PATCH 017/324] Doing a pass at demoting logs (#1531) A few more candidates to think about demoting - Publisher mute changes - Forwarder -> layer lock/upgrade/downgrade/overshoot adjusting - StreamAllocator --- pkg/rtc/participant.go | 10 +++++----- pkg/rtc/participant_signal.go | 2 +- pkg/rtc/uptrackmanager.go | 2 +- pkg/sfu/buffer/fps.go | 2 +- pkg/sfu/buffer/rtpstats.go | 8 ++++---- pkg/sfu/forwarder.go | 18 +++++++++++------- pkg/sfu/streamtrackermanager.go | 4 ++-- 7 files changed, 25 insertions(+), 21 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 101eeecac..f2b7892d0 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -480,7 +480,7 @@ func (p *ParticipantImpl) OnClaimsChanged(callback func(types.LocalParticipant)) // HandleOffer an offer from remote participant, used when clients make the initial connection func (p *ParticipantImpl) HandleOffer(offer webrtc.SessionDescription) { - p.params.Logger.Infow("received offer", "transport", livekit.SignalTarget_PUBLISHER) + p.params.Logger.Debugw("received offer", "transport", livekit.SignalTarget_PUBLISHER) shouldPend := false if p.MigrateState() == types.MigrateStateInit { shouldPend = true @@ -494,7 +494,7 @@ func (p *ParticipantImpl) HandleOffer(offer webrtc.SessionDescription) { // HandleAnswer handles a client answer response, with subscriber PC, server initiates the // offer and client answers func (p *ParticipantImpl) HandleAnswer(answer webrtc.SessionDescription) { - p.params.Logger.Infow("received answer", "transport", livekit.SignalTarget_SUBSCRIBER) + p.params.Logger.Debugw("received answer", "transport", livekit.SignalTarget_SUBSCRIBER) /* from server received join request to client answer * 1. server send join response & offer @@ -508,7 +508,7 @@ func (p *ParticipantImpl) HandleAnswer(answer webrtc.SessionDescription) { } func (p *ParticipantImpl) onPublisherAnswer(answer webrtc.SessionDescription) error { - p.params.Logger.Infow("sending answer", "transport", livekit.SignalTarget_PUBLISHER) + p.params.Logger.Debugw("sending answer", "transport", livekit.SignalTarget_PUBLISHER) answer = p.configurePublisherAnswer(answer) if err := p.writeMessage(&livekit.SignalResponse{ Message: &livekit.SignalResponse_Answer{ @@ -1121,7 +1121,7 @@ func (p *ParticipantImpl) setIsPublisher(isPublisher bool) { // when the server has an offer for participant func (p *ParticipantImpl) onSubscriberOffer(offer webrtc.SessionDescription) error { - p.params.Logger.Infow("sending offer", "transport", livekit.SignalTarget_SUBSCRIBER) + p.params.Logger.Debugw("sending offer", "transport", livekit.SignalTarget_SUBSCRIBER) return p.writeMessage(&livekit.SignalResponse{ Message: &livekit.SignalResponse_Offer{ Offer: ToProtoSessionDescription(offer), @@ -1894,7 +1894,7 @@ func (p *ParticipantImpl) publisherRTCPWorker() { // read from rtcpChan for pkts := range p.rtcpCh { if pkts == nil { - p.params.Logger.Infow("exiting publisher RTCP worker") + p.params.Logger.Debugw("exiting publisher RTCP worker") return } diff --git a/pkg/rtc/participant_signal.go b/pkg/rtc/participant_signal.go index 60c5e4e79..cc1c0ea62 100644 --- a/pkg/rtc/participant_signal.go +++ b/pkg/rtc/participant_signal.go @@ -265,7 +265,7 @@ func (p *ParticipantImpl) writeMessage(msg *livekit.SignalResponse) error { sink := p.getResponseSink() if sink == nil { - p.params.Logger.Infow("could not send message to participant", "messageType", fmt.Sprintf("%T", msg.Message)) + p.params.Logger.Debugw("could not send message to participant", "messageType", fmt.Sprintf("%T", msg.Message)) return nil } diff --git a/pkg/rtc/uptrackmanager.go b/pkg/rtc/uptrackmanager.go index 93309a83a..76c380a5b 100644 --- a/pkg/rtc/uptrackmanager.go +++ b/pkg/rtc/uptrackmanager.go @@ -145,7 +145,7 @@ func (u *UpTrackManager) UpdateSubscriptionPermission( if u.subscriptionPermission != nil { perms = u.subscriptionPermission.String() } - u.params.Logger.Infow( + u.params.Logger.Debugw( "skipping older subscription permission version", "existingValue", perms, "existingVersion", u.subscriptionPermissionVersion.ToProto().String(), diff --git a/pkg/sfu/buffer/fps.go b/pkg/sfu/buffer/fps.go index 6efffe682..c44efc559 100644 --- a/pkg/sfu/buffer/fps.go +++ b/pkg/sfu/buffer/fps.go @@ -222,7 +222,7 @@ func (f *FrameRateCalculatorDD) RecvPacket(ep *ExtPacket) bool { } if ep.DependencyDescriptor == nil { - f.logger.Infow("dependency descriptor is nil") + f.logger.Debugw("dependency descriptor is nil") return false } diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index 8a68a1232..2a59a1c44 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -21,7 +21,7 @@ const ( FirstSnapshotId = 1 SnInfoSize = 2048 SnInfoMask = SnInfoSize - 1 - TooLargeOWD = 400 * time.Millisecond + TooLargeOWDDelta = 400 * time.Millisecond ) type RTPFlowState struct { @@ -704,8 +704,8 @@ func (r *RTPStats) SetRtcpSenderReportData(srData *RTCPSenderReportData) { owd := srData.ArrivalTime.Sub(srData.NTPTimestamp.Time()) if r.srDataExt != nil { prevOwd := r.srDataExt.SenderReportData.ArrivalTime.Sub(r.srDataExt.SenderReportData.NTPTimestamp.Time()) - if time.Duration(math.Abs(float64(owd)-float64(prevOwd))) > TooLargeOWD { - r.logger.Infow("large one-way-delay", "owd", owd, "prevOwd", prevOwd) + if time.Duration(math.Abs(float64(owd)-float64(prevOwd))) > TooLargeOWDDelta { + r.logger.Debugw("large delta in one-way-delay", "owd", owd, "prevOwd", prevOwd) } } @@ -756,7 +756,7 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32, srDataExt *RTCPSenderReportD smoothedLocalTimeOfLatestSenderReportNTP := srDataExt.SenderReportData.NTPTimestamp.Time().Add(srDataExt.SmoothedOWD) if smoothedLocalTimeOfLatestSenderReportNTP.After(now) { - r.logger.Infow("smoothed time of NTP is ahead", + r.logger.Debugw("smoothed time of NTP is ahead", "now", now, "smoothed", smoothedLocalTimeOfLatestSenderReportNTP, "diff", smoothedLocalTimeOfLatestSenderReportNTP.Sub(now), diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index c5dd4df47..296f5be3b 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -252,7 +252,7 @@ func (f *Forwarder) SetMaxPublishedLayer(maxPublishedLayer int32) { } f.maxPublishedLayer = maxPublishedLayer - f.logger.Infow("setting max published layer", "maxPublishedLayer", f.maxPublishedLayer) + f.logger.Debugw("setting max published layer", "maxPublishedLayer", f.maxPublishedLayer) } func (f *Forwarder) SetMaxTemporalLayerSeen(maxTemporalLayerSeen int32) { @@ -264,7 +264,7 @@ func (f *Forwarder) SetMaxTemporalLayerSeen(maxTemporalLayerSeen int32) { } f.maxTemporalLayerSeen = maxTemporalLayerSeen - f.logger.Infow("setting max temporal layer seen", "maxTemporalLayerSeen", f.maxTemporalLayerSeen) + f.logger.Debugw("setting max temporal layer seen", "maxTemporalLayerSeen", f.maxTemporalLayerSeen) } func (f *Forwarder) OnParkedLayersExpired(fn func()) { @@ -414,7 +414,7 @@ func (f *Forwarder) SetMaxSpatialLayer(spatialLayer int32) (bool, VideoLayers, V return false, f.maxLayers, f.currentLayers } - f.logger.Infow("setting max spatial layer", "layer", spatialLayer) + f.logger.Debugw("setting max spatial layer", "layer", spatialLayer) f.maxLayers.Spatial = spatialLayer f.clearParkedLayers() @@ -430,7 +430,7 @@ func (f *Forwarder) SetMaxTemporalLayer(temporalLayer int32) (bool, VideoLayers, return false, f.maxLayers, f.currentLayers } - f.logger.Infow("setting max temporal layer", "layer", temporalLayer) + f.logger.Debugw("setting max temporal layer", "layer", temporalLayer) f.maxLayers.Temporal = temporalLayer f.clearParkedLayers() @@ -1269,7 +1269,11 @@ func (f *Forwarder) updateAllocation(alloc VideoAllocation, reason string) Video alloc.pauseReason != f.lastAllocation.pauseReason || alloc.targetLayers != f.lastAllocation.targetLayers || alloc.requestLayerSpatial != f.lastAllocation.requestLayerSpatial { - f.logger.Infow(fmt.Sprintf("stream allocation: %s", reason), "allocation", alloc) + if reason == "optimal" { + f.logger.Debugw(fmt.Sprintf("stream allocation: %s", reason), "allocation", alloc) + } else { + f.logger.Infow(fmt.Sprintf("stream allocation: %s", reason), "allocation", alloc) + } } f.lastAllocation = alloc @@ -1416,11 +1420,11 @@ func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer i last := f.rtpMunger.GetLast() td = refTS - last.LastTS if td == 0 || td > (1<<31) { - f.logger.Infow("reference timestamp out-of-order, using default", "lastTS", last.LastTS, "refTS", refTS, "td", int32(td)) + f.logger.Debugw("reference timestamp out-of-order, using default", "lastTS", last.LastTS, "refTS", refTS, "td", int32(td)) td = 1 } } else { - f.logger.Infow("reference timestamp get error, using default", "error", err) + f.logger.Debugw("reference timestamp get error, using default", "error", err) } } diff --git a/pkg/sfu/streamtrackermanager.go b/pkg/sfu/streamtrackermanager.go index d04b03a54..b2d1f894a 100644 --- a/pkg/sfu/streamtrackermanager.go +++ b/pkg/sfu/streamtrackermanager.go @@ -409,7 +409,7 @@ func (s *StreamTrackerManager) addAvailableLayer(layer int32) { // check if new layer is the max layer isMaxLayerChange := s.availableLayers[len(s.availableLayers)-1] == layer - s.logger.Infow( + s.logger.Debugw( "available layers changed - layer seen", "added", layer, "availableLayers", s.availableLayers, @@ -441,7 +441,7 @@ func (s *StreamTrackerManager) removeAvailableLayer(layer int32) { sort.Slice(newLayers, func(i, j int) bool { return newLayers[i] < newLayers[j] }) s.availableLayers = newLayers - s.logger.Infow( + s.logger.Debugw( "available layers changed - layer gone", "removed", layer, "availableLayers", newLayers, From e8c7506d601b519c7a620fb90965cbad431f848a Mon Sep 17 00:00:00 2001 From: David Colburn Date: Mon, 20 Mar 2023 13:46:47 -0700 Subject: [PATCH 018/324] update deprecated egress client warning (#1533) --- pkg/service/egress.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/service/egress.go b/pkg/service/egress.go index 1b2c97ec3..80008f82b 100644 --- a/pkg/service/egress.go +++ b/pkg/service/egress.go @@ -186,7 +186,7 @@ func (s *egressLauncher) StartEgressWithClusterId(ctx context.Context, clusterId if s.psrpcClient != nil { info, err = s.psrpcClient.StartEgress(ctx, clusterId, req) } else { - logger.Infow("using deprecated egress client") + logger.Warnw("Using deprecated egress client. Upgrade egress to v1.5.6+ and use egress:use_psrpc:true in your livekit config", nil) // SendRequest will transform rpc.StartEgressRequest into deprecated livekit.StartEgressRequest info, err = s.clientDeprecated.SendRequest(ctx, req) } From 1a78dba3e02d9d45bd0fb0b4c639b91d5ff23188 Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Tue, 21 Mar 2023 09:50:43 +0800 Subject: [PATCH 019/324] Detect client short ice connection (#1532) --- pkg/rtc/participant.go | 2 +- pkg/rtc/transport.go | 4 ++-- pkg/rtc/transportmanager.go | 22 +++++++++++++++++++++- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index f2b7892d0..50c4f3bce 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -793,7 +793,7 @@ func (p *ParticipantImpl) ICERestart(iceConfig *livekit.ICEConfig, reason liveki t.(types.LocalMediaTrack).Restart() } - p.TransportManager.ICERestart(iceConfig, reason == livekit.ReconnectReason_RR_PUBLISHER_FAILED || reason == livekit.ReconnectReason_RR_SUBSCRIBER_FAILED) + p.TransportManager.ICERestart(iceConfig, reason) } func (p *ParticipantImpl) OnICEConfigChanged(f func(participant types.LocalParticipant, iceConfig *livekit.ICEConfig)) { diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index cbc70ba2c..c4afdd558 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -495,7 +495,7 @@ func (t *PCTransport) resetShortConn() { t.lock.Unlock() } -func (t *PCTransport) isShortConnection(at time.Time) (bool, time.Duration) { +func (t *PCTransport) IsShortConnection(at time.Time) (bool, time.Duration) { t.lock.RLock() defer t.lock.RUnlock() @@ -568,7 +568,7 @@ func (t *PCTransport) handleConnectionFailed(forceShortConn bool) { isShort := forceShortConn if !isShort { var duration time.Duration - isShort, duration = t.isShortConnection(time.Now()) + isShort, duration = t.IsShortConnection(time.Now()) if isShort { pair, err := t.getSelectedPair() if err != nil { diff --git a/pkg/rtc/transportmanager.go b/pkg/rtc/transportmanager.go index d4d347ba3..b3002212e 100644 --- a/pkg/rtc/transportmanager.go +++ b/pkg/rtc/transportmanager.go @@ -456,11 +456,31 @@ func (t *TransportManager) NegotiateSubscriber(force bool) { t.subscriber.Negotiate(force) } -func (t *TransportManager) ICERestart(iceConfig *livekit.ICEConfig, resetShortConnection bool) { +func (t *TransportManager) ICERestart(iceConfig *livekit.ICEConfig, reason livekit.ReconnectReason) { if iceConfig != nil { t.SetICEConfig(iceConfig) } + var ( + isShort bool + duration time.Duration + resetShortConnection bool + ) + switch reason { + case livekit.ReconnectReason_RR_PUBLISHER_FAILED: + resetShortConnection = true + isShort, duration = t.publisher.IsShortConnection(time.Now()) + + case livekit.ReconnectReason_RR_SUBSCRIBER_FAILED: + resetShortConnection = true + isShort, duration = t.subscriber.IsShortConnection(time.Now()) + } + + if isShort { + t.params.Logger.Infow("short connection by client ice restart", "duration", duration, "reason", reason) + t.handleConnectionFailed(isShort) + } + if resetShortConnection { t.publisher.ResetShortConnOnICERestart() t.subscriber.ResetShortConnOnICERestart() From c76c35474cbc97bf87a4504c93874970541458b4 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 21 Mar 2023 11:50:47 +0530 Subject: [PATCH 020/324] Init RTT/jitter in snapshot, else get 0 some times (#1534) --- pkg/sfu/buffer/rtpstats.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index 2a59a1c44..46af6d5e9 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -1313,9 +1313,9 @@ func (r *RTPStats) getAndResetSnapshot(snapshotId uint32) (*Snapshot, *Snapshot) nacks: r.nacks, plis: r.plis, firs: r.firs, - maxJitter: 0.0, - maxJitterOverridden: 0.0, - maxRtt: 0, + maxJitter: r.jitter, + maxJitterOverridden: r.jitterOverridden, + maxRtt: r.rtt, } // make a copy so that it can be used independently now := *r.snapshots[snapshotId] From f782c8956d65d4f11ae97036b0c2df0894348faf Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 22 Mar 2023 11:36:30 +0530 Subject: [PATCH 021/324] Extend range of `GOOD` scores. (#1536) Empirically, the experience is not bad for a larger range. So, triggering POOR too early causes confusion. --- pkg/sfu/connectionquality/connectionstats.go | 10 ++-- .../connectionquality/connectionstats_test.go | 59 ++++++++++--------- pkg/sfu/connectionquality/scorer.go | 32 +++++++--- 3 files changed, 58 insertions(+), 43 deletions(-) diff --git a/pkg/sfu/connectionquality/connectionstats.go b/pkg/sfu/connectionquality/connectionstats.go index d2e4e30a3..98562ae1e 100644 --- a/pkg/sfu/connectionquality/connectionstats.go +++ b/pkg/sfu/connectionquality/connectionstats.go @@ -252,23 +252,23 @@ func getPacketLossWeight(mimeType string, isFecEnabled bool) float64 { plw := float64(0.0) switch { case strings.EqualFold(mimeType, webrtc.MimeTypeOpus): - // 2.5%: fall to GOOD, 5%: fall to POOR + // 2.5%: fall to GOOD, 7.5%: fall to POOR plw = 8.0 if isFecEnabled { - // 3.75%: fall to GOOD, 7.5%: fall to POOR + // 3.75%: fall to GOOD, 11.25%: fall to POOR plw /= 1.5 } case strings.EqualFold(mimeType, "audio/red"): - // 6.66%: fall to GOOD, 13.33%: fall to POOR + // 6.66%: fall to GOOD, 20.0%: fall to POOR plw = 3.0 if isFecEnabled { - // 10%: fall to GOOD, 20%: fall to POOR + // 10%: fall to GOOD, 30.0%: fall to POOR plw /= 1.5 } case strings.HasPrefix(strings.ToLower(mimeType), "video/"): - // 2%: fall to GOOD, 4%: fall to POOR + // 2%: fall to GOOD, 6%: fall to POOR plw = 10.0 } diff --git a/pkg/sfu/connectionquality/connectionstats_test.go b/pkg/sfu/connectionquality/connectionstats_test.go index 91667cb39..567ef485e 100644 --- a/pkg/sfu/connectionquality/connectionstats_test.go +++ b/pkg/sfu/connectionquality/connectionstats_test.go @@ -63,11 +63,12 @@ func TestConnectionQuality(t *testing.T) { } cs.updateScore(streams, now.Add(duration)) mos, quality = cs.GetScoreAndQuality() - require.Greater(t, float32(3.2), mos) + require.Greater(t, float32(2.1), mos) require.Equal(t, livekit.ConnectionQuality_POOR, quality) - // should stay at POOR quality for one iteration even if the conditions improve - // due to significant loss (12%) in the previous window + // should climb to GOOD quality in one iteration if the conditions improve. + // although significant loss (12%) in the previous window, lowest score is + // bound so that climbing back does not take too long even under excellent conditions. now = now.Add(duration) streams = map[uint32]*buffer.StreamStatsWithLayers{ 1: &buffer.StreamStatsWithLayers{ @@ -80,10 +81,10 @@ func TestConnectionQuality(t *testing.T) { } cs.updateScore(streams, now.Add(duration)) mos, quality = cs.GetScoreAndQuality() - require.Greater(t, float32(3.2), mos) - require.Equal(t, livekit.ConnectionQuality_POOR, quality) + require.Greater(t, float32(4.1), mos) + require.Equal(t, livekit.ConnectionQuality_GOOD, quality) - // should climb up to GOOD if conditions continue to be good + // should stay at GOOD if conditions continue to be good now = now.Add(duration) streams = map[uint32]*buffer.StreamStatsWithLayers{ 1: &buffer.StreamStatsWithLayers{ @@ -178,7 +179,7 @@ func TestConnectionQuality(t *testing.T) { } cs.updateScore(streams, now.Add(duration)) mos, quality = cs.GetScoreAndQuality() - require.Greater(t, float32(3.2), mos) + require.Greater(t, float32(2.1), mos) require.Equal(t, livekit.ConnectionQuality_POOR, quality) now = now.Add(duration) @@ -218,7 +219,7 @@ func TestConnectionQuality(t *testing.T) { } cs.updateScore(streams, now.Add(duration)) mos, quality = cs.GetScoreAndQuality() - require.Greater(t, float32(3.2), mos) + require.Greater(t, float32(2.1), mos) require.Equal(t, livekit.ConnectionQuality_POOR, quality) // mute/unmute to bring quality back up @@ -357,7 +358,7 @@ func TestConnectionQuality(t *testing.T) { expectedQualities []expectedQuality }{ // NOTE: Because of EWMA (Exponentially Weighted Moving Average), these cut off points are not exact - // "audio/opus" - no fec - 0 <= loss < 2.5%: EXCELLENT, 2.5% <= loss < 5%: GOOD, >= 5%: POOR + // "audio/opus" - no fec - 0 <= loss < 2.5%: EXCELLENT, 2.5% <= loss < 7.5%: GOOD, >= 7.5%: POOR { name: "audio/opus - no fec", mimeType: "audio/opus", @@ -375,13 +376,13 @@ func TestConnectionQuality(t *testing.T) { expectedQuality: livekit.ConnectionQuality_GOOD, }, { - packetLossPercentage: 5.2, - expectedMOS: 3.2, + packetLossPercentage: 9.2, + expectedMOS: 2.1, expectedQuality: livekit.ConnectionQuality_POOR, }, }, }, - // "audio/opus" - fec - 0 <= loss < 3.75%: EXCELLENT, 3.75% <= loss < 7.5%: GOOD, >= 7.5%: POOR + // "audio/opus" - fec - 0 <= loss < 3.75%: EXCELLENT, 3.75% <= loss < 11.25%: GOOD, >= 11.25%: POOR { name: "audio/opus - fec", mimeType: "audio/opus", @@ -399,13 +400,13 @@ func TestConnectionQuality(t *testing.T) { expectedQuality: livekit.ConnectionQuality_GOOD, }, { - packetLossPercentage: 8.2, - expectedMOS: 3.2, + packetLossPercentage: 13.2, + expectedMOS: 2.1, expectedQuality: livekit.ConnectionQuality_POOR, }, }, }, - // "audio/red" - no fec - 0 <= loss < 6.66%: EXCELLENT, 6.66% <= loss < 13.33%: GOOD, >= 13.33%: POOR + // "audio/red" - no fec - 0 <= loss < 6.66%: EXCELLENT, 6.66% <= loss < 20%: GOOD, >= 20%: POOR { name: "audio/red - no fec", mimeType: "audio/red", @@ -423,13 +424,13 @@ func TestConnectionQuality(t *testing.T) { expectedQuality: livekit.ConnectionQuality_GOOD, }, { - packetLossPercentage: 16.0, - expectedMOS: 3.2, + packetLossPercentage: 23.0, + expectedMOS: 2.1, expectedQuality: livekit.ConnectionQuality_POOR, }, }, }, - // "audio/red" - fec - 0 <= loss < 10%: EXCELLENT, 10% <= loss < 20%: GOOD, >= 20%: POOR + // "audio/red" - fec - 0 <= loss < 10%: EXCELLENT, 10% <= loss < 30%: GOOD, >= 30%: POOR { name: "audio/red - fec", mimeType: "audio/red", @@ -447,13 +448,13 @@ func TestConnectionQuality(t *testing.T) { expectedQuality: livekit.ConnectionQuality_GOOD, }, { - packetLossPercentage: 22.0, - expectedMOS: 3.2, + packetLossPercentage: 36.0, + expectedMOS: 2.1, expectedQuality: livekit.ConnectionQuality_POOR, }, }, }, - // "video/*" - 0 <= loss < 2%: EXCELLENT, 2% <= loss < 4%: GOOD, >= 4%: POOR + // "video/*" - 0 <= loss < 2%: EXCELLENT, 2% <= loss < 6%: GOOD, >= 6%: POOR { name: "video/*", mimeType: "video/vp8", @@ -471,8 +472,8 @@ func TestConnectionQuality(t *testing.T) { expectedQuality: livekit.ConnectionQuality_GOOD, }, { - packetLossPercentage: 5.0, - expectedMOS: 3.2, + packetLossPercentage: 7.0, + expectedMOS: 2.1, expectedQuality: livekit.ConnectionQuality_POOR, }, }, @@ -523,8 +524,8 @@ func TestConnectionQuality(t *testing.T) { }{ // NOTE: Because of EWMA (Exponentially Weighted Moving Average), these cut off points are not exact // 1.0 <= expectedBits / actualBits < ~2.7 = EXCELLENT - // ~2.7 <= expectedBits / actualBits < ~7.5 = GOOD - // expectedBits / actualBits >= ~7.5 = POOR + // ~2.7 <= expectedBits / actualBits < ~20.1 = GOOD + // expectedBits / actualBits >= ~20.1 = POOR { name: "excellent", transitions: []transition{ @@ -566,8 +567,8 @@ func TestConnectionQuality(t *testing.T) { offset: 3 * time.Second, }, }, - bytes: uint64(math.Ceil(8_000_000.0 / 8.0 / 13.0)), - expectedMOS: 3.2, + bytes: uint64(math.Ceil(8_000_000.0 / 8.0 / 43.0)), + expectedMOS: 2.1, expectedQuality: livekit.ConnectionQuality_POOR, }, } @@ -650,11 +651,11 @@ func TestConnectionQuality(t *testing.T) { distance: 2.0, }, { - distance: 2.0, + distance: 2.2, offset: 1 * time.Second, }, }, - expectedMOS: 3.2, + expectedMOS: 2.1, expectedQuality: livekit.ConnectionQuality_POOR, }, } diff --git a/pkg/sfu/connectionquality/scorer.go b/pkg/sfu/connectionquality/scorer.go index 1a2fb6825..5582a7832 100644 --- a/pkg/sfu/connectionquality/scorer.go +++ b/pkg/sfu/connectionquality/scorer.go @@ -14,12 +14,13 @@ const ( MaxMOS = float32(4.5) maxScore = float64(100.0) - poorScore = float64(50.0) + poorScore = float64(30.0) + minScore = float64(20.0) increaseFactor = float64(0.4) // slow increase decreaseFactor = float64(0.8) // fast decrease - distanceWeight = float64(25.0) // each spatial layer missed drops a quality level + distanceWeight = float64(35.0) // each spatial layer missed drops a quality level unmuteTimeThreshold = float64(0.5) ) @@ -75,7 +76,7 @@ func (w *windowStat) calculateBitrateScore(expectedBitrate int64) float64 { if w.bytes != 0 { // using the ratio of expectedBitrate / actualBitrate // the quality inflection points are approximately - // GOOD at ~2.7x, POOR at ~7.5x + // GOOD at ~2.7x, POOR at ~20.1x score = maxScore - 20*math.Log(float64(expectedBitrate)/float64(w.bytes*8)) if score > maxScore { score = maxScore @@ -264,12 +265,19 @@ func (q *qualityScorer) Update(stat *windowStat, at time.Time) { reason = "layer" score = layerScore } + + factor := increaseFactor + if score < q.score { + factor = decreaseFactor + } + score = factor*score + (1.0-factor)*q.score } - factor := increaseFactor - if score < q.score { - factor = decreaseFactor + if score < minScore { + // lower bound to prevent score from becoming very small values due to extreme conditions. + // Without a lower bound, it can get so low that it takes a long time to climb back to + // better quality even under excellent conditions. + score = minScore } - score = factor*score + (1.0-factor)*q.score // WARNING NOTE: comparing protobuf enum values directly (livekit.ConnectionQuality) if scoreToConnectionQuality(q.score) > scoreToConnectionQuality(score) { q.params.Logger.Infow( @@ -453,13 +461,19 @@ func (q *qualityScorer) GetMOSAndQuality() (float32, livekit.ConnectionQuality) // ------------------------------------------ func scoreToConnectionQuality(score float64) livekit.ConnectionQuality { - // R-factor -> livekit.ConnectionQuality scale mapping based on + // R-factor -> livekit.ConnectionQuality scale mapping roughly based on // https://www.itu.int/ITU-T/2005-2008/com12/emodelv1/tut.htm + // + // As there are only three levels in livekit.ConnectionQuality scale, + // using a larger range for middling quality. Empirical evidence suggests + // that a score of 60 does not correspond to `POOR` quality. Repair + // mechanisms and use of algorithms like de-jittering makes the experience + // better even under harsh conditions. if score > 80.0 { return livekit.ConnectionQuality_EXCELLENT } - if score > 60.0 { + if score > 40.0 { return livekit.ConnectionQuality_GOOD } From e7c5872758528780b45cefcf18e10b5d61da819a Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 22 Mar 2023 11:59:32 +0530 Subject: [PATCH 022/324] Dependent RTT/jitter control. (#1537) --- pkg/sfu/connectionquality/connectionstats.go | 18 +++-- .../connectionquality/connectionstats_test.go | 74 +++++++++++++++++-- pkg/sfu/connectionquality/scorer.go | 23 ++++-- pkg/sfu/downtrack.go | 9 ++- pkg/sfu/receiver.go | 9 ++- 5 files changed, 105 insertions(+), 28 deletions(-) diff --git a/pkg/sfu/connectionquality/connectionstats.go b/pkg/sfu/connectionquality/connectionstats.go index 98562ae1e..8608da3d6 100644 --- a/pkg/sfu/connectionquality/connectionstats.go +++ b/pkg/sfu/connectionquality/connectionstats.go @@ -21,11 +21,13 @@ const ( ) type ConnectionStatsParams struct { - UpdateInterval time.Duration - MimeType string - IsFECEnabled bool - GetDeltaStats func() map[uint32]*buffer.StreamStatsWithLayers - Logger logger.Logger + UpdateInterval time.Duration + MimeType string + IsFECEnabled bool + IsDependentRTT bool + IsDependentJitter bool + GetDeltaStats func() map[uint32]*buffer.StreamStatsWithLayers + Logger logger.Logger } type ConnectionStats struct { @@ -49,8 +51,10 @@ func NewConnectionStats(params ConnectionStatsParams) *ConnectionStats { return &ConnectionStats{ params: params, scorer: newQualityScorer(qualityScorerParams{ - PacketLossWeight: getPacketLossWeight(params.MimeType, params.IsFECEnabled), // LK-TODO: have to notify codec change? - Logger: params.Logger, + PacketLossWeight: getPacketLossWeight(params.MimeType, params.IsFECEnabled), // LK-TODO: have to notify codec change? + IsDependentRTT: params.IsDependentRTT, + IsDependentJitter: params.IsDependentJitter, + Logger: params.Logger, }), done: core.NewFuse(), } diff --git a/pkg/sfu/connectionquality/connectionstats_test.go b/pkg/sfu/connectionquality/connectionstats_test.go index 567ef485e..075d24dbc 100644 --- a/pkg/sfu/connectionquality/connectionstats_test.go +++ b/pkg/sfu/connectionquality/connectionstats_test.go @@ -11,17 +11,19 @@ import ( "github.com/stretchr/testify/require" ) -func newConnectionStats(mimeType string, isFECEnabled bool) *ConnectionStats { +func newConnectionStats(mimeType string, isFECEnabled bool, isDependentRTT bool, isDependentJitter bool) *ConnectionStats { return NewConnectionStats(ConnectionStatsParams{ - MimeType: mimeType, - IsFECEnabled: isFECEnabled, - Logger: logger.GetLogger(), + MimeType: mimeType, + IsFECEnabled: isFECEnabled, + IsDependentRTT: isDependentRTT, + IsDependentJitter: isDependentJitter, + Logger: logger.GetLogger(), }) } func TestConnectionQuality(t *testing.T) { t.Run("quality scorer state machine", func(t *testing.T) { - cs := newConnectionStats("audio/opus", false) + cs := newConnectionStats("audio/opus", false, false, false) duration := 5 * time.Second now := time.Now() @@ -344,6 +346,62 @@ func TestConnectionQuality(t *testing.T) { require.Equal(t, livekit.ConnectionQuality_GOOD, quality) }) + t.Run("quality scorer dependent rtt", func(t *testing.T) { + cs := newConnectionStats("audio/opus", false, true, false) + + duration := 5 * time.Second + now := time.Now() + cs.Start(&livekit.TrackInfo{Type: livekit.TrackType_AUDIO}, now.Add(-duration)) + cs.UpdateMute(false, now.Add(-1*time.Second)) + + // RTT does not knock quality down because it is dependent and hence not taken into account + // at 2% loss, quality should stay at EXCELLENT purely based on loss. With high RTT (700 ms) + // quality should drop to GOOD if RTT were taken into consieration + streams := map[uint32]*buffer.StreamStatsWithLayers{ + 1: &buffer.StreamStatsWithLayers{ + RTPStats: &buffer.RTPDeltaInfo{ + StartTime: now, + Duration: duration, + Packets: 250, + PacketsLost: 5, + RttMax: 700, + }, + }, + } + cs.updateScore(streams, now.Add(duration)) + mos, quality := cs.GetScoreAndQuality() + require.Greater(t, float32(4.6), mos) + require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality) + }) + + t.Run("quality scorer dependent jitter", func(t *testing.T) { + cs := newConnectionStats("audio/opus", false, false, true) + + duration := 5 * time.Second + now := time.Now() + cs.Start(&livekit.TrackInfo{Type: livekit.TrackType_AUDIO}, now.Add(-duration)) + cs.UpdateMute(false, now.Add(-1*time.Second)) + + // Jitter does not knock quality down because it is dependent and hence not taken into account + // at 2% loss, quality should stay at EXCELLENT purely based on loss. With high jitter (200 ms) + // quality should drop to GOOD if jitter were taken into consieration + streams := map[uint32]*buffer.StreamStatsWithLayers{ + 1: &buffer.StreamStatsWithLayers{ + RTPStats: &buffer.RTPDeltaInfo{ + StartTime: now, + Duration: duration, + Packets: 250, + PacketsLost: 5, + JitterMax: 200, + }, + }, + } + cs.updateScore(streams, now.Add(duration)) + mos, quality := cs.GetScoreAndQuality() + require.Greater(t, float32(4.6), mos) + require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality) + }) + t.Run("codecs - packet", func(t *testing.T) { type expectedQuality struct { packetLossPercentage float64 @@ -482,7 +540,7 @@ func TestConnectionQuality(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - cs := newConnectionStats(tc.mimeType, tc.isFECEnabled) + cs := newConnectionStats(tc.mimeType, tc.isFECEnabled, false, false) duration := 5 * time.Second now := time.Now() @@ -575,7 +633,7 @@ func TestConnectionQuality(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - cs := newConnectionStats("video/vp8", false) + cs := newConnectionStats("video/vp8", false, false, false) duration := 5 * time.Second now := time.Now() @@ -662,7 +720,7 @@ func TestConnectionQuality(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - cs := newConnectionStats("video/vp8", false) + cs := newConnectionStats("video/vp8", false, false, false) duration := 5 * time.Second now := time.Now() diff --git a/pkg/sfu/connectionquality/scorer.go b/pkg/sfu/connectionquality/scorer.go index 5582a7832..d89329cfe 100644 --- a/pkg/sfu/connectionquality/scorer.go +++ b/pkg/sfu/connectionquality/scorer.go @@ -38,10 +38,21 @@ type windowStat struct { jitterMax float64 } -func (w *windowStat) calculatePacketScore(plw float64) float64 { +func (w *windowStat) calculatePacketScore(plw float64, isDependentRTT bool, isDependentJitter bool) float64 { // this is based on simplified E-model based on packet loss, rtt, jitter as // outlined at https://www.pingman.com/kb/article/how-is-mos-calculated-in-pingplotter-pro-50.html. - effectiveDelay := (float64(w.rttMax) / 2.0) + ((w.jitterMax * 2.0) / 1000.0) + effectiveDelay := 0.0 + // discount the dependent factors if dependency indicated. + // for example, + // 1. in the up stream, RTT cannot be measured without RTCP-XR, it is using down stream RTT. + // 2. in the down stream, up stream jitter affects it. although jitter can be adjusted to account for up stream + // jitter, this lever can be used to discount jitter in scoring. + if !isDependentRTT { + effectiveDelay += float64(w.rttMax) / 2.0 + } + if !isDependentJitter { + effectiveDelay += (w.jitterMax * 2.0) / 1000.0 + } delayEffect := effectiveDelay / 40.0 if effectiveDelay > 160.0 { delayEffect = (effectiveDelay - 120.0) / 10.0 @@ -114,8 +125,10 @@ type layerTransition struct { } type qualityScorerParams struct { - PacketLossWeight float64 - Logger logger.Logger + PacketLossWeight float64 + IsDependentRTT bool + IsDependentJitter bool + Logger logger.Logger } type qualityScorer struct { @@ -245,7 +258,7 @@ func (q *qualityScorer) Update(stat *windowStat, at time.Time) { reason = "dry" score = poorScore } else { - packetScore := stat.calculatePacketScore(q.getPacketLossWeight(stat)) + packetScore := stat.calculatePacketScore(q.getPacketLossWeight(stat), q.params.IsDependentRTT, q.params.IsDependentJitter) bitrateScore := stat.calculateBitrateScore(expectedBitrate) layerScore := math.Max(math.Min(maxScore, maxScore-(expectedDistance*distanceWeight)), 0.0) diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 79abe27cb..1191bf5e0 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -271,10 +271,11 @@ func NewDownTrack( }) d.connectionStats = connectionquality.NewConnectionStats(connectionquality.ConnectionStatsParams{ - MimeType: codecs[0].MimeType, // LK-TODO have to notify on codec change - IsFECEnabled: strings.EqualFold(codecs[0].MimeType, webrtc.MimeTypeOpus) && strings.Contains(strings.ToLower(codecs[0].SDPFmtpLine), "fec"), - GetDeltaStats: d.getDeltaStats, - Logger: d.logger, + MimeType: codecs[0].MimeType, // LK-TODO have to notify on codec change + IsFECEnabled: strings.EqualFold(codecs[0].MimeType, webrtc.MimeTypeOpus) && strings.Contains(strings.ToLower(codecs[0].SDPFmtpLine), "fec"), + IsDependentJitter: true, + GetDeltaStats: d.getDeltaStats, + Logger: d.logger.WithValues("direction", "down"), }) d.connectionStats.OnStatsUpdate(func(_cs *connectionquality.ConnectionStats, stat *livekit.AnalyticsStat) { if d.onStatsUpdate != nil { diff --git a/pkg/sfu/receiver.go b/pkg/sfu/receiver.go index bc5ea1705..05d7303e2 100644 --- a/pkg/sfu/receiver.go +++ b/pkg/sfu/receiver.go @@ -204,10 +204,11 @@ func NewWebRTCReceiver( }) w.connectionStats = connectionquality.NewConnectionStats(connectionquality.ConnectionStatsParams{ - MimeType: w.codec.MimeType, - IsFECEnabled: strings.EqualFold(w.codec.MimeType, webrtc.MimeTypeOpus) && strings.Contains(strings.ToLower(w.codec.SDPFmtpLine), "fec"), - GetDeltaStats: w.getDeltaStats, - Logger: w.logger, + MimeType: w.codec.MimeType, + IsFECEnabled: strings.EqualFold(w.codec.MimeType, webrtc.MimeTypeOpus) && strings.Contains(strings.ToLower(w.codec.SDPFmtpLine), "fec"), + IsDependentRTT: true, + GetDeltaStats: w.getDeltaStats, + Logger: w.logger.WithValues("direction", "up"), }) w.connectionStats.OnStatsUpdate(func(_cs *connectionquality.ConnectionStats, stat *livekit.AnalyticsStat) { if w.onStatsUpdate != nil { From 23c03f6add7d47485b4421d3617e1e82df6d0362 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 22 Mar 2023 15:35:24 +0530 Subject: [PATCH 023/324] Fix av1 forwarding. (#1538) --- pkg/sfu/forwarder.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 296f5be3b..27266091c 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -1477,9 +1477,10 @@ func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer in if f.ddLayerSelector != nil { if selected := f.ddLayerSelector.Select(extPkt, tp); !selected { tp.shouldDrop = true + f.rtpMunger.UpdateAndGetSnTs(extPkt) // call to update highest incoming sequence number and other internal structures f.rtpMunger.PacketDropped(extPkt) return tp, nil - } else if tp.isSwitchingToTargetLayer { + } else if f.targetLayers.Spatial != f.currentLayers.Spatial && f.targetLayers.Spatial == layer && (extPkt.KeyFrame || tp.isSwitchingToTargetLayer) { // lock to target layer f.logger.Infow( "locking to target layer", @@ -1609,7 +1610,8 @@ func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer in } } - if f.currentLayers.Spatial != layer { + // if we have layer selector, let it decide whether to drop or not + if f.ddLayerSelector == nil && f.currentLayers.Spatial != layer { tp.shouldDrop = true return tp, nil } From 0ea88e40250b436b42b670974e8b6a60659cda91 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 22 Mar 2023 23:08:26 +0530 Subject: [PATCH 024/324] Ensure sequence number continuity (#1539) * Ensure sequence number continuity When using Go SDK (livekit-cli or egress) as a client, SFU sends blank frames when audio track is muted to ensure that Pion OnTrack fires on GoSDK side. That resulted in a huge sequence number/time stamp jump when the real stream started. Ensure continuity by creating random sequence number/time stamp when starting with a blank frames. And when sequence number/time stamp is initialized using SetLastSnTs, continue sequence if it was already initialized. * remove debug --- pkg/sfu/rtpmunger.go | 36 +++++++++++++----- pkg/sfu/rtpmunger_test.go | 79 +++++++++++++++++++++++++-------------- 2 files changed, 78 insertions(+), 37 deletions(-) diff --git a/pkg/sfu/rtpmunger.go b/pkg/sfu/rtpmunger.go index d42a4fe5f..1cd27d745 100644 --- a/pkg/sfu/rtpmunger.go +++ b/pkg/sfu/rtpmunger.go @@ -2,6 +2,7 @@ package sfu import ( "fmt" + "math/rand" "github.com/livekit/protocol/logger" @@ -41,17 +42,19 @@ type SnTs struct { // ---------------------------------------------------------------------- type RTPMungerState struct { - LastSN uint16 - LastTS uint32 + Started bool + LastSN uint16 + LastTS uint32 } func (r RTPMungerState) String() string { - return fmt.Sprintf("RTPMungerState{lastSN: %d, lastTS: %d)", r.LastSN, r.LastTS) + return fmt.Sprintf("RTPMungerState{started: %v, lastSN: %d, lastTS: %d)", r.Started, r.LastSN, r.LastTS) } // ---------------------------------------------------------------------- type RTPMungerParams struct { + started bool highestIncomingSN uint16 lastSN uint16 snOffset uint16 @@ -81,6 +84,7 @@ func NewRTPMunger(logger logger.Logger) *RTPMunger { func (r *RTPMunger) GetParams() RTPMungerParams { return RTPMungerParams{ + started: r.started, highestIncomingSN: r.highestIncomingSN, lastSN: r.lastSN, snOffset: r.snOffset, @@ -92,20 +96,28 @@ func (r *RTPMunger) GetParams() RTPMungerParams { func (r *RTPMunger) GetLast() RTPMungerState { return RTPMungerState{ - LastSN: r.lastSN, - LastTS: r.lastTS, + Started: r.started, + LastSN: r.lastSN, + LastTS: r.lastTS, } } func (r *RTPMunger) SeedLast(state RTPMungerState) { + r.started = state.Started r.lastSN = state.LastSN r.lastTS = state.LastTS } func (r *RTPMunger) SetLastSnTs(extPkt *buffer.ExtPacket) { r.highestIncomingSN = extPkt.Packet.SequenceNumber - 1 - r.lastSN = extPkt.Packet.SequenceNumber - r.lastTS = extPkt.Packet.Timestamp + if !r.started { + r.lastSN = extPkt.Packet.SequenceNumber + r.lastTS = extPkt.Packet.Timestamp + } else { + r.snOffset = extPkt.Packet.SequenceNumber - r.lastSN - 1 + r.tsOffset = extPkt.Packet.Timestamp - r.lastTS - 1 + } + r.started = true } func (r *RTPMunger) UpdateSnTsOffsets(extPkt *buffer.ExtPacket, snAdjust uint16, tsAdjust uint32) { @@ -122,7 +134,7 @@ func (r *RTPMunger) PacketDropped(extPkt *buffer.ExtPacket) { if r.highestIncomingSN != extPkt.Packet.SequenceNumber { return } - r.snOffset += 1 + r.snOffset++ r.lastSN = extPkt.Packet.SequenceNumber - r.snOffset r.snOffsetsWritePtr = (r.snOffsetsWritePtr - 1) & SnOffsetCacheMask @@ -177,7 +189,7 @@ func (r *RTPMunger) UpdateAndGetSnTs(extPkt *buffer.ExtPacket) (*TranslationPara // sequence number offset. if len(extPkt.Packet.Payload) == 0 { r.highestIncomingSN = extPkt.Packet.SequenceNumber - r.snOffset += 1 + r.snOffset++ return &TranslationParamsRTP{ snOrdering: SequenceNumberOrderingContiguous, @@ -241,6 +253,12 @@ func (r *RTPMunger) UpdateAndGetPaddingSnTs(num int, clockRate uint32, frameRate tsOffset = 1 } + if !r.started { + r.lastSN = uint16(rand.Intn(1<<14)) + uint16(1<<15) // a random number in third quartile of sequence number space + r.lastTS = uint32(rand.Intn(1<<30)) + uint32(1<<31) // a random number in third quartile of time stamp space + r.started = true + } + vals := make([]SnTs, num) for i := 0; i < num; i++ { vals[i].sequenceNumber = r.lastSN + uint16(i) + 1 diff --git a/pkg/sfu/rtpmunger_test.go b/pkg/sfu/rtpmunger_test.go index faa68efe9..1d59759ab 100644 --- a/pkg/sfu/rtpmunger_test.go +++ b/pkg/sfu/rtpmunger_test.go @@ -27,14 +27,18 @@ func TestSetLastSnTs(t *testing.T) { require.NotNil(t, extPkt) r.SetLastSnTs(extPkt) - require.True(t, r.highestIncomingSN == 23332) - require.True(t, r.lastSN == 23333) - require.True(t, r.lastTS == 0xabcdef) + require.Equal(t, uint16(23332), r.highestIncomingSN) + require.Equal(t, uint16(23333), r.lastSN) + require.Equal(t, uint32(0xabcdef), r.lastTS) require.Equal(t, uint16(0), r.snOffset) require.Equal(t, uint32(0), r.tsOffset) + require.True(t, r.started) + + // force re-start + r.started = false params = &testutils.TestExtPacketParams{ - SequenceNumber: 0, + SequenceNumber: 43, Timestamp: 0xabcdef, SSRC: 0x12345678, } @@ -43,11 +47,30 @@ func TestSetLastSnTs(t *testing.T) { require.NotNil(t, extPkt) r.SetLastSnTs(extPkt) - require.True(t, r.highestIncomingSN == 65535) - require.True(t, r.lastSN == 0) - require.True(t, r.lastTS == 0xabcdef) + require.Equal(t, uint16(42), r.highestIncomingSN) + require.Equal(t, uint16(43), r.lastSN) + require.Equal(t, uint32(0xabcdef), r.lastTS) require.Equal(t, uint16(0), r.snOffset) require.Equal(t, uint32(0), r.tsOffset) + require.True(t, r.started) + + // set on a started munger + params = &testutils.TestExtPacketParams{ + SequenceNumber: 23457, + Timestamp: 0xabcdef, + SSRC: 0x12345678, + } + extPkt, err = testutils.GetTestExtPacket(params) + require.NoError(t, err) + require.NotNil(t, extPkt) + + r.SetLastSnTs(extPkt) + require.Equal(t, uint16(23456), r.highestIncomingSN) + require.Equal(t, uint16(43), r.lastSN) + require.Equal(t, uint32(0xabcdef), r.lastTS) + require.Equal(t, uint16(23413), r.snOffset) + require.Equal(t, uint32(0xffffffff), r.tsOffset) + require.True(t, r.started) } func TestUpdateSnTsOffsets(t *testing.T) { @@ -68,9 +91,9 @@ func TestUpdateSnTsOffsets(t *testing.T) { } extPkt, _ = testutils.GetTestExtPacket(params) r.UpdateSnTsOffsets(extPkt, 1, 1) - require.True(t, r.highestIncomingSN == 33332) - require.True(t, r.lastSN == 23333) - require.True(t, r.lastTS == 0xabcdef) + require.Equal(t, uint16(33332), r.highestIncomingSN) + require.Equal(t, uint16(23333), r.lastSN) + require.Equal(t, uint32(0xabcdef), r.lastTS) require.Equal(t, uint16(9999), r.snOffset) require.Equal(t, uint32(0xffffffff), r.tsOffset) } @@ -207,8 +230,8 @@ func TestPaddingOnlyPacket(t *testing.T) { require.Error(t, err) require.ErrorIs(t, err, ErrPaddingOnlyPacket) require.Equal(t, tpExpected, *tp) - require.True(t, r.highestIncomingSN == 23333) - require.True(t, r.lastSN == 23333) + require.Equal(t, uint16(23333), r.highestIncomingSN) + require.Equal(t, uint16(23333), r.lastSN) require.Equal(t, uint16(1), r.snOffset) // padding only packet with a gap should not report an error @@ -228,8 +251,8 @@ func TestPaddingOnlyPacket(t *testing.T) { tp, err = r.UpdateAndGetSnTs(extPkt) require.NoError(t, err) require.Equal(t, tpExpected, *tp) - require.True(t, r.highestIncomingSN == 23335) - require.True(t, r.lastSN == 23334) + require.Equal(t, uint16(23335), r.highestIncomingSN) + require.Equal(t, uint16(23334), r.lastSN) require.Equal(t, uint16(1), r.snOffset) } @@ -266,8 +289,8 @@ func TestGapInSequenceNumber(t *testing.T) { tp, err := r.UpdateAndGetSnTs(extPkt) require.NoError(t, err) require.Equal(t, tpExpected, *tp) - require.True(t, r.highestIncomingSN == 1) - require.True(t, r.lastSN == 1) + require.Equal(t, uint16(1), r.highestIncomingSN) + require.Equal(t, uint16(1), r.lastSN) require.Equal(t, uint16(0), r.snOffset) // ensure missing sequence numbers got recorded in cache @@ -294,8 +317,8 @@ func TestGapInSequenceNumber(t *testing.T) { tp, err = r.UpdateAndGetSnTs(extPkt) require.ErrorIs(t, err, ErrPaddingOnlyPacket) require.Equal(t, tpExpected, *tp) - require.True(t, r.highestIncomingSN == 2) - require.True(t, r.lastSN == 1) + require.Equal(t, uint16(2), r.highestIncomingSN) + require.Equal(t, uint16(1), r.lastSN) require.Equal(t, uint16(1), r.snOffset) // a packet with a gap should be adding to missing cache @@ -316,8 +339,8 @@ func TestGapInSequenceNumber(t *testing.T) { tp, err = r.UpdateAndGetSnTs(extPkt) require.NoError(t, err) require.Equal(t, tpExpected, *tp) - require.True(t, r.highestIncomingSN == 4) - require.True(t, r.lastSN == 3) + require.Equal(t, uint16(4), r.highestIncomingSN) + require.Equal(t, uint16(3), r.lastSN) require.Equal(t, uint16(1), r.snOffset) // another contiguous padding only packet should be dropped @@ -335,8 +358,8 @@ func TestGapInSequenceNumber(t *testing.T) { tp, err = r.UpdateAndGetSnTs(extPkt) require.ErrorIs(t, err, ErrPaddingOnlyPacket) require.Equal(t, tpExpected, *tp) - require.True(t, r.highestIncomingSN == 5) - require.True(t, r.lastSN == 3) + require.Equal(t, uint16(5), r.highestIncomingSN) + require.Equal(t, uint16(3), r.lastSN) require.Equal(t, uint16(2), r.snOffset) // a packet with a gap should be adding to missing cache @@ -357,8 +380,8 @@ func TestGapInSequenceNumber(t *testing.T) { tp, err = r.UpdateAndGetSnTs(extPkt) require.NoError(t, err) require.Equal(t, tpExpected, *tp) - require.True(t, r.highestIncomingSN == 7) - require.True(t, r.lastSN == 5) + require.Equal(t, uint16(7), r.highestIncomingSN) + require.Equal(t, uint16(5), r.lastSN) require.Equal(t, uint16(2), r.snOffset) // check the missing packets @@ -378,8 +401,8 @@ func TestGapInSequenceNumber(t *testing.T) { tp, err = r.UpdateAndGetSnTs(extPkt) require.NoError(t, err) require.Equal(t, tpExpected, *tp) - require.True(t, r.highestIncomingSN == 7) - require.True(t, r.lastSN == 5) + require.Equal(t, uint16(7), r.highestIncomingSN) + require.Equal(t, uint16(5), r.lastSN) require.Equal(t, uint16(2), r.snOffset) params = &testutils.TestExtPacketParams{ @@ -398,8 +421,8 @@ func TestGapInSequenceNumber(t *testing.T) { tp, err = r.UpdateAndGetSnTs(extPkt) require.NoError(t, err) require.Equal(t, tpExpected, *tp) - require.True(t, r.highestIncomingSN == 7) - require.True(t, r.lastSN == 5) + require.Equal(t, uint16(7), r.highestIncomingSN) + require.Equal(t, uint16(5), r.lastSN) require.Equal(t, uint16(2), r.snOffset) } From 191a9e8014827ad0f2987ec7d9666db844a64786 Mon Sep 17 00:00:00 2001 From: David Colburn Date: Wed, 22 Mar 2023 16:53:23 -0700 Subject: [PATCH 025/324] update core to 0.0.5 (#1540) * update core * sort imports * fix typos * redundant types --- go.mod | 2 +- go.sum | 4 +- pkg/config/config.go | 26 +-- pkg/routing/utils_test.go | 3 +- pkg/rtc/config.go | 6 +- pkg/rtc/dynacastmanager.go | 2 +- pkg/rtc/dynacastmanager_test.go | 5 +- pkg/rtc/mediatrackreceiver.go | 2 +- pkg/rtc/participant.go | 8 +- pkg/rtc/participant_internal_test.go | 2 +- pkg/rtc/room.go | 12 +- pkg/rtc/transport.go | 8 +- pkg/rtc/transport_test.go | 2 +- pkg/rtc/transportmanager.go | 4 +- pkg/service/ioinfo.go | 2 +- pkg/service/redisstore.go | 7 +- pkg/sfu/buffer/buffer.go | 10 +- pkg/sfu/buffer/buffer_test.go | 3 +- pkg/sfu/buffer/datastats_test.go | 3 +- pkg/sfu/buffer/fps.go | 2 +- pkg/sfu/buffer/fps_test.go | 4 +- pkg/sfu/buffer/rtpstats.go | 4 +- pkg/sfu/buffer/videolayerutils.go | 4 +- pkg/sfu/buffer/videolayerutils_test.go | 201 +++++++++--------- pkg/sfu/connectionquality/connectionstats.go | 20 +- .../connectionquality/connectionstats_test.go | 49 ++--- pkg/sfu/connectionquality/scorer.go | 20 +- .../dependencydescriptor/bitstreamreader.go | 30 +-- .../dependencydescriptor/bitstreamwriter.go | 2 +- .../dependencydescriptorextension.go | 10 +- .../dependencydescriptorextension_test.go | 2 +- .../dependencydescriptorreader.go | 2 +- .../dependencydescriptorwriter.go | 22 +- pkg/sfu/downtrack.go | 6 +- pkg/sfu/forwarder_test.go | 6 +- pkg/sfu/streamtracker/streamtracker.go | 3 +- pkg/sfu/streamtracker/streamtracker_frame.go | 2 +- pkg/sfu/streamtrackermanager.go | 1 + pkg/sfu/videolayerselector.go | 16 +- pkg/telemetry/signalanddatastats.go | 16 +- test/client/client.go | 2 +- 41 files changed, 272 insertions(+), 263 deletions(-) diff --git a/go.mod b/go.mod index bcaa2e41f..aecb0e2cd 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/dustin/go-humanize v1.0.1 github.com/elliotchance/orderedmap/v2 v2.2.0 github.com/florianl/go-tc v0.4.2 - github.com/frostbyte73/core v0.0.4 + github.com/frostbyte73/core v0.0.5 github.com/gammazero/deque v0.1.0 github.com/gammazero/workerpool v1.1.2 github.com/google/wire v0.5.0 diff --git a/go.sum b/go.sum index 212a673ca..c3f9d7c71 100644 --- a/go.sum +++ b/go.sum @@ -87,8 +87,8 @@ github.com/florianl/go-tc v0.4.2 h1:jan5zcOWCLhA9SRBHZhQ0SSAq7cmDUagiRPngAi5AOQ= github.com/florianl/go-tc v0.4.2/go.mod h1:2W1jSMFryiYlpQigr4ZpSSpE9XNze+bW7cTsCXWbMwo= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= -github.com/frostbyte73/core v0.0.4 h1:CwwoYfKPdNSO/QbOOMWMRSYoNW14ov4XHnt094AuMX8= -github.com/frostbyte73/core v0.0.4/go.mod h1:mqHHSVFS5DE6kSdhU1/s9Mm0YCnLB8Ou2DD/eX1Zbr4= +github.com/frostbyte73/core v0.0.5 h1:+oHjXDyQyQzEx04mtmmafYP07n7EToKpUGafWbNVQ9I= +github.com/frostbyte73/core v0.0.5/go.mod h1:mqHHSVFS5DE6kSdhU1/s9Mm0YCnLB8Ou2DD/eX1Zbr4= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/gammazero/deque v0.1.0 h1:f9LnNmq66VDeuAlSAapemq/U7hJ2jpIWa4c09q8Dlik= diff --git a/pkg/config/config.go b/pkg/config/config.go index 8aad90afb..0195543bd 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -314,30 +314,30 @@ func NewConfig(confString string, strictMode bool, c *cli.Context, baseFlags []c 2: 1 * time.Second, }, PacketTracker: map[int32]StreamTrackerPacketConfig{ - 0: StreamTrackerPacketConfig{ + 0: { SamplesRequired: 1, CyclesRequired: 4, CycleDuration: 500 * time.Millisecond, }, - 1: StreamTrackerPacketConfig{ + 1: { SamplesRequired: 5, CyclesRequired: 20, CycleDuration: 500 * time.Millisecond, }, - 2: StreamTrackerPacketConfig{ + 2: { SamplesRequired: 5, CyclesRequired: 20, CycleDuration: 500 * time.Millisecond, }, }, FrameTracker: map[int32]StreamTrackerFrameConfig{ - 0: StreamTrackerFrameConfig{ + 0: { MinFPS: 5.0, }, - 1: StreamTrackerFrameConfig{ + 1: { MinFPS: 5.0, }, - 2: StreamTrackerFrameConfig{ + 2: { MinFPS: 5.0, }, }, @@ -350,30 +350,30 @@ func NewConfig(confString string, strictMode bool, c *cli.Context, baseFlags []c 2: 4 * time.Second, }, PacketTracker: map[int32]StreamTrackerPacketConfig{ - 0: StreamTrackerPacketConfig{ + 0: { SamplesRequired: 1, CyclesRequired: 1, CycleDuration: 2 * time.Second, }, - 1: StreamTrackerPacketConfig{ + 1: { SamplesRequired: 1, CyclesRequired: 1, CycleDuration: 2 * time.Second, }, - 2: StreamTrackerPacketConfig{ + 2: { SamplesRequired: 1, CyclesRequired: 1, CycleDuration: 2 * time.Second, }, }, FrameTracker: map[int32]StreamTrackerFrameConfig{ - 0: StreamTrackerFrameConfig{ + 0: { MinFPS: 0.5, }, - 1: StreamTrackerFrameConfig{ + 1: { MinFPS: 0.5, }, - 2: StreamTrackerFrameConfig{ + 2: { MinFPS: 0.5, }, }, @@ -576,7 +576,7 @@ func (conf *Config) ValidateKeys() error { func GenerateCLIFlags(existingFlags []cli.Flag, hidden bool) ([]cli.Flag, error) { blankConfig := &Config{} - flags := []cli.Flag{} + flags := make([]cli.Flag, 0) for name, value := range blankConfig.ToCLIFlagNames(existingFlags) { kind := value.Kind() if kind == reflect.Ptr { diff --git a/pkg/routing/utils_test.go b/pkg/routing/utils_test.go index 127d3fd13..ae40b27e8 100644 --- a/pkg/routing/utils_test.go +++ b/pkg/routing/utils_test.go @@ -3,8 +3,9 @@ package routing import ( "testing" - "github.com/livekit/protocol/livekit" "github.com/stretchr/testify/require" + + "github.com/livekit/protocol/livekit" ) func TestUtils_ParticipantKey(t *testing.T) { diff --git a/pkg/rtc/config.go b/pkg/rtc/config.go index f5c885603..027c05637 100644 --- a/pkg/rtc/config.go +++ b/pkg/rtc/config.go @@ -340,7 +340,7 @@ func getNAT1to1IPsForConf(conf *config.Config, ipFilter func(net.IP) bool) ([]st }(ip) } - var firstResloved bool + var firstResolved bool natMapping := make(map[string]string) timeout := time.NewTimer(5 * time.Second) defer timeout.Stop() @@ -349,8 +349,8 @@ done: for { select { case mapping := <-addrCh: - if !firstResloved { - firstResloved = true + if !firstResolved { + firstResolved = true timeout.Reset(1 * time.Second) } if local, ok := natMapping[mapping.externalIP]; ok { diff --git a/pkg/rtc/dynacastmanager.go b/pkg/rtc/dynacastmanager.go index aed49e63f..76427798a 100644 --- a/pkg/rtc/dynacastmanager.go +++ b/pkg/rtc/dynacastmanager.go @@ -88,7 +88,7 @@ func (d *DynacastManager) Close() { } } -// THere are situations like track unmute or streaming from a sifferent node +// THere are situations like track unmute or streaming from a different node // where subscribed quality needs to sent to the provider immediately. // This bypasses any debouncing and forces a subscribed quality update // with immediate effect. diff --git a/pkg/rtc/dynacastmanager_test.go b/pkg/rtc/dynacastmanager_test.go index ae600575a..ad7b065bb 100644 --- a/pkg/rtc/dynacastmanager_test.go +++ b/pkg/rtc/dynacastmanager_test.go @@ -6,10 +6,11 @@ import ( "testing" "time" - "github.com/livekit/livekit-server/pkg/rtc/types" - "github.com/livekit/protocol/livekit" "github.com/pion/webrtc/v3" "github.com/stretchr/testify/require" + + "github.com/livekit/livekit-server/pkg/rtc/types" + "github.com/livekit/protocol/livekit" ) func TestSubscribedMaxQuality(t *testing.T) { diff --git a/pkg/rtc/mediatrackreceiver.go b/pkg/rtc/mediatrackreceiver.go index 31a6dcf3c..e1b54c1e5 100644 --- a/pkg/rtc/mediatrackreceiver.go +++ b/pkg/rtc/mediatrackreceiver.go @@ -54,7 +54,7 @@ func (m mediaTrackReceiverState) String() string { } } -//----------------------------------------------------- +// ----------------------------------------------------- type simulcastReceiver struct { sfu.TrackReceiver diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 50c4f3bce..61f3eb3f0 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -687,7 +687,7 @@ func (p *ParticipantImpl) IsClosed() bool { return p.isClosed.Load() } -// Negotiate subscriber SDP with client, if force is true, will cencel pending +// Negotiate subscriber SDP with client, if force is true, will cancel pending // negotiate task and negotiate immediately func (p *ParticipantImpl) Negotiate(force bool) { if p.MigrateState() != types.MigrateStateInit { @@ -1363,7 +1363,7 @@ func (p *ParticipantImpl) onStreamStateChange(update *sfu.StreamStateUpdate) err }) } -func (p *ParticipantImpl) onSubscribedMaxQualityChange(trackID livekit.TrackID, subscribedQualities []*livekit.SubscribedCodec, maxSubscribedQualites []types.SubscribedCodecQuality) error { +func (p *ParticipantImpl) onSubscribedMaxQualityChange(trackID livekit.TrackID, subscribedQualities []*livekit.SubscribedCodec, maxSubscribedQualities []types.SubscribedCodecQuality) error { if p.params.DisableDynacast { return nil } @@ -1394,7 +1394,7 @@ func (p *ParticipantImpl) onSubscribedMaxQualityChange(trackID livekit.TrackID, } } - for _, maxSubscribedQuality := range maxSubscribedQualites { + for _, maxSubscribedQuality := range maxSubscribedQualities { ti := &livekit.TrackInfo{ Sid: string(trackID), Type: livekit.TrackType_VIDEO, @@ -1417,7 +1417,7 @@ func (p *ParticipantImpl) onSubscribedMaxQualityChange(trackID livekit.TrackID, "sending max subscribed quality", "trackID", trackID, "qualities", subscribedQualities, - "max", maxSubscribedQualites, + "max", maxSubscribedQualities, ) return p.writeMessage(&livekit.SignalResponse{ Message: &livekit.SignalResponse_SubscribedQualityUpdate{ diff --git a/pkg/rtc/participant_internal_test.go b/pkg/rtc/participant_internal_test.go index 509282d03..f04c277b5 100644 --- a/pkg/rtc/participant_internal_test.go +++ b/pkg/rtc/participant_internal_test.go @@ -361,7 +361,7 @@ func TestSetStableTrackID(t *testing.T) { } func TestDisableCodecs(t *testing.T) { - participant := newParticipantForTestWithOpts(livekit.ParticipantIdentity("123"), &participantOpts{ + participant := newParticipantForTestWithOpts("123", &participantOpts{ publisher: false, clientConf: &livekit.ClientConfiguration{ DisabledCodecs: &livekit.DisabledCodecs{ diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index f319010e4..bd2f55fe4 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -1162,11 +1162,11 @@ func BroadcastDataPacketForRoom(r types.Room, source types.LocalParticipant, dp var dpData []byte participants := r.GetLocalParticipants() - cap := len(dest) - if cap == 0 { - cap = len(participants) + capacity := len(dest) + if capacity == 0 { + capacity = len(participants) } - destParticpants := make([]types.LocalParticipant, 0, cap) + destParticipants := make([]types.LocalParticipant, 0, capacity) for _, op := range participants { if op.State() != livekit.ParticipantInfo_ACTIVE { @@ -1195,10 +1195,10 @@ func BroadcastDataPacketForRoom(r types.Room, source types.LocalParticipant, dp return } } - destParticpants = append(destParticpants, op) + destParticipants = append(destParticipants, op) } - utils.ParallelExec(destParticpants, dataForwardLoadBalanceThreshold, 1, func(op types.LocalParticipant) { + utils.ParallelExec(destParticipants, dataForwardLoadBalanceThreshold, 1, func(op types.LocalParticipant) { err := op.SendDataPacket(dp, dpData) if err != nil && !errors.Is(err, io.ErrClosedPipe) { op.GetLogger().Infow("send data packet error", "error", err) diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index c4afdd558..14055d0a3 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -1263,7 +1263,7 @@ func (t *PCTransport) initPCWithPreviousAnswer(previousAnswer webrtc.SessionDesc } tr.SetMid(mid) - // save mid -> senders for migration resue + // save mid -> senders for migration reuse sender := tr.Sender() senders[mid] = sender @@ -1294,7 +1294,7 @@ func (t *PCTransport) SetPreviousSdp(offer, answer *webrtc.SessionDescription) { } return } else if offer != nil { - // in migration case, can't reuse tranceiver before negotiated except track subscribed at previous node + // in migration case, can't reuse transceiver before negotiated except track subscribed at previous node t.canReuseTransceiver = false if err := t.parseTrackMid(*offer, senders); err != nil { t.params.Logger.Errorw("parse previous offer failed", err, "offer", offer.SDP) @@ -1318,12 +1318,12 @@ func (t *PCTransport) parseTrackMid(offer webrtc.SessionDescription, senders map } if split := strings.Split(msid, " "); len(split) == 2 { - trackid := split[1] + trackID := split[1] mid := lksdp.GetMidValue(m) if mid == "" { return ErrMidNotFound } - t.previousTrackDescription[trackid] = &trackDescription{ + t.previousTrackDescription[trackID] = &trackDescription{ mid: mid, sender: senders[mid], } diff --git a/pkg/rtc/transport_test.go b/pkg/rtc/transport_test.go index ef8d66d21..5e388facb 100644 --- a/pkg/rtc/transport_test.go +++ b/pkg/rtc/transport_test.go @@ -246,7 +246,7 @@ func TestFirstAnswerMissedDuringICERestart(t *testing.T) { // exchange ICE handleICEExchange(t, transportA, transportB) - // first anwser missed + // first answer missed var firstAnswerReceived atomic.Bool transportB.OnAnswer(func(sd webrtc.SessionDescription) error { if firstAnswerReceived.Load() { diff --git a/pkg/rtc/transportmanager.go b/pkg/rtc/transportmanager.go index b3002212e..cee3c3661 100644 --- a/pkg/rtc/transportmanager.go +++ b/pkg/rtc/transportmanager.go @@ -363,7 +363,7 @@ func (t *TransportManager) GetUnmatchMediaForOffer(offer webrtc.SessionDescripti answer := lastAnswer.(webrtc.SessionDescription) parsedAnswer, err1 := answer.Unmarshal() if err1 != nil { - // should not happend + // should not happen t.params.Logger.Errorw("failed to parse last answer", err) return } @@ -572,7 +572,7 @@ func (t *TransportManager) handleConnectionFailed(isShortLived bool) { } // - // Checking only `PreferenceSubcriber` field although any connection failure (PUBLISHER OR SUBSCRIBER) will + // Checking only `PreferenceSubscriber` field although any connection failure (PUBLISHER OR SUBSCRIBER) will // flow through here. // // As both transports are switched to the same type on any failure, checking just subscriber should be fine. diff --git a/pkg/service/ioinfo.go b/pkg/service/ioinfo.go index b9e49e4b6..cc0aa404b 100644 --- a/pkg/service/ioinfo.go +++ b/pkg/service/ioinfo.go @@ -118,7 +118,7 @@ func (s *IOInfoService) loadIngressFromInfoRequest(req *rpc.GetIngressInfoReques } else if req.StreamKey != "" { info, err = s.is.LoadIngressFromStreamKey(context.Background(), req.StreamKey) } else { - err = errors.New("request needs to specity either IngressId or StreamKey") + err = errors.New("request needs to specify either IngressId or StreamKey") } return info, err } diff --git a/pkg/service/redisstore.go b/pkg/service/redisstore.go index 2558bc4a4..bd3f835d2 100644 --- a/pkg/service/redisstore.go +++ b/pkg/service/redisstore.go @@ -27,10 +27,9 @@ const ( RoomInternalKey = "room_internal" // EgressKey is a hash of egressID => egress info - EgressKey = "egress" - EndedEgressKey = "ended_egress" - RoomEgressPrefix = "egress:room:" - DeprecatedRoomEgressPrefix = "room_egress:" + EgressKey = "egress" + EndedEgressKey = "ended_egress" + RoomEgressPrefix = "egress:room:" // IngressKey is a hash of ingressID => ingress info IngressKey = "ingress" diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go index dde05a51e..48ee591e2 100644 --- a/pkg/sfu/buffer/buffer.go +++ b/pkg/sfu/buffer/buffer.go @@ -92,7 +92,7 @@ type Buffer struct { // logger logger logger.Logger - // depencency descriptor + // dependency descriptor ddExt uint8 ddParser *DependencyDescriptorParser maxLayerChangedCB func(int32, int32) @@ -436,15 +436,15 @@ func (b *Buffer) patchExtPacket(ep *ExtPacket, buf []byte) *ExtPacket { ep.RawPacket = buf[:n] // patch RTP packet to point payload to new buffer - rtp := *ep.Packet + pkt := *ep.Packet payloadStart := ep.Packet.Header.MarshalSize() payloadEnd := payloadStart + len(ep.Packet.Payload) if payloadEnd > n { b.logger.Warnw("unexpected marshal size", nil, "max", n, "need", payloadEnd) return nil } - rtp.Payload = buf[payloadStart:payloadEnd] - ep.Packet = &rtp + pkt.Payload = buf[payloadStart:payloadEnd] + ep.Packet = &pkt return ep } @@ -755,7 +755,7 @@ func (b *Buffer) GetAudioLevel() (float64, bool) { } // TODO : now we rely on stream tracker for layer change, dependency still -// work for that too. Do we keep it unchange or use both methods? +// work for that too. Do we keep it unchanged or use both methods? func (b *Buffer) OnMaxLayerChanged(fn func(int32, int32)) { b.maxLayerChangedCB = fn } diff --git a/pkg/sfu/buffer/buffer_test.go b/pkg/sfu/buffer/buffer_test.go index a3207336e..68d8de3a9 100644 --- a/pkg/sfu/buffer/buffer_test.go +++ b/pkg/sfu/buffer/buffer_test.go @@ -6,11 +6,12 @@ import ( "testing" "time" - "github.com/livekit/mediatransportutil/pkg/nack" "github.com/pion/rtcp" "github.com/pion/rtp" "github.com/pion/webrtc/v3" "github.com/stretchr/testify/require" + + "github.com/livekit/mediatransportutil/pkg/nack" ) var vp8Codec = webrtc.RTPCodecParameters{ diff --git a/pkg/sfu/buffer/datastats_test.go b/pkg/sfu/buffer/datastats_test.go index 72992d35f..f2369b7c5 100644 --- a/pkg/sfu/buffer/datastats_test.go +++ b/pkg/sfu/buffer/datastats_test.go @@ -4,9 +4,10 @@ import ( "testing" "time" - "github.com/livekit/protocol/livekit" "github.com/stretchr/testify/require" "google.golang.org/protobuf/proto" + + "github.com/livekit/protocol/livekit" ) func TestDataStats(t *testing.T) { diff --git a/pkg/sfu/buffer/fps.go b/pkg/sfu/buffer/fps.go index c44efc559..ddd6fb467 100644 --- a/pkg/sfu/buffer/fps.go +++ b/pkg/sfu/buffer/fps.go @@ -115,7 +115,7 @@ func (f *FrameRateCalculatorVP8) RecvPacket(ep *ExtPacket) bool { func (f *FrameRateCalculatorVP8) calc() bool { var rateCounter int - for currentTemporal := int32(0); currentTemporal <= int32(DefaultMaxLayerTemporal); currentTemporal++ { + for currentTemporal := int32(0); currentTemporal <= DefaultMaxLayerTemporal; currentTemporal++ { if f.frameRates[currentTemporal] > 0 { rateCounter++ continue diff --git a/pkg/sfu/buffer/fps_test.go b/pkg/sfu/buffer/fps_test.go index a28f1073b..5f0ff79ce 100644 --- a/pkg/sfu/buffer/fps_test.go +++ b/pkg/sfu/buffer/fps_test.go @@ -147,7 +147,7 @@ func TestFpsVP8(t *testing.T) { testCase := c t.Run(name, func(t *testing.T) { fps := testCase.fps - frames := [][]*testFrameInfo{} + frames := make([][]*testFrameInfo, 0) vp8calcs := make([]*FrameRateCalculatorVP8, len(fps)) for i := range vp8calcs { vp8calcs[i] = NewFrameRateCalculatorVP8(90000, logger.GetLogger()) @@ -178,7 +178,7 @@ func TestFpsVP8(t *testing.T) { } t.Run("packet lost and duplicate", func(t *testing.T) { fps := [][]float32{{7.5, 15}, {7.5, 15}, {15, 30}} - frames := [][]*testFrameInfo{} + frames := make([][]*testFrameInfo, 0) vp8calcs := make([]*FrameRateCalculatorVP8, len(fps)) for i := range vp8calcs { vp8calcs[i] = NewFrameRateCalculatorVP8(90000, logger.GetLogger()) diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index 46af6d5e9..49e4bda6b 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -801,7 +801,7 @@ func (r *RTPStats) SnapshotRtcpReceptionReport(ssrc uint32, proxyFracLost uint8, packetsLost := uint32(0) if r.params.IsReceiverReportDriven { - // receiver report drvien should not be set for streams that need to generate reception report, but including code here for consistency + // receiver report driven should not be set for streams that need to generate reception report, but including code here for consistency packetsLost = now.packetsLostOverridden - then.packetsLostOverridden if int32(packetsLost) < 0 { packetsLost = 0 @@ -825,7 +825,7 @@ func (r *RTPStats) SnapshotRtcpReceptionReport(ssrc uint32, proxyFracLost uint8, jitter := r.jitter if r.params.IsReceiverReportDriven { - // receiver report drvien should not be set for streams that need to generate reception report, but including code here for consistency + // receiver report driven should not be set for streams that need to generate reception report, but including code here for consistency jitter = r.jitterOverridden } diff --git a/pkg/sfu/buffer/videolayerutils.go b/pkg/sfu/buffer/videolayerutils.go index b2bc54f23..32abdb45d 100644 --- a/pkg/sfu/buffer/videolayerutils.go +++ b/pkg/sfu/buffer/videolayerutils.go @@ -84,7 +84,7 @@ func RidToSpatialLayer(rid string, trackInfo *livekit.TrackInfo) int32 { logger.Warnw("unexpected rid f with only two qualities, low and high", nil) return 1 case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]: - logger.Warnw("unexpected rid f with only two qualities, medum and high", nil) + logger.Warnw("unexpected rid f with only two qualities, medium and high", nil) return 1 default: @@ -156,7 +156,7 @@ func SpatialLayerToRid(layer int32, trackInfo *livekit.TrackInfo) string { logger.Warnw("unexpected layer 2 with only two qualities, low and high", nil) return HalfResolution case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]: - logger.Warnw("unexpected layer 2 with only two qualities, medum and high", nil) + logger.Warnw("unexpected layer 2 with only two qualities, medium and high", nil) return HalfResolution default: diff --git a/pkg/sfu/buffer/videolayerutils_test.go b/pkg/sfu/buffer/videolayerutils_test.go index 7e3b08792..b57bb2de0 100644 --- a/pkg/sfu/buffer/videolayerutils_test.go +++ b/pkg/sfu/buffer/videolayerutils_test.go @@ -3,8 +3,9 @@ package buffer import ( "testing" - "github.com/livekit/protocol/livekit" "github.com/stretchr/testify/require" + + "github.com/livekit/protocol/livekit" ) func TestRidConversion(t *testing.T) { @@ -21,123 +22,123 @@ func TestRidConversion(t *testing.T) { "no track info", nil, map[string]RidAndLayer{ - "": RidAndLayer{rid: QuarterResolution, layer: 0}, - QuarterResolution: RidAndLayer{rid: QuarterResolution, layer: 0}, - HalfResolution: RidAndLayer{rid: HalfResolution, layer: 1}, - FullResolution: RidAndLayer{rid: FullResolution, layer: 2}, + "": {rid: QuarterResolution, layer: 0}, + QuarterResolution: {rid: QuarterResolution, layer: 0}, + HalfResolution: {rid: HalfResolution, layer: 1}, + FullResolution: {rid: FullResolution, layer: 2}, }, }, { "no layers", &livekit.TrackInfo{}, map[string]RidAndLayer{ - "": RidAndLayer{rid: QuarterResolution, layer: 0}, - QuarterResolution: RidAndLayer{rid: QuarterResolution, layer: 0}, - HalfResolution: RidAndLayer{rid: HalfResolution, layer: 1}, - FullResolution: RidAndLayer{rid: FullResolution, layer: 2}, + "": {rid: QuarterResolution, layer: 0}, + QuarterResolution: {rid: QuarterResolution, layer: 0}, + HalfResolution: {rid: HalfResolution, layer: 1}, + FullResolution: {rid: FullResolution, layer: 2}, }, }, { "single layer, low", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - &livekit.VideoLayer{Quality: livekit.VideoQuality_LOW}, + {Quality: livekit.VideoQuality_LOW}, }, }, map[string]RidAndLayer{ - "": RidAndLayer{rid: QuarterResolution, layer: 0}, - QuarterResolution: RidAndLayer{rid: QuarterResolution, layer: 0}, - HalfResolution: RidAndLayer{rid: QuarterResolution, layer: 0}, - FullResolution: RidAndLayer{rid: QuarterResolution, layer: 0}, + "": {rid: QuarterResolution, layer: 0}, + QuarterResolution: {rid: QuarterResolution, layer: 0}, + HalfResolution: {rid: QuarterResolution, layer: 0}, + FullResolution: {rid: QuarterResolution, layer: 0}, }, }, { "single layer, medium", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - &livekit.VideoLayer{Quality: livekit.VideoQuality_MEDIUM}, + {Quality: livekit.VideoQuality_MEDIUM}, }, }, map[string]RidAndLayer{ - "": RidAndLayer{rid: QuarterResolution, layer: 0}, - QuarterResolution: RidAndLayer{rid: QuarterResolution, layer: 0}, - HalfResolution: RidAndLayer{rid: QuarterResolution, layer: 0}, - FullResolution: RidAndLayer{rid: QuarterResolution, layer: 0}, + "": {rid: QuarterResolution, layer: 0}, + QuarterResolution: {rid: QuarterResolution, layer: 0}, + HalfResolution: {rid: QuarterResolution, layer: 0}, + FullResolution: {rid: QuarterResolution, layer: 0}, }, }, { "single layer, high", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - &livekit.VideoLayer{Quality: livekit.VideoQuality_HIGH}, + {Quality: livekit.VideoQuality_HIGH}, }, }, map[string]RidAndLayer{ - "": RidAndLayer{rid: QuarterResolution, layer: 0}, - QuarterResolution: RidAndLayer{rid: QuarterResolution, layer: 0}, - HalfResolution: RidAndLayer{rid: QuarterResolution, layer: 0}, - FullResolution: RidAndLayer{rid: QuarterResolution, layer: 0}, + "": {rid: QuarterResolution, layer: 0}, + QuarterResolution: {rid: QuarterResolution, layer: 0}, + HalfResolution: {rid: QuarterResolution, layer: 0}, + FullResolution: {rid: QuarterResolution, layer: 0}, }, }, { "two layers, low and medium", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - &livekit.VideoLayer{Quality: livekit.VideoQuality_LOW}, - &livekit.VideoLayer{Quality: livekit.VideoQuality_MEDIUM}, + {Quality: livekit.VideoQuality_LOW}, + {Quality: livekit.VideoQuality_MEDIUM}, }, }, map[string]RidAndLayer{ - "": RidAndLayer{rid: QuarterResolution, layer: 0}, - QuarterResolution: RidAndLayer{rid: QuarterResolution, layer: 0}, - HalfResolution: RidAndLayer{rid: HalfResolution, layer: 1}, - FullResolution: RidAndLayer{rid: HalfResolution, layer: 1}, + "": {rid: QuarterResolution, layer: 0}, + QuarterResolution: {rid: QuarterResolution, layer: 0}, + HalfResolution: {rid: HalfResolution, layer: 1}, + FullResolution: {rid: HalfResolution, layer: 1}, }, }, { "two layers, low and high", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - &livekit.VideoLayer{Quality: livekit.VideoQuality_LOW}, - &livekit.VideoLayer{Quality: livekit.VideoQuality_HIGH}, + {Quality: livekit.VideoQuality_LOW}, + {Quality: livekit.VideoQuality_HIGH}, }, }, map[string]RidAndLayer{ - "": RidAndLayer{rid: QuarterResolution, layer: 0}, - QuarterResolution: RidAndLayer{rid: QuarterResolution, layer: 0}, - HalfResolution: RidAndLayer{rid: HalfResolution, layer: 1}, - FullResolution: RidAndLayer{rid: HalfResolution, layer: 1}, + "": {rid: QuarterResolution, layer: 0}, + QuarterResolution: {rid: QuarterResolution, layer: 0}, + HalfResolution: {rid: HalfResolution, layer: 1}, + FullResolution: {rid: HalfResolution, layer: 1}, }, }, { "two layers, medium and high", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - &livekit.VideoLayer{Quality: livekit.VideoQuality_MEDIUM}, - &livekit.VideoLayer{Quality: livekit.VideoQuality_HIGH}, + {Quality: livekit.VideoQuality_MEDIUM}, + {Quality: livekit.VideoQuality_HIGH}, }, }, map[string]RidAndLayer{ - "": RidAndLayer{rid: QuarterResolution, layer: 0}, - QuarterResolution: RidAndLayer{rid: QuarterResolution, layer: 0}, - HalfResolution: RidAndLayer{rid: HalfResolution, layer: 1}, - FullResolution: RidAndLayer{rid: HalfResolution, layer: 1}, + "": {rid: QuarterResolution, layer: 0}, + QuarterResolution: {rid: QuarterResolution, layer: 0}, + HalfResolution: {rid: HalfResolution, layer: 1}, + FullResolution: {rid: HalfResolution, layer: 1}, }, }, { "three layers", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - &livekit.VideoLayer{Quality: livekit.VideoQuality_LOW}, - &livekit.VideoLayer{Quality: livekit.VideoQuality_MEDIUM}, - &livekit.VideoLayer{Quality: livekit.VideoQuality_HIGH}, + {Quality: livekit.VideoQuality_LOW}, + {Quality: livekit.VideoQuality_MEDIUM}, + {Quality: livekit.VideoQuality_HIGH}, }, }, map[string]RidAndLayer{ - "": RidAndLayer{rid: QuarterResolution, layer: 0}, - QuarterResolution: RidAndLayer{rid: QuarterResolution, layer: 0}, - HalfResolution: RidAndLayer{rid: HalfResolution, layer: 1}, - FullResolution: RidAndLayer{rid: FullResolution, layer: 2}, + "": {rid: QuarterResolution, layer: 0}, + QuarterResolution: {rid: QuarterResolution, layer: 0}, + HalfResolution: {rid: HalfResolution, layer: 1}, + FullResolution: {rid: FullResolution, layer: 2}, }, }, } @@ -169,114 +170,114 @@ func TestQualityConversion(t *testing.T) { "no track info", nil, map[livekit.VideoQuality]QualityAndLayer{ - livekit.VideoQuality_LOW: QualityAndLayer{quality: livekit.VideoQuality_LOW, layer: 0}, - livekit.VideoQuality_MEDIUM: QualityAndLayer{quality: livekit.VideoQuality_MEDIUM, layer: 1}, - livekit.VideoQuality_HIGH: QualityAndLayer{quality: livekit.VideoQuality_HIGH, layer: 2}, + livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_LOW, layer: 0}, + livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 1}, + livekit.VideoQuality_HIGH: {quality: livekit.VideoQuality_HIGH, layer: 2}, }, }, { "no layers", &livekit.TrackInfo{}, map[livekit.VideoQuality]QualityAndLayer{ - livekit.VideoQuality_LOW: QualityAndLayer{quality: livekit.VideoQuality_LOW, layer: 0}, - livekit.VideoQuality_MEDIUM: QualityAndLayer{quality: livekit.VideoQuality_MEDIUM, layer: 1}, - livekit.VideoQuality_HIGH: QualityAndLayer{quality: livekit.VideoQuality_HIGH, layer: 2}, + livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_LOW, layer: 0}, + livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 1}, + livekit.VideoQuality_HIGH: {quality: livekit.VideoQuality_HIGH, layer: 2}, }, }, { "single layer, low", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - &livekit.VideoLayer{Quality: livekit.VideoQuality_LOW}, + {Quality: livekit.VideoQuality_LOW}, }, }, map[livekit.VideoQuality]QualityAndLayer{ - livekit.VideoQuality_LOW: QualityAndLayer{quality: livekit.VideoQuality_LOW, layer: 0}, - livekit.VideoQuality_MEDIUM: QualityAndLayer{quality: livekit.VideoQuality_LOW, layer: 0}, - livekit.VideoQuality_HIGH: QualityAndLayer{quality: livekit.VideoQuality_LOW, layer: 0}, + livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_LOW, layer: 0}, + livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_LOW, layer: 0}, + livekit.VideoQuality_HIGH: {quality: livekit.VideoQuality_LOW, layer: 0}, }, }, { "single layer, medium", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - &livekit.VideoLayer{Quality: livekit.VideoQuality_MEDIUM}, + {Quality: livekit.VideoQuality_MEDIUM}, }, }, map[livekit.VideoQuality]QualityAndLayer{ - livekit.VideoQuality_LOW: QualityAndLayer{quality: livekit.VideoQuality_MEDIUM, layer: 0}, - livekit.VideoQuality_MEDIUM: QualityAndLayer{quality: livekit.VideoQuality_MEDIUM, layer: 0}, - livekit.VideoQuality_HIGH: QualityAndLayer{quality: livekit.VideoQuality_MEDIUM, layer: 0}, + livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_MEDIUM, layer: 0}, + livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 0}, + livekit.VideoQuality_HIGH: {quality: livekit.VideoQuality_MEDIUM, layer: 0}, }, }, { "single layer, high", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - &livekit.VideoLayer{Quality: livekit.VideoQuality_HIGH}, + {Quality: livekit.VideoQuality_HIGH}, }, }, map[livekit.VideoQuality]QualityAndLayer{ - livekit.VideoQuality_LOW: QualityAndLayer{quality: livekit.VideoQuality_HIGH, layer: 0}, - livekit.VideoQuality_MEDIUM: QualityAndLayer{quality: livekit.VideoQuality_HIGH, layer: 0}, - livekit.VideoQuality_HIGH: QualityAndLayer{quality: livekit.VideoQuality_HIGH, layer: 0}, + livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_HIGH, layer: 0}, + livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_HIGH, layer: 0}, + livekit.VideoQuality_HIGH: {quality: livekit.VideoQuality_HIGH, layer: 0}, }, }, { "two layers, low and medium", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - &livekit.VideoLayer{Quality: livekit.VideoQuality_LOW}, - &livekit.VideoLayer{Quality: livekit.VideoQuality_MEDIUM}, + {Quality: livekit.VideoQuality_LOW}, + {Quality: livekit.VideoQuality_MEDIUM}, }, }, map[livekit.VideoQuality]QualityAndLayer{ - livekit.VideoQuality_LOW: QualityAndLayer{quality: livekit.VideoQuality_LOW, layer: 0}, - livekit.VideoQuality_MEDIUM: QualityAndLayer{quality: livekit.VideoQuality_MEDIUM, layer: 1}, - livekit.VideoQuality_HIGH: QualityAndLayer{quality: livekit.VideoQuality_MEDIUM, layer: 1}, + livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_LOW, layer: 0}, + livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 1}, + livekit.VideoQuality_HIGH: {quality: livekit.VideoQuality_MEDIUM, layer: 1}, }, }, { "two layers, low and high", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - &livekit.VideoLayer{Quality: livekit.VideoQuality_LOW}, - &livekit.VideoLayer{Quality: livekit.VideoQuality_HIGH}, + {Quality: livekit.VideoQuality_LOW}, + {Quality: livekit.VideoQuality_HIGH}, }, }, map[livekit.VideoQuality]QualityAndLayer{ - livekit.VideoQuality_LOW: QualityAndLayer{quality: livekit.VideoQuality_LOW, layer: 0}, - livekit.VideoQuality_MEDIUM: QualityAndLayer{quality: livekit.VideoQuality_HIGH, layer: 1}, - livekit.VideoQuality_HIGH: QualityAndLayer{quality: livekit.VideoQuality_HIGH, layer: 1}, + livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_LOW, layer: 0}, + livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_HIGH, layer: 1}, + livekit.VideoQuality_HIGH: {quality: livekit.VideoQuality_HIGH, layer: 1}, }, }, { "two layers, medium and high", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - &livekit.VideoLayer{Quality: livekit.VideoQuality_MEDIUM}, - &livekit.VideoLayer{Quality: livekit.VideoQuality_HIGH}, + {Quality: livekit.VideoQuality_MEDIUM}, + {Quality: livekit.VideoQuality_HIGH}, }, }, map[livekit.VideoQuality]QualityAndLayer{ - livekit.VideoQuality_LOW: QualityAndLayer{quality: livekit.VideoQuality_MEDIUM, layer: 0}, - livekit.VideoQuality_MEDIUM: QualityAndLayer{quality: livekit.VideoQuality_MEDIUM, layer: 0}, - livekit.VideoQuality_HIGH: QualityAndLayer{quality: livekit.VideoQuality_HIGH, layer: 1}, + livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_MEDIUM, layer: 0}, + livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 0}, + livekit.VideoQuality_HIGH: {quality: livekit.VideoQuality_HIGH, layer: 1}, }, }, { "three layers", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - &livekit.VideoLayer{Quality: livekit.VideoQuality_LOW}, - &livekit.VideoLayer{Quality: livekit.VideoQuality_MEDIUM}, - &livekit.VideoLayer{Quality: livekit.VideoQuality_HIGH}, + {Quality: livekit.VideoQuality_LOW}, + {Quality: livekit.VideoQuality_MEDIUM}, + {Quality: livekit.VideoQuality_HIGH}, }, }, map[livekit.VideoQuality]QualityAndLayer{ - livekit.VideoQuality_LOW: QualityAndLayer{quality: livekit.VideoQuality_LOW, layer: 0}, - livekit.VideoQuality_MEDIUM: QualityAndLayer{quality: livekit.VideoQuality_MEDIUM, layer: 1}, - livekit.VideoQuality_HIGH: QualityAndLayer{quality: livekit.VideoQuality_HIGH, layer: 2}, + livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_LOW, layer: 0}, + livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 1}, + livekit.VideoQuality_HIGH: {quality: livekit.VideoQuality_HIGH, layer: 2}, }, }, } @@ -322,7 +323,7 @@ func TestVideoQualityToRidConversion(t *testing.T) { "single layer, low", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - &livekit.VideoLayer{Quality: livekit.VideoQuality_LOW}, + {Quality: livekit.VideoQuality_LOW}, }, }, map[livekit.VideoQuality]string{ @@ -335,7 +336,7 @@ func TestVideoQualityToRidConversion(t *testing.T) { "single layer, medium", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - &livekit.VideoLayer{Quality: livekit.VideoQuality_MEDIUM}, + {Quality: livekit.VideoQuality_MEDIUM}, }, }, map[livekit.VideoQuality]string{ @@ -348,7 +349,7 @@ func TestVideoQualityToRidConversion(t *testing.T) { "single layer, high", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - &livekit.VideoLayer{Quality: livekit.VideoQuality_HIGH}, + {Quality: livekit.VideoQuality_HIGH}, }, }, map[livekit.VideoQuality]string{ @@ -361,8 +362,8 @@ func TestVideoQualityToRidConversion(t *testing.T) { "two layers, low and medium", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - &livekit.VideoLayer{Quality: livekit.VideoQuality_LOW}, - &livekit.VideoLayer{Quality: livekit.VideoQuality_MEDIUM}, + {Quality: livekit.VideoQuality_LOW}, + {Quality: livekit.VideoQuality_MEDIUM}, }, }, map[livekit.VideoQuality]string{ @@ -375,8 +376,8 @@ func TestVideoQualityToRidConversion(t *testing.T) { "two layers, low and high", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - &livekit.VideoLayer{Quality: livekit.VideoQuality_LOW}, - &livekit.VideoLayer{Quality: livekit.VideoQuality_HIGH}, + {Quality: livekit.VideoQuality_LOW}, + {Quality: livekit.VideoQuality_HIGH}, }, }, map[livekit.VideoQuality]string{ @@ -389,8 +390,8 @@ func TestVideoQualityToRidConversion(t *testing.T) { "two layers, medium and high", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - &livekit.VideoLayer{Quality: livekit.VideoQuality_MEDIUM}, - &livekit.VideoLayer{Quality: livekit.VideoQuality_HIGH}, + {Quality: livekit.VideoQuality_MEDIUM}, + {Quality: livekit.VideoQuality_HIGH}, }, }, map[livekit.VideoQuality]string{ @@ -403,9 +404,9 @@ func TestVideoQualityToRidConversion(t *testing.T) { "three layers", &livekit.TrackInfo{ Layers: []*livekit.VideoLayer{ - &livekit.VideoLayer{Quality: livekit.VideoQuality_LOW}, - &livekit.VideoLayer{Quality: livekit.VideoQuality_MEDIUM}, - &livekit.VideoLayer{Quality: livekit.VideoQuality_HIGH}, + {Quality: livekit.VideoQuality_LOW}, + {Quality: livekit.VideoQuality_MEDIUM}, + {Quality: livekit.VideoQuality_HIGH}, }, }, map[livekit.VideoQuality]string{ diff --git a/pkg/sfu/connectionquality/connectionstats.go b/pkg/sfu/connectionquality/connectionstats.go index 8608da3d6..2a39a3f33 100644 --- a/pkg/sfu/connectionquality/connectionstats.go +++ b/pkg/sfu/connectionquality/connectionstats.go @@ -6,13 +6,12 @@ import ( "time" "github.com/frostbyte73/core" + "github.com/pion/webrtc/v3" "go.uber.org/atomic" + "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" - "github.com/pion/webrtc/v3" - - "github.com/livekit/livekit-server/pkg/sfu/buffer" ) const ( @@ -227,13 +226,14 @@ func (cs *ConnectionStats) updateStatsWorker() { tk := time.NewTicker(interval) defer tk.Stop() + done := cs.done.Watch() for { select { - case <-cs.done.Watch(): + case <-done: return case <-tk.C: - if cs.done.IsClosed() { + if cs.done.IsBroken() { return } @@ -247,13 +247,15 @@ func (cs *ConnectionStats) updateStatsWorker() { // how much weight to give to packet loss rate when calculating score. // It is codec dependent. // For audio: -// o Opus without FEC or RED suffers the most through packet loss, hence has the highest weight -// o RED with two packet redundancy can absorb two out of every three packets lost, so packet loss is not as detrimental and therefore lower weight +// +// o Opus without FEC or RED suffers the most through packet loss, hence has the highest weight +// o RED with two packet redundancy can absorb two out of every three packets lost, so packet loss is not as detrimental and therefore lower weight // // For video: -// o No in-built codec repair available, hence same for all codecs +// +// o No in-built codec repair available, hence same for all codecs func getPacketLossWeight(mimeType string, isFecEnabled bool) float64 { - plw := float64(0.0) + var plw float64 switch { case strings.EqualFold(mimeType, webrtc.MimeTypeOpus): // 2.5%: fall to GOOD, 7.5%: fall to POOR diff --git a/pkg/sfu/connectionquality/connectionstats_test.go b/pkg/sfu/connectionquality/connectionstats_test.go index 075d24dbc..6ffa34277 100644 --- a/pkg/sfu/connectionquality/connectionstats_test.go +++ b/pkg/sfu/connectionquality/connectionstats_test.go @@ -5,10 +5,11 @@ import ( "testing" "time" + "github.com/stretchr/testify/require" + "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" - "github.com/stretchr/testify/require" ) func newConnectionStats(mimeType string, isFECEnabled bool, isDependentRTT bool, isDependentJitter bool) *ConnectionStats { @@ -38,7 +39,7 @@ func TestConnectionQuality(t *testing.T) { // best conditions (no loss, jitter/rtt = 0) - quality should stay EXCELLENT streams := map[uint32]*buffer.StreamStatsWithLayers{ - 1: &buffer.StreamStatsWithLayers{ + 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, Duration: duration, @@ -54,7 +55,7 @@ func TestConnectionQuality(t *testing.T) { // introduce loss and the score should drop - 12% loss for Opus -> POOR now = now.Add(duration) streams = map[uint32]*buffer.StreamStatsWithLayers{ - 1: &buffer.StreamStatsWithLayers{ + 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, Duration: duration, @@ -73,7 +74,7 @@ func TestConnectionQuality(t *testing.T) { // bound so that climbing back does not take too long even under excellent conditions. now = now.Add(duration) streams = map[uint32]*buffer.StreamStatsWithLayers{ - 1: &buffer.StreamStatsWithLayers{ + 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, Duration: duration, @@ -89,7 +90,7 @@ func TestConnectionQuality(t *testing.T) { // should stay at GOOD if conditions continue to be good now = now.Add(duration) streams = map[uint32]*buffer.StreamStatsWithLayers{ - 1: &buffer.StreamStatsWithLayers{ + 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, Duration: duration, @@ -105,7 +106,7 @@ func TestConnectionQuality(t *testing.T) { // should climb up to EXCELLENT if conditions continue to be good now = now.Add(duration) streams = map[uint32]*buffer.StreamStatsWithLayers{ - 1: &buffer.StreamStatsWithLayers{ + 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, Duration: duration, @@ -121,7 +122,7 @@ func TestConnectionQuality(t *testing.T) { // introduce loss and the score should drop - 5% loss for Opus -> GOOD now = now.Add(duration) streams = map[uint32]*buffer.StreamStatsWithLayers{ - 1: &buffer.StreamStatsWithLayers{ + 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, Duration: duration, @@ -138,7 +139,7 @@ func TestConnectionQuality(t *testing.T) { // should stay at GOOD quality for another iteration even if the conditions improve now = now.Add(duration) streams = map[uint32]*buffer.StreamStatsWithLayers{ - 1: &buffer.StreamStatsWithLayers{ + 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, Duration: duration, @@ -154,7 +155,7 @@ func TestConnectionQuality(t *testing.T) { // should climb up to EXCELLENT if conditions continue to be good now = now.Add(duration) streams = map[uint32]*buffer.StreamStatsWithLayers{ - 1: &buffer.StreamStatsWithLayers{ + 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, Duration: duration, @@ -170,7 +171,7 @@ func TestConnectionQuality(t *testing.T) { // mute when quality is POOR should return quality to EXCELLENT now = now.Add(duration) streams = map[uint32]*buffer.StreamStatsWithLayers{ - 1: &buffer.StreamStatsWithLayers{ + 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, Duration: duration, @@ -195,7 +196,7 @@ func TestConnectionQuality(t *testing.T) { cs.UpdateMute(false, now.Add(3*time.Second)) streams = map[uint32]*buffer.StreamStatsWithLayers{ - 1: &buffer.StreamStatsWithLayers{ + 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, Duration: duration, @@ -211,7 +212,7 @@ func TestConnectionQuality(t *testing.T) { // next update with no packets should knock quality down now = now.Add(duration) streams = map[uint32]*buffer.StreamStatsWithLayers{ - 1: &buffer.StreamStatsWithLayers{ + 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, Duration: duration, @@ -232,7 +233,7 @@ func TestConnectionQuality(t *testing.T) { // with lesser number of packet (simulating DTX). // even higher loss (like 10%) should only knock down quality to GOOD, typically would be POOR at that loss rate streams = map[uint32]*buffer.StreamStatsWithLayers{ - 1: &buffer.StreamStatsWithLayers{ + 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, Duration: duration, @@ -254,7 +255,7 @@ func TestConnectionQuality(t *testing.T) { // RTT and jitter can knock quality down. // at 2% loss, quality should stay at EXCELLENT purely based on loss, but with added RTT/jitter, should drop to GOOD streams = map[uint32]*buffer.StreamStatsWithLayers{ - 1: &buffer.StreamStatsWithLayers{ + 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, Duration: duration, @@ -280,7 +281,7 @@ func TestConnectionQuality(t *testing.T) { cs.AddBitrateTransition(2_000_000, now.Add(2*time.Second)) streams = map[uint32]*buffer.StreamStatsWithLayers{ - 1: &buffer.StreamStatsWithLayers{ + 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, Duration: duration, @@ -306,7 +307,7 @@ func TestConnectionQuality(t *testing.T) { cs.AddBitrateTransition(2_000_000, now.Add(2*time.Second)) streams = map[uint32]*buffer.StreamStatsWithLayers{ - 1: &buffer.StreamStatsWithLayers{ + 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, Duration: duration, @@ -331,7 +332,7 @@ func TestConnectionQuality(t *testing.T) { cs.AddBitrateTransition(2_000_000, now.Add(2*time.Second)) streams = map[uint32]*buffer.StreamStatsWithLayers{ - 1: &buffer.StreamStatsWithLayers{ + 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, Duration: duration, @@ -356,9 +357,9 @@ func TestConnectionQuality(t *testing.T) { // RTT does not knock quality down because it is dependent and hence not taken into account // at 2% loss, quality should stay at EXCELLENT purely based on loss. With high RTT (700 ms) - // quality should drop to GOOD if RTT were taken into consieration + // quality should drop to GOOD if RTT were taken into consideration streams := map[uint32]*buffer.StreamStatsWithLayers{ - 1: &buffer.StreamStatsWithLayers{ + 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, Duration: duration, @@ -384,9 +385,9 @@ func TestConnectionQuality(t *testing.T) { // Jitter does not knock quality down because it is dependent and hence not taken into account // at 2% loss, quality should stay at EXCELLENT purely based on loss. With high jitter (200 ms) - // quality should drop to GOOD if jitter were taken into consieration + // quality should drop to GOOD if jitter were taken into consideration streams := map[uint32]*buffer.StreamStatsWithLayers{ - 1: &buffer.StreamStatsWithLayers{ + 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, Duration: duration, @@ -548,7 +549,7 @@ func TestConnectionQuality(t *testing.T) { for _, eq := range tc.expectedQualities { streams := map[uint32]*buffer.StreamStatsWithLayers{ - 123: &buffer.StreamStatsWithLayers{ + 123: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, Duration: duration, @@ -644,7 +645,7 @@ func TestConnectionQuality(t *testing.T) { } streams := map[uint32]*buffer.StreamStatsWithLayers{ - 123: &buffer.StreamStatsWithLayers{ + 123: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, Duration: duration, @@ -731,7 +732,7 @@ func TestConnectionQuality(t *testing.T) { } streams := map[uint32]*buffer.StreamStatsWithLayers{ - 123: &buffer.StreamStatsWithLayers{ + 123: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, Duration: duration, diff --git a/pkg/sfu/connectionquality/scorer.go b/pkg/sfu/connectionquality/scorer.go index d89329cfe..7711e89ca 100644 --- a/pkg/sfu/connectionquality/scorer.go +++ b/pkg/sfu/connectionquality/scorer.go @@ -63,7 +63,7 @@ func (w *windowStat) calculatePacketScore(plw float64, isDependentRTT bool, isDe actualLost = 0 } - lossEffect := float64(0.0) + var lossEffect float64 if w.packetsExpected > 0 { lossEffect = float64(actualLost) * 100.0 / float64(w.packetsExpected) } @@ -83,7 +83,7 @@ func (w *windowStat) calculateBitrateScore(expectedBitrate int64) float64 { return maxScore } - score := float64(0.0) + var score float64 if w.bytes != 0 { // using the ratio of expectedBitrate / actualBitrate // the quality inflection points are approximately @@ -369,7 +369,7 @@ func (q *qualityScorer) getExpectedBitsAndUpdateTransitions(at time.Time) int64 } var startedAt time.Time - totalBits := float64(0.0) + var totalBits float64 for idx := 0; idx < len(q.bitrateTransitions)-1; idx++ { bt := &q.bitrateTransitions[idx] btNext := &q.bitrateTransitions[idx+1] @@ -391,8 +391,8 @@ func (q *qualityScorer) getExpectedBitsAndUpdateTransitions(at time.Time) int64 } totalBits += at.Sub(startedAt).Seconds() * float64(bt.bitrate) - // set up last bit rate as the startig bit rate for next analysis window - q.bitrateTransitions = []bitrateTransition{bitrateTransition{ + // set up last bit rate as the starting bit rate for next analysis window + q.bitrateTransitions = []bitrateTransition{{ startedAt: at, bitrate: bt.bitrate, }} @@ -406,7 +406,7 @@ func (q *qualityScorer) getExpectedDistanceAndUpdateTransitions(at time.Time) fl } var startedAt time.Time - totalDistance := float64(0.0) + var totalDistance float64 totalDuration := time.Duration(0) for idx := 0; idx < len(q.layerTransitions)-1; idx++ { lt := &q.layerTransitions[idx] @@ -425,7 +425,7 @@ func (q *qualityScorer) getExpectedDistanceAndUpdateTransitions(at time.Time) fl // negative distances are overshoot, that does not compensate for shortfalls, so use optimal, i. e. 0 distance when overshooting dist = 0.0 } - totalDistance += dur.Seconds() * float64(dist) + totalDistance += dur.Seconds() * dist } // last transition @@ -442,10 +442,10 @@ func (q *qualityScorer) getExpectedDistanceAndUpdateTransitions(at time.Time) fl if dist < 0.0 { dist = 0.0 } - totalDistance += dur.Seconds() * float64(dist) + totalDistance += dur.Seconds() * dist - // set up last distance as the startig distance for next analysis window - q.layerTransitions = []layerTransition{layerTransition{ + // set up last distance as the starting distance for next analysis window + q.layerTransitions = []layerTransition{{ startedAt: at, distance: lt.distance, }} diff --git a/pkg/sfu/dependencydescriptor/bitstreamreader.go b/pkg/sfu/dependencydescriptor/bitstreamreader.go index 84700f6b8..1ce60e390 100644 --- a/pkg/sfu/dependencydescriptor/bitstreamreader.go +++ b/pkg/sfu/dependencydescriptor/bitstreamreader.go @@ -6,17 +6,17 @@ import ( ) type BitStreamReader struct { - buf []byte - pos int - remaingBits int + buf []byte + pos int + remainingBits int } func NewBitStreamReader(buf []byte) *BitStreamReader { - return &BitStreamReader{buf: buf, remaingBits: len(buf) * 8} + return &BitStreamReader{buf: buf, remainingBits: len(buf) * 8} } -func (b *BitStreamReader) RemaningBits() int { - return b.remaingBits +func (b *BitStreamReader) RemainingBits() int { + return b.remainingBits } // Reads `bits` from the bitstream. `bits` must be in range [0, 64]. @@ -27,17 +27,17 @@ func (b *BitStreamReader) ReadBits(bits int) (uint64, error) { return 0, errors.New("invalid number of bits, expected 0-64") } - if b.remaingBits < bits { - b.remaingBits -= bits + if b.remainingBits < bits { + b.remainingBits -= bits return 0, io.EOF } - remainingBitsInFirstByte := b.remaingBits % 8 - b.remaingBits -= bits + remainingBitsInFirstByte := b.remainingBits % 8 + b.remainingBits -= bits if bits < remainingBitsInFirstByte { // Reading fewer bits than what's left in the current byte, just // return the portion of this byte that is needed. - offset := (remainingBitsInFirstByte - bits) + offset := remainingBitsInFirstByte - bits return uint64((b.buf[b.pos] >> offset) & ((1 << bits) - 1)), nil } var result uint64 @@ -69,11 +69,11 @@ func (b *BitStreamReader) ReadBool() (bool, error) { } func (b *BitStreamReader) Ok() bool { - return b.remaingBits >= 0 + return b.remainingBits >= 0 } func (b *BitStreamReader) Invalidate() { - b.remaingBits = -1 + b.remainingBits = -1 } // Reads value in range [0, `num_values` - 1]. @@ -107,8 +107,8 @@ func (b *BitStreamReader) ReadNonSymmetric(numValues uint32) (uint32, error) { return uint32((val << 1) + bit - uint64(numMinBitsValues)), nil } -func (b *BitStreamReader) ReadedBytes() int { - if b.remaingBits%8 > 0 { +func (b *BitStreamReader) BytesRead() int { + if b.remainingBits%8 > 0 { return b.pos + 1 } return b.pos diff --git a/pkg/sfu/dependencydescriptor/bitstreamwriter.go b/pkg/sfu/dependencydescriptor/bitstreamwriter.go index 671fcdc50..0461792aa 100644 --- a/pkg/sfu/dependencydescriptor/bitstreamwriter.go +++ b/pkg/sfu/dependencydescriptor/bitstreamwriter.go @@ -27,7 +27,7 @@ func (w *BitStreamWriter) WriteBits(val uint64, bitCount int) error { totalBits := bitCount // push bits to the highest bits of uint64 - val <<= (64 - bitCount) + val <<= 64 - bitCount buf := w.buf[w.pos:] diff --git a/pkg/sfu/dependencydescriptor/dependencydescriptorextension.go b/pkg/sfu/dependencydescriptor/dependencydescriptorextension.go index 8316b6b01..4d6e339cc 100644 --- a/pkg/sfu/dependencydescriptor/dependencydescriptorextension.go +++ b/pkg/sfu/dependencydescriptor/dependencydescriptorextension.go @@ -85,17 +85,17 @@ func (d *DependencyDescriptor) String() string { type DecodeTargetIndication int const ( - DecodeTargetNotPresent DecodeTargetIndication = iota // DecodeTargetInfo symbol '-' - DecodeTargetDiscadable // DecodeTargetInfo symbol 'D' - DecodeTargetSwitch // DecodeTargetInfo symbol 'S' - DecodeTargetRequired // DecodeTargetInfo symbol 'R' + DecodeTargetNotPresent DecodeTargetIndication = iota // DecodeTargetInfo symbol '-' + DecodeTargetDiscardable // DecodeTargetInfo symbol 'D' + DecodeTargetSwitch // DecodeTargetInfo symbol 'S' + DecodeTargetRequired // DecodeTargetInfo symbol 'R' ) func (i DecodeTargetIndication) String() string { switch i { case DecodeTargetNotPresent: return "-" - case DecodeTargetDiscadable: + case DecodeTargetDiscardable: return "D" case DecodeTargetSwitch: return "S" diff --git a/pkg/sfu/dependencydescriptor/dependencydescriptorextension_test.go b/pkg/sfu/dependencydescriptor/dependencydescriptorextension_test.go index 25cb1b3bc..6580c8842 100644 --- a/pkg/sfu/dependencydescriptor/dependencydescriptorextension_test.go +++ b/pkg/sfu/dependencydescriptor/dependencydescriptorextension_test.go @@ -35,7 +35,7 @@ func TestDependencyDescriptorUnmarshal(t *testing.T) { } var ddVal DependencyDescriptor - var d DependencyDescriptorExtension = DependencyDescriptorExtension{ + var d = DependencyDescriptorExtension{ Structure: structure, Descriptor: &ddVal, } diff --git a/pkg/sfu/dependencydescriptor/dependencydescriptorreader.go b/pkg/sfu/dependencydescriptor/dependencydescriptorreader.go index fdfff8f89..99f3fd68a 100644 --- a/pkg/sfu/dependencydescriptor/dependencydescriptorreader.go +++ b/pkg/sfu/dependencydescriptor/dependencydescriptorreader.go @@ -59,7 +59,7 @@ func (r *DependencyDescriptorReader) Parse() (int, error) { if err != nil { return 0, err } - return r.buffer.ReadedBytes(), nil + return r.buffer.BytesRead(), nil } func (r *DependencyDescriptorReader) readMandatoryFields() error { diff --git a/pkg/sfu/dependencydescriptor/dependencydescriptorwriter.go b/pkg/sfu/dependencydescriptor/dependencydescriptorwriter.go index 0f6972898..0c4d5164f 100644 --- a/pkg/sfu/dependencydescriptor/dependencydescriptorwriter.go +++ b/pkg/sfu/dependencydescriptor/dependencydescriptorwriter.go @@ -47,7 +47,7 @@ func (w *DependencyDescriptorWriter) Write() error { return err } - if w.hasExtenedFields() { + if w.hasExtendedFields() { if err := w.writeExtendedFields(); err != nil { return err } @@ -57,15 +57,15 @@ func (w *DependencyDescriptorWriter) Write() error { } } - remaingBits := w.writer.RemainingBits() + remainingBits := w.writer.RemainingBits() // Zero remaining memory to avoid leaving it uninitialized. - if remaingBits%64 != 0 { - if err := w.writeBits(0, remaingBits%64); err != nil { + if remainingBits%64 != 0 { + if err := w.writeBits(0, remainingBits%64); err != nil { return err } } - for i := 0; i < remaingBits/64; i++ { + for i := 0; i < remainingBits/64; i++ { if err := w.writeBits(0, 64); err != nil { return err } @@ -101,10 +101,10 @@ func (w *DependencyDescriptorWriter) findBestTemplate() error { } // Search if there any better template that have small extra size. - w.bestTemplate = w.caculateMatch(firstSameLayerIdx, firstSameLayer) + w.bestTemplate = w.calculateMatch(firstSameLayerIdx, firstSameLayer) for i := firstSameLayerIdx + 1; i <= lastSameLayerIdx; i++ { t := w.structure.Templates[i] - match := w.caculateMatch(i, t) + match := w.calculateMatch(i, t) if match.ExtraSizeBits < w.bestTemplate.ExtraSizeBits { w.bestTemplate = match } @@ -127,7 +127,7 @@ func (w *DependencyDescriptorWriter) findBestTemplate() error { // return true // } -func (w *DependencyDescriptorWriter) caculateMatch(idx int, template *FrameDependencyTemplate) TemplateMatch { +func (w *DependencyDescriptorWriter) calculateMatch(idx int, template *FrameDependencyTemplate) TemplateMatch { var result TemplateMatch result.TemplateIdx = idx result.NeedCustomFdiffs = w.descriptor.FrameDependencies.FrameDiffs != nil && !reflect.DeepEqual(w.descriptor.FrameDependencies.FrameDiffs, template.FrameDiffs) @@ -191,13 +191,13 @@ func (w *DependencyDescriptorWriter) writeBool(val bool) error { } func (w *DependencyDescriptorWriter) writeBits(val uint64, bitCount int) error { - if err := w.writer.WriteBits(uint64(val), bitCount); err != nil { + if err := w.writer.WriteBits(val, bitCount); err != nil { return err } return nil } -func (w *DependencyDescriptorWriter) hasExtenedFields() bool { +func (w *DependencyDescriptorWriter) hasExtendedFields() bool { return w.bestTemplate.ExtraSizeBits > 0 || w.descriptor.AttachedStructure != nil || w.descriptor.ActiveDecodeTargetsBitmask != nil } @@ -448,7 +448,7 @@ const mandatoryFieldSize = 1 + 1 + 6 + 16 func (w *DependencyDescriptorWriter) ValueSizeBits() int { valueSizeBits := mandatoryFieldSize + w.bestTemplate.ExtraSizeBits - if w.hasExtenedFields() { + if w.hasExtendedFields() { valueSizeBits += 5 if w.descriptor.AttachedStructure != nil { valueSizeBits += w.structureSizeBits() diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 1191bf5e0..e1469d4df 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -143,7 +143,7 @@ type DownTrackStreamAllocatorListener interface { // subscribed max video layer changed OnSubscribedLayersChanged(dt *DownTrack, layers VideoLayers) - // target video layer reaached + // target video layer reached OnTargetLayerReached(dt *DownTrack) // packet(s) sent @@ -760,7 +760,7 @@ func (d *DownTrack) handleMute(muted bool, isPub bool, changed bool, maxLayers V // 2. down track(s) notifying max layer // 3. out-of-band notification about max layer sent back to the publisher // 4. publisher starts layer(s) - // Ideally, on publisher mute, whatever layers were active reamin active and + // Ideally, on publisher mute, whatever layers were active remain active and // can be restarted by publisher immediately on unmute. // // Note that while publisher mute is active, subscriber changes can also happen @@ -1503,7 +1503,7 @@ func (d *DownTrack) writeRTPHeaderExtensions(hdr *rtp.Header, extraExtensions .. hdr.Extensions = []rtp.Extension{} for _, ext := range extraExtensions { - hdr.SetExtension(uint8(ext.id), ext.payload) + hdr.SetExtension(ext.id, ext.payload) } if d.absSendTimeID != 0 { diff --git a/pkg/sfu/forwarder_test.go b/pkg/sfu/forwarder_test.go index 2c3784ea3..5d0695a1f 100644 --- a/pkg/sfu/forwarder_test.go +++ b/pkg/sfu/forwarder_test.go @@ -518,7 +518,7 @@ func TestForwarderProvisionalAllocate(t *testing.T) { } expectedResult = VideoAllocation{ bandwidthRequested: bitrates[1][3], - bandwidthDelta: bitrates[1][3] - 1, // 1 is the last allocation bandwith requested + bandwidthDelta: bitrates[1][3] - 1, // 1 is the last allocation bandwidth requested bitrates: bitrates, targetLayers: expectedTargetLayers, requestLayerSpatial: expectedTargetLayers.Spatial, @@ -562,7 +562,7 @@ func TestForwarderProvisionalAllocate(t *testing.T) { expectedResult = VideoAllocation{ pauseReason: VideoPauseReasonFeedDry, bandwidthRequested: bitrates[0][2], - bandwidthDelta: bitrates[0][2] - 8, // 8 is the last allocation bandwith requested + bandwidthDelta: bitrates[0][2] - 8, // 8 is the last allocation bandwidth requested bitrates: bitrates, targetLayers: expectedTargetLayers, requestLayerSpatial: expectedTargetLayers.Spatial, @@ -787,7 +787,7 @@ func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { require.Equal(t, expectedLayers, f.TargetLayers()) // - // Test continuting at current layers when feed is dry + // Test continuing at current layers when feed is dry // bitrates = Bitrates{ {0, 0, 0, 0}, diff --git a/pkg/sfu/streamtracker/streamtracker.go b/pkg/sfu/streamtracker/streamtracker.go index 9f3a611a2..b1d16dae6 100644 --- a/pkg/sfu/streamtracker/streamtracker.go +++ b/pkg/sfu/streamtracker/streamtracker.go @@ -5,8 +5,9 @@ import ( "sync" "time" - "github.com/livekit/protocol/logger" "go.uber.org/atomic" + + "github.com/livekit/protocol/logger" ) // ------------------------------------------------------------ diff --git a/pkg/sfu/streamtracker/streamtracker_frame.go b/pkg/sfu/streamtracker/streamtracker_frame.go index 02d9ad7ae..87273bcc1 100644 --- a/pkg/sfu/streamtracker/streamtracker_frame.go +++ b/pkg/sfu/streamtracker/streamtracker_frame.go @@ -151,7 +151,7 @@ func (s *StreamTrackerFrame) updateEstimatedFrameRate() float64 { s.oldestTS = s.newestTS s.numFrames = 1 - factor := float64(1.0) + factor := 1.0 switch { case s.estimatedFrameRate < frameRate: // slow increase, prevents shortening eval interval too quickly on frame rate going up diff --git a/pkg/sfu/streamtrackermanager.go b/pkg/sfu/streamtrackermanager.go index b2d1f894a..62c4f82e4 100644 --- a/pkg/sfu/streamtrackermanager.go +++ b/pkg/sfu/streamtrackermanager.go @@ -7,6 +7,7 @@ import ( "time" "github.com/frostbyte73/core" + "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/livekit-server/pkg/sfu/streamtracker" diff --git a/pkg/sfu/videolayerselector.go b/pkg/sfu/videolayerselector.go index 507bfe49f..ec08065ac 100644 --- a/pkg/sfu/videolayerselector.go +++ b/pkg/sfu/videolayerselector.go @@ -78,7 +78,7 @@ func (s *DDVideoLayerSelector) Select(expPkt *buffer.ExtPacket, tp *TranslationP } if currentTarget < 0 { - // s.logger.Debugw(fmt.Sprintf("drop packet for no target found, deocdeTargets %v, selected layer %v, s:%d, t:%d", + // s.logger.Debugw(fmt.Sprintf("drop packet for no target found, decodeTargets %v, selected layer %v, s:%d, t:%d", // s.decodeTargetLayer, s.layer, expPkt.DependencyDescriptor.FrameDependencies.SpatialId, expPkt.DependencyDescriptor.FrameDependencies.TemporalId)) // no active decode target, forward all packets return false @@ -178,25 +178,25 @@ func (s *DDVideoLayerSelector) updateDependencyStructure(structure *dd.FrameDepe // DD-TODO : use generic wrapper when updated to go 1.18 type Uint16Wrapper struct { - last_value *uint16 + lastValue *uint16 lastUnwrapped int32 } func (w *Uint16Wrapper) Unwrap(value uint16) int32 { - if w.last_value == nil { - w.last_value = &value + if w.lastValue == nil { + w.lastValue = &value w.lastUnwrapped = int32(value) - return int32(*w.last_value) + return int32(*w.lastValue) } - diff := value - *w.last_value + diff := value - *w.lastValue w.lastUnwrapped += int32(diff) - if diff == 0x8000 && value < *w.last_value { + if diff == 0x8000 && value < *w.lastValue { w.lastUnwrapped -= 0x10000 } else if diff > 0x8000 { w.lastUnwrapped -= 0x10000 } - *w.last_value = value + *w.lastValue = value return w.lastUnwrapped } diff --git a/pkg/telemetry/signalanddatastats.go b/pkg/telemetry/signalanddatastats.go index b218582cd..0ff189ca7 100644 --- a/pkg/telemetry/signalanddatastats.go +++ b/pkg/telemetry/signalanddatastats.go @@ -53,31 +53,31 @@ func (s *BytesTrackStats) Report() { s.report(true) } -func (p *BytesTrackStats) report(force bool) { +func (s *BytesTrackStats) report(force bool) { now := time.Now() if !force { - lr := p.lastStatsReport.Load().(*time.Time) + lr := s.lastStatsReport.Load().(*time.Time) if time.Since(*lr) < statsReportInterval { return } - if !p.lastStatsReport.CompareAndSwap(lr, &now) { + if !s.lastStatsReport.CompareAndSwap(lr, &now) { return } } else { - p.lastStatsReport.Store(&now) + s.lastStatsReport.Store(&now) } - if recv := p.recv.Swap(0); recv > 0 { - p.telemetry.TrackStats(StatsKeyForData(livekit.StreamType_UPSTREAM, p.pID, p.trackID), &livekit.AnalyticsStat{ + if recv := s.recv.Swap(0); recv > 0 { + s.telemetry.TrackStats(StatsKeyForData(livekit.StreamType_UPSTREAM, s.pID, s.trackID), &livekit.AnalyticsStat{ Streams: []*livekit.AnalyticsStream{ {PrimaryBytes: recv}, }, }) } - if send := p.send.Swap(0); send > 0 { - p.telemetry.TrackStats(StatsKeyForData(livekit.StreamType_DOWNSTREAM, p.pID, p.trackID), &livekit.AnalyticsStat{ + if send := s.send.Swap(0); send > 0 { + s.telemetry.TrackStats(StatsKeyForData(livekit.StreamType_DOWNSTREAM, s.pID, s.trackID), &livekit.AnalyticsStat{ Streams: []*livekit.AnalyticsStream{ {PrimaryBytes: send}, }, diff --git a/test/client/client.go b/test/client/client.go index b886d90f8..76a347e87 100644 --- a/test/client/client.go +++ b/test/client/client.go @@ -630,7 +630,7 @@ func (c *RTCClient) ensurePublisherConnected() error { } } -func (c *RTCClient) handleDataMessage(kind livekit.DataPacket_Kind, data []byte) { +func (c *RTCClient) handleDataMessage(_ livekit.DataPacket_Kind, data []byte) { dp := &livekit.DataPacket{} err := proto.Unmarshal(data, dp) if err != nil { From e76f7f65aa34daf80cbfac6e5bb76abc52b297c4 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 23 Mar 2023 00:08:33 -0700 Subject: [PATCH 026/324] Update module github.com/livekit/protocol to v1.5.1 (#1482) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index aecb0e2cd..14da7bb87 100644 --- a/go.mod +++ b/go.mod @@ -18,9 +18,9 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230130133657-96cfb115473a - github.com/livekit/protocol v1.5.1-0.20230314035739-6d1cd857eb3b + github.com/livekit/protocol v1.5.1 github.com/livekit/psrpc v0.2.10-0.20230310095745-5cd63568998d - github.com/mackerelio/go-osstat v0.2.3 + github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.14.0 github.com/maxbrunsfeld/counterfeiter/v6 v6.6.1 github.com/mitchellh/go-homedir v1.1.0 @@ -49,7 +49,7 @@ require ( go.uber.org/atomic v1.10.0 go.uber.org/zap v1.24.0 golang.org/x/sync v0.1.0 - google.golang.org/protobuf v1.29.1 + google.golang.org/protobuf v1.30.0 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go.sum b/go.sum index c3f9d7c71..7c54fa60d 100644 --- a/go.sum +++ b/go.sum @@ -233,12 +233,12 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20230130133657-96cfb115473a h1:5UkGQpskXp7HcBmyrCwWtO7ygDWbqtjN09Yva4l/nyE= github.com/livekit/mediatransportutil v0.0.0-20230130133657-96cfb115473a/go.mod h1:1Dlx20JPoIKGP45eo+yuj0HjeE25zmyeX/EWHiPCjFw= -github.com/livekit/protocol v1.5.1-0.20230314035739-6d1cd857eb3b h1:rDm6mdo22cGAFNLwgNhyGwkjxR2civBlspf5//7LIeQ= -github.com/livekit/protocol v1.5.1-0.20230314035739-6d1cd857eb3b/go.mod h1:hkK/G0wwFiLUGp9F5kxeQxq2CQuIzkmfBwKhTsc71us= +github.com/livekit/protocol v1.5.1 h1:K/p0ByfXuPv6yLeTcoQJn7pHI7bW4IKOSyb2ueG/qq0= +github.com/livekit/protocol v1.5.1/go.mod h1:m5PkhcDT0EWdhatB0MpjmMaxyySjfE5NyZQC/LJWfEM= github.com/livekit/psrpc v0.2.10-0.20230310095745-5cd63568998d h1:3wfbd8zi7zGQCR+xfG3r2k9m2RwXUiIzR0SN4BHewwU= github.com/livekit/psrpc v0.2.10-0.20230310095745-5cd63568998d/go.mod h1:K0j8f1PgLShR7Lx80KbmwFkDH2BvOnycXGV0OSRURKc= -github.com/mackerelio/go-osstat v0.2.3 h1:jAMXD5erlDE39kdX2CU7YwCGRcxIO33u/p8+Fhe5dJw= -github.com/mackerelio/go-osstat v0.2.3/go.mod h1:DQbPOnsss9JHIXgBStc/dnhhir3gbd3YH+Dbdi7ptMA= +github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= +github.com/mackerelio/go-osstat v0.2.4/go.mod h1:Zy+qzGdZs3A9cuIqmgbJvwbmLQH9dJvtio5ZjJTbdlQ= github.com/magefile/mage v1.14.0 h1:6QDX3g6z1YvJ4olPhT1wksUcSa/V0a1B+pJb73fBjyo= github.com/magefile/mage v1.14.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= @@ -759,8 +759,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.29.1 h1:7QBf+IK2gx70Ap/hDsOmam3GE0v9HicjfEdAxE62UoM= -google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From db2e9f1f8be2d95f6595b3a7c2b3d484745fcf08 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 23 Mar 2023 13:16:14 +0530 Subject: [PATCH 027/324] Use layer 2 for SVC always. (#1542) This fixes the case of screen share forwarding. We should probably also look at proper AddTrack. The problem was that - AddTrack used two layers for screen share from JS sample app - Track was published with rid = f. Given that and the track info, consistent layer mapping set the layer as 1. - `getBufferLocked` always uses the highest layer for SVC - Between the two, when down track was requesting PLI, there was no buffer at the requested layer and hence no PLI went out. A few other notes - Tried locking SVC to layer 0 (instead of layer 2), but that resulted in PLI layer lock spamming. It did not happen in v1.3.0 of the server though. Not sure what causes that. Need to investigate later. But, that does not happen when using layer 2 buffer as SVC buffer. - When using layer 2 for SVC, the PLI throttle config will be using that of layer 2. Is that okay? - `buffer` structure should maintain more stats about spatial layers for SVC case so that layer stats can be reported to analytics/scoring etc. - In general, `buffer` may need some more hooks to make it SVC aware so that it can handle various spetial layer aware/specific bits. --- pkg/sfu/connectionquality/scorer.go | 5 ++++- pkg/sfu/receiver.go | 7 ++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/pkg/sfu/connectionquality/scorer.go b/pkg/sfu/connectionquality/scorer.go index 7711e89ca..af7861d15 100644 --- a/pkg/sfu/connectionquality/scorer.go +++ b/pkg/sfu/connectionquality/scorer.go @@ -252,13 +252,14 @@ func (q *qualityScorer) Update(stat *windowStat, at time.Time) { return } + plw := q.getPacketLossWeight(stat) reason := "none" var score float64 if stat.packetsExpected == 0 { reason = "dry" score = poorScore } else { - packetScore := stat.calculatePacketScore(q.getPacketLossWeight(stat), q.params.IsDependentRTT, q.params.IsDependentJitter) + packetScore := stat.calculatePacketScore(plw, q.params.IsDependentRTT, q.params.IsDependentJitter) bitrateScore := stat.calculateBitrateScore(expectedBitrate) layerScore := math.Max(math.Min(maxScore, maxScore-(expectedDistance*distanceWeight)), 0.0) @@ -302,6 +303,8 @@ func (q *qualityScorer) Update(stat *windowStat, at time.Time) { "score", score, "quality", scoreToConnectionQuality(score), "stat", stat, + "packetLossWeight", plw, + "maxPPS", q.maxPPS, "expectedBitrate", expectedBitrate, "expectedDistance", expectedDistance, ) diff --git a/pkg/sfu/receiver.go b/pkg/sfu/receiver.go index 05d7303e2..67f00f22c 100644 --- a/pkg/sfu/receiver.go +++ b/pkg/sfu/receiver.go @@ -298,7 +298,12 @@ func (w *WebRTCReceiver) AddUpTrack(track *webrtc.TrackRemote, buff *buffer.Buff } layer := int32(0) - if w.Kind() == webrtc.RTPCodecTypeVideo { + // for svc codecs, use layer full quality instead. + // we only have buffer for full quality + if w.isSVC { + layer = int32(len(w.buffers)) - 1 + } + if w.Kind() == webrtc.RTPCodecTypeVideo && !w.isSVC { layer = buffer.RidToSpatialLayer(track.RID(), w.trackInfo) } buff.SetLogger(w.logger.WithValues("layer", layer)) From d7750a60ec566bc4f8c4bc24eee6f99baa978d38 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 23 Mar 2023 17:23:14 +0530 Subject: [PATCH 028/324] Unify the forwarder between dependency descriptor and no DD case. (#1543) --- pkg/sfu/downtrack.go | 8 +- pkg/sfu/forwarder.go | 180 ++++++++++++++++++++----------------------- pkg/sfu/receiver.go | 11 +-- 3 files changed, 92 insertions(+), 107 deletions(-) diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index e1469d4df..9a4507fae 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -1277,10 +1277,10 @@ func (d *DownTrack) handleRTCP(bytes []byte) { pliOnce := true sendPliOnce := func() { if pliOnce { - targetLayers := d.forwarder.TargetLayers() - if targetLayers != InvalidLayers && !d.forwarder.IsAnyMuted() { - d.logger.Debugw("sending PLI RTCP", "layer", targetLayers.Spatial) - d.receiver.SendPLI(targetLayers.Spatial, false) + _, layer := d.forwarder.CheckSync() + if layer != InvalidLayerSpatial && !d.forwarder.IsAnyMuted() { + d.logger.Debugw("sending PLI RTCP", "layer", layer) + d.receiver.SendPLI(layer, false) d.isNACKThrottled.Store(true) d.rtpStats.UpdatePliTime() pliOnce = false diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 27266091c..69aab504c 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -1480,94 +1480,37 @@ func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer in f.rtpMunger.UpdateAndGetSnTs(extPkt) // call to update highest incoming sequence number and other internal structures f.rtpMunger.PacketDropped(extPkt) return tp, nil - } else if f.targetLayers.Spatial != f.currentLayers.Spatial && f.targetLayers.Spatial == layer && (extPkt.KeyFrame || tp.isSwitchingToTargetLayer) { - // lock to target layer - f.logger.Infow( - "locking to target layer", - "current", f.currentLayers, - "target", f.targetLayers, - "req", f.requestLayerSpatial, - "feed", extPkt.Packet.SSRC, - ) - f.currentLayers.Spatial = f.targetLayers.Spatial - if !f.isTemporalSupported { - f.currentLayers.Temporal = f.targetLayers.Temporal - } - // TODO : we switch to target layer immediately now since we assume all frame chain is integrity - // if we have frame chain check, should switch only if target chain is not broken and decodable - // if f.ddLayerSelector != nil { - // f.ddLayerSelector.SelectLayer(f.currentLayers) - // } - if f.currentLayers.Spatial >= f.maxLayers.Spatial { - tp.isSwitchingToMaxLayer = true - } } - } else { - if f.currentLayers.Spatial != f.targetLayers.Spatial { - // Three things to check when not locked to target - // 1. Resumable layer - don't need a key frame - // 2. Opportunistic layer upgrade - needs a key frame - // 3. Need to downgrade - needs a key frame - found := false - if f.parkedLayers.IsValid() { - if f.parkedLayers.Spatial == layer { - f.logger.Infow( - "resuming at parked layer", - "current", f.currentLayers, - "target", f.targetLayers, - "parked", f.parkedLayers, - "feed", extPkt.Packet.SSRC, - ) - f.currentLayers = f.parkedLayers - found = true - } - } else { - if extPkt.KeyFrame { - if layer > f.currentLayers.Spatial && layer <= f.targetLayers.Spatial { - f.logger.Infow( - "upgrading layer", - "current", f.currentLayers, - "target", f.targetLayers, - "max", f.maxLayers, - "layer", layer, - "req", f.requestLayerSpatial, - "maxPublished", f.maxPublishedLayer, - "feed", extPkt.Packet.SSRC, - ) - found = true - } + } - if layer < f.currentLayers.Spatial && layer >= f.targetLayers.Spatial { - f.logger.Infow( - "downgrading layer", - "current", f.currentLayers, - "target", f.targetLayers, - "max", f.maxLayers, - "layer", layer, - "req", f.requestLayerSpatial, - "maxPublished", f.maxPublishedLayer, - "feed", extPkt.Packet.SSRC, - ) - found = true - } - - if found { - f.currentLayers.Spatial = layer - if !f.isTemporalSupported { - f.currentLayers.Temporal = extPkt.Temporal - } - } - } + // at this point, either + // 1. dependency description has selected the layer for forwarding OR + // 2. non-dependency deescriptor is yet to make decision, but it can potentially switch to the incoming layer and start forwarding + // + // both cases cases upgrade/downgrade to current layer under the right conditions + if f.currentLayers.Spatial != f.targetLayers.Spatial { + // Three things to check when not locked to target + // 1. Resumable layer - don't need a key frame + // 2. Opportunistic layer upgrade - needs a key frame if not using depedency descriptor + // 3. Need to downgrade - needs a key frame if not using dependency descriptor + found := false + if f.parkedLayers.IsValid() { + if f.parkedLayers.Spatial == layer { + f.logger.Infow( + "resuming at parked layer", + "current", f.currentLayers, + "target", f.targetLayers, + "parked", f.parkedLayers, + "feed", extPkt.Packet.SSRC, + ) + f.currentLayers = f.parkedLayers + found = true } - - if found { - tp.isSwitchingToTargetLayer = true - f.clearParkedLayers() - if f.currentLayers.Spatial >= f.maxLayers.Spatial { - tp.isSwitchingToMaxLayer = true - + } else { + if extPkt.KeyFrame || tp.isSwitchingToTargetLayer { + if layer > f.currentLayers.Spatial && layer <= f.targetLayers.Spatial { f.logger.Infow( - "reached max layer", + "upgrading layer", "current", f.currentLayers, "target", f.targetLayers, "max", f.maxLayers, @@ -1576,19 +1519,40 @@ func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer in "maxPublished", f.maxPublishedLayer, "feed", extPkt.Packet.SSRC, ) + found = true } - if f.currentLayers.Spatial >= f.maxLayers.Spatial || f.currentLayers.Spatial == f.maxPublishedLayer { - f.targetLayers.Spatial = f.currentLayers.Spatial + if layer < f.currentLayers.Spatial && layer >= f.targetLayers.Spatial { + f.logger.Infow( + "downgrading layer", + "current", f.currentLayers, + "target", f.targetLayers, + "max", f.maxLayers, + "layer", layer, + "req", f.requestLayerSpatial, + "maxPublished", f.maxPublishedLayer, + "feed", extPkt.Packet.SSRC, + ) + found = true + } + + if found { + f.currentLayers.Spatial = layer + if !f.isTemporalSupported { + f.currentLayers.Temporal = f.targetLayers.Temporal + } } } } - // if locked to higher than max layer due to overshoot, check if it can be dialed back - if f.currentLayers.Spatial > f.maxLayers.Spatial { - if layer <= f.maxLayers.Spatial && extPkt.KeyFrame { + if found { + tp.isSwitchingToTargetLayer = true + f.clearParkedLayers() + if f.currentLayers.Spatial >= f.maxLayers.Spatial { + tp.isSwitchingToMaxLayer = true + f.logger.Infow( - "adjusting overshoot", + "reached max layer", "current", f.currentLayers, "target", f.targetLayers, "max", f.maxLayers, @@ -1597,14 +1561,40 @@ func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer in "maxPublished", f.maxPublishedLayer, "feed", extPkt.Packet.SSRC, ) - f.currentLayers.Spatial = layer + } - if f.currentLayers.Spatial >= f.maxLayers.Spatial { - tp.isSwitchingToMaxLayer = true + if f.currentLayers.Spatial >= f.maxLayers.Spatial || f.currentLayers.Spatial == f.maxPublishedLayer { + f.targetLayers.Spatial = f.currentLayers.Spatial + if f.ddLayerSelector != nil { + f.ddLayerSelector.SelectLayer(f.targetLayers) } + } + } + } - if f.currentLayers.Spatial >= f.maxLayers.Spatial || f.currentLayers.Spatial == f.maxPublishedLayer { - f.targetLayers.Spatial = layer + // if locked to higher than max layer due to overshoot, check if it can be dialed back + if f.currentLayers.Spatial > f.maxLayers.Spatial { + if layer <= f.maxLayers.Spatial && (extPkt.KeyFrame || tp.isSwitchingToTargetLayer) { + f.logger.Infow( + "adjusting overshoot", + "current", f.currentLayers, + "target", f.targetLayers, + "max", f.maxLayers, + "layer", layer, + "req", f.requestLayerSpatial, + "maxPublished", f.maxPublishedLayer, + "feed", extPkt.Packet.SSRC, + ) + f.currentLayers.Spatial = layer + + if f.currentLayers.Spatial >= f.maxLayers.Spatial { + tp.isSwitchingToMaxLayer = true + } + + if f.currentLayers.Spatial >= f.maxLayers.Spatial || f.currentLayers.Spatial == f.maxPublishedLayer { + f.targetLayers.Spatial = layer + if f.ddLayerSelector != nil { + f.ddLayerSelector.SelectLayer(f.targetLayers) } } } diff --git a/pkg/sfu/receiver.go b/pkg/sfu/receiver.go index 67f00f22c..c8c5d9a63 100644 --- a/pkg/sfu/receiver.go +++ b/pkg/sfu/receiver.go @@ -298,11 +298,6 @@ func (w *WebRTCReceiver) AddUpTrack(track *webrtc.TrackRemote, buff *buffer.Buff } layer := int32(0) - // for svc codecs, use layer full quality instead. - // we only have buffer for full quality - if w.isSVC { - layer = int32(len(w.buffers)) - 1 - } if w.Kind() == webrtc.RTPCodecTypeVideo && !w.isSVC { layer = buffer.RidToSpatialLayer(track.RID(), w.trackInfo) } @@ -510,10 +505,10 @@ func (w *WebRTCReceiver) getBuffer(layer int32) *buffer.Buffer { } func (w *WebRTCReceiver) getBufferLocked(layer int32) *buffer.Buffer { - // for svc codecs, use layer full quality instead. - // we only have buffer for full quality + // for svc codecs, use layer = 0 always. + // spatial layers are in-built and handled by single buffer if w.isSVC { - layer = int32(len(w.buffers)) - 1 + layer = 0 } if int(layer) >= len(w.buffers) { From d8356e012e462adcec26878e93b4cb5f07d66eb2 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Thu, 23 Mar 2023 23:57:11 -0700 Subject: [PATCH 029/324] Give proper grace period when recorder is still in the room (#1547) When a recorder is in the room, we would skip grace period due to a bug with how LastLeftAt was set. This would cause RoomComposite templates to exit immediately if the last participant in the room reconnects. --- pkg/rtc/room.go | 30 +++++++++++++++++------------- pkg/rtc/room_test.go | 8 +++++--- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index bd2f55fe4..ebdd590ef 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -27,14 +27,18 @@ import ( const ( DefaultEmptyTimeout = 5 * 60 // 5m - DefaultRoomDepartureGrace = 20 - AudioLevelQuantization = 8 // ideally power of 2 to minimize float decimal + AudioLevelQuantization = 8 // ideally power of 2 to minimize float decimal invAudioLevelQuantization = 1.0 / AudioLevelQuantization subscriberUpdateInterval = 3 * time.Second dataForwardLoadBalanceThreshold = 20 ) +var ( + // var to allow unit test override + RoomDepartureGrace uint32 = 20 +) + type broadcastOptions struct { skipSource bool immediate bool @@ -464,6 +468,11 @@ func (r *Room) RemoveParticipant(identity livekit.ParticipantIdentity, pID livek // send broadcast only if it's not already closed sendUpdates := !p.IsDisconnected() + // remove all published tracks + for _, t := range p.GetPublishedTracks() { + r.trackManager.RemoveTrack(t) + } + p.OnTrackUpdated(nil) p.OnTrackPublished(nil) p.OnTrackUnpublished(nil) @@ -476,11 +485,7 @@ func (r *Room) RemoveParticipant(identity livekit.ParticipantIdentity, pID livek r.Logger.Debugw("closing participant for removal", "pID", p.ID(), "participant", p.Identity()) _ = p.Close(true, reason) - r.lock.RLock() - if len(r.participants) == 0 { - r.leftAt.Store(time.Now().Unix()) - } - r.lock.RUnlock() + r.leftAt.Store(time.Now().Unix()) if sendUpdates { if r.onParticipantChanged != nil { @@ -597,16 +602,15 @@ func (r *Room) CloseIfEmpty() { } } - timeout := r.protoRoom.EmptyTimeout + var timeout uint32 var elapsed int64 - if r.FirstJoinedAt() > 0 { - // exit 20s after + if r.FirstJoinedAt() > 0 && r.LastLeftAt() > 0 { elapsed = time.Now().Unix() - r.LastLeftAt() - if timeout > DefaultRoomDepartureGrace { - timeout = DefaultRoomDepartureGrace - } + // need to give time in case participant is reconnecting + timeout = RoomDepartureGrace } else { elapsed = time.Now().Unix() - r.protoRoom.CreationTime + timeout = r.protoRoom.EmptyTimeout } r.lock.Unlock() diff --git a/pkg/rtc/room_test.go b/pkg/rtc/room_test.go index 2452eb6cc..9f4a4fd10 100644 --- a/pkg/rtc/room_test.go +++ b/pkg/rtc/room_test.go @@ -33,6 +33,8 @@ func init() { serverlogger.InitFromConfig(config.LoggingConfig{ Config: logger.Config{Level: "debug"}, }) + // allow immediate closure in testing + RoomDepartureGrace = 1 } var iceServersForRoom = []*livekit.ICEServer{{Urls: []string{"stun:stun.l.google.com:19302"}}} @@ -59,11 +61,11 @@ func TestJoinedState(t *testing.T) { require.LessOrEqual(t, s, rm.LastLeftAt()) }) - t.Run("LastLeftAt should not be set when there are still participants in the room", func(t *testing.T) { + t.Run("LastLeftAt should be set when there are still participants in the room", func(t *testing.T) { rm := newRoomWithParticipants(t, testRoomOpts{num: 2}) p0 := rm.GetParticipants()[0] rm.RemoveParticipant(p0.Identity(), p0.ID(), types.ParticipantCloseReasonClientRequestLeave) - require.EqualValues(t, 0, rm.LastLeftAt()) + require.Greater(t, rm.LastLeftAt(), int64(0)) }) } @@ -349,7 +351,7 @@ func TestRoomClosure(t *testing.T) { rm.protoRoom.EmptyTimeout = 0 rm.RemoveParticipant(p.Identity(), p.ID(), types.ParticipantCloseReasonClientRequestLeave) - time.Sleep(defaultDelay) + time.Sleep(time.Duration(RoomDepartureGrace)*time.Second + defaultDelay) rm.CloseIfEmpty() require.Len(t, rm.GetParticipants(), 0) From f05a3a047a4ee04bc89bf5deaaf8e1cc5a88d95c Mon Sep 17 00:00:00 2001 From: davidliu Date: Sat, 25 Mar 2023 01:27:37 +0900 Subject: [PATCH 030/324] add handling for react native and rust sdk client infos (#1544) --- pkg/service/rtcservice.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/service/rtcservice.go b/pkg/service/rtcservice.go index 6a609057e..d712cd17b 100644 --- a/pkg/service/rtcservice.go +++ b/pkg/service/rtcservice.go @@ -390,6 +390,10 @@ func (s *RTCService) ParseClientInfo(r *http.Request) *livekit.ClientInfo { ci.Sdk = livekit.ClientInfo_GO case "unity": ci.Sdk = livekit.ClientInfo_UNITY + case "reactnative": + ci.Sdk = livekit.ClientInfo_REACT_NATIVE + case "rust": + ci.Sdk = livekit.ClientInfo_RUST } ci.Version = values.Get("version") @@ -407,6 +411,7 @@ func (s *RTCService) ParseClientInfo(r *http.Request) *livekit.ClientInfo { // attempt to parse types for SDKs that support browser as a platform if ci.Sdk == livekit.ClientInfo_JS || + ci.Sdk == livekit.ClientInfo_REACT_NATIVE || ci.Sdk == livekit.ClientInfo_FLUTTER || ci.Sdk == livekit.ClientInfo_UNITY { client := s.parser.Parse(r.UserAgent()) From d269b8c935053b890a95e06ab5e89b9f9935b9f9 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Fri, 24 Mar 2023 11:16:36 -0700 Subject: [PATCH 031/324] Update README with heading and demo changes (#1541) --- README.md | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index d3aad5166..1019ad0a3 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ -# LiveKit: High-performance WebRTC +# LiveKit: Real-time video, audio and data for developers -LiveKit is an open source project that provides scalable, multi-user conferencing based on WebRTC. It's designed to -provide everything you need to build real-time video/audio/data capabilities in your applications. +[LiveKit](https://livekit.io) is an open source project that provides scalable, multi-user conferencing based on WebRTC. +It's designed to provide everything you need to build real-time video/audio/data capabilities in your applications. LiveKit's server is written in Go, using the awesome [Pion WebRTC](https://github.com/pion/webrtc) implementation. @@ -9,7 +9,7 @@ LiveKit's server is written in Go, using the awesome [Pion WebRTC](https://githu [![Slack community](https://img.shields.io/endpoint?url=https%3A%2F%2Flivekit.io%2Fbadges%2Fslack)](https://livekit.io/join-slack) [![Twitter Follow](https://img.shields.io/twitter/follow/livekitted)](https://twitter.com/livekitted) [![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/livekit/livekit)](https://github.com/livekit/livekit/releases/latest) -[![GitHub Workflow Status](https://img.shields.io/github/workflow/status/livekit/livekit/Test)](https://github.com/livekit/livekit/actions/workflows/buildtest.yaml) +[![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/livekit/livekit/buildtest.yaml?branch=master)](https://github.com/livekit/livekit/actions/workflows/buildtest.yaml) [![License](https://img.shields.io/github/license/livekit/livekit)](https://github.com/livekit/livekit/blob/master/LICENSE) ## Features @@ -32,10 +32,11 @@ LiveKit's server is written in Go, using the awesome [Pion WebRTC](https://githu https://docs.livekit.io -## Try it live +## Live Demos -Head to [our playground](https://livekit.io/playground) and give it a spin. Build a Zoom-like conferencing app in under -100 lines of code! +- [LiveKit Meet](https://meet.livekit.io) ([source](https://github.com/livekit/meet)) +- [Spatial Audio](https://spatial-audio-demo.livekit.io/) ([source](https://github.com/livekit-examples/spatial-audio)) +- Livestreaming from OBS Studio ([source](https://github.com/livekit-examples/livestream)) ## SDKs & Tools @@ -107,8 +108,9 @@ Client SDKs enable your frontend to include interactive, multi-user experiences. Compose example + - Flutter + Flutter (all platforms) client-sdk-flutter @@ -139,6 +141,15 @@ Client SDKs enable your frontend to include interactive, multi-user experiences. native + + + Rust + + client-sdk-rust + + + + ### Server SDKs @@ -159,8 +170,9 @@ enabling you to build automations that behave like end-users. ### Ecosystem & Tools -- [Egress](https://github.com/livekit/egress) - export and record your rooms - [CLI](https://github.com/livekit/livekit-cli) - command line interface & load tester +- [Egress](https://github.com/livekit/egress) - export and record your rooms +- [Ingress](https://github.com/livekit/ingress) - ingest streams from RTMP / OBS Studio - [Docker image](https://hub.docker.com/r/livekit/livekit-server) - [Helm charts](https://github.com/livekit/livekit-helm) @@ -235,11 +247,14 @@ simulation. ## Deployment ### Use LiveKit Cloud -LiveKit Cloud is the fastest and most reliable way to run LiveKit. Every project gets free monthly bandwidth and transcoding credits. + +LiveKit Cloud is the fastest and most reliable way to run LiveKit. Every project gets free monthly bandwidth and +transcoding credits. Sign up for [LiveKit Cloud](https://cloud.livekit.io/). ### Self-host + Read our [deployment docs](https://docs.livekit.io/deploy/) for more information. ## Building from source From 576eb7abbd9aad6f424abea93f2900da89d76146 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Sat, 25 Mar 2023 23:02:08 -0700 Subject: [PATCH 032/324] Update urfave/cli to fix 1 return value with livekit-server --help (#1549) Fixes #1513 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 14da7bb87..ed787301f 100644 --- a/go.mod +++ b/go.mod @@ -44,7 +44,7 @@ require ( github.com/thoas/go-funk v0.9.3 github.com/twitchtv/twirp v8.1.3+incompatible github.com/ua-parser/uap-go v0.0.0-20211112212520-00c877edfe0f - github.com/urfave/cli/v2 v2.24.2 + github.com/urfave/cli/v2 v2.25.0 github.com/urfave/negroni/v3 v3.0.0 go.uber.org/atomic v1.10.0 go.uber.org/zap v1.24.0 diff --git a/go.sum b/go.sum index 7c54fa60d..115bc0cb2 100644 --- a/go.sum +++ b/go.sum @@ -401,8 +401,8 @@ github.com/twitchtv/twirp v8.1.3+incompatible h1:+F4TdErPgSUbMZMwp13Q/KgDVuI7HJX github.com/twitchtv/twirp v8.1.3+incompatible/go.mod h1:RRJoFSAmTEh2weEqWtpPE3vFK5YBhA6bqp2l1kfCC5A= github.com/ua-parser/uap-go v0.0.0-20211112212520-00c877edfe0f h1:A+MmlgpvrHLeUP8dkBVn4Pnf5Bp5Yk2OALm7SEJLLE8= github.com/ua-parser/uap-go v0.0.0-20211112212520-00c877edfe0f/go.mod h1:OBcG9bn7sHtXgarhUEb3OfCnNsgtGnkVf41ilSZ3K3E= -github.com/urfave/cli/v2 v2.24.2 h1:q1VA+ofZ8SWfEKB9xXHUD4QZaeI9e+ItEqSbfH2JBXk= -github.com/urfave/cli/v2 v2.24.2/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= +github.com/urfave/cli/v2 v2.25.0 h1:ykdZKuQey2zq0yin/l7JOm9Mh+pg72ngYMeB0ABn6q8= +github.com/urfave/cli/v2 v2.25.0/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= github.com/urfave/negroni/v3 v3.0.0 h1:Vo8CeZfu1lFR9gW8GnAb6dOGCJyijfil9j/jKKc/JhU= github.com/urfave/negroni/v3 v3.0.0/go.mod h1:jWvnX03kcSjDBl/ShB0iHvx5uOs7mAzZXW+JvJ5XYAs= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= From 2fce780ce8863bb59d31c4da5f995e425f847da4 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 25 Mar 2023 23:17:04 -0700 Subject: [PATCH 033/324] Update go deps (#1402) * Update go deps Generated by renovateBot * use generics with Deque --------- Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: David Zhao --- go.mod | 10 +++++----- go.sum | 22 +++++++++++----------- pkg/rtc/supervisor/publication_monitor.go | 6 +++--- pkg/sfu/buffer/buffer.go | 4 ++-- pkg/sfu/prober.go | 6 +++--- 5 files changed, 24 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index ed787301f..732d17b2b 100644 --- a/go.mod +++ b/go.mod @@ -4,20 +4,20 @@ go 1.18 require ( github.com/bep/debounce v1.2.1 - github.com/d5/tengo/v2 v2.13.0 + github.com/d5/tengo/v2 v2.14.0 github.com/dustin/go-humanize v1.0.1 github.com/elliotchance/orderedmap/v2 v2.2.0 github.com/florianl/go-tc v0.4.2 github.com/frostbyte73/core v0.0.5 - github.com/gammazero/deque v0.1.0 - github.com/gammazero/workerpool v1.1.2 + github.com/gammazero/deque v0.2.1 + github.com/gammazero/workerpool v1.1.3 github.com/google/wire v0.5.0 github.com/gorilla/websocket v1.5.0 github.com/hashicorp/go-version v1.6.0 - github.com/hashicorp/golang-lru/v2 v2.0.1 + github.com/hashicorp/golang-lru/v2 v2.0.2 github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 - github.com/livekit/mediatransportutil v0.0.0-20230130133657-96cfb115473a + github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 github.com/livekit/protocol v1.5.1 github.com/livekit/psrpc v0.2.10-0.20230310095745-5cd63568998d github.com/mackerelio/go-osstat v0.2.4 diff --git a/go.sum b/go.sum index 115bc0cb2..0796fe2db 100644 --- a/go.sum +++ b/go.sum @@ -64,8 +64,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/d5/tengo/v2 v2.13.0 h1:4pZ5mR4vjOejpp+PMeIMpjZdObK7iwWoLTpVyhT+0Jk= -github.com/d5/tengo/v2 v2.13.0/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8= +github.com/d5/tengo/v2 v2.14.0 h1:ZTUyb1tGvxXGah1xDdCjaIq6ZdG4Z8PP1ZlBSN5OChg= +github.com/d5/tengo/v2 v2.14.0/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -91,10 +91,10 @@ github.com/frostbyte73/core v0.0.5 h1:+oHjXDyQyQzEx04mtmmafYP07n7EToKpUGafWbNVQ9 github.com/frostbyte73/core v0.0.5/go.mod h1:mqHHSVFS5DE6kSdhU1/s9Mm0YCnLB8Ou2DD/eX1Zbr4= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/gammazero/deque v0.1.0 h1:f9LnNmq66VDeuAlSAapemq/U7hJ2jpIWa4c09q8Dlik= -github.com/gammazero/deque v0.1.0/go.mod h1:KQw7vFau1hHuM8xmI9RbgKFbAsQFWmBpqQ2KenFLk6M= -github.com/gammazero/workerpool v1.1.2 h1:vuioDQbgrz4HoaCi2q1HLlOXdpbap5AET7xu5/qj87g= -github.com/gammazero/workerpool v1.1.2/go.mod h1:UelbXcO0zCIGFcufcirHhq2/xtLXJdQ29qZNlXG9OjQ= +github.com/gammazero/deque v0.2.1 h1:qSdsbG6pgp6nL7A0+K/B7s12mcCY/5l5SIUpMOl+dC0= +github.com/gammazero/deque v0.2.1/go.mod h1:LFroj8x4cMYCukHJDbxFCkT+r9AndaJnFMuZDV34tuU= +github.com/gammazero/workerpool v1.1.3 h1:WixN4xzukFoN0XSeXF6puqEqFTl2mECI9S6W44HWy9Q= +github.com/gammazero/workerpool v1.1.3/go.mod h1:wPjyBLDbyKnUn2XwwyD3EEwo9dHutia9/fwNmSHWACc= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -183,8 +183,8 @@ github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mO github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4= -github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/golang-lru/v2 v2.0.2 h1:Dwmkdr5Nc/oBiXgJS3CDHNhJtIHkuZ3DZF5twqnfBdU= +github.com/hashicorp/golang-lru/v2 v2.0.2/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/josharian/native v0.0.0-20200817173448-b6b71def0850/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= @@ -231,8 +231,8 @@ github.com/lithammer/shortuuid/v4 v4.0.0 h1:QRbbVkfgNippHOS8PXDkti4NaWeyYfcBTHtw github.com/lithammer/shortuuid/v4 v4.0.0/go.mod h1:Zs8puNcrvf2rV9rTH51ZLLcj7ZXqQI3lv67aw4KiB1Y= github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkDaKb5iXdynYrzB84ErPPO4LbRASk58= github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= -github.com/livekit/mediatransportutil v0.0.0-20230130133657-96cfb115473a h1:5UkGQpskXp7HcBmyrCwWtO7ygDWbqtjN09Yva4l/nyE= -github.com/livekit/mediatransportutil v0.0.0-20230130133657-96cfb115473a/go.mod h1:1Dlx20JPoIKGP45eo+yuj0HjeE25zmyeX/EWHiPCjFw= +github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 h1:QlQFyMwCDgjyySsrgmrMcVbEBA6KZcyTzvK+z346tUA= +github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26/go.mod h1:eDA41kiySZoG+wy4Etsjb3w0jjLx69i/vAmSjG4bteA= github.com/livekit/protocol v1.5.1 h1:K/p0ByfXuPv6yLeTcoQJn7pHI7bW4IKOSyb2ueG/qq0= github.com/livekit/protocol v1.5.1/go.mod h1:m5PkhcDT0EWdhatB0MpjmMaxyySjfE5NyZQC/LJWfEM= github.com/livekit/psrpc v0.2.10-0.20230310095745-5cd63568998d h1:3wfbd8zi7zGQCR+xfG3r2k9m2RwXUiIzR0SN4BHewwU= @@ -420,7 +420,7 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= +go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= diff --git a/pkg/rtc/supervisor/publication_monitor.go b/pkg/rtc/supervisor/publication_monitor.go index b51e8efd1..f7af4ed23 100644 --- a/pkg/rtc/supervisor/publication_monitor.go +++ b/pkg/rtc/supervisor/publication_monitor.go @@ -34,7 +34,7 @@ type PublicationMonitor struct { params PublicationMonitorParams lock sync.RWMutex - desiredPublishes deque.Deque + desiredPublishes deque.Deque[*publish] isConnected bool @@ -128,7 +128,7 @@ func (p *PublicationMonitor) Check() error { p.lock.RLock() var pub *publish if p.desiredPublishes.Len() > 0 { - pub = p.desiredPublishes.Front().(*publish) + pub = p.desiredPublishes.Front() } isMuted := p.isMuted @@ -160,7 +160,7 @@ func (p *PublicationMonitor) update() { for { var pub *publish if p.desiredPublishes.Len() > 0 { - pub = p.desiredPublishes.PopFront().(*publish) + pub = p.desiredPublishes.PopFront() } if pub == nil { diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go index 48ee591e2..5a04d66c1 100644 --- a/pkg/sfu/buffer/buffer.go +++ b/pkg/sfu/buffer/buffer.go @@ -52,7 +52,7 @@ type Buffer struct { videoPool *sync.Pool audioPool *sync.Pool codecType webrtc.RTPCodecType - extPackets deque.Deque + extPackets deque.Deque[*ExtPacket] pPackets []pendingPacket closeOnce sync.Once mediaSSRC uint32 @@ -290,7 +290,7 @@ func (b *Buffer) ReadExtended(buf []byte) (*ExtPacket, error) { } b.Lock() if b.extPackets.Len() > 0 { - ep := b.extPackets.PopFront().(*ExtPacket) + ep := b.extPackets.PopFront() ep = b.patchExtPacket(ep, buf) if ep == nil { b.Unlock() diff --git a/pkg/sfu/prober.go b/pkg/sfu/prober.go index 42dd61d1e..05bacd349 100644 --- a/pkg/sfu/prober.go +++ b/pkg/sfu/prober.go @@ -132,7 +132,7 @@ type Prober struct { clusterId atomic.Uint32 clustersMu sync.RWMutex - clusters deque.Deque + clusters deque.Deque[*Cluster] activeCluster *Cluster activeStateQueue []bool activeStateQueueInProcess atomic.Bool @@ -238,7 +238,7 @@ func (p *Prober) getFrontCluster() *Cluster { if p.clusters.Len() == 0 { p.activeCluster = nil } else { - p.activeCluster = p.clusters.Front().(*Cluster) + p.activeCluster = p.clusters.Front() p.activeCluster.Start() } return p.activeCluster @@ -253,7 +253,7 @@ func (p *Prober) popFrontCluster(cluster *Cluster) { return } - if p.clusters.Front().(*Cluster) == cluster { + if p.clusters.Front() == cluster { p.clusters.PopFront() } From afdae26972e70c72c5fbcaeee28fc0c134d5563a Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sun, 26 Mar 2023 12:52:50 +0530 Subject: [PATCH 034/324] Log timestamp jumps greater than 0.5 seconds. (#1551) Would be good to check if this happens and if it correlates to any A/V sync reports. --- pkg/sfu/forwarder.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 69aab504c..7cb65dcbc 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -1422,6 +1422,9 @@ func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer i if td == 0 || td > (1<<31) { f.logger.Debugw("reference timestamp out-of-order, using default", "lastTS", last.LastTS, "refTS", refTS, "td", int32(td)) td = 1 + } else if td > uint32(0.5*float32(f.codec.ClockRate)) { + // log jumps greater than 0.5 seconds + f.logger.Debugw("reference timestamp too far ahead", "lastTS", last.LastTS, "refTS", refTS, "td", td) } } else { f.logger.Debugw("reference timestamp get error, using default", "error", err) From f63962c2cc3713adf941db339728ac372cba4859 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sun, 26 Mar 2023 23:13:17 +0530 Subject: [PATCH 035/324] Pure code movement (#1552) --- pkg/rtc/mediatrackreceiver.go | 2 +- pkg/rtc/participant.go | 5 +- pkg/rtc/transport.go | 10 +- pkg/rtc/transportmanager.go | 3 +- pkg/sfu/buffer/videolayer.go | 5 + pkg/sfu/downtrack.go | 26 +- pkg/sfu/forwarder.go | 473 +++++---- pkg/sfu/forwarder_test.go | 948 +++++++++--------- pkg/sfu/receiver.go | 8 +- pkg/sfu/streamallocator/channelobserver.go | 181 ++++ pkg/sfu/{ => streamallocator}/prober.go | 2 +- .../{ => streamallocator}/streamallocator.go | 690 +------------ pkg/sfu/streamallocator/streamstateupdate.go | 63 ++ pkg/sfu/streamallocator/track.go | 255 +++++ pkg/sfu/streamallocator/trenddetector.go | 157 +++ pkg/sfu/streamtrackermanager.go | 26 +- pkg/sfu/videolayerselector.go | 14 +- 17 files changed, 1450 insertions(+), 1418 deletions(-) create mode 100644 pkg/sfu/streamallocator/channelobserver.go rename pkg/sfu/{ => streamallocator}/prober.go (99%) rename pkg/sfu/{ => streamallocator}/streamallocator.go (65%) create mode 100644 pkg/sfu/streamallocator/streamstateupdate.go create mode 100644 pkg/sfu/streamallocator/track.go create mode 100644 pkg/sfu/streamallocator/trenddetector.go diff --git a/pkg/rtc/mediatrackreceiver.go b/pkg/rtc/mediatrackreceiver.go index e1b54c1e5..0ae04a970 100644 --- a/pkg/rtc/mediatrackreceiver.go +++ b/pkg/rtc/mediatrackreceiver.go @@ -240,7 +240,7 @@ func (t *MediaTrackReceiver) SetLayerSsrc(mime string, rid string, ssrc uint32) defer t.lock.Unlock() layer := buffer.RidToSpatialLayer(rid, t.params.TrackInfo) - if layer == sfu.InvalidLayerSpatial { + if layer == buffer.InvalidLayerSpatial { // non-simulcast case will not have `rid` layer = 0 } diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 61f3eb3f0..7838f8531 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -24,6 +24,7 @@ import ( "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/livekit-server/pkg/sfu/connectionquality" + "github.com/livekit/livekit-server/pkg/sfu/streamallocator" "github.com/livekit/livekit-server/pkg/telemetry" "github.com/livekit/mediatransportutil/pkg/twcc" "github.com/livekit/protocol/auth" @@ -1338,7 +1339,7 @@ func (p *ParticipantImpl) subscriberRTCPWorker() { } } -func (p *ParticipantImpl) onStreamStateChange(update *sfu.StreamStateUpdate) error { +func (p *ParticipantImpl) onStreamStateChange(update *streamallocator.StreamStateUpdate) error { if len(update.StreamStates) == 0 { return nil } @@ -1346,7 +1347,7 @@ func (p *ParticipantImpl) onStreamStateChange(update *sfu.StreamStateUpdate) err streamStateUpdate := &livekit.StreamStateUpdate{} for _, streamStateInfo := range update.StreamStates { state := livekit.StreamState_ACTIVE - if streamStateInfo.State == sfu.StreamStatePaused { + if streamStateInfo.State == streamallocator.StreamStatePaused { state = livekit.StreamState_PAUSED } streamStateUpdate.StreamStates = append(streamStateUpdate.StreamStates, &livekit.StreamStateInfo{ diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index 14055d0a3..379ac5be6 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -26,7 +26,7 @@ import ( "github.com/livekit/livekit-server/pkg/config" serverlogger "github.com/livekit/livekit-server/pkg/logger" "github.com/livekit/livekit-server/pkg/rtc/types" - "github.com/livekit/livekit-server/pkg/sfu" + "github.com/livekit/livekit-server/pkg/sfu/streamallocator" "github.com/livekit/livekit-server/pkg/telemetry" "github.com/livekit/livekit-server/pkg/telemetry/prometheus" ) @@ -182,7 +182,7 @@ type PCTransport struct { onNegotiationFailed func() // stream allocator for subscriber PC - streamAllocator *sfu.StreamAllocator + streamAllocator *streamallocator.StreamAllocator previousAnswer *webrtc.SessionDescription // track id -> description map in previous offer sdp @@ -366,7 +366,7 @@ func NewPCTransport(params TransportParams) (*PCTransport, error) { canReuseTransceiver: true, } if params.IsSendSide { - t.streamAllocator = sfu.NewStreamAllocator(sfu.StreamAllocatorParams{ + t.streamAllocator = streamallocator.NewStreamAllocator(streamallocator.StreamAllocatorParams{ Config: params.CongestionControlConfig, Logger: params.Logger, }) @@ -1085,7 +1085,7 @@ func (t *PCTransport) ResetShortConnOnICERestart() { t.resetShortConnOnICERestart.Store(true) } -func (t *PCTransport) OnStreamStateChange(f func(update *sfu.StreamStateUpdate) error) { +func (t *PCTransport) OnStreamStateChange(f func(update *streamallocator.StreamStateUpdate) error) { if t.streamAllocator == nil { return } @@ -1098,7 +1098,7 @@ func (t *PCTransport) AddTrackToStreamAllocator(subTrack types.SubscribedTrack) return } - t.streamAllocator.AddTrack(subTrack.DownTrack(), sfu.AddTrackParams{ + t.streamAllocator.AddTrack(subTrack.DownTrack(), streamallocator.AddTrackParams{ Source: subTrack.MediaTrack().Source(), IsSimulcast: subTrack.MediaTrack().IsSimulcast(), PublisherID: subTrack.MediaTrack().PublisherID(), diff --git a/pkg/rtc/transportmanager.go b/pkg/rtc/transportmanager.go index cee3c3661..cc883a1ba 100644 --- a/pkg/rtc/transportmanager.go +++ b/pkg/rtc/transportmanager.go @@ -17,6 +17,7 @@ import ( "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/sfu" + "github.com/livekit/livekit-server/pkg/sfu/streamallocator" "github.com/livekit/livekit-server/pkg/telemetry" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" @@ -243,7 +244,7 @@ func (t *TransportManager) OnSubscriberInitialConnected(f func()) { t.onSubscriberInitialConnected = f } -func (t *TransportManager) OnSubscriberStreamStateChange(f func(update *sfu.StreamStateUpdate) error) { +func (t *TransportManager) OnSubscriberStreamStateChange(f func(update *streamallocator.StreamStateUpdate) error) { t.subscriber.OnStreamStateChange(f) } diff --git a/pkg/sfu/buffer/videolayer.go b/pkg/sfu/buffer/videolayer.go index dce60597f..15dd655f5 100644 --- a/pkg/sfu/buffer/videolayer.go +++ b/pkg/sfu/buffer/videolayer.go @@ -15,6 +15,11 @@ var ( Spatial: InvalidLayerSpatial, Temporal: InvalidLayerTemporal, } + + DefaultMaxLayers = VideoLayer{ + Spatial: DefaultMaxLayerSpatial, + Temporal: DefaultMaxLayerTemporal, + } ) type VideoLayer struct { diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 9a4507fae..c1d72f28c 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -141,7 +141,7 @@ type DownTrackStreamAllocatorListener interface { OnSubscriptionChanged(dt *DownTrack) // subscribed max video layer changed - OnSubscribedLayersChanged(dt *DownTrack, layers VideoLayers) + OnSubscribedLayersChanged(dt *DownTrack, layers buffer.VideoLayer) // target video layer reached OnTargetLayerReached(dt *DownTrack) @@ -508,7 +508,7 @@ func (d *DownTrack) stopKeyFrameRequester() { } func (d *DownTrack) keyFrameRequester(generation uint32, layer int32) { - if d.IsClosed() || layer == InvalidLayerSpatial { + if d.IsClosed() || layer == buffer.InvalidLayerSpatial { return } interval := 2 * d.rtpStats.GetRtt() @@ -739,7 +739,7 @@ func (d *DownTrack) PubMute(pubMuted bool) { d.handleMute(pubMuted, true, changed, maxLayers) } -func (d *DownTrack) handleMute(muted bool, isPub bool, changed bool, maxLayers VideoLayers) { +func (d *DownTrack) handleMute(muted bool, isPub bool, changed bool, maxLayers buffer.VideoLayer) { if !changed { return } @@ -767,7 +767,7 @@ func (d *DownTrack) handleMute(muted bool, isPub bool, changed bool, maxLayers V // and that could turn on/off layers on publisher side. // if !isPub && d.onMaxSubscribedLayerChanged != nil && d.kind == webrtc.RTPCodecTypeVideo { - notifyLayer := InvalidLayerSpatial + notifyLayer := buffer.InvalidLayerSpatial if !muted { // // When unmuting, don't wait for layer lock as @@ -861,7 +861,7 @@ func (d *DownTrack) CloseWithFlush(flush bool) { d.logger.Infow("rtp stats", "direction", "downstream", "mime", d.mime, "ssrc", d.ssrc, "stats", d.rtpStats.ToString()) if d.onMaxSubscribedLayerChanged != nil && d.kind == webrtc.RTPCodecTypeVideo { - d.onMaxSubscribedLayerChanged(d, InvalidLayerSpatial) + d.onMaxSubscribedLayerChanged(d, buffer.InvalidLayerSpatial) } if d.onCloseHandler != nil { @@ -905,7 +905,7 @@ func (d *DownTrack) SetMaxTemporalLayer(temporalLayer int32) { } } -func (d *DownTrack) MaxLayers() VideoLayers { +func (d *DownTrack) MaxLayers() buffer.VideoLayer { return d.forwarder.MaxLayers() } @@ -1008,7 +1008,7 @@ func (d *DownTrack) AllocateOptimal(allowOvershoot bool) VideoAllocation { al, brs := d.receiver.GetLayeredBitrate() allocation := d.forwarder.AllocateOptimal(al, brs, allowOvershoot) d.maybeStartKeyFrameRequester() - d.maybeAddTransition(allocation.bandwidthNeeded, allocation.distanceToDesired) + d.maybeAddTransition(allocation.BandwidthNeeded, allocation.DistanceToDesired) return allocation } @@ -1017,7 +1017,7 @@ func (d *DownTrack) ProvisionalAllocatePrepare() { d.forwarder.ProvisionalAllocatePrepare(al, brs) } -func (d *DownTrack) ProvisionalAllocate(availableChannelCapacity int64, layers VideoLayers, allowPause bool, allowOvershoot bool) int64 { +func (d *DownTrack) ProvisionalAllocate(availableChannelCapacity int64, layers buffer.VideoLayer, allowPause bool, allowOvershoot bool) int64 { return d.forwarder.ProvisionalAllocate(availableChannelCapacity, layers, allowPause, allowOvershoot) } @@ -1036,7 +1036,7 @@ func (d *DownTrack) ProvisionalAllocateGetBestWeightedTransition() VideoTransiti func (d *DownTrack) ProvisionalAllocateCommit() VideoAllocation { allocation := d.forwarder.ProvisionalAllocateCommit() d.maybeStartKeyFrameRequester() - d.maybeAddTransition(allocation.bandwidthNeeded, allocation.distanceToDesired) + d.maybeAddTransition(allocation.BandwidthNeeded, allocation.DistanceToDesired) return allocation } @@ -1044,7 +1044,7 @@ func (d *DownTrack) AllocateNextHigher(availableChannelCapacity int64, allowOver al, brs := d.receiver.GetLayeredBitrate() allocation, available := d.forwarder.AllocateNextHigher(availableChannelCapacity, al, brs, allowOvershoot) d.maybeStartKeyFrameRequester() - d.maybeAddTransition(allocation.bandwidthNeeded, allocation.distanceToDesired) + d.maybeAddTransition(allocation.BandwidthNeeded, allocation.DistanceToDesired) return allocation, available } @@ -1059,7 +1059,7 @@ func (d *DownTrack) Pause() VideoAllocation { al, brs := d.receiver.GetLayeredBitrate() allocation := d.forwarder.Pause(al, brs) d.maybeStartKeyFrameRequester() - d.maybeAddTransition(allocation.bandwidthNeeded, allocation.distanceToDesired) + d.maybeAddTransition(allocation.BandwidthNeeded, allocation.DistanceToDesired) return allocation } @@ -1278,7 +1278,7 @@ func (d *DownTrack) handleRTCP(bytes []byte) { sendPliOnce := func() { if pliOnce { _, layer := d.forwarder.CheckSync() - if layer != InvalidLayerSpatial && !d.forwarder.IsAnyMuted() { + if layer != buffer.InvalidLayerSpatial && !d.forwarder.IsAnyMuted() { d.logger.Debugw("sending PLI RTCP", "layer", layer) d.receiver.SendPLI(layer, false) d.isNACKThrottled.Store(true) @@ -1637,7 +1637,7 @@ func (d *DownTrack) onBindAndConnected() { if d.connected.Load() && d.bound.Load() && !d.bindAndConnectedOnce.Swap(true) { if d.kind == webrtc.RTPCodecTypeVideo { _, layer := d.forwarder.CheckSync() - if layer != InvalidLayerSpatial { + if layer != buffer.InvalidLayerSpatial { d.receiver.SendPLI(layer, true) } } diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 7cb65dcbc..fde7e1ccb 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -55,39 +55,39 @@ func (v VideoPauseReason) String() string { // ------------------------------------------------------------------- type VideoAllocation struct { - pauseReason VideoPauseReason - isDeficient bool - bandwidthRequested int64 - bandwidthDelta int64 - bandwidthNeeded int64 - bitrates Bitrates - targetLayers VideoLayers - requestLayerSpatial int32 - maxLayers VideoLayers - distanceToDesired float64 + PauseReason VideoPauseReason + IsDeficient bool + BandwidthRequested int64 + BandwidthDelta int64 + BandwidthNeeded int64 + Bitrates Bitrates + TargetLayers buffer.VideoLayer + RequestLayerSpatial int32 + MaxLayers buffer.VideoLayer + DistanceToDesired float64 } func (v VideoAllocation) String() string { return fmt.Sprintf("VideoAllocation{pause: %s, def: %+v, bwr: %d, del: %d, bwn: %d, rates: %+v, target: %s, req: %d, max: %s, dist: %0.2f}", - v.pauseReason, - v.isDeficient, - v.bandwidthRequested, - v.bandwidthDelta, - v.bandwidthNeeded, - v.bitrates, - v.targetLayers, - v.requestLayerSpatial, - v.maxLayers, - v.distanceToDesired, + v.PauseReason, + v.IsDeficient, + v.BandwidthRequested, + v.BandwidthDelta, + v.BandwidthNeeded, + v.Bitrates, + v.TargetLayers, + v.RequestLayerSpatial, + v.MaxLayers, + v.DistanceToDesired, ) } var ( VideoAllocationDefault = VideoAllocation{ - pauseReason: VideoPauseReasonFeedDry, // start with no feed till feed is seen - targetLayers: InvalidLayers, - requestLayerSpatial: InvalidLayerSpatial, - maxLayers: InvalidLayers, + PauseReason: VideoPauseReasonFeedDry, // start with no feed till feed is seen + TargetLayers: buffer.InvalidLayers, + RequestLayerSpatial: buffer.InvalidLayerSpatial, + MaxLayers: buffer.InvalidLayers, } ) @@ -99,23 +99,23 @@ type VideoAllocationProvisional struct { maxPublishedLayer int32 maxTemporalLayerSeen int32 availableLayers []int32 - bitrates Bitrates - maxLayers VideoLayers - currentLayers VideoLayers - parkedLayers VideoLayers - allocatedLayers VideoLayers + Bitrates Bitrates + maxLayers buffer.VideoLayer + currentLayers buffer.VideoLayer + parkedLayers buffer.VideoLayer + allocatedLayers buffer.VideoLayer } // ------------------------------------------------------------------- type VideoTransition struct { - from VideoLayers - to VideoLayers - bandwidthDelta int64 + From buffer.VideoLayer + To buffer.VideoLayer + BandwidthDelta int64 } func (v VideoTransition) String() string { - return fmt.Sprintf("VideoTransition{from: %s, to: %s, del: %d}", v.from, v.to, v.bandwidthDelta) + return fmt.Sprintf("VideoTransition{from: %s, to: %s, del: %d}", v.From, v.To, v.BandwidthDelta) } // ------------------------------------------------------------------- @@ -137,27 +137,6 @@ type TranslationParams struct { // ------------------------------------------------------------------- -type VideoLayers = buffer.VideoLayer - -const ( - InvalidLayerSpatial = buffer.InvalidLayerSpatial - InvalidLayerTemporal = buffer.InvalidLayerTemporal - - DefaultMaxLayerSpatial = buffer.DefaultMaxLayerSpatial - DefaultMaxLayerTemporal = buffer.DefaultMaxLayerTemporal -) - -var ( - InvalidLayers = buffer.InvalidLayers - - DefaultMaxLayers = VideoLayers{ - Spatial: DefaultMaxLayerSpatial, - Temporal: DefaultMaxLayerTemporal, - } -) - -// ------------------------------------------------------------------- - type ForwarderState struct { Started bool RTP RTPMungerState @@ -187,11 +166,11 @@ type Forwarder struct { lastSSRC uint32 referenceLayerSpatial int32 - maxLayers VideoLayers - currentLayers VideoLayers - targetLayers VideoLayers + maxLayers buffer.VideoLayer + currentLayers buffer.VideoLayer + targetLayers buffer.VideoLayer requestLayerSpatial int32 - parkedLayers VideoLayers // layers that can resume without key frame + parkedLayers buffer.VideoLayer // layers that can resume without key frame parkedLayersTimer *time.Timer provisional *VideoAllocationProvisional @@ -218,16 +197,16 @@ func NewForwarder( logger: logger, getReferenceLayerRTPTimestamp: getReferenceLayerRTPTimestamp, - maxPublishedLayer: InvalidLayerSpatial, - maxTemporalLayerSeen: InvalidLayerTemporal, + maxPublishedLayer: buffer.InvalidLayerSpatial, + maxTemporalLayerSeen: buffer.InvalidLayerTemporal, - referenceLayerSpatial: InvalidLayerSpatial, + referenceLayerSpatial: buffer.InvalidLayerSpatial, // start off with nothing, let streamallocator/opportunistic forwarder set the target - currentLayers: InvalidLayers, - targetLayers: InvalidLayers, - requestLayerSpatial: InvalidLayerSpatial, - parkedLayers: InvalidLayers, + currentLayers: buffer.InvalidLayers, + targetLayers: buffer.InvalidLayers, + requestLayerSpatial: buffer.InvalidLayerSpatial, + parkedLayers: buffer.InvalidLayers, lastAllocation: VideoAllocationDefault, @@ -235,9 +214,9 @@ func NewForwarder( } if f.kind == webrtc.RTPCodecTypeVideo { - f.maxLayers = VideoLayers{Spatial: InvalidLayerSpatial, Temporal: DefaultMaxLayerTemporal} + f.maxLayers = buffer.VideoLayer{Spatial: buffer.InvalidLayerSpatial, Temporal: buffer.DefaultMaxLayerTemporal} } else { - f.maxLayers = InvalidLayers + f.maxLayers = buffer.InvalidLayers } return f @@ -337,7 +316,7 @@ func (f *Forwarder) SeedState(state ForwarderState) { f.started = true } -func (f *Forwarder) Mute(muted bool) (bool, VideoLayers) { +func (f *Forwarder) Mute(muted bool) (bool, buffer.VideoLayer) { f.lock.Lock() defer f.lock.Unlock() @@ -363,7 +342,7 @@ func (f *Forwarder) IsMuted() bool { return f.muted } -func (f *Forwarder) PubMute(pubMuted bool) (bool, VideoLayers) { +func (f *Forwarder) PubMute(pubMuted bool) (bool, buffer.VideoLayer) { f.lock.Lock() defer f.lock.Unlock() @@ -385,7 +364,7 @@ func (f *Forwarder) PubMute(pubMuted bool) (bool, VideoLayers) { // On unmute, park current layers as streaming can continue without a key frame when publisher starts the stream. if !pubMuted && f.targetLayers.IsValid() && f.currentLayers.Spatial == f.targetLayers.Spatial { f.setupParkedLayers(f.targetLayers) - f.currentLayers = InvalidLayers + f.currentLayers = buffer.InvalidLayers } } @@ -406,7 +385,7 @@ func (f *Forwarder) IsAnyMuted() bool { return f.muted || f.pubMuted } -func (f *Forwarder) SetMaxSpatialLayer(spatialLayer int32) (bool, VideoLayers, VideoLayers) { +func (f *Forwarder) SetMaxSpatialLayer(spatialLayer int32) (bool, buffer.VideoLayer, buffer.VideoLayer) { f.lock.Lock() defer f.lock.Unlock() @@ -422,7 +401,7 @@ func (f *Forwarder) SetMaxSpatialLayer(spatialLayer int32) (bool, VideoLayers, V return true, f.maxLayers, f.currentLayers } -func (f *Forwarder) SetMaxTemporalLayer(temporalLayer int32) (bool, VideoLayers, VideoLayers) { +func (f *Forwarder) SetMaxTemporalLayer(temporalLayer int32) (bool, buffer.VideoLayer, buffer.VideoLayer) { f.lock.Lock() defer f.lock.Unlock() @@ -438,21 +417,21 @@ func (f *Forwarder) SetMaxTemporalLayer(temporalLayer int32) (bool, VideoLayers, return true, f.maxLayers, f.currentLayers } -func (f *Forwarder) MaxLayers() VideoLayers { +func (f *Forwarder) MaxLayers() buffer.VideoLayer { f.lock.RLock() defer f.lock.RUnlock() return f.maxLayers } -func (f *Forwarder) CurrentLayers() VideoLayers { +func (f *Forwarder) CurrentLayers() buffer.VideoLayer { f.lock.RLock() defer f.lock.RUnlock() return f.currentLayers } -func (f *Forwarder) TargetLayers() VideoLayers { +func (f *Forwarder) TargetLayers() buffer.VideoLayer { f.lock.RLock() defer f.lock.RUnlock() @@ -467,7 +446,7 @@ func (f *Forwarder) GetReferenceLayerSpatial() int32 { } func (f *Forwarder) isDeficientLocked() bool { - return f.lastAllocation.isDeficient + return f.lastAllocation.IsDeficient } func (f *Forwarder) IsDeficient() bool { @@ -482,7 +461,7 @@ func (f *Forwarder) BandwidthRequested(brs Bitrates) int64 { defer f.lock.RUnlock() if !f.targetLayers.IsValid() { - if f.targetLayers != InvalidLayers { + if f.targetLayers != buffer.InvalidLayers { f.logger.Warnw( "unexpected target layers", nil, "target", f.targetLayers, @@ -530,17 +509,17 @@ func (f *Forwarder) AllocateOptimal(availableLayers []int32, brs Bitrates, allow } alloc := VideoAllocation{ - pauseReason: VideoPauseReasonNone, - bitrates: brs, - targetLayers: InvalidLayers, - requestLayerSpatial: f.requestLayerSpatial, - maxLayers: f.maxLayers, + PauseReason: VideoPauseReasonNone, + Bitrates: brs, + TargetLayers: buffer.InvalidLayers, + RequestLayerSpatial: f.requestLayerSpatial, + MaxLayers: f.maxLayers, } optimalBandwidthNeeded := getOptimalBandwidthNeeded(f.muted, f.pubMuted, f.maxPublishedLayer, brs, f.maxLayers) if optimalBandwidthNeeded == 0 { - alloc.pauseReason = VideoPauseReasonFeedDry + alloc.PauseReason = VideoPauseReasonFeedDry } - alloc.bandwidthNeeded = optimalBandwidthNeeded + alloc.BandwidthNeeded = optimalBandwidthNeeded opportunisticAlloc := func() { // opportunistically latch on to anything @@ -548,29 +527,29 @@ func (f *Forwarder) AllocateOptimal(availableLayers []int32, brs Bitrates, allow if allowOvershoot && f.maxPublishedLayer > maxSpatial { maxSpatial = f.maxPublishedLayer } - alloc.targetLayers = VideoLayers{ + alloc.TargetLayers = buffer.VideoLayer{ Spatial: int32(math.Min(float64(f.maxPublishedLayer), float64(maxSpatial))), - Temporal: DefaultMaxLayerTemporal, + Temporal: buffer.DefaultMaxLayerTemporal, } } switch { - case !f.maxLayers.IsValid() || f.maxPublishedLayer == InvalidLayerSpatial: + case !f.maxLayers.IsValid() || f.maxPublishedLayer == buffer.InvalidLayerSpatial: // nothing to do when max layers are not valid OR max publisher layer is invalid case f.muted: - alloc.pauseReason = VideoPauseReasonMuted + alloc.PauseReason = VideoPauseReasonMuted case f.pubMuted: - alloc.pauseReason = VideoPauseReasonPubMuted + alloc.PauseReason = VideoPauseReasonPubMuted // leave it at current layers for opportunistic resume - alloc.targetLayers = f.currentLayers - alloc.requestLayerSpatial = alloc.targetLayers.Spatial + alloc.TargetLayers = f.currentLayers + alloc.RequestLayerSpatial = alloc.TargetLayers.Spatial case f.parkedLayers.IsValid(): // if parked on a layer, let it continue - alloc.targetLayers = f.parkedLayers - alloc.requestLayerSpatial = alloc.targetLayers.Spatial + alloc.TargetLayers = f.parkedLayers + alloc.RequestLayerSpatial = alloc.TargetLayers.Spatial case len(availableLayers) == 0: // feed may be dry @@ -579,12 +558,12 @@ func (f *Forwarder) AllocateOptimal(availableLayers []int32, brs Bitrates, allow // Covers the cases of // 1. mis-detection of layer stop - can continue streaming // 2. current layer resuming - can latch on when it starts - alloc.targetLayers = f.currentLayers - alloc.requestLayerSpatial = alloc.targetLayers.Spatial + alloc.TargetLayers = f.currentLayers + alloc.RequestLayerSpatial = alloc.TargetLayers.Spatial } else { // opportunistically latch on to anything opportunisticAlloc() - alloc.requestLayerSpatial = int32(math.Min(float64(f.maxLayers.Spatial), float64(f.maxPublishedLayer))) + alloc.RequestLayerSpatial = int32(math.Min(float64(f.maxLayers.Spatial), float64(f.maxPublishedLayer))) } default: @@ -601,60 +580,60 @@ func (f *Forwarder) AllocateOptimal(availableLayers []int32, brs Bitrates, allow if !isCurrentLayerAvailable && f.currentLayers.IsValid() { // current layer maybe stopped, move to highest available for _, l := range availableLayers { - if l > alloc.targetLayers.Spatial { - alloc.targetLayers.Spatial = l + if l > alloc.TargetLayers.Spatial { + alloc.TargetLayers.Spatial = l } } - alloc.targetLayers.Temporal = DefaultMaxLayerTemporal + alloc.TargetLayers.Temporal = buffer.DefaultMaxLayerTemporal - alloc.requestLayerSpatial = alloc.targetLayers.Spatial + alloc.RequestLayerSpatial = alloc.TargetLayers.Spatial } else { requestLayerSpatial := int32(math.Min(float64(f.maxLayers.Spatial), float64(f.maxPublishedLayer))) if f.currentLayers.IsValid() && requestLayerSpatial == f.requestLayerSpatial && f.currentLayers.Spatial == f.requestLayerSpatial { // current is locked to desired, stay there - alloc.targetLayers = f.currentLayers - alloc.requestLayerSpatial = f.requestLayerSpatial + alloc.TargetLayers = f.currentLayers + alloc.RequestLayerSpatial = f.requestLayerSpatial } else { // opportunistically latch on to anything opportunisticAlloc() - alloc.requestLayerSpatial = requestLayerSpatial + alloc.RequestLayerSpatial = requestLayerSpatial } } } - if !alloc.targetLayers.IsValid() { - alloc.targetLayers = InvalidLayers - alloc.requestLayerSpatial = InvalidLayerSpatial + if !alloc.TargetLayers.IsValid() { + alloc.TargetLayers = buffer.InvalidLayers + alloc.RequestLayerSpatial = buffer.InvalidLayerSpatial } - if alloc.targetLayers.IsValid() { - alloc.bandwidthRequested = optimalBandwidthNeeded + if alloc.TargetLayers.IsValid() { + alloc.BandwidthRequested = optimalBandwidthNeeded } - alloc.bandwidthDelta = alloc.bandwidthRequested - f.lastAllocation.bandwidthRequested - alloc.distanceToDesired = getDistanceToDesired( + alloc.BandwidthDelta = alloc.BandwidthRequested - f.lastAllocation.BandwidthRequested + alloc.DistanceToDesired = getDistanceToDesired( f.muted, f.pubMuted, f.maxPublishedLayer, f.maxTemporalLayerSeen, availableLayers, brs, - alloc.targetLayers, + alloc.TargetLayers, f.maxLayers, ) return f.updateAllocation(alloc, "optimal") } -func (f *Forwarder) ProvisionalAllocatePrepare(availableLayers []int32, bitrates Bitrates) { +func (f *Forwarder) ProvisionalAllocatePrepare(availableLayers []int32, Bitrates Bitrates) { f.lock.Lock() defer f.lock.Unlock() f.provisional = &VideoAllocationProvisional{ - allocatedLayers: InvalidLayers, + allocatedLayers: buffer.InvalidLayers, muted: f.muted, pubMuted: f.pubMuted, maxPublishedLayer: f.maxPublishedLayer, maxTemporalLayerSeen: f.maxTemporalLayerSeen, - bitrates: bitrates, + Bitrates: Bitrates, maxLayers: f.maxLayers, currentLayers: f.currentLayers, parkedLayers: f.parkedLayers, @@ -664,22 +643,22 @@ func (f *Forwarder) ProvisionalAllocatePrepare(availableLayers []int32, bitrates copy(f.provisional.availableLayers, availableLayers) } -func (f *Forwarder) ProvisionalAllocate(availableChannelCapacity int64, layers VideoLayers, allowPause bool, allowOvershoot bool) int64 { +func (f *Forwarder) ProvisionalAllocate(availableChannelCapacity int64, layers buffer.VideoLayer, allowPause bool, allowOvershoot bool) int64 { f.lock.Lock() defer f.lock.Unlock() - if f.provisional.muted || f.provisional.pubMuted || f.provisional.maxPublishedLayer == InvalidLayerSpatial || !f.provisional.maxLayers.IsValid() || (!allowOvershoot && layers.GreaterThan(f.provisional.maxLayers)) { + if f.provisional.muted || f.provisional.pubMuted || f.provisional.maxPublishedLayer == buffer.InvalidLayerSpatial || !f.provisional.maxLayers.IsValid() || (!allowOvershoot && layers.GreaterThan(f.provisional.maxLayers)) { return 0 } - requiredBitrate := f.provisional.bitrates[layers.Spatial][layers.Temporal] + requiredBitrate := f.provisional.Bitrates[layers.Spatial][layers.Temporal] if requiredBitrate == 0 { return 0 } alreadyAllocatedBitrate := int64(0) if f.provisional.allocatedLayers.IsValid() { - alreadyAllocatedBitrate = f.provisional.bitrates[f.provisional.allocatedLayers.Spatial][f.provisional.allocatedLayers.Temporal] + alreadyAllocatedBitrate = f.provisional.Bitrates[f.provisional.allocatedLayers.Spatial][f.provisional.allocatedLayers.Temporal] } // a layer under maximum fits, take it @@ -727,28 +706,28 @@ func (f *Forwarder) ProvisionalAllocateGetCooperativeTransition(allowOvershoot b defer f.lock.Unlock() if f.provisional.muted || f.provisional.pubMuted { - f.provisional.allocatedLayers = InvalidLayers + f.provisional.allocatedLayers = buffer.InvalidLayers if f.provisional.pubMuted { // leave it at current for opportunistic forwarding, there is still bandwidth saving with publisher mute f.provisional.allocatedLayers = f.provisional.currentLayers } return VideoTransition{ - from: f.targetLayers, - to: f.provisional.allocatedLayers, - bandwidthDelta: 0 - f.lastAllocation.bandwidthRequested, + From: f.targetLayers, + To: f.provisional.allocatedLayers, + BandwidthDelta: 0 - f.lastAllocation.BandwidthRequested, } } // check if we should preserve current target if f.targetLayers.IsValid() { // what is the highest that is available - maximalLayers := InvalidLayers + maximalLayers := buffer.InvalidLayers maximalBandwidthRequired := int64(0) for s := f.provisional.maxLayers.Spatial; s >= 0; s-- { for t := f.provisional.maxLayers.Temporal; t >= 0; t-- { - if f.provisional.bitrates[s][t] != 0 { - maximalLayers = VideoLayers{Spatial: s, Temporal: t} - maximalBandwidthRequired = f.provisional.bitrates[s][t] + if f.provisional.Bitrates[s][t] != 0 { + maximalLayers = buffer.VideoLayer{Spatial: s, Temporal: t} + maximalBandwidthRequired = f.provisional.Bitrates[s][t] break } } @@ -759,14 +738,14 @@ func (f *Forwarder) ProvisionalAllocateGetCooperativeTransition(allowOvershoot b } if maximalLayers.IsValid() { - if !f.targetLayers.GreaterThan(maximalLayers) && f.provisional.bitrates[f.targetLayers.Spatial][f.targetLayers.Temporal] != 0 { + if !f.targetLayers.GreaterThan(maximalLayers) && f.provisional.Bitrates[f.targetLayers.Spatial][f.targetLayers.Temporal] != 0 { // currently streaming and maybe wanting an upgrade (f.targetLayers <= maximalLayers), // just preserve current target in the cooperative scheme of things f.provisional.allocatedLayers = f.targetLayers return VideoTransition{ - from: f.targetLayers, - to: f.targetLayers, - bandwidthDelta: 0, + From: f.targetLayers, + To: f.targetLayers, + BandwidthDelta: 0, } } @@ -774,9 +753,9 @@ func (f *Forwarder) ProvisionalAllocateGetCooperativeTransition(allowOvershoot b // maximalLayers < f.targetLayers, make the down move f.provisional.allocatedLayers = maximalLayers return VideoTransition{ - from: f.targetLayers, - to: maximalLayers, - bandwidthDelta: maximalBandwidthRequired - f.lastAllocation.bandwidthRequested, + From: f.targetLayers, + To: maximalLayers, + BandwidthDelta: maximalBandwidthRequired - f.lastAllocation.BandwidthRequested, } } } @@ -785,14 +764,14 @@ func (f *Forwarder) ProvisionalAllocateGetCooperativeTransition(allowOvershoot b findNextLayer := func( minSpatial, maxSpatial int32, minTemporal, maxTemporal int32, - ) (VideoLayers, int64) { - layers := InvalidLayers + ) (buffer.VideoLayer, int64) { + layers := buffer.InvalidLayers bw := int64(0) for s := minSpatial; s <= maxSpatial; s++ { for t := minTemporal; t <= maxTemporal; t++ { - if f.provisional.bitrates[s][t] != 0 { - layers = VideoLayers{Spatial: s, Temporal: t} - bw = f.provisional.bitrates[s][t] + if f.provisional.Bitrates[s][t] != 0 { + layers = buffer.VideoLayer{Spatial: s, Temporal: t} + bw = f.provisional.Bitrates[s][t] break } } @@ -819,8 +798,8 @@ func (f *Forwarder) ProvisionalAllocateGetCooperativeTransition(allowOvershoot b // could not find a minimal layer, overshoot if allowed if bandwidthRequired == 0 && f.provisional.maxLayers.IsValid() && allowOvershoot { targetLayers, bandwidthRequired = findNextLayer( - f.provisional.maxLayers.Spatial+1, DefaultMaxLayerSpatial, - 0, DefaultMaxLayerTemporal, + f.provisional.maxLayers.Spatial+1, buffer.DefaultMaxLayerSpatial, + 0, buffer.DefaultMaxLayerTemporal, ) } } @@ -836,9 +815,9 @@ func (f *Forwarder) ProvisionalAllocateGetCooperativeTransition(allowOvershoot b f.provisional.allocatedLayers = targetLayers return VideoTransition{ - from: f.targetLayers, - to: targetLayers, - bandwidthDelta: bandwidthRequired - f.lastAllocation.bandwidthRequested, + From: f.targetLayers, + To: targetLayers, + BandwidthDelta: bandwidthRequired - f.lastAllocation.BandwidthRequested, } } @@ -862,32 +841,32 @@ func (f *Forwarder) ProvisionalAllocateGetBestWeightedTransition() VideoTransiti defer f.lock.Unlock() if f.provisional.muted || f.provisional.pubMuted { - f.provisional.allocatedLayers = InvalidLayers + f.provisional.allocatedLayers = buffer.InvalidLayers if f.provisional.pubMuted { // leave it at current for opportunistic forwarding, there is still bandwidth saving with publisher mute f.provisional.allocatedLayers = f.provisional.currentLayers } return VideoTransition{ - from: f.targetLayers, - to: f.provisional.allocatedLayers, - bandwidthDelta: 0 - f.lastAllocation.bandwidthRequested, + From: f.targetLayers, + To: f.provisional.allocatedLayers, + BandwidthDelta: 0 - f.lastAllocation.BandwidthRequested, } } - maxReachableLayerTemporal := InvalidLayerTemporal + maxReachableLayerTemporal := buffer.InvalidLayerTemporal for t := f.provisional.maxLayers.Temporal; t >= 0; t-- { for s := f.provisional.maxLayers.Spatial; s >= 0; s-- { - if f.provisional.bitrates[s][t] != 0 { + if f.provisional.Bitrates[s][t] != 0 { maxReachableLayerTemporal = t break } } - if maxReachableLayerTemporal != InvalidLayerTemporal { + if maxReachableLayerTemporal != buffer.InvalidLayerTemporal { break } } - if maxReachableLayerTemporal == InvalidLayerTemporal { + if maxReachableLayerTemporal == buffer.InvalidLayerTemporal { // feed has gone dry, just leave target at current to enable opportunistic forwarding in case current resumes. // Note that this is giving back bits and opportunistic forwarding resuming might trigger congestion again, // but that should be handled by stream allocator. @@ -897,15 +876,15 @@ func (f *Forwarder) ProvisionalAllocateGetBestWeightedTransition() VideoTransiti f.provisional.allocatedLayers = f.provisional.currentLayers } return VideoTransition{ - from: f.targetLayers, - to: f.provisional.allocatedLayers, - bandwidthDelta: 0 - f.lastAllocation.bandwidthRequested, + From: f.targetLayers, + To: f.provisional.allocatedLayers, + BandwidthDelta: 0 - f.lastAllocation.BandwidthRequested, } } // starting from minimum to target, find transition which gives the best // transition taking into account bits saved vs cost of such a transition - bestLayers := InvalidLayers + bestLayers := buffer.InvalidLayers bestBandwidthDelta := int64(0) bestValue := float32(0) for s := int32(0); s <= f.targetLayers.Spatial; s++ { @@ -914,7 +893,7 @@ func (f *Forwarder) ProvisionalAllocateGetBestWeightedTransition() VideoTransiti break } - bandwidthDelta := int64(math.Max(float64(0), float64(f.lastAllocation.bandwidthRequested-f.provisional.bitrates[s][t]))) + BandwidthDelta := int64(math.Max(float64(0), float64(f.lastAllocation.BandwidthRequested-f.provisional.Bitrates[s][t]))) transitionCost := int32(0) if f.targetLayers.Spatial != s { @@ -925,21 +904,21 @@ func (f *Forwarder) ProvisionalAllocateGetBestWeightedTransition() VideoTransiti value := float32(0) if (transitionCost + qualityCost) != 0 { - value = float32(bandwidthDelta) / float32(transitionCost+qualityCost) + value = float32(BandwidthDelta) / float32(transitionCost+qualityCost) } - if value > bestValue || (value == bestValue && bandwidthDelta > bestBandwidthDelta) { + if value > bestValue || (value == bestValue && BandwidthDelta > bestBandwidthDelta) { bestValue = value - bestBandwidthDelta = bandwidthDelta - bestLayers = VideoLayers{Spatial: s, Temporal: t} + bestBandwidthDelta = BandwidthDelta + bestLayers = buffer.VideoLayer{Spatial: s, Temporal: t} } } } f.provisional.allocatedLayers = bestLayers return VideoTransition{ - from: f.targetLayers, - to: bestLayers, - bandwidthDelta: bestBandwidthDelta, + From: f.targetLayers, + To: bestLayers, + BandwidthDelta: bestBandwidthDelta, } } @@ -951,24 +930,24 @@ func (f *Forwarder) ProvisionalAllocateCommit() VideoAllocation { f.provisional.muted, f.provisional.pubMuted, f.provisional.maxPublishedLayer, - f.provisional.bitrates, + f.provisional.Bitrates, f.provisional.maxLayers, ) alloc := VideoAllocation{ - bandwidthRequested: 0, - bandwidthDelta: -f.lastAllocation.bandwidthRequested, - bitrates: f.provisional.bitrates, - bandwidthNeeded: optimalBandwidthNeeded, - targetLayers: f.provisional.allocatedLayers, - requestLayerSpatial: f.provisional.allocatedLayers.Spatial, - maxLayers: f.provisional.maxLayers, - distanceToDesired: getDistanceToDesired( + BandwidthRequested: 0, + BandwidthDelta: -f.lastAllocation.BandwidthRequested, + Bitrates: f.provisional.Bitrates, + BandwidthNeeded: optimalBandwidthNeeded, + TargetLayers: f.provisional.allocatedLayers, + RequestLayerSpatial: f.provisional.allocatedLayers.Spatial, + MaxLayers: f.provisional.maxLayers, + DistanceToDesired: getDistanceToDesired( f.provisional.muted, f.provisional.pubMuted, f.provisional.maxPublishedLayer, f.provisional.maxTemporalLayerSeen, f.provisional.availableLayers, - f.provisional.bitrates, + f.provisional.Bitrates, f.provisional.allocatedLayers, f.provisional.maxLayers, ), @@ -976,47 +955,47 @@ func (f *Forwarder) ProvisionalAllocateCommit() VideoAllocation { switch { case f.provisional.muted: - alloc.pauseReason = VideoPauseReasonMuted + alloc.PauseReason = VideoPauseReasonMuted case f.provisional.pubMuted: - alloc.pauseReason = VideoPauseReasonPubMuted + alloc.PauseReason = VideoPauseReasonPubMuted case optimalBandwidthNeeded == 0: if f.provisional.allocatedLayers.IsValid() { // overshoot - alloc.bandwidthRequested = f.provisional.bitrates[f.provisional.allocatedLayers.Spatial][f.provisional.allocatedLayers.Temporal] - alloc.bandwidthDelta = alloc.bandwidthRequested - f.lastAllocation.bandwidthRequested + alloc.BandwidthRequested = f.provisional.Bitrates[f.provisional.allocatedLayers.Spatial][f.provisional.allocatedLayers.Temporal] + alloc.BandwidthDelta = alloc.BandwidthRequested - f.lastAllocation.BandwidthRequested } else { - alloc.pauseReason = VideoPauseReasonFeedDry + alloc.PauseReason = VideoPauseReasonFeedDry // leave target at current for opportunistic forwarding if f.provisional.currentLayers.IsValid() && f.provisional.currentLayers.Spatial <= f.provisional.maxLayers.Spatial { f.provisional.allocatedLayers = f.provisional.currentLayers - alloc.targetLayers = f.provisional.allocatedLayers - alloc.requestLayerSpatial = alloc.targetLayers.Spatial + alloc.TargetLayers = f.provisional.allocatedLayers + alloc.RequestLayerSpatial = alloc.TargetLayers.Spatial } } default: if f.provisional.allocatedLayers.IsValid() { - alloc.bandwidthRequested = f.provisional.bitrates[f.provisional.allocatedLayers.Spatial][f.provisional.allocatedLayers.Temporal] + alloc.BandwidthRequested = f.provisional.Bitrates[f.provisional.allocatedLayers.Spatial][f.provisional.allocatedLayers.Temporal] } - alloc.bandwidthDelta = alloc.bandwidthRequested - f.lastAllocation.bandwidthRequested + alloc.BandwidthDelta = alloc.BandwidthRequested - f.lastAllocation.BandwidthRequested if f.provisional.allocatedLayers.GreaterThan(f.provisional.maxLayers) || - alloc.bandwidthRequested >= getOptimalBandwidthNeeded( + alloc.BandwidthRequested >= getOptimalBandwidthNeeded( f.provisional.muted, f.provisional.pubMuted, f.provisional.maxPublishedLayer, - f.provisional.bitrates, + f.provisional.Bitrates, f.provisional.maxLayers, ) { // could be greater than optimal if overshooting - alloc.isDeficient = false + alloc.IsDeficient = false } else { - alloc.isDeficient = true + alloc.IsDeficient = true if !f.provisional.allocatedLayers.IsValid() { - alloc.pauseReason = VideoPauseReasonBandwidth + alloc.PauseReason = VideoPauseReasonBandwidth } } } @@ -1056,27 +1035,27 @@ func (f *Forwarder) AllocateNextHigher(availableChannelCapacity int64, available ) (bool, VideoAllocation, bool) { for s := minSpatial; s <= maxSpatial; s++ { for t := minTemporal; t <= maxTemporal; t++ { - bandwidthRequested := brs[s][t] - if bandwidthRequested == 0 { + BandwidthRequested := brs[s][t] + if BandwidthRequested == 0 { continue } - if !allowOvershoot && bandwidthRequested-alreadyAllocated > availableChannelCapacity { + if !allowOvershoot && BandwidthRequested-alreadyAllocated > availableChannelCapacity { // next higher available layer does not fit, return return true, f.lastAllocation, false } - targetLayers := VideoLayers{Spatial: s, Temporal: t} + targetLayers := buffer.VideoLayer{Spatial: s, Temporal: t} alloc := VideoAllocation{ - isDeficient: true, - bandwidthRequested: bandwidthRequested, - bandwidthDelta: bandwidthRequested - alreadyAllocated, - bandwidthNeeded: optimalBandwidthNeeded, - bitrates: brs, - targetLayers: targetLayers, - requestLayerSpatial: targetLayers.Spatial, - maxLayers: f.maxLayers, - distanceToDesired: getDistanceToDesired( + IsDeficient: true, + BandwidthRequested: BandwidthRequested, + BandwidthDelta: BandwidthRequested - alreadyAllocated, + BandwidthNeeded: optimalBandwidthNeeded, + Bitrates: brs, + TargetLayers: targetLayers, + RequestLayerSpatial: targetLayers.Spatial, + MaxLayers: f.maxLayers, + DistanceToDesired: getDistanceToDesired( f.muted, f.pubMuted, f.maxPublishedLayer, @@ -1087,8 +1066,8 @@ func (f *Forwarder) AllocateNextHigher(availableChannelCapacity int64, available f.maxLayers, ), } - if targetLayers.GreaterThan(f.maxLayers) || bandwidthRequested >= optimalBandwidthNeeded { - alloc.isDeficient = false + if targetLayers.GreaterThan(f.maxLayers) || BandwidthRequested >= optimalBandwidthNeeded { + alloc.IsDeficient = false } return true, f.updateAllocation(alloc, "next-higher"), true @@ -1124,8 +1103,8 @@ func (f *Forwarder) AllocateNextHigher(availableChannelCapacity int64, available if allowOvershoot && f.maxLayers.IsValid() { done, allocation, boosted = doAllocation( - f.maxLayers.Spatial+1, DefaultMaxLayerSpatial, - 0, DefaultMaxLayerTemporal, + f.maxLayers.Spatial+1, buffer.DefaultMaxLayerSpatial, + 0, buffer.DefaultMaxLayerTemporal, ) if done { return allocation, boosted @@ -1164,15 +1143,15 @@ func (f *Forwarder) GetNextHigherTransition(brs Bitrates, allowOvershoot bool) ( ) (bool, VideoTransition, bool) { for s := minSpatial; s <= maxSpatial; s++ { for t := minTemporal; t <= maxTemporal; t++ { - bandwidthRequested := brs[s][t] - if bandwidthRequested == 0 { + BandwidthRequested := brs[s][t] + if BandwidthRequested == 0 { continue } transition := VideoTransition{ - from: f.targetLayers, - to: VideoLayers{Spatial: s, Temporal: t}, - bandwidthDelta: bandwidthRequested - alreadyAllocated, + From: f.targetLayers, + To: buffer.VideoLayer{Spatial: s, Temporal: t}, + BandwidthDelta: BandwidthRequested - alreadyAllocated, } return true, transition, true @@ -1208,8 +1187,8 @@ func (f *Forwarder) GetNextHigherTransition(brs Bitrates, allowOvershoot bool) ( if allowOvershoot && f.maxLayers.IsValid() { done, transition, isAvailable = findNextHigher( - f.maxLayers.Spatial+1, DefaultMaxLayerSpatial, - 0, DefaultMaxLayerTemporal, + f.maxLayers.Spatial+1, buffer.DefaultMaxLayerSpatial, + 0, buffer.DefaultMaxLayerTemporal, ) if done { return transition, isAvailable @@ -1225,39 +1204,39 @@ func (f *Forwarder) Pause(availableLayers []int32, brs Bitrates) VideoAllocation optimalBandwidthNeeded := getOptimalBandwidthNeeded(f.muted, f.pubMuted, f.maxPublishedLayer, brs, f.maxLayers) alloc := VideoAllocation{ - bandwidthRequested: 0, - bandwidthDelta: 0 - f.lastAllocation.bandwidthRequested, - bitrates: brs, - bandwidthNeeded: optimalBandwidthNeeded, - targetLayers: InvalidLayers, - requestLayerSpatial: InvalidLayerSpatial, - maxLayers: f.maxLayers, - distanceToDesired: getDistanceToDesired( + BandwidthRequested: 0, + BandwidthDelta: 0 - f.lastAllocation.BandwidthRequested, + Bitrates: brs, + BandwidthNeeded: optimalBandwidthNeeded, + TargetLayers: buffer.InvalidLayers, + RequestLayerSpatial: buffer.InvalidLayerSpatial, + MaxLayers: f.maxLayers, + DistanceToDesired: getDistanceToDesired( f.muted, f.pubMuted, f.maxPublishedLayer, f.maxTemporalLayerSeen, availableLayers, brs, - InvalidLayers, + buffer.InvalidLayers, f.maxLayers, ), } switch { case f.muted: - alloc.pauseReason = VideoPauseReasonMuted + alloc.PauseReason = VideoPauseReasonMuted case f.pubMuted: - alloc.pauseReason = VideoPauseReasonPubMuted + alloc.PauseReason = VideoPauseReasonPubMuted case optimalBandwidthNeeded == 0: - alloc.pauseReason = VideoPauseReasonFeedDry + alloc.PauseReason = VideoPauseReasonFeedDry default: // pausing due to lack of bandwidth - alloc.isDeficient = true - alloc.pauseReason = VideoPauseReasonBandwidth + alloc.IsDeficient = true + alloc.PauseReason = VideoPauseReasonBandwidth } f.clearParkedLayers() @@ -1265,10 +1244,10 @@ func (f *Forwarder) Pause(availableLayers []int32, brs Bitrates) VideoAllocation } func (f *Forwarder) updateAllocation(alloc VideoAllocation, reason string) VideoAllocation { - if alloc.isDeficient != f.lastAllocation.isDeficient || - alloc.pauseReason != f.lastAllocation.pauseReason || - alloc.targetLayers != f.lastAllocation.targetLayers || - alloc.requestLayerSpatial != f.lastAllocation.requestLayerSpatial { + if alloc.IsDeficient != f.lastAllocation.IsDeficient || + alloc.PauseReason != f.lastAllocation.PauseReason || + alloc.TargetLayers != f.lastAllocation.TargetLayers || + alloc.RequestLayerSpatial != f.lastAllocation.RequestLayerSpatial { if reason == "optimal" { f.logger.Debugw(fmt.Sprintf("stream allocation: %s", reason), "allocation", alloc) } else { @@ -1277,7 +1256,7 @@ func (f *Forwarder) updateAllocation(alloc VideoAllocation, reason string) Video } f.lastAllocation = alloc - f.setTargetLayers(f.lastAllocation.targetLayers, f.lastAllocation.requestLayerSpatial) + f.setTargetLayers(f.lastAllocation.TargetLayers, f.lastAllocation.RequestLayerSpatial) if !f.targetLayers.IsValid() { f.resyncLocked() } @@ -1285,7 +1264,7 @@ func (f *Forwarder) updateAllocation(alloc VideoAllocation, reason string) Video return f.lastAllocation } -func (f *Forwarder) setTargetLayers(targetLayers VideoLayers, requestLayerSpatial int32) { +func (f *Forwarder) setTargetLayers(targetLayers buffer.VideoLayer, requestLayerSpatial int32) { f.targetLayers = targetLayers if f.ddLayerSelector != nil { f.ddLayerSelector.SelectLayer(targetLayers) @@ -1302,20 +1281,20 @@ func (f *Forwarder) Resync() { } func (f *Forwarder) resyncLocked() { - f.currentLayers = InvalidLayers + f.currentLayers = buffer.InvalidLayers f.lastSSRC = 0 f.clearParkedLayers() } func (f *Forwarder) clearParkedLayers() { - f.parkedLayers = InvalidLayers + f.parkedLayers = buffer.InvalidLayers if f.parkedLayersTimer != nil { f.parkedLayersTimer.Stop() f.parkedLayersTimer = nil } } -func (f *Forwarder) setupParkedLayers(parkedLayers VideoLayers) { +func (f *Forwarder) setupParkedLayers(parkedLayers buffer.VideoLayer) { f.clearParkedLayers() f.parkedLayers = parkedLayers @@ -1339,7 +1318,7 @@ func (f *Forwarder) CheckSync() (locked bool, layer int32) { return } -func (f *Forwarder) FilterRTX(nacks []uint16) (filtered []uint16, disallowedLayers [DefaultMaxLayerSpatial + 1]bool) { +func (f *Forwarder) FilterRTX(nacks []uint16) (filtered []uint16, disallowedLayers [buffer.DefaultMaxLayerSpatial + 1]bool) { if !FlagFilterRTX { filtered = nacks return @@ -1358,7 +1337,7 @@ func (f *Forwarder) FilterRTX(nacks []uint16) (filtered []uint16, disallowedLaye // // Without the curb, when congestion hits, RTX rate could be so high that it further congests the channel. // - for layer := int32(0); layer < DefaultMaxLayerSpatial+1; layer++ { + for layer := int32(0); layer < buffer.DefaultMaxLayerSpatial+1; layer++ { if f.isDeficientLocked() && (f.targetLayers.Spatial < f.currentLayers.Spatial || layer > f.currentLayers.Spatial) { disallowedLayers[layer] = true } @@ -1406,7 +1385,7 @@ func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer i f.vp8Munger.SetLast(extPkt) } } else { - if f.referenceLayerSpatial == InvalidLayerSpatial { + if f.referenceLayerSpatial == buffer.InvalidLayerSpatial { // on a resume, reference layer may not be set, so only set when it is invalid f.referenceLayerSpatial = layer } @@ -1715,8 +1694,8 @@ func (f *Forwarder) GetRTPMungerParams() RTPMungerParams { // ----------------------------------------------------------------------------- -func getOptimalBandwidthNeeded(muted bool, pubMuted bool, maxPublishedLayer int32, brs Bitrates, maxLayers VideoLayers) int64 { - if muted || pubMuted || maxPublishedLayer == InvalidLayerSpatial { +func getOptimalBandwidthNeeded(muted bool, pubMuted bool, maxPublishedLayer int32, brs Bitrates, maxLayers buffer.VideoLayer) int64 { + if muted || pubMuted || maxPublishedLayer == buffer.InvalidLayerSpatial { return 0 } @@ -1745,17 +1724,17 @@ func getDistanceToDesired( maxTemporalLayerSeen int32, availableLayers []int32, brs Bitrates, - targetLayers VideoLayers, - maxLayers VideoLayers, + targetLayers buffer.VideoLayer, + maxLayers buffer.VideoLayer, ) float64 { - if muted || pubMuted || maxPublishedLayer == InvalidLayerSpatial || maxTemporalLayerSeen == InvalidLayerTemporal || !maxLayers.IsValid() { + if muted || pubMuted || maxPublishedLayer == buffer.InvalidLayerSpatial || maxTemporalLayerSeen == buffer.InvalidLayerTemporal || !maxLayers.IsValid() { return 0.0 } adjustedMaxLayers := maxLayers - maxAvailableSpatial := InvalidLayerSpatial - maxAvailableTemporal := InvalidLayerTemporal + maxAvailableSpatial := buffer.InvalidLayerSpatial + maxAvailableTemporal := buffer.InvalidLayerTemporal // max available spatial is min(subscribedMax, publishedMax, availableMax) // subscribedMax = subscriber requested max spatial layer @@ -1791,7 +1770,7 @@ done: // subscribedMax = subscriber requested max temporal layer // temporalLayerSeenMax = max temporal layer ever published/seen // availableMax = based on bit rate measurement, available max temporal in the adjusted max spatial layer - if adjustedMaxLayers.Spatial != InvalidLayerSpatial { + if adjustedMaxLayers.Spatial != buffer.InvalidLayerSpatial { for t := int32(len(brs[0])) - 1; t >= 0; t-- { if brs[adjustedMaxLayers.Spatial][t] != 0 { maxAvailableTemporal = t @@ -1808,13 +1787,13 @@ done: } if !adjustedMaxLayers.IsValid() { - adjustedMaxLayers = VideoLayers{Spatial: 0, Temporal: 0} + adjustedMaxLayers = buffer.VideoLayer{Spatial: 0, Temporal: 0} } // adjust target layers if they are invalid, i. e. not streaming adjustedTargetLayers := targetLayers if !targetLayers.IsValid() { - adjustedTargetLayers = VideoLayers{Spatial: 0, Temporal: 0} + adjustedTargetLayers = buffer.VideoLayer{Spatial: 0, Temporal: 0} } distance := diff --git a/pkg/sfu/forwarder_test.go b/pkg/sfu/forwarder_test.go index 5d0695a1f..81778adae 100644 --- a/pkg/sfu/forwarder_test.go +++ b/pkg/sfu/forwarder_test.go @@ -13,8 +13,8 @@ import ( ) func disable(f *Forwarder) { - f.currentLayers = InvalidLayers - f.targetLayers = InvalidLayers + f.currentLayers = buffer.InvalidLayers + f.targetLayers = buffer.InvalidLayers } func newForwarder(codec webrtc.RTPCodecCapability, kind webrtc.RTPCodecType) *Forwarder { @@ -40,74 +40,74 @@ func TestForwarderMute(t *testing.T) { func TestForwarderLayersAudio(t *testing.T) { f := newForwarder(testutils.TestOpusCodec, webrtc.RTPCodecTypeAudio) - require.Equal(t, InvalidLayers, f.MaxLayers()) + require.Equal(t, buffer.InvalidLayers, f.MaxLayers()) - require.Equal(t, InvalidLayers, f.CurrentLayers()) - require.Equal(t, InvalidLayers, f.TargetLayers()) + require.Equal(t, buffer.InvalidLayers, f.CurrentLayers()) + require.Equal(t, buffer.InvalidLayers, f.TargetLayers()) changed, maxLayers, currentLayers := f.SetMaxSpatialLayer(1) require.False(t, changed) - require.Equal(t, InvalidLayers, maxLayers) - require.Equal(t, InvalidLayers, currentLayers) + require.Equal(t, buffer.InvalidLayers, maxLayers) + require.Equal(t, buffer.InvalidLayers, currentLayers) changed, maxLayers, currentLayers = f.SetMaxTemporalLayer(1) require.False(t, changed) - require.Equal(t, InvalidLayers, maxLayers) - require.Equal(t, InvalidLayers, currentLayers) + require.Equal(t, buffer.InvalidLayers, maxLayers) + require.Equal(t, buffer.InvalidLayers, currentLayers) - require.Equal(t, InvalidLayers, f.MaxLayers()) + require.Equal(t, buffer.InvalidLayers, f.MaxLayers()) } func TestForwarderLayersVideo(t *testing.T) { f := newForwarder(testutils.TestVP8Codec, webrtc.RTPCodecTypeVideo) maxLayers := f.MaxLayers() - expectedLayers := VideoLayers{Spatial: InvalidLayerSpatial, Temporal: DefaultMaxLayerTemporal} + expectedLayers := buffer.VideoLayer{Spatial: buffer.InvalidLayerSpatial, Temporal: buffer.DefaultMaxLayerTemporal} require.Equal(t, expectedLayers, maxLayers) - require.Equal(t, InvalidLayers, f.CurrentLayers()) - require.Equal(t, InvalidLayers, f.TargetLayers()) + require.Equal(t, buffer.InvalidLayers, f.CurrentLayers()) + require.Equal(t, buffer.InvalidLayers, f.TargetLayers()) - expectedLayers = VideoLayers{ - Spatial: DefaultMaxLayerSpatial, - Temporal: DefaultMaxLayerTemporal, + expectedLayers = buffer.VideoLayer{ + Spatial: buffer.DefaultMaxLayerSpatial, + Temporal: buffer.DefaultMaxLayerTemporal, } - changed, maxLayers, currentLayers := f.SetMaxSpatialLayer(DefaultMaxLayerSpatial) + changed, maxLayers, currentLayers := f.SetMaxSpatialLayer(buffer.DefaultMaxLayerSpatial) require.True(t, changed) require.Equal(t, expectedLayers, maxLayers) - require.Equal(t, InvalidLayers, currentLayers) + require.Equal(t, buffer.InvalidLayers, currentLayers) - changed, maxLayers, currentLayers = f.SetMaxSpatialLayer(DefaultMaxLayerSpatial - 1) + changed, maxLayers, currentLayers = f.SetMaxSpatialLayer(buffer.DefaultMaxLayerSpatial - 1) require.True(t, changed) - expectedLayers = VideoLayers{ - Spatial: DefaultMaxLayerSpatial - 1, - Temporal: DefaultMaxLayerTemporal, + expectedLayers = buffer.VideoLayer{ + Spatial: buffer.DefaultMaxLayerSpatial - 1, + Temporal: buffer.DefaultMaxLayerTemporal, } require.Equal(t, expectedLayers, maxLayers) require.Equal(t, expectedLayers, f.MaxLayers()) - require.Equal(t, InvalidLayers, currentLayers) + require.Equal(t, buffer.InvalidLayers, currentLayers) - f.currentLayers = VideoLayers{Spatial: 0, Temporal: 1} - changed, maxLayers, currentLayers = f.SetMaxSpatialLayer(DefaultMaxLayerSpatial - 1) + f.currentLayers = buffer.VideoLayer{Spatial: 0, Temporal: 1} + changed, maxLayers, currentLayers = f.SetMaxSpatialLayer(buffer.DefaultMaxLayerSpatial - 1) require.False(t, changed) require.Equal(t, expectedLayers, maxLayers) require.Equal(t, expectedLayers, f.MaxLayers()) - require.Equal(t, VideoLayers{Spatial: 0, Temporal: 1}, currentLayers) + require.Equal(t, buffer.VideoLayer{Spatial: 0, Temporal: 1}, currentLayers) - changed, maxLayers, currentLayers = f.SetMaxTemporalLayer(DefaultMaxLayerTemporal) + changed, maxLayers, currentLayers = f.SetMaxTemporalLayer(buffer.DefaultMaxLayerTemporal) require.False(t, changed) require.Equal(t, expectedLayers, maxLayers) - require.Equal(t, VideoLayers{Spatial: 0, Temporal: 1}, currentLayers) + require.Equal(t, buffer.VideoLayer{Spatial: 0, Temporal: 1}, currentLayers) - changed, maxLayers, currentLayers = f.SetMaxTemporalLayer(DefaultMaxLayerTemporal - 1) + changed, maxLayers, currentLayers = f.SetMaxTemporalLayer(buffer.DefaultMaxLayerTemporal - 1) require.True(t, changed) - expectedLayers = VideoLayers{ - Spatial: DefaultMaxLayerSpatial - 1, - Temporal: DefaultMaxLayerTemporal - 1, + expectedLayers = buffer.VideoLayer{ + Spatial: buffer.DefaultMaxLayerSpatial - 1, + Temporal: buffer.DefaultMaxLayerTemporal - 1, } require.Equal(t, expectedLayers, maxLayers) require.Equal(t, expectedLayers, f.MaxLayers()) - require.Equal(t, VideoLayers{Spatial: 0, Temporal: 1}, currentLayers) + require.Equal(t, buffer.VideoLayer{Spatial: 0, Temporal: 1}, currentLayers) } func TestForwarderAllocateOptimal(t *testing.T) { @@ -121,53 +121,53 @@ func TestForwarderAllocateOptimal(t *testing.T) { } // invalid max layers - f.maxLayers = InvalidLayers + f.maxLayers = buffer.InvalidLayers expectedResult := VideoAllocation{ - pauseReason: VideoPauseReasonFeedDry, - bandwidthRequested: 0, - bandwidthDelta: 0, - bitrates: bitrates, - targetLayers: InvalidLayers, - requestLayerSpatial: InvalidLayerSpatial, - maxLayers: InvalidLayers, - distanceToDesired: 0, + PauseReason: VideoPauseReasonFeedDry, + BandwidthRequested: 0, + BandwidthDelta: 0, + Bitrates: bitrates, + TargetLayers: buffer.InvalidLayers, + RequestLayerSpatial: buffer.InvalidLayerSpatial, + MaxLayers: buffer.InvalidLayers, + DistanceToDesired: 0, } result := f.AllocateOptimal(nil, bitrates, true) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - f.SetMaxSpatialLayer(DefaultMaxLayerSpatial) - f.SetMaxTemporalLayer(DefaultMaxLayerTemporal) + f.SetMaxSpatialLayer(buffer.DefaultMaxLayerSpatial) + f.SetMaxTemporalLayer(buffer.DefaultMaxLayerTemporal) - // should still have target at InvalidLayers until max publisher layer is available + // should still have target at buffer.InvalidLayers until max publisher layer is available expectedResult = VideoAllocation{ - pauseReason: VideoPauseReasonFeedDry, - bandwidthRequested: 0, - bandwidthDelta: 0, - bitrates: bitrates, - targetLayers: InvalidLayers, - requestLayerSpatial: InvalidLayerSpatial, - maxLayers: DefaultMaxLayers, - distanceToDesired: 0, + PauseReason: VideoPauseReasonFeedDry, + BandwidthRequested: 0, + BandwidthDelta: 0, + Bitrates: bitrates, + TargetLayers: buffer.InvalidLayers, + RequestLayerSpatial: buffer.InvalidLayerSpatial, + MaxLayers: buffer.DefaultMaxLayers, + DistanceToDesired: 0, } result = f.AllocateOptimal(nil, bitrates, true) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - f.SetMaxPublishedLayer(DefaultMaxLayerSpatial) + f.SetMaxPublishedLayer(buffer.DefaultMaxLayerSpatial) // muted should not consume any bandwidth f.Mute(true) disable(f) expectedResult = VideoAllocation{ - pauseReason: VideoPauseReasonMuted, - bandwidthRequested: 0, - bandwidthDelta: 0, - bitrates: bitrates, - targetLayers: InvalidLayers, - requestLayerSpatial: InvalidLayerSpatial, - maxLayers: DefaultMaxLayers, - distanceToDesired: 0, + PauseReason: VideoPauseReasonMuted, + BandwidthRequested: 0, + BandwidthDelta: 0, + Bitrates: bitrates, + TargetLayers: buffer.InvalidLayers, + RequestLayerSpatial: buffer.InvalidLayerSpatial, + MaxLayers: buffer.DefaultMaxLayers, + DistanceToDesired: 0, } result = f.AllocateOptimal(nil, bitrates, true) require.Equal(t, expectedResult, result) @@ -179,14 +179,14 @@ func TestForwarderAllocateOptimal(t *testing.T) { f.PubMute(true) disable(f) expectedResult = VideoAllocation{ - pauseReason: VideoPauseReasonPubMuted, - bandwidthRequested: 0, - bandwidthDelta: 0, - bitrates: bitrates, - targetLayers: InvalidLayers, - requestLayerSpatial: InvalidLayerSpatial, - maxLayers: DefaultMaxLayers, - distanceToDesired: 0, + PauseReason: VideoPauseReasonPubMuted, + BandwidthRequested: 0, + BandwidthDelta: 0, + Bitrates: bitrates, + TargetLayers: buffer.InvalidLayers, + RequestLayerSpatial: buffer.InvalidLayerSpatial, + MaxLayers: buffer.DefaultMaxLayers, + DistanceToDesired: 0, } result = f.AllocateOptimal(nil, bitrates, true) require.Equal(t, expectedResult, result) @@ -195,209 +195,209 @@ func TestForwarderAllocateOptimal(t *testing.T) { f.PubMute(false) // when parked layers valid, should stay there - f.parkedLayers = VideoLayers{ + f.parkedLayers = buffer.VideoLayer{ Spatial: 0, Temporal: 1, } expectedResult = VideoAllocation{ - pauseReason: VideoPauseReasonFeedDry, - bandwidthRequested: 0, - bandwidthDelta: 0, - bitrates: emptyBitrates, - targetLayers: f.parkedLayers, - requestLayerSpatial: f.parkedLayers.Spatial, - maxLayers: DefaultMaxLayers, - distanceToDesired: 0, + PauseReason: VideoPauseReasonFeedDry, + BandwidthRequested: 0, + BandwidthDelta: 0, + Bitrates: emptyBitrates, + TargetLayers: f.parkedLayers, + RequestLayerSpatial: f.parkedLayers.Spatial, + MaxLayers: buffer.DefaultMaxLayers, + DistanceToDesired: 0, } result = f.AllocateOptimal(nil, emptyBitrates, true) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) require.Equal(t, f.parkedLayers, f.TargetLayers()) - f.parkedLayers = InvalidLayers + f.parkedLayers = buffer.InvalidLayers // when max layers changes, target is opportunistic, but requested spatial layer should be at max f.SetMaxTemporalLayerSeen(3) - f.maxLayers = VideoLayers{Spatial: 1, Temporal: 3} + f.maxLayers = buffer.VideoLayer{Spatial: 1, Temporal: 3} expectedResult = VideoAllocation{ - pauseReason: VideoPauseReasonNone, - bandwidthRequested: bitrates[1][3], - bandwidthDelta: bitrates[1][3], - bandwidthNeeded: bitrates[1][3], - bitrates: bitrates, - targetLayers: DefaultMaxLayers, - requestLayerSpatial: f.maxLayers.Spatial, - maxLayers: f.maxLayers, - distanceToDesired: -1, + PauseReason: VideoPauseReasonNone, + BandwidthRequested: bitrates[1][3], + BandwidthDelta: bitrates[1][3], + BandwidthNeeded: bitrates[1][3], + Bitrates: bitrates, + TargetLayers: buffer.DefaultMaxLayers, + RequestLayerSpatial: f.maxLayers.Spatial, + MaxLayers: f.maxLayers, + DistanceToDesired: -1, } result = f.AllocateOptimal(nil, bitrates, true) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, DefaultMaxLayers, f.TargetLayers()) + require.Equal(t, buffer.DefaultMaxLayers, f.TargetLayers()) // reset max layers for rest of the tests below - f.maxLayers = DefaultMaxLayers + f.maxLayers = buffer.DefaultMaxLayers // when feed is dry and current is not valid, should set up for opportunistic forwarding // NOTE: feed is dry due to availableLayers = nil, some valid bitrates may be passed in here for testing purposes only disable(f) - expectedTargetLayers := VideoLayers{ + expectedTargetLayers := buffer.VideoLayer{ Spatial: 2, - Temporal: DefaultMaxLayerTemporal, + Temporal: buffer.DefaultMaxLayerTemporal, } expectedResult = VideoAllocation{ - pauseReason: VideoPauseReasonNone, - bandwidthRequested: bitrates[2][1], - bandwidthDelta: bitrates[2][1] - bitrates[1][3], - bandwidthNeeded: bitrates[2][1], - bitrates: bitrates, - targetLayers: expectedTargetLayers, - requestLayerSpatial: expectedTargetLayers.Spatial, - maxLayers: DefaultMaxLayers, - distanceToDesired: -0.5, + PauseReason: VideoPauseReasonNone, + BandwidthRequested: bitrates[2][1], + BandwidthDelta: bitrates[2][1] - bitrates[1][3], + BandwidthNeeded: bitrates[2][1], + Bitrates: bitrates, + TargetLayers: expectedTargetLayers, + RequestLayerSpatial: expectedTargetLayers.Spatial, + MaxLayers: buffer.DefaultMaxLayers, + DistanceToDesired: -0.5, } result = f.AllocateOptimal(nil, bitrates, true) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) require.Equal(t, expectedTargetLayers, f.TargetLayers()) - f.targetLayers = VideoLayers{Spatial: 0, Temporal: 0} // set to valid to trigger paths in tests below - f.currentLayers = VideoLayers{Spatial: 0, Temporal: 3} // set to valid to trigger paths in tests below + f.targetLayers = buffer.VideoLayer{Spatial: 0, Temporal: 0} // set to valid to trigger paths in tests below + f.currentLayers = buffer.VideoLayer{Spatial: 0, Temporal: 3} // set to valid to trigger paths in tests below // when feed is dry and current is valid, should stay at current - expectedTargetLayers = VideoLayers{ + expectedTargetLayers = buffer.VideoLayer{ Spatial: 0, Temporal: 3, } expectedResult = VideoAllocation{ - pauseReason: VideoPauseReasonFeedDry, - bandwidthRequested: 0, - bandwidthDelta: 0 - bitrates[2][1], - bitrates: emptyBitrates, - targetLayers: expectedTargetLayers, - requestLayerSpatial: expectedTargetLayers.Spatial, - maxLayers: DefaultMaxLayers, - distanceToDesired: -0.75, + PauseReason: VideoPauseReasonFeedDry, + BandwidthRequested: 0, + BandwidthDelta: 0 - bitrates[2][1], + Bitrates: emptyBitrates, + TargetLayers: expectedTargetLayers, + RequestLayerSpatial: expectedTargetLayers.Spatial, + MaxLayers: buffer.DefaultMaxLayers, + DistanceToDesired: -0.75, } result = f.AllocateOptimal(nil, emptyBitrates, true) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) require.Equal(t, expectedTargetLayers, f.TargetLayers()) - f.currentLayers = InvalidLayers + f.currentLayers = buffer.InvalidLayers // opportunistic target if feed is not dry and current is not valid, i. e. not forwarding expectedResult = VideoAllocation{ - pauseReason: VideoPauseReasonNone, - bandwidthRequested: bitrates[2][1], - bandwidthDelta: bitrates[2][1], - bandwidthNeeded: bitrates[2][1], - bitrates: bitrates, - targetLayers: DefaultMaxLayers, - requestLayerSpatial: 2, - maxLayers: DefaultMaxLayers, - distanceToDesired: -0.5, + PauseReason: VideoPauseReasonNone, + BandwidthRequested: bitrates[2][1], + BandwidthDelta: bitrates[2][1], + BandwidthNeeded: bitrates[2][1], + Bitrates: bitrates, + TargetLayers: buffer.DefaultMaxLayers, + RequestLayerSpatial: 2, + MaxLayers: buffer.DefaultMaxLayers, + DistanceToDesired: -0.5, } result = f.AllocateOptimal([]int32{0, 1}, bitrates, true) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, DefaultMaxLayers, f.TargetLayers()) + require.Equal(t, buffer.DefaultMaxLayers, f.TargetLayers()) // if feed is not dry and current is not locked, should be opportunistic (with and without overshoot) - f.targetLayers = InvalidLayers + f.targetLayers = buffer.InvalidLayers expectedResult = VideoAllocation{ - pauseReason: VideoPauseReasonFeedDry, - bandwidthRequested: 0, - bandwidthDelta: 0 - bitrates[2][1], - bitrates: emptyBitrates, - targetLayers: DefaultMaxLayers, - requestLayerSpatial: 2, - maxLayers: DefaultMaxLayers, - distanceToDesired: -1.0, + PauseReason: VideoPauseReasonFeedDry, + BandwidthRequested: 0, + BandwidthDelta: 0 - bitrates[2][1], + Bitrates: emptyBitrates, + TargetLayers: buffer.DefaultMaxLayers, + RequestLayerSpatial: 2, + MaxLayers: buffer.DefaultMaxLayers, + DistanceToDesired: -1.0, } result = f.AllocateOptimal([]int32{0, 1}, emptyBitrates, false) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - f.targetLayers = InvalidLayers - expectedTargetLayers = VideoLayers{ + f.targetLayers = buffer.InvalidLayers + expectedTargetLayers = buffer.VideoLayer{ Spatial: 2, - Temporal: DefaultMaxLayerTemporal, + Temporal: buffer.DefaultMaxLayerTemporal, } expectedResult = VideoAllocation{ - pauseReason: VideoPauseReasonNone, - bandwidthRequested: bitrates[2][1], - bandwidthDelta: bitrates[2][1], - bandwidthNeeded: bitrates[2][1], - bitrates: bitrates, - targetLayers: expectedTargetLayers, - requestLayerSpatial: 2, - maxLayers: DefaultMaxLayers, - distanceToDesired: -0.5, + PauseReason: VideoPauseReasonNone, + BandwidthRequested: bitrates[2][1], + BandwidthDelta: bitrates[2][1], + BandwidthNeeded: bitrates[2][1], + Bitrates: bitrates, + TargetLayers: expectedTargetLayers, + RequestLayerSpatial: 2, + MaxLayers: buffer.DefaultMaxLayers, + DistanceToDesired: -0.5, } result = f.AllocateOptimal([]int32{0, 1}, bitrates, true) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) // switches to highest available if feed is not dry and current is valid and current is not available - f.currentLayers = VideoLayers{Spatial: 0, Temporal: 1} - expectedTargetLayers = VideoLayers{ + f.currentLayers = buffer.VideoLayer{Spatial: 0, Temporal: 1} + expectedTargetLayers = buffer.VideoLayer{ Spatial: 1, - Temporal: DefaultMaxLayerTemporal, + Temporal: buffer.DefaultMaxLayerTemporal, } expectedResult = VideoAllocation{ - pauseReason: VideoPauseReasonNone, - bandwidthRequested: bitrates[2][1], - bandwidthDelta: 0, - bandwidthNeeded: bitrates[2][1], - bitrates: bitrates, - targetLayers: expectedTargetLayers, - requestLayerSpatial: 1, - maxLayers: DefaultMaxLayers, - distanceToDesired: 0.5, + PauseReason: VideoPauseReasonNone, + BandwidthRequested: bitrates[2][1], + BandwidthDelta: 0, + BandwidthNeeded: bitrates[2][1], + Bitrates: bitrates, + TargetLayers: expectedTargetLayers, + RequestLayerSpatial: 1, + MaxLayers: buffer.DefaultMaxLayers, + DistanceToDesired: 0.5, } result = f.AllocateOptimal([]int32{1}, bitrates, true) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) // stays the same if feed is not dry and current is valid, available and locked - f.maxLayers = VideoLayers{Spatial: 0, Temporal: 1} - f.currentLayers = VideoLayers{Spatial: 0, Temporal: 1} + f.maxLayers = buffer.VideoLayer{Spatial: 0, Temporal: 1} + f.currentLayers = buffer.VideoLayer{Spatial: 0, Temporal: 1} f.requestLayerSpatial = 0 - expectedTargetLayers = VideoLayers{ + expectedTargetLayers = buffer.VideoLayer{ Spatial: 0, Temporal: 1, } expectedResult = VideoAllocation{ - pauseReason: VideoPauseReasonFeedDry, - bandwidthRequested: 0, - bandwidthDelta: 0 - bitrates[2][1], - bitrates: emptyBitrates, - targetLayers: expectedTargetLayers, - requestLayerSpatial: 0, - maxLayers: f.maxLayers, - distanceToDesired: 0.0, + PauseReason: VideoPauseReasonFeedDry, + BandwidthRequested: 0, + BandwidthDelta: 0 - bitrates[2][1], + Bitrates: emptyBitrates, + TargetLayers: expectedTargetLayers, + RequestLayerSpatial: 0, + MaxLayers: f.maxLayers, + DistanceToDesired: 0.0, } result = f.AllocateOptimal([]int32{0, 1}, emptyBitrates, true) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) // opportunistic if feed is not dry and current is valid, but request layer has changed - f.maxLayers = VideoLayers{Spatial: 2, Temporal: 1} - f.currentLayers = VideoLayers{Spatial: 0, Temporal: 1} + f.maxLayers = buffer.VideoLayer{Spatial: 2, Temporal: 1} + f.currentLayers = buffer.VideoLayer{Spatial: 0, Temporal: 1} f.requestLayerSpatial = 0 - expectedTargetLayers = VideoLayers{ + expectedTargetLayers = buffer.VideoLayer{ Spatial: 2, Temporal: 3, } expectedResult = VideoAllocation{ - pauseReason: VideoPauseReasonFeedDry, - bandwidthRequested: 0, - bandwidthDelta: 0, - bitrates: emptyBitrates, - targetLayers: expectedTargetLayers, - requestLayerSpatial: 2, - maxLayers: f.maxLayers, - distanceToDesired: -1.5, + PauseReason: VideoPauseReasonFeedDry, + BandwidthRequested: 0, + BandwidthDelta: 0, + Bitrates: emptyBitrates, + TargetLayers: expectedTargetLayers, + RequestLayerSpatial: 2, + MaxLayers: f.maxLayers, + DistanceToDesired: -1.5, } result = f.AllocateOptimal([]int32{0, 1}, emptyBitrates, true) require.Equal(t, expectedResult, result) @@ -406,10 +406,10 @@ func TestForwarderAllocateOptimal(t *testing.T) { func TestForwarderProvisionalAllocate(t *testing.T) { f := newForwarder(testutils.TestVP8Codec, webrtc.RTPCodecTypeVideo) - f.SetMaxSpatialLayer(DefaultMaxLayerSpatial) - f.SetMaxTemporalLayer(DefaultMaxLayerTemporal) - f.SetMaxPublishedLayer(DefaultMaxLayerSpatial) - f.SetMaxTemporalLayerSeen(DefaultMaxLayerTemporal) + f.SetMaxSpatialLayer(buffer.DefaultMaxLayerSpatial) + f.SetMaxTemporalLayer(buffer.DefaultMaxLayerTemporal) + f.SetMaxPublishedLayer(buffer.DefaultMaxLayerSpatial) + f.SetMaxTemporalLayerSeen(buffer.DefaultMaxLayerTemporal) bitrates := Bitrates{ {1, 2, 3, 4}, @@ -419,37 +419,37 @@ func TestForwarderProvisionalAllocate(t *testing.T) { f.ProvisionalAllocatePrepare(nil, bitrates) - usedBitrate := f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 0, Temporal: 0}, true, false) + usedBitrate := f.ProvisionalAllocate(bitrates[2][3], buffer.VideoLayer{Spatial: 0, Temporal: 0}, true, false) require.Equal(t, bitrates[0][0], usedBitrate) - usedBitrate = f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 2, Temporal: 3}, true, false) + usedBitrate = f.ProvisionalAllocate(bitrates[2][3], buffer.VideoLayer{Spatial: 2, Temporal: 3}, true, false) require.Equal(t, bitrates[2][3]-bitrates[0][0], usedBitrate) - usedBitrate = f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 0, Temporal: 3}, true, false) + usedBitrate = f.ProvisionalAllocate(bitrates[2][3], buffer.VideoLayer{Spatial: 0, Temporal: 3}, true, false) require.Equal(t, bitrates[0][3]-bitrates[2][3], usedBitrate) - usedBitrate = f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 1, Temporal: 2}, true, false) + usedBitrate = f.ProvisionalAllocate(bitrates[2][3], buffer.VideoLayer{Spatial: 1, Temporal: 2}, true, false) require.Equal(t, bitrates[1][2]-bitrates[0][3], usedBitrate) // available not enough to reach (2, 2), allocating at (2, 2) should not succeed - usedBitrate = f.ProvisionalAllocate(bitrates[2][2]-bitrates[1][2]-1, VideoLayers{Spatial: 2, Temporal: 2}, true, false) + usedBitrate = f.ProvisionalAllocate(bitrates[2][2]-bitrates[1][2]-1, buffer.VideoLayer{Spatial: 2, Temporal: 2}, true, false) require.Equal(t, int64(0), usedBitrate) // committing should set target to (1, 2) - expectedTargetLayers := VideoLayers{ + expectedTargetLayers := buffer.VideoLayer{ Spatial: 1, Temporal: 2, } expectedResult := VideoAllocation{ - isDeficient: true, - bandwidthRequested: bitrates[1][2], - bandwidthDelta: bitrates[1][2], - bandwidthNeeded: bitrates[2][3], - bitrates: bitrates, - targetLayers: expectedTargetLayers, - requestLayerSpatial: expectedTargetLayers.Spatial, - maxLayers: DefaultMaxLayers, - distanceToDesired: 1.25, + IsDeficient: true, + BandwidthRequested: bitrates[1][2], + BandwidthDelta: bitrates[1][2], + BandwidthNeeded: bitrates[2][3], + Bitrates: bitrates, + TargetLayers: expectedTargetLayers, + RequestLayerSpatial: expectedTargetLayers.Spatial, + MaxLayers: buffer.DefaultMaxLayers, + DistanceToDesired: 1.25, } result := f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) @@ -457,26 +457,26 @@ func TestForwarderProvisionalAllocate(t *testing.T) { require.Equal(t, expectedTargetLayers, f.TargetLayers()) // when nothing fits and pausing disallowed, should allocate (0, 0) - f.targetLayers = InvalidLayers + f.targetLayers = buffer.InvalidLayers f.ProvisionalAllocatePrepare(nil, bitrates) - usedBitrate = f.ProvisionalAllocate(0, VideoLayers{Spatial: 0, Temporal: 0}, false, false) + usedBitrate = f.ProvisionalAllocate(0, buffer.VideoLayer{Spatial: 0, Temporal: 0}, false, false) require.Equal(t, int64(1), usedBitrate) // committing should set target to (0, 0) - expectedTargetLayers = VideoLayers{ + expectedTargetLayers = buffer.VideoLayer{ Spatial: 0, Temporal: 0, } expectedResult = VideoAllocation{ - isDeficient: true, - bandwidthRequested: bitrates[0][0], - bandwidthDelta: bitrates[0][0] - bitrates[1][2], - bandwidthNeeded: bitrates[2][3], - bitrates: bitrates, - targetLayers: expectedTargetLayers, - requestLayerSpatial: expectedTargetLayers.Spatial, - maxLayers: DefaultMaxLayers, - distanceToDesired: 2.75, + IsDeficient: true, + BandwidthRequested: bitrates[0][0], + BandwidthDelta: bitrates[0][0] - bitrates[1][2], + BandwidthNeeded: bitrates[2][3], + Bitrates: bitrates, + TargetLayers: expectedTargetLayers, + RequestLayerSpatial: expectedTargetLayers.Spatial, + MaxLayers: buffer.DefaultMaxLayers, + DistanceToDesired: 2.75, } result = f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) @@ -496,34 +496,34 @@ func TestForwarderProvisionalAllocate(t *testing.T) { f.ProvisionalAllocatePrepare(nil, bitrates) - usedBitrate = f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 0, Temporal: 0}, false, true) + usedBitrate = f.ProvisionalAllocate(bitrates[2][3], buffer.VideoLayer{Spatial: 0, Temporal: 0}, false, true) require.Equal(t, int64(0), usedBitrate) // overshoot should succeed - usedBitrate = f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 2, Temporal: 3}, false, true) + usedBitrate = f.ProvisionalAllocate(bitrates[2][3], buffer.VideoLayer{Spatial: 2, Temporal: 3}, false, true) require.Equal(t, bitrates[2][3], usedBitrate) // overshoot should succeed - this should win as this is lesser overshoot - usedBitrate = f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 1, Temporal: 3}, false, true) + usedBitrate = f.ProvisionalAllocate(bitrates[2][3], buffer.VideoLayer{Spatial: 1, Temporal: 3}, false, true) require.Equal(t, bitrates[1][3]-bitrates[2][3], usedBitrate) // committing should set target to (1, 3) - expectedTargetLayers = VideoLayers{ + expectedTargetLayers = buffer.VideoLayer{ Spatial: 1, Temporal: 3, } - expectedMaxLayers := VideoLayers{ + expectedMaxLayers := buffer.VideoLayer{ Spatial: 0, Temporal: 3, } expectedResult = VideoAllocation{ - bandwidthRequested: bitrates[1][3], - bandwidthDelta: bitrates[1][3] - 1, // 1 is the last allocation bandwidth requested - bitrates: bitrates, - targetLayers: expectedTargetLayers, - requestLayerSpatial: expectedTargetLayers.Spatial, - maxLayers: expectedMaxLayers, - distanceToDesired: -1.75, + BandwidthRequested: bitrates[1][3], + BandwidthDelta: bitrates[1][3] - 1, // 1 is the last allocation bandwidth requested + Bitrates: bitrates, + TargetLayers: expectedTargetLayers, + RequestLayerSpatial: expectedTargetLayers.Spatial, + MaxLayers: expectedMaxLayers, + DistanceToDesired: -1.75, } result = f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) @@ -539,35 +539,35 @@ func TestForwarderProvisionalAllocate(t *testing.T) { {0, 0, 0, 0}, } - f.currentLayers = VideoLayers{Spatial: 0, Temporal: 2} + f.currentLayers = buffer.VideoLayer{Spatial: 0, Temporal: 2} f.ProvisionalAllocatePrepare(nil, bitrates) // all the provisional allocations should not succeed because the feed is dry - usedBitrate = f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 0, Temporal: 0}, false, true) + usedBitrate = f.ProvisionalAllocate(bitrates[2][3], buffer.VideoLayer{Spatial: 0, Temporal: 0}, false, true) require.Equal(t, int64(0), usedBitrate) // overshoot should not succeed - usedBitrate = f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 2, Temporal: 3}, false, true) + usedBitrate = f.ProvisionalAllocate(bitrates[2][3], buffer.VideoLayer{Spatial: 2, Temporal: 3}, false, true) require.Equal(t, int64(0), usedBitrate) // overshoot should not succeed - usedBitrate = f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 1, Temporal: 3}, false, true) + usedBitrate = f.ProvisionalAllocate(bitrates[2][3], buffer.VideoLayer{Spatial: 1, Temporal: 3}, false, true) require.Equal(t, int64(0), usedBitrate) // committing should set target to (0, 2), i. e. leave it at current for opportunistic forwarding - expectedTargetLayers = VideoLayers{ + expectedTargetLayers = buffer.VideoLayer{ Spatial: 0, Temporal: 2, } expectedResult = VideoAllocation{ - pauseReason: VideoPauseReasonFeedDry, - bandwidthRequested: bitrates[0][2], - bandwidthDelta: bitrates[0][2] - 8, // 8 is the last allocation bandwidth requested - bitrates: bitrates, - targetLayers: expectedTargetLayers, - requestLayerSpatial: expectedTargetLayers.Spatial, - maxLayers: expectedMaxLayers, - distanceToDesired: 0.25, + PauseReason: VideoPauseReasonFeedDry, + BandwidthRequested: bitrates[0][2], + BandwidthDelta: bitrates[0][2] - 8, // 8 is the last allocation bandwidth requested + Bitrates: bitrates, + TargetLayers: expectedTargetLayers, + RequestLayerSpatial: expectedTargetLayers.Spatial, + MaxLayers: expectedMaxLayers, + DistanceToDesired: 0.25, } result = f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) @@ -577,42 +577,42 @@ func TestForwarderProvisionalAllocate(t *testing.T) { // // Same case as above, but current is above max, so target should go to invalid // - f.currentLayers = VideoLayers{Spatial: 1, Temporal: 2} + f.currentLayers = buffer.VideoLayer{Spatial: 1, Temporal: 2} f.ProvisionalAllocatePrepare(nil, bitrates) // all the provisional allocations below should not succeed because the feed is dry - usedBitrate = f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 0, Temporal: 0}, false, true) + usedBitrate = f.ProvisionalAllocate(bitrates[2][3], buffer.VideoLayer{Spatial: 0, Temporal: 0}, false, true) require.Equal(t, int64(0), usedBitrate) // overshoot should not succeed - usedBitrate = f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 2, Temporal: 3}, false, true) + usedBitrate = f.ProvisionalAllocate(bitrates[2][3], buffer.VideoLayer{Spatial: 2, Temporal: 3}, false, true) require.Equal(t, int64(0), usedBitrate) // overshoot should not succeed - usedBitrate = f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 1, Temporal: 3}, false, true) + usedBitrate = f.ProvisionalAllocate(bitrates[2][3], buffer.VideoLayer{Spatial: 1, Temporal: 3}, false, true) require.Equal(t, int64(0), usedBitrate) expectedResult = VideoAllocation{ - pauseReason: VideoPauseReasonFeedDry, - bandwidthRequested: 0, - bandwidthDelta: 0, - bitrates: bitrates, - targetLayers: InvalidLayers, - requestLayerSpatial: InvalidLayerSpatial, - maxLayers: expectedMaxLayers, - distanceToDesired: 0.25, + PauseReason: VideoPauseReasonFeedDry, + BandwidthRequested: 0, + BandwidthDelta: 0, + Bitrates: bitrates, + TargetLayers: buffer.InvalidLayers, + RequestLayerSpatial: buffer.InvalidLayerSpatial, + MaxLayers: expectedMaxLayers, + DistanceToDesired: 0.25, } result = f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, InvalidLayers, f.TargetLayers()) - require.Equal(t, InvalidLayers, f.CurrentLayers()) + require.Equal(t, buffer.InvalidLayers, f.TargetLayers()) + require.Equal(t, buffer.InvalidLayers, f.CurrentLayers()) } func TestForwarderProvisionalAllocateMute(t *testing.T) { f := newForwarder(testutils.TestVP8Codec, webrtc.RTPCodecTypeVideo) - f.SetMaxSpatialLayer(DefaultMaxLayerSpatial) - f.SetMaxTemporalLayer(DefaultMaxLayerTemporal) + f.SetMaxSpatialLayer(buffer.DefaultMaxLayerSpatial) + f.SetMaxTemporalLayer(buffer.DefaultMaxLayerTemporal) bitrates := Bitrates{ {1, 2, 3, 4}, @@ -623,35 +623,35 @@ func TestForwarderProvisionalAllocateMute(t *testing.T) { f.Mute(true) f.ProvisionalAllocatePrepare(nil, bitrates) - usedBitrate := f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 0, Temporal: 0}, true, false) + usedBitrate := f.ProvisionalAllocate(bitrates[2][3], buffer.VideoLayer{Spatial: 0, Temporal: 0}, true, false) require.Equal(t, int64(0), usedBitrate) - usedBitrate = f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 1, Temporal: 2}, true, true) + usedBitrate = f.ProvisionalAllocate(bitrates[2][3], buffer.VideoLayer{Spatial: 1, Temporal: 2}, true, true) require.Equal(t, int64(0), usedBitrate) - // committing should set target to InvalidLayers as track is muted + // committing should set target to buffer.InvalidLayers as track is muted expectedResult := VideoAllocation{ - pauseReason: VideoPauseReasonMuted, - bandwidthRequested: 0, - bandwidthDelta: 0, - bitrates: bitrates, - targetLayers: InvalidLayers, - requestLayerSpatial: InvalidLayerSpatial, - maxLayers: DefaultMaxLayers, - distanceToDesired: 0, + PauseReason: VideoPauseReasonMuted, + BandwidthRequested: 0, + BandwidthDelta: 0, + Bitrates: bitrates, + TargetLayers: buffer.InvalidLayers, + RequestLayerSpatial: buffer.InvalidLayerSpatial, + MaxLayers: buffer.DefaultMaxLayers, + DistanceToDesired: 0, } result := f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, InvalidLayers, f.TargetLayers()) + require.Equal(t, buffer.InvalidLayers, f.TargetLayers()) } func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { f := newForwarder(testutils.TestVP8Codec, webrtc.RTPCodecTypeVideo) - f.SetMaxSpatialLayer(DefaultMaxLayerSpatial) - f.SetMaxTemporalLayer(DefaultMaxLayerTemporal) - f.SetMaxPublishedLayer(DefaultMaxLayerSpatial) - f.SetMaxTemporalLayerSeen(DefaultMaxLayerTemporal) + f.SetMaxSpatialLayer(buffer.DefaultMaxLayerSpatial) + f.SetMaxTemporalLayer(buffer.DefaultMaxLayerTemporal) + f.SetMaxPublishedLayer(buffer.DefaultMaxLayerSpatial) + f.SetMaxTemporalLayerSeen(buffer.DefaultMaxLayerTemporal) bitrates := Bitrates{ {1, 2, 3, 4}, @@ -661,27 +661,27 @@ func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { f.ProvisionalAllocatePrepare(nil, bitrates) - // from scratch (InvalidLayers) should give back layer (0, 0) + // from scratch (buffer.InvalidLayers) should give back layer (0, 0) expectedTransition := VideoTransition{ - from: InvalidLayers, - to: VideoLayers{Spatial: 0, Temporal: 0}, - bandwidthDelta: 1, + From: buffer.InvalidLayers, + To: buffer.VideoLayer{Spatial: 0, Temporal: 0}, + BandwidthDelta: 1, } transition := f.ProvisionalAllocateGetCooperativeTransition(false) require.Equal(t, expectedTransition, transition) // committing should set target to (0, 0) - expectedLayers := VideoLayers{Spatial: 0, Temporal: 0} + expectedLayers := buffer.VideoLayer{Spatial: 0, Temporal: 0} expectedResult := VideoAllocation{ - isDeficient: true, - bandwidthRequested: 1, - bandwidthDelta: 1, - bandwidthNeeded: bitrates[2][1], - bitrates: bitrates, - targetLayers: expectedLayers, - requestLayerSpatial: expectedLayers.Spatial, - maxLayers: DefaultMaxLayers, - distanceToDesired: 2.25, + IsDeficient: true, + BandwidthRequested: 1, + BandwidthDelta: 1, + BandwidthNeeded: bitrates[2][1], + Bitrates: bitrates, + TargetLayers: expectedLayers, + RequestLayerSpatial: expectedLayers.Spatial, + MaxLayers: buffer.DefaultMaxLayers, + DistanceToDesired: 2.25, } result := f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) @@ -689,28 +689,28 @@ func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { require.Equal(t, expectedLayers, f.TargetLayers()) // a higher target that is already streaming, just maintain it - targetLayers := VideoLayers{Spatial: 2, Temporal: 1} + targetLayers := buffer.VideoLayer{Spatial: 2, Temporal: 1} f.targetLayers = targetLayers - f.lastAllocation.bandwidthRequested = 10 + f.lastAllocation.BandwidthRequested = 10 expectedTransition = VideoTransition{ - from: targetLayers, - to: targetLayers, - bandwidthDelta: 0, + From: targetLayers, + To: targetLayers, + BandwidthDelta: 0, } transition = f.ProvisionalAllocateGetCooperativeTransition(false) require.Equal(t, expectedTransition, transition) // committing should set target to (2, 1) - expectedLayers = VideoLayers{Spatial: 2, Temporal: 1} + expectedLayers = buffer.VideoLayer{Spatial: 2, Temporal: 1} expectedResult = VideoAllocation{ - bandwidthRequested: 10, - bandwidthDelta: 0, - bitrates: bitrates, - bandwidthNeeded: bitrates[2][1], - targetLayers: expectedLayers, - requestLayerSpatial: expectedLayers.Spatial, - maxLayers: DefaultMaxLayers, - distanceToDesired: 0.0, + BandwidthRequested: 10, + BandwidthDelta: 0, + Bitrates: bitrates, + BandwidthNeeded: bitrates[2][1], + TargetLayers: expectedLayers, + RequestLayerSpatial: expectedLayers.Spatial, + MaxLayers: buffer.DefaultMaxLayers, + DistanceToDesired: 0.0, } result = f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) @@ -718,12 +718,12 @@ func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { require.Equal(t, expectedLayers, f.TargetLayers()) // from a target that has become unavailable, should switch to lower available layer - targetLayers = VideoLayers{Spatial: 2, Temporal: 2} + targetLayers = buffer.VideoLayer{Spatial: 2, Temporal: 2} f.targetLayers = targetLayers expectedTransition = VideoTransition{ - from: targetLayers, - to: VideoLayers{Spatial: 2, Temporal: 1}, - bandwidthDelta: 0, + From: targetLayers, + To: buffer.VideoLayer{Spatial: 2, Temporal: 1}, + BandwidthDelta: 0, } transition = f.ProvisionalAllocateGetCooperativeTransition(false) require.Equal(t, expectedTransition, transition) @@ -734,11 +734,11 @@ func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { f.Mute(true) f.ProvisionalAllocatePrepare(nil, bitrates) - // mute should send target to InvalidLayers + // mute should send target to buffer.InvalidLayers expectedTransition = VideoTransition{ - from: VideoLayers{Spatial: 2, Temporal: 1}, - to: InvalidLayers, - bandwidthDelta: -10, + From: buffer.VideoLayer{Spatial: 2, Temporal: 1}, + To: buffer.InvalidLayers, + BandwidthDelta: -10, } transition = f.ProvisionalAllocateGetCooperativeTransition(false) require.Equal(t, expectedTransition, transition) @@ -757,29 +757,29 @@ func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { {9, 10, 0, 0}, } - f.targetLayers = InvalidLayers + f.targetLayers = buffer.InvalidLayers f.ProvisionalAllocatePrepare(nil, bitrates) - // from scratch (InvalidLayers) should go to a layer past maximum as overshoot is allowed + // from scratch (buffer.InvalidLayers) should go to a layer past maximum as overshoot is allowed expectedTransition = VideoTransition{ - from: InvalidLayers, - to: VideoLayers{Spatial: 1, Temporal: 0}, - bandwidthDelta: 5, + From: buffer.InvalidLayers, + To: buffer.VideoLayer{Spatial: 1, Temporal: 0}, + BandwidthDelta: 5, } transition = f.ProvisionalAllocateGetCooperativeTransition(true) require.Equal(t, expectedTransition, transition) // committing should set target to (1, 0) - expectedLayers = VideoLayers{Spatial: 1, Temporal: 0} - expectedMaxLayers := VideoLayers{Spatial: 0, Temporal: DefaultMaxLayerTemporal} + expectedLayers = buffer.VideoLayer{Spatial: 1, Temporal: 0} + expectedMaxLayers := buffer.VideoLayer{Spatial: 0, Temporal: buffer.DefaultMaxLayerTemporal} expectedResult = VideoAllocation{ - bandwidthRequested: 5, - bandwidthDelta: 5, - bitrates: bitrates, - targetLayers: expectedLayers, - requestLayerSpatial: expectedLayers.Spatial, - maxLayers: expectedMaxLayers, - distanceToDesired: -1.0, + BandwidthRequested: 5, + BandwidthDelta: 5, + Bitrates: bitrates, + TargetLayers: expectedLayers, + RequestLayerSpatial: expectedLayers.Spatial, + MaxLayers: expectedMaxLayers, + DistanceToDesired: -1.0, } result = f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) @@ -795,30 +795,30 @@ func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { {0, 0, 0, 0}, } - f.currentLayers = VideoLayers{Spatial: 0, Temporal: 2} - f.targetLayers = InvalidLayers + f.currentLayers = buffer.VideoLayer{Spatial: 0, Temporal: 2} + f.targetLayers = buffer.InvalidLayers f.ProvisionalAllocatePrepare(nil, bitrates) - // from scratch (InvalidLayers) should go to current layer - // NOTE: targetLayer is set to InvalidLayers for testing, but in practice current layers valid and target layers invalid should not happen + // from scratch (buffer.InvalidLayers) should go to current layer + // NOTE: targetLayer is set to buffer.InvalidLayers for testing, but in practice current layers valid and target layers invalid should not happen expectedTransition = VideoTransition{ - from: InvalidLayers, - to: VideoLayers{Spatial: 0, Temporal: 2}, - bandwidthDelta: -5, // 5 was the bandwidth needed for the last allocation + From: buffer.InvalidLayers, + To: buffer.VideoLayer{Spatial: 0, Temporal: 2}, + BandwidthDelta: -5, // 5 was the bandwidth needed for the last allocation } transition = f.ProvisionalAllocateGetCooperativeTransition(true) require.Equal(t, expectedTransition, transition) // committing should set target to (0, 2) - expectedLayers = VideoLayers{Spatial: 0, Temporal: 2} + expectedLayers = buffer.VideoLayer{Spatial: 0, Temporal: 2} expectedResult = VideoAllocation{ - bandwidthRequested: 0, - bandwidthDelta: -5, - bitrates: bitrates, - targetLayers: expectedLayers, - requestLayerSpatial: expectedLayers.Spatial, - maxLayers: expectedMaxLayers, - distanceToDesired: -0.5, + BandwidthRequested: 0, + BandwidthDelta: -5, + Bitrates: bitrates, + TargetLayers: expectedLayers, + RequestLayerSpatial: expectedLayers.Spatial, + MaxLayers: expectedMaxLayers, + DistanceToDesired: -0.5, } result = f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) @@ -827,13 +827,13 @@ func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { // committing should set target to current layers to enable opportunistic forwarding expectedResult = VideoAllocation{ - bandwidthRequested: 0, - bandwidthDelta: 0, - bitrates: bitrates, - targetLayers: expectedLayers, - requestLayerSpatial: expectedLayers.Spatial, - maxLayers: expectedMaxLayers, - distanceToDesired: -0.5, + BandwidthRequested: 0, + BandwidthDelta: 0, + Bitrates: bitrates, + TargetLayers: expectedLayers, + RequestLayerSpatial: expectedLayers.Spatial, + MaxLayers: expectedMaxLayers, + DistanceToDesired: -0.5, } result = f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) @@ -843,8 +843,8 @@ func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { func TestForwarderProvisionalAllocateGetBestWeightedTransition(t *testing.T) { f := newForwarder(testutils.TestVP8Codec, webrtc.RTPCodecTypeVideo) - f.SetMaxSpatialLayer(DefaultMaxLayerSpatial) - f.SetMaxTemporalLayer(DefaultMaxLayerTemporal) + f.SetMaxSpatialLayer(buffer.DefaultMaxLayerSpatial) + f.SetMaxTemporalLayer(buffer.DefaultMaxLayerTemporal) bitrates := Bitrates{ {1, 2, 3, 4}, @@ -854,12 +854,12 @@ func TestForwarderProvisionalAllocateGetBestWeightedTransition(t *testing.T) { f.ProvisionalAllocatePrepare(nil, bitrates) - f.targetLayers = VideoLayers{Spatial: 2, Temporal: 2} - f.lastAllocation.bandwidthRequested = bitrates[2][2] + f.targetLayers = buffer.VideoLayer{Spatial: 2, Temporal: 2} + f.lastAllocation.BandwidthRequested = bitrates[2][2] expectedTransition := VideoTransition{ - from: f.targetLayers, - to: VideoLayers{Spatial: 2, Temporal: 0}, - bandwidthDelta: 2, + From: f.targetLayers, + To: buffer.VideoLayer{Spatial: 2, Temporal: 0}, + BandwidthDelta: 2, } transition := f.ProvisionalAllocateGetBestWeightedTransition() require.Equal(t, expectedTransition, transition) @@ -867,9 +867,9 @@ func TestForwarderProvisionalAllocateGetBestWeightedTransition(t *testing.T) { func TestForwarderAllocateNextHigher(t *testing.T) { f := newForwarder(testutils.TestOpusCodec, webrtc.RTPCodecTypeAudio) - f.SetMaxSpatialLayer(DefaultMaxLayerSpatial) - f.SetMaxTemporalLayer(DefaultMaxLayerTemporal) - f.SetMaxPublishedLayer(DefaultMaxLayerSpatial) + f.SetMaxSpatialLayer(buffer.DefaultMaxLayerSpatial) + f.SetMaxTemporalLayer(buffer.DefaultMaxLayerTemporal) + f.SetMaxPublishedLayer(buffer.DefaultMaxLayerSpatial) emptyBitrates := Bitrates{} bitrates := Bitrates{ @@ -878,81 +878,81 @@ func TestForwarderAllocateNextHigher(t *testing.T) { {0, 7, 0, 0}, } - result, boosted := f.AllocateNextHigher(ChannelCapacityInfinity, nil, bitrates, false) + result, boosted := f.AllocateNextHigher(100_000_000, nil, bitrates, false) require.Equal(t, VideoAllocationDefault, result) // no layer for audio require.False(t, boosted) f = newForwarder(testutils.TestVP8Codec, webrtc.RTPCodecTypeVideo) - f.SetMaxSpatialLayer(DefaultMaxLayerSpatial) - f.SetMaxTemporalLayer(DefaultMaxLayerTemporal) - f.SetMaxPublishedLayer(DefaultMaxLayerSpatial) - f.SetMaxTemporalLayerSeen(DefaultMaxLayerTemporal) + f.SetMaxSpatialLayer(buffer.DefaultMaxLayerSpatial) + f.SetMaxTemporalLayer(buffer.DefaultMaxLayerTemporal) + f.SetMaxPublishedLayer(buffer.DefaultMaxLayerSpatial) + f.SetMaxTemporalLayerSeen(buffer.DefaultMaxLayerTemporal) // when not in deficient state, does not boost - result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, nil, bitrates, false) + result, boosted = f.AllocateNextHigher(100_000_000, nil, bitrates, false) require.Equal(t, VideoAllocationDefault, result) require.False(t, boosted) // if layers have not caught up, should not allocate next layer even if deficient - f.targetLayers = VideoLayers{ + f.targetLayers = buffer.VideoLayer{ Spatial: 0, Temporal: 0, } - result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, nil, bitrates, false) + result, boosted = f.AllocateNextHigher(100_000_000, nil, bitrates, false) require.Equal(t, VideoAllocationDefault, result) require.False(t, boosted) - f.lastAllocation.isDeficient = true - f.currentLayers = VideoLayers{ + f.lastAllocation.IsDeficient = true + f.currentLayers = buffer.VideoLayer{ Spatial: 0, Temporal: 0, } // move from (0, 0) -> (0, 1), i.e. a higher temporal layer is available in the same spatial layer - expectedTargetLayers := VideoLayers{ + expectedTargetLayers := buffer.VideoLayer{ Spatial: 0, Temporal: 1, } expectedResult := VideoAllocation{ - isDeficient: true, - bandwidthRequested: 3, - bandwidthDelta: 1, - bandwidthNeeded: bitrates[2][1], - bitrates: bitrates, - targetLayers: expectedTargetLayers, - requestLayerSpatial: expectedTargetLayers.Spatial, - maxLayers: DefaultMaxLayers, - distanceToDesired: 2.0, + IsDeficient: true, + BandwidthRequested: 3, + BandwidthDelta: 1, + BandwidthNeeded: bitrates[2][1], + Bitrates: bitrates, + TargetLayers: expectedTargetLayers, + RequestLayerSpatial: expectedTargetLayers.Spatial, + MaxLayers: buffer.DefaultMaxLayers, + DistanceToDesired: 2.0, } - result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, nil, bitrates, false) + result, boosted = f.AllocateNextHigher(100_000_000, nil, bitrates, false) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) require.Equal(t, expectedTargetLayers, f.TargetLayers()) require.True(t, boosted) // empty bitrates cannot increase layer, i. e. last allocation is left unchanged - result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, nil, emptyBitrates, false) + result, boosted = f.AllocateNextHigher(100_000_000, nil, emptyBitrates, false) require.Equal(t, expectedResult, result) require.False(t, boosted) // move from (0, 1) -> (1, 0), i.e. a higher spatial layer is available f.currentLayers.Temporal = 1 - expectedTargetLayers = VideoLayers{ + expectedTargetLayers = buffer.VideoLayer{ Spatial: 1, Temporal: 0, } expectedResult = VideoAllocation{ - isDeficient: true, - bandwidthRequested: 4, - bandwidthDelta: 1, - bandwidthNeeded: bitrates[2][1], - bitrates: bitrates, - targetLayers: expectedTargetLayers, - requestLayerSpatial: expectedTargetLayers.Spatial, - maxLayers: DefaultMaxLayers, - distanceToDesired: 1.25, + IsDeficient: true, + BandwidthRequested: 4, + BandwidthDelta: 1, + BandwidthNeeded: bitrates[2][1], + Bitrates: bitrates, + TargetLayers: expectedTargetLayers, + RequestLayerSpatial: expectedTargetLayers.Spatial, + MaxLayers: buffer.DefaultMaxLayers, + DistanceToDesired: 1.25, } - result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, nil, bitrates, false) + result, boosted = f.AllocateNextHigher(100_000_000, nil, bitrates, false) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) require.Equal(t, expectedTargetLayers, f.TargetLayers()) @@ -961,22 +961,22 @@ func TestForwarderAllocateNextHigher(t *testing.T) { // next higher, move from (1, 0) -> (1, 3), still deficient though f.currentLayers.Spatial = 1 f.currentLayers.Temporal = 0 - expectedTargetLayers = VideoLayers{ + expectedTargetLayers = buffer.VideoLayer{ Spatial: 1, Temporal: 3, } expectedResult = VideoAllocation{ - isDeficient: true, - bandwidthRequested: 5, - bandwidthDelta: 1, - bandwidthNeeded: bitrates[2][1], - bitrates: bitrates, - targetLayers: expectedTargetLayers, - requestLayerSpatial: expectedTargetLayers.Spatial, - maxLayers: DefaultMaxLayers, - distanceToDesired: 0.5, + IsDeficient: true, + BandwidthRequested: 5, + BandwidthDelta: 1, + BandwidthNeeded: bitrates[2][1], + Bitrates: bitrates, + TargetLayers: expectedTargetLayers, + RequestLayerSpatial: expectedTargetLayers.Spatial, + MaxLayers: buffer.DefaultMaxLayers, + DistanceToDesired: 0.5, } - result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, nil, bitrates, false) + result, boosted = f.AllocateNextHigher(100_000_000, nil, bitrates, false) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) require.Equal(t, expectedTargetLayers, f.TargetLayers()) @@ -984,21 +984,21 @@ func TestForwarderAllocateNextHigher(t *testing.T) { // next higher, move from (1, 3) -> (2, 1), optimal allocation f.currentLayers.Temporal = 3 - expectedTargetLayers = VideoLayers{ + expectedTargetLayers = buffer.VideoLayer{ Spatial: 2, Temporal: 1, } expectedResult = VideoAllocation{ - bandwidthRequested: 7, - bandwidthDelta: 2, - bitrates: bitrates, - bandwidthNeeded: bitrates[2][1], - targetLayers: expectedTargetLayers, - requestLayerSpatial: expectedTargetLayers.Spatial, - maxLayers: DefaultMaxLayers, - distanceToDesired: 0.0, + BandwidthRequested: 7, + BandwidthDelta: 2, + Bitrates: bitrates, + BandwidthNeeded: bitrates[2][1], + TargetLayers: expectedTargetLayers, + RequestLayerSpatial: expectedTargetLayers.Spatial, + MaxLayers: buffer.DefaultMaxLayers, + DistanceToDesired: 0.0, } - result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, nil, bitrates, false) + result, boosted = f.AllocateNextHigher(100_000_000, nil, bitrates, false) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) require.Equal(t, expectedTargetLayers, f.TargetLayers()) @@ -1007,7 +1007,7 @@ func TestForwarderAllocateNextHigher(t *testing.T) { // ask again, should return not boosted as there is no room to go higher f.currentLayers.Spatial = 2 f.currentLayers.Temporal = 1 - result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, nil, bitrates, false) + result, boosted = f.AllocateNextHigher(100_000_000, nil, bitrates, false) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) require.Equal(t, expectedTargetLayers, f.TargetLayers()) @@ -1015,25 +1015,25 @@ func TestForwarderAllocateNextHigher(t *testing.T) { // turn off everything, allocating next layer should result in streaming lowest layers disable(f) - f.lastAllocation.isDeficient = true - f.lastAllocation.bandwidthRequested = 0 + f.lastAllocation.IsDeficient = true + f.lastAllocation.BandwidthRequested = 0 - expectedTargetLayers = VideoLayers{ + expectedTargetLayers = buffer.VideoLayer{ Spatial: 0, Temporal: 0, } expectedResult = VideoAllocation{ - isDeficient: true, - bandwidthRequested: 2, - bandwidthDelta: 2, - bandwidthNeeded: bitrates[2][1], - bitrates: bitrates, - targetLayers: expectedTargetLayers, - requestLayerSpatial: expectedTargetLayers.Spatial, - maxLayers: DefaultMaxLayers, - distanceToDesired: 2.25, + IsDeficient: true, + BandwidthRequested: 2, + BandwidthDelta: 2, + BandwidthNeeded: bitrates[2][1], + Bitrates: bitrates, + TargetLayers: expectedTargetLayers, + RequestLayerSpatial: expectedTargetLayers.Spatial, + MaxLayers: buffer.DefaultMaxLayers, + DistanceToDesired: 2.25, } - result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, nil, bitrates, false) + result, boosted = f.AllocateNextHigher(100_000_000, nil, bitrates, false) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) require.Equal(t, expectedTargetLayers, f.TargetLayers()) @@ -1041,15 +1041,15 @@ func TestForwarderAllocateNextHigher(t *testing.T) { // no new available capacity cannot bump up layer expectedResult = VideoAllocation{ - isDeficient: true, - bandwidthRequested: 2, - bandwidthDelta: 2, - bandwidthNeeded: bitrates[2][1], - bitrates: bitrates, - targetLayers: expectedTargetLayers, - requestLayerSpatial: expectedTargetLayers.Spatial, - maxLayers: DefaultMaxLayers, - distanceToDesired: 2.25, + IsDeficient: true, + BandwidthRequested: 2, + BandwidthDelta: 2, + BandwidthNeeded: bitrates[2][1], + Bitrates: bitrates, + TargetLayers: expectedTargetLayers, + RequestLayerSpatial: expectedTargetLayers.Spatial, + MaxLayers: buffer.DefaultMaxLayers, + DistanceToDesired: 2.25, } result, boosted = f.AllocateNextHigher(0, nil, bitrates, false) require.Equal(t, expectedResult, result) @@ -1068,22 +1068,22 @@ func TestForwarderAllocateNextHigher(t *testing.T) { f.currentLayers = f.targetLayers - expectedTargetLayers = VideoLayers{ + expectedTargetLayers = buffer.VideoLayer{ Spatial: 1, Temporal: 0, } - expectedMaxLayers := VideoLayers{ + expectedMaxLayers := buffer.VideoLayer{ Spatial: 0, - Temporal: DefaultMaxLayerTemporal, + Temporal: buffer.DefaultMaxLayerTemporal, } expectedResult = VideoAllocation{ - bandwidthRequested: bitrates[1][0], - bandwidthDelta: bitrates[1][0], - bitrates: bitrates, - targetLayers: expectedTargetLayers, - requestLayerSpatial: expectedTargetLayers.Spatial, - maxLayers: expectedMaxLayers, - distanceToDesired: -1.0, + BandwidthRequested: bitrates[1][0], + BandwidthDelta: bitrates[1][0], + Bitrates: bitrates, + TargetLayers: expectedTargetLayers, + RequestLayerSpatial: expectedTargetLayers.Spatial, + MaxLayers: expectedMaxLayers, + DistanceToDesired: -1.0, } // overshoot should return (1, 0) even if there is not enough capacity result, boosted = f.AllocateNextHigher(bitrates[1][0]-1, nil, bitrates, true) @@ -1095,10 +1095,10 @@ func TestForwarderAllocateNextHigher(t *testing.T) { func TestForwarderPause(t *testing.T) { f := newForwarder(testutils.TestVP8Codec, webrtc.RTPCodecTypeVideo) - f.SetMaxSpatialLayer(DefaultMaxLayerSpatial) - f.SetMaxTemporalLayer(DefaultMaxLayerTemporal) - f.SetMaxPublishedLayer(DefaultMaxLayerSpatial) - f.SetMaxTemporalLayerSeen(DefaultMaxLayerTemporal) + f.SetMaxSpatialLayer(buffer.DefaultMaxLayerSpatial) + f.SetMaxTemporalLayer(buffer.DefaultMaxLayerTemporal) + f.SetMaxPublishedLayer(buffer.DefaultMaxLayerSpatial) + f.SetMaxTemporalLayerSeen(buffer.DefaultMaxLayerTemporal) bitrates := Bitrates{ {1, 2, 3, 4}, @@ -1107,33 +1107,33 @@ func TestForwarderPause(t *testing.T) { } f.ProvisionalAllocatePrepare(nil, bitrates) - f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 0, Temporal: 0}, true, false) + f.ProvisionalAllocate(bitrates[2][3], buffer.VideoLayer{Spatial: 0, Temporal: 0}, true, false) // should have set target at (0, 0) f.ProvisionalAllocateCommit() expectedResult := VideoAllocation{ - pauseReason: VideoPauseReasonBandwidth, - isDeficient: true, - bandwidthRequested: 0, - bandwidthDelta: 0 - bitrates[0][0], - bandwidthNeeded: bitrates[2][3], - bitrates: bitrates, - targetLayers: InvalidLayers, - requestLayerSpatial: InvalidLayerSpatial, - maxLayers: DefaultMaxLayers, - distanceToDesired: 3, + PauseReason: VideoPauseReasonBandwidth, + IsDeficient: true, + BandwidthRequested: 0, + BandwidthDelta: 0 - bitrates[0][0], + BandwidthNeeded: bitrates[2][3], + Bitrates: bitrates, + TargetLayers: buffer.InvalidLayers, + RequestLayerSpatial: buffer.InvalidLayerSpatial, + MaxLayers: buffer.DefaultMaxLayers, + DistanceToDesired: 3, } result := f.Pause(nil, bitrates) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, InvalidLayers, f.TargetLayers()) + require.Equal(t, buffer.InvalidLayers, f.TargetLayers()) } func TestForwarderPauseMute(t *testing.T) { f := newForwarder(testutils.TestVP8Codec, webrtc.RTPCodecTypeVideo) - f.SetMaxSpatialLayer(DefaultMaxLayerSpatial) - f.SetMaxTemporalLayer(DefaultMaxLayerTemporal) - f.SetMaxPublishedLayer(DefaultMaxLayerSpatial) + f.SetMaxSpatialLayer(buffer.DefaultMaxLayerSpatial) + f.SetMaxTemporalLayer(buffer.DefaultMaxLayerTemporal) + f.SetMaxPublishedLayer(buffer.DefaultMaxLayerSpatial) bitrates := Bitrates{ {1, 2, 3, 4}, @@ -1142,25 +1142,25 @@ func TestForwarderPauseMute(t *testing.T) { } f.ProvisionalAllocatePrepare(nil, bitrates) - f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 0, Temporal: 0}, true, true) + f.ProvisionalAllocate(bitrates[2][3], buffer.VideoLayer{Spatial: 0, Temporal: 0}, true, true) // should have set target at (0, 0) f.ProvisionalAllocateCommit() f.Mute(true) expectedResult := VideoAllocation{ - pauseReason: VideoPauseReasonMuted, - bandwidthRequested: 0, - bandwidthDelta: 0 - bitrates[0][0], - bitrates: bitrates, - targetLayers: InvalidLayers, - requestLayerSpatial: InvalidLayerSpatial, - maxLayers: DefaultMaxLayers, - distanceToDesired: 0, + PauseReason: VideoPauseReasonMuted, + BandwidthRequested: 0, + BandwidthDelta: 0 - bitrates[0][0], + Bitrates: bitrates, + TargetLayers: buffer.InvalidLayers, + RequestLayerSpatial: buffer.InvalidLayerSpatial, + MaxLayers: buffer.DefaultMaxLayers, + DistanceToDesired: 0, } result := f.Pause(nil, bitrates) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, InvalidLayers, f.TargetLayers()) + require.Equal(t, buffer.InvalidLayers, f.TargetLayers()) } func TestForwarderGetTranslationParamsMuted(t *testing.T) { @@ -1365,7 +1365,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { require.Equal(t, expectedTP, *actualTP) // although target layer matches, not a key frame, so should drop - f.targetLayers = VideoLayers{ + f.targetLayers = buffer.VideoLayer{ Spatial: 0, Temporal: 1, } @@ -1620,7 +1620,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { // switching SSRC (happens for new layer or new track source) // should lock onto the new source, but sequence number should be contiguous - f.targetLayers = VideoLayers{ + f.targetLayers = buffer.VideoLayer{ Spatial: 1, Temporal: 1, } @@ -1706,11 +1706,11 @@ func TestForwardGetSnTsForPadding(t *testing.T) { } extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8) - f.targetLayers = VideoLayers{ + f.targetLayers = buffer.VideoLayer{ Spatial: 0, Temporal: 1, } - f.currentLayers = InvalidLayers + f.currentLayers = buffer.InvalidLayers // send it through so that forwarder locks onto stream _, _ = f.GetTranslationParams(extPkt, 0) @@ -1773,11 +1773,11 @@ func TestForwardGetSnTsForBlankFrames(t *testing.T) { } extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8) - f.targetLayers = VideoLayers{ + f.targetLayers = buffer.VideoLayer{ Spatial: 0, Temporal: 1, } - f.currentLayers = InvalidLayers + f.currentLayers = buffer.InvalidLayers // send it through so that forwarder locks onto stream _, _ = f.GetTranslationParams(extPkt, 0) @@ -1843,11 +1843,11 @@ func TestForwardGetPaddingVP8(t *testing.T) { } extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8) - f.targetLayers = VideoLayers{ + f.targetLayers = buffer.VideoLayer{ Spatial: 0, Temporal: 1, } - f.currentLayers = InvalidLayers + f.currentLayers = buffer.InvalidLayers // send it through so that forwarder locks onto stream _, _ = f.GetTranslationParams(extPkt, 0) diff --git a/pkg/sfu/receiver.go b/pkg/sfu/receiver.go index c8c5d9a63..3d2cc9c31 100644 --- a/pkg/sfu/receiver.go +++ b/pkg/sfu/receiver.go @@ -30,7 +30,7 @@ var ( type AudioLevelHandle func(level uint8, duration uint32) -type Bitrates [DefaultMaxLayerSpatial + 1][DefaultMaxLayerTemporal + 1]int64 +type Bitrates [buffer.DefaultMaxLayerSpatial + 1][buffer.DefaultMaxLayerTemporal + 1]int64 // TrackReceiver defines an interface receive media from remote peer type TrackReceiver interface { @@ -94,11 +94,11 @@ type WebRTCReceiver struct { twcc *twcc.Responder bufferMu sync.RWMutex - buffers [DefaultMaxLayerSpatial + 1]*buffer.Buffer + buffers [buffer.DefaultMaxLayerSpatial + 1]*buffer.Buffer rtt uint32 upTrackMu sync.RWMutex - upTracks [DefaultMaxLayerSpatial + 1]*webrtc.TrackRemote + upTracks [buffer.DefaultMaxLayerSpatial + 1]*webrtc.TrackRemote lbThreshold int @@ -391,7 +391,7 @@ func (w *WebRTCReceiver) SetMaxExpectedSpatialLayer(layer int32) { w.streamTrackerManager.SetMaxExpectedSpatialLayer(layer) now := time.Now() - if layer == InvalidLayerSpatial { + if layer == buffer.InvalidLayerSpatial { w.connectionStats.UpdateLayerMute(true, now) } else { w.connectionStats.UpdateLayerMute(false, now) diff --git a/pkg/sfu/streamallocator/channelobserver.go b/pkg/sfu/streamallocator/channelobserver.go new file mode 100644 index 000000000..f19bbb692 --- /dev/null +++ b/pkg/sfu/streamallocator/channelobserver.go @@ -0,0 +1,181 @@ +package streamallocator + +import ( + "fmt" + "time" + + "github.com/livekit/protocol/logger" +) + +// ------------------------------------------------ + +type ChannelTrend int + +const ( + ChannelTrendNeutral ChannelTrend = iota + ChannelTrendClearing + ChannelTrendCongesting +) + +func (c ChannelTrend) String() string { + switch c { + case ChannelTrendNeutral: + return "NEUTRAL" + case ChannelTrendClearing: + return "CLEARING" + case ChannelTrendCongesting: + return "CONGESTING" + default: + return fmt.Sprintf("%d", int(c)) + } +} + +// ------------------------------------------------ + +type ChannelCongestionReason int + +const ( + ChannelCongestionReasonNone ChannelCongestionReason = iota + ChannelCongestionReasonEstimate + ChannelCongestionReasonLoss +) + +func (c ChannelCongestionReason) String() string { + switch c { + case ChannelCongestionReasonNone: + return "NONE" + case ChannelCongestionReasonEstimate: + return "ESTIMATE" + case ChannelCongestionReasonLoss: + return "LOSS" + default: + return fmt.Sprintf("%d", int(c)) + } +} + +// ------------------------------------------------ + +type ChannelObserverParams struct { + Name string + EstimateRequiredSamples int + EstimateDownwardTrendThreshold float64 + EstimateCollapseValues bool + NackWindowMinDuration time.Duration + NackWindowMaxDuration time.Duration + NackRatioThreshold float64 +} + +type ChannelObserver struct { + params ChannelObserverParams + logger logger.Logger + + estimateTrend *TrendDetector + + nackWindowStartTime time.Time + packets uint32 + repeatedNacks uint32 +} + +func NewChannelObserver(params ChannelObserverParams, logger logger.Logger) *ChannelObserver { + return &ChannelObserver{ + params: params, + logger: logger, + estimateTrend: NewTrendDetector(TrendDetectorParams{ + Name: params.Name + "-estimate", + Logger: logger, + RequiredSamples: params.EstimateRequiredSamples, + DownwardTrendThreshold: params.EstimateDownwardTrendThreshold, + CollapseValues: params.EstimateCollapseValues, + }), + } +} + +func (c *ChannelObserver) SeedEstimate(estimate int64) { + c.estimateTrend.Seed(estimate) +} + +func (c *ChannelObserver) SeedNack(packets uint32, repeatedNacks uint32) { + c.packets = packets + c.repeatedNacks = repeatedNacks +} + +func (c *ChannelObserver) AddEstimate(estimate int64) { + c.estimateTrend.AddValue(estimate) +} + +func (c *ChannelObserver) AddNack(packets uint32, repeatedNacks uint32) { + if c.params.NackWindowMaxDuration != 0 && !c.nackWindowStartTime.IsZero() && time.Since(c.nackWindowStartTime) > c.params.NackWindowMaxDuration { + c.nackWindowStartTime = time.Time{} + c.packets = 0 + c.repeatedNacks = 0 + } + + // + // Start NACK monitoring window only when a repeated NACK happens. + // This allows locking tightly to when NACKs start happening and + // check if the NACKs keep adding up (potentially a sign of congestion) + // or isolated losses + // + if c.repeatedNacks == 0 && repeatedNacks != 0 { + c.nackWindowStartTime = time.Now() + } + + if !c.nackWindowStartTime.IsZero() { + c.packets += packets + c.repeatedNacks += repeatedNacks + } +} + +func (c *ChannelObserver) GetLowestEstimate() int64 { + return c.estimateTrend.GetLowest() +} + +func (c *ChannelObserver) GetHighestEstimate() int64 { + return c.estimateTrend.GetHighest() +} + +func (c *ChannelObserver) GetNackRatio() (uint32, uint32, float64) { + ratio := 0.0 + if c.packets != 0 { + ratio = float64(c.repeatedNacks) / float64(c.packets) + if ratio > 1.0 { + ratio = 1.0 + } + } + + return c.packets, c.repeatedNacks, ratio +} + +func (c *ChannelObserver) GetTrend() (ChannelTrend, ChannelCongestionReason) { + estimateDirection := c.estimateTrend.GetDirection() + packets, repeatedNacks, nackRatio := c.GetNackRatio() + + switch { + case estimateDirection == TrendDirectionDownward: + c.logger.Debugw( + "stream allocator: channel observer: estimate is trending downward", + "name", c.params.Name, + "estimate", c.estimateTrend.ToString(), + "packets", packets, + "repeatedNacks", repeatedNacks, + "ratio", nackRatio, + ) + return ChannelTrendCongesting, ChannelCongestionReasonEstimate + case c.params.NackWindowMinDuration != 0 && !c.nackWindowStartTime.IsZero() && time.Since(c.nackWindowStartTime) > c.params.NackWindowMinDuration && nackRatio > c.params.NackRatioThreshold: + c.logger.Debugw( + "stream allocator: channel observer: high rate of repeated NACKs", + "name", c.params.Name, + "estimate", c.estimateTrend.ToString(), + "packets", packets, + "repeatedNacks", repeatedNacks, + "ratio", nackRatio, + ) + return ChannelTrendCongesting, ChannelCongestionReasonLoss + case estimateDirection == TrendDirectionUpward: + return ChannelTrendClearing, ChannelCongestionReasonNone + } + + return ChannelTrendNeutral, ChannelCongestionReasonNone +} + +// ------------------------------------------------ diff --git a/pkg/sfu/prober.go b/pkg/sfu/streamallocator/prober.go similarity index 99% rename from pkg/sfu/prober.go rename to pkg/sfu/streamallocator/prober.go index 05bacd349..0e3661bf0 100644 --- a/pkg/sfu/prober.go +++ b/pkg/sfu/streamallocator/prober.go @@ -103,7 +103,7 @@ // window being long(ish). But, RTT should be much shorter especially if // the subscriber peer connection of the client is able to connect to // the nearest data center. -package sfu +package streamallocator import ( "fmt" diff --git a/pkg/sfu/streamallocator.go b/pkg/sfu/streamallocator/streamallocator.go similarity index 65% rename from pkg/sfu/streamallocator.go rename to pkg/sfu/streamallocator/streamallocator.go index bf5ba725f..329b3f2c2 100644 --- a/pkg/sfu/streamallocator.go +++ b/pkg/sfu/streamallocator/streamallocator.go @@ -1,4 +1,4 @@ -package sfu +package streamallocator import ( "fmt" @@ -16,6 +16,8 @@ import ( "github.com/livekit/protocol/logger" "github.com/livekit/livekit-server/pkg/config" + "github.com/livekit/livekit-server/pkg/sfu" + "github.com/livekit/livekit-server/pkg/sfu/buffer" ) const ( @@ -223,12 +225,12 @@ type AddTrackParams struct { PublisherID livekit.ParticipantID } -func (s *StreamAllocator) AddTrack(downTrack *DownTrack, params AddTrackParams) { +func (s *StreamAllocator) AddTrack(downTrack *sfu.DownTrack, params AddTrackParams) { if downTrack.Kind() != webrtc.RTPCodecTypeVideo { return } - track := newTrack(downTrack, params.Source, params.IsSimulcast, params.PublisherID, s.params.Logger) + track := NewTrack(downTrack, params.Source, params.IsSimulcast, params.PublisherID, s.params.Logger) track.SetPriority(params.Priority) s.videoTracksMu.Lock() @@ -244,7 +246,7 @@ func (s *StreamAllocator) AddTrack(downTrack *DownTrack, params AddTrackParams) s.maybePostEventAllocateTrack(downTrack) } -func (s *StreamAllocator) RemoveTrack(downTrack *DownTrack) { +func (s *StreamAllocator) RemoveTrack(downTrack *sfu.DownTrack) { s.videoTracksMu.Lock() if existing := s.videoTracks[livekit.TrackID(downTrack.ID())]; existing != nil && existing.DownTrack() == downTrack { delete(s.videoTracks, livekit.TrackID(downTrack.ID())) @@ -257,7 +259,7 @@ func (s *StreamAllocator) RemoveTrack(downTrack *DownTrack) { }) } -func (s *StreamAllocator) SetTrackPriority(downTrack *DownTrack, priority uint8) { +func (s *StreamAllocator) SetTrackPriority(downTrack *sfu.DownTrack, priority uint8) { s.videoTracksMu.Lock() if track := s.videoTracks[livekit.TrackID(downTrack.ID())]; track != nil { changed := track.SetPriority(priority) @@ -280,7 +282,7 @@ func (s *StreamAllocator) resetState() { } // called when a new REMB is received (receive side bandwidth estimation) -func (s *StreamAllocator) OnREMB(downTrack *DownTrack, remb *rtcp.ReceiverEstimatedMaximumBitrate) { +func (s *StreamAllocator) OnREMB(downTrack *sfu.DownTrack, remb *rtcp.ReceiverEstimatedMaximumBitrate) { // // Channel capacity is estimated at a peer connection level. All down tracks // in the peer connection will end up calling this for a REMB report with @@ -360,7 +362,7 @@ func (s *StreamAllocator) OnREMB(downTrack *DownTrack, remb *rtcp.ReceiverEstima } // called when a new transport-cc feedback is received -func (s *StreamAllocator) OnTransportCCFeedback(downTrack *DownTrack, fb *rtcp.TransportLayerCC) { +func (s *StreamAllocator) OnTransportCCFeedback(downTrack *sfu.DownTrack, fb *rtcp.TransportLayerCC) { if s.bwe != nil { s.bwe.WriteRTCP([]rtcp.Packet{fb}, nil) } @@ -375,27 +377,27 @@ func (s *StreamAllocator) onTargetBitrateChange(bitrate int) { } // called when feeding track's layer availability changes -func (s *StreamAllocator) OnAvailableLayersChanged(downTrack *DownTrack) { +func (s *StreamAllocator) OnAvailableLayersChanged(downTrack *sfu.DownTrack) { s.maybePostEventAllocateTrack(downTrack) } // called when feeding track's bitrate measurement of any layer is available -func (s *StreamAllocator) OnBitrateAvailabilityChanged(downTrack *DownTrack) { +func (s *StreamAllocator) OnBitrateAvailabilityChanged(downTrack *sfu.DownTrack) { s.maybePostEventAllocateTrack(downTrack) } // called when feeding track's max publisher layer changes -func (s *StreamAllocator) OnMaxPublishedLayerChanged(downTrack *DownTrack) { +func (s *StreamAllocator) OnMaxPublishedLayerChanged(downTrack *sfu.DownTrack) { s.maybePostEventAllocateTrack(downTrack) } // called when subscription settings changes (muting/unmuting of track) -func (s *StreamAllocator) OnSubscriptionChanged(downTrack *DownTrack) { +func (s *StreamAllocator) OnSubscriptionChanged(downTrack *sfu.DownTrack) { s.maybePostEventAllocateTrack(downTrack) } // called when subscribed layers changes (limiting max layers) -func (s *StreamAllocator) OnSubscribedLayersChanged(downTrack *DownTrack, layers VideoLayers) { +func (s *StreamAllocator) OnSubscribedLayersChanged(downTrack *sfu.DownTrack, layers buffer.VideoLayer) { shouldPost := false s.videoTracksMu.Lock() if track := s.videoTracks[livekit.TrackID(downTrack.ID())]; track != nil { @@ -414,7 +416,7 @@ func (s *StreamAllocator) OnSubscribedLayersChanged(downTrack *DownTrack, layers } // called when forwarder finds a target layer -func (s *StreamAllocator) OnTargetLayerReached(downTrack *DownTrack) { +func (s *StreamAllocator) OnTargetLayerReached(downTrack *sfu.DownTrack) { s.postEvent(Event{ Signal: streamAllocatorSignalTargetLayerFound, TrackID: livekit.TrackID(downTrack.ID()), @@ -422,7 +424,7 @@ func (s *StreamAllocator) OnTargetLayerReached(downTrack *DownTrack) { } // called when a video DownTrack sends a packet -func (s *StreamAllocator) OnPacketsSent(downTrack *DownTrack, size int) { +func (s *StreamAllocator) OnPacketsSent(downTrack *sfu.DownTrack, size int) { s.prober.PacketsSent(size) } @@ -454,7 +456,7 @@ func (s *StreamAllocator) OnActiveChanged(isActive bool) { } } -func (s *StreamAllocator) maybePostEventAllocateTrack(downTrack *DownTrack) { +func (s *StreamAllocator) maybePostEventAllocateTrack(downTrack *sfu.DownTrack) { shouldPost := false s.videoTracksMu.Lock() if track := s.videoTracks[livekit.TrackID(downTrack.ID())]; track != nil { @@ -762,7 +764,7 @@ func (s *StreamAllocator) allocateTrack(track *Track) { if !s.params.Config.Enabled || s.state == streamAllocatorStateStable || !track.IsManaged() { update := NewStreamStateUpdate() allocation := track.AllocateOptimal(FlagAllowOvershootWhileOptimal) - if allocation.pauseReason == VideoPauseReasonBandwidth && track.SetPaused(true) { + if allocation.PauseReason == sfu.VideoPauseReasonBandwidth && track.SetPaused(true) { update.HandleStreamingChange(true, track) } s.maybeSendUpdate(update) @@ -780,16 +782,16 @@ func (s *StreamAllocator) allocateTrack(track *Track) { transition := track.ProvisionalAllocateGetCooperativeTransition(FlagAllowOvershootWhileDeficient) // track is currently streaming at minimum - if transition.bandwidthDelta == 0 { + if transition.BandwidthDelta == 0 { return } // downgrade, giving back bits - if transition.from.GreaterThan(transition.to) { + if transition.From.GreaterThan(transition.To) { allocation := track.ProvisionalAllocateCommit() update := NewStreamStateUpdate() - if allocation.pauseReason == VideoPauseReasonBandwidth && track.SetPaused(true) { + if allocation.PauseReason == sfu.VideoPauseReasonBandwidth && track.SetPaused(true) { update.HandleStreamingChange(true, track) } s.maybeSendUpdate(update) @@ -817,22 +819,22 @@ func (s *StreamAllocator) allocateTrack(track *Track) { for _, t := range minDistanceSorted { tx := t.ProvisionalAllocateGetBestWeightedTransition() - if tx.bandwidthDelta < 0 { + if tx.BandwidthDelta < 0 { contributingTracks = append(contributingTracks, t) - bandwidthAcquired += -tx.bandwidthDelta - if bandwidthAcquired >= transition.bandwidthDelta { + bandwidthAcquired += -tx.BandwidthDelta + if bandwidthAcquired >= transition.BandwidthDelta { break } } } update := NewStreamStateUpdate() - if bandwidthAcquired >= transition.bandwidthDelta { + if bandwidthAcquired >= transition.BandwidthDelta { // commit the tracks that contributed for _, t := range contributingTracks { allocation := t.ProvisionalAllocateCommit() - if allocation.pauseReason == VideoPauseReasonBandwidth && track.SetPaused(true) { + if allocation.PauseReason == sfu.VideoPauseReasonBandwidth && track.SetPaused(true) { update.HandleStreamingChange(true, t) } } @@ -841,9 +843,9 @@ func (s *StreamAllocator) allocateTrack(track *Track) { } // commit the track that needs change if enough could be acquired or pause not allowed - if !s.params.Config.AllowPause || bandwidthAcquired >= transition.bandwidthDelta { + if !s.params.Config.AllowPause || bandwidthAcquired >= transition.BandwidthDelta { allocation := track.ProvisionalAllocateCommit() - if allocation.pauseReason == VideoPauseReasonBandwidth && track.SetPaused(true) { + if allocation.PauseReason == sfu.VideoPauseReasonBandwidth && track.SetPaused(true) { update.HandleStreamingChange(true, track) } } @@ -915,11 +917,11 @@ func (s *StreamAllocator) maybeBoostDeficientTracks() { continue } - if allocation.pauseReason == VideoPauseReasonBandwidth && track.SetPaused(true) { + if allocation.PauseReason == sfu.VideoPauseReasonBandwidth && track.SetPaused(true) { update.HandleStreamingChange(true, track) } - availableChannelCapacity -= allocation.bandwidthDelta + availableChannelCapacity -= allocation.BandwidthDelta if availableChannelCapacity <= 0 { break } @@ -971,12 +973,12 @@ func (s *StreamAllocator) allocateAllTracks() { } allocation := track.AllocateOptimal(FlagAllowOvershootExemptTrackWhileDeficient) - if allocation.pauseReason == VideoPauseReasonBandwidth && track.SetPaused(true) { + if allocation.PauseReason == sfu.VideoPauseReasonBandwidth && track.SetPaused(true) { update.HandleStreamingChange(true, track) } // LK-TODO: optimistic allocation before bitrate is available will return 0. How to account for that? - availableChannelCapacity -= allocation.bandwidthRequested + availableChannelCapacity -= allocation.BandwidthRequested } if availableChannelCapacity < 0 { @@ -990,7 +992,7 @@ func (s *StreamAllocator) allocateAllTracks() { } allocation := track.Pause() - if allocation.pauseReason == VideoPauseReasonBandwidth && track.SetPaused(true) { + if allocation.PauseReason == sfu.VideoPauseReasonBandwidth && track.SetPaused(true) { update.HandleStreamingChange(true, track) } } @@ -1000,9 +1002,9 @@ func (s *StreamAllocator) allocateAllTracks() { track.ProvisionalAllocatePrepare() } - for spatial := int32(0); spatial <= DefaultMaxLayerSpatial; spatial++ { - for temporal := int32(0); temporal <= DefaultMaxLayerTemporal; temporal++ { - layers := VideoLayers{ + for spatial := int32(0); spatial <= buffer.DefaultMaxLayerSpatial; spatial++ { + for temporal := int32(0); temporal <= buffer.DefaultMaxLayerTemporal; temporal++ { + layers := buffer.VideoLayer{ Spatial: spatial, Temporal: temporal, } @@ -1019,7 +1021,7 @@ func (s *StreamAllocator) allocateAllTracks() { for _, track := range sorted { allocation := track.ProvisionalAllocateCommit() - if allocation.pauseReason == VideoPauseReasonBandwidth && track.SetPaused(true) { + if allocation.PauseReason == sfu.VideoPauseReasonBandwidth && track.SetPaused(true) { update.HandleStreamingChange(true, track) } } @@ -1172,7 +1174,7 @@ func (s *StreamAllocator) maybeProbeWithMedia() { } update := NewStreamStateUpdate() - if allocation.pauseReason == VideoPauseReasonBandwidth && track.SetPaused(true) { + if allocation.PauseReason == sfu.VideoPauseReasonBandwidth && track.SetPaused(true) { update.HandleStreamingChange(true, track) } s.maybeSendUpdate(update) @@ -1186,11 +1188,11 @@ func (s *StreamAllocator) maybeProbeWithPadding() { // use deficient track farthest from desired layers to find how much to probe for _, track := range s.getMaxDistanceSortedDeficient() { transition, available := track.GetNextHigherTransition(FlagAllowOvershootInProbe) - if !available || transition.bandwidthDelta < 0 { + if !available || transition.BandwidthDelta < 0 { continue } - probeRateBps := (transition.bandwidthDelta * ProbePct) / 100 + probeRateBps := (transition.BandwidthDelta * ProbePct) / 100 if probeRateBps < ProbeMinBps { probeRateBps = ProbeMinBps } @@ -1263,615 +1265,3 @@ func (s *StreamAllocator) getMaxDistanceSortedDeficient() MaxDistanceSorter { } // ------------------------------------------------ - -type StreamState int - -const ( - StreamStateActive StreamState = iota - StreamStatePaused -) - -func (s StreamState) String() string { - switch s { - case StreamStateActive: - return "active" - case StreamStatePaused: - return "paused" - default: - return "unknown" - } -} - -type StreamStateInfo struct { - ParticipantID livekit.ParticipantID - TrackID livekit.TrackID - State StreamState -} - -type StreamStateUpdate struct { - StreamStates []*StreamStateInfo -} - -func NewStreamStateUpdate() *StreamStateUpdate { - return &StreamStateUpdate{} -} - -func (s *StreamStateUpdate) HandleStreamingChange(isPaused bool, track *Track) { - if isPaused { - s.StreamStates = append(s.StreamStates, &StreamStateInfo{ - ParticipantID: track.PublisherID(), - TrackID: track.ID(), - State: StreamStatePaused, - }) - } else { - s.StreamStates = append(s.StreamStates, &StreamStateInfo{ - ParticipantID: track.PublisherID(), - TrackID: track.ID(), - State: StreamStateActive, - }) - } -} - -func (s *StreamStateUpdate) Empty() bool { - return len(s.StreamStates) == 0 -} - -// ------------------------------------------------ - -type Track struct { - downTrack *DownTrack - source livekit.TrackSource - isSimulcast bool - priority uint8 - publisherID livekit.ParticipantID - logger logger.Logger - - maxLayers VideoLayers - - totalPackets uint32 - totalRepeatedNacks uint32 - - isDirty bool - - isPaused bool -} - -func newTrack( - downTrack *DownTrack, - source livekit.TrackSource, - isSimulcast bool, - publisherID livekit.ParticipantID, - logger logger.Logger, -) *Track { - t := &Track{ - downTrack: downTrack, - source: source, - isSimulcast: isSimulcast, - publisherID: publisherID, - logger: logger, - isPaused: true, - } - t.SetPriority(0) - t.SetMaxLayers(downTrack.MaxLayers()) - - return t -} - -func (t *Track) SetDirty(isDirty bool) bool { - if t.isDirty == isDirty { - return false - } - - t.isDirty = isDirty - return true -} - -func (t *Track) SetPaused(isPaused bool) bool { - if t.isPaused == isPaused { - return false - } - - t.isPaused = isPaused - return true -} - -func (t *Track) SetPriority(priority uint8) bool { - if priority == 0 { - switch t.source { - case livekit.TrackSource_SCREEN_SHARE: - priority = PriorityDefaultScreenshare - default: - priority = PriorityDefaultVideo - } - } - - if t.priority == priority { - return false - } - - t.priority = priority - return true -} - -func (t *Track) Priority() uint8 { - return t.priority -} - -func (t *Track) DownTrack() *DownTrack { - return t.downTrack -} - -func (t *Track) IsManaged() bool { - return t.source != livekit.TrackSource_SCREEN_SHARE || t.isSimulcast -} - -func (t *Track) ID() livekit.TrackID { - return livekit.TrackID(t.downTrack.ID()) -} - -func (t *Track) PublisherID() livekit.ParticipantID { - return t.publisherID -} - -func (t *Track) SetMaxLayers(layers VideoLayers) bool { - if t.maxLayers == layers { - return false - } - - t.maxLayers = layers - return true -} - -func (t *Track) WritePaddingRTP(bytesToSend int) int { - return t.downTrack.WritePaddingRTP(bytesToSend, false) -} - -func (t *Track) AllocateOptimal(allowOvershoot bool) VideoAllocation { - return t.downTrack.AllocateOptimal(allowOvershoot) -} - -func (t *Track) ProvisionalAllocatePrepare() { - t.downTrack.ProvisionalAllocatePrepare() -} - -func (t *Track) ProvisionalAllocate(availableChannelCapacity int64, layers VideoLayers, allowPause bool, allowOvershoot bool) int64 { - return t.downTrack.ProvisionalAllocate(availableChannelCapacity, layers, allowPause, allowOvershoot) -} - -func (t *Track) ProvisionalAllocateGetCooperativeTransition(allowOvershoot bool) VideoTransition { - return t.downTrack.ProvisionalAllocateGetCooperativeTransition(allowOvershoot) -} - -func (t *Track) ProvisionalAllocateGetBestWeightedTransition() VideoTransition { - return t.downTrack.ProvisionalAllocateGetBestWeightedTransition() -} - -func (t *Track) ProvisionalAllocateCommit() VideoAllocation { - return t.downTrack.ProvisionalAllocateCommit() -} - -func (t *Track) AllocateNextHigher(availableChannelCapacity int64, allowOvershoot bool) (VideoAllocation, bool) { - return t.downTrack.AllocateNextHigher(availableChannelCapacity, allowOvershoot) -} - -func (t *Track) GetNextHigherTransition(allowOvershoot bool) (VideoTransition, bool) { - return t.downTrack.GetNextHigherTransition(allowOvershoot) -} - -func (t *Track) Pause() VideoAllocation { - return t.downTrack.Pause() -} - -func (t *Track) IsDeficient() bool { - return t.downTrack.IsDeficient() -} - -func (t *Track) BandwidthRequested() int64 { - return t.downTrack.BandwidthRequested() -} - -func (t *Track) DistanceToDesired() float64 { - return t.downTrack.DistanceToDesired() -} - -func (t *Track) GetNackDelta() (uint32, uint32) { - totalPackets, totalRepeatedNacks := t.downTrack.GetNackStats() - - packetDelta := totalPackets - t.totalPackets - t.totalPackets = totalPackets - - nackDelta := totalRepeatedNacks - t.totalRepeatedNacks - t.totalRepeatedNacks = totalRepeatedNacks - - return packetDelta, nackDelta -} - -// ------------------------------------------------ - -type TrackSorter []*Track - -func (t TrackSorter) Len() int { - return len(t) -} - -func (t TrackSorter) Swap(i, j int) { - t[i], t[j] = t[j], t[i] -} - -func (t TrackSorter) Less(i, j int) bool { - // - // TrackSorter is used to allocate layer-by-layer. - // So, higher priority track should come earlier so that it gets an earlier shot at each layer - // - if t[i].priority != t[j].priority { - return t[i].priority > t[j].priority - } - - if t[i].maxLayers.Spatial != t[j].maxLayers.Spatial { - return t[i].maxLayers.Spatial > t[j].maxLayers.Spatial - } - - return t[i].maxLayers.Temporal > t[j].maxLayers.Temporal -} - -// ------------------------------------------------ - -type MaxDistanceSorter []*Track - -func (m MaxDistanceSorter) Len() int { - return len(m) -} - -func (m MaxDistanceSorter) Swap(i, j int) { - m[i], m[j] = m[j], m[i] -} - -func (m MaxDistanceSorter) Less(i, j int) bool { - // - // MaxDistanceSorter is used to find a deficient track to use for probing during recovery from congestion. - // So, higher priority track should come earlier so that they have a chance to recover sooner. - // - if m[i].priority != m[j].priority { - return m[i].priority > m[j].priority - } - - return m[i].DistanceToDesired() > m[j].DistanceToDesired() -} - -// ------------------------------------------------ - -type MinDistanceSorter []*Track - -func (m MinDistanceSorter) Len() int { - return len(m) -} - -func (m MinDistanceSorter) Swap(i, j int) { - m[i], m[j] = m[j], m[i] -} - -func (m MinDistanceSorter) Less(i, j int) bool { - // - // MinDistanceSorter is used to find excess bandwidth in cooperative allocation. - // So, lower priority track should come earlier so that they contribute bandwidth to higher priority tracks. - // - if m[i].priority != m[j].priority { - return m[i].priority < m[j].priority - } - - return m[i].DistanceToDesired() < m[j].DistanceToDesired() -} - -// ------------------------------------------------ - -type ChannelTrend int - -const ( - ChannelTrendNeutral ChannelTrend = iota - ChannelTrendClearing - ChannelTrendCongesting -) - -func (c ChannelTrend) String() string { - switch c { - case ChannelTrendNeutral: - return "NEUTRAL" - case ChannelTrendClearing: - return "CLEARING" - case ChannelTrendCongesting: - return "CONGESTING" - default: - return fmt.Sprintf("%d", int(c)) - } -} - -type ChannelCongestionReason int - -const ( - ChannelCongestionReasonNone ChannelCongestionReason = iota - ChannelCongestionReasonEstimate - ChannelCongestionReasonLoss -) - -func (c ChannelCongestionReason) String() string { - switch c { - case ChannelCongestionReasonNone: - return "NONE" - case ChannelCongestionReasonEstimate: - return "ESTIMATE" - case ChannelCongestionReasonLoss: - return "LOSS" - default: - return fmt.Sprintf("%d", int(c)) - } -} - -type ChannelObserverParams struct { - Name string - EstimateRequiredSamples int - EstimateDownwardTrendThreshold float64 - EstimateCollapseValues bool - NackWindowMinDuration time.Duration - NackWindowMaxDuration time.Duration - NackRatioThreshold float64 -} - -type ChannelObserver struct { - params ChannelObserverParams - logger logger.Logger - - estimateTrend *TrendDetector - - nackWindowStartTime time.Time - packets uint32 - repeatedNacks uint32 -} - -func NewChannelObserver(params ChannelObserverParams, logger logger.Logger) *ChannelObserver { - return &ChannelObserver{ - params: params, - logger: logger, - estimateTrend: NewTrendDetector(TrendDetectorParams{ - Name: params.Name + "-estimate", - Logger: logger, - RequiredSamples: params.EstimateRequiredSamples, - DownwardTrendThreshold: params.EstimateDownwardTrendThreshold, - CollapseValues: params.EstimateCollapseValues, - }), - } -} - -func (c *ChannelObserver) SeedEstimate(estimate int64) { - c.estimateTrend.Seed(estimate) -} - -func (c *ChannelObserver) SeedNack(packets uint32, repeatedNacks uint32) { - c.packets = packets - c.repeatedNacks = repeatedNacks -} - -func (c *ChannelObserver) AddEstimate(estimate int64) { - c.estimateTrend.AddValue(estimate) -} - -func (c *ChannelObserver) AddNack(packets uint32, repeatedNacks uint32) { - if c.params.NackWindowMaxDuration != 0 && !c.nackWindowStartTime.IsZero() && time.Since(c.nackWindowStartTime) > c.params.NackWindowMaxDuration { - c.nackWindowStartTime = time.Time{} - c.packets = 0 - c.repeatedNacks = 0 - } - - // - // Start NACK monitoring window only when a repeated NACK happens. - // This allows locking tightly to when NACKs start happening and - // check if the NACKs keep adding up (potentially a sign of congestion) - // or isolated losses - // - if c.repeatedNacks == 0 && repeatedNacks != 0 { - c.nackWindowStartTime = time.Now() - } - - if !c.nackWindowStartTime.IsZero() { - c.packets += packets - c.repeatedNacks += repeatedNacks - } -} - -func (c *ChannelObserver) GetLowestEstimate() int64 { - return c.estimateTrend.GetLowest() -} - -func (c *ChannelObserver) GetHighestEstimate() int64 { - return c.estimateTrend.GetHighest() -} - -func (c *ChannelObserver) GetNackRatio() (uint32, uint32, float64) { - ratio := 0.0 - if c.packets != 0 { - ratio = float64(c.repeatedNacks) / float64(c.packets) - if ratio > 1.0 { - ratio = 1.0 - } - } - - return c.packets, c.repeatedNacks, ratio -} - -func (c *ChannelObserver) GetTrend() (ChannelTrend, ChannelCongestionReason) { - estimateDirection := c.estimateTrend.GetDirection() - packets, repeatedNacks, nackRatio := c.GetNackRatio() - - switch { - case estimateDirection == TrendDirectionDownward: - c.logger.Debugw( - "stream allocator: channel observer: estimate is trending downward", - "name", c.params.Name, - "estimate", c.estimateTrend.ToString(), - "packets", packets, - "repeatedNacks", repeatedNacks, - "ratio", nackRatio, - ) - return ChannelTrendCongesting, ChannelCongestionReasonEstimate - case c.params.NackWindowMinDuration != 0 && !c.nackWindowStartTime.IsZero() && time.Since(c.nackWindowStartTime) > c.params.NackWindowMinDuration && nackRatio > c.params.NackRatioThreshold: - c.logger.Debugw( - "stream allocator: channel observer: high rate of repeated NACKs", - "name", c.params.Name, - "estimate", c.estimateTrend.ToString(), - "packets", packets, - "repeatedNacks", repeatedNacks, - "ratio", nackRatio, - ) - return ChannelTrendCongesting, ChannelCongestionReasonLoss - case estimateDirection == TrendDirectionUpward: - return ChannelTrendClearing, ChannelCongestionReasonNone - } - - return ChannelTrendNeutral, ChannelCongestionReasonNone -} - -// ------------------------------------------------ - -type TrendDirection int - -const ( - TrendDirectionNeutral TrendDirection = iota - TrendDirectionUpward - TrendDirectionDownward -) - -func (t TrendDirection) String() string { - switch t { - case TrendDirectionNeutral: - return "NEUTRAL" - case TrendDirectionUpward: - return "UPWARD" - case TrendDirectionDownward: - return "DOWNWARD" - default: - return fmt.Sprintf("%d", int(t)) - } -} - -type TrendDetectorParams struct { - Name string - Logger logger.Logger - RequiredSamples int - DownwardTrendThreshold float64 - CollapseValues bool -} - -type TrendDetector struct { - params TrendDetectorParams - - startTime time.Time - numSamples int - values []int64 - lowestValue int64 - highestValue int64 - - direction TrendDirection -} - -func NewTrendDetector(params TrendDetectorParams) *TrendDetector { - return &TrendDetector{ - params: params, - startTime: time.Now(), - direction: TrendDirectionNeutral, - } -} - -func (t *TrendDetector) Seed(value int64) { - if len(t.values) != 0 { - return - } - - t.values = append(t.values, value) -} - -func (t *TrendDetector) AddValue(value int64) { - t.numSamples++ - if t.lowestValue == 0 || value < t.lowestValue { - t.lowestValue = value - } - if value > t.highestValue { - t.highestValue = value - } - - // ignore duplicate values - if t.params.CollapseValues && len(t.values) != 0 && t.values[len(t.values)-1] == value { - return - } - - if len(t.values) == t.params.RequiredSamples { - t.values = t.values[1:] - } - t.values = append(t.values, value) - - t.updateDirection() -} - -func (t *TrendDetector) GetLowest() int64 { - return t.lowestValue -} - -func (t *TrendDetector) GetHighest() int64 { - return t.highestValue -} - -func (t *TrendDetector) GetValues() []int64 { - return t.values -} - -func (t *TrendDetector) GetDirection() TrendDirection { - return t.direction -} - -func (t *TrendDetector) ToString() string { - now := time.Now() - elapsed := now.Sub(t.startTime).Seconds() - str := fmt.Sprintf("n: %s", t.params.Name) - str += fmt.Sprintf(", t: %+v|%+v|%.2fs", t.startTime.Format(time.UnixDate), now.Format(time.UnixDate), elapsed) - str += fmt.Sprintf(", v: %d|%d|%d|%+v|%.2f", t.numSamples, t.lowestValue, t.highestValue, t.values, kendallsTau(t.values)) - return str -} - -func (t *TrendDetector) updateDirection() { - if len(t.values) < t.params.RequiredSamples { - t.direction = TrendDirectionNeutral - return - } - - // using Kendall's Tau to find trend - kt := kendallsTau(t.values) - - t.direction = TrendDirectionNeutral - switch { - case kt > 0: - t.direction = TrendDirectionUpward - case kt < t.params.DownwardTrendThreshold: - t.direction = TrendDirectionDownward - } -} - -// ------------------------------------------------ - -func kendallsTau(values []int64) float64 { - concordantPairs := 0 - discordantPairs := 0 - - for i := 0; i < len(values)-1; i++ { - for j := i + 1; j < len(values); j++ { - if values[i] < values[j] { - concordantPairs++ - } else if values[i] > values[j] { - discordantPairs++ - } - } - } - - if (concordantPairs + discordantPairs) == 0 { - return 0.0 - } - - return (float64(concordantPairs) - float64(discordantPairs)) / (float64(concordantPairs) + float64(discordantPairs)) -} diff --git a/pkg/sfu/streamallocator/streamstateupdate.go b/pkg/sfu/streamallocator/streamstateupdate.go new file mode 100644 index 000000000..e5b3156f7 --- /dev/null +++ b/pkg/sfu/streamallocator/streamstateupdate.go @@ -0,0 +1,63 @@ +package streamallocator + +import ( + "github.com/livekit/protocol/livekit" +) + +// ------------------------------------------------ + +type StreamState int + +const ( + StreamStateActive StreamState = iota + StreamStatePaused +) + +func (s StreamState) String() string { + switch s { + case StreamStateActive: + return "active" + case StreamStatePaused: + return "paused" + default: + return "unknown" + } +} + +// ------------------------------------------------ + +type StreamStateInfo struct { + ParticipantID livekit.ParticipantID + TrackID livekit.TrackID + State StreamState +} + +type StreamStateUpdate struct { + StreamStates []*StreamStateInfo +} + +func NewStreamStateUpdate() *StreamStateUpdate { + return &StreamStateUpdate{} +} + +func (s *StreamStateUpdate) HandleStreamingChange(isPaused bool, track *Track) { + if isPaused { + s.StreamStates = append(s.StreamStates, &StreamStateInfo{ + ParticipantID: track.PublisherID(), + TrackID: track.ID(), + State: StreamStatePaused, + }) + } else { + s.StreamStates = append(s.StreamStates, &StreamStateInfo{ + ParticipantID: track.PublisherID(), + TrackID: track.ID(), + State: StreamStateActive, + }) + } +} + +func (s *StreamStateUpdate) Empty() bool { + return len(s.StreamStates) == 0 +} + +// ------------------------------------------------ diff --git a/pkg/sfu/streamallocator/track.go b/pkg/sfu/streamallocator/track.go new file mode 100644 index 000000000..70df54990 --- /dev/null +++ b/pkg/sfu/streamallocator/track.go @@ -0,0 +1,255 @@ +package streamallocator + +import ( + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" + + "github.com/livekit/livekit-server/pkg/sfu" + "github.com/livekit/livekit-server/pkg/sfu/buffer" +) + +type Track struct { + downTrack *sfu.DownTrack + source livekit.TrackSource + isSimulcast bool + priority uint8 + publisherID livekit.ParticipantID + logger logger.Logger + + maxLayers buffer.VideoLayer + + totalPackets uint32 + totalRepeatedNacks uint32 + + isDirty bool + + isPaused bool +} + +func NewTrack( + downTrack *sfu.DownTrack, + source livekit.TrackSource, + isSimulcast bool, + publisherID livekit.ParticipantID, + logger logger.Logger, +) *Track { + t := &Track{ + downTrack: downTrack, + source: source, + isSimulcast: isSimulcast, + publisherID: publisherID, + logger: logger, + isPaused: true, + } + t.SetPriority(0) + t.SetMaxLayers(downTrack.MaxLayers()) + + return t +} + +func (t *Track) SetDirty(isDirty bool) bool { + if t.isDirty == isDirty { + return false + } + + t.isDirty = isDirty + return true +} + +func (t *Track) SetPaused(isPaused bool) bool { + if t.isPaused == isPaused { + return false + } + + t.isPaused = isPaused + return true +} + +func (t *Track) SetPriority(priority uint8) bool { + if priority == 0 { + switch t.source { + case livekit.TrackSource_SCREEN_SHARE: + priority = PriorityDefaultScreenshare + default: + priority = PriorityDefaultVideo + } + } + + if t.priority == priority { + return false + } + + t.priority = priority + return true +} + +func (t *Track) Priority() uint8 { + return t.priority +} + +func (t *Track) DownTrack() *sfu.DownTrack { + return t.downTrack +} + +func (t *Track) IsManaged() bool { + return t.source != livekit.TrackSource_SCREEN_SHARE || t.isSimulcast +} + +func (t *Track) ID() livekit.TrackID { + return livekit.TrackID(t.downTrack.ID()) +} + +func (t *Track) PublisherID() livekit.ParticipantID { + return t.publisherID +} + +func (t *Track) SetMaxLayers(layers buffer.VideoLayer) bool { + if t.maxLayers == layers { + return false + } + + t.maxLayers = layers + return true +} + +func (t *Track) WritePaddingRTP(bytesToSend int) int { + return t.downTrack.WritePaddingRTP(bytesToSend, false) +} + +func (t *Track) AllocateOptimal(allowOvershoot bool) sfu.VideoAllocation { + return t.downTrack.AllocateOptimal(allowOvershoot) +} + +func (t *Track) ProvisionalAllocatePrepare() { + t.downTrack.ProvisionalAllocatePrepare() +} + +func (t *Track) ProvisionalAllocate(availableChannelCapacity int64, layers buffer.VideoLayer, allowPause bool, allowOvershoot bool) int64 { + return t.downTrack.ProvisionalAllocate(availableChannelCapacity, layers, allowPause, allowOvershoot) +} + +func (t *Track) ProvisionalAllocateGetCooperativeTransition(allowOvershoot bool) sfu.VideoTransition { + return t.downTrack.ProvisionalAllocateGetCooperativeTransition(allowOvershoot) +} + +func (t *Track) ProvisionalAllocateGetBestWeightedTransition() sfu.VideoTransition { + return t.downTrack.ProvisionalAllocateGetBestWeightedTransition() +} + +func (t *Track) ProvisionalAllocateCommit() sfu.VideoAllocation { + return t.downTrack.ProvisionalAllocateCommit() +} + +func (t *Track) AllocateNextHigher(availableChannelCapacity int64, allowOvershoot bool) (sfu.VideoAllocation, bool) { + return t.downTrack.AllocateNextHigher(availableChannelCapacity, allowOvershoot) +} + +func (t *Track) GetNextHigherTransition(allowOvershoot bool) (sfu.VideoTransition, bool) { + return t.downTrack.GetNextHigherTransition(allowOvershoot) +} + +func (t *Track) Pause() sfu.VideoAllocation { + return t.downTrack.Pause() +} + +func (t *Track) IsDeficient() bool { + return t.downTrack.IsDeficient() +} + +func (t *Track) BandwidthRequested() int64 { + return t.downTrack.BandwidthRequested() +} + +func (t *Track) DistanceToDesired() float64 { + return t.downTrack.DistanceToDesired() +} + +func (t *Track) GetNackDelta() (uint32, uint32) { + totalPackets, totalRepeatedNacks := t.downTrack.GetNackStats() + + packetDelta := totalPackets - t.totalPackets + t.totalPackets = totalPackets + + nackDelta := totalRepeatedNacks - t.totalRepeatedNacks + t.totalRepeatedNacks = totalRepeatedNacks + + return packetDelta, nackDelta +} + +// ------------------------------------------------ + +type TrackSorter []*Track + +func (t TrackSorter) Len() int { + return len(t) +} + +func (t TrackSorter) Swap(i, j int) { + t[i], t[j] = t[j], t[i] +} + +func (t TrackSorter) Less(i, j int) bool { + // + // TrackSorter is used to allocate layer-by-layer. + // So, higher priority track should come earlier so that it gets an earlier shot at each layer + // + if t[i].priority != t[j].priority { + return t[i].priority > t[j].priority + } + + if t[i].maxLayers.Spatial != t[j].maxLayers.Spatial { + return t[i].maxLayers.Spatial > t[j].maxLayers.Spatial + } + + return t[i].maxLayers.Temporal > t[j].maxLayers.Temporal +} + +// ------------------------------------------------ + +type MaxDistanceSorter []*Track + +func (m MaxDistanceSorter) Len() int { + return len(m) +} + +func (m MaxDistanceSorter) Swap(i, j int) { + m[i], m[j] = m[j], m[i] +} + +func (m MaxDistanceSorter) Less(i, j int) bool { + // + // MaxDistanceSorter is used to find a deficient track to use for probing during recovery from congestion. + // So, higher priority track should come earlier so that they have a chance to recover sooner. + // + if m[i].priority != m[j].priority { + return m[i].priority > m[j].priority + } + + return m[i].DistanceToDesired() > m[j].DistanceToDesired() +} + +// ------------------------------------------------ + +type MinDistanceSorter []*Track + +func (m MinDistanceSorter) Len() int { + return len(m) +} + +func (m MinDistanceSorter) Swap(i, j int) { + m[i], m[j] = m[j], m[i] +} + +func (m MinDistanceSorter) Less(i, j int) bool { + // + // MinDistanceSorter is used to find excess bandwidth in cooperative allocation. + // So, lower priority track should come earlier so that they contribute bandwidth to higher priority tracks. + // + if m[i].priority != m[j].priority { + return m[i].priority < m[j].priority + } + + return m[i].DistanceToDesired() < m[j].DistanceToDesired() +} + +// ------------------------------------------------ diff --git a/pkg/sfu/streamallocator/trenddetector.go b/pkg/sfu/streamallocator/trenddetector.go new file mode 100644 index 000000000..959b7d9de --- /dev/null +++ b/pkg/sfu/streamallocator/trenddetector.go @@ -0,0 +1,157 @@ +package streamallocator + +import ( + "fmt" + "time" + + "github.com/livekit/protocol/logger" +) + +// ------------------------------------------------ + +type TrendDirection int + +const ( + TrendDirectionNeutral TrendDirection = iota + TrendDirectionUpward + TrendDirectionDownward +) + +func (t TrendDirection) String() string { + switch t { + case TrendDirectionNeutral: + return "NEUTRAL" + case TrendDirectionUpward: + return "UPWARD" + case TrendDirectionDownward: + return "DOWNWARD" + default: + return fmt.Sprintf("%d", int(t)) + } +} + +// ------------------------------------------------ + +type TrendDetectorParams struct { + Name string + Logger logger.Logger + RequiredSamples int + DownwardTrendThreshold float64 + CollapseValues bool +} + +type TrendDetector struct { + params TrendDetectorParams + + startTime time.Time + numSamples int + values []int64 + lowestValue int64 + highestValue int64 + + direction TrendDirection +} + +func NewTrendDetector(params TrendDetectorParams) *TrendDetector { + return &TrendDetector{ + params: params, + startTime: time.Now(), + direction: TrendDirectionNeutral, + } +} + +func (t *TrendDetector) Seed(value int64) { + if len(t.values) != 0 { + return + } + + t.values = append(t.values, value) +} + +func (t *TrendDetector) AddValue(value int64) { + t.numSamples++ + if t.lowestValue == 0 || value < t.lowestValue { + t.lowestValue = value + } + if value > t.highestValue { + t.highestValue = value + } + + // ignore duplicate values + if t.params.CollapseValues && len(t.values) != 0 && t.values[len(t.values)-1] == value { + return + } + + if len(t.values) == t.params.RequiredSamples { + t.values = t.values[1:] + } + t.values = append(t.values, value) + + t.updateDirection() +} + +func (t *TrendDetector) GetLowest() int64 { + return t.lowestValue +} + +func (t *TrendDetector) GetHighest() int64 { + return t.highestValue +} + +func (t *TrendDetector) GetValues() []int64 { + return t.values +} + +func (t *TrendDetector) GetDirection() TrendDirection { + return t.direction +} + +func (t *TrendDetector) ToString() string { + now := time.Now() + elapsed := now.Sub(t.startTime).Seconds() + str := fmt.Sprintf("n: %s", t.params.Name) + str += fmt.Sprintf(", t: %+v|%+v|%.2fs", t.startTime.Format(time.UnixDate), now.Format(time.UnixDate), elapsed) + str += fmt.Sprintf(", v: %d|%d|%d|%+v|%.2f", t.numSamples, t.lowestValue, t.highestValue, t.values, kendallsTau(t.values)) + return str +} + +func (t *TrendDetector) updateDirection() { + if len(t.values) < t.params.RequiredSamples { + t.direction = TrendDirectionNeutral + return + } + + // using Kendall's Tau to find trend + kt := kendallsTau(t.values) + + t.direction = TrendDirectionNeutral + switch { + case kt > 0: + t.direction = TrendDirectionUpward + case kt < t.params.DownwardTrendThreshold: + t.direction = TrendDirectionDownward + } +} + +// ------------------------------------------------ + +func kendallsTau(values []int64) float64 { + concordantPairs := 0 + discordantPairs := 0 + + for i := 0; i < len(values)-1; i++ { + for j := i + 1; j < len(values); j++ { + if values[i] < values[j] { + concordantPairs++ + } else if values[i] > values[j] { + discordantPairs++ + } + } + } + + if (concordantPairs + discordantPairs) == 0 { + return 0.0 + } + + return (float64(concordantPairs) - float64(discordantPairs)) / (float64(concordantPairs) + float64(discordantPairs)) +} diff --git a/pkg/sfu/streamtrackermanager.go b/pkg/sfu/streamtrackermanager.go index 62c4f82e4..abd4a9554 100644 --- a/pkg/sfu/streamtrackermanager.go +++ b/pkg/sfu/streamtrackermanager.go @@ -36,14 +36,14 @@ type StreamTrackerManager struct { maxPublishedLayer int32 maxTemporalLayerSeen int32 - trackers [DefaultMaxLayerSpatial + 1]*streamtracker.StreamTracker + trackers [buffer.DefaultMaxLayerSpatial + 1]*streamtracker.StreamTracker availableLayers []int32 maxExpectedLayer int32 paused bool senderReportMu sync.RWMutex - senderReports [DefaultMaxLayerSpatial + 1]*buffer.RTCPSenderReportDataExt + senderReports [buffer.DefaultMaxLayerSpatial + 1]*buffer.RTCPSenderReportDataExt closed core.Fuse @@ -61,8 +61,8 @@ func NewStreamTrackerManager( logger: logger, trackInfo: trackInfo, isSVC: isSVC, - maxPublishedLayer: InvalidLayerSpatial, - maxTemporalLayerSeen: InvalidLayerTemporal, + maxPublishedLayer: buffer.InvalidLayerSpatial, + maxTemporalLayerSeen: buffer.InvalidLayerTemporal, clockRate: clockRate, closed: core.NewFuse(), } @@ -295,12 +295,12 @@ func (s *StreamTrackerManager) DistanceToDesired() float64 { al, brs := s.getLayeredBitrateLocked() - maxLayers := InvalidLayers + maxLayers := buffer.InvalidLayers done: for s := int32(len(brs)) - 1; s >= 0; s-- { for t := int32(len(brs[0])) - 1; t >= 0; t-- { if brs[s][t] != 0 { - maxLayers = VideoLayers{ + maxLayers = buffer.VideoLayer{ Spatial: s, Temporal: t, } @@ -319,7 +319,7 @@ done: adjustedMaxLayers := maxLayers if !maxLayers.IsValid() { - adjustedMaxLayers = VideoLayers{Spatial: 0, Temporal: 0} + adjustedMaxLayers = buffer.VideoLayer{Spatial: 0, Temporal: 0} } distance := @@ -351,7 +351,7 @@ func (s *StreamTrackerManager) getLayeredBitrateLocked() ([]int32, Bitrates) { for i, tracker := range s.trackers { if tracker != nil { - tls := make([]int64, DefaultMaxLayerTemporal+1) + tls := make([]int64, buffer.DefaultMaxLayerTemporal+1) if s.hasSpatialLayerLocked(int32(i)) { tls = tracker.BitrateTemporalCumulative() } @@ -428,12 +428,12 @@ func (s *StreamTrackerManager) addAvailableLayer(layer int32) { func (s *StreamTrackerManager) removeAvailableLayer(layer int32) { s.lock.Lock() - prevMaxLayer := InvalidLayerSpatial + prevMaxLayer := buffer.InvalidLayerSpatial if len(s.availableLayers) > 0 { prevMaxLayer = s.availableLayers[len(s.availableLayers)-1] } - newLayers := make([]int32, 0, DefaultMaxLayerSpatial+1) + newLayers := make([]int32, 0, buffer.DefaultMaxLayerSpatial+1) for _, l := range s.availableLayers { if l != layer { newLayers = append(newLayers, l) @@ -448,7 +448,7 @@ func (s *StreamTrackerManager) removeAvailableLayer(layer int32) { "availableLayers", newLayers, ) - curMaxLayer := InvalidLayerSpatial + curMaxLayer := buffer.InvalidLayerSpatial if len(s.availableLayers) > 0 { curMaxLayer = s.availableLayers[len(s.availableLayers)-1] } @@ -466,7 +466,7 @@ func (s *StreamTrackerManager) removeAvailableLayer(layer int32) { } func (s *StreamTrackerManager) maxExpectedLayerFromTrackInfo() { - s.maxExpectedLayer = InvalidLayerSpatial + s.maxExpectedLayer = buffer.InvalidLayerSpatial for _, layer := range s.trackInfo.Layers { spatialLayer := buffer.VideoQualityToSpatialLayer(layer.Quality, s.trackInfo) if spatialLayer > s.maxExpectedLayer { @@ -547,7 +547,7 @@ func (s *StreamTrackerManager) GetMaxTemporalLayerSeen() int32 { } func (s *StreamTrackerManager) updateMaxTemporalLayerSeen(brs Bitrates) { - maxTemporalLayerSeen := InvalidLayerTemporal + maxTemporalLayerSeen := buffer.InvalidLayerTemporal done: for t := int32(len(brs[0])) - 1; t >= 0; t-- { for s := int32(len(brs)) - 1; s >= 0; s-- { diff --git a/pkg/sfu/videolayerselector.go b/pkg/sfu/videolayerselector.go index ec08065ac..1368bc870 100644 --- a/pkg/sfu/videolayerselector.go +++ b/pkg/sfu/videolayerselector.go @@ -11,7 +11,7 @@ import ( type targetLayer struct { Target int - Layer VideoLayers + Layer buffer.VideoLayer } type DDVideoLayerSelector struct { @@ -22,7 +22,7 @@ type DDVideoLayerSelector struct { // expectKeyFrame bool decodeTargetLayer []targetLayer - layer VideoLayers + layer buffer.VideoLayer activeDecodeTargetsBitmask *uint32 structure *dd.FrameDependencyStructure } @@ -30,7 +30,7 @@ type DDVideoLayerSelector struct { func NewDDVideoLayerSelector(logger logger.Logger) *DDVideoLayerSelector { return &DDVideoLayerSelector{ logger: logger, - layer: VideoLayers{Spatial: 2, Temporal: 2}, + layer: buffer.VideoLayer{Spatial: 2, Temporal: 2}, } } @@ -48,7 +48,7 @@ func (s *DDVideoLayerSelector) Select(expPkt *buffer.ExtPacket, tp *TranslationP } // forward all packets before locking - if s.layer == InvalidLayers { + if s.layer == buffer.InvalidLayers { return true } @@ -122,8 +122,8 @@ func (s *DDVideoLayerSelector) Select(expPkt *buffer.ExtPacket, tp *TranslationP return true } -func (s *DDVideoLayerSelector) SelectLayer(layer VideoLayers) { - // layer = VideoLayers{1, 1} +func (s *DDVideoLayerSelector) SelectLayer(layer buffer.VideoLayer) { + // layer = buffer.VideoLayer{1, 1} s.layer = layer activeBitMask := uint32(0) var maxSpatial, maxTemporal int32 @@ -152,7 +152,7 @@ func (s *DDVideoLayerSelector) updateDependencyStructure(structure *dd.FrameDepe s.decodeTargetLayer = s.decodeTargetLayer[:0] for target := 0; target < structure.NumDecodeTargets; target++ { - layer := VideoLayers{Spatial: 0, Temporal: 0} + layer := buffer.VideoLayer{Spatial: 0, Temporal: 0} for _, t := range structure.Templates { if t.DecodeTargetIndications[target] != dd.DecodeTargetNotPresent { if layer.Spatial < int32(t.SpatialId) { From 96607a170a7abb08ece2064a626c9f6573a5d1c5 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 26 Mar 2023 17:18:47 -0700 Subject: [PATCH 036/324] Update module github.com/urfave/cli/v2 to v2.25.1 (#1553) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 732d17b2b..4db549ab4 100644 --- a/go.mod +++ b/go.mod @@ -44,7 +44,7 @@ require ( github.com/thoas/go-funk v0.9.3 github.com/twitchtv/twirp v8.1.3+incompatible github.com/ua-parser/uap-go v0.0.0-20211112212520-00c877edfe0f - github.com/urfave/cli/v2 v2.25.0 + github.com/urfave/cli/v2 v2.25.1 github.com/urfave/negroni/v3 v3.0.0 go.uber.org/atomic v1.10.0 go.uber.org/zap v1.24.0 diff --git a/go.sum b/go.sum index 0796fe2db..bbb34e0e5 100644 --- a/go.sum +++ b/go.sum @@ -401,8 +401,8 @@ github.com/twitchtv/twirp v8.1.3+incompatible h1:+F4TdErPgSUbMZMwp13Q/KgDVuI7HJX github.com/twitchtv/twirp v8.1.3+incompatible/go.mod h1:RRJoFSAmTEh2weEqWtpPE3vFK5YBhA6bqp2l1kfCC5A= github.com/ua-parser/uap-go v0.0.0-20211112212520-00c877edfe0f h1:A+MmlgpvrHLeUP8dkBVn4Pnf5Bp5Yk2OALm7SEJLLE8= github.com/ua-parser/uap-go v0.0.0-20211112212520-00c877edfe0f/go.mod h1:OBcG9bn7sHtXgarhUEb3OfCnNsgtGnkVf41ilSZ3K3E= -github.com/urfave/cli/v2 v2.25.0 h1:ykdZKuQey2zq0yin/l7JOm9Mh+pg72ngYMeB0ABn6q8= -github.com/urfave/cli/v2 v2.25.0/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= +github.com/urfave/cli/v2 v2.25.1 h1:zw8dSP7ghX0Gmm8vugrs6q9Ku0wzweqPyshy+syu9Gw= +github.com/urfave/cli/v2 v2.25.1/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= github.com/urfave/negroni/v3 v3.0.0 h1:Vo8CeZfu1lFR9gW8GnAb6dOGCJyijfil9j/jKKc/JhU= github.com/urfave/negroni/v3 v3.0.0/go.mod h1:jWvnX03kcSjDBl/ShB0iHvx5uOs7mAzZXW+JvJ5XYAs= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= From 108b251045ad820a47511575e857c03caa98e545 Mon Sep 17 00:00:00 2001 From: David Colburn Date: Mon, 27 Mar 2023 16:34:44 -0700 Subject: [PATCH 037/324] egress updated webhook (#1555) --- pkg/service/ioinfo.go | 26 ++++-------- pkg/telemetry/events.go | 9 ++++ .../telemetryfakes/fake_telemetry_service.go | 41 +++++++++++++++++++ pkg/telemetry/telemetryservice.go | 1 + 4 files changed, 60 insertions(+), 17 deletions(-) diff --git a/pkg/service/ioinfo.go b/pkg/service/ioinfo.go index cc0aa404b..e8fc77f9c 100644 --- a/pkg/service/ioinfo.go +++ b/pkg/service/ioinfo.go @@ -3,7 +3,6 @@ package service import ( "context" "errors" - "time" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/emptypb" @@ -68,22 +67,17 @@ func (s *IOInfoService) Start() error { } func (s *IOInfoService) UpdateEgressInfo(ctx context.Context, info *livekit.EgressInfo) (*emptypb.Empty, error) { + err := s.es.UpdateEgress(ctx, info) + switch info.Status { + case livekit.EgressStatus_EGRESS_ACTIVE: + s.telemetry.EgressUpdated(ctx, info) + case livekit.EgressStatus_EGRESS_COMPLETE, livekit.EgressStatus_EGRESS_FAILED, livekit.EgressStatus_EGRESS_ABORTED, livekit.EgressStatus_EGRESS_LIMIT_REACHED: - // make sure endedAt is set so it eventually gets deleted - if info.EndedAt == 0 { - info.EndedAt = time.Now().UnixNano() - } - - if err := s.es.UpdateEgress(ctx, info); err != nil { - logger.Errorw("could not update egress", err) - return nil, err - } - // log results if info.Error != "" { logger.Errorw("egress failed", errors.New(info.Error), "egressID", info.EgressId) @@ -92,12 +86,10 @@ func (s *IOInfoService) UpdateEgressInfo(ctx context.Context, info *livekit.Egre } s.telemetry.EgressEnded(ctx, info) - - default: - if err := s.es.UpdateEgress(ctx, info); err != nil { - logger.Errorw("could not update egress", err) - return nil, err - } + } + if err != nil { + logger.Errorw("could not update egress", err) + return nil, err } return &emptypb.Empty{}, nil diff --git a/pkg/telemetry/events.go b/pkg/telemetry/events.go index 1c78a5486..75f7c8ce2 100644 --- a/pkg/telemetry/events.go +++ b/pkg/telemetry/events.go @@ -411,6 +411,15 @@ func (t *telemetryService) EgressStarted(ctx context.Context, info *livekit.Egre }) } +func (t *telemetryService) EgressUpdated(ctx context.Context, info *livekit.EgressInfo) { + t.enqueue(func() { + t.NotifyEvent(ctx, &livekit.WebhookEvent{ + Event: webhook.EventEgressUpdated, + EgressInfo: info, + }) + }) +} + func (t *telemetryService) EgressEnded(ctx context.Context, info *livekit.EgressInfo) { t.enqueue(func() { t.NotifyEvent(ctx, &livekit.WebhookEvent{ diff --git a/pkg/telemetry/telemetryfakes/fake_telemetry_service.go b/pkg/telemetry/telemetryfakes/fake_telemetry_service.go index 4e6cff95e..923916164 100644 --- a/pkg/telemetry/telemetryfakes/fake_telemetry_service.go +++ b/pkg/telemetry/telemetryfakes/fake_telemetry_service.go @@ -22,6 +22,12 @@ type FakeTelemetryService struct { arg1 context.Context arg2 *livekit.EgressInfo } + EgressUpdatedStub func(context.Context, *livekit.EgressInfo) + egressUpdatedMutex sync.RWMutex + egressUpdatedArgsForCall []struct { + arg1 context.Context + arg2 *livekit.EgressInfo + } FlushStatsStub func() flushStatsMutex sync.RWMutex flushStatsArgsForCall []struct { @@ -274,6 +280,39 @@ func (fake *FakeTelemetryService) EgressStartedArgsForCall(i int) (context.Conte return argsForCall.arg1, argsForCall.arg2 } +func (fake *FakeTelemetryService) EgressUpdated(arg1 context.Context, arg2 *livekit.EgressInfo) { + fake.egressUpdatedMutex.Lock() + fake.egressUpdatedArgsForCall = append(fake.egressUpdatedArgsForCall, struct { + arg1 context.Context + arg2 *livekit.EgressInfo + }{arg1, arg2}) + stub := fake.EgressUpdatedStub + fake.recordInvocation("EgressUpdated", []interface{}{arg1, arg2}) + fake.egressUpdatedMutex.Unlock() + if stub != nil { + fake.EgressUpdatedStub(arg1, arg2) + } +} + +func (fake *FakeTelemetryService) EgressUpdatedCallCount() int { + fake.egressUpdatedMutex.RLock() + defer fake.egressUpdatedMutex.RUnlock() + return len(fake.egressUpdatedArgsForCall) +} + +func (fake *FakeTelemetryService) EgressUpdatedCalls(stub func(context.Context, *livekit.EgressInfo)) { + fake.egressUpdatedMutex.Lock() + defer fake.egressUpdatedMutex.Unlock() + fake.EgressUpdatedStub = stub +} + +func (fake *FakeTelemetryService) EgressUpdatedArgsForCall(i int) (context.Context, *livekit.EgressInfo) { + fake.egressUpdatedMutex.RLock() + defer fake.egressUpdatedMutex.RUnlock() + argsForCall := fake.egressUpdatedArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + func (fake *FakeTelemetryService) FlushStats() { fake.flushStatsMutex.Lock() fake.flushStatsArgsForCall = append(fake.flushStatsArgsForCall, struct { @@ -1109,6 +1148,8 @@ func (fake *FakeTelemetryService) Invocations() map[string][][]interface{} { defer fake.egressEndedMutex.RUnlock() fake.egressStartedMutex.RLock() defer fake.egressStartedMutex.RUnlock() + fake.egressUpdatedMutex.RLock() + defer fake.egressUpdatedMutex.RUnlock() fake.flushStatsMutex.RLock() defer fake.flushStatsMutex.RUnlock() fake.notifyEventMutex.RLock() diff --git a/pkg/telemetry/telemetryservice.go b/pkg/telemetry/telemetryservice.go index bcef252ae..d78664bf6 100644 --- a/pkg/telemetry/telemetryservice.go +++ b/pkg/telemetry/telemetryservice.go @@ -54,6 +54,7 @@ type TelemetryService interface { TrackPublishRTPStats(ctx context.Context, participantID livekit.ParticipantID, trackID livekit.TrackID, mimeType string, layer int, stats *livekit.RTPStats) TrackSubscribeRTPStats(ctx context.Context, participantID livekit.ParticipantID, trackID livekit.TrackID, mimeType string, stats *livekit.RTPStats) EgressStarted(ctx context.Context, info *livekit.EgressInfo) + EgressUpdated(ctx context.Context, info *livekit.EgressInfo) EgressEnded(ctx context.Context, info *livekit.EgressInfo) // helpers From de86ccb3df55b1687a76946dc9401c32de97799e Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 28 Mar 2023 07:18:31 +0530 Subject: [PATCH 038/324] Calculate stats duration. (#1554) --- pkg/sfu/connectionquality/connectionstats.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pkg/sfu/connectionquality/connectionstats.go b/pkg/sfu/connectionquality/connectionstats.go index 2a39a3f33..1ef2a5f42 100644 --- a/pkg/sfu/connectionquality/connectionstats.go +++ b/pkg/sfu/connectionquality/connectionstats.go @@ -106,13 +106,15 @@ func (cs *ConnectionStats) ReceiverReportReceived(at time.Time) { } func (cs *ConnectionStats) updateScore(streams map[uint32]*buffer.StreamStatsWithLayers, at time.Time) float32 { + var endedAt time.Time var stat windowStat for _, s := range streams { if stat.startedAt.IsZero() || stat.startedAt.After(s.RTPStats.StartTime) { stat.startedAt = s.RTPStats.StartTime } - if stat.duration < s.RTPStats.Duration { - stat.duration = s.RTPStats.Duration + streamEndedAt := s.RTPStats.StartTime.Add(s.RTPStats.Duration) + if endedAt.IsZero() || endedAt.Before(streamEndedAt) { + endedAt = streamEndedAt } stat.packetsExpected += s.RTPStats.Packets + s.RTPStats.PacketsPadding stat.packetsLost += s.RTPStats.PacketsLost @@ -125,6 +127,9 @@ func (cs *ConnectionStats) updateScore(streams map[uint32]*buffer.StreamStatsWit } stat.bytes += s.RTPStats.Bytes - s.RTPStats.HeaderBytes // only use media payload size } + if !stat.startedAt.IsZero() && !endedAt.IsZero() { + stat.duration = endedAt.Sub(stat.startedAt) + } cs.scorer.Update(&stat, at) mos, _ := cs.scorer.GetMOSAndQuality() From 2c439b3063afacebce5395bb15cbc2ac5e4cabb9 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 29 Mar 2023 07:43:28 +0530 Subject: [PATCH 039/324] Fix sequence number offset on packet drop (#1556) --- pkg/sfu/downtrack.go | 2 +- pkg/sfu/rtpmunger.go | 6 -- pkg/sfu/rtpmunger_test.go | 29 +++++++- pkg/sfu/streamallocator/streamallocator.go | 6 +- pkg/sfu/vp8munger.go | 84 ++++++++++++++-------- pkg/sfu/vp8munger_test.go | 51 ++++++------- 6 files changed, 112 insertions(+), 66 deletions(-) diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index c1d72f28c..4e89eed3e 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -1051,7 +1051,7 @@ func (d *DownTrack) AllocateNextHigher(availableChannelCapacity int64, allowOver func (d *DownTrack) GetNextHigherTransition(allowOvershoot bool) (VideoTransition, bool) { _, brs := d.receiver.GetLayeredBitrate() transition, available := d.forwarder.GetNextHigherTransition(brs, allowOvershoot) - d.logger.Debugw("stream: get next higher layer", "transition", transition, "available", available) + d.logger.Debugw("stream: get next higher layer", "transition", transition, "available", available, "bitrates", brs) return transition, available } diff --git a/pkg/sfu/rtpmunger.go b/pkg/sfu/rtpmunger.go index 1cd27d745..35ea2b25f 100644 --- a/pkg/sfu/rtpmunger.go +++ b/pkg/sfu/rtpmunger.go @@ -136,12 +136,6 @@ func (r *RTPMunger) PacketDropped(extPkt *buffer.ExtPacket) { } r.snOffset++ r.lastSN = extPkt.Packet.SequenceNumber - r.snOffset - - r.snOffsetsWritePtr = (r.snOffsetsWritePtr - 1) & SnOffsetCacheMask - r.snOffsetsOccupancy-- - if r.snOffsetsOccupancy < 0 { - r.logger.Warnw("sequence number offset cache is invalid", nil, "occupancy", r.snOffsetsOccupancy) - } } func (r *RTPMunger) UpdateAndGetSnTs(extPkt *buffer.ExtPacket) (*TranslationParamsRTP, error) { diff --git a/pkg/sfu/rtpmunger_test.go b/pkg/sfu/rtpmunger_test.go index 1d59759ab..e1e772023 100644 --- a/pkg/sfu/rtpmunger_test.go +++ b/pkg/sfu/rtpmunger_test.go @@ -105,6 +105,7 @@ func TestPacketDropped(t *testing.T) { SequenceNumber: 23333, Timestamp: 0xabcdef, SSRC: 0x12345678, + PayloadSize: 10, } extPkt, _ := testutils.GetTestExtPacket(params) r.SetLastSnTs(extPkt) @@ -114,6 +115,9 @@ func TestPacketDropped(t *testing.T) { require.Equal(t, uint16(0), r.snOffset) require.Equal(t, uint32(0), r.tsOffset) + r.UpdateAndGetSnTs(extPkt) // update sequence number offset + require.Equal(t, 1, r.snOffsetsWritePtr) + // drop a non-head packet, should cause no change in internals params = &testutils.TestExtPacketParams{ SequenceNumber: 33333, @@ -122,7 +126,7 @@ func TestPacketDropped(t *testing.T) { } extPkt, _ = testutils.GetTestExtPacket(params) r.PacketDropped(extPkt) - require.Equal(t, r.highestIncomingSN, uint16(23332)) + require.Equal(t, r.highestIncomingSN, uint16(23333)) require.Equal(t, r.lastSN, uint16(23333)) require.Equal(t, uint16(0), r.snOffset) @@ -131,12 +135,33 @@ func TestPacketDropped(t *testing.T) { SequenceNumber: 44444, Timestamp: 0xabcdef, SSRC: 0x12345678, + PayloadSize: 20, } extPkt, _ = testutils.GetTestExtPacket(params) - r.highestIncomingSN = 44444 + + r.UpdateAndGetSnTs(extPkt) // update sequence number offset + snOffsetWritePtr := (44444 - 23333 + 1) & SnOffsetCacheMask + require.Equal(t, snOffsetWritePtr, r.snOffsetsWritePtr) + require.Equal(t, SnOffsetCacheSize, r.snOffsetsOccupancy) + r.PacketDropped(extPkt) require.Equal(t, r.lastSN, uint16(44443)) require.Equal(t, uint16(1), r.snOffset) + + params = &testutils.TestExtPacketParams{ + SequenceNumber: 44445, + Timestamp: 0xabcdef, + SSRC: 0x12345678, + PayloadSize: 20, + } + extPkt, _ = testutils.GetTestExtPacket(params) + + r.UpdateAndGetSnTs(extPkt) // update sequence number offset + require.Equal(t, uint16(1), r.snOffsets[snOffsetWritePtr]) + snOffsetWritePtr = (snOffsetWritePtr + 1) & SnOffsetCacheMask + require.Equal(t, snOffsetWritePtr, r.snOffsetsWritePtr) + require.Equal(t, r.lastSN, uint16(44444)) + require.Equal(t, uint16(1), r.snOffset) } func TestOutOfOrderSequenceNumber(t *testing.T) { diff --git a/pkg/sfu/streamallocator/streamallocator.go b/pkg/sfu/streamallocator/streamallocator.go index 329b3f2c2..e5f7d5e3f 100644 --- a/pkg/sfu/streamallocator/streamallocator.go +++ b/pkg/sfu/streamallocator/streamallocator.go @@ -723,12 +723,10 @@ func (s *StreamAllocator) handleNewEstimateInNonProbe() { } var estimateToCommit int64 - var packets, repeatedNacks uint32 - var nackRatio float64 expectedBandwidthUsage := s.getExpectedBandwidthUsage() + packets, repeatedNacks, nackRatio := s.channelObserver.GetNackRatio() switch reason { case ChannelCongestionReasonLoss: - packets, repeatedNacks, nackRatio = s.channelObserver.GetNackRatio() estimateToCommit = int64(float64(expectedBandwidthUsage) * (1.0 - NackRatioAttenuator*nackRatio)) default: estimateToCommit = s.lastReceivedEstimate @@ -1110,7 +1108,7 @@ func (s *StreamAllocator) initProbe(probeRateBps int64) { "committed", s.committedChannelCapacity, "lastReceived", s.lastReceivedEstimate, "probeRateBps", probeRateBps, - "goalBps", expectedBandwidthUsage+probeRateBps, + "goalBps", s.probeGoalBps, ) } diff --git a/pkg/sfu/vp8munger.go b/pkg/sfu/vp8munger.go index 1dfa24d12..a631a6237 100644 --- a/pkg/sfu/vp8munger.go +++ b/pkg/sfu/vp8munger.go @@ -10,6 +10,12 @@ import ( "github.com/livekit/livekit-server/pkg/sfu/buffer" ) +const ( + missingPictureIdsThreshold = 50 + droppedPictureIdsThreshold = 20 + exemptedPictureIdsThreshold = 20 +) + // VP8 munger type TranslationParamsVP8 struct { Header *buffer.VP8 @@ -47,8 +53,9 @@ type VP8MungerParams struct { keyIdxOffset uint8 keyIdxUsed int - missingPictureIds *orderedmap.OrderedMap[int32, int32] - lastDroppedPictureId int32 + missingPictureIds *orderedmap.OrderedMap[int32, int32] + droppedPictureIds *orderedmap.OrderedMap[int32, bool] + exemptedPictureIds *orderedmap.OrderedMap[int32, bool] } type VP8Munger struct { @@ -61,8 +68,9 @@ func NewVP8Munger(logger logger.Logger) *VP8Munger { return &VP8Munger{ logger: logger, VP8MungerParams: VP8MungerParams{ - missingPictureIds: orderedmap.NewOrderedMap[int32, int32](), - lastDroppedPictureId: -1, + missingPictureIds: orderedmap.NewOrderedMap[int32, int32](), + droppedPictureIds: orderedmap.NewOrderedMap[int32, bool](), + exemptedPictureIds: orderedmap.NewOrderedMap[int32, bool](), }, } } @@ -112,8 +120,6 @@ func (v *VP8Munger) SetLast(extPkt *buffer.ExtPacket) { if v.keyIdxUsed == 1 { v.lastKeyIdx = vp8.KEYIDX } - - v.lastDroppedPictureId = -1 } func (v *VP8Munger) UpdateOffsets(extPkt *buffer.ExtPacket) { @@ -135,10 +141,10 @@ func (v *VP8Munger) UpdateOffsets(extPkt *buffer.ExtPacket) { v.keyIdxOffset = (vp8.KEYIDX - v.lastKeyIdx - 1) & 0x1f } - // clear missing picture ids on layer switch + // clear picture id caches on layer switch v.missingPictureIds = orderedmap.NewOrderedMap[int32, int32]() - - v.lastDroppedPictureId = -1 + v.droppedPictureIds = orderedmap.NewOrderedMap[int32, bool]() + v.exemptedPictureIds = orderedmap.NewOrderedMap[int32, bool]() } func (v *VP8Munger) UpdateAndGet(extPkt *buffer.ExtPacket, ordering SequenceNumberOrdering, maxTemporalLayer int32) (*TranslationParamsVP8, error) { @@ -200,34 +206,56 @@ func (v *VP8Munger) UpdateAndGet(extPkt *buffer.ExtPacket, ordering SequenceNumb // and check if that was the last packet of Picture 10), it could get complicated when // the gap is larger. if ordering == SequenceNumberOrderingGap { - // can drop packet if it belongs to the last dropped picture. - // Example: - // o Packet 10 - Picture 11 - TID that should be dropped - // o Packet 11 - missing - // o Packet 12 - Picture 11 - will be reported as GAP, but belongs to a picture that was dropped and hence can be dropped - // If Packet 11 comes around, it will be reported as OUT_OF_ORDER, but the missing - // picture id cache will not have an entry and hence will be dropped. - if extPictureId == v.lastDroppedPictureId { - return nil, ErrFilteredVP8TemporalLayer - } else { - for lostPictureId := prevMaxPictureId; lostPictureId <= extPictureId; lostPictureId++ { + for lostPictureId := prevMaxPictureId; lostPictureId <= extPictureId; lostPictureId++ { + // Record missing only if picture id was not dropped. This is to avoid a subsequent packet of dropped frame going through. + // A sequence like this + // o Packet 10 - Picture 11 - TID that should be dropped + // o Packet 11 - missing - belongs to Picture 11 still + // o Packet 12 - Picture 12 - will be reported as GAP, so missing picture id mapping will be set up for Picture 11 also. + // o Next packet - Packet 11 - this will use the wrong offset from missing pictures cache + _, ok := v.droppedPictureIds.Get(lostPictureId) + if !ok { v.missingPictureIds.Set(lostPictureId, v.pictureIdOffset) } + } + // trim cache if necessary + for v.missingPictureIds.Len() > missingPictureIdsThreshold { + el := v.missingPictureIds.Front() + v.missingPictureIds.Delete(el.Key) + } + + // if there is a gap, packet is forwarded irrespective of temporal layer as it cannot be determined + // which layer the missing packets belong to. A layer could have multiple packets. So, keep track + // of pictures that are forwarded even though they will be filterd out based on temporal layer + // requirements. That allows forwarding of the complete picture. + if vp8.TIDPresent == 1 && vp8.TID > uint8(maxTemporalLayer) { + v.exemptedPictureIds.Set(extPictureId, true) // trim cache if necessary - for v.missingPictureIds.Len() > 50 { - el := v.missingPictureIds.Front() - v.missingPictureIds.Delete(el.Key) + for v.exemptedPictureIds.Len() > exemptedPictureIdsThreshold { + el := v.exemptedPictureIds.Front() + v.exemptedPictureIds.Delete(el.Key) } } } else { if vp8.TIDPresent == 1 && vp8.TID > uint8(maxTemporalLayer) { - // adjust only once per picture as a picture could have multiple packets - if vp8.PictureIDPresent == 1 && prevMaxPictureId != extPictureId { - v.lastDroppedPictureId = extPictureId - v.pictureIdOffset += 1 + // drop only if not exempted + _, ok := v.exemptedPictureIds.Get(extPictureId) + if !ok { + // adjust only once per picture as a picture could have multiple packets + if vp8.PictureIDPresent == 1 && prevMaxPictureId != extPictureId { + // keep track of dropped picture ids so that they do not get into the missing picture cache + v.droppedPictureIds.Set(extPictureId, true) + // trim cache if necessary + for v.droppedPictureIds.Len() > droppedPictureIdsThreshold { + el := v.droppedPictureIds.Front() + v.droppedPictureIds.Delete(el.Key) + } + + v.pictureIdOffset += 1 + } + return nil, ErrFilteredVP8TemporalLayer } - return nil, ErrFilteredVP8TemporalLayer } } diff --git a/pkg/sfu/vp8munger_test.go b/pkg/sfu/vp8munger_test.go index 20fbcd922..0b32fc4db 100644 --- a/pkg/sfu/vp8munger_test.go +++ b/pkg/sfu/vp8munger_test.go @@ -65,17 +65,16 @@ func TestSetLast(t *testing.T) { totalWrap: 0, lastWrap: 0, }, - extLastPictureId: 13467, - pictureIdOffset: 0, - pictureIdUsed: 1, - lastTl0PicIdx: 233, - tl0PicIdxOffset: 0, - tl0PicIdxUsed: 1, - tidUsed: 1, - lastKeyIdx: 23, - keyIdxOffset: 0, - keyIdxUsed: 1, - lastDroppedPictureId: -1, + extLastPictureId: 13467, + pictureIdOffset: 0, + pictureIdUsed: 1, + lastTl0PicIdx: 233, + tl0PicIdxOffset: 0, + tl0PicIdxUsed: 1, + tidUsed: 1, + lastKeyIdx: 23, + keyIdxOffset: 0, + keyIdxUsed: 1, }, } @@ -140,17 +139,16 @@ func TestUpdateOffsets(t *testing.T) { totalWrap: 0, lastWrap: 0, }, - extLastPictureId: 13467, - pictureIdOffset: 345 - 13467 - 1, - pictureIdUsed: 1, - lastTl0PicIdx: 233, - tl0PicIdxOffset: (12 - 233 - 1) & 0xff, - tl0PicIdxUsed: 1, - tidUsed: 1, - lastKeyIdx: 23, - keyIdxOffset: (4 - 23 - 1) & 0x1f, - keyIdxUsed: 1, - lastDroppedPictureId: -1, + extLastPictureId: 13467, + pictureIdOffset: 345 - 13467 - 1, + pictureIdUsed: 1, + lastTl0PicIdx: 233, + tl0PicIdxOffset: (12 - 233 - 1) & 0xff, + tl0PicIdxUsed: 1, + tidUsed: 1, + lastKeyIdx: 23, + keyIdxOffset: (4 - 23 - 1) & 0x1f, + keyIdxUsed: 1, }, } require.True(t, compare(&expectedVP8Munger, v)) @@ -289,7 +287,8 @@ func TestTemporalLayerFiltering(t *testing.T) { require.Error(t, err) require.ErrorIs(t, err, ErrFilteredVP8TemporalLayer) require.Nil(t, tp) - require.EqualValues(t, 13467, v.lastDroppedPictureId) + dropped, _ := v.droppedPictureIds.Get(13467) + require.True(t, dropped) require.EqualValues(t, 1, v.pictureIdOffset) // another packet with the same picture id. @@ -301,7 +300,8 @@ func TestTemporalLayerFiltering(t *testing.T) { require.Error(t, err) require.ErrorIs(t, err, ErrFilteredVP8TemporalLayer) require.Nil(t, tp) - require.EqualValues(t, 13467, v.lastDroppedPictureId) + dropped, _ = v.droppedPictureIds.Get(13467) + require.True(t, dropped) require.EqualValues(t, 1, v.pictureIdOffset) // another packet with the same picture id, but a gap in sequence number. @@ -313,7 +313,8 @@ func TestTemporalLayerFiltering(t *testing.T) { require.Error(t, err) require.ErrorIs(t, err, ErrFilteredVP8TemporalLayer) require.Nil(t, tp) - require.EqualValues(t, 13467, v.lastDroppedPictureId) + dropped, _ = v.droppedPictureIds.Get(13467) + require.True(t, dropped) require.EqualValues(t, 1, v.pictureIdOffset) } From de80f521de79f44b428c16d9c602578987e4c8e6 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 29 Mar 2023 22:32:39 +0530 Subject: [PATCH 040/324] Increase sequence number cache to handle higb rate tracks. (#1560) Hopefully temporary while we can find a better solution. Adds 36 KB per SSRC. So, if a node can handle 10K SSRCs (roughly 10K tracks), that will be 360 MB of extra memory. --- pkg/sfu/buffer/rtpstats.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index 49e4bda6b..ebd68f242 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -19,7 +19,7 @@ const ( GapHistogramNumBins = 101 NumSequenceNumbers = 65536 FirstSnapshotId = 1 - SnInfoSize = 2048 + SnInfoSize = 8192 SnInfoMask = SnInfoSize - 1 TooLargeOWDDelta = 400 * time.Millisecond ) From aaab3b8ce868768ebc8c72a988538cec7066bf2c Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Wed, 29 Mar 2023 16:34:45 -0700 Subject: [PATCH 041/324] fix signal client message buffer size (#1561) * fix signal client message buffer size * update psrpc dep --- go.mod | 12 ++++++------ go.sum | 25 ++++++++++++------------- pkg/routing/signal.go | 16 ++++++++++------ 3 files changed, 28 insertions(+), 25 deletions(-) diff --git a/go.mod b/go.mod index 4db549ab4..e2a6f9cb7 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 github.com/livekit/protocol v1.5.1 - github.com/livekit/psrpc v0.2.10-0.20230310095745-5cd63568998d + github.com/livekit/psrpc v0.2.10-0.20230329223620-b33ff56abdc6 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.14.0 github.com/maxbrunsfeld/counterfeiter/v6 v6.6.1 @@ -73,8 +73,8 @@ require ( github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/mdlayher/netlink v1.7.1 // indirect github.com/mdlayher/socket v0.4.0 // indirect - github.com/nats-io/nats.go v1.24.0 // indirect - github.com/nats-io/nkeys v0.3.0 // indirect + github.com/nats-io/nats.go v1.25.0 // indirect + github.com/nats-io/nkeys v0.4.4 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/pion/datachannel v1.5.5 // indirect github.com/pion/dtls/v2 v2.2.6 // indirect @@ -91,14 +91,14 @@ require ( github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect go.uber.org/multierr v1.6.0 // indirect golang.org/x/crypto v0.7.0 // indirect - golang.org/x/exp v0.0.0-20230310171629-522b1b587ee0 // indirect + golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect golang.org/x/mod v0.8.0 // indirect golang.org/x/net v0.8.0 // indirect golang.org/x/sys v0.6.0 // indirect golang.org/x/text v0.8.0 // indirect golang.org/x/tools v0.6.0 // indirect - google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect - google.golang.org/grpc v1.53.0 // indirect + google.golang.org/genproto v0.0.0-20230327215041-6ac7f18bb9d5 // indirect + google.golang.org/grpc v1.54.0 // indirect gopkg.in/square/go-jose.v2 v2.6.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index bbb34e0e5..488b7b88a 100644 --- a/go.sum +++ b/go.sum @@ -235,8 +235,8 @@ github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 h1:QlQF github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26/go.mod h1:eDA41kiySZoG+wy4Etsjb3w0jjLx69i/vAmSjG4bteA= github.com/livekit/protocol v1.5.1 h1:K/p0ByfXuPv6yLeTcoQJn7pHI7bW4IKOSyb2ueG/qq0= github.com/livekit/protocol v1.5.1/go.mod h1:m5PkhcDT0EWdhatB0MpjmMaxyySjfE5NyZQC/LJWfEM= -github.com/livekit/psrpc v0.2.10-0.20230310095745-5cd63568998d h1:3wfbd8zi7zGQCR+xfG3r2k9m2RwXUiIzR0SN4BHewwU= -github.com/livekit/psrpc v0.2.10-0.20230310095745-5cd63568998d/go.mod h1:K0j8f1PgLShR7Lx80KbmwFkDH2BvOnycXGV0OSRURKc= +github.com/livekit/psrpc v0.2.10-0.20230329223620-b33ff56abdc6 h1:pd7xvJGPy/hBA7Jl94QfJ2gtnDjbZeYmJVH19ovDEdY= +github.com/livekit/psrpc v0.2.10-0.20230329223620-b33ff56abdc6/go.mod h1:K0j8f1PgLShR7Lx80KbmwFkDH2BvOnycXGV0OSRURKc= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= github.com/mackerelio/go-osstat v0.2.4/go.mod h1:Zy+qzGdZs3A9cuIqmgbJvwbmLQH9dJvtio5ZjJTbdlQ= github.com/magefile/mage v1.14.0 h1:6QDX3g6z1YvJ4olPhT1wksUcSa/V0a1B+pJb73fBjyo= @@ -278,10 +278,10 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nats-io/jwt/v2 v2.3.0 h1:z2mA1a7tIf5ShggOFlR1oBPgd6hGqcDYsISxZByUzdI= github.com/nats-io/nats-server/v2 v2.9.8 h1:jgxZsv+A3Reb3MgwxaINcNq/za8xZInKhDg9Q0cGN1o= -github.com/nats-io/nats.go v1.24.0 h1:CRiD8L5GOQu/DcfkmgBcTTIQORMwizF+rPk6T0RaHVQ= -github.com/nats-io/nats.go v1.24.0/go.mod h1:dVQF+BK3SzUZpwyzHedXsvH3EO38aVKuOPkkHlv5hXA= -github.com/nats-io/nkeys v0.3.0 h1:cgM5tL53EvYRU+2YLXIK0G2mJtK12Ft9oeooSZMA2G8= -github.com/nats-io/nkeys v0.3.0/go.mod h1:gvUNGjVcM2IPr5rCsRsC6Wb3Hr2CQAm08dsxtV6A5y4= +github.com/nats-io/nats.go v1.25.0 h1:t5/wCPGciR7X3Mu8QOi4jiJaXaWM8qtkLu4lzGZvYHE= +github.com/nats-io/nats.go v1.25.0/go.mod h1:D2WALIhz7V8M0pH8Scx8JZXlg6Oqz5VG+nQkK8nJdvg= +github.com/nats-io/nkeys v0.4.4 h1:xvBJ8d69TznjcQl9t6//Q5xXuVhyYiSos6RPtvQNTwA= +github.com/nats-io/nkeys v0.4.4/go.mod h1:XUkxdLPTufzlihbamfzQ7mw/VGx6ObUs+0bN5sNvt64= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= @@ -431,7 +431,6 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= @@ -447,8 +446,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230310171629-522b1b587ee0 h1:LGJsf5LRplCck6jUCH3dBL2dmycNruWNF5xugkSlfXw= -golang.org/x/exp v0.0.0-20230310171629-522b1b587ee0/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug= +golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -731,8 +730,8 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 h1:DdoeryqhaXp1LtT/emMP1BRJPHHKFi5akj/nbx/zNTA= -google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= +google.golang.org/genproto v0.0.0-20230327215041-6ac7f18bb9d5 h1:Kd6tRRHXw8z4TlPlWi+NaK10gsePL6GdZBQChptOLGA= +google.golang.org/genproto v0.0.0-20230327215041-6ac7f18bb9d5/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -745,8 +744,8 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= -google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= +google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= +google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/pkg/routing/signal.go b/pkg/routing/signal.go index 4d0f8b812..56800b8a1 100644 --- a/pkg/routing/signal.go +++ b/pkg/routing/signal.go @@ -31,12 +31,16 @@ type signalClient struct { } func NewSignalClient(nodeID livekit.NodeID, bus psrpc.MessageBus, config config.SignalRelayConfig) (SignalClient, error) { - ri := middleware.NewStreamRetryInterceptorFactory(middleware.RetryOptions{ - MaxAttempts: config.MaxAttempts, - Timeout: config.Timeout, - Backoff: config.Backoff, - }) - c, err := rpc.NewTypedSignalClient(nodeID, bus, psrpc.WithClientStreamInterceptors(ri)) + c, err := rpc.NewTypedSignalClient( + nodeID, + bus, + psrpc.WithClientStreamInterceptors(middleware.NewStreamRetryInterceptorFactory(middleware.RetryOptions{ + MaxAttempts: config.MaxAttempts, + Timeout: config.Timeout, + Backoff: config.Backoff, + })), + psrpc.WithClientChannelSize(config.StreamBufferSize), + ) if err != nil { return nil, err } From e67d9ca201c8fb10434e7ede26327e5efc42d4a1 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 29 Mar 2023 22:04:45 -0700 Subject: [PATCH 042/324] Update pion deps (#1557) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index e2a6f9cb7..25cbd4013 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/maxbrunsfeld/counterfeiter/v6 v6.6.1 github.com/mitchellh/go-homedir v1.1.0 github.com/olekukonko/tablewriter v0.0.5 - github.com/pion/ice/v2 v2.3.1 + github.com/pion/ice/v2 v2.3.2 github.com/pion/interceptor v0.1.12 github.com/pion/logging v0.2.2 github.com/pion/rtcp v1.2.10 @@ -34,7 +34,7 @@ require ( github.com/pion/stun v0.4.0 github.com/pion/transport/v2 v2.0.2 github.com/pion/turn/v2 v2.1.0 - github.com/pion/webrtc/v3 v3.1.58 + github.com/pion/webrtc/v3 v3.1.59 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.14.0 github.com/redis/go-redis/v9 v9.0.2 diff --git a/go.sum b/go.sum index 488b7b88a..263c18f56 100644 --- a/go.sum +++ b/go.sum @@ -300,8 +300,8 @@ github.com/pion/datachannel v1.5.5 h1:10ef4kwdjije+M9d7Xm9im2Y3O6A6ccQb0zcqZcJew github.com/pion/datachannel v1.5.5/go.mod h1:iMz+lECmfdCMqFRhXhcA/219B0SQlbpoR2V118yimL0= github.com/pion/dtls/v2 v2.2.6 h1:yXMxKr0Skd+Ub6A8UqXTRLSywskx93ooMRHsQUtd+Z4= github.com/pion/dtls/v2 v2.2.6/go.mod h1:t8fWJCIquY5rlQZwA2yWxUS1+OCrAdXrhVKXB5oD/wY= -github.com/pion/ice/v2 v2.3.1 h1:FQCmUfZe2Jpe7LYStVBOP6z1DiSzbIateih3TztgTjc= -github.com/pion/ice/v2 v2.3.1/go.mod h1:aq2kc6MtYNcn4XmMhobAv6hTNJiHzvD0yXRz80+bnP8= +github.com/pion/ice/v2 v2.3.2 h1:vh+fi4RkZ8H5fB4brZ/jm3j4BqFgMmNs+aB3X52Hu7M= +github.com/pion/ice/v2 v2.3.2/go.mod h1:AMIpuJqcpe+UwloocNebmTSWhCZM1TUCo9v7nW50jX0= github.com/pion/interceptor v0.1.12 h1:CslaNriCFUItiXS5o+hh5lpL0t0ytQkFnUcbbCs2Zq8= github.com/pion/interceptor v0.1.12/go.mod h1:bDtgAD9dRkBZpWHGKaoKb42FhDHTG2rX8Ii9LRALLVA= github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= @@ -332,8 +332,8 @@ github.com/pion/turn/v2 v2.1.0 h1:5wGHSgGhJhP/RpabkUb/T9PdsAjkGLS6toYz5HNzoSI= github.com/pion/turn/v2 v2.1.0/go.mod h1:yrT5XbXSGX1VFSF31A3c1kCNB5bBZgk/uu5LET162qs= github.com/pion/udp/v2 v2.0.1 h1:xP0z6WNux1zWEjhC7onRA3EwwSliXqu1ElUZAQhUP54= github.com/pion/udp/v2 v2.0.1/go.mod h1:B7uvTMP00lzWdyMr/1PVZXtV3wpPIxBRd4Wl6AksXn8= -github.com/pion/webrtc/v3 v3.1.58 h1:husXqiKQuk6gbOqJlPHs185OskAyxUW6iAEgHghgCrc= -github.com/pion/webrtc/v3 v3.1.58/go.mod h1:jJdqoqGBlZiE3y8Z1tg1fjSkyEDCZLL+foypUBn0Lhk= +github.com/pion/webrtc/v3 v3.1.59 h1:B3YFo8q6dwBYKA2LUjWRChP59Qtt+xvv1Ul7UPDp6Zc= +github.com/pion/webrtc/v3 v3.1.59/go.mod h1:rJGgStRoFyFOWJULHLayaimsG+jIEoenhJ5MB5gIFqw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= From 59961c1992ef9204b2bc67894e5af73c1d3855f8 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 30 Mar 2023 13:34:52 +0530 Subject: [PATCH 043/324] Aggregate method for RTPDeltaInfo (#1562) --- pkg/sfu/buffer/rtpstats.go | 98 +++++++++++++++++++ pkg/sfu/connectionquality/connectionstats.go | 35 +++---- .../connectionquality/connectionstats_test.go | 10 +- 3 files changed, 121 insertions(+), 22 deletions(-) diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index ebd68f242..8c4c9da37 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -1492,3 +1492,101 @@ func AggregateRTPStats(statsList []*livekit.RTPStats) *livekit.RTPStats { RttMax: maxRtt, } } + +func AggregateRTPDeltaInfo(deltaInfoList []*RTPDeltaInfo) *RTPDeltaInfo { + if len(deltaInfoList) == 0 { + return nil + } + + startTime := time.Time{} + endTime := time.Time{} + + packets := uint32(0) + bytes := uint64(0) + headerBytes := uint64(0) + + packetsDuplicate := uint32(0) + bytesDuplicate := uint64(0) + headerBytesDuplicate := uint64(0) + + packetsPadding := uint32(0) + bytesPadding := uint64(0) + headerBytesPadding := uint64(0) + + packetsLost := uint32(0) + packetsMissing := uint32(0) + + frames := uint32(0) + + maxRtt := uint32(0) + maxJitter := float64(0) + + nacks := uint32(0) + plis := uint32(0) + firs := uint32(0) + + for _, deltaInfo := range deltaInfoList { + if startTime.IsZero() || startTime.After(deltaInfo.StartTime) { + startTime = deltaInfo.StartTime + } + + endedAt := deltaInfo.StartTime.Add(deltaInfo.Duration) + if endTime.IsZero() || endTime.Before(endedAt) { + endTime = endedAt + } + + packets += deltaInfo.Packets + bytes += deltaInfo.Bytes + headerBytes += deltaInfo.HeaderBytes + + packetsDuplicate += deltaInfo.PacketsDuplicate + bytesDuplicate += deltaInfo.BytesDuplicate + headerBytesDuplicate += deltaInfo.HeaderBytesDuplicate + + packetsPadding += deltaInfo.PacketsPadding + bytesPadding += deltaInfo.BytesPadding + headerBytesPadding += deltaInfo.HeaderBytesPadding + + packetsLost += deltaInfo.PacketsLost + packetsMissing += deltaInfo.PacketsMissing + + frames += deltaInfo.Frames + + if deltaInfo.RttMax > maxRtt { + maxRtt = deltaInfo.RttMax + } + + if deltaInfo.JitterMax > maxJitter { + maxJitter = deltaInfo.JitterMax + } + + nacks += deltaInfo.Nacks + plis += deltaInfo.Plis + firs += deltaInfo.Firs + } + if startTime.IsZero() || endTime.IsZero() { + return nil + } + + return &RTPDeltaInfo{ + StartTime: startTime, + Duration: endTime.Sub(startTime), + Packets: packets, + Bytes: bytes, + HeaderBytes: headerBytes, + PacketsDuplicate: packetsDuplicate, + BytesDuplicate: bytesDuplicate, + HeaderBytesDuplicate: headerBytesDuplicate, + PacketsPadding: packetsPadding, + BytesPadding: bytesPadding, + HeaderBytesPadding: headerBytesPadding, + PacketsLost: packetsLost, + PacketsMissing: packetsMissing, + Frames: frames, + RttMax: maxRtt, + JitterMax: maxJitter, + Nacks: nacks, + Plis: plis, + Firs: firs, + } +} diff --git a/pkg/sfu/connectionquality/connectionstats.go b/pkg/sfu/connectionquality/connectionstats.go index 1ef2a5f42..185acc6e6 100644 --- a/pkg/sfu/connectionquality/connectionstats.go +++ b/pkg/sfu/connectionquality/connectionstats.go @@ -106,29 +106,22 @@ func (cs *ConnectionStats) ReceiverReportReceived(at time.Time) { } func (cs *ConnectionStats) updateScore(streams map[uint32]*buffer.StreamStatsWithLayers, at time.Time) float32 { - var endedAt time.Time - var stat windowStat + deltaInfoList := make([]*buffer.RTPDeltaInfo, 0, len(streams)) for _, s := range streams { - if stat.startedAt.IsZero() || stat.startedAt.After(s.RTPStats.StartTime) { - stat.startedAt = s.RTPStats.StartTime - } - streamEndedAt := s.RTPStats.StartTime.Add(s.RTPStats.Duration) - if endedAt.IsZero() || endedAt.Before(streamEndedAt) { - endedAt = streamEndedAt - } - stat.packetsExpected += s.RTPStats.Packets + s.RTPStats.PacketsPadding - stat.packetsLost += s.RTPStats.PacketsLost - stat.packetsMissing += s.RTPStats.PacketsMissing - if stat.rttMax < s.RTPStats.RttMax { - stat.rttMax = s.RTPStats.RttMax - } - if stat.jitterMax < s.RTPStats.JitterMax { - stat.jitterMax = s.RTPStats.JitterMax - } - stat.bytes += s.RTPStats.Bytes - s.RTPStats.HeaderBytes // only use media payload size + deltaInfoList = append(deltaInfoList, s.RTPStats) } - if !stat.startedAt.IsZero() && !endedAt.IsZero() { - stat.duration = endedAt.Sub(stat.startedAt) + agg := buffer.AggregateRTPDeltaInfo(deltaInfoList) + + var stat windowStat + if agg != nil { + stat.startedAt = agg.StartTime + stat.duration = agg.Duration + stat.packetsExpected = agg.Packets + agg.PacketsPadding + stat.packetsLost = agg.PacketsLost + stat.packetsMissing = agg.PacketsMissing + stat.bytes = agg.Bytes - agg.HeaderBytes // only use media payload size + stat.rttMax = agg.RttMax + stat.jitterMax = agg.JitterMax } cs.scorer.Update(&stat, at) diff --git a/pkg/sfu/connectionquality/connectionstats_test.go b/pkg/sfu/connectionquality/connectionstats_test.go index 6ffa34277..2ed5f3789 100644 --- a/pkg/sfu/connectionquality/connectionstats_test.go +++ b/pkg/sfu/connectionquality/connectionstats_test.go @@ -59,10 +59,18 @@ func TestConnectionQuality(t *testing.T) { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, Duration: duration, - Packets: 250, + Packets: 120, PacketsLost: 30, }, }, + 2: { + RTPStats: &buffer.RTPDeltaInfo{ + StartTime: now, + Duration: duration, + Packets: 130, + PacketsLost: 0, + }, + }, } cs.updateScore(streams, now.Add(duration)) mos, quality = cs.GetScoreAndQuality() From 82fd3e865e71b86359767e2e7da93bc60b5eaa28 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Thu, 30 Mar 2023 17:10:32 -0700 Subject: [PATCH 044/324] Fix deadlock caused by subscription manager test (#1563) We held the lock if subscription length check fails --- pkg/rtc/subscriptionmanager_test.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/pkg/rtc/subscriptionmanager_test.go b/pkg/rtc/subscriptionmanager_test.go index aa4441c1e..0132abb80 100644 --- a/pkg/rtc/subscriptionmanager_test.go +++ b/pkg/rtc/subscriptionmanager_test.go @@ -225,14 +225,20 @@ func TestUnsubscribe(t *testing.T) { require.False(t, s.isDesired()) require.Eventually(t, func() bool { - return !s.needsUnsubscribe() + if s.needsUnsubscribe() { + return false + } + sm.lock.RLock() + subLen := len(sm.subscriptions) + sm.lock.RUnlock() + if subLen != 0 { + return false + } + return true }, subSettleTimeout, subCheckInterval, "Track was not unsubscribed") // no traces should be left require.Len(t, sm.GetSubscribedTracks(), 0) - sm.lock.RLock() - require.Len(t, sm.subscriptions, 0) - sm.lock.RUnlock() require.False(t, res.TrackChangedNotifier.HasObservers()) tm := sm.params.Telemetry.(*telemetryfakes.FakeTelemetryService) From b5b896a6c66571ea00b37abf8f8d9cbf2ec94b89 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Thu, 30 Mar 2023 23:16:36 -0700 Subject: [PATCH 045/324] Version 1.4.0 (#1564) --- CHANGELOG | 34 ++++++++++++++++++++++++++++++++++ version/version.go | 2 +- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index 28b5fb8be..d8b45f488 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,6 +2,40 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.4.0] - 2023-03-27 +### Added +- Added config to disable active RED encoding. Use NACK instead #1476 #1477 +- Added option to skip TCP fallback if TCP RTT is high #1484 +- psrpc based signaling between signal and RTC #1485 +- Connection quality algorithm revamp #1490 #1491 #1493 #1496 #1497 #1500 #1505 #1507 #1509 #1516 #1520 #1521 #1527 #1528 #1536 +- Support for topics in data channel messages #1489 +- Added active filter to ListEgress #1517 +- Handling for React Native and Rust SDK ClientInfo #1544 + +### Fixed +- Fixed unsubscribed speakers stuck as speaking to clients #1475 +- Do not include packet in RED if timestamp is too far back #1478 +- Prevent PLI layer lock getting stuck #1481 +- Fix a case of changing video quality not succeeding #1483 +- Resync on pub muted for audio to avoid jump in sequence numbers on unmute #1487 +- Fixed a case of data race #1492 +- Inform reconnecting participant about recently disconnected users #1495 +- Send room update that may be missed by reconnected participant #1499 +- Fixed regression for AV1 forwarding #1538 +- Ensure sequence number continuity #1539 +- Give proper grace period when recorder is still in the room #1547 +- Fix sequence number offset on packet drop #1556 +- Fix signal client message buffer size #1561 + +### Changed +- Reduce lock scope getting RTCP sender reports #1473 +- Avoid duplicate queueReconcile in subscription manager #1474 +- Do not log TURN errors with prefix "error when handling datagram" #1494 +- Improvements to TCP fallback mode #1498 +- Unify forwarder between dependency descriptor and no DD case. #1543 +- Increase sequence number cache to handle high rate tracks #1560 + + ## [1.3.5] - 2023-02-25 ### Added - Allow for strict ACKs to be disabled or subscriber peer connections #1410 diff --git a/version/version.go b/version/version.go index 1f31716a0..ea5a66e4e 100644 --- a/version/version.go +++ b/version/version.go @@ -1,3 +1,3 @@ package version -const Version = "1.3.5" +const Version = "1.4.0" From c45e23be3fb78f8c2f131a46141a5de76db08369 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Fri, 31 Mar 2023 00:45:50 -0700 Subject: [PATCH 046/324] store participant res sink in interface typed field (#1567) --- pkg/rtc/participant.go | 12 ++++++------ pkg/rtc/participant_signal.go | 19 ++++++------------- 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 7838f8531..3e140499c 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -93,12 +93,12 @@ type ParticipantParams struct { type ParticipantImpl struct { params ParticipantParams - isClosed atomic.Bool - state atomic.Value // livekit.ParticipantInfo_State - resSink atomic.Value // routing.MessageSink - resSinkValid atomic.Bool - grants *auth.ClaimGrants - isPublisher atomic.Bool + isClosed atomic.Bool + state atomic.Value // livekit.ParticipantInfo_State + resSinkMu sync.Mutex + resSink routing.MessageSink + grants *auth.ClaimGrants + isPublisher atomic.Bool // when first connected connectedAt time.Time diff --git a/pkg/rtc/participant_signal.go b/pkg/rtc/participant_signal.go index cc1c0ea62..544399a1e 100644 --- a/pkg/rtc/participant_signal.go +++ b/pkg/rtc/participant_signal.go @@ -12,22 +12,15 @@ import ( ) func (p *ParticipantImpl) getResponseSink() routing.MessageSink { - if !p.resSinkValid.Load() { - return nil - } - sink := p.resSink.Load() - if s, ok := sink.(routing.MessageSink); ok { - return s - } - return nil + p.resSinkMu.Lock() + defer p.resSinkMu.Unlock() + return p.resSink } func (p *ParticipantImpl) SetResponseSink(sink routing.MessageSink) { - p.resSinkValid.Store(sink != nil) - if sink != nil { - // cannot store nil into atomic.Value - p.resSink.Store(sink) - } + p.resSinkMu.Lock() + defer p.resSinkMu.Unlock() + p.resSink = sink } func (p *ParticipantImpl) SendJoinResponse(joinResponse *livekit.JoinResponse) error { From 8be2b88ff6543cc1d6311e9af76a0183529de9e9 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Fri, 31 Mar 2023 18:28:15 -0700 Subject: [PATCH 047/324] Config docs for signal relay (#1566) --- config-sample.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/config-sample.yaml b/config-sample.yaml index 8cc964656..00d072f53 100644 --- a/config-sample.yaml +++ b/config-sample.yaml @@ -163,6 +163,21 @@ keys: # urls: # - https://your-host.com/handler +# Signal Relay +# since v1.4.0, a more reliable, psrpc based signal relay is available +# this gives us the ability to reliably proxy messages between a signal server and RTC node +# signal_relay: +# # disabled by default. will be enabled by default in future versions +# enabled: true +# # number of attempts for each message before giving up +# max_attempts: 3 +# # amount of time to wait for RTC node to ack +# timeout: 100ms +# # amount of time to add to timeout for each retry +# backoff: 500ms +# # number of messages to buffer before dropping +# stream_buffer_size: 1000 + # customize audio level sensitivity # audio: # # minimum level to be considered active, 0-127, where 0 is loudest From 602f987ed7b105582caf32fcb1e910989c0c7efa Mon Sep 17 00:00:00 2001 From: David Zhao Date: Fri, 31 Mar 2023 22:47:23 -0700 Subject: [PATCH 048/324] Switch up ordering DTLS elliptic curves to reduce likelihood of filtering (#1568) tl;dr. Pion-based traffic is impacted because TOR Snowflake uses Pion https://github.com/pion/dtls/pull/474 --- pkg/rtc/transport.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index 379ac5be6..c98f6241d 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -8,6 +8,7 @@ import ( "time" "github.com/bep/debounce" + "github.com/pion/dtls/v2/pkg/crypto/elliptic" "github.com/pion/ice/v2" "github.com/pion/interceptor" "github.com/pion/interceptor/pkg/cc" @@ -241,6 +242,10 @@ func newPeerConnection(params TransportParams, onBandwidthEstimator func(estimat se := params.Config.SettingEngine se.DisableMediaEngineCopy(true) + // Change elliptic curve to improve connectivity + // https://github.com/pion/dtls/pull/474 + se.SetDTLSEllipticCurves(elliptic.X25519, elliptic.P384, elliptic.P256) + // // Disable SRTP replay protection (https://datatracker.ietf.org/doc/html/rfc3711#page-15). // Needed due to lack of RTX stream support in Pion. From 1b09fc9721b2a03b73fe3d65c861b02d6a5686bc Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Sat, 1 Apr 2023 12:39:13 -0700 Subject: [PATCH 049/324] fix panic in rtpstats (#1570) --- pkg/sfu/buffer/rtpstats.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index 8c4c9da37..f1176f5d2 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -1526,6 +1526,10 @@ func AggregateRTPDeltaInfo(deltaInfoList []*RTPDeltaInfo) *RTPDeltaInfo { firs := uint32(0) for _, deltaInfo := range deltaInfoList { + if deltaInfo == nil { + continue + } + if startTime.IsZero() || startTime.After(deltaInfo.StartTime) { startTime = deltaInfo.StartTime } From 31427f75949b1895451b92974f0bf3ce10d457a7 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 2 Apr 2023 13:43:05 -0700 Subject: [PATCH 050/324] Update module github.com/redis/go-redis/v9 to v9.0.3 (#1572) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 25cbd4013..1038a7f0f 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,7 @@ require ( github.com/maxbrunsfeld/counterfeiter/v6 v6.6.1 github.com/mitchellh/go-homedir v1.1.0 github.com/olekukonko/tablewriter v0.0.5 + github.com/pion/dtls/v2 v2.2.6 github.com/pion/ice/v2 v2.3.2 github.com/pion/interceptor v0.1.12 github.com/pion/logging v0.2.2 @@ -37,7 +38,7 @@ require ( github.com/pion/webrtc/v3 v3.1.59 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.14.0 - github.com/redis/go-redis/v9 v9.0.2 + github.com/redis/go-redis/v9 v9.0.3 github.com/rs/cors v1.8.3 github.com/sebest/xff v0.0.0-20210106013422-671bd2870b3a github.com/stretchr/testify v1.8.2 @@ -77,7 +78,6 @@ require ( github.com/nats-io/nkeys v0.4.4 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/pion/datachannel v1.5.5 // indirect - github.com/pion/dtls/v2 v2.2.6 // indirect github.com/pion/mdns v0.0.7 // indirect github.com/pion/randutil v0.1.0 // indirect github.com/pion/sctp v1.8.6 // indirect diff --git a/go.sum b/go.sum index 263c18f56..ccb5323c4 100644 --- a/go.sum +++ b/go.sum @@ -45,8 +45,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bep/debounce v1.2.1 h1:v67fRdBA9UQu2NhLFXrSg0Brw7CexQekrBwDMM8bzeY= github.com/bep/debounce v1.2.1/go.mod h1:H8yggRPQKLUhUoqrJC1bO2xNya7vanpDl7xR3ISbCJ0= -github.com/bsm/ginkgo/v2 v2.5.0 h1:aOAnND1T40wEdAtkGSkvSICWeQ8L3UASX7YVCqQx+eQ= -github.com/bsm/gomega v1.20.0 h1:JhAwLmtRzXFTx2AkALSLa8ijZafntmhSoU63Ok18Uq8= +github.com/bsm/ginkgo/v2 v2.7.0 h1:ItPMPH90RbmZJt5GtkcNvIRuGEdwlBItdNVoyzaNQao= +github.com/bsm/gomega v1.26.0 h1:LhQm+AFcgV2M0WyKroMASzAzCAJVpAxQXv4SaI9a69Y= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -366,8 +366,8 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= -github.com/redis/go-redis/v9 v9.0.2 h1:BA426Zqe/7r56kCcvxYLWe1mkaz71LKF77GwgFzSxfE= -github.com/redis/go-redis/v9 v9.0.2/go.mod h1:/xDTe9EF1LM61hek62Poq2nzQSGj0xSrEtEHbBQevps= +github.com/redis/go-redis/v9 v9.0.3 h1:+7mmR26M0IvyLxGZUHxu4GiBkJkVDid0Un+j4ScYu4k= +github.com/redis/go-redis/v9 v9.0.3/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= From 1cb6cc3ed7acf53f26d01c69dba74a56634edd27 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Mon, 3 Apr 2023 15:24:53 -0700 Subject: [PATCH 051/324] Do not sample per participant to reduce memory usage (#1576) --- pkg/rtc/utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/rtc/utils.go b/pkg/rtc/utils.go index 36d7cfc7b..95db5606d 100644 --- a/pkg/rtc/utils.go +++ b/pkg/rtc/utils.go @@ -140,7 +140,7 @@ func LoggerWithParticipant(l logger.Logger, identity livekit.ParticipantIdentity } values = append(values, "remote", isRemote) // enable sampling per participant - return l.WithItemSampler().WithValues(values...) + return l.WithValues(values...) } func LoggerWithRoom(l logger.Logger, name livekit.RoomName, roomID livekit.RoomID) logger.Logger { From 8cfba1308b1d3d673bfb8c8522a618e188496b40 Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Tue, 4 Apr 2023 09:28:47 +0800 Subject: [PATCH 052/324] Add test case for munged sdp (#1574) * Add test case for munged sdp * clean code --- pkg/rtc/participant_internal_test.go | 179 +++++++++++++++++++++++++++ pkg/rtc/transport_test.go | 37 ++++++ 2 files changed, 216 insertions(+) diff --git a/pkg/rtc/participant_internal_test.go b/pkg/rtc/participant_internal_test.go index f04c277b5..a182606e8 100644 --- a/pkg/rtc/participant_internal_test.go +++ b/pkg/rtc/participant_internal_test.go @@ -1,6 +1,7 @@ package rtc import ( + "fmt" "strings" "testing" "time" @@ -425,6 +426,184 @@ func TestDisableCodecs(t *testing.T) { require.False(t, found264) } +func TestPreferVideoCodecForPublisher(t *testing.T) { + participant := newParticipantForTestWithOpts("123", &participantOpts{ + publisher: true, + }) + participant.SetMigrateState(types.MigrateStateComplete) + + pc, err := webrtc.NewPeerConnection(webrtc.Configuration{}) + require.NoError(t, err) + defer pc.Close() + + for i := 0; i < 2; i++ { + // publish h264 track without client preferred codec + trackCid := fmt.Sprintf("preferh264video%d", i) + participant.AddTrack(&livekit.AddTrackRequest{ + Type: livekit.TrackType_VIDEO, + Name: "video", + Width: 1280, + Height: 720, + Source: livekit.TrackSource_CAMERA, + SimulcastCodecs: []*livekit.SimulcastCodec{ + { + Codec: "h264", + Cid: trackCid, + }, + }, + }) + + track, err := webrtc.NewTrackLocalStaticRTP(webrtc.RTPCodecCapability{MimeType: "video/vp8"}, trackCid, trackCid) + require.NoError(t, err) + transceiver, err := pc.AddTransceiverFromTrack(track, webrtc.RTPTransceiverInit{Direction: webrtc.RTPTransceiverDirectionSendrecv}) + require.NoError(t, err) + sdp, err := pc.CreateOffer(nil) + require.NoError(t, err) + pc.SetLocalDescription(sdp) + codecs := transceiver.Receiver().GetParameters().Codecs + + // h264 should not be preferred + require.NotEqual(t, codecs[0].MimeType, "video/h264") + + sink := &routingfakes.FakeMessageSink{} + participant.SetResponseSink(sink) + var answer webrtc.SessionDescription + var answerReceived atomic.Bool + sink.WriteMessageStub = func(msg proto.Message) error { + if res, ok := msg.(*livekit.SignalResponse); ok { + if res.GetAnswer() != nil { + answer = FromProtoSessionDescription(res.GetAnswer()) + pc.SetRemoteDescription(answer) + answerReceived.Store(true) + } + } + return nil + } + participant.HandleOffer(sdp) + + require.Eventually(t, func() bool { return answerReceived.Load() }, 5*time.Second, 10*time.Millisecond) + + var h264Preferred bool + parsed, err := answer.Unmarshal() + require.NoError(t, err) + var videoSectionIndex int + for _, m := range parsed.MediaDescriptions { + if m.MediaName.Media == "video" { + if videoSectionIndex == i { + codecs, err := codecsFromMediaDescription(m) + require.NoError(t, err) + if strings.EqualFold(codecs[0].Name, "h264") { + h264Preferred = true + break + } + } + videoSectionIndex++ + } + } + + require.Truef(t, h264Preferred, "h264 should be preferred for video section %d, answer sdp: \n%s", i, answer.SDP) + } +} + +func TestPreferAudioCodecForRed(t *testing.T) { + participant := newParticipantForTestWithOpts("123", &participantOpts{ + publisher: true, + }) + participant.SetMigrateState(types.MigrateStateComplete) + + me := webrtc.MediaEngine{} + me.RegisterDefaultCodecs() + require.NoError(t, me.RegisterCodec(webrtc.RTPCodecParameters{ + RTPCodecCapability: redCodecCapability, + PayloadType: 63, + }, webrtc.RTPCodecTypeAudio)) + + api := webrtc.NewAPI(webrtc.WithMediaEngine(&me)) + pc, err := api.NewPeerConnection(webrtc.Configuration{}) + require.NoError(t, err) + defer pc.Close() + + for i, disableRed := range []bool{false, true} { + t.Run(fmt.Sprintf("disableRed=%v", disableRed), func(t *testing.T) { + trackCid := fmt.Sprintf("audiotrack%d", i) + participant.AddTrack(&livekit.AddTrackRequest{ + Type: livekit.TrackType_AUDIO, + DisableRed: disableRed, + Cid: trackCid, + }) + track, err := webrtc.NewTrackLocalStaticRTP(webrtc.RTPCodecCapability{MimeType: "audio/opus"}, trackCid, trackCid) + require.NoError(t, err) + transceiver, err := pc.AddTransceiverFromTrack(track, webrtc.RTPTransceiverInit{Direction: webrtc.RTPTransceiverDirectionSendrecv}) + require.NoError(t, err) + codecs := transceiver.Sender().GetParameters().Codecs + for i, c := range codecs { + if c.MimeType == "audio/opus" && i != 0 { + codecs[0], codecs[i] = codecs[i], codecs[0] + break + } + } + transceiver.SetCodecPreferences(codecs) + sdp, err := pc.CreateOffer(nil) + require.NoError(t, err) + pc.SetLocalDescription(sdp) + // opus should be preferred + require.Equal(t, codecs[0].MimeType, "audio/opus", sdp) + + sink := &routingfakes.FakeMessageSink{} + participant.SetResponseSink(sink) + var answer webrtc.SessionDescription + var answerReceived atomic.Bool + sink.WriteMessageStub = func(msg proto.Message) error { + if res, ok := msg.(*livekit.SignalResponse); ok { + if res.GetAnswer() != nil { + answer = FromProtoSessionDescription(res.GetAnswer()) + pc.SetRemoteDescription(answer) + answerReceived.Store(true) + } + } + return nil + } + participant.HandleOffer(sdp) + + require.Eventually(t, func() bool { return answerReceived.Load() }, 5*time.Second, 10*time.Millisecond) + + var redPreferred bool + parsed, err := answer.Unmarshal() + require.NoError(t, err) + var audioSectionIndex int + for _, m := range parsed.MediaDescriptions { + if m.MediaName.Media == "audio" { + if audioSectionIndex == i { + codecs, err := codecsFromMediaDescription(m) + require.NoError(t, err) + // nack is always enabled. if red is preferred, server will not generate nack request + var nackEnabled bool + for _, c := range codecs { + if c.Name == "opus" { + for _, fb := range c.RTCPFeedback { + if strings.Contains(fb, "nack") { + nackEnabled = true + break + } + } + } + } + require.True(t, nackEnabled, "nack should be enabled for opus") + + if strings.EqualFold(codecs[0].Name, "red") { + redPreferred = true + break + } + } + audioSectionIndex++ + } + } + require.Equalf(t, !disableRed, redPreferred, "offer : \n%s\nanswer sdp: \n%s", sdp, answer.SDP) + }) + } + +} + type participantOpts struct { permissions *livekit.ParticipantPermission protocolVersion types.ProtocolVersion diff --git a/pkg/rtc/transport_test.go b/pkg/rtc/transport_test.go index 5e388facb..e7365531e 100644 --- a/pkg/rtc/transport_test.go +++ b/pkg/rtc/transport_test.go @@ -506,3 +506,40 @@ func connectTransports(t *testing.T, offerer, answerer *PCTransport, isICERestar return answerer.pc.ICEConnectionState() == webrtc.ICEConnectionStateConnected }, 10*time.Second, time.Millisecond*10, "answerer did not become connected") } + +func TestConfigureAudioTransceiver(t *testing.T) { + pc, err := webrtc.NewPeerConnection(webrtc.Configuration{}) + require.NoError(t, err) + defer pc.Close() + + for _, testcase := range []struct { + nack bool + stereo bool + }{ + {false, false}, + {true, false}, + {false, true}, + {true, true}, + } { + t.Run(fmt.Sprintf("nack=%v,stereo=%v", testcase.nack, testcase.stereo), func(t *testing.T) { + tr, err := pc.AddTransceiverFromKind(webrtc.RTPCodecTypeAudio, webrtc.RtpTransceiverInit{Direction: webrtc.RTPTransceiverDirectionSendonly}) + require.NoError(t, err) + + configureAudioTransceiver(tr, testcase.stereo, testcase.nack) + codecs := tr.Sender().GetParameters().Codecs + for _, codec := range codecs { + if strings.Contains(codec.MimeType, webrtc.MimeTypeOpus) { + require.Equal(t, testcase.stereo, strings.Contains(codec.SDPFmtpLine, "sprop-stereo=1")) + var nackEnabled bool + for _, fb := range codec.RTCPFeedback { + if fb.Type == webrtc.TypeRTCPFBNACK { + nackEnabled = true + break + } + } + require.Equal(t, testcase.nack, nackEnabled) + } + } + }) + } +} From 793e61ac142fc2ccaee6190824a5ce7417da5277 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 4 Apr 2023 09:58:57 +0530 Subject: [PATCH 053/324] Use bandwidth requested from last allocation. (#1577) * Use bandwidth requested from last allocation. With overshoot/opportunistic forwarding, It is possible that bitrate at target layers is 0. So, use bandwidth requested from last allocation which shouold have a correct value. Still need to think about using the latest bit rates to get the requested bandwidth. It is possible that bitrates have changed since last allocation. That was the idea behind using the latest bitrates, but it could return 0. Accounting for it runs into a few scenarios. Last allocation has number from last allocation and is a good indicator of the need. * race --- pkg/rtc/participant_internal_test.go | 4 ++-- pkg/sfu/downtrack.go | 3 +-- pkg/sfu/forwarder.go | 18 ++---------------- 3 files changed, 5 insertions(+), 20 deletions(-) diff --git a/pkg/rtc/participant_internal_test.go b/pkg/rtc/participant_internal_test.go index a182606e8..4d294c646 100644 --- a/pkg/rtc/participant_internal_test.go +++ b/pkg/rtc/participant_internal_test.go @@ -469,7 +469,7 @@ func TestPreferVideoCodecForPublisher(t *testing.T) { participant.SetResponseSink(sink) var answer webrtc.SessionDescription var answerReceived atomic.Bool - sink.WriteMessageStub = func(msg proto.Message) error { + sink.WriteMessageCalls(func(msg proto.Message) error { if res, ok := msg.(*livekit.SignalResponse); ok { if res.GetAnswer() != nil { answer = FromProtoSessionDescription(res.GetAnswer()) @@ -478,7 +478,7 @@ func TestPreferVideoCodecForPublisher(t *testing.T) { } } return nil - } + }) participant.HandleOffer(sdp) require.Eventually(t, func() bool { return answerReceived.Load() }, 5*time.Second, 10*time.Millisecond) diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 4e89eed3e..0fee2e8ca 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -995,8 +995,7 @@ func (d *DownTrack) IsDeficient() bool { } func (d *DownTrack) BandwidthRequested() int64 { - _, brs := d.receiver.GetLayeredBitrate() - return d.forwarder.BandwidthRequested(brs) + return d.forwarder.BandwidthRequested() } func (d *DownTrack) DistanceToDesired() float64 { diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index fde7e1ccb..1ef7674f2 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -456,25 +456,11 @@ func (f *Forwarder) IsDeficient() bool { return f.isDeficientLocked() } -func (f *Forwarder) BandwidthRequested(brs Bitrates) int64 { +func (f *Forwarder) BandwidthRequested() int64 { f.lock.RLock() defer f.lock.RUnlock() - if !f.targetLayers.IsValid() { - if f.targetLayers != buffer.InvalidLayers { - f.logger.Warnw( - "unexpected target layers", nil, - "target", f.targetLayers, - "current", f.currentLayers, - "parked", f.parkedLayers, - "max", f.maxLayers, - "lastAllocation", f.lastAllocation, - ) - } - return 0 - } - - return brs[f.targetLayers.Spatial][f.targetLayers.Temporal] + return f.lastAllocation.BandwidthRequested } func (f *Forwarder) DistanceToDesired(availableLayers []int32, brs Bitrates) float64 { From fc6a306031310988338690c58711b9fc58484a33 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Tue, 4 Apr 2023 19:32:49 -0700 Subject: [PATCH 054/324] Create a helper for retrieving a user's actual IP (#1579) --- go.mod | 1 - go.sum | 2 -- pkg/service/rtcservice.go | 6 +----- pkg/service/utils.go | 16 ++++++++++++++++ 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 1038a7f0f..ced4643ac 100644 --- a/go.mod +++ b/go.mod @@ -40,7 +40,6 @@ require ( github.com/prometheus/client_golang v1.14.0 github.com/redis/go-redis/v9 v9.0.3 github.com/rs/cors v1.8.3 - github.com/sebest/xff v0.0.0-20210106013422-671bd2870b3a github.com/stretchr/testify v1.8.2 github.com/thoas/go-funk v0.9.3 github.com/twitchtv/twirp v8.1.3+incompatible diff --git a/go.sum b/go.sum index ccb5323c4..189e10f24 100644 --- a/go.sum +++ b/go.sum @@ -377,8 +377,6 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/sclevine/spec v1.4.0 h1:z/Q9idDcay5m5irkZ28M7PtQM4aOISzOpj4bUPkDee8= -github.com/sebest/xff v0.0.0-20210106013422-671bd2870b3a h1:iLcLb5Fwwz7g/DLK89F+uQBDeAhHhwdzB5fSlVdhGcM= -github.com/sebest/xff v0.0.0-20210106013422-671bd2870b3a/go.mod h1:wozgYq9WEBQBaIJe4YZ0qTSFAMxmcwBhQH0fO0R34Z0= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= diff --git a/pkg/service/rtcservice.go b/pkg/service/rtcservice.go index d712cd17b..22592a4c5 100644 --- a/pkg/service/rtcservice.go +++ b/pkg/service/rtcservice.go @@ -12,7 +12,6 @@ import ( "time" "github.com/gorilla/websocket" - "github.com/sebest/xff" "github.com/ua-parser/uap-go/uaparser" "github.com/livekit/livekit-server/pkg/utils" @@ -404,10 +403,7 @@ func (s *RTCService) ParseClientInfo(r *http.Request) *livekit.ClientInfo { ci.DeviceModel = values.Get("device_model") ci.Network = values.Get("network") // get real address (forwarded http header) - check Cloudflare headers first, fall back to X-Forwarded-For - ci.Address = r.Header.Get("CF-Connecting-IP") - if len(ci.Address) == 0 { - ci.Address = xff.GetRemoteAddr(r) - } + ci.Address = GetClientIP(r) // attempt to parse types for SDKs that support browser as a platform if ci.Sdk == livekit.ClientInfo_JS || diff --git a/pkg/service/utils.go b/pkg/service/utils.go index 07eca562d..42f0c9c41 100644 --- a/pkg/service/utils.go +++ b/pkg/service/utils.go @@ -1,6 +1,7 @@ package service import ( + "net" "net/http" "regexp" @@ -22,3 +23,18 @@ func IsValidDomain(domain string) bool { domainRegexp := regexp.MustCompile(`^(?i)[a-z0-9-]+(\.[a-z0-9-]+)+\.?$`) return domainRegexp.MatchString(domain) } + +func GetClientIP(r *http.Request) string { + // CF proxy typically is first thing the user reaches + if ip := r.Header.Get("CF-Connecting-IP"); ip != "" { + return ip + } + if ip := r.Header.Get("X-Forwarded-For"); ip != "" { + return ip + } + if ip := r.Header.Get("X-Real-IP"); ip != "" { + return ip + } + ip, _, _ := net.SplitHostPort(r.RemoteAddr) + return ip +} From 5564bc531f22d89cf4885f889a80641b901c4453 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Wed, 5 Apr 2023 03:42:59 -0700 Subject: [PATCH 055/324] write signal messages from media without blocking (#1580) --- pkg/service/signal.go | 68 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 55 insertions(+), 13 deletions(-) diff --git a/pkg/service/signal.go b/pkg/service/signal.go index 835788b34..b4838945e 100644 --- a/pkg/service/signal.go +++ b/pkg/service/signal.go @@ -2,6 +2,8 @@ package service import ( "context" + "fmt" + "sync" "github.com/pkg/errors" "google.golang.org/protobuf/proto" @@ -106,6 +108,12 @@ func (r *signalService) RelaySignal(stream psrpc.ServerStream[*rpc.RelaySignalRe return errors.Wrap(err, "failed to read participant from session") } + l := logger.GetLogger().WithValues( + "room", ss.RoomName, + "participant", ss.Identity, + "connectionID", ss.ConnectionId, + ) + reqChan := routing.NewDefaultMessageChannel() defer reqChan.Close() @@ -115,14 +123,13 @@ func (r *signalService) RelaySignal(stream psrpc.ServerStream[*rpc.RelaySignalRe *pi, livekit.ConnectionID(ss.ConnectionId), reqChan, - &relaySignalResponseSink{stream}, + &relaySignalResponseSink{ + ServerStream: stream, + logger: l, + }, ) if err != nil { - logger.Errorw("could not handle new participant", err, - "room", ss.RoomName, - "participant", ss.Identity, - "connectionID", ss.ConnectionId, - ) + l.Errorw("could not handle new participant", err) } for msg := range stream.Channel() { @@ -131,16 +138,17 @@ func (r *signalService) RelaySignal(stream psrpc.ServerStream[*rpc.RelaySignalRe } } - logger.Debugw("participant signal stream closed", - "room", ss.RoomName, - "participant", ss.Identity, - "connectionID", ss.ConnectionId, - ) + l.Debugw("participant signal stream closed") return } type relaySignalResponseSink struct { psrpc.ServerStream[*rpc.RelaySignalResponse, *rpc.RelaySignalRequest] + logger logger.Logger + + mu sync.Mutex + queue []*livekit.SignalResponse + writing bool } func (s *relaySignalResponseSink) Close() { @@ -151,6 +159,40 @@ func (s *relaySignalResponseSink) IsClosed() bool { return s.Context().Err() != nil } -func (s *relaySignalResponseSink) WriteMessage(msg proto.Message) error { - return s.Send(&rpc.RelaySignalResponse{Response: msg.(*livekit.SignalResponse)}) +func (s *relaySignalResponseSink) write() { + for { + s.mu.Lock() + var msg *livekit.SignalResponse + if len(s.queue) != 0 && !s.IsClosed() { + msg = s.queue[0] + s.queue = s.queue[1:] + } else { + s.writing = false + s.mu.Unlock() + return + } + s.mu.Unlock() + + if err := s.Send(&rpc.RelaySignalResponse{Response: msg}); err != nil { + s.logger.Warnw( + "could not send message to participant", err, + "messageType", fmt.Sprintf("%T", msg.Message), + ) + } + } +} + +func (s *relaySignalResponseSink) WriteMessage(msg proto.Message) error { + if err := s.Context().Err(); err != nil { + return err + } + + s.mu.Lock() + s.queue = append(s.queue, msg.(*livekit.SignalResponse)) + if !s.writing { + s.writing = true + go s.write() + } + s.mu.Unlock() + return nil } From 6636e3766423e4a20d00870ac10f7acb1e243695 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Wed, 5 Apr 2023 03:50:43 -0700 Subject: [PATCH 056/324] add prometheus psrpc metrics observer (#1571) * add prometheus psrpc metrics observer * record rpc error counts * update psrpc * update protocol --- go.mod | 12 ++-- go.sum | 26 +++---- pkg/routing/signal.go | 6 +- pkg/service/signal.go | 18 +++-- pkg/telemetry/prometheus/node.go | 1 + pkg/telemetry/prometheus/psrpc.go | 111 ++++++++++++++++++++++++++++++ 6 files changed, 148 insertions(+), 26 deletions(-) create mode 100644 pkg/telemetry/prometheus/psrpc.go diff --git a/go.mod b/go.mod index ced4643ac..9f16bab62 100644 --- a/go.mod +++ b/go.mod @@ -18,8 +18,8 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 - github.com/livekit/protocol v1.5.1 - github.com/livekit/psrpc v0.2.10-0.20230329223620-b33ff56abdc6 + github.com/livekit/protocol v1.5.2-0.20230405103303-8c8b87686b2c + github.com/livekit/psrpc v0.2.10 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.14.0 github.com/maxbrunsfeld/counterfeiter/v6 v6.6.1 @@ -61,7 +61,8 @@ require ( github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/eapache/channels v1.1.0 // indirect github.com/eapache/queue v1.1.0 // indirect - github.com/go-logr/logr v1.2.3 // indirect + github.com/go-jose/go-jose/v3 v3.0.0 // indirect + github.com/go-logr/logr v1.2.4 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/subcommands v1.2.0 // indirect @@ -93,11 +94,10 @@ require ( golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect golang.org/x/mod v0.8.0 // indirect golang.org/x/net v0.8.0 // indirect - golang.org/x/sys v0.6.0 // indirect + golang.org/x/sys v0.7.0 // indirect golang.org/x/text v0.8.0 // indirect golang.org/x/tools v0.6.0 // indirect - google.golang.org/genproto v0.0.0-20230327215041-6ac7f18bb9d5 // indirect + google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd // indirect google.golang.org/grpc v1.54.0 // indirect - gopkg.in/square/go-jose.v2 v2.6.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 189e10f24..4cf3986b3 100644 --- a/go.sum +++ b/go.sum @@ -98,6 +98,8 @@ github.com/gammazero/workerpool v1.1.3/go.mod h1:wPjyBLDbyKnUn2XwwyD3EEwo9dHutia github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-jose/go-jose/v3 v3.0.0 h1:s6rrhirfEP/CGIoc6p+PZAeogN2SxKav6Wp7+dyMWVo= +github.com/go-jose/go-jose/v3 v3.0.0/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= @@ -106,8 +108,8 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -233,10 +235,10 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 h1:QlQFyMwCDgjyySsrgmrMcVbEBA6KZcyTzvK+z346tUA= github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26/go.mod h1:eDA41kiySZoG+wy4Etsjb3w0jjLx69i/vAmSjG4bteA= -github.com/livekit/protocol v1.5.1 h1:K/p0ByfXuPv6yLeTcoQJn7pHI7bW4IKOSyb2ueG/qq0= -github.com/livekit/protocol v1.5.1/go.mod h1:m5PkhcDT0EWdhatB0MpjmMaxyySjfE5NyZQC/LJWfEM= -github.com/livekit/psrpc v0.2.10-0.20230329223620-b33ff56abdc6 h1:pd7xvJGPy/hBA7Jl94QfJ2gtnDjbZeYmJVH19ovDEdY= -github.com/livekit/psrpc v0.2.10-0.20230329223620-b33ff56abdc6/go.mod h1:K0j8f1PgLShR7Lx80KbmwFkDH2BvOnycXGV0OSRURKc= +github.com/livekit/protocol v1.5.2-0.20230405103303-8c8b87686b2c h1:fkBV/qBvTSZogAh+dJ75WYqrAGHOpNGIi7Z1iuulHMQ= +github.com/livekit/protocol v1.5.2-0.20230405103303-8c8b87686b2c/go.mod h1:UFgAWejoO4eshaaDe2jynTdQWwSktNO+8Wx19V7bs+o= +github.com/livekit/psrpc v0.2.10 h1:Ud9GzMYkKhMbB6c3RkOqe8mlRtSqtW0StyICp1jApDM= +github.com/livekit/psrpc v0.2.10/go.mod h1:K0j8f1PgLShR7Lx80KbmwFkDH2BvOnycXGV0OSRURKc= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= github.com/mackerelio/go-osstat v0.2.4/go.mod h1:Zy+qzGdZs3A9cuIqmgbJvwbmLQH9dJvtio5ZjJTbdlQ= github.com/magefile/mage v1.14.0 h1:6QDX3g6z1YvJ4olPhT1wksUcSa/V0a1B+pJb73fBjyo= @@ -388,6 +390,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= @@ -427,6 +430,7 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -605,8 +609,8 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -728,8 +732,8 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20230327215041-6ac7f18bb9d5 h1:Kd6tRRHXw8z4TlPlWi+NaK10gsePL6GdZBQChptOLGA= -google.golang.org/genproto v0.0.0-20230327215041-6ac7f18bb9d5/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd h1:sLpv7bNL1AsX3fdnWh9WVh7ejIzXdOc1RRHGeAmeStU= +google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -765,8 +769,6 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI= -gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/routing/signal.go b/pkg/routing/signal.go index 56800b8a1..1bfe7b365 100644 --- a/pkg/routing/signal.go +++ b/pkg/routing/signal.go @@ -7,6 +7,7 @@ import ( "google.golang.org/protobuf/proto" "github.com/livekit/livekit-server/pkg/config" + "github.com/livekit/livekit-server/pkg/telemetry/prometheus" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/rpc" @@ -34,11 +35,12 @@ func NewSignalClient(nodeID livekit.NodeID, bus psrpc.MessageBus, config config. c, err := rpc.NewTypedSignalClient( nodeID, bus, - psrpc.WithClientStreamInterceptors(middleware.NewStreamRetryInterceptorFactory(middleware.RetryOptions{ + middleware.WithClientMetrics(prometheus.PSRPCMetricsObserver{}), + middleware.WithStreamRetries(middleware.RetryOptions{ MaxAttempts: config.MaxAttempts, Timeout: config.Timeout, Backoff: config.Backoff, - })), + }), psrpc.WithClientChannelSize(config.StreamBufferSize), ) if err != nil { diff --git a/pkg/service/signal.go b/pkg/service/signal.go index b4838945e..3e9eab948 100644 --- a/pkg/service/signal.go +++ b/pkg/service/signal.go @@ -38,12 +38,18 @@ func NewSignalServer( config config.SignalRelayConfig, sessionHandler SessionHandler, ) (*SignalServer, error) { - ri := middleware.NewStreamRetryInterceptorFactory(middleware.RetryOptions{ - MaxAttempts: config.MaxAttempts, - Timeout: config.Timeout, - Backoff: config.Backoff, - }) - s, err := rpc.NewTypedSignalServer(nodeID, &signalService{region, sessionHandler}, bus, psrpc.WithServerStreamInterceptors(ri), psrpc.WithServerChannelSize(config.StreamBufferSize)) + s, err := rpc.NewTypedSignalServer( + nodeID, + &signalService{region, sessionHandler}, + bus, + middleware.WithServerMetrics(prometheus.PSRPCMetricsObserver{}), + psrpc.WithServerStreamInterceptors(middleware.NewStreamRetryInterceptorFactory(middleware.RetryOptions{ + MaxAttempts: config.MaxAttempts, + Timeout: config.Timeout, + Backoff: config.Backoff, + })), + psrpc.WithServerChannelSize(config.StreamBufferSize), + ) if err != nil { return nil, err } diff --git a/pkg/telemetry/prometheus/node.go b/pkg/telemetry/prometheus/node.go index 9393052c1..54e7f9d33 100644 --- a/pkg/telemetry/prometheus/node.go +++ b/pkg/telemetry/prometheus/node.go @@ -95,6 +95,7 @@ func Init(nodeID string, nodeType livekit.NodeType, env string) { initPacketStats(nodeID, nodeType, env) initRoomStats(nodeID, nodeType, env) + initPSRPCStats(nodeID, nodeType, env) } func GetUpdatedNodeStats(prev *livekit.NodeStats, prevAverage *livekit.NodeStats) (*livekit.NodeStats, bool, error) { diff --git a/pkg/telemetry/prometheus/psrpc.go b/pkg/telemetry/prometheus/psrpc.go new file mode 100644 index 000000000..1ac3d9a72 --- /dev/null +++ b/pkg/telemetry/prometheus/psrpc.go @@ -0,0 +1,111 @@ +package prometheus + +import ( + "time" + + "github.com/prometheus/client_golang/prometheus" + + "github.com/livekit/protocol/livekit" + "github.com/livekit/psrpc" + "github.com/livekit/psrpc/middleware" +) + +var ( + psrpcRequestTime *prometheus.HistogramVec + psrpcStreamSendTime *prometheus.HistogramVec + psrpcStreamReceiveTotal *prometheus.CounterVec + psrpcStreamCurrent *prometheus.GaugeVec + psrpcErrorTotal *prometheus.CounterVec +) + +func initPSRPCStats(nodeID string, nodeType livekit.NodeType, env string) { + labels := []string{"role", "kind", "service", "method"} + streamLabels := []string{"role", "service", "method"} + + psrpcRequestTime = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: livekitNamespace, + Subsystem: "psrpc", + Name: "request_time_ms", + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + Buckets: []float64{10, 50, 100, 300, 500, 1000, 1500, 2000, 5000, 10000}, + }, labels) + psrpcStreamSendTime = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: livekitNamespace, + Subsystem: "psrpc", + Name: "stream_send_time_ms", + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + Buckets: []float64{10, 50, 100, 300, 500, 1000, 1500, 2000, 5000, 10000}, + }, streamLabels) + psrpcStreamReceiveTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: livekitNamespace, + Subsystem: "psrpc", + Name: "stream_receive_total", + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + }, streamLabels) + psrpcStreamCurrent = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: livekitNamespace, + Subsystem: "psrpc", + Name: "stream_count", + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + }, streamLabels) + psrpcErrorTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: livekitNamespace, + Subsystem: "psrpc", + Name: "error_total", + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + }, labels) + + prometheus.MustRegister(psrpcRequestTime) + prometheus.MustRegister(psrpcStreamSendTime) + prometheus.MustRegister(psrpcStreamReceiveTotal) + prometheus.MustRegister(psrpcStreamCurrent) + prometheus.MustRegister(psrpcErrorTotal) +} + +var _ middleware.MetricsObserver = PSRPCMetricsObserver{} + +type PSRPCMetricsObserver struct{} + +func (o PSRPCMetricsObserver) OnUnaryRequest(role middleware.MetricRole, info psrpc.RPCInfo, duration time.Duration, err error) { + if err != nil { + psrpcErrorTotal.WithLabelValues(role.String(), "rpc", info.Service, info.Method).Inc() + } else if role == middleware.ClientRole { + psrpcRequestTime.WithLabelValues(role.String(), "rpc", info.Service, info.Method).Observe(float64(duration.Milliseconds())) + } else { + psrpcRequestTime.WithLabelValues(role.String(), "rpc", info.Service, info.Method).Observe(float64(duration.Milliseconds())) + } +} + +func (o PSRPCMetricsObserver) OnMultiRequest(role middleware.MetricRole, info psrpc.RPCInfo, duration time.Duration, responseCount int, errorCount int) { + if responseCount == 0 { + psrpcErrorTotal.WithLabelValues(role.String(), "multirpc", info.Service, info.Method).Inc() + } else if role == middleware.ClientRole { + psrpcRequestTime.WithLabelValues(role.String(), "multirpc", info.Service, info.Method).Observe(float64(duration.Milliseconds())) + } else { + psrpcRequestTime.WithLabelValues(role.String(), "multirpc", info.Service, info.Method).Observe(float64(duration.Milliseconds())) + } +} + +func (o PSRPCMetricsObserver) OnStreamSend(role middleware.MetricRole, info psrpc.RPCInfo, duration time.Duration, err error) { + if err != nil { + psrpcErrorTotal.WithLabelValues(role.String(), "stream", info.Service, info.Method).Inc() + } else { + psrpcStreamSendTime.WithLabelValues(role.String(), info.Service, info.Method).Observe(float64(duration.Milliseconds())) + } +} + +func (o PSRPCMetricsObserver) OnStreamRecv(role middleware.MetricRole, info psrpc.RPCInfo, err error) { + if err != nil { + psrpcErrorTotal.WithLabelValues(role.String(), "stream", info.Service, info.Method).Inc() + } else { + psrpcStreamReceiveTotal.WithLabelValues(role.String(), info.Service, info.Method).Inc() + } +} + +func (o PSRPCMetricsObserver) OnStreamOpen(role middleware.MetricRole, info psrpc.RPCInfo) { + psrpcStreamCurrent.WithLabelValues(role.String(), info.Service, info.Method).Inc() +} + +func (o PSRPCMetricsObserver) OnStreamClose(role middleware.MetricRole, info psrpc.RPCInfo) { + psrpcStreamCurrent.WithLabelValues(role.String(), info.Service, info.Method).Dec() +} From 55520622287a8d19461fa63d9becf597723323d7 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Wed, 5 Apr 2023 12:29:52 -0700 Subject: [PATCH 057/324] drain signal stream before closing (#1582) * drain signal stream before closing * update psrpc * cleanup --- go.mod | 2 +- go.sum | 4 ++-- pkg/service/signal.go | 26 ++++++++++++++++++-------- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 9f16bab62..0a6a22e92 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 github.com/livekit/protocol v1.5.2-0.20230405103303-8c8b87686b2c - github.com/livekit/psrpc v0.2.10 + github.com/livekit/psrpc v0.2.11-0.20230405191830-d76f71512630 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.14.0 github.com/maxbrunsfeld/counterfeiter/v6 v6.6.1 diff --git a/go.sum b/go.sum index 4cf3986b3..a63672866 100644 --- a/go.sum +++ b/go.sum @@ -237,8 +237,8 @@ github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 h1:QlQF github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26/go.mod h1:eDA41kiySZoG+wy4Etsjb3w0jjLx69i/vAmSjG4bteA= github.com/livekit/protocol v1.5.2-0.20230405103303-8c8b87686b2c h1:fkBV/qBvTSZogAh+dJ75WYqrAGHOpNGIi7Z1iuulHMQ= github.com/livekit/protocol v1.5.2-0.20230405103303-8c8b87686b2c/go.mod h1:UFgAWejoO4eshaaDe2jynTdQWwSktNO+8Wx19V7bs+o= -github.com/livekit/psrpc v0.2.10 h1:Ud9GzMYkKhMbB6c3RkOqe8mlRtSqtW0StyICp1jApDM= -github.com/livekit/psrpc v0.2.10/go.mod h1:K0j8f1PgLShR7Lx80KbmwFkDH2BvOnycXGV0OSRURKc= +github.com/livekit/psrpc v0.2.11-0.20230405191830-d76f71512630 h1:Rm5KLZgQxWnTidY+H8MsAV6sk1iiFxeXqPFgSLkMing= +github.com/livekit/psrpc v0.2.11-0.20230405191830-d76f71512630/go.mod h1:K0j8f1PgLShR7Lx80KbmwFkDH2BvOnycXGV0OSRURKc= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= github.com/mackerelio/go-osstat v0.2.4/go.mod h1:Zy+qzGdZs3A9cuIqmgbJvwbmLQH9dJvtio5ZjJTbdlQ= github.com/magefile/mage v1.14.0 h1:6QDX3g6z1YvJ4olPhT1wksUcSa/V0a1B+pJb73fBjyo= diff --git a/pkg/service/signal.go b/pkg/service/signal.go index 3e9eab948..2423260d6 100644 --- a/pkg/service/signal.go +++ b/pkg/service/signal.go @@ -152,13 +152,19 @@ type relaySignalResponseSink struct { psrpc.ServerStream[*rpc.RelaySignalResponse, *rpc.RelaySignalRequest] logger logger.Logger - mu sync.Mutex - queue []*livekit.SignalResponse - writing bool + mu sync.Mutex + queue []*livekit.SignalResponse + writing bool + draining bool } func (s *relaySignalResponseSink) Close() { - s.ServerStream.Close(nil) + s.mu.Lock() + s.draining = true + if !s.writing { + s.ServerStream.Close(nil) + } + s.mu.Unlock() } func (s *relaySignalResponseSink) IsClosed() bool { @@ -173,6 +179,9 @@ func (s *relaySignalResponseSink) write() { msg = s.queue[0] s.queue = s.queue[1:] } else { + if s.draining { + s.ServerStream.Close(nil) + } s.writing = false s.mu.Unlock() return @@ -189,16 +198,17 @@ func (s *relaySignalResponseSink) write() { } func (s *relaySignalResponseSink) WriteMessage(msg proto.Message) error { - if err := s.Context().Err(); err != nil { - return err + s.mu.Lock() + defer s.mu.Unlock() + + if s.draining || s.IsClosed() { + return psrpc.ErrStreamClosed } - s.mu.Lock() s.queue = append(s.queue, msg.(*livekit.SignalResponse)) if !s.writing { s.writing = true go s.write() } - s.mu.Unlock() return nil } From 234f7ea5cbcee0954a4e9f9ff0c7bc270ee97776 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Wed, 5 Apr 2023 14:41:52 -0700 Subject: [PATCH 058/324] read batched signal messages (#1583) * batch signal messages * update protcol --- go.mod | 2 +- go.sum | 4 ++-- pkg/routing/redisrouter.go | 8 ++++---- pkg/routing/signal.go | 5 +++++ 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 0a6a22e92..55bdf34be 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 - github.com/livekit/protocol v1.5.2-0.20230405103303-8c8b87686b2c + github.com/livekit/protocol v1.5.2-0.20230405195605-927c9ea2b4c6 github.com/livekit/psrpc v0.2.11-0.20230405191830-d76f71512630 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.14.0 diff --git a/go.sum b/go.sum index a63672866..c963460f4 100644 --- a/go.sum +++ b/go.sum @@ -235,8 +235,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 h1:QlQFyMwCDgjyySsrgmrMcVbEBA6KZcyTzvK+z346tUA= github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26/go.mod h1:eDA41kiySZoG+wy4Etsjb3w0jjLx69i/vAmSjG4bteA= -github.com/livekit/protocol v1.5.2-0.20230405103303-8c8b87686b2c h1:fkBV/qBvTSZogAh+dJ75WYqrAGHOpNGIi7Z1iuulHMQ= -github.com/livekit/protocol v1.5.2-0.20230405103303-8c8b87686b2c/go.mod h1:UFgAWejoO4eshaaDe2jynTdQWwSktNO+8Wx19V7bs+o= +github.com/livekit/protocol v1.5.2-0.20230405195605-927c9ea2b4c6 h1:rvkmoc5s+VJTpShWkY+QWFQ5XhLDMFFyZIZrrr7PgJE= +github.com/livekit/protocol v1.5.2-0.20230405195605-927c9ea2b4c6/go.mod h1:UFgAWejoO4eshaaDe2jynTdQWwSktNO+8Wx19V7bs+o= github.com/livekit/psrpc v0.2.11-0.20230405191830-d76f71512630 h1:Rm5KLZgQxWnTidY+H8MsAV6sk1iiFxeXqPFgSLkMing= github.com/livekit/psrpc v0.2.11-0.20230405191830-d76f71512630/go.mod h1:K0j8f1PgLShR7Lx80KbmwFkDH2BvOnycXGV0OSRURKc= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= diff --git a/pkg/routing/redisrouter.go b/pkg/routing/redisrouter.go index be1a8e025..d53c52997 100644 --- a/pkg/routing/redisrouter.go +++ b/pkg/routing/redisrouter.go @@ -149,10 +149,6 @@ func (r *RedisRouter) StartParticipantSignal(ctx context.Context, roomName livek return } - if r.usePSRPCSignal { - return r.StartParticipantSignalWithNodeID(ctx, roomName, pi, livekit.NodeID(rtcNode.Id)) - } - // create a new connection id connectionID = livekit.ConnectionID(utils.NewGuid("CO_")) pKey := participantKeyLegacy(roomName, pi.Identity) @@ -163,6 +159,10 @@ func (r *RedisRouter) StartParticipantSignal(ctx context.Context, roomName livek return } + if r.usePSRPCSignal { + return r.StartParticipantSignalWithNodeID(ctx, roomName, pi, livekit.NodeID(rtcNode.Id)) + } + // index by connectionID, since there may be multiple connections for the participant // set up response channel before sending StartSession and be ready to receive responses. resChan := r.getOrCreateMessageChannel(r.responseChannels, string(connectionID)) diff --git a/pkg/routing/signal.go b/pkg/routing/signal.go index 1bfe7b365..418852c3d 100644 --- a/pkg/routing/signal.go +++ b/pkg/routing/signal.go @@ -105,6 +105,11 @@ func (r *signalClient) StartParticipantSignal( if err = resChan.WriteMessage(msg.Response); err != nil { break } + for _, res := range msg.Responses { + if err = resChan.WriteMessage(res); err != nil { + break + } + } } logger.Debugw("participant signal stream closed", From 6b0cb33c53afdfa006eec94c4ba363c78444de81 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Wed, 5 Apr 2023 16:08:25 -0700 Subject: [PATCH 059/324] set participant node for redis router in signal service (#1584) --- pkg/routing/localrouter.go | 8 ++++---- pkg/routing/redisrouter.go | 35 +++++++++++++++++++++-------------- pkg/routing/utils.go | 4 ++-- pkg/routing/utils_test.go | 10 +++++----- pkg/service/signal.go | 11 +++++++++++ 5 files changed, 43 insertions(+), 25 deletions(-) diff --git a/pkg/routing/localrouter.go b/pkg/routing/localrouter.go index a70cec005..2c5c39a98 100644 --- a/pkg/routing/localrouter.go +++ b/pkg/routing/localrouter.go @@ -106,14 +106,14 @@ func (r *LocalRouter) WriteParticipantRTC(_ context.Context, roomName livekit.Ro r.rtcMessageChan = NewMessageChannel(localRTCChannelSize) } r.lock.Unlock() - msg.ParticipantKey = string(participantKeyLegacy(roomName, identity)) - msg.ParticipantKeyB62 = string(participantKey(roomName, identity)) + msg.ParticipantKey = string(ParticipantKeyLegacy(roomName, identity)) + msg.ParticipantKeyB62 = string(ParticipantKey(roomName, identity)) return r.writeRTCMessage(r.rtcMessageChan, msg) } func (r *LocalRouter) WriteRoomRTC(ctx context.Context, roomName livekit.RoomName, msg *livekit.RTCNodeMessage) error { - msg.ParticipantKey = string(participantKeyLegacy(roomName, "")) - msg.ParticipantKeyB62 = string(participantKey(roomName, "")) + msg.ParticipantKey = string(ParticipantKeyLegacy(roomName, "")) + msg.ParticipantKeyB62 = string(ParticipantKey(roomName, "")) return r.WriteNodeRTC(ctx, r.currentNode.Id, msg) } diff --git a/pkg/routing/redisrouter.go b/pkg/routing/redisrouter.go index d53c52997..c5da2a54b 100644 --- a/pkg/routing/redisrouter.go +++ b/pkg/routing/redisrouter.go @@ -149,20 +149,27 @@ func (r *RedisRouter) StartParticipantSignal(ctx context.Context, roomName livek return } + if r.usePSRPCSignal { + connectionID, reqSink, resSource, err = r.StartParticipantSignalWithNodeID(ctx, roomName, pi, livekit.NodeID(rtcNode.Id)) + if err != nil { + return + } + + // map signal & rtc nodes + err = r.setParticipantSignalNode(connectionID, r.currentNode.Id) + return + } + // create a new connection id connectionID = livekit.ConnectionID(utils.NewGuid("CO_")) - pKey := participantKeyLegacy(roomName, pi.Identity) - pKeyB62 := participantKey(roomName, pi.Identity) + pKey := ParticipantKeyLegacy(roomName, pi.Identity) + pKeyB62 := ParticipantKey(roomName, pi.Identity) // map signal & rtc nodes if err = r.setParticipantSignalNode(connectionID, r.currentNode.Id); err != nil { return } - if r.usePSRPCSignal { - return r.StartParticipantSignalWithNodeID(ctx, roomName, pi, livekit.NodeID(rtcNode.Id)) - } - // index by connectionID, since there may be multiple connections for the participant // set up response channel before sending StartSession and be ready to receive responses. resChan := r.getOrCreateMessageChannel(r.responseChannels, string(connectionID)) @@ -185,16 +192,16 @@ func (r *RedisRouter) StartParticipantSignal(ctx context.Context, roomName livek } func (r *RedisRouter) WriteParticipantRTC(_ context.Context, roomName livekit.RoomName, identity livekit.ParticipantIdentity, msg *livekit.RTCNodeMessage) error { - pkey := participantKeyLegacy(roomName, identity) - pkeyB62 := participantKey(roomName, identity) + pkey := ParticipantKeyLegacy(roomName, identity) + pkeyB62 := ParticipantKey(roomName, identity) rtcNode, err := r.getParticipantRTCNode(pkey, pkeyB62) if err != nil { return err } rtcSink := NewRTCNodeSink(r.rc, livekit.NodeID(rtcNode), pkey, pkeyB62) - msg.ParticipantKey = string(participantKeyLegacy(roomName, identity)) - msg.ParticipantKeyB62 = string(participantKey(roomName, identity)) + msg.ParticipantKey = string(ParticipantKeyLegacy(roomName, identity)) + msg.ParticipantKeyB62 = string(ParticipantKey(roomName, identity)) return r.writeRTCMessage(rtcSink, msg) } @@ -203,8 +210,8 @@ func (r *RedisRouter) WriteRoomRTC(ctx context.Context, roomName livekit.RoomNam if err != nil { return err } - msg.ParticipantKey = string(participantKeyLegacy(roomName, "")) - msg.ParticipantKeyB62 = string(participantKey(roomName, "")) + msg.ParticipantKey = string(ParticipantKeyLegacy(roomName, "")) + msg.ParticipantKeyB62 = string(ParticipantKey(roomName, "")) return r.WriteNodeRTC(ctx, node.Id, msg) } @@ -229,7 +236,7 @@ func (r *RedisRouter) startParticipantRTC(ss *livekit.StartSession, participantK return err } - if err := r.setParticipantRTCNode(participantKey, participantKeyB62, rtcNode.Id); err != nil { + if err := r.SetParticipantRTCNode(participantKey, participantKeyB62, rtcNode.Id); err != nil { return err } @@ -328,7 +335,7 @@ func (r *RedisRouter) Stop() { r.cancel() } -func (r *RedisRouter) setParticipantRTCNode(participantKey livekit.ParticipantKey, participantKeyB62 livekit.ParticipantKey, nodeID string) error { +func (r *RedisRouter) SetParticipantRTCNode(participantKey livekit.ParticipantKey, participantKeyB62 livekit.ParticipantKey, nodeID string) error { var err error if participantKey != "" { err1 := r.rc.Set(r.ctx, participantRTCKey(participantKey), nodeID, participantMappingTTL).Err() diff --git a/pkg/routing/utils.go b/pkg/routing/utils.go index 8db4f649c..38a458b63 100644 --- a/pkg/routing/utils.go +++ b/pkg/routing/utils.go @@ -9,7 +9,7 @@ import ( "github.com/livekit/protocol/livekit" ) -func participantKeyLegacy(roomName livekit.RoomName, identity livekit.ParticipantIdentity) livekit.ParticipantKey { +func ParticipantKeyLegacy(roomName livekit.RoomName, identity livekit.ParticipantIdentity) livekit.ParticipantKey { return livekit.ParticipantKey(string(roomName) + "|" + string(identity)) } @@ -25,7 +25,7 @@ func parseParticipantKeyLegacy(pkey livekit.ParticipantKey) (roomName livekit.Ro return } -func participantKey(roomName livekit.RoomName, identity livekit.ParticipantIdentity) livekit.ParticipantKey { +func ParticipantKey(roomName livekit.RoomName, identity livekit.ParticipantIdentity) livekit.ParticipantKey { return livekit.ParticipantKey(encode(string(roomName), string(identity))) } diff --git a/pkg/routing/utils_test.go b/pkg/routing/utils_test.go index ae40b27e8..10a21a60f 100644 --- a/pkg/routing/utils_test.go +++ b/pkg/routing/utils_test.go @@ -10,7 +10,7 @@ import ( func TestUtils_ParticipantKey(t *testing.T) { // encode/decode empty - encoded := participantKey("", "") + encoded := ParticipantKey("", "") roomName, identity, err := parseParticipantKey(encoded) require.NoError(t, err) require.Equal(t, livekit.RoomName(""), roomName) @@ -21,28 +21,28 @@ func TestUtils_ParticipantKey(t *testing.T) { require.Error(t, err) // encode/decode without delimiter - encoded = participantKey("room1", "identity1") + encoded = ParticipantKey("room1", "identity1") roomName, identity, err = parseParticipantKey(encoded) require.NoError(t, err) require.Equal(t, livekit.RoomName("room1"), roomName) require.Equal(t, livekit.ParticipantIdentity("identity1"), identity) // encode/decode with delimiter in roomName - encoded = participantKey("room1|alter_room1", "identity1") + encoded = ParticipantKey("room1|alter_room1", "identity1") roomName, identity, err = parseParticipantKey(encoded) require.NoError(t, err) require.Equal(t, livekit.RoomName("room1|alter_room1"), roomName) require.Equal(t, livekit.ParticipantIdentity("identity1"), identity) // encode/decode with delimiter in identity - encoded = participantKey("room1", "identity1|alter-identity1") + encoded = ParticipantKey("room1", "identity1|alter-identity1") roomName, identity, err = parseParticipantKey(encoded) require.NoError(t, err) require.Equal(t, livekit.RoomName("room1"), roomName) require.Equal(t, livekit.ParticipantIdentity("identity1|alter-identity1"), identity) // encode/decode with delimiter in both and multiple delimiters in both - encoded = participantKey("room1|alter_room1|again_room1", "identity1|alter-identity1|again-identity1") + encoded = ParticipantKey("room1|alter_room1|again_room1", "identity1|alter-identity1|again-identity1") roomName, identity, err = parseParticipantKey(encoded) require.NoError(t, err) require.Equal(t, livekit.RoomName("room1|alter_room1|again_room1"), roomName) diff --git a/pkg/service/signal.go b/pkg/service/signal.go index 2423260d6..407c173e1 100644 --- a/pkg/service/signal.go +++ b/pkg/service/signal.go @@ -77,6 +77,17 @@ func NewDefaultSignalServer( responseSink routing.MessageSink, ) error { prometheus.IncrementParticipantRtcInit(1) + + if rr, ok := router.(*routing.RedisRouter); ok { + pKey := routing.ParticipantKeyLegacy(roomName, pi.Identity) + pKeyB62 := routing.ParticipantKey(roomName, pi.Identity) + + // RTC session should start on this node + if err := rr.SetParticipantRTCNode(pKey, pKeyB62, currentNode.Id); err != nil { + return err + } + } + return roomManager.StartSession(ctx, roomName, pi, requestSource, responseSink) } From 2b16589b79ec21c95e68154594894e0823b4d482 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Wed, 5 Apr 2023 21:43:25 -0700 Subject: [PATCH 060/324] Version 1.4.1 (#1585) * Version 1.4.1 * fix change description --- CHANGELOG | 12 ++++++++++++ version/version.go | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index d8b45f488..d2c66ec7b 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,6 +2,18 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.4.1] - 2023-04-05 +### Added +- Added prometheus metrics for internal signaling API #1571 + +### Fixed +- Fix regressions in RTC when using redis with psrpc signaling #1584 #1582 #1580 #1567 +- Fix required bitrate assessment under channel congestion #1577 + +### Changed +- Improve DTLS reliability in regions with internet filters #1568 +- Reduce memory usage from logging #1576 + ## [1.4.0] - 2023-03-27 ### Added - Added config to disable active RED encoding. Use NACK instead #1476 #1477 diff --git a/version/version.go b/version/version.go index ea5a66e4e..bf695e80d 100644 --- a/version/version.go +++ b/version/version.go @@ -1,3 +1,3 @@ package version -const Version = "1.4.0" +const Version = "1.4.1" From fb301e6e759aa65a000f45e76562cf3602bf00ea Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Thu, 6 Apr 2023 21:51:33 +0800 Subject: [PATCH 061/324] Add vp9 svc support by Dependency Descriptor (#1586) * Add VP9 SVC support * Fix preferred fps does not work * Fix forwarder test --- pkg/rtc/mediaengine.go | 18 ++++++++---------- pkg/rtc/participant_sdp.go | 2 +- pkg/sfu/buffer/buffer.go | 2 ++ pkg/sfu/buffer/helpers.go | 26 ++++++++++++++++++++++++++ pkg/sfu/forwarder.go | 10 +++++----- pkg/sfu/forwarder_test.go | 4 ++-- 6 files changed, 44 insertions(+), 18 deletions(-) diff --git a/pkg/rtc/mediaengine.go b/pkg/rtc/mediaengine.go index de637e9df..1b01b8874 100644 --- a/pkg/rtc/mediaengine.go +++ b/pkg/rtc/mediaengine.go @@ -40,16 +40,14 @@ func registerCodecs(me *webrtc.MediaEngine, codecs []*livekit.Codec, rtcpFeedbac RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP8, ClockRate: 90000, RTCPFeedback: rtcpFeedback.Video}, PayloadType: 96, }, - /* - { - RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP9, ClockRate: 90000, SDPFmtpLine: "profile-id=0", RTCPFeedback: rtcpFeedback.Video}, - PayloadType: 98, - }, - { - RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP9, ClockRate: 90000, SDPFmtpLine: "profile-id=1", RTCPFeedback: rtcpFeedback.Video}, - PayloadType: 100, - }, - */ + { + RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP9, ClockRate: 90000, SDPFmtpLine: "profile-id=0", RTCPFeedback: rtcpFeedback.Video}, + PayloadType: 98, + }, + { + RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP9, ClockRate: 90000, SDPFmtpLine: "profile-id=1", RTCPFeedback: rtcpFeedback.Video}, + PayloadType: 100, + }, { RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264, ClockRate: 90000, SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f", RTCPFeedback: rtcpFeedback.Video}, PayloadType: 125, diff --git a/pkg/rtc/participant_sdp.go b/pkg/rtc/participant_sdp.go index 6ae4b8715..3edb71b9e 100644 --- a/pkg/rtc/participant_sdp.go +++ b/pkg/rtc/participant_sdp.go @@ -132,7 +132,7 @@ func (p *ParticipantImpl) setCodecPreferencesVideoForPublisher(offer webrtc.Sess mime = strings.ToUpper(mime) // remove dd extension if av1 not preferred - if !strings.Contains(mime, "AV1") { + if !strings.Contains(mime, "AV1") && !strings.Contains(mime, "VP9") { for i, attr := range unmatchVideo.Attributes { if strings.Contains(attr.Value, dd.ExtensionUrl) { unmatchVideo.Attributes[i] = unmatchVideo.Attributes[len(unmatchVideo.Attributes)-1] diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go index 5a04d66c1..1cfc7b2de 100644 --- a/pkg/sfu/buffer/buffer.go +++ b/pkg/sfu/buffer/buffer.go @@ -564,6 +564,8 @@ func (b *Buffer) getExtPacket(rtpPacket *rtp.Packet, arrivalTime int64) *ExtPack ep.KeyFrame = IsH264Keyframe(rtpPacket.Payload) case "video/av1": ep.KeyFrame = IsAV1Keyframe(rtpPacket.Payload) + case "video/vp9": + ep.KeyFrame = IsVP9Keyframe(rtpPacket.Payload) } if ep.KeyFrame { diff --git a/pkg/sfu/buffer/helpers.go b/pkg/sfu/buffer/helpers.go index cacc7ad1f..9545ab049 100644 --- a/pkg/sfu/buffer/helpers.go +++ b/pkg/sfu/buffer/helpers.go @@ -4,6 +4,8 @@ import ( "encoding/binary" "errors" + "github.com/pion/rtp/codecs" + "github.com/livekit/protocol/logger" ) @@ -351,4 +353,28 @@ func IsAV1Keyframe(payload []byte) bool { } } +// IsVP9Keyframe detects if vp9 payload is a keyframe +// taken from https://github.com/jech/galene/blob/master/codecs/codecs.go +// all credits belongs to Juliusz Chroboczek @jech and the awesome Galene SFU +func IsVP9Keyframe(payload []byte) bool { + var vp9 codecs.VP9Packet + _, err := vp9.Unmarshal(payload) + if err != nil || len(vp9.Payload) < 1 { + return false + } + if !vp9.B { + return false + } + + if (vp9.Payload[0] & 0xc0) != 0x80 { + return false + } + + profile := (vp9.Payload[0] >> 4) & 0x3 + if profile != 3 { + return (vp9.Payload[0] & 0xC) == 0 + } + return (vp9.Payload[0] & 0x6) == 0 +} + // ------------------------------------- diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 1ef7674f2..1104fb6d0 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -273,8 +273,8 @@ func (f *Forwarder) DetermineCodec(codec webrtc.RTPCodecCapability) { case "video/vp8": f.isTemporalSupported = true f.vp8Munger = NewVP8Munger(f.logger) - case "video/av1": - // TODO : we only enable dd layer selector for av1 now, at future we can + case "video/av1", "video/vp9": + // TODO : we only enable dd layer selector for av1 and vp9 now, at future we can // enable it for vp8 too f.ddLayerSelector = NewDDVideoLayerSelector(f.logger) } @@ -515,7 +515,7 @@ func (f *Forwarder) AllocateOptimal(availableLayers []int32, brs Bitrates, allow } alloc.TargetLayers = buffer.VideoLayer{ Spatial: int32(math.Min(float64(f.maxPublishedLayer), float64(maxSpatial))), - Temporal: buffer.DefaultMaxLayerTemporal, + Temporal: f.maxLayers.Temporal, } } @@ -570,14 +570,14 @@ func (f *Forwarder) AllocateOptimal(availableLayers []int32, brs Bitrates, allow alloc.TargetLayers.Spatial = l } } - alloc.TargetLayers.Temporal = buffer.DefaultMaxLayerTemporal + alloc.TargetLayers.Temporal = f.maxLayers.Temporal alloc.RequestLayerSpatial = alloc.TargetLayers.Spatial } else { requestLayerSpatial := int32(math.Min(float64(f.maxLayers.Spatial), float64(f.maxPublishedLayer))) if f.currentLayers.IsValid() && requestLayerSpatial == f.requestLayerSpatial && f.currentLayers.Spatial == f.requestLayerSpatial { // current is locked to desired, stay there - alloc.TargetLayers = f.currentLayers + alloc.TargetLayers = buffer.VideoLayer{Spatial: f.requestLayerSpatial, Temporal: f.maxLayers.Temporal} alloc.RequestLayerSpatial = f.requestLayerSpatial } else { // opportunistically latch on to anything diff --git a/pkg/sfu/forwarder_test.go b/pkg/sfu/forwarder_test.go index 81778adae..c543824ad 100644 --- a/pkg/sfu/forwarder_test.go +++ b/pkg/sfu/forwarder_test.go @@ -387,7 +387,7 @@ func TestForwarderAllocateOptimal(t *testing.T) { f.requestLayerSpatial = 0 expectedTargetLayers = buffer.VideoLayer{ Spatial: 2, - Temporal: 3, + Temporal: 1, } expectedResult = VideoAllocation{ PauseReason: VideoPauseReasonFeedDry, @@ -397,7 +397,7 @@ func TestForwarderAllocateOptimal(t *testing.T) { TargetLayers: expectedTargetLayers, RequestLayerSpatial: 2, MaxLayers: f.maxLayers, - DistanceToDesired: -1.5, + DistanceToDesired: -1, } result = f.AllocateOptimal([]int32{0, 1}, emptyBitrates, true) require.Equal(t, expectedResult, result) From 57b931e9bd0dce9227d96509a736d4035b0fb7ec Mon Sep 17 00:00:00 2001 From: David Zhao Date: Fri, 7 Apr 2023 19:54:30 -0700 Subject: [PATCH 062/324] Fix return code when no panics have occurred (#1589) Actually fix #1513 --- cmd/server/main.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/server/main.go b/cmd/server/main.go index 994e71b34..31abfaae2 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -103,8 +103,9 @@ func init() { func main() { defer func() { - rtc.Recover(logger.GetLogger()) - os.Exit(1) + if rtc.Recover(logger.GetLogger()) != nil { + os.Exit(1) + } }() generatedFlags, err := config.GenerateCLIFlags(baseFlags, true) From e32eaa451fa4441ba6b9990cdb42b5eba3bed379 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sat, 8 Apr 2023 10:57:57 +0530 Subject: [PATCH 063/324] Refactor video layer selector (#1588) * WIP commit * WIP commit * fix test * FPS for VP9 * WIP commit * test changes * WIP commit * h264 * codec munger * forwarder state * clean up a bit * dd interface * WIP commit * WIP commit * WIP commit * WIP commit * more TODO notes * overshoot interface * clean up * clean up isTemporalSupported * wait for key frame to resume * clean up VP8 payload descriptor stuff * temporal layer selector * comment out vp9 and av1 * space * fix test compile * append bytes * fix tests * fix test --- pkg/rtc/participant_sdp.go | 4 +- pkg/sfu/buffer/buffer.go | 37 +- pkg/sfu/buffer/fps.go | 162 +++- pkg/sfu/buffer/helpers.go | 209 ++++-- pkg/sfu/buffer/helpers_test.go | 2 +- pkg/sfu/codecmunger/codecmunger.go | 25 + pkg/sfu/codecmunger/null.go | 34 + pkg/sfu/{vp8munger.go => codecmunger/vp8.go} | 221 +++--- .../vp8_test.go} | 498 ++++++------- pkg/sfu/downtrack.go | 95 +-- pkg/sfu/forwarder.go | 699 ++++++++---------- pkg/sfu/forwarder_test.go | 625 +++++++++------- pkg/sfu/sequencer.go | 38 +- pkg/sfu/sequencer_test.go | 81 -- pkg/sfu/streamallocator/streamallocator.go | 18 +- pkg/sfu/testutils/data.go | 10 +- pkg/sfu/videolayerselector.go | 202 ----- pkg/sfu/videolayerselector/base.go | 120 +++ .../dependencydescriptor.go | 278 +++++++ pkg/sfu/videolayerselector/null.go | 15 + pkg/sfu/videolayerselector/simulcast.go | 143 ++++ .../temporallayerselector/null.go | 17 + .../temporallayerselector.go | 7 + .../temporallayerselector/vp8.go | 42 ++ .../videolayerselector/videolayerselector.go | 46 ++ pkg/sfu/videolayerselector/vp9.go | 102 +++ 26 files changed, 2183 insertions(+), 1547 deletions(-) create mode 100644 pkg/sfu/codecmunger/codecmunger.go create mode 100644 pkg/sfu/codecmunger/null.go rename pkg/sfu/{vp8munger.go => codecmunger/vp8.go} (71%) rename pkg/sfu/{vp8munger_test.go => codecmunger/vp8_test.go} (51%) delete mode 100644 pkg/sfu/videolayerselector.go create mode 100644 pkg/sfu/videolayerselector/base.go create mode 100644 pkg/sfu/videolayerselector/dependencydescriptor.go create mode 100644 pkg/sfu/videolayerselector/null.go create mode 100644 pkg/sfu/videolayerselector/simulcast.go create mode 100644 pkg/sfu/videolayerselector/temporallayerselector/null.go create mode 100644 pkg/sfu/videolayerselector/temporallayerselector/temporallayerselector.go create mode 100644 pkg/sfu/videolayerselector/temporallayerselector/vp8.go create mode 100644 pkg/sfu/videolayerselector/videolayerselector.go create mode 100644 pkg/sfu/videolayerselector/vp9.go diff --git a/pkg/rtc/participant_sdp.go b/pkg/rtc/participant_sdp.go index 3edb71b9e..524a487fb 100644 --- a/pkg/rtc/participant_sdp.go +++ b/pkg/rtc/participant_sdp.go @@ -131,8 +131,8 @@ func (p *ParticipantImpl) setCodecPreferencesVideoForPublisher(offer webrtc.Sess p.pendingTracksLock.RUnlock() mime = strings.ToUpper(mime) - // remove dd extension if av1 not preferred - if !strings.Contains(mime, "AV1") && !strings.Contains(mime, "VP9") { + // remove dd extension if av1/vp9 not preferred + if !strings.Contains(strings.ToLower(mime), "av1") && !strings.Contains(strings.ToLower(mime), "vp9") { for i, attr := range unmatchVideo.Attributes { if strings.Contains(attr.Value, dd.ExtensionUrl) { unmatchVideo.Attributes[i] = unmatchVideo.Attributes[len(unmatchVideo.Attributes)-1] diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go index 1cfc7b2de..26fb22aca 100644 --- a/pkg/sfu/buffer/buffer.go +++ b/pkg/sfu/buffer/buffer.go @@ -10,6 +10,7 @@ import ( "github.com/gammazero/deque" "github.com/pion/rtcp" "github.com/pion/rtp" + "github.com/pion/rtp/codecs" "github.com/pion/sdp/v3" "github.com/pion/webrtc/v3" "go.uber.org/atomic" @@ -193,8 +194,17 @@ func (b *Buffer) Bind(params webrtc.RTPParameters, codec webrtc.RTPCodecCapabili case strings.HasPrefix(b.mime, "video/"): b.codecType = webrtc.RTPCodecTypeVideo b.bucket = bucket.NewBucket(b.videoPool.Get().(*[]byte)) - if b.frameRateCalculator[0] == nil && strings.EqualFold(codec.MimeType, webrtc.MimeTypeVP8) { - b.frameRateCalculator[0] = NewFrameRateCalculatorVP8(b.clockRate, b.logger) + if b.frameRateCalculator[0] == nil { + if strings.EqualFold(codec.MimeType, webrtc.MimeTypeVP8) { + b.frameRateCalculator[0] = NewFrameRateCalculatorVP8(b.clockRate, b.logger) + } + + if strings.EqualFold(codec.MimeType, webrtc.MimeTypeVP9) { + frc := NewFrameRateCalculatorVP9(b.clockRate, b.logger) + for i := range b.frameRateCalculator { + b.frameRateCalculator[i] = frc.GetFrameRateCalculatorForSpatial(int32(i)) + } + } } default: @@ -560,12 +570,25 @@ func (b *Buffer) getExtPacket(rtpPacket *rtp.Packet, arrivalTime int64) *ExtPack ep.Spatial = InvalidLayerSpatial // vp8 don't have spatial scalability, reset to -1 } ep.Payload = vp8Packet - case "video/h264": - ep.KeyFrame = IsH264Keyframe(rtpPacket.Payload) - case "video/av1": - ep.KeyFrame = IsAV1Keyframe(rtpPacket.Payload) case "video/vp9": - ep.KeyFrame = IsVP9Keyframe(rtpPacket.Payload) + if ep.DependencyDescriptor == nil { + var vp9Packet codecs.VP9Packet + _, err := vp9Packet.Unmarshal(rtpPacket.Payload) + if err != nil { + b.logger.Warnw("could not unmarshal VP9 packet", err) + return nil + } + ep.VideoLayer = VideoLayer{ + Spatial: int32(vp9Packet.SID), + Temporal: int32(vp9Packet.TID), + } + ep.Payload = vp9Packet + } + ep.KeyFrame = IsVP9KeyFrame(rtpPacket.Payload) + case "video/h264": + ep.KeyFrame = IsH264KeyFrame(rtpPacket.Payload) + case "video/av1": + ep.KeyFrame = IsAV1KeyFrame(rtpPacket.Payload) } if ep.KeyFrame { diff --git a/pkg/sfu/buffer/fps.go b/pkg/sfu/buffer/fps.go index ddd6fb467..f8f192227 100644 --- a/pkg/sfu/buffer/fps.go +++ b/pkg/sfu/buffer/fps.go @@ -4,6 +4,7 @@ import ( "container/list" "github.com/livekit/protocol/logger" + "github.com/pion/rtp/codecs" ) var minFramesForCalculation = [DefaultMaxLayerTemporal + 1]int{8, 15, 40} @@ -24,8 +25,9 @@ type FrameRateCalculator interface { } // ----------------------------- -// FrameRateCalculator based on PictureID in VP8 -type FrameRateCalculatorVP8 struct { + +// FrameRateCalculator based on PictureID in VPx +type frameRateCalculatorVPx struct { frameRates [DefaultMaxLayerTemporal + 1]float32 clockRate uint32 logger logger.Logger @@ -36,27 +38,21 @@ type FrameRateCalculatorVP8 struct { completed bool } -func NewFrameRateCalculatorVP8(clockRate uint32, logger logger.Logger) *FrameRateCalculatorVP8 { - return &FrameRateCalculatorVP8{ +func newFrameRateCalculatorVPx(clockRate uint32, logger logger.Logger) *frameRateCalculatorVPx { + return &frameRateCalculatorVPx{ clockRate: clockRate, logger: logger, } } -func (f *FrameRateCalculatorVP8) Completed() bool { +func (f *frameRateCalculatorVPx) Completed() bool { return f.completed } -func (f *FrameRateCalculatorVP8) RecvPacket(ep *ExtPacket) bool { +func (f *frameRateCalculatorVPx) RecvPacket(ep *ExtPacket, fn uint16) bool { if f.completed { return true } - vp8, ok := ep.Payload.(VP8) - if !ok { - f.logger.Debugw("no vp8 payload", "sn", ep.Packet.SequenceNumber) - return false - } - fn := vp8.PictureID if ep.Temporal >= int32(len(f.frameRates)) { f.logger.Warnw("invalid temporal layer", nil, "temporal", ep.Temporal) @@ -113,7 +109,7 @@ func (f *FrameRateCalculatorVP8) RecvPacket(ep *ExtPacket) bool { return f.calc() } -func (f *FrameRateCalculatorVP8) calc() bool { +func (f *frameRateCalculatorVPx) calc() bool { var rateCounter int for currentTemporal := int32(0); currentTemporal <= DefaultMaxLayerTemporal; currentTemporal++ { if f.frameRates[currentTemporal] > 0 { @@ -156,14 +152,13 @@ func (f *FrameRateCalculatorVP8) calc() bool { if f.frameRates[2] > 0 && f.frameRates[2] > f.frameRates[1]*3 { f.frameRates[1] = f.frameRates[2] / 2 } - f.logger.Debugw("frame rate calculated", "rate", f.frameRates) f.reset() return true } return false } -func (f *FrameRateCalculatorVP8) reset() { +func (f *frameRateCalculatorVPx) reset() { for i := range f.firstFrames { f.firstFrames[i] = nil f.secondFrames[i] = nil @@ -175,20 +170,145 @@ func (f *FrameRateCalculatorVP8) reset() { f.baseFrame = nil } -func (f *FrameRateCalculatorVP8) GetFrameRate() []float32 { +func (f *frameRateCalculatorVPx) GetFrameRate() []float32 { return f.frameRates[:] } // ----------------------------- -// FrameRateCalculator based on Dependency descriptor +// FrameRateCalculator based on PictureID in VP8 +type FrameRateCalculatorVP8 struct { + *frameRateCalculatorVPx + logger logger.Logger +} + +func NewFrameRateCalculatorVP8(clockRate uint32, logger logger.Logger) *FrameRateCalculatorVP8 { + return &FrameRateCalculatorVP8{ + frameRateCalculatorVPx: newFrameRateCalculatorVPx(clockRate, logger), + logger: logger, + } +} + +func (f *FrameRateCalculatorVP8) RecvPacket(ep *ExtPacket) bool { + if f.frameRateCalculatorVPx.Completed() { + return true + } + + vp8, ok := ep.Payload.(VP8) + if !ok { + f.logger.Debugw("no vp8 payload", "sn", ep.Packet.SequenceNumber) + return false + } + success := f.frameRateCalculatorVPx.RecvPacket(ep, vp8.PictureID) + + if f.frameRateCalculatorVPx.Completed() { + f.logger.Debugw("frame rate calculated", "rate", f.frameRateCalculatorVPx.GetFrameRate()) + } + + return success +} + +// ----------------------------- + +// FrameRateCalculator based on PictureID in VP9 +type FrameRateCalculatorVP9 struct { + logger logger.Logger + completed bool + + // VP9-TODO - this is assuming three spatial layers. As `completed` marker relies on all layers being finished, have to assume this. FIX. + // Maybe look at number of layers in livekit.TrackInfo and declare completed once advertised layers are measured + frameRateCalculatorsVPx [DefaultMaxLayerSpatial + 1]*frameRateCalculatorVPx +} + +func NewFrameRateCalculatorVP9(clockRate uint32, logger logger.Logger) *FrameRateCalculatorVP9 { + f := &FrameRateCalculatorVP9{ + logger: logger, + } + + for i := range f.frameRateCalculatorsVPx { + f.frameRateCalculatorsVPx[i] = newFrameRateCalculatorVPx(clockRate, logger) + } + + return f +} + +func (f *FrameRateCalculatorVP9) Completed() bool { + return f.completed +} + +func (f *FrameRateCalculatorVP9) RecvPacket(ep *ExtPacket) bool { + if f.completed { + return true + } + + vp9, ok := ep.Payload.(codecs.VP9Packet) + if !ok { + f.logger.Debugw("no vp9 payload", "sn", ep.Packet.SequenceNumber) + return false + } + + if ep.Spatial < 0 || ep.Spatial >= int32(len(f.frameRateCalculatorsVPx)) || f.frameRateCalculatorsVPx[ep.Spatial] == nil { + f.logger.Debugw("invalid spatial layer", "sn", ep.Packet.SequenceNumber, "spatial", ep.Spatial) + return false + } + + success := f.frameRateCalculatorsVPx[ep.Spatial].RecvPacket(ep, vp9.PictureID) + + completed := true + for _, frc := range f.frameRateCalculatorsVPx { + if !frc.Completed() { + completed = false + break + } + } + + if completed { + f.completed = true + + var frameRates [DefaultMaxLayerSpatial + 1][]float32 + for i := range f.frameRateCalculatorsVPx { + frameRates[i] = f.frameRateCalculatorsVPx[i].GetFrameRate() + } + f.logger.Debugw("frame rate calculated", "rate", frameRates) + } + + return success +} + +func (f *FrameRateCalculatorVP9) GetFrameRateForSpatial(spatial int32) []float32 { + if spatial < 0 || spatial >= int32(len(f.frameRateCalculatorsVPx)) || f.frameRateCalculatorsVPx[spatial] == nil { + return nil + } + return f.frameRateCalculatorsVPx[spatial].GetFrameRate() +} + +func (f *FrameRateCalculatorVP9) GetFrameRateCalculatorForSpatial(spatial int32) *FrameRateCalculatorForVP9Layer { + return &FrameRateCalculatorForVP9Layer{ + FrameRateCalculatorVP9: f, + spatial: spatial, + } +} + +// ----------------------------- + +type FrameRateCalculatorForVP9Layer struct { + *FrameRateCalculatorVP9 + spatial int32 +} + +func (f *FrameRateCalculatorForVP9Layer) GetFrameRate() []float32 { + return f.FrameRateCalculatorVP9.GetFrameRateForSpatial(f.spatial) +} + +// ----------------------------------------------- + +// FrameRateCalculator based on Dependency descriptor type FrameRateCalculatorDD struct { frameRates [DefaultMaxLayerSpatial + 1][DefaultMaxLayerTemporal + 1]float32 clockRate uint32 logger logger.Logger firstFrames [DefaultMaxLayerSpatial + 1][DefaultMaxLayerTemporal + 1]*frameInfo secondFrames [DefaultMaxLayerSpatial + 1][DefaultMaxLayerTemporal + 1]*frameInfo - spatial int fnReceived [256]*frameInfo baseFrame *frameInfo completed bool @@ -385,7 +505,7 @@ func (f *FrameRateCalculatorDD) calc() bool { f.completed = true f.close() - f.logger.Debugw("frame rate calculated", "spatial", f.spatial, "rate", f.frameRates) + f.logger.Debugw("frame rate calculated", "rate", f.frameRates) return true } return false @@ -424,6 +544,8 @@ func (f *FrameRateCalculatorDD) GetFrameRateCalculatorForSpatial(spatial int32) } } +// ----------------------------------------------- + type FrameRateCalculatorForDDLayer struct { *FrameRateCalculatorDD spatial int32 @@ -432,3 +554,5 @@ type FrameRateCalculatorForDDLayer struct { func (f *FrameRateCalculatorForDDLayer) GetFrameRate() []float32 { return f.FrameRateCalculatorDD.GetFrameRateForSpatial(f.spatial) } + +// ----------------------------------------------- diff --git a/pkg/sfu/buffer/helpers.go b/pkg/sfu/buffer/helpers.go index 9545ab049..01bb33403 100644 --- a/pkg/sfu/buffer/helpers.go +++ b/pkg/sfu/buffer/helpers.go @@ -4,8 +4,6 @@ import ( "encoding/binary" "errors" - "github.com/pion/rtp/codecs" - "github.com/livekit/protocol/logger" ) @@ -35,22 +33,23 @@ var ( */ type VP8 struct { FirstByte byte + S bool - PictureIDPresent int - PictureID uint16 /* 8 or 16 bits, picture ID */ - MBit bool + I bool + M bool + PictureID uint16 /* 8 or 16 bits, picture ID */ - TL0PICIDXPresent int - TL0PICIDX uint8 /* 8 bits temporal level zero index */ + L bool + TL0PICIDX uint8 /* 8 bits temporal level zero index */ // Optional Header If either of the T or K bits are set to 1, // the TID/Y/KEYIDX extension field MUST be present. - TIDPresent int - TID uint8 /* 2 bits temporal layer idx */ - Y uint8 + T bool + TID uint8 /* 2 bits temporal layer idx */ + Y bool - KEYIDXPresent int - KEYIDX uint8 /* 5 bits of key frame idx */ + K bool + KEYIDX uint8 /* 5 bits of key frame idx */ HeaderSize int @@ -65,96 +64,94 @@ func (v *VP8) Unmarshal(payload []byte) error { } payloadLen := len(payload) - if payloadLen < 1 { return errShortPacket } idx := 0 v.FirstByte = payload[idx] - S := payload[idx]&0x10 > 0 + v.S = payload[idx]&0x10 > 0 // Check for extended bit control if payload[idx]&0x80 > 0 { idx++ if payloadLen < idx+1 { return errShortPacket } - I := payload[idx]&0x80 > 0 - L := payload[idx]&0x40 > 0 - T := payload[idx]&0x20 > 0 - K := payload[idx]&0x10 > 0 - if L && !T { + v.I = payload[idx]&0x80 > 0 + v.L = payload[idx]&0x40 > 0 + v.T = payload[idx]&0x20 > 0 + v.K = payload[idx]&0x10 > 0 + if v.L && !v.T { return errInvalidPacket } - // Check for PictureID - if I { + + if v.I { idx++ if payloadLen < idx+1 { return errShortPacket } - v.PictureIDPresent = 1 pid := payload[idx] & 0x7f - // Check if m is 1, then Picture ID is 15 bits - if payload[idx]&0x80 > 0 { + // if m is 1, then Picture ID is 15 bits + v.M = payload[idx]&0x80 > 0 + if v.M { idx++ if payloadLen < idx+1 { return errShortPacket } - v.MBit = true v.PictureID = binary.BigEndian.Uint16([]byte{pid, payload[idx]}) } else { v.PictureID = uint16(pid) } } - // Check if TL0PICIDX is present - if L { + + if v.L { idx++ if payloadLen < idx+1 { return errShortPacket } - v.TL0PICIDXPresent = 1 - - if idx >= payloadLen { - return errShortPacket - } v.TL0PICIDX = payload[idx] } - if T || K { + + if v.T || v.K { idx++ if payloadLen < idx+1 { return errShortPacket } - if T { - v.TIDPresent = 1 + + if v.T { v.TID = (payload[idx] & 0xc0) >> 6 - v.Y = (payload[idx] & 0x20) >> 5 + v.Y = (payload[idx] & 0x20) > 0 } - if K { - v.KEYIDXPresent = 1 + + if v.K { v.KEYIDX = payload[idx] & 0x1f } } - if idx >= payloadLen { - return errShortPacket - } idx++ if payloadLen < idx+1 { return errShortPacket } + // Check is packet is a keyframe by looking at P bit in vp8 payload - v.IsKeyFrame = payload[idx]&0x01 == 0 && S + v.IsKeyFrame = payload[idx]&0x01 == 0 && v.S } else { idx++ if payloadLen < idx+1 { return errShortPacket } // Check is packet is a keyframe by looking at P bit in vp8 payload - v.IsKeyFrame = payload[idx]&0x01 == 0 && S + v.IsKeyFrame = payload[idx]&0x01 == 0 && v.S } v.HeaderSize = idx return nil } +func (v *VP8) Marshal() ([]byte, error) { + buf := make([]byte, v.HeaderSize) + err := v.MarshalTo(buf) + return buf, err +} + func (v *VP8) MarshalTo(buf []byte) error { if len(buf) < v.HeaderSize { return errShortPacket @@ -162,13 +159,17 @@ func (v *VP8) MarshalTo(buf []byte) error { idx := 0 buf[idx] = v.FirstByte - if (v.PictureIDPresent + v.TL0PICIDXPresent + v.TIDPresent + v.KEYIDXPresent) != 0 { + if v.I || v.L || v.T || v.K { buf[idx] |= 0x80 // X bit idx++ - buf[idx] = byte(v.PictureIDPresent<<7) | byte(v.TL0PICIDXPresent<<6) | byte(v.TIDPresent<<5) | byte(v.KEYIDXPresent<<4) + + xpos := idx + xval := byte(0) + idx++ - if v.PictureIDPresent == 1 { - if v.MBit { + if v.I { + xval |= (1 << 7) + if v.M { buf[idx] = 0x80 | byte((v.PictureID>>8)&0x7f) buf[idx+1] = byte(v.PictureID & 0xff) idx += 2 @@ -177,20 +178,31 @@ func (v *VP8) MarshalTo(buf []byte) error { idx++ } } - if v.TL0PICIDXPresent == 1 { + + if v.L { + xval |= (1 << 6) buf[idx] = v.TL0PICIDX idx++ } - if v.TIDPresent == 1 || v.KEYIDXPresent == 1 { + + if v.T || v.K { buf[idx] = 0 - if v.TIDPresent == 1 { - buf[idx] = v.TID<<6 | v.Y<<5 + if v.T { + xval |= (1 << 5) + buf[idx] = v.TID << 6 + if v.Y { + buf[idx] |= (1 << 5) + } } - if v.KEYIDXPresent == 1 { + + if v.K { + xval |= (1 << 4) buf[idx] |= v.KEYIDX & 0x1f } idx++ } + + buf[xpos] = xval } else { buf[idx] &^= 0x80 // X bit idx++ @@ -199,7 +211,9 @@ func (v *VP8) MarshalTo(buf []byte) error { return nil } -func VP8PictureIdSizeDiff(mBit1 bool, mBit2 bool) int { +// ------------------------------------- + +func VPxPictureIdSizeDiff(mBit1 bool, mBit2 bool) int { if mBit1 == mBit2 { return 0 } @@ -211,10 +225,12 @@ func VP8PictureIdSizeDiff(mBit1 bool, mBit2 bool) int { return -1 } -// IsH264Keyframe detects if h264 payload is a keyframe +// ------------------------------------- + +// IsH264KeyFrame detects if h264 payload is a keyframe // this code was taken from https://github.com/jech/galene/blob/codecs/rtpconn/rtpreader.go#L45 // all credits belongs to Juliusz Chroboczek @jech and the awesome Galene SFU -func IsH264Keyframe(payload []byte) bool { +func IsH264KeyFrame(payload []byte) bool { if len(payload) < 1 { return false } @@ -278,10 +294,65 @@ func IsH264Keyframe(payload []byte) bool { return false } -// IsAV1Keyframe detects if av1 payload is a keyframe +// ------------------------------------- + +func IsVP9KeyFrame(payload []byte) bool { + payloadLen := len(payload) + if payloadLen < 1 { + return false + } + + idx := 0 + I := payload[idx]&0x80 > 0 + P := payload[idx]&0x40 > 0 + L := payload[idx]&0x20 > 0 + F := payload[idx]&0x10 > 0 + B := payload[idx]&0x08 > 0 + + if F && !I { + return false + } + + // Check for PictureID + if I { + idx++ + if payloadLen < idx+1 { + return false + } + // Check if m is 1, then Picture ID is 15 bits + if payload[idx]&0x80 > 0 { + idx++ + if payloadLen < idx+1 { + return false + } + } + } + + // Check if TL0PICIDX is present + sid := -1 + if L { + idx++ + if payloadLen < idx+1 { + return false + } + + tid := (payload[idx] >> 5) & 0x7 + if !P && tid != 0 { + return false + } + + sid = int((payload[idx] >> 1) & 0x7) + } + + return !P && (!L || (L && sid == 0)) && B +} + +// ------------------------------------- + +// IsAV1KeyFrame detects if av1 payload is a keyframe // taken from https://github.com/jech/galene/blob/master/codecs/codecs.go // all credits belongs to Juliusz Chroboczek @jech and the awesome Galene SFU -func IsAV1Keyframe(payload []byte) bool { +func IsAV1KeyFrame(payload []byte) bool { if len(payload) < 2 { return false } @@ -353,28 +424,4 @@ func IsAV1Keyframe(payload []byte) bool { } } -// IsVP9Keyframe detects if vp9 payload is a keyframe -// taken from https://github.com/jech/galene/blob/master/codecs/codecs.go -// all credits belongs to Juliusz Chroboczek @jech and the awesome Galene SFU -func IsVP9Keyframe(payload []byte) bool { - var vp9 codecs.VP9Packet - _, err := vp9.Unmarshal(payload) - if err != nil || len(vp9.Payload) < 1 { - return false - } - if !vp9.B { - return false - } - - if (vp9.Payload[0] & 0xc0) != 0x80 { - return false - } - - profile := (vp9.Payload[0] >> 4) & 0x3 - if profile != 3 { - return (vp9.Payload[0] & 0xC) == 0 - } - return (vp9.Payload[0] & 0x6) == 0 -} - // ------------------------------------- diff --git a/pkg/sfu/buffer/helpers_test.go b/pkg/sfu/buffer/helpers_test.go index d52bce5c5..6ce0ad860 100644 --- a/pkg/sfu/buffer/helpers_test.go +++ b/pkg/sfu/buffer/helpers_test.go @@ -75,7 +75,7 @@ func TestVP8Helper_Unmarshal(t *testing.T) { t.Errorf("Unmarshal() error = %v, wantErr %v", err, tt.wantErr) } if tt.checkTemporal { - require.Equal(t, tt.temporalSupport, p.TIDPresent == 1) + require.Equal(t, tt.temporalSupport, p.T) } if tt.checkKeyFrame { require.Equal(t, tt.keyFrame, p.IsKeyFrame) diff --git a/pkg/sfu/codecmunger/codecmunger.go b/pkg/sfu/codecmunger/codecmunger.go new file mode 100644 index 000000000..eec4f2437 --- /dev/null +++ b/pkg/sfu/codecmunger/codecmunger.go @@ -0,0 +1,25 @@ +package codecmunger + +import ( + "errors" + + "github.com/livekit/livekit-server/pkg/sfu/buffer" +) + +var ( + ErrNotVP8 = errors.New("not VP8") + ErrOutOfOrderVP8PictureIdCacheMiss = errors.New("out-of-order VP8 picture id not found in cache") + ErrFilteredVP8TemporalLayer = errors.New("filtered VP8 temporal layer") +) + +type CodecMunger interface { + GetState() interface{} + SeedState(state interface{}) + + SetLast(extPkt *buffer.ExtPacket) + UpdateOffsets(extPkt *buffer.ExtPacket) + + UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap bool, maxTemporal int32) ([]byte, error) + + UpdateAndGetPadding(newPicture bool) ([]byte, error) +} diff --git a/pkg/sfu/codecmunger/null.go b/pkg/sfu/codecmunger/null.go new file mode 100644 index 000000000..e6b3f00cb --- /dev/null +++ b/pkg/sfu/codecmunger/null.go @@ -0,0 +1,34 @@ +package codecmunger + +import ( + "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/protocol/logger" +) + +type Null struct { +} + +func NewNull(_logger logger.Logger) *Null { + return &Null{} +} + +func (n *Null) GetState() interface{} { + return nil +} + +func (n *Null) SeedState(_state interface{}) { +} + +func (n *Null) SetLast(_extPkt *buffer.ExtPacket) { +} + +func (n *Null) UpdateOffsets(_extPkt *buffer.ExtPacket) { +} + +func (n *Null) UpdateAndGet(_extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap bool, maxTemporal int32) ([]byte, error) { + return nil, nil +} + +func (n *Null) UpdateAndGetPadding(newPicture bool) ([]byte, error) { + return nil, nil +} diff --git a/pkg/sfu/vp8munger.go b/pkg/sfu/codecmunger/vp8.go similarity index 71% rename from pkg/sfu/vp8munger.go rename to pkg/sfu/codecmunger/vp8.go index a631a6237..271dac71e 100644 --- a/pkg/sfu/vp8munger.go +++ b/pkg/sfu/codecmunger/vp8.go @@ -1,4 +1,4 @@ -package sfu +package codecmunger import ( "fmt" @@ -16,67 +16,56 @@ const ( exemptedPictureIdsThreshold = 20 ) -// VP8 munger -type TranslationParamsVP8 struct { - Header *buffer.VP8 -} - // ----------------------------------------------------------- -type VP8MungerState struct { +type VP8State struct { ExtLastPictureId int32 - PictureIdUsed int + PictureIdUsed bool LastTl0PicIdx uint8 - Tl0PicIdxUsed int - TidUsed int + Tl0PicIdxUsed bool + TidUsed bool LastKeyIdx uint8 - KeyIdxUsed int + KeyIdxUsed bool } -func (v VP8MungerState) String() string { - return fmt.Sprintf("VP8MungerState{extLastPictureId: %d, pictureIdUsed: %+v, lastTl0PicIdx: %d, tl0PicIdxUsed: %+v, tidUsed: %+v, lastKeyIdx: %d, keyIdxUsed: %+v)", +func (v VP8State) String() string { + return fmt.Sprintf("VP8State{extLastPictureId: %d, pictureIdUsed: %+v, lastTl0PicIdx: %d, tl0PicIdxUsed: %+v, tidUsed: %+v, lastKeyIdx: %d, keyIdxUsed: %+v)", v.ExtLastPictureId, v.PictureIdUsed, v.LastTl0PicIdx, v.Tl0PicIdxUsed, v.TidUsed, v.LastKeyIdx, v.KeyIdxUsed) } // ----------------------------------------------------------- -type VP8MungerParams struct { +type VP8 struct { + logger logger.Logger + pictureIdWrapHandler VP8PictureIdWrapHandler extLastPictureId int32 pictureIdOffset int32 - pictureIdUsed int + pictureIdUsed bool lastTl0PicIdx uint8 tl0PicIdxOffset uint8 - tl0PicIdxUsed int - tidUsed int + tl0PicIdxUsed bool + tidUsed bool lastKeyIdx uint8 keyIdxOffset uint8 - keyIdxUsed int + keyIdxUsed bool missingPictureIds *orderedmap.OrderedMap[int32, int32] droppedPictureIds *orderedmap.OrderedMap[int32, bool] exemptedPictureIds *orderedmap.OrderedMap[int32, bool] } -type VP8Munger struct { - logger logger.Logger - - VP8MungerParams -} - -func NewVP8Munger(logger logger.Logger) *VP8Munger { - return &VP8Munger{ - logger: logger, - VP8MungerParams: VP8MungerParams{ - missingPictureIds: orderedmap.NewOrderedMap[int32, int32](), - droppedPictureIds: orderedmap.NewOrderedMap[int32, bool](), - exemptedPictureIds: orderedmap.NewOrderedMap[int32, bool](), - }, +func NewVP8(logger logger.Logger) *VP8 { + return &VP8{ + logger: logger, + missingPictureIds: orderedmap.NewOrderedMap[int32, int32](), + droppedPictureIds: orderedmap.NewOrderedMap[int32, bool](), + exemptedPictureIds: orderedmap.NewOrderedMap[int32, bool](), } } -func (v *VP8Munger) GetLast() VP8MungerState { - return VP8MungerState{ +func (v *VP8) GetState() interface{} { + return VP8State{ ExtLastPictureId: v.extLastPictureId, PictureIdUsed: v.pictureIdUsed, LastTl0PicIdx: v.lastTl0PicIdx, @@ -87,57 +76,59 @@ func (v *VP8Munger) GetLast() VP8MungerState { } } -func (v *VP8Munger) SeedLast(state VP8MungerState) { - v.extLastPictureId = state.ExtLastPictureId - v.pictureIdUsed = state.PictureIdUsed - v.lastTl0PicIdx = state.LastTl0PicIdx - v.tl0PicIdxUsed = state.Tl0PicIdxUsed - v.tidUsed = state.TidUsed - v.lastKeyIdx = state.LastKeyIdx - v.keyIdxUsed = state.KeyIdxUsed +func (v *VP8) SeedState(seed interface{}) { + if state, ok := seed.(VP8State); ok { + v.extLastPictureId = state.ExtLastPictureId + v.pictureIdUsed = state.PictureIdUsed + v.lastTl0PicIdx = state.LastTl0PicIdx + v.tl0PicIdxUsed = state.Tl0PicIdxUsed + v.tidUsed = state.TidUsed + v.lastKeyIdx = state.LastKeyIdx + v.keyIdxUsed = state.KeyIdxUsed + } } -func (v *VP8Munger) SetLast(extPkt *buffer.ExtPacket) { +func (v *VP8) SetLast(extPkt *buffer.ExtPacket) { vp8, ok := extPkt.Payload.(buffer.VP8) if !ok { return } - v.pictureIdUsed = vp8.PictureIDPresent - if v.pictureIdUsed == 1 { - v.pictureIdWrapHandler.Init(int32(vp8.PictureID)-1, vp8.MBit) + v.pictureIdUsed = vp8.I + if v.pictureIdUsed { + v.pictureIdWrapHandler.Init(int32(vp8.PictureID)-1, vp8.M) v.extLastPictureId = int32(vp8.PictureID) } - v.tl0PicIdxUsed = vp8.TL0PICIDXPresent - if v.tl0PicIdxUsed == 1 { + v.tl0PicIdxUsed = vp8.L + if v.tl0PicIdxUsed { v.lastTl0PicIdx = vp8.TL0PICIDX } - v.tidUsed = vp8.TIDPresent + v.tidUsed = vp8.T - v.keyIdxUsed = vp8.KEYIDXPresent - if v.keyIdxUsed == 1 { + v.keyIdxUsed = vp8.K + if v.keyIdxUsed { v.lastKeyIdx = vp8.KEYIDX } } -func (v *VP8Munger) UpdateOffsets(extPkt *buffer.ExtPacket) { +func (v *VP8) UpdateOffsets(extPkt *buffer.ExtPacket) { vp8, ok := extPkt.Payload.(buffer.VP8) if !ok { return } - if v.pictureIdUsed == 1 { - v.pictureIdWrapHandler.Init(int32(vp8.PictureID)-1, vp8.MBit) + if v.pictureIdUsed { + v.pictureIdWrapHandler.Init(int32(vp8.PictureID)-1, vp8.M) v.pictureIdOffset = int32(vp8.PictureID) - v.extLastPictureId - 1 } - if v.tl0PicIdxUsed == 1 { + if v.tl0PicIdxUsed { v.tl0PicIdxOffset = vp8.TL0PICIDX - v.lastTl0PicIdx - 1 } - if v.keyIdxUsed == 1 { + if v.keyIdxUsed { v.keyIdxOffset = (vp8.KEYIDX - v.lastKeyIdx - 1) & 0x1f } @@ -147,16 +138,16 @@ func (v *VP8Munger) UpdateOffsets(extPkt *buffer.ExtPacket) { v.exemptedPictureIds = orderedmap.NewOrderedMap[int32, bool]() } -func (v *VP8Munger) UpdateAndGet(extPkt *buffer.ExtPacket, ordering SequenceNumberOrdering, maxTemporalLayer int32) (*TranslationParamsVP8, error) { +func (v *VP8) UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap bool, maxTemporalLayer int32) ([]byte, error) { vp8, ok := extPkt.Payload.(buffer.VP8) if !ok { return nil, ErrNotVP8 } - extPictureId := v.pictureIdWrapHandler.Unwrap(vp8.PictureID, vp8.MBit) + extPictureId := v.pictureIdWrapHandler.Unwrap(vp8.PictureID, vp8.M) // if out-of-order, look up missing picture id cache - if ordering == SequenceNumberOrderingOutOfOrder { + if snOutOfOrder { pictureIdOffset, ok := v.missingPictureIds.Get(extPictureId) if !ok { return nil, ErrOutOfOrderVP8PictureIdCacheMiss @@ -170,27 +161,25 @@ func (v *VP8Munger) UpdateAndGet(extPkt *buffer.ExtPacket, ordering SequenceNumb mungedPictureId := uint16((extPictureId - pictureIdOffset) & 0x7fff) vp8Packet := &buffer.VP8{ - FirstByte: vp8.FirstByte, - PictureIDPresent: vp8.PictureIDPresent, - PictureID: mungedPictureId, - MBit: mungedPictureId > 127, - TL0PICIDXPresent: vp8.TL0PICIDXPresent, - TL0PICIDX: vp8.TL0PICIDX - v.tl0PicIdxOffset, - TIDPresent: vp8.TIDPresent, - TID: vp8.TID, - Y: vp8.Y, - KEYIDXPresent: vp8.KEYIDXPresent, - KEYIDX: vp8.KEYIDX - v.keyIdxOffset, - IsKeyFrame: vp8.IsKeyFrame, - HeaderSize: vp8.HeaderSize + buffer.VP8PictureIdSizeDiff(mungedPictureId > 127, vp8.MBit), + FirstByte: vp8.FirstByte, + I: vp8.I, + M: mungedPictureId > 127, + PictureID: mungedPictureId, + L: vp8.L, + TL0PICIDX: vp8.TL0PICIDX - v.tl0PicIdxOffset, + T: vp8.T, + TID: vp8.TID, + Y: vp8.Y, + K: vp8.K, + KEYIDX: vp8.KEYIDX - v.keyIdxOffset, + IsKeyFrame: vp8.IsKeyFrame, + HeaderSize: vp8.HeaderSize + buffer.VPxPictureIdSizeDiff(mungedPictureId > 127, vp8.M), } - return &TranslationParamsVP8{ - Header: vp8Packet, - }, nil + return vp8Packet.Marshal() } prevMaxPictureId := v.pictureIdWrapHandler.MaxPictureId() - v.pictureIdWrapHandler.UpdateMaxPictureId(extPictureId, vp8.MBit) + v.pictureIdWrapHandler.UpdateMaxPictureId(extPictureId, vp8.M) // if there is a gap in sequence number, record possible pictures that // the missing packets can belong to in missing picture id cache. @@ -205,7 +194,7 @@ func (v *VP8Munger) UpdateAndGet(extPkt *buffer.ExtPacket, ordering SequenceNumb // it is possible to deduce that (for example by looking at previous packet's RTP marker // and check if that was the last packet of Picture 10), it could get complicated when // the gap is larger. - if ordering == SequenceNumberOrderingGap { + if snHasGap { for lostPictureId := prevMaxPictureId; lostPictureId <= extPictureId; lostPictureId++ { // Record missing only if picture id was not dropped. This is to avoid a subsequent packet of dropped frame going through. // A sequence like this @@ -229,7 +218,7 @@ func (v *VP8Munger) UpdateAndGet(extPkt *buffer.ExtPacket, ordering SequenceNumb // which layer the missing packets belong to. A layer could have multiple packets. So, keep track // of pictures that are forwarded even though they will be filterd out based on temporal layer // requirements. That allows forwarding of the complete picture. - if vp8.TIDPresent == 1 && vp8.TID > uint8(maxTemporalLayer) { + if vp8.T && vp8.TID > uint8(maxTemporalLayer) { v.exemptedPictureIds.Set(extPictureId, true) // trim cache if necessary for v.exemptedPictureIds.Len() > exemptedPictureIdsThreshold { @@ -238,12 +227,12 @@ func (v *VP8Munger) UpdateAndGet(extPkt *buffer.ExtPacket, ordering SequenceNumb } } } else { - if vp8.TIDPresent == 1 && vp8.TID > uint8(maxTemporalLayer) { + if vp8.T && vp8.TID > uint8(maxTemporalLayer) { // drop only if not exempted _, ok := v.exemptedPictureIds.Get(extPictureId) if !ok { // adjust only once per picture as a picture could have multiple packets - if vp8.PictureIDPresent == 1 && prevMaxPictureId != extPictureId { + if vp8.I && prevMaxPictureId != extPictureId { // keep track of dropped picture ids so that they do not get into the missing picture cache v.droppedPictureIds.Set(extPictureId, true) // trim cache if necessary @@ -275,38 +264,36 @@ func (v *VP8Munger) UpdateAndGet(extPkt *buffer.ExtPacket, ordering SequenceNumb v.lastKeyIdx = mungedKeyIdx vp8Packet := &buffer.VP8{ - FirstByte: vp8.FirstByte, - PictureIDPresent: vp8.PictureIDPresent, - PictureID: mungedPictureId, - MBit: mungedPictureId > 127, - TL0PICIDXPresent: vp8.TL0PICIDXPresent, - TL0PICIDX: mungedTl0PicIdx, - TIDPresent: vp8.TIDPresent, - TID: vp8.TID, - Y: vp8.Y, - KEYIDXPresent: vp8.KEYIDXPresent, - KEYIDX: mungedKeyIdx, - IsKeyFrame: vp8.IsKeyFrame, - HeaderSize: vp8.HeaderSize + buffer.VP8PictureIdSizeDiff(mungedPictureId > 127, vp8.MBit), + FirstByte: vp8.FirstByte, + I: vp8.I, + M: mungedPictureId > 127, + PictureID: mungedPictureId, + L: vp8.L, + TL0PICIDX: mungedTl0PicIdx, + T: vp8.T, + TID: vp8.TID, + Y: vp8.Y, + K: vp8.K, + KEYIDX: mungedKeyIdx, + IsKeyFrame: vp8.IsKeyFrame, + HeaderSize: vp8.HeaderSize + buffer.VPxPictureIdSizeDiff(mungedPictureId > 127, vp8.M), } - return &TranslationParamsVP8{ - Header: vp8Packet, - }, nil + return vp8Packet.Marshal() } -func (v *VP8Munger) UpdateAndGetPadding(newPicture bool) *buffer.VP8 { +func (v *VP8) UpdateAndGetPadding(newPicture bool) ([]byte, error) { offset := 0 if newPicture { offset = 1 } headerSize := 1 - if (v.pictureIdUsed + v.tl0PicIdxUsed + v.tidUsed + v.keyIdxUsed) != 0 { + if v.pictureIdUsed || v.tl0PicIdxUsed || v.tidUsed || v.keyIdxUsed { headerSize += 1 } extPictureId := v.extLastPictureId - if v.pictureIdUsed == 1 { + if v.pictureIdUsed { extPictureId = v.extLastPictureId + int32(offset) v.extLastPictureId = extPictureId v.pictureIdOffset -= int32(offset) @@ -319,44 +306,44 @@ func (v *VP8Munger) UpdateAndGetPadding(newPicture bool) *buffer.VP8 { pictureId := uint16(extPictureId & 0x7fff) tl0PicIdx := uint8(0) - if v.tl0PicIdxUsed == 1 { + if v.tl0PicIdxUsed { tl0PicIdx = v.lastTl0PicIdx + uint8(offset) v.lastTl0PicIdx = tl0PicIdx v.tl0PicIdxOffset -= uint8(offset) headerSize += 1 } - if (v.tidUsed + v.keyIdxUsed) != 0 { + if v.tidUsed || v.keyIdxUsed { headerSize += 1 } keyIdx := uint8(0) - if v.keyIdxUsed == 1 { + if v.keyIdxUsed { keyIdx = (v.lastKeyIdx + uint8(offset)) & 0x1f v.lastKeyIdx = keyIdx v.keyIdxOffset -= uint8(offset) } vp8Packet := &buffer.VP8{ - FirstByte: 0x10, // partition 0, start of VP8 Partition, reference frame - PictureIDPresent: v.pictureIdUsed, - PictureID: pictureId, - MBit: pictureId > 127, - TL0PICIDXPresent: v.tl0PicIdxUsed, - TL0PICIDX: tl0PicIdx, - TIDPresent: v.tidUsed, - TID: 0, - Y: 1, - KEYIDXPresent: v.keyIdxUsed, - KEYIDX: keyIdx, - IsKeyFrame: true, - HeaderSize: headerSize, + FirstByte: 0x10, // partition 0, start of VP8 Partition, reference frame + I: v.pictureIdUsed, + M: pictureId > 127, + PictureID: pictureId, + L: v.tl0PicIdxUsed, + TL0PICIDX: tl0PicIdx, + T: v.tidUsed, + TID: 0, + Y: true, + K: v.keyIdxUsed, + KEYIDX: keyIdx, + IsKeyFrame: true, + HeaderSize: headerSize, } - return vp8Packet + return vp8Packet.Marshal() } // for testing only -func (v *VP8Munger) PictureIdOffset(extPictureId int32) (int32, bool) { +func (v *VP8) PictureIdOffset(extPictureId int32) (int32, bool) { return v.missingPictureIds.Get(extPictureId) } diff --git a/pkg/sfu/vp8munger_test.go b/pkg/sfu/codecmunger/vp8_test.go similarity index 51% rename from pkg/sfu/vp8munger_test.go rename to pkg/sfu/codecmunger/vp8_test.go index 0b32fc4db..93c27086c 100644 --- a/pkg/sfu/vp8munger_test.go +++ b/pkg/sfu/codecmunger/vp8_test.go @@ -1,4 +1,4 @@ -package sfu +package codecmunger import ( "reflect" @@ -12,7 +12,7 @@ import ( "github.com/livekit/livekit-server/pkg/sfu/testutils" ) -func compare(expected *VP8Munger, actual *VP8Munger) bool { +func compare(expected *VP8, actual *VP8) bool { return reflect.DeepEqual(expected.pictureIdWrapHandler, actual.pictureIdWrapHandler) && expected.extLastPictureId == actual.extLastPictureId && expected.pictureIdOffset == actual.pictureIdOffset && @@ -26,12 +26,12 @@ func compare(expected *VP8Munger, actual *VP8Munger) bool { expected.keyIdxUsed == actual.keyIdxUsed } -func newVP8Munger() *VP8Munger { - return NewVP8Munger(logger.GetLogger()) +func newVP8() *VP8 { + return NewVP8(logger.GetLogger()) } func TestSetLast(t *testing.T) { - v := newVP8Munger() + v := newVP8() params := &testutils.TestExtPacketParams{ SequenceNumber: 23333, @@ -39,51 +39,49 @@ func TestSetLast(t *testing.T) { SSRC: 0x12345678, } vp8 := &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 13467, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 13, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: true, + FirstByte: 25, + I: true, + M: true, + PictureID: 13467, + L: true, + TL0PICIDX: 233, + T: true, + TID: 13, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: true, } extPkt, err := testutils.GetTestExtPacketVP8(params, vp8) require.NoError(t, err) require.NotNil(t, extPkt) - expectedVP8Munger := VP8Munger{ - VP8MungerParams: VP8MungerParams{ - pictureIdWrapHandler: VP8PictureIdWrapHandler{ - maxPictureId: 13466, - maxMBit: true, - totalWrap: 0, - lastWrap: 0, - }, - extLastPictureId: 13467, - pictureIdOffset: 0, - pictureIdUsed: 1, - lastTl0PicIdx: 233, - tl0PicIdxOffset: 0, - tl0PicIdxUsed: 1, - tidUsed: 1, - lastKeyIdx: 23, - keyIdxOffset: 0, - keyIdxUsed: 1, + expectedVP8 := VP8{ + pictureIdWrapHandler: VP8PictureIdWrapHandler{ + maxPictureId: 13466, + maxMBit: true, + totalWrap: 0, + lastWrap: 0, }, + extLastPictureId: 13467, + pictureIdOffset: 0, + pictureIdUsed: true, + lastTl0PicIdx: 233, + tl0PicIdxOffset: 0, + tl0PicIdxUsed: true, + tidUsed: true, + lastKeyIdx: 23, + keyIdxOffset: 0, + keyIdxUsed: true, } v.SetLast(extPkt) - require.True(t, compare(&expectedVP8Munger, v)) + require.True(t, compare(&expectedVP8, v)) } func TestUpdateOffsets(t *testing.T) { - v := newVP8Munger() + v := newVP8() params := &testutils.TestExtPacketParams{ SequenceNumber: 23333, @@ -91,19 +89,19 @@ func TestUpdateOffsets(t *testing.T) { SSRC: 0x12345678, } vp8 := &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 13467, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 13, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: true, + FirstByte: 25, + I: true, + M: true, + PictureID: 13467, + L: true, + TL0PICIDX: 233, + T: true, + TID: 13, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: true, } extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8) v.SetLast(extPkt) @@ -114,48 +112,46 @@ func TestUpdateOffsets(t *testing.T) { SSRC: 0x87654321, } vp8 = &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 345, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 12, - TIDPresent: 1, - TID: 13, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 4, - HeaderSize: 6, - IsKeyFrame: true, + FirstByte: 25, + I: true, + M: true, + PictureID: 345, + L: true, + TL0PICIDX: 12, + T: true, + TID: 13, + Y: true, + K: true, + KEYIDX: 4, + HeaderSize: 6, + IsKeyFrame: true, } extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) v.UpdateOffsets(extPkt) - expectedVP8Munger := VP8Munger{ - VP8MungerParams: VP8MungerParams{ - pictureIdWrapHandler: VP8PictureIdWrapHandler{ - maxPictureId: 344, - maxMBit: true, - totalWrap: 0, - lastWrap: 0, - }, - extLastPictureId: 13467, - pictureIdOffset: 345 - 13467 - 1, - pictureIdUsed: 1, - lastTl0PicIdx: 233, - tl0PicIdxOffset: (12 - 233 - 1) & 0xff, - tl0PicIdxUsed: 1, - tidUsed: 1, - lastKeyIdx: 23, - keyIdxOffset: (4 - 23 - 1) & 0x1f, - keyIdxUsed: 1, + expectedVP8 := VP8{ + pictureIdWrapHandler: VP8PictureIdWrapHandler{ + maxPictureId: 344, + maxMBit: true, + totalWrap: 0, + lastWrap: 0, }, + extLastPictureId: 13467, + pictureIdOffset: 345 - 13467 - 1, + pictureIdUsed: true, + lastTl0PicIdx: 233, + tl0PicIdxOffset: (12 - 233 - 1) & 0xff, + tl0PicIdxUsed: true, + tidUsed: true, + lastKeyIdx: 23, + keyIdxOffset: (4 - 23 - 1) & 0x1f, + keyIdxUsed: true, } - require.True(t, compare(&expectedVP8Munger, v)) + require.True(t, compare(&expectedVP8, v)) } func TestOutOfOrderPictureId(t *testing.T) { - v := newVP8Munger() + v := newVP8() params := &testutils.TestExtPacketParams{ SequenceNumber: 23333, @@ -163,58 +159,57 @@ func TestOutOfOrderPictureId(t *testing.T) { SSRC: 0x12345678, } vp8 := &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 13467, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 1, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: true, + FirstByte: 25, + I: true, + M: true, + PictureID: 13467, + L: true, + TL0PICIDX: 233, + T: true, + TID: 1, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: true, } extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8) v.SetLast(extPkt) - v.UpdateAndGet(extPkt, SequenceNumberOrderingContiguous, 2) + v.UpdateAndGet(extPkt, false, false, 2) // out-of-order sequence number not in the missing picture id cache vp8.PictureID = 13466 extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) - tp, err := v.UpdateAndGet(extPkt, SequenceNumberOrderingOutOfOrder, 2) + codecBytes, err := v.UpdateAndGet(extPkt, true, false, 2) require.Error(t, err) require.ErrorIs(t, err, ErrOutOfOrderVP8PictureIdCacheMiss) - require.Nil(t, tp) + require.Nil(t, codecBytes) // create a hole in picture id vp8.PictureID = 13469 extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) - tpExpected := TranslationParamsVP8{ - Header: &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 13469, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 1, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: true, - }, + expectedVP8 := &buffer.VP8{ + FirstByte: 25, + I: true, + M: true, + PictureID: 13469, + L: true, + TL0PICIDX: 233, + T: true, + TID: 1, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: true, } - tp, err = v.UpdateAndGet(extPkt, SequenceNumberOrderingGap, 2) + marshalledVP8, err := expectedVP8.Marshal() require.NoError(t, err) - require.NotNil(t, tp) - require.Equal(t, tpExpected, *tp) + codecBytes, err = v.UpdateAndGet(extPkt, false, true, 2) + require.NoError(t, err) + require.Equal(t, marshalledVP8, codecBytes) // all three, the last, the current and the in-between should have been added to missing picture id cache value, ok := v.PictureIdOffset(13467) @@ -233,31 +228,30 @@ func TestOutOfOrderPictureId(t *testing.T) { vp8.PictureID = 13468 extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) - tpExpected = TranslationParamsVP8{ - Header: &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 13468, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 1, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: true, - }, + expectedVP8 = &buffer.VP8{ + FirstByte: 25, + I: true, + M: true, + PictureID: 13468, + L: true, + TL0PICIDX: 233, + T: true, + TID: 1, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: true, } - tp, err = v.UpdateAndGet(extPkt, SequenceNumberOrderingOutOfOrder, 2) + marshalledVP8, err = expectedVP8.Marshal() require.NoError(t, err) - require.NotNil(t, tp) - require.Equal(t, tpExpected, *tp) + codecBytes, err = v.UpdateAndGet(extPkt, true, false, 2) + require.NoError(t, err) + require.Equal(t, marshalledVP8, codecBytes) } func TestTemporalLayerFiltering(t *testing.T) { - v := newVP8Munger() + v := newVP8() params := &testutils.TestExtPacketParams{ SequenceNumber: 23333, @@ -265,25 +259,25 @@ func TestTemporalLayerFiltering(t *testing.T) { SSRC: 0x12345678, } vp8 := &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 13467, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 1, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: true, + FirstByte: 25, + I: true, + M: true, + PictureID: 13467, + L: true, + TL0PICIDX: 233, + T: true, + TID: 1, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: true, } extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8) v.SetLast(extPkt) // translate - tp, err := v.UpdateAndGet(extPkt, SequenceNumberOrderingContiguous, 0) + tp, err := v.UpdateAndGet(extPkt, false, false, 0) require.Error(t, err) require.ErrorIs(t, err, ErrFilteredVP8TemporalLayer) require.Nil(t, tp) @@ -296,7 +290,7 @@ func TestTemporalLayerFiltering(t *testing.T) { params.SequenceNumber = 23334 extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) - tp, err = v.UpdateAndGet(extPkt, SequenceNumberOrderingContiguous, 0) + tp, err = v.UpdateAndGet(extPkt, false, false, 0) require.Error(t, err) require.ErrorIs(t, err, ErrFilteredVP8TemporalLayer) require.Nil(t, tp) @@ -309,7 +303,7 @@ func TestTemporalLayerFiltering(t *testing.T) { params.SequenceNumber = 23337 extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) - tp, err = v.UpdateAndGet(extPkt, SequenceNumberOrderingContiguous, 0) + tp, err = v.UpdateAndGet(extPkt, false, false, 0) require.Error(t, err) require.ErrorIs(t, err, ErrFilteredVP8TemporalLayer) require.Nil(t, tp) @@ -319,7 +313,7 @@ func TestTemporalLayerFiltering(t *testing.T) { } func TestGapInSequenceNumberSamePicture(t *testing.T) { - v := newVP8Munger() + v := newVP8() params := &testutils.TestExtPacketParams{ SequenceNumber: 65533, @@ -328,65 +322,65 @@ func TestGapInSequenceNumberSamePicture(t *testing.T) { PayloadSize: 33, } vp8 := &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 13467, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 1, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: true, + FirstByte: 25, + I: true, + M: true, + PictureID: 13467, + L: true, + TL0PICIDX: 233, + T: true, + TID: 1, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: true, } extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8) v.SetLast(extPkt) - tpExpected := TranslationParamsVP8{ - Header: &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 13467, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 1, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: true, - }, + expectedVP8 := &buffer.VP8{ + FirstByte: 25, + I: true, + M: true, + PictureID: 13467, + L: true, + TL0PICIDX: 233, + T: true, + TID: 1, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: true, } - tp, err := v.UpdateAndGet(extPkt, SequenceNumberOrderingContiguous, 2) + marshalledVP8, err := expectedVP8.Marshal() require.NoError(t, err) - require.Equal(t, tpExpected, *tp) + codecBytes, err := v.UpdateAndGet(extPkt, false, false, 2) + require.NoError(t, err) + require.Equal(t, marshalledVP8, codecBytes) // telling there is a gap in sequence number will add pictures to missing picture cache - tpExpected = TranslationParamsVP8{ - Header: &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 13467, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 1, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: true, - }, + expectedVP8 = &buffer.VP8{ + FirstByte: 25, + I: true, + M: true, + PictureID: 13467, + L: true, + TL0PICIDX: 233, + T: true, + TID: 1, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: true, } - tp, err = v.UpdateAndGet(extPkt, SequenceNumberOrderingGap, 2) + marshalledVP8, err = expectedVP8.Marshal() require.NoError(t, err) - require.Equal(t, tpExpected, *tp) + codecBytes, err = v.UpdateAndGet(extPkt, false, true, 2) + require.NoError(t, err) + require.Equal(t, marshalledVP8, codecBytes) value, ok := v.PictureIdOffset(13467) require.True(t, ok) @@ -394,7 +388,7 @@ func TestGapInSequenceNumberSamePicture(t *testing.T) { } func TestUpdateAndGetPadding(t *testing.T) { - v := newVP8Munger() + v := newVP8() params := &testutils.TestExtPacketParams{ SequenceNumber: 23333, @@ -403,61 +397,67 @@ func TestUpdateAndGetPadding(t *testing.T) { PayloadSize: 20, } vp8 := &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 13467, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 13, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: true, + FirstByte: 25, + I: true, + M: true, + PictureID: 13467, + L: true, + TL0PICIDX: 233, + T: true, + TID: 13, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: true, } extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8) v.SetLast(extPkt) // getting padding with repeat of last picture - blankVP8 := v.UpdateAndGetPadding(false) + blankBytes, err := v.UpdateAndGetPadding(false) + require.NoError(t, err) expectedVP8 := buffer.VP8{ - FirstByte: 16, - PictureIDPresent: 1, - PictureID: 13467, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 0, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: true, + FirstByte: 16, + I: true, + M: true, + PictureID: 13467, + L: true, + TL0PICIDX: 233, + T: true, + TID: 0, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: true, } - require.Equal(t, expectedVP8, *blankVP8) + marshalledVP8, err := expectedVP8.Marshal() + require.NoError(t, err) + require.Equal(t, marshalledVP8, blankBytes) // getting padding with new picture - blankVP8 = v.UpdateAndGetPadding(true) + blankBytes, err = v.UpdateAndGetPadding(true) + require.NoError(t, err) expectedVP8 = buffer.VP8{ - FirstByte: 16, - PictureIDPresent: 1, - PictureID: 13468, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 234, - TIDPresent: 1, - TID: 0, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 24, - HeaderSize: 6, - IsKeyFrame: true, + FirstByte: 16, + I: true, + M: true, + PictureID: 13468, + L: true, + TL0PICIDX: 234, + T: true, + TID: 0, + Y: true, + K: true, + KEYIDX: 24, + HeaderSize: 6, + IsKeyFrame: true, } - require.Equal(t, expectedVP8, *blankVP8) + marshalledVP8, err = expectedVP8.Marshal() + require.NoError(t, err) + require.Equal(t, marshalledVP8, blankBytes) } func TestVP8PictureIdWrapHandler(t *testing.T) { diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 0fee2e8ca..3c75064fc 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -65,11 +65,7 @@ var ( ErrPaddingOnlyPacket = errors.New("padding only packet that need not be forwarded") ErrDuplicatePacket = errors.New("duplicate packet") ErrPaddingNotOnFrameBoundary = errors.New("padding cannot send on non-frame boundary") - ErrNotVP8 = errors.New("not VP8") - ErrOutOfOrderVP8PictureIdCacheMiss = errors.New("out-of-order VP8 picture id not found in cache") - ErrFilteredVP8TemporalLayer = errors.New("filtered VP8 temporal layer") ErrDownTrackAlreadyBound = errors.New("already bound") - ErrDownTrackClosed = errors.New("downtrack closed") ) var ( @@ -143,8 +139,8 @@ type DownTrackStreamAllocatorListener interface { // subscribed max video layer changed OnSubscribedLayersChanged(dt *DownTrack, layers buffer.VideoLayer) - // target video layer reached - OnTargetLayerReached(dt *DownTrack) + // stream resumed + OnResume(dt *DownTrack) // packet(s) sent OnPacketsSent(dt *DownTrack, size int) @@ -209,8 +205,7 @@ type DownTrack struct { connectionStats *connectionquality.ConnectionStats deltaStatsSnapshotId uint32 - // Debug info - pktsDropped atomic.Uint32 + // for throttling error logs writeIOErrors atomic.Uint32 isNACKThrottled atomic.Bool @@ -342,13 +337,14 @@ func (d *DownTrack) Bind(t webrtc.TrackLocalContext) (webrtc.RTPCodecParameters, } d.codec = codec.RTPCodecCapability - d.forwarder.DetermineCodec(d.codec) if d.onBinding != nil { d.onBinding() } d.bound.Store(true) d.bindLock.Unlock() + d.forwarder.DetermineCodec(d.codec, d.receiver.HeaderExtensions()) + d.logger.Debugw("downtrack bound") d.onBindAndConnected() @@ -551,9 +547,6 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { tp, err := d.forwarder.GetTranslationParams(extPkt, layer) if tp.shouldDrop { - if tp.isDroppingRelevant { - d.pktsDropped.Inc() - } if err != nil { d.logger.Errorw("write rtp packet failed", err) } @@ -561,39 +554,32 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { } payload := extPkt.Packet.Payload - if tp.vp8 != nil { + if len(tp.codecBytes) != 0 { incomingVP8, _ := extPkt.Payload.(buffer.VP8) pool = PacketFactory.Get().(*[]byte) - payload, err = d.translateVP8PacketTo(extPkt.Packet, &incomingVP8, tp.vp8.Header, pool) - if err != nil { - d.pktsDropped.Inc() - d.logger.Errorw("write rtp packet failed", err) - return err - } + payload = d.translateVP8PacketTo(extPkt.Packet, &incomingVP8, tp.codecBytes, pool) } var meta *packetMeta if d.sequencer != nil { meta = d.sequencer.push(extPkt.Packet.SequenceNumber, tp.rtp.sequenceNumber, tp.rtp.timestamp, int8(layer)) - if meta != nil && tp.vp8 != nil { - meta.packVP8(tp.vp8.Header) + if meta != nil { + meta.codecBytes = append(meta.codecBytes, tp.codecBytes...) } } hdr, err := d.getTranslatedRTPHeader(extPkt, tp) if err != nil { - d.pktsDropped.Inc() d.logger.Errorw("write rtp packet failed", err) return err } if meta != nil && d.dependencyDescriptorID != 0 { - meta.ddBytes = hdr.GetExtension(uint8(d.dependencyDescriptorID)) + meta.ddBytes = append(meta.ddBytes, tp.ddBytes...) } _, err = d.writeStream.WriteRTP(hdr, payload) if err != nil { - d.pktsDropped.Inc() if errors.Is(err, io.ErrClosedPipe) { writeIOErrors := d.writeIOErrors.Inc() if (writeIOErrors % 100) == 1 { @@ -611,22 +597,23 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { d.onMaxSubscribedLayerChanged(d, layer) } - if extPkt.KeyFrame || tp.isSwitchingToTargetLayer { + if extPkt.KeyFrame { d.isNACKThrottled.Store(false) if extPkt.KeyFrame { d.rtpStats.UpdateKeyFrame(1) d.logger.Debugw("forwarding key frame", "layer", layer) } + // SVC-TODO - no need for key frame always when using SVC locked, _ := d.forwarder.CheckSync() if locked { d.stopKeyFrameRequester() } + } - if tp.isSwitchingToTargetLayer { - if sal := d.getStreamAllocatorListener(); sal != nil { - sal.OnTargetLayerReached(d) - } + if tp.isResuming { + if sal := d.getStreamAllocatorListener(); sal != nil { + sal.OnResume(d) } } @@ -1222,20 +1209,18 @@ func (d *DownTrack) writeOpusRedBlankFrame(hdr *rtp.Header, frameEndNeeded bool) } func (d *DownTrack) writeVP8BlankFrame(hdr *rtp.Header, frameEndNeeded bool) (int, error) { - blankVP8 := d.forwarder.GetPaddingVP8(frameEndNeeded) + blankVP8, err := d.forwarder.GetPadding(frameEndNeeded) + if err != nil { + return 0, err + } // 8x8 key frame // Used even when closing out a previous frame. Looks like receivers // do not care about content (it will probably end up being an undecodable // frame, but that should be okay as there are key frames following) - payload := make([]byte, blankVP8.HeaderSize+len(VP8KeyFrame8x8)) - vp8Header := payload[:blankVP8.HeaderSize] - err := blankVP8.MarshalTo(vp8Header) - if err != nil { - return 0, err - } - - copy(payload[blankVP8.HeaderSize:], VP8KeyFrame8x8) + payload := make([]byte, len(blankVP8)+len(VP8KeyFrame8x8)) + copy(payload[:len(blankVP8)], blankVP8) + copy(payload[len(blankVP8):], VP8KeyFrame8x8) _, err = d.writeStream.WriteRTP(hdr, payload) if err == nil { @@ -1451,13 +1436,9 @@ func (d *DownTrack) retransmitPackets(nacks []uint16) { continue } - translatedVP8 := meta.unpackVP8() + translatedVP8 := meta.codecBytes pool = PacketFactory.Get().(*[]byte) - payload, err = d.translateVP8PacketTo(&pkt, &incomingVP8, translatedVP8, pool) - if err != nil { - d.logger.Errorw("translating VP8 packet err", err) - continue - } + payload = d.translateVP8PacketTo(&pkt, &incomingVP8, translatedVP8, pool) } var extraExtensions []extensionData @@ -1533,16 +1514,11 @@ func (d *DownTrack) getTranslatedRTPHeader(extPkt *buffer.ExtPacket, tp *Transla } var extension []extensionData - if d.dependencyDescriptorID != 0 && tp.ddExtension != nil { - bytes, err := tp.ddExtension.Marshal() - if err != nil { - d.logger.Warnw("error marshalling dependency descriptor extension", err) - } else { - extension = append(extension, extensionData{ - id: uint8(d.dependencyDescriptorID), - payload: bytes, - }) - } + if d.dependencyDescriptorID != 0 && len(tp.ddBytes) != 0 { + extension = append(extension, extensionData{ + id: uint8(d.dependencyDescriptorID), + payload: tp.ddBytes, + }) } err := d.writeRTPHeaderExtensions(&hdr, extension...) if err != nil { @@ -1552,14 +1528,14 @@ func (d *DownTrack) getTranslatedRTPHeader(extPkt *buffer.ExtPacket, tp *Transla return &hdr, nil } -func (d *DownTrack) translateVP8PacketTo(pkt *rtp.Packet, incomingVP8 *buffer.VP8, translatedVP8 *buffer.VP8, outbuf *[]byte) ([]byte, error) { - buf := (*outbuf)[:len(pkt.Payload)+translatedVP8.HeaderSize-incomingVP8.HeaderSize] +func (d *DownTrack) translateVP8PacketTo(pkt *rtp.Packet, incomingVP8 *buffer.VP8, translatedVP8 []byte, outbuf *[]byte) []byte { + buf := (*outbuf)[:len(pkt.Payload)+len(translatedVP8)-incomingVP8.HeaderSize] srcPayload := pkt.Payload[incomingVP8.HeaderSize:] - dstPayload := buf[translatedVP8.HeaderSize:] + dstPayload := buf[len(translatedVP8):] copy(dstPayload, srcPayload) - err := translatedVP8.MarshalTo(buf[:translatedVP8.HeaderSize]) - return buf, err + copy(buf[:len(translatedVP8)], translatedVP8) + return buf } func (d *DownTrack) DebugInfo() map[string]interface{} { @@ -1572,7 +1548,6 @@ func (d *DownTrack) DebugInfo() map[string]interface{} { "TSOffset": rtpMungerParams.tsOffset, "LastMarker": rtpMungerParams.lastMarker, "LastPli": d.rtpStats.LastPli(), - "PacketsDropped": d.pktsDropped.Load(), } senderReport := d.CreateSenderReport() diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 1104fb6d0..f29ad906f 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -12,7 +12,10 @@ import ( "github.com/livekit/protocol/logger" "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/livekit-server/pkg/sfu/codecmunger" dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + "github.com/livekit/livekit-server/pkg/sfu/videolayerselector" + "github.com/livekit/livekit-server/pkg/sfu/videolayerselector/temporallayerselector" ) // Forwarder @@ -94,16 +97,15 @@ var ( // ------------------------------------------------------------------- type VideoAllocationProvisional struct { - muted bool - pubMuted bool - maxPublishedLayer int32 - maxTemporalLayerSeen int32 - availableLayers []int32 - Bitrates Bitrates - maxLayers buffer.VideoLayer - currentLayers buffer.VideoLayer - parkedLayers buffer.VideoLayer - allocatedLayers buffer.VideoLayer + muted bool + pubMuted bool + maxSeenLayer buffer.VideoLayer + availableLayers []int32 + Bitrates Bitrates + maxLayers buffer.VideoLayer + currentLayers buffer.VideoLayer + parkedLayers buffer.VideoLayer + allocatedLayers buffer.VideoLayer } // ------------------------------------------------------------------- @@ -122,17 +124,12 @@ func (v VideoTransition) String() string { type TranslationParams struct { shouldDrop bool - isDroppingRelevant bool + isResuming bool isSwitchingToMaxLayer bool rtp *TranslationParamsRTP - vp8 *TranslationParamsVP8 - ddExtension *dd.DependencyDescriptorExtension + codecBytes []byte + ddBytes []byte marker bool - - // indicates this frame has 'Switch' decode indication for target layer - // TODO : in theory, we need check frame chain is not broken for the target - // but we don't have frame queue now, so just use decode target indication - isSwitchingToTargetLayer bool } // ------------------------------------------------------------------- @@ -140,11 +137,16 @@ type TranslationParams struct { type ForwarderState struct { Started bool RTP RTPMungerState - VP8 VP8MungerState + Codec interface{} } func (f ForwarderState) String() string { - return fmt.Sprintf("ForwarderState{started: %v, rtp: %s, vp8: %s}", f.Started, f.RTP.String(), f.VP8.String()) + codecString := "" + switch codecState := f.Codec.(type) { + case codecmunger.VP8State: + codecString = codecState.String() + } + return fmt.Sprintf("ForwarderState{started: %v, rtp: %s, codec: %s}", f.Started, f.RTP.String(), codecString) } // ------------------------------------------------------------------- @@ -159,30 +161,21 @@ type Forwarder struct { muted bool pubMuted bool - maxPublishedLayer int32 - maxTemporalLayerSeen int32 - started bool lastSSRC uint32 referenceLayerSpatial int32 - maxLayers buffer.VideoLayer - currentLayers buffer.VideoLayer - targetLayers buffer.VideoLayer - requestLayerSpatial int32 - parkedLayers buffer.VideoLayer // layers that can resume without key frame - parkedLayersTimer *time.Timer + parkedLayersTimer *time.Timer provisional *VideoAllocationProvisional lastAllocation VideoAllocation rtpMunger *RTPMunger - vp8Munger *VP8Munger - isTemporalSupported bool + vls videolayerselector.VideoLayerSelector - ddLayerSelector *DDVideoLayerSelector + codecMunger codecmunger.CodecMunger onParkedLayersExpired func() } @@ -196,29 +189,16 @@ func NewForwarder( kind: kind, logger: logger, getReferenceLayerRTPTimestamp: getReferenceLayerRTPTimestamp, - - maxPublishedLayer: buffer.InvalidLayerSpatial, - maxTemporalLayerSeen: buffer.InvalidLayerTemporal, - - referenceLayerSpatial: buffer.InvalidLayerSpatial, - - // start off with nothing, let streamallocator/opportunistic forwarder set the target - currentLayers: buffer.InvalidLayers, - targetLayers: buffer.InvalidLayers, - requestLayerSpatial: buffer.InvalidLayerSpatial, - parkedLayers: buffer.InvalidLayers, - - lastAllocation: VideoAllocationDefault, - - rtpMunger: NewRTPMunger(logger), + referenceLayerSpatial: buffer.InvalidLayerSpatial, + lastAllocation: VideoAllocationDefault, + rtpMunger: NewRTPMunger(logger), + vls: videolayerselector.NewNull(logger), + codecMunger: codecmunger.NewNull(logger), } if f.kind == webrtc.RTPCodecTypeVideo { - f.maxLayers = buffer.VideoLayer{Spatial: buffer.InvalidLayerSpatial, Temporal: buffer.DefaultMaxLayerTemporal} - } else { - f.maxLayers = buffer.InvalidLayers + f.vls.SetMaxTemporal(buffer.DefaultMaxLayerTemporal) } - return f } @@ -226,24 +206,26 @@ func (f *Forwarder) SetMaxPublishedLayer(maxPublishedLayer int32) { f.lock.Lock() defer f.lock.Unlock() - if maxPublishedLayer <= f.maxPublishedLayer { + existingMaxSeen := f.vls.GetMaxSeen() + if maxPublishedLayer <= existingMaxSeen.Spatial { return } - f.maxPublishedLayer = maxPublishedLayer - f.logger.Debugw("setting max published layer", "maxPublishedLayer", f.maxPublishedLayer) + f.vls.SetMaxSeenSpatial(maxPublishedLayer) + f.logger.Debugw("setting max published layer", "maxPublishedLayer", maxPublishedLayer) } func (f *Forwarder) SetMaxTemporalLayerSeen(maxTemporalLayerSeen int32) { f.lock.Lock() defer f.lock.Unlock() - if maxTemporalLayerSeen <= f.maxTemporalLayerSeen { + existingMaxSeen := f.vls.GetMaxSeen() + if maxTemporalLayerSeen <= existingMaxSeen.Temporal { return } - f.maxTemporalLayerSeen = maxTemporalLayerSeen - f.logger.Debugw("setting max temporal layer seen", "maxTemporalLayerSeen", f.maxTemporalLayerSeen) + f.vls.SetMaxSeenTemporal(maxTemporalLayerSeen) + f.logger.Debugw("setting max temporal layer seen", "maxTemporalLayerSeen", maxTemporalLayerSeen) } func (f *Forwarder) OnParkedLayersExpired(fn func()) { @@ -260,7 +242,7 @@ func (f *Forwarder) getOnParkedLayersExpired() func() { return f.onParkedLayersExpired } -func (f *Forwarder) DetermineCodec(codec webrtc.RTPCodecCapability) { +func (f *Forwarder) DetermineCodec(codec webrtc.RTPCodecCapability, extensions []webrtc.RTPHeaderExtensionParameter) { f.lock.Lock() defer f.lock.Unlock() @@ -271,12 +253,49 @@ func (f *Forwarder) DetermineCodec(codec webrtc.RTPCodecCapability) { switch strings.ToLower(codec.MimeType) { case "video/vp8": - f.isTemporalSupported = true - f.vp8Munger = NewVP8Munger(f.logger) - case "video/av1", "video/vp9": - // TODO : we only enable dd layer selector for av1 and vp9 now, at future we can - // enable it for vp8 too - f.ddLayerSelector = NewDDVideoLayerSelector(f.logger) + f.codecMunger = codecmunger.NewVP8(f.logger) + if f.vls != nil { + f.vls = videolayerselector.NewSimulcastFromNull(f.vls) + } else { + f.vls = videolayerselector.NewSimulcast(f.logger) + } + f.vls.SetTemporalLayerSelector(temporallayerselector.NewVP8(f.logger)) + case "video/h264": + if f.vls != nil { + f.vls = videolayerselector.NewSimulcastFromNull(f.vls) + } else { + f.vls = videolayerselector.NewSimulcast(f.logger) + } + case "video/vp9": + isDDAvailable := false + searchDone: + for _, ext := range extensions { + switch ext.URI { + case dd.ExtensionUrl: + isDDAvailable = true + break searchDone + } + } + if isDDAvailable { + if f.vls != nil { + f.vls = videolayerselector.NewDependencyDescriptorFromNull(f.vls) + } else { + f.vls = videolayerselector.NewDependencyDescriptor(f.logger) + } + } else { + if f.vls != nil { + f.vls = videolayerselector.NewVP9FromNull(f.vls) + } else { + f.vls = videolayerselector.NewVP9(f.logger) + } + } + case "video/av1": + // DD-TODO : we only enable dd layer selector for av1/vp9 now, in the future we can enable it for vp8 too + if f.vls != nil { + f.vls = videolayerselector.NewDependencyDescriptorFromNull(f.vls) + } else { + f.vls = videolayerselector.NewDependencyDescriptor(f.logger) + } } } @@ -291,10 +310,7 @@ func (f *Forwarder) GetState() ForwarderState { state := ForwarderState{ Started: f.started, RTP: f.rtpMunger.GetLast(), - } - - if f.vp8Munger != nil { - state.VP8 = f.vp8Munger.GetLast() + Codec: f.codecMunger.GetState(), } return state @@ -309,9 +325,7 @@ func (f *Forwarder) SeedState(state ForwarderState) { defer f.lock.Unlock() f.rtpMunger.SeedLast(state.RTP) - if f.vp8Munger != nil { - f.vp8Munger.SeedLast(state.VP8) - } + f.codecMunger.SeedState(state.Codec) f.started = true } @@ -321,7 +335,7 @@ func (f *Forwarder) Mute(muted bool) (bool, buffer.VideoLayer) { defer f.lock.Unlock() if f.muted == muted { - return false, f.maxLayers + return false, f.vls.GetMax() } f.logger.Debugw("setting forwarder mute", "muted", muted) @@ -332,7 +346,7 @@ func (f *Forwarder) Mute(muted bool) (bool, buffer.VideoLayer) { f.resyncLocked() } - return true, f.maxLayers + return true, f.vls.GetMax() } func (f *Forwarder) IsMuted() bool { @@ -347,7 +361,7 @@ func (f *Forwarder) PubMute(pubMuted bool) (bool, buffer.VideoLayer) { defer f.lock.Unlock() if f.pubMuted == pubMuted { - return false, f.maxLayers + return false, f.vls.GetMax() } f.logger.Debugw("setting forwarder pub mute", "pubMuted", pubMuted) @@ -362,13 +376,14 @@ func (f *Forwarder) PubMute(pubMuted bool) (bool, buffer.VideoLayer) { } else { // Do not resync on publisher mute as forwarding can continue on unmute using same layers. // On unmute, park current layers as streaming can continue without a key frame when publisher starts the stream. - if !pubMuted && f.targetLayers.IsValid() && f.currentLayers.Spatial == f.targetLayers.Spatial { - f.setupParkedLayers(f.targetLayers) - f.currentLayers = buffer.InvalidLayers + targetLayer := f.vls.GetTarget() + if !pubMuted && targetLayer.IsValid() && f.vls.GetCurrent().Spatial == targetLayer.Spatial { + f.setupParkedLayers(targetLayer) + f.vls.SetCurrent(buffer.InvalidLayers) } } - return true, f.maxLayers + return true, f.vls.GetMax() } func (f *Forwarder) IsPubMuted() bool { @@ -389,53 +404,63 @@ func (f *Forwarder) SetMaxSpatialLayer(spatialLayer int32) (bool, buffer.VideoLa f.lock.Lock() defer f.lock.Unlock() - if f.kind == webrtc.RTPCodecTypeAudio || spatialLayer == f.maxLayers.Spatial { - return false, f.maxLayers, f.currentLayers + if f.kind == webrtc.RTPCodecTypeAudio { + return false, buffer.InvalidLayers, buffer.InvalidLayers + } + + existingMax := f.vls.GetMax() + if spatialLayer == existingMax.Spatial { + return false, existingMax, f.vls.GetCurrent() } f.logger.Debugw("setting max spatial layer", "layer", spatialLayer) - f.maxLayers.Spatial = spatialLayer + f.vls.SetMaxSpatial(spatialLayer) f.clearParkedLayers() - return true, f.maxLayers, f.currentLayers + return true, f.vls.GetMax(), f.vls.GetCurrent() } func (f *Forwarder) SetMaxTemporalLayer(temporalLayer int32) (bool, buffer.VideoLayer, buffer.VideoLayer) { f.lock.Lock() defer f.lock.Unlock() - if f.kind == webrtc.RTPCodecTypeAudio || temporalLayer == f.maxLayers.Temporal { - return false, f.maxLayers, f.currentLayers + if f.kind == webrtc.RTPCodecTypeAudio { + return false, buffer.InvalidLayers, buffer.InvalidLayers + } + + existingMax := f.vls.GetMax() + if temporalLayer == existingMax.Temporal { + return false, existingMax, f.vls.GetCurrent() } f.logger.Debugw("setting max temporal layer", "layer", temporalLayer) - f.maxLayers.Temporal = temporalLayer + f.vls.SetMaxTemporal(temporalLayer) f.clearParkedLayers() - return true, f.maxLayers, f.currentLayers + return true, f.vls.GetMax(), f.vls.GetCurrent() } func (f *Forwarder) MaxLayers() buffer.VideoLayer { f.lock.RLock() defer f.lock.RUnlock() - return f.maxLayers + return f.vls.GetMax() } func (f *Forwarder) CurrentLayers() buffer.VideoLayer { f.lock.RLock() defer f.lock.RUnlock() - return f.currentLayers + return f.vls.GetCurrent() } func (f *Forwarder) TargetLayers() buffer.VideoLayer { f.lock.RLock() defer f.lock.RUnlock() - return f.targetLayers + return f.vls.GetTarget() } func (f *Forwarder) GetReferenceLayerSpatial() int32 { @@ -470,12 +495,11 @@ func (f *Forwarder) DistanceToDesired(availableLayers []int32, brs Bitrates) flo return getDistanceToDesired( f.muted, f.pubMuted, - f.maxPublishedLayer, - f.maxTemporalLayerSeen, + f.vls.GetMaxSeen(), availableLayers, brs, - f.targetLayers, - f.maxLayers, + f.vls.GetTarget(), + f.vls.GetMax(), ) } @@ -483,7 +507,7 @@ func (f *Forwarder) GetOptimalBandwidthNeeded(brs Bitrates) int64 { f.lock.RLock() defer f.lock.RUnlock() - return getOptimalBandwidthNeeded(f.muted, f.pubMuted, f.maxPublishedLayer, brs, f.maxLayers) + return getOptimalBandwidthNeeded(f.muted, f.pubMuted, f.vls.GetMaxSeen().Spatial, brs, f.vls.GetMax()) } func (f *Forwarder) AllocateOptimal(availableLayers []int32, brs Bitrates, allowOvershoot bool) VideoAllocation { @@ -494,14 +518,19 @@ func (f *Forwarder) AllocateOptimal(availableLayers []int32, brs Bitrates, allow return f.lastAllocation } + maxLayer := f.vls.GetMax() + maxSeenLayer := f.vls.GetMaxSeen() + parkedLayer := f.vls.GetParked() + currentLayer := f.vls.GetCurrent() + requestSpatial := f.vls.GetRequestSpatial() alloc := VideoAllocation{ PauseReason: VideoPauseReasonNone, Bitrates: brs, TargetLayers: buffer.InvalidLayers, - RequestLayerSpatial: f.requestLayerSpatial, - MaxLayers: f.maxLayers, + RequestLayerSpatial: requestSpatial, + MaxLayers: maxLayer, } - optimalBandwidthNeeded := getOptimalBandwidthNeeded(f.muted, f.pubMuted, f.maxPublishedLayer, brs, f.maxLayers) + optimalBandwidthNeeded := getOptimalBandwidthNeeded(f.muted, f.pubMuted, maxSeenLayer.Spatial, brs, maxLayer) if optimalBandwidthNeeded == 0 { alloc.PauseReason = VideoPauseReasonFeedDry } @@ -509,19 +538,19 @@ func (f *Forwarder) AllocateOptimal(availableLayers []int32, brs Bitrates, allow opportunisticAlloc := func() { // opportunistically latch on to anything - maxSpatial := f.maxLayers.Spatial - if allowOvershoot && f.maxPublishedLayer > maxSpatial { - maxSpatial = f.maxPublishedLayer + maxSpatial := maxLayer.Spatial + if allowOvershoot && f.vls.IsOvershootOkay() && maxSeenLayer.Spatial > maxSpatial { + maxSpatial = maxSeenLayer.Spatial } alloc.TargetLayers = buffer.VideoLayer{ - Spatial: int32(math.Min(float64(f.maxPublishedLayer), float64(maxSpatial))), - Temporal: f.maxLayers.Temporal, + Spatial: int32(math.Min(float64(maxSeenLayer.Spatial), float64(maxSpatial))), + Temporal: maxLayer.Temporal, } } switch { - case !f.maxLayers.IsValid() || f.maxPublishedLayer == buffer.InvalidLayerSpatial: - // nothing to do when max layers are not valid OR max publisher layer is invalid + case !maxLayer.IsValid() || maxSeenLayer.Spatial == buffer.InvalidLayerSpatial: + // nothing to do when max layers are not valid OR max published layer is invalid case f.muted: alloc.PauseReason = VideoPauseReasonMuted @@ -529,56 +558,59 @@ func (f *Forwarder) AllocateOptimal(availableLayers []int32, brs Bitrates, allow case f.pubMuted: alloc.PauseReason = VideoPauseReasonPubMuted // leave it at current layers for opportunistic resume - alloc.TargetLayers = f.currentLayers + alloc.TargetLayers = currentLayer alloc.RequestLayerSpatial = alloc.TargetLayers.Spatial - case f.parkedLayers.IsValid(): + case parkedLayer.IsValid(): // if parked on a layer, let it continue - alloc.TargetLayers = f.parkedLayers + alloc.TargetLayers = parkedLayer alloc.RequestLayerSpatial = alloc.TargetLayers.Spatial case len(availableLayers) == 0: // feed may be dry - if f.currentLayers.IsValid() { + if currentLayer.IsValid() { // let it continue at current layer if valid. // Covers the cases of // 1. mis-detection of layer stop - can continue streaming // 2. current layer resuming - can latch on when it starts - alloc.TargetLayers = f.currentLayers + alloc.TargetLayers = currentLayer alloc.RequestLayerSpatial = alloc.TargetLayers.Spatial } else { // opportunistically latch on to anything opportunisticAlloc() - alloc.RequestLayerSpatial = int32(math.Min(float64(f.maxLayers.Spatial), float64(f.maxPublishedLayer))) + alloc.RequestLayerSpatial = int32(math.Min(float64(maxLayer.Spatial), float64(maxSeenLayer.Spatial))) } default: isCurrentLayerAvailable := false - if f.currentLayers.IsValid() { + if currentLayer.IsValid() { for _, l := range availableLayers { - if l == f.currentLayers.Spatial { + if l == currentLayer.Spatial { isCurrentLayerAvailable = true break } } } - if !isCurrentLayerAvailable && f.currentLayers.IsValid() { + if !isCurrentLayerAvailable && currentLayer.IsValid() { // current layer maybe stopped, move to highest available for _, l := range availableLayers { if l > alloc.TargetLayers.Spatial { alloc.TargetLayers.Spatial = l } } - alloc.TargetLayers.Temporal = f.maxLayers.Temporal + alloc.TargetLayers.Temporal = maxLayer.Temporal alloc.RequestLayerSpatial = alloc.TargetLayers.Spatial } else { - requestLayerSpatial := int32(math.Min(float64(f.maxLayers.Spatial), float64(f.maxPublishedLayer))) - if f.currentLayers.IsValid() && requestLayerSpatial == f.requestLayerSpatial && f.currentLayers.Spatial == f.requestLayerSpatial { + requestLayerSpatial := int32(math.Min(float64(maxLayer.Spatial), float64(maxSeenLayer.Spatial))) + if currentLayer.IsValid() && requestLayerSpatial == requestSpatial && currentLayer.Spatial == requestSpatial { // current is locked to desired, stay there - alloc.TargetLayers = buffer.VideoLayer{Spatial: f.requestLayerSpatial, Temporal: f.maxLayers.Temporal} - alloc.RequestLayerSpatial = f.requestLayerSpatial + alloc.TargetLayers = buffer.VideoLayer{ + Spatial: requestSpatial, + Temporal: maxLayer.Temporal, + } + alloc.RequestLayerSpatial = requestSpatial } else { // opportunistically latch on to anything opportunisticAlloc() @@ -598,12 +630,11 @@ func (f *Forwarder) AllocateOptimal(availableLayers []int32, brs Bitrates, allow alloc.DistanceToDesired = getDistanceToDesired( f.muted, f.pubMuted, - f.maxPublishedLayer, - f.maxTemporalLayerSeen, + f.vls.GetMaxSeen(), availableLayers, brs, alloc.TargetLayers, - f.maxLayers, + f.vls.GetMax(), ) return f.updateAllocation(alloc, "optimal") @@ -614,15 +645,14 @@ func (f *Forwarder) ProvisionalAllocatePrepare(availableLayers []int32, Bitrates defer f.lock.Unlock() f.provisional = &VideoAllocationProvisional{ - allocatedLayers: buffer.InvalidLayers, - muted: f.muted, - pubMuted: f.pubMuted, - maxPublishedLayer: f.maxPublishedLayer, - maxTemporalLayerSeen: f.maxTemporalLayerSeen, - Bitrates: Bitrates, - maxLayers: f.maxLayers, - currentLayers: f.currentLayers, - parkedLayers: f.parkedLayers, + allocatedLayers: buffer.InvalidLayers, + muted: f.muted, + pubMuted: f.pubMuted, + maxSeenLayer: f.vls.GetMaxSeen(), + Bitrates: Bitrates, + maxLayers: f.vls.GetMax(), + currentLayers: f.vls.GetCurrent(), + parkedLayers: f.vls.GetParked(), } f.provisional.availableLayers = make([]int32, len(availableLayers)) @@ -633,7 +663,11 @@ func (f *Forwarder) ProvisionalAllocate(availableChannelCapacity int64, layers b f.lock.Lock() defer f.lock.Unlock() - if f.provisional.muted || f.provisional.pubMuted || f.provisional.maxPublishedLayer == buffer.InvalidLayerSpatial || !f.provisional.maxLayers.IsValid() || (!allowOvershoot && layers.GreaterThan(f.provisional.maxLayers)) { + if f.provisional.muted || + f.provisional.pubMuted || + f.provisional.maxSeenLayer.Spatial == buffer.InvalidLayerSpatial || + !f.provisional.maxLayers.IsValid() || + ((!allowOvershoot || !f.vls.IsOvershootOkay()) && layers.GreaterThan(f.provisional.maxLayers)) { return 0 } @@ -654,10 +688,11 @@ func (f *Forwarder) ProvisionalAllocate(availableChannelCapacity int64, layers b } // - // Given layer does not fit. But overshoot is allowed. + // Given layer does not fit. + // // Could be one of // 1. a layer below maximum that does not fit - // 2. a layer above maximum which may or may not fit. + // 2. a layer above maximum which may or may not fit, but overshoot is allowed. // In any of those cases, take the lowest possible layer if pause is not allowed // if !allowPause && (!f.provisional.allocatedLayers.IsValid() || !layers.GreaterThan(f.provisional.allocatedLayers)) { @@ -698,14 +733,15 @@ func (f *Forwarder) ProvisionalAllocateGetCooperativeTransition(allowOvershoot b f.provisional.allocatedLayers = f.provisional.currentLayers } return VideoTransition{ - From: f.targetLayers, + From: f.vls.GetTarget(), To: f.provisional.allocatedLayers, BandwidthDelta: 0 - f.lastAllocation.BandwidthRequested, } } // check if we should preserve current target - if f.targetLayers.IsValid() { + targetLayer := f.vls.GetTarget() + if targetLayer.IsValid() { // what is the highest that is available maximalLayers := buffer.InvalidLayers maximalBandwidthRequired := int64(0) @@ -724,22 +760,22 @@ func (f *Forwarder) ProvisionalAllocateGetCooperativeTransition(allowOvershoot b } if maximalLayers.IsValid() { - if !f.targetLayers.GreaterThan(maximalLayers) && f.provisional.Bitrates[f.targetLayers.Spatial][f.targetLayers.Temporal] != 0 { - // currently streaming and maybe wanting an upgrade (f.targetLayers <= maximalLayers), + if !targetLayer.GreaterThan(maximalLayers) && f.provisional.Bitrates[targetLayer.Spatial][targetLayer.Temporal] != 0 { + // currently streaming and maybe wanting an upgrade (targetLayer <= maximalLayers), // just preserve current target in the cooperative scheme of things - f.provisional.allocatedLayers = f.targetLayers + f.provisional.allocatedLayers = targetLayer return VideoTransition{ - From: f.targetLayers, - To: f.targetLayers, + From: targetLayer, + To: targetLayer, BandwidthDelta: 0, } } - if f.targetLayers.GreaterThan(maximalLayers) { - // maximalLayers < f.targetLayers, make the down move + if targetLayer.GreaterThan(maximalLayers) { + // maximalLayers < targetLayer, make the down move f.provisional.allocatedLayers = maximalLayers return VideoTransition{ - From: f.targetLayers, + From: targetLayer, To: maximalLayers, BandwidthDelta: maximalBandwidthRequired - f.lastAllocation.BandwidthRequested, } @@ -770,20 +806,19 @@ func (f *Forwarder) ProvisionalAllocateGetCooperativeTransition(allowOvershoot b return layers, bw } - targetLayers := f.targetLayers bandwidthRequired := int64(0) - if !targetLayers.IsValid() { + if !targetLayer.IsValid() { // currently not streaming, find minimal // NOTE: a layer in feed could have paused and there could be other options than going back to minimal, // but the cooperative scheme knocks things back to minimal - targetLayers, bandwidthRequired = findNextLayer( + targetLayer, bandwidthRequired = findNextLayer( 0, f.provisional.maxLayers.Spatial, 0, f.provisional.maxLayers.Temporal, ) // could not find a minimal layer, overshoot if allowed - if bandwidthRequired == 0 && f.provisional.maxLayers.IsValid() && allowOvershoot { - targetLayers, bandwidthRequired = findNextLayer( + if bandwidthRequired == 0 && f.provisional.maxLayers.IsValid() && allowOvershoot && f.vls.IsOvershootOkay() { + targetLayer, bandwidthRequired = findNextLayer( f.provisional.maxLayers.Spatial+1, buffer.DefaultMaxLayerSpatial, 0, buffer.DefaultMaxLayerTemporal, ) @@ -791,18 +826,18 @@ func (f *Forwarder) ProvisionalAllocateGetCooperativeTransition(allowOvershoot b } // if nothing available, just leave target at current to enable opportunistic forwarding in case current resumes - if !targetLayers.IsValid() { + if !targetLayer.IsValid() { if f.provisional.parkedLayers.IsValid() { - targetLayers = f.provisional.parkedLayers + targetLayer = f.provisional.parkedLayers } else { - targetLayers = f.provisional.currentLayers + targetLayer = f.provisional.currentLayers } } - f.provisional.allocatedLayers = targetLayers + f.provisional.allocatedLayers = targetLayer return VideoTransition{ - From: f.targetLayers, - To: targetLayers, + From: f.vls.GetTarget(), + To: targetLayer, BandwidthDelta: bandwidthRequired - f.lastAllocation.BandwidthRequested, } } @@ -826,6 +861,7 @@ func (f *Forwarder) ProvisionalAllocateGetBestWeightedTransition() VideoTransiti f.lock.Lock() defer f.lock.Unlock() + targetLayer := f.vls.GetTarget() if f.provisional.muted || f.provisional.pubMuted { f.provisional.allocatedLayers = buffer.InvalidLayers if f.provisional.pubMuted { @@ -833,7 +869,7 @@ func (f *Forwarder) ProvisionalAllocateGetBestWeightedTransition() VideoTransiti f.provisional.allocatedLayers = f.provisional.currentLayers } return VideoTransition{ - From: f.targetLayers, + From: targetLayer, To: f.provisional.allocatedLayers, BandwidthDelta: 0 - f.lastAllocation.BandwidthRequested, } @@ -862,7 +898,7 @@ func (f *Forwarder) ProvisionalAllocateGetBestWeightedTransition() VideoTransiti f.provisional.allocatedLayers = f.provisional.currentLayers } return VideoTransition{ - From: f.targetLayers, + From: targetLayer, To: f.provisional.allocatedLayers, BandwidthDelta: 0 - f.lastAllocation.BandwidthRequested, } @@ -873,20 +909,21 @@ func (f *Forwarder) ProvisionalAllocateGetBestWeightedTransition() VideoTransiti bestLayers := buffer.InvalidLayers bestBandwidthDelta := int64(0) bestValue := float32(0) - for s := int32(0); s <= f.targetLayers.Spatial; s++ { - for t := int32(0); t <= f.targetLayers.Temporal; t++ { - if s == f.targetLayers.Spatial && t == f.targetLayers.Temporal { + for s := int32(0); s <= targetLayer.Spatial; s++ { + for t := int32(0); t <= targetLayer.Temporal; t++ { + if s == targetLayer.Spatial && t == targetLayer.Temporal { break } BandwidthDelta := int64(math.Max(float64(0), float64(f.lastAllocation.BandwidthRequested-f.provisional.Bitrates[s][t]))) transitionCost := int32(0) - if f.targetLayers.Spatial != s { + // LK-TODO: SVC will need a different cost transition + if targetLayer.Spatial != s { transitionCost = TransitionCostSpatial } - qualityCost := (maxReachableLayerTemporal+1)*(f.targetLayers.Spatial-s) + (f.targetLayers.Temporal - t) + qualityCost := (maxReachableLayerTemporal+1)*(targetLayer.Spatial-s) + (targetLayer.Temporal - t) value := float32(0) if (transitionCost + qualityCost) != 0 { @@ -902,7 +939,7 @@ func (f *Forwarder) ProvisionalAllocateGetBestWeightedTransition() VideoTransiti f.provisional.allocatedLayers = bestLayers return VideoTransition{ - From: f.targetLayers, + From: targetLayer, To: bestLayers, BandwidthDelta: bestBandwidthDelta, } @@ -915,7 +952,7 @@ func (f *Forwarder) ProvisionalAllocateCommit() VideoAllocation { optimalBandwidthNeeded := getOptimalBandwidthNeeded( f.provisional.muted, f.provisional.pubMuted, - f.provisional.maxPublishedLayer, + f.provisional.maxSeenLayer.Spatial, f.provisional.Bitrates, f.provisional.maxLayers, ) @@ -930,8 +967,7 @@ func (f *Forwarder) ProvisionalAllocateCommit() VideoAllocation { DistanceToDesired: getDistanceToDesired( f.provisional.muted, f.provisional.pubMuted, - f.provisional.maxPublishedLayer, - f.provisional.maxTemporalLayerSeen, + f.provisional.maxSeenLayer, f.provisional.availableLayers, f.provisional.Bitrates, f.provisional.allocatedLayers, @@ -972,7 +1008,7 @@ func (f *Forwarder) ProvisionalAllocateCommit() VideoAllocation { alloc.BandwidthRequested >= getOptimalBandwidthNeeded( f.provisional.muted, f.provisional.pubMuted, - f.provisional.maxPublishedLayer, + f.provisional.maxSeenLayer.Spatial, f.provisional.Bitrates, f.provisional.maxLayers, ) { @@ -1004,15 +1040,18 @@ func (f *Forwarder) AllocateNextHigher(availableChannelCapacity int64, available } // if targets are still pending, don't increase - if f.targetLayers.IsValid() && f.targetLayers != f.currentLayers { + targetLayer := f.vls.GetTarget() + if targetLayer.IsValid() && targetLayer != f.vls.GetCurrent() { return f.lastAllocation, false } - optimalBandwidthNeeded := getOptimalBandwidthNeeded(f.muted, f.pubMuted, f.maxPublishedLayer, brs, f.maxLayers) + maxLayer := f.vls.GetMax() + maxSeenLayer := f.vls.GetMaxSeen() + optimalBandwidthNeeded := getOptimalBandwidthNeeded(f.muted, f.pubMuted, maxSeenLayer.Spatial, brs, maxLayer) alreadyAllocated := int64(0) - if f.targetLayers.IsValid() { - alreadyAllocated = brs[f.targetLayers.Spatial][f.targetLayers.Temporal] + if targetLayer.IsValid() { + alreadyAllocated = brs[targetLayer.Spatial][targetLayer.Temporal] } doAllocation := func( @@ -1021,38 +1060,37 @@ func (f *Forwarder) AllocateNextHigher(availableChannelCapacity int64, available ) (bool, VideoAllocation, bool) { for s := minSpatial; s <= maxSpatial; s++ { for t := minTemporal; t <= maxTemporal; t++ { - BandwidthRequested := brs[s][t] - if BandwidthRequested == 0 { + bandwidthRequested := brs[s][t] + if bandwidthRequested == 0 { continue } - if !allowOvershoot && BandwidthRequested-alreadyAllocated > availableChannelCapacity { + if (!allowOvershoot || !f.vls.IsOvershootOkay()) && bandwidthRequested-alreadyAllocated > availableChannelCapacity { // next higher available layer does not fit, return return true, f.lastAllocation, false } - targetLayers := buffer.VideoLayer{Spatial: s, Temporal: t} + newTargetLayer := buffer.VideoLayer{Spatial: s, Temporal: t} alloc := VideoAllocation{ IsDeficient: true, - BandwidthRequested: BandwidthRequested, - BandwidthDelta: BandwidthRequested - alreadyAllocated, + BandwidthRequested: bandwidthRequested, + BandwidthDelta: bandwidthRequested - alreadyAllocated, BandwidthNeeded: optimalBandwidthNeeded, Bitrates: brs, - TargetLayers: targetLayers, - RequestLayerSpatial: targetLayers.Spatial, - MaxLayers: f.maxLayers, + TargetLayers: newTargetLayer, + RequestLayerSpatial: newTargetLayer.Spatial, + MaxLayers: maxLayer, DistanceToDesired: getDistanceToDesired( f.muted, f.pubMuted, - f.maxPublishedLayer, - f.maxTemporalLayerSeen, + maxSeenLayer, availableLayers, brs, - targetLayers, - f.maxLayers, + newTargetLayer, + maxLayer, ), } - if targetLayers.GreaterThan(f.maxLayers) || BandwidthRequested >= optimalBandwidthNeeded { + if newTargetLayer.GreaterThan(maxLayer) || bandwidthRequested >= optimalBandwidthNeeded { alloc.IsDeficient = false } @@ -1068,10 +1106,10 @@ func (f *Forwarder) AllocateNextHigher(availableChannelCapacity int64, available boosted := false // try moving temporal layer up in currently streaming spatial layer - if f.targetLayers.IsValid() { + if targetLayer.IsValid() { done, allocation, boosted = doAllocation( - f.targetLayers.Spatial, f.targetLayers.Spatial, - f.targetLayers.Temporal+1, f.maxLayers.Temporal, + targetLayer.Spatial, targetLayer.Spatial, + targetLayer.Temporal+1, maxLayer.Temporal, ) if done { return allocation, boosted @@ -1080,16 +1118,16 @@ func (f *Forwarder) AllocateNextHigher(availableChannelCapacity int64, available // try moving spatial layer up if temporal layer move up is not available done, allocation, boosted = doAllocation( - f.targetLayers.Spatial+1, f.maxLayers.Spatial, - 0, f.maxLayers.Temporal, + targetLayer.Spatial+1, maxLayer.Spatial, + 0, maxLayer.Temporal, ) if done { return allocation, boosted } - if allowOvershoot && f.maxLayers.IsValid() { + if allowOvershoot && f.vls.IsOvershootOkay() && maxLayer.IsValid() { done, allocation, boosted = doAllocation( - f.maxLayers.Spatial+1, buffer.DefaultMaxLayerSpatial, + maxLayer.Spatial+1, buffer.DefaultMaxLayerSpatial, 0, buffer.DefaultMaxLayerTemporal, ) if done { @@ -1114,13 +1152,14 @@ func (f *Forwarder) GetNextHigherTransition(brs Bitrates, allowOvershoot bool) ( } // if targets are still pending, don't increase - if f.targetLayers.IsValid() && f.targetLayers != f.currentLayers { + targetLayer := f.vls.GetTarget() + if targetLayer.IsValid() && targetLayer != f.vls.GetCurrent() { return VideoTransition{}, false } alreadyAllocated := int64(0) - if f.targetLayers.IsValid() { - alreadyAllocated = brs[f.targetLayers.Spatial][f.targetLayers.Temporal] + if targetLayer.IsValid() { + alreadyAllocated = brs[targetLayer.Spatial][targetLayer.Temporal] } findNextHigher := func( @@ -1129,15 +1168,15 @@ func (f *Forwarder) GetNextHigherTransition(brs Bitrates, allowOvershoot bool) ( ) (bool, VideoTransition, bool) { for s := minSpatial; s <= maxSpatial; s++ { for t := minTemporal; t <= maxTemporal; t++ { - BandwidthRequested := brs[s][t] - if BandwidthRequested == 0 { + bandwidthRequested := brs[s][t] + if bandwidthRequested == 0 { continue } transition := VideoTransition{ - From: f.targetLayers, + From: targetLayer, To: buffer.VideoLayer{Spatial: s, Temporal: t}, - BandwidthDelta: BandwidthRequested - alreadyAllocated, + BandwidthDelta: bandwidthRequested - alreadyAllocated, } return true, transition, true @@ -1152,10 +1191,11 @@ func (f *Forwarder) GetNextHigherTransition(brs Bitrates, allowOvershoot bool) ( isAvailable := false // try moving temporal layer up in currently streaming spatial layer - if f.targetLayers.IsValid() { + maxLayer := f.vls.GetMax() + if targetLayer.IsValid() { done, transition, isAvailable = findNextHigher( - f.targetLayers.Spatial, f.targetLayers.Spatial, - f.targetLayers.Temporal+1, f.maxLayers.Temporal, + targetLayer.Spatial, targetLayer.Spatial, + targetLayer.Temporal+1, maxLayer.Temporal, ) if done { return transition, isAvailable @@ -1164,16 +1204,16 @@ func (f *Forwarder) GetNextHigherTransition(brs Bitrates, allowOvershoot bool) ( // try moving spatial layer up if temporal layer move up is not available done, transition, isAvailable = findNextHigher( - f.targetLayers.Spatial+1, f.maxLayers.Spatial, - 0, f.maxLayers.Temporal, + targetLayer.Spatial+1, maxLayer.Spatial, + 0, maxLayer.Temporal, ) if done { return transition, isAvailable } - if allowOvershoot && f.maxLayers.IsValid() { + if allowOvershoot && f.vls.IsOvershootOkay() && maxLayer.IsValid() { done, transition, isAvailable = findNextHigher( - f.maxLayers.Spatial+1, buffer.DefaultMaxLayerSpatial, + maxLayer.Spatial+1, buffer.DefaultMaxLayerSpatial, 0, buffer.DefaultMaxLayerTemporal, ) if done { @@ -1188,7 +1228,9 @@ func (f *Forwarder) Pause(availableLayers []int32, brs Bitrates) VideoAllocation f.lock.Lock() defer f.lock.Unlock() - optimalBandwidthNeeded := getOptimalBandwidthNeeded(f.muted, f.pubMuted, f.maxPublishedLayer, brs, f.maxLayers) + maxLayer := f.vls.GetMax() + maxSeenLayer := f.vls.GetMaxSeen() + optimalBandwidthNeeded := getOptimalBandwidthNeeded(f.muted, f.pubMuted, maxSeenLayer.Spatial, brs, maxLayer) alloc := VideoAllocation{ BandwidthRequested: 0, BandwidthDelta: 0 - f.lastAllocation.BandwidthRequested, @@ -1196,16 +1238,15 @@ func (f *Forwarder) Pause(availableLayers []int32, brs Bitrates) VideoAllocation BandwidthNeeded: optimalBandwidthNeeded, TargetLayers: buffer.InvalidLayers, RequestLayerSpatial: buffer.InvalidLayerSpatial, - MaxLayers: f.maxLayers, + MaxLayers: maxLayer, DistanceToDesired: getDistanceToDesired( f.muted, f.pubMuted, - f.maxPublishedLayer, - f.maxTemporalLayerSeen, + maxSeenLayer, availableLayers, brs, buffer.InvalidLayers, - f.maxLayers, + maxLayer, ), } @@ -1230,6 +1271,11 @@ func (f *Forwarder) Pause(availableLayers []int32, brs Bitrates) VideoAllocation } func (f *Forwarder) updateAllocation(alloc VideoAllocation, reason string) VideoAllocation { + // restrict target temporal to 0 if codec does not support temporal layers + if alloc.TargetLayers.IsValid() && strings.ToLower(f.codec.MimeType) == "video/h264" { + alloc.TargetLayers.Temporal = 0 + } + if alloc.IsDeficient != f.lastAllocation.IsDeficient || alloc.PauseReason != f.lastAllocation.PauseReason || alloc.TargetLayers != f.lastAllocation.TargetLayers || @@ -1243,7 +1289,7 @@ func (f *Forwarder) updateAllocation(alloc VideoAllocation, reason string) Video f.lastAllocation = alloc f.setTargetLayers(f.lastAllocation.TargetLayers, f.lastAllocation.RequestLayerSpatial) - if !f.targetLayers.IsValid() { + if !f.vls.GetTarget().IsValid() { f.resyncLocked() } @@ -1251,12 +1297,8 @@ func (f *Forwarder) updateAllocation(alloc VideoAllocation, reason string) Video } func (f *Forwarder) setTargetLayers(targetLayers buffer.VideoLayer, requestLayerSpatial int32) { - f.targetLayers = targetLayers - if f.ddLayerSelector != nil { - f.ddLayerSelector.SelectLayer(targetLayers) - } - - f.requestLayerSpatial = requestLayerSpatial + f.vls.SetTarget(targetLayers) + f.vls.SetRequestSpatial(requestLayerSpatial) } func (f *Forwarder) Resync() { @@ -1267,13 +1309,13 @@ func (f *Forwarder) Resync() { } func (f *Forwarder) resyncLocked() { - f.currentLayers = buffer.InvalidLayers + f.vls.SetCurrent(buffer.InvalidLayers) f.lastSSRC = 0 f.clearParkedLayers() } func (f *Forwarder) clearParkedLayers() { - f.parkedLayers = buffer.InvalidLayers + f.vls.SetParked(buffer.InvalidLayers) if f.parkedLayersTimer != nil { f.parkedLayersTimer.Stop() f.parkedLayersTimer = nil @@ -1283,13 +1325,14 @@ func (f *Forwarder) clearParkedLayers() { func (f *Forwarder) setupParkedLayers(parkedLayers buffer.VideoLayer) { f.clearParkedLayers() - f.parkedLayers = parkedLayers + f.vls.SetParked(parkedLayers) f.parkedLayersTimer = time.AfterFunc(ParkedLayersWaitDuration, func() { f.lock.Lock() + notify := f.vls.GetParked().IsValid() f.clearParkedLayers() f.lock.Unlock() - if onParkedLayersExpired := f.getOnParkedLayersExpired(); onParkedLayersExpired != nil { + if onParkedLayersExpired := f.getOnParkedLayersExpired(); onParkedLayersExpired != nil && notify { onParkedLayersExpired() } }) @@ -1299,8 +1342,8 @@ func (f *Forwarder) CheckSync() (locked bool, layer int32) { f.lock.RLock() defer f.lock.RUnlock() - layer = f.requestLayerSpatial - locked = f.requestLayerSpatial == f.currentLayers.Spatial || f.parkedLayers.IsValid() + layer = f.vls.GetRequestSpatial() + locked = layer == f.vls.GetCurrent().Spatial || f.vls.GetParked().IsValid() return } @@ -1323,8 +1366,10 @@ func (f *Forwarder) FilterRTX(nacks []uint16) (filtered []uint16, disallowedLaye // // Without the curb, when congestion hits, RTX rate could be so high that it further congests the channel. // + currentLayer := f.vls.GetCurrent() + targetLayer := f.vls.GetTarget() for layer := int32(0); layer < buffer.DefaultMaxLayerSpatial+1; layer++ { - if f.isDeficientLocked() && (f.targetLayers.Spatial < f.currentLayers.Spatial || layer > f.currentLayers.Spatial) { + if f.isDeficientLocked() && (targetLayer.Spatial < currentLayer.Spatial || layer > currentLayer.Spatial) { disallowedLayers[layer] = true } } @@ -1367,9 +1412,7 @@ func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer i f.started = true f.referenceLayerSpatial = layer f.rtpMunger.SetLastSnTs(extPkt) - if f.vp8Munger != nil { - f.vp8Munger.SetLast(extPkt) - } + f.codecMunger.SetLast(extPkt) } else { if f.referenceLayerSpatial == buffer.InvalidLayerSpatial { // on a resume, reference layer may not be set, so only set when it is invalid @@ -1397,9 +1440,7 @@ func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer i } f.rtpMunger.UpdateSnTsOffsets(extPkt, 1, td) - if f.vp8Munger != nil { - f.vp8Munger.UpdateOffsets(extPkt) - } + f.codecMunger.UpdateOffsets(extPkt) } f.logger.Debugw("switching feed", "from", f.lastSSRC, "to", extPkt.Packet.SSRC) @@ -1413,13 +1454,8 @@ func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer i if err != nil { tp.shouldDrop = true if err == ErrPaddingOnlyPacket || err == ErrDuplicatePacket || err == ErrOutOfOrderSequenceNumberCacheMiss { - if err == ErrOutOfOrderSequenceNumberCacheMiss { - tp.isDroppingRelevant = true - } return tp, nil } - - tp.isDroppingRelevant = true return tp, err } @@ -1436,145 +1472,26 @@ func (f *Forwarder) getTranslationParamsAudio(extPkt *buffer.ExtPacket, layer in func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer int32) (*TranslationParams, error) { tp := &TranslationParams{} - if !f.targetLayers.IsValid() { + if !f.vls.GetTarget().IsValid() { // stream is paused by streamallocator tp.shouldDrop = true return tp, nil } - if f.ddLayerSelector != nil { - if selected := f.ddLayerSelector.Select(extPkt, tp); !selected { - tp.shouldDrop = true + result := f.vls.Select(extPkt, layer) + if !result.IsSelected { + tp.shouldDrop = true + if f.started && result.IsRelevant { f.rtpMunger.UpdateAndGetSnTs(extPkt) // call to update highest incoming sequence number and other internal structures f.rtpMunger.PacketDropped(extPkt) - return tp, nil } - } - - // at this point, either - // 1. dependency description has selected the layer for forwarding OR - // 2. non-dependency deescriptor is yet to make decision, but it can potentially switch to the incoming layer and start forwarding - // - // both cases cases upgrade/downgrade to current layer under the right conditions - if f.currentLayers.Spatial != f.targetLayers.Spatial { - // Three things to check when not locked to target - // 1. Resumable layer - don't need a key frame - // 2. Opportunistic layer upgrade - needs a key frame if not using depedency descriptor - // 3. Need to downgrade - needs a key frame if not using dependency descriptor - found := false - if f.parkedLayers.IsValid() { - if f.parkedLayers.Spatial == layer { - f.logger.Infow( - "resuming at parked layer", - "current", f.currentLayers, - "target", f.targetLayers, - "parked", f.parkedLayers, - "feed", extPkt.Packet.SSRC, - ) - f.currentLayers = f.parkedLayers - found = true - } - } else { - if extPkt.KeyFrame || tp.isSwitchingToTargetLayer { - if layer > f.currentLayers.Spatial && layer <= f.targetLayers.Spatial { - f.logger.Infow( - "upgrading layer", - "current", f.currentLayers, - "target", f.targetLayers, - "max", f.maxLayers, - "layer", layer, - "req", f.requestLayerSpatial, - "maxPublished", f.maxPublishedLayer, - "feed", extPkt.Packet.SSRC, - ) - found = true - } - - if layer < f.currentLayers.Spatial && layer >= f.targetLayers.Spatial { - f.logger.Infow( - "downgrading layer", - "current", f.currentLayers, - "target", f.targetLayers, - "max", f.maxLayers, - "layer", layer, - "req", f.requestLayerSpatial, - "maxPublished", f.maxPublishedLayer, - "feed", extPkt.Packet.SSRC, - ) - found = true - } - - if found { - f.currentLayers.Spatial = layer - if !f.isTemporalSupported { - f.currentLayers.Temporal = f.targetLayers.Temporal - } - } - } - } - - if found { - tp.isSwitchingToTargetLayer = true - f.clearParkedLayers() - if f.currentLayers.Spatial >= f.maxLayers.Spatial { - tp.isSwitchingToMaxLayer = true - - f.logger.Infow( - "reached max layer", - "current", f.currentLayers, - "target", f.targetLayers, - "max", f.maxLayers, - "layer", layer, - "req", f.requestLayerSpatial, - "maxPublished", f.maxPublishedLayer, - "feed", extPkt.Packet.SSRC, - ) - } - - if f.currentLayers.Spatial >= f.maxLayers.Spatial || f.currentLayers.Spatial == f.maxPublishedLayer { - f.targetLayers.Spatial = f.currentLayers.Spatial - if f.ddLayerSelector != nil { - f.ddLayerSelector.SelectLayer(f.targetLayers) - } - } - } - } - - // if locked to higher than max layer due to overshoot, check if it can be dialed back - if f.currentLayers.Spatial > f.maxLayers.Spatial { - if layer <= f.maxLayers.Spatial && (extPkt.KeyFrame || tp.isSwitchingToTargetLayer) { - f.logger.Infow( - "adjusting overshoot", - "current", f.currentLayers, - "target", f.targetLayers, - "max", f.maxLayers, - "layer", layer, - "req", f.requestLayerSpatial, - "maxPublished", f.maxPublishedLayer, - "feed", extPkt.Packet.SSRC, - ) - f.currentLayers.Spatial = layer - - if f.currentLayers.Spatial >= f.maxLayers.Spatial { - tp.isSwitchingToMaxLayer = true - } - - if f.currentLayers.Spatial >= f.maxLayers.Spatial || f.currentLayers.Spatial == f.maxPublishedLayer { - f.targetLayers.Spatial = layer - if f.ddLayerSelector != nil { - f.ddLayerSelector.SelectLayer(f.targetLayers) - } - } - } - } - - // if we have layer selector, let it decide whether to drop or not - if f.ddLayerSelector == nil && f.currentLayers.Spatial != layer { - tp.shouldDrop = true return tp, nil } + tp.isSwitchingToMaxLayer = result.IsSwitchingToMaxSpatial + tp.isResuming = result.IsResuming + tp.marker = result.RTPMarker - if FlagPauseOnDowngrade && f.targetLayers.Spatial < f.currentLayers.Spatial && f.isDeficientLocked() { + if FlagPauseOnDowngrade && f.isDeficientLocked() && f.vls.GetTarget().Spatial < f.vls.GetCurrent().Spatial { // // If target layer is lower than both the current and // maximum subscribed layer, it is due to bandwidth @@ -1594,45 +1511,36 @@ func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer in // To differentiate between the two cases, drop only when in DEFICIENT state. // tp.shouldDrop = true - tp.isDroppingRelevant = true return tp, nil } _, err := f.getTranslationParamsCommon(extPkt, layer, tp) - if tp.shouldDrop || f.vp8Munger == nil || len(extPkt.Packet.Payload) == 0 { + if tp.shouldDrop || len(extPkt.Packet.Payload) == 0 { return tp, err } - // catch up temporal layer if necessary - if f.currentLayers.Temporal != f.targetLayers.Temporal { - incomingVP8, ok := extPkt.Payload.(buffer.VP8) - if ok { - if incomingVP8.TIDPresent == 0 || incomingVP8.TID <= uint8(f.targetLayers.Temporal) { - f.currentLayers.Temporal = f.targetLayers.Temporal - } - } - } - - tpVP8, err := f.vp8Munger.UpdateAndGet(extPkt, tp.rtp.snOrdering, f.currentLayers.Temporal) + // codec specific forwarding check and any needed packet munging + codecBytes, err := f.codecMunger.UpdateAndGet( + extPkt, + tp.rtp.snOrdering == SequenceNumberOrderingOutOfOrder, + tp.rtp.snOrdering == SequenceNumberOrderingGap, + f.vls.SelectTemporal(extPkt), + ) if err != nil { tp.rtp = nil tp.shouldDrop = true - if err == ErrFilteredVP8TemporalLayer || err == ErrOutOfOrderVP8PictureIdCacheMiss { - if err == ErrFilteredVP8TemporalLayer { + if err == codecmunger.ErrFilteredVP8TemporalLayer || err == codecmunger.ErrOutOfOrderVP8PictureIdCacheMiss { + if err == codecmunger.ErrFilteredVP8TemporalLayer { // filtered temporal layer, update sequence number offset to prevent holes f.rtpMunger.PacketDropped(extPkt) } - if err == ErrOutOfOrderVP8PictureIdCacheMiss { - tp.isDroppingRelevant = true - } return tp, nil } - tp.isDroppingRelevant = true return tp, err } - tp.vp8 = tpVP8 + tp.codecBytes = codecBytes return tp, nil } @@ -1646,7 +1554,7 @@ func (f *Forwarder) GetSnTsForPadding(num int) ([]SnTs, error) { // force a frame marker as a restart of the stream will // start with a key frame which will reset the decoder. forceMarker := false - if !f.targetLayers.IsValid() { + if !f.vls.GetTarget().IsValid() { forceMarker = true } return f.rtpMunger.UpdateAndGetPaddingSnTs(num, 0, 0, forceMarker) @@ -1664,11 +1572,11 @@ func (f *Forwarder) GetSnTsForBlankFrames(frameRate uint32, numPackets int) ([]S return snts, frameEndNeeded, err } -func (f *Forwarder) GetPaddingVP8(frameEndNeeded bool) *buffer.VP8 { +func (f *Forwarder) GetPadding(frameEndNeeded bool) ([]byte, error) { f.lock.Lock() defer f.lock.Unlock() - return f.vp8Munger.UpdateAndGetPadding(!frameEndNeeded) + return f.codecMunger.UpdateAndGetPadding(!frameEndNeeded) } func (f *Forwarder) GetRTPMungerParams() RTPMungerParams { @@ -1706,14 +1614,13 @@ func getOptimalBandwidthNeeded(muted bool, pubMuted bool, maxPublishedLayer int3 func getDistanceToDesired( muted bool, pubMuted bool, - maxPublishedLayer int32, - maxTemporalLayerSeen int32, + maxSeenLayer buffer.VideoLayer, availableLayers []int32, brs Bitrates, targetLayers buffer.VideoLayer, maxLayers buffer.VideoLayer, ) float64 { - if muted || pubMuted || maxPublishedLayer == buffer.InvalidLayerSpatial || maxTemporalLayerSeen == buffer.InvalidLayerTemporal || !maxLayers.IsValid() { + if muted || pubMuted || !maxSeenLayer.IsValid() || !maxLayers.IsValid() { return 0.0 } @@ -1740,7 +1647,7 @@ done: for _, layer := range availableLayers { if layer > maxAvailableSpatial { maxAvailableSpatial = layer - maxAvailableTemporal = maxTemporalLayerSeen // till bit rate measurement is available, assume max seen as temporal + maxAvailableTemporal = maxSeenLayer.Temporal // till bit rate measurement is available, assume max seen as temporal } } @@ -1748,8 +1655,8 @@ done: adjustedMaxLayers.Spatial = maxAvailableSpatial } - if maxPublishedLayer < adjustedMaxLayers.Spatial { - adjustedMaxLayers.Spatial = maxPublishedLayer + if maxSeenLayer.Spatial < adjustedMaxLayers.Spatial { + adjustedMaxLayers.Spatial = maxSeenLayer.Spatial } // max available temporal is min(subscribedMax, temporalLayerSeenMax, availableMax) @@ -1768,8 +1675,8 @@ done: adjustedMaxLayers.Temporal = maxAvailableTemporal } - if maxTemporalLayerSeen < adjustedMaxLayers.Temporal { - adjustedMaxLayers.Temporal = maxTemporalLayerSeen + if maxSeenLayer.Temporal < adjustedMaxLayers.Temporal { + adjustedMaxLayers.Temporal = maxSeenLayer.Temporal } if !adjustedMaxLayers.IsValid() { @@ -1783,11 +1690,11 @@ done: } distance := - ((adjustedMaxLayers.Spatial - adjustedTargetLayers.Spatial) * (maxTemporalLayerSeen + 1)) + + ((adjustedMaxLayers.Spatial - adjustedTargetLayers.Spatial) * (maxSeenLayer.Temporal + 1)) + (adjustedMaxLayers.Temporal - adjustedTargetLayers.Temporal) if !targetLayers.IsValid() { distance++ } - return float64(distance) / float64(maxTemporalLayerSeen+1) + return float64(distance) / float64(maxSeenLayer.Temporal+1) } diff --git a/pkg/sfu/forwarder_test.go b/pkg/sfu/forwarder_test.go index c543824ad..697a0f573 100644 --- a/pkg/sfu/forwarder_test.go +++ b/pkg/sfu/forwarder_test.go @@ -13,13 +13,13 @@ import ( ) func disable(f *Forwarder) { - f.currentLayers = buffer.InvalidLayers - f.targetLayers = buffer.InvalidLayers + f.vls.SetCurrent(buffer.InvalidLayers) + f.vls.SetTarget(buffer.InvalidLayers) } func newForwarder(codec webrtc.RTPCodecCapability, kind webrtc.RTPCodecType) *Forwarder { f := NewForwarder(kind, logger.GetLogger(), nil) - f.DetermineCodec(codec) + f.DetermineCodec(codec, nil) return f } @@ -87,7 +87,7 @@ func TestForwarderLayersVideo(t *testing.T) { require.Equal(t, expectedLayers, f.MaxLayers()) require.Equal(t, buffer.InvalidLayers, currentLayers) - f.currentLayers = buffer.VideoLayer{Spatial: 0, Temporal: 1} + f.vls.SetCurrent(buffer.VideoLayer{Spatial: 0, Temporal: 1}) changed, maxLayers, currentLayers = f.SetMaxSpatialLayer(buffer.DefaultMaxLayerSpatial - 1) require.False(t, changed) require.Equal(t, expectedLayers, maxLayers) @@ -121,7 +121,7 @@ func TestForwarderAllocateOptimal(t *testing.T) { } // invalid max layers - f.maxLayers = buffer.InvalidLayers + f.vls.SetMax(buffer.InvalidLayers) expectedResult := VideoAllocation{ PauseReason: VideoPauseReasonFeedDry, BandwidthRequested: 0, @@ -195,29 +195,29 @@ func TestForwarderAllocateOptimal(t *testing.T) { f.PubMute(false) // when parked layers valid, should stay there - f.parkedLayers = buffer.VideoLayer{ + f.vls.SetParked(buffer.VideoLayer{ Spatial: 0, Temporal: 1, - } + }) expectedResult = VideoAllocation{ PauseReason: VideoPauseReasonFeedDry, BandwidthRequested: 0, BandwidthDelta: 0, Bitrates: emptyBitrates, - TargetLayers: f.parkedLayers, - RequestLayerSpatial: f.parkedLayers.Spatial, + TargetLayers: f.vls.GetParked(), + RequestLayerSpatial: f.vls.GetParked().Spatial, MaxLayers: buffer.DefaultMaxLayers, DistanceToDesired: 0, } result = f.AllocateOptimal(nil, emptyBitrates, true) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, f.parkedLayers, f.TargetLayers()) - f.parkedLayers = buffer.InvalidLayers + require.Equal(t, f.vls.GetParked(), f.TargetLayers()) + f.vls.SetParked(buffer.InvalidLayers) // when max layers changes, target is opportunistic, but requested spatial layer should be at max f.SetMaxTemporalLayerSeen(3) - f.maxLayers = buffer.VideoLayer{Spatial: 1, Temporal: 3} + f.vls.SetMax(buffer.VideoLayer{Spatial: 1, Temporal: 3}) expectedResult = VideoAllocation{ PauseReason: VideoPauseReasonNone, BandwidthRequested: bitrates[1][3], @@ -225,8 +225,8 @@ func TestForwarderAllocateOptimal(t *testing.T) { BandwidthNeeded: bitrates[1][3], Bitrates: bitrates, TargetLayers: buffer.DefaultMaxLayers, - RequestLayerSpatial: f.maxLayers.Spatial, - MaxLayers: f.maxLayers, + RequestLayerSpatial: f.vls.GetMax().Spatial, + MaxLayers: f.vls.GetMax(), DistanceToDesired: -1, } result = f.AllocateOptimal(nil, bitrates, true) @@ -235,7 +235,7 @@ func TestForwarderAllocateOptimal(t *testing.T) { require.Equal(t, buffer.DefaultMaxLayers, f.TargetLayers()) // reset max layers for rest of the tests below - f.maxLayers = buffer.DefaultMaxLayers + f.vls.SetMax(buffer.DefaultMaxLayers) // when feed is dry and current is not valid, should set up for opportunistic forwarding // NOTE: feed is dry due to availableLayers = nil, some valid bitrates may be passed in here for testing purposes only @@ -260,8 +260,8 @@ func TestForwarderAllocateOptimal(t *testing.T) { require.Equal(t, expectedResult, f.lastAllocation) require.Equal(t, expectedTargetLayers, f.TargetLayers()) - f.targetLayers = buffer.VideoLayer{Spatial: 0, Temporal: 0} // set to valid to trigger paths in tests below - f.currentLayers = buffer.VideoLayer{Spatial: 0, Temporal: 3} // set to valid to trigger paths in tests below + f.vls.SetTarget(buffer.VideoLayer{Spatial: 0, Temporal: 0}) // set to valid to trigger paths in tests below + f.vls.SetCurrent(buffer.VideoLayer{Spatial: 0, Temporal: 3}) // set to valid to trigger paths in tests below // when feed is dry and current is valid, should stay at current expectedTargetLayers = buffer.VideoLayer{ @@ -283,7 +283,7 @@ func TestForwarderAllocateOptimal(t *testing.T) { require.Equal(t, expectedResult, f.lastAllocation) require.Equal(t, expectedTargetLayers, f.TargetLayers()) - f.currentLayers = buffer.InvalidLayers + f.vls.SetCurrent(buffer.InvalidLayers) // opportunistic target if feed is not dry and current is not valid, i. e. not forwarding expectedResult = VideoAllocation{ @@ -303,7 +303,7 @@ func TestForwarderAllocateOptimal(t *testing.T) { require.Equal(t, buffer.DefaultMaxLayers, f.TargetLayers()) // if feed is not dry and current is not locked, should be opportunistic (with and without overshoot) - f.targetLayers = buffer.InvalidLayers + f.vls.SetTarget(buffer.InvalidLayers) expectedResult = VideoAllocation{ PauseReason: VideoPauseReasonFeedDry, BandwidthRequested: 0, @@ -318,7 +318,7 @@ func TestForwarderAllocateOptimal(t *testing.T) { require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - f.targetLayers = buffer.InvalidLayers + f.vls.SetTarget(buffer.InvalidLayers) expectedTargetLayers = buffer.VideoLayer{ Spatial: 2, Temporal: buffer.DefaultMaxLayerTemporal, @@ -339,7 +339,7 @@ func TestForwarderAllocateOptimal(t *testing.T) { require.Equal(t, expectedResult, f.lastAllocation) // switches to highest available if feed is not dry and current is valid and current is not available - f.currentLayers = buffer.VideoLayer{Spatial: 0, Temporal: 1} + f.vls.SetCurrent(buffer.VideoLayer{Spatial: 0, Temporal: 1}) expectedTargetLayers = buffer.VideoLayer{ Spatial: 1, Temporal: buffer.DefaultMaxLayerTemporal, @@ -360,9 +360,9 @@ func TestForwarderAllocateOptimal(t *testing.T) { require.Equal(t, expectedResult, f.lastAllocation) // stays the same if feed is not dry and current is valid, available and locked - f.maxLayers = buffer.VideoLayer{Spatial: 0, Temporal: 1} - f.currentLayers = buffer.VideoLayer{Spatial: 0, Temporal: 1} - f.requestLayerSpatial = 0 + f.vls.SetMax(buffer.VideoLayer{Spatial: 0, Temporal: 1}) + f.vls.SetCurrent(buffer.VideoLayer{Spatial: 0, Temporal: 1}) + f.vls.SetRequestSpatial(0) expectedTargetLayers = buffer.VideoLayer{ Spatial: 0, Temporal: 1, @@ -374,7 +374,7 @@ func TestForwarderAllocateOptimal(t *testing.T) { Bitrates: emptyBitrates, TargetLayers: expectedTargetLayers, RequestLayerSpatial: 0, - MaxLayers: f.maxLayers, + MaxLayers: f.vls.GetMax(), DistanceToDesired: 0.0, } result = f.AllocateOptimal([]int32{0, 1}, emptyBitrates, true) @@ -382,9 +382,9 @@ func TestForwarderAllocateOptimal(t *testing.T) { require.Equal(t, expectedResult, f.lastAllocation) // opportunistic if feed is not dry and current is valid, but request layer has changed - f.maxLayers = buffer.VideoLayer{Spatial: 2, Temporal: 1} - f.currentLayers = buffer.VideoLayer{Spatial: 0, Temporal: 1} - f.requestLayerSpatial = 0 + f.vls.SetMax(buffer.VideoLayer{Spatial: 2, Temporal: 1}) + f.vls.SetCurrent(buffer.VideoLayer{Spatial: 0, Temporal: 1}) + f.vls.SetRequestSpatial(0) expectedTargetLayers = buffer.VideoLayer{ Spatial: 2, Temporal: 1, @@ -396,7 +396,7 @@ func TestForwarderAllocateOptimal(t *testing.T) { Bitrates: emptyBitrates, TargetLayers: expectedTargetLayers, RequestLayerSpatial: 2, - MaxLayers: f.maxLayers, + MaxLayers: f.vls.GetMax(), DistanceToDesired: -1, } result = f.AllocateOptimal([]int32{0, 1}, emptyBitrates, true) @@ -457,7 +457,7 @@ func TestForwarderProvisionalAllocate(t *testing.T) { require.Equal(t, expectedTargetLayers, f.TargetLayers()) // when nothing fits and pausing disallowed, should allocate (0, 0) - f.targetLayers = buffer.InvalidLayers + f.vls.SetTarget(buffer.InvalidLayers) f.ProvisionalAllocatePrepare(nil, bitrates) usedBitrate = f.ProvisionalAllocate(0, buffer.VideoLayer{Spatial: 0, Temporal: 0}, false, false) require.Equal(t, int64(1), usedBitrate) @@ -539,7 +539,7 @@ func TestForwarderProvisionalAllocate(t *testing.T) { {0, 0, 0, 0}, } - f.currentLayers = buffer.VideoLayer{Spatial: 0, Temporal: 2} + f.vls.SetCurrent(buffer.VideoLayer{Spatial: 0, Temporal: 2}) f.ProvisionalAllocatePrepare(nil, bitrates) // all the provisional allocations should not succeed because the feed is dry @@ -577,7 +577,7 @@ func TestForwarderProvisionalAllocate(t *testing.T) { // // Same case as above, but current is above max, so target should go to invalid // - f.currentLayers = buffer.VideoLayer{Spatial: 1, Temporal: 2} + f.vls.SetCurrent(buffer.VideoLayer{Spatial: 1, Temporal: 2}) f.ProvisionalAllocatePrepare(nil, bitrates) // all the provisional allocations below should not succeed because the feed is dry @@ -690,7 +690,7 @@ func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { // a higher target that is already streaming, just maintain it targetLayers := buffer.VideoLayer{Spatial: 2, Temporal: 1} - f.targetLayers = targetLayers + f.vls.SetTarget(targetLayers) f.lastAllocation.BandwidthRequested = 10 expectedTransition = VideoTransition{ From: targetLayers, @@ -719,7 +719,7 @@ func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { // from a target that has become unavailable, should switch to lower available layer targetLayers = buffer.VideoLayer{Spatial: 2, Temporal: 2} - f.targetLayers = targetLayers + f.vls.SetTarget(targetLayers) expectedTransition = VideoTransition{ From: targetLayers, To: buffer.VideoLayer{Spatial: 2, Temporal: 1}, @@ -757,7 +757,7 @@ func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { {9, 10, 0, 0}, } - f.targetLayers = buffer.InvalidLayers + f.vls.SetTarget(buffer.InvalidLayers) f.ProvisionalAllocatePrepare(nil, bitrates) // from scratch (buffer.InvalidLayers) should go to a layer past maximum as overshoot is allowed @@ -795,8 +795,8 @@ func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { {0, 0, 0, 0}, } - f.currentLayers = buffer.VideoLayer{Spatial: 0, Temporal: 2} - f.targetLayers = buffer.InvalidLayers + f.vls.SetCurrent(buffer.VideoLayer{Spatial: 0, Temporal: 2}) + f.vls.SetTarget(buffer.InvalidLayers) f.ProvisionalAllocatePrepare(nil, bitrates) // from scratch (buffer.InvalidLayers) should go to current layer @@ -854,10 +854,10 @@ func TestForwarderProvisionalAllocateGetBestWeightedTransition(t *testing.T) { f.ProvisionalAllocatePrepare(nil, bitrates) - f.targetLayers = buffer.VideoLayer{Spatial: 2, Temporal: 2} + f.vls.SetTarget(buffer.VideoLayer{Spatial: 2, Temporal: 2}) f.lastAllocation.BandwidthRequested = bitrates[2][2] expectedTransition := VideoTransition{ - From: f.targetLayers, + From: f.TargetLayers(), To: buffer.VideoLayer{Spatial: 2, Temporal: 0}, BandwidthDelta: 2, } @@ -894,19 +894,19 @@ func TestForwarderAllocateNextHigher(t *testing.T) { require.False(t, boosted) // if layers have not caught up, should not allocate next layer even if deficient - f.targetLayers = buffer.VideoLayer{ + f.vls.SetTarget(buffer.VideoLayer{ Spatial: 0, Temporal: 0, - } + }) result, boosted = f.AllocateNextHigher(100_000_000, nil, bitrates, false) require.Equal(t, VideoAllocationDefault, result) require.False(t, boosted) f.lastAllocation.IsDeficient = true - f.currentLayers = buffer.VideoLayer{ + f.vls.SetCurrent(buffer.VideoLayer{ Spatial: 0, Temporal: 0, - } + }) // move from (0, 0) -> (0, 1), i.e. a higher temporal layer is available in the same spatial layer expectedTargetLayers := buffer.VideoLayer{ @@ -936,7 +936,7 @@ func TestForwarderAllocateNextHigher(t *testing.T) { require.False(t, boosted) // move from (0, 1) -> (1, 0), i.e. a higher spatial layer is available - f.currentLayers.Temporal = 1 + f.vls.SetCurrent(buffer.VideoLayer{Spatial: f.vls.GetCurrent().Spatial, Temporal: 1}) expectedTargetLayers = buffer.VideoLayer{ Spatial: 1, Temporal: 0, @@ -959,8 +959,7 @@ func TestForwarderAllocateNextHigher(t *testing.T) { require.True(t, boosted) // next higher, move from (1, 0) -> (1, 3), still deficient though - f.currentLayers.Spatial = 1 - f.currentLayers.Temporal = 0 + f.vls.SetCurrent(buffer.VideoLayer{Spatial: 1, Temporal: 0}) expectedTargetLayers = buffer.VideoLayer{ Spatial: 1, Temporal: 3, @@ -983,7 +982,7 @@ func TestForwarderAllocateNextHigher(t *testing.T) { require.True(t, boosted) // next higher, move from (1, 3) -> (2, 1), optimal allocation - f.currentLayers.Temporal = 3 + f.vls.SetCurrent(buffer.VideoLayer{Spatial: f.vls.GetCurrent().Spatial, Temporal: 3}) expectedTargetLayers = buffer.VideoLayer{ Spatial: 2, Temporal: 1, @@ -1005,8 +1004,7 @@ func TestForwarderAllocateNextHigher(t *testing.T) { require.True(t, boosted) // ask again, should return not boosted as there is no room to go higher - f.currentLayers.Spatial = 2 - f.currentLayers.Temporal = 1 + f.vls.SetCurrent(buffer.VideoLayer{Spatial: 2, Temporal: 1}) result, boosted = f.AllocateNextHigher(100_000_000, nil, bitrates, false) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) @@ -1066,7 +1064,7 @@ func TestForwarderAllocateNextHigher(t *testing.T) { {9, 10, 11, 12}, } - f.currentLayers = f.targetLayers + f.vls.SetCurrent(f.vls.GetTarget()) expectedTargetLayers = buffer.VideoLayer{ Spatial: 1, @@ -1227,8 +1225,7 @@ func TestForwarderGetTranslationParamsAudio(t *testing.T) { extPkt, _ = testutils.GetTestExtPacket(params) expectedTP = TranslationParams{ - shouldDrop: true, - isDroppingRelevant: true, + shouldDrop: true, } actualTP, err = f.GetTranslationParams(extPkt, 0) require.NoError(t, err) @@ -1338,21 +1335,22 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { Timestamp: 0xabcdef, SSRC: 0x12345678, PayloadSize: 20, + SetMarker: true, } vp8 := &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 13467, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 1, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: false, + FirstByte: 25, + I: true, + M: true, + PictureID: 13467, + L: true, + TL0PICIDX: 233, + T: true, + TID: 0, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: false, } extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8) @@ -1365,10 +1363,10 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { require.Equal(t, expectedTP, *actualTP) // although target layer matches, not a key frame, so should drop - f.targetLayers = buffer.VideoLayer{ + f.vls.SetTarget(buffer.VideoLayer{ Spatial: 0, Temporal: 1, - } + }) expectedTP = TranslationParams{ shouldDrop: true, } @@ -1376,48 +1374,50 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { require.NoError(t, err) require.Equal(t, expectedTP, *actualTP) - // should lock onto packet (target layer and key frame) + // should lock onto packet (key frame) vp8 = &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 13467, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 1, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: true, + FirstByte: 25, + I: true, + M: true, + PictureID: 13467, + L: true, + TL0PICIDX: 233, + T: true, + TID: 0, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: true, } extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) + expectedVP8 := &buffer.VP8{ + FirstByte: 25, + I: true, + M: true, + PictureID: 13467, + L: true, + TL0PICIDX: 233, + T: true, + TID: 0, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: true, + } + marshalledVP8, err := expectedVP8.Marshal() + require.NoError(t, err) expectedTP = TranslationParams{ - isSwitchingToMaxLayer: true, - isSwitchingToTargetLayer: true, + isSwitchingToMaxLayer: true, + isResuming: true, rtp: &TranslationParamsRTP{ snOrdering: SequenceNumberOrderingContiguous, sequenceNumber: 23333, timestamp: 0xabcdef, }, - vp8: &TranslationParamsVP8{ - Header: &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 13467, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 1, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: true, - }, - }, + codecBytes: marshalledVP8, + marker: true, } actualTP, err = f.GetTranslationParams(extPkt, 0) require.NoError(t, err) @@ -1428,6 +1428,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { // send a duplicate, should be dropped expectedTP = TranslationParams{ shouldDrop: true, + marker: true, } actualTP, err = f.GetTranslationParams(extPkt, 0) require.NoError(t, err) @@ -1442,8 +1443,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { } extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) expectedTP = TranslationParams{ - shouldDrop: true, - isDroppingRelevant: true, + shouldDrop: true, } actualTP, err = f.GetTranslationParams(extPkt, 0) require.NoError(t, err) @@ -1471,35 +1471,36 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { PayloadSize: 20, } extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) + expectedVP8 = &buffer.VP8{ + FirstByte: 25, + I: true, + M: true, + PictureID: 13467, + L: true, + TL0PICIDX: 233, + T: true, + TID: 0, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: true, + } + marshalledVP8, err = expectedVP8.Marshal() + require.NoError(t, err) expectedTP = TranslationParams{ rtp: &TranslationParamsRTP{ snOrdering: SequenceNumberOrderingContiguous, sequenceNumber: 23334, timestamp: 0xabcdef, }, - vp8: &TranslationParamsVP8{ - Header: &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 13467, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 1, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: true, - }, - }, + codecBytes: marshalledVP8, } actualTP, err = f.GetTranslationParams(extPkt, 0) require.NoError(t, err) require.Equal(t, expectedTP, *actualTP) - // temporal layer higher than target, should be dropped + // temporal layer matching target, should be forwarded params = &testutils.TestExtPacketParams{ SequenceNumber: 23336, Timestamp: 0xabcdef, @@ -1507,19 +1508,72 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { PayloadSize: 20, } vp8 = &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 13468, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 2, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: true, + FirstByte: 25, + S: true, + I: true, + M: true, + PictureID: 13468, + L: true, + TL0PICIDX: 233, + T: true, + TID: 1, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: true, + } + extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) + expectedVP8 = &buffer.VP8{ + FirstByte: 25, + I: true, + M: true, + PictureID: 13468, + L: true, + TL0PICIDX: 233, + T: true, + TID: 1, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: true, + } + marshalledVP8, err = expectedVP8.Marshal() + require.NoError(t, err) + expectedTP = TranslationParams{ + rtp: &TranslationParamsRTP{ + snOrdering: SequenceNumberOrderingContiguous, + sequenceNumber: 23335, + timestamp: 0xabcdef, + }, + codecBytes: marshalledVP8, + } + actualTP, err = f.GetTranslationParams(extPkt, 0) + require.NoError(t, err) + require.Equal(t, expectedTP, *actualTP) + + // temporal layer higher than target, should be dropped + params = &testutils.TestExtPacketParams{ + SequenceNumber: 23337, + Timestamp: 0xabcdef, + SSRC: 0x12345678, + PayloadSize: 20, + } + vp8 = &buffer.VP8{ + FirstByte: 25, + I: true, + M: true, + PictureID: 13468, + L: true, + TL0PICIDX: 233, + T: true, + TID: 2, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: true, } extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) expectedTP = TranslationParams{ @@ -1531,50 +1585,51 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { // RTP sequence number and VP8 picture id should be contiguous after dropping higher temporal layer picture params = &testutils.TestExtPacketParams{ - SequenceNumber: 23337, + SequenceNumber: 23338, Timestamp: 0xabcdef, SSRC: 0x12345678, PayloadSize: 20, } vp8 = &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 13469, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 234, - TIDPresent: 1, - TID: 0, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: false, + FirstByte: 25, + I: true, + M: true, + PictureID: 13469, + L: true, + TL0PICIDX: 234, + T: true, + TID: 0, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: false, } extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) + expectedVP8 = &buffer.VP8{ + FirstByte: 25, + I: true, + M: true, + PictureID: 13469, + L: true, + TL0PICIDX: 234, + T: true, + TID: 0, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: false, + } + marshalledVP8, err = expectedVP8.Marshal() + require.NoError(t, err) expectedTP = TranslationParams{ rtp: &TranslationParamsRTP{ snOrdering: SequenceNumberOrderingContiguous, - sequenceNumber: 23335, + sequenceNumber: 23336, timestamp: 0xabcdef, }, - vp8: &TranslationParamsVP8{ - Header: &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 13468, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 234, - TIDPresent: 1, - TID: 0, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: false, - }, - }, + codecBytes: marshalledVP8, } actualTP, err = f.GetTranslationParams(extPkt, 0) require.NoError(t, err) @@ -1582,7 +1637,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { // padding only packet after a gap should be forwarded params = &testutils.TestExtPacketParams{ - SequenceNumber: 23339, + SequenceNumber: 23340, Timestamp: 0xabcdef, SSRC: 0x12345678, } @@ -1591,7 +1646,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { expectedTP = TranslationParams{ rtp: &TranslationParamsRTP{ snOrdering: SequenceNumberOrderingGap, - sequenceNumber: 23337, + sequenceNumber: 23338, timestamp: 0xabcdef, }, } @@ -1601,7 +1656,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { // out-of-order should be forwarded using cache, even if it is padding only params = &testutils.TestExtPacketParams{ - SequenceNumber: 23338, + SequenceNumber: 23339, Timestamp: 0xabcdef, SSRC: 0x12345678, } @@ -1610,7 +1665,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { expectedTP = TranslationParams{ rtp: &TranslationParamsRTP{ snOrdering: SequenceNumberOrderingOutOfOrder, - sequenceNumber: 23336, + sequenceNumber: 23337, timestamp: 0xabcdef, }, } @@ -1620,10 +1675,10 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { // switching SSRC (happens for new layer or new track source) // should lock onto the new source, but sequence number should be contiguous - f.targetLayers = buffer.VideoLayer{ + f.vls.SetTarget(buffer.VideoLayer{ Spatial: 1, Temporal: 1, - } + }) params = &testutils.TestExtPacketParams{ SequenceNumber: 123, @@ -1632,47 +1687,47 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { PayloadSize: 20, } vp8 = &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 45, - MBit: false, - TL0PICIDXPresent: 1, - TL0PICIDX: 12, - TIDPresent: 1, - TID: 0, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 30, - HeaderSize: 5, - IsKeyFrame: true, + FirstByte: 25, + I: true, + M: false, + PictureID: 45, + L: true, + TL0PICIDX: 12, + T: true, + TID: 0, + Y: true, + K: true, + KEYIDX: 30, + HeaderSize: 5, + IsKeyFrame: true, } extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) + expectedVP8 = &buffer.VP8{ + FirstByte: 25, + I: true, + M: true, + PictureID: 13470, + L: true, + TL0PICIDX: 235, + T: true, + TID: 0, + Y: true, + K: true, + KEYIDX: 24, + HeaderSize: 6, + IsKeyFrame: true, + } + marshalledVP8, err = expectedVP8.Marshal() + require.NoError(t, err) expectedTP = TranslationParams{ - isSwitchingToMaxLayer: true, - isSwitchingToTargetLayer: true, + isSwitchingToMaxLayer: true, rtp: &TranslationParamsRTP{ snOrdering: SequenceNumberOrderingContiguous, - sequenceNumber: 23338, + sequenceNumber: 23339, timestamp: 0xabcdf0, }, - vp8: &TranslationParamsVP8{ - Header: &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 13469, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 235, - TIDPresent: 1, - TID: 0, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 24, - HeaderSize: 6, - IsKeyFrame: true, - }, - }, + codecBytes: marshalledVP8, } actualTP, err = f.GetTranslationParams(extPkt, 1) require.NoError(t, err) @@ -1690,27 +1745,27 @@ func TestForwardGetSnTsForPadding(t *testing.T) { PayloadSize: 20, } vp8 := &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 13467, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 13, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: true, + FirstByte: 25, + I: true, + M: true, + PictureID: 13467, + L: true, + TL0PICIDX: 233, + T: true, + TID: 0, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: true, } extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8) - f.targetLayers = buffer.VideoLayer{ + f.vls.SetTarget(buffer.VideoLayer{ Spatial: 0, Temporal: 1, - } - f.currentLayers = buffer.InvalidLayers + }) + f.vls.SetCurrent(buffer.InvalidLayers) // send it through so that forwarder locks onto stream _, _ = f.GetTranslationParams(extPkt, 0) @@ -1757,27 +1812,27 @@ func TestForwardGetSnTsForBlankFrames(t *testing.T) { PayloadSize: 20, } vp8 := &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 13467, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 13, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: true, + FirstByte: 25, + I: true, + M: true, + PictureID: 13467, + L: true, + TL0PICIDX: 233, + T: true, + TID: 0, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: true, } extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8) - f.targetLayers = buffer.VideoLayer{ + f.vls.SetTarget(buffer.VideoLayer{ Spatial: 0, Temporal: 1, - } - f.currentLayers = buffer.InvalidLayers + }) + f.vls.SetCurrent(buffer.InvalidLayers) // send it through so that forwarder locks onto stream _, _ = f.GetTranslationParams(extPkt, 0) @@ -1827,66 +1882,72 @@ func TestForwardGetPaddingVP8(t *testing.T) { PayloadSize: 20, } vp8 := &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 13467, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 13, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: true, + FirstByte: 25, + I: true, + M: true, + PictureID: 13467, + L: true, + TL0PICIDX: 233, + T: true, + TID: 13, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: true, } extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8) - f.targetLayers = buffer.VideoLayer{ + f.vls.SetTarget(buffer.VideoLayer{ Spatial: 0, Temporal: 1, - } - f.currentLayers = buffer.InvalidLayers + }) + f.vls.SetCurrent(buffer.InvalidLayers) // send it through so that forwarder locks onto stream _, _ = f.GetTranslationParams(extPkt, 0) // getting padding with frame end needed, should repeat the last picture id expectedVP8 := buffer.VP8{ - FirstByte: 16, - PictureIDPresent: 1, - PictureID: 13467, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 0, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: true, + FirstByte: 16, + I: true, + M: true, + PictureID: 13467, + L: true, + TL0PICIDX: 233, + T: true, + TID: 0, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: true, } - blankVP8 := f.GetPaddingVP8(true) - require.Equal(t, expectedVP8, *blankVP8) + blankVP8, err := f.GetPadding(true) + require.NoError(t, err) + marshalledVP8, err := expectedVP8.Marshal() + require.NoError(t, err) + require.Equal(t, marshalledVP8, blankVP8) // getting padding with no frame end needed, should get next picture id expectedVP8 = buffer.VP8{ - FirstByte: 16, - PictureIDPresent: 1, - PictureID: 13468, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 234, - TIDPresent: 1, - TID: 0, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 24, - HeaderSize: 6, - IsKeyFrame: true, + FirstByte: 16, + I: true, + M: true, + PictureID: 13468, + L: true, + TL0PICIDX: 234, + T: true, + TID: 0, + Y: true, + K: true, + KEYIDX: 24, + HeaderSize: 6, + IsKeyFrame: true, } - blankVP8 = f.GetPaddingVP8(false) - require.Equal(t, expectedVP8, *blankVP8) + blankVP8, err = f.GetPadding(false) + require.NoError(t, err) + marshalledVP8, err = expectedVP8.Marshal() + require.NoError(t, err) + require.Equal(t, marshalledVP8, blankVP8) } diff --git a/pkg/sfu/sequencer.go b/pkg/sfu/sequencer.go index 0fb7693af..4f8558512 100644 --- a/pkg/sfu/sequencer.go +++ b/pkg/sfu/sequencer.go @@ -6,8 +6,6 @@ import ( "time" "github.com/livekit/protocol/logger" - - "github.com/livekit/livekit-server/pkg/sfu/buffer" ) const ( @@ -51,45 +49,11 @@ type packetMeta struct { // Spatial layer of packet layer int8 // Information that differs depending on the codec - misc uint64 + codecBytes []byte // Dependency Descriptor of packet ddBytes []byte } -func (p *packetMeta) packVP8(vp8 *buffer.VP8) { - p.misc = uint64(vp8.FirstByte)<<56 | - uint64(vp8.PictureIDPresent&0x1)<<55 | - uint64(vp8.TL0PICIDXPresent&0x1)<<54 | - uint64(vp8.TIDPresent&0x1)<<53 | - uint64(vp8.KEYIDXPresent&0x1)<<52 | - uint64(btoi(vp8.MBit)&0x1)<<51 | - uint64(btoi(vp8.IsKeyFrame)&0x1)<<50 | - uint64(vp8.PictureID&0x7FFF)<<32 | - uint64(vp8.TL0PICIDX&0xFF)<<24 | - uint64(vp8.TID&0x3)<<22 | - uint64(vp8.Y&0x1)<<21 | - uint64(vp8.KEYIDX&0x1F)<<16 | - uint64(vp8.HeaderSize&0xFF)<<8 -} - -func (p *packetMeta) unpackVP8() *buffer.VP8 { - return &buffer.VP8{ - FirstByte: byte(p.misc >> 56), - PictureIDPresent: int((p.misc >> 55) & 0x1), - PictureID: uint16((p.misc >> 32) & 0x7FFF), - MBit: itob(int((p.misc >> 51) & 0x1)), - TL0PICIDXPresent: int((p.misc >> 54) & 0x1), - TL0PICIDX: uint8((p.misc >> 24) & 0xFF), - TIDPresent: int((p.misc >> 53) & 0x1), - TID: uint8((p.misc >> 22) & 0x3), - Y: uint8((p.misc >> 21) & 0x1), - KEYIDXPresent: int((p.misc >> 52) & 0x1), - KEYIDX: uint8((p.misc >> 16) & 0x1F), - HeaderSize: int((p.misc >> 8) & 0xFF), - IsKeyFrame: itob(int((p.misc >> 50) & 0x1)), - } -} - // Sequencer stores the packet sequence received by the down track type sequencer struct { sync.Mutex diff --git a/pkg/sfu/sequencer_test.go b/pkg/sfu/sequencer_test.go index 51cbc0eb4..776434004 100644 --- a/pkg/sfu/sequencer_test.go +++ b/pkg/sfu/sequencer_test.go @@ -8,8 +8,6 @@ import ( "github.com/stretchr/testify/require" "github.com/livekit/protocol/logger" - - "github.com/livekit/livekit-server/pkg/sfu/buffer" ) func Test_sequencer(t *testing.T) { @@ -104,82 +102,3 @@ func Test_sequencer_getNACKSeqNo(t *testing.T) { }) } } - -func Test_packetMeta_VP8(t *testing.T) { - p := &packetMeta{} - - vp8 := &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 55467, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 13, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: true, - } - - p.packVP8(vp8) - - // booleans are not packed, so they will be `false` in unpacked. - // Also, TID is only two bits, so it should be modulo 3. - expectedVP8 := &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 55467 % 32768, - MBit: true, - TL0PICIDXPresent: 1, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 13 % 3, - Y: 1, - KEYIDXPresent: 1, - KEYIDX: 23, - HeaderSize: 6, - IsKeyFrame: true, - } - unpackedVP8 := p.unpackVP8() - require.Equal(t, expectedVP8, unpackedVP8) - - // short picture id and no TL0PICIDX - vp8 = &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 63, - MBit: false, - TL0PICIDXPresent: 0, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 2, - Y: 1, - KEYIDXPresent: 0, - KEYIDX: 23, - HeaderSize: 23, - IsKeyFrame: true, - } - - p.packVP8(vp8) - - expectedVP8 = &buffer.VP8{ - FirstByte: 25, - PictureIDPresent: 1, - PictureID: 63, - MBit: false, - TL0PICIDXPresent: 0, - TL0PICIDX: 233, - TIDPresent: 1, - TID: 2, - Y: 1, - KEYIDXPresent: 0, - KEYIDX: 23, - HeaderSize: 23, - IsKeyFrame: true, - } - unpackedVP8 = p.unpackVP8() - require.Equal(t, expectedVP8, unpackedVP8) -} diff --git a/pkg/sfu/streamallocator/streamallocator.go b/pkg/sfu/streamallocator/streamallocator.go index e5f7d5e3f..cb0e0915f 100644 --- a/pkg/sfu/streamallocator/streamallocator.go +++ b/pkg/sfu/streamallocator/streamallocator.go @@ -98,7 +98,7 @@ const ( streamAllocatorSignalPeriodicPing streamAllocatorSignalSendProbe streamAllocatorSignalProbeClusterDone - streamAllocatorSignalTargetLayerFound + streamAllocatorSignalResume ) func (s streamAllocatorSignal) String() string { @@ -117,8 +117,8 @@ func (s streamAllocatorSignal) String() string { return "SEND_PROBE" case streamAllocatorSignalProbeClusterDone: return "PROBE_CLUSTER_DONE" - case streamAllocatorSignalTargetLayerFound: - return "TARGET_LAYER_FOUND" + case streamAllocatorSignalResume: + return "RESUME" default: return fmt.Sprintf("%d", int(s)) } @@ -415,10 +415,10 @@ func (s *StreamAllocator) OnSubscribedLayersChanged(downTrack *sfu.DownTrack, la } } -// called when forwarder finds a target layer -func (s *StreamAllocator) OnTargetLayerReached(downTrack *sfu.DownTrack) { +// called when forwarder resumes a track +func (s *StreamAllocator) OnResume(downTrack *sfu.DownTrack) { s.postEvent(Event{ - Signal: streamAllocatorSignalTargetLayerFound, + Signal: streamAllocatorSignalResume, TrackID: livekit.TrackID(downTrack.ID()), }) } @@ -529,8 +529,8 @@ func (s *StreamAllocator) handleEvent(event *Event) { s.handleSignalSendProbe(event) case streamAllocatorSignalProbeClusterDone: s.handleSignalProbeClusterDone(event) - case streamAllocatorSignalTargetLayerFound: - s.handleSignalTargetLayerFound(event) + case streamAllocatorSignalResume: + s.handleSignalResume(event) } } @@ -630,7 +630,7 @@ func (s *StreamAllocator) handleSignalProbeClusterDone(event *Event) { s.probeEndTime = s.lastProbeStartTime.Add(queueWait) } -func (s *StreamAllocator) handleSignalTargetLayerFound(event *Event) { +func (s *StreamAllocator) handleSignalResume(event *Event) { s.videoTracksMu.Lock() track := s.videoTracks[event.TrackID] s.videoTracksMu.Unlock() diff --git a/pkg/sfu/testutils/data.go b/pkg/sfu/testutils/data.go index a3131eeee..8b243b40f 100644 --- a/pkg/sfu/testutils/data.go +++ b/pkg/sfu/testutils/data.go @@ -19,6 +19,7 @@ type TestExtPacketParams struct { PayloadSize int PaddingSize byte ArrivalTime int64 + VideoLayer buffer.VideoLayer } // ----------------------------------------------------------- @@ -44,10 +45,11 @@ func GetTestExtPacket(params *TestExtPacketParams) (*buffer.ExtPacket, error) { } ep := &buffer.ExtPacket{ - Arrival: params.ArrivalTime, - Packet: &packet, - KeyFrame: params.IsKeyFrame, - RawPacket: raw, + VideoLayer: params.VideoLayer, + Arrival: params.ArrivalTime, + Packet: &packet, + KeyFrame: params.IsKeyFrame, + RawPacket: raw, } return ep, nil diff --git a/pkg/sfu/videolayerselector.go b/pkg/sfu/videolayerselector.go deleted file mode 100644 index 1368bc870..000000000 --- a/pkg/sfu/videolayerselector.go +++ /dev/null @@ -1,202 +0,0 @@ -package sfu - -import ( - "fmt" - "sort" - - "github.com/livekit/livekit-server/pkg/sfu/buffer" - dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" - "github.com/livekit/protocol/logger" -) - -type targetLayer struct { - Target int - Layer buffer.VideoLayer -} - -type DDVideoLayerSelector struct { - logger logger.Logger - - // DD-TODO : fields for frame chain detect - // frameNumberWrapper Uint16Wrapper - // expectKeyFrame bool - - decodeTargetLayer []targetLayer - layer buffer.VideoLayer - activeDecodeTargetsBitmask *uint32 - structure *dd.FrameDependencyStructure -} - -func NewDDVideoLayerSelector(logger logger.Logger) *DDVideoLayerSelector { - return &DDVideoLayerSelector{ - logger: logger, - layer: buffer.VideoLayer{Spatial: 2, Temporal: 2}, - } -} - -func (s *DDVideoLayerSelector) Select(expPkt *buffer.ExtPacket, tp *TranslationParams) (selected bool) { - tp.marker = expPkt.Packet.Marker - if expPkt.DependencyDescriptor == nil { - // packet don't have dependency descriptor, pass check - return true - } - - if expPkt.DependencyDescriptor.AttachedStructure != nil { - // update decode target layer and active decode targets - // DD-TODO : these targets info can be shared by all the downtracks, no need calculate in every selector - s.updateDependencyStructure(expPkt.DependencyDescriptor.AttachedStructure) - } - - // forward all packets before locking - if s.layer == buffer.InvalidLayers { - return true - } - - // DD-TODO : we don't have a rtp queue to ensure the order of packets now, - // so we don't know packet is lost/out of order, that cause us can't detect - // frame integrity, entire frame is forwareded, whether frame chain is broken. - // So use a simple check here, assume all the reference frame is forwarded and - // only check DTI of the active decode target. - // it is not effeciency, at last we need check frame chain integrity. - - activeDecodeTargets := expPkt.DependencyDescriptor.ActiveDecodeTargetsBitmask - if activeDecodeTargets != nil { - s.logger.Debugw("active decode targets", "activeDecodeTargets", *activeDecodeTargets) - } - - currentTarget := -1 - for _, dt := range s.decodeTargetLayer { - // find target match with selected layer - if dt.Layer.Spatial <= s.layer.Spatial && dt.Layer.Temporal <= s.layer.Temporal { - if activeDecodeTargets == nil || ((*activeDecodeTargets)&(1< maxSpatial { - maxSpatial = dt.Layer.Spatial - } - if dt.Layer.Temporal > maxTemporal { - maxTemporal = dt.Layer.Temporal - } - if dt.Layer.Spatial <= layer.Spatial && dt.Layer.Temporal <= layer.Temporal { - activeBitMask |= 1 << dt.Target - } - } - if layer.Spatial == maxSpatial && layer.Temporal == maxTemporal { - // all the decode targets are selected - s.activeDecodeTargetsBitmask = nil - } else { - s.activeDecodeTargetsBitmask = &activeBitMask - } - s.logger.Debugw("select layer ", "layer", layer, "activeDecodeTargetsBitmask", s.activeDecodeTargetsBitmask) -} - -func (s *DDVideoLayerSelector) updateDependencyStructure(structure *dd.FrameDependencyStructure) { - s.structure = structure - s.decodeTargetLayer = s.decodeTargetLayer[:0] - - for target := 0; target < structure.NumDecodeTargets; target++ { - layer := buffer.VideoLayer{Spatial: 0, Temporal: 0} - for _, t := range structure.Templates { - if t.DecodeTargetIndications[target] != dd.DecodeTargetNotPresent { - if layer.Spatial < int32(t.SpatialId) { - layer.Spatial = int32(t.SpatialId) - } - if layer.Temporal < int32(t.TemporalId) { - layer.Temporal = int32(t.TemporalId) - } - } - } - s.decodeTargetLayer = append(s.decodeTargetLayer, targetLayer{target, layer}) - } - - // sort decode target layer by spatial and temporal from high to low - sort.Slice(s.decodeTargetLayer, func(i, j int) bool { - if s.decodeTargetLayer[i].Layer.Spatial == s.decodeTargetLayer[j].Layer.Spatial { - return s.decodeTargetLayer[i].Layer.Temporal > s.decodeTargetLayer[j].Layer.Temporal - } - return s.decodeTargetLayer[i].Layer.Spatial > s.decodeTargetLayer[j].Layer.Spatial - }) - s.logger.Debugw(fmt.Sprintf("update decode targets: %v", s.decodeTargetLayer)) -} - -// DD-TODO : use generic wrapper when updated to go 1.18 -type Uint16Wrapper struct { - lastValue *uint16 - lastUnwrapped int32 -} - -func (w *Uint16Wrapper) Unwrap(value uint16) int32 { - if w.lastValue == nil { - w.lastValue = &value - w.lastUnwrapped = int32(value) - return int32(*w.lastValue) - } - - diff := value - *w.lastValue - w.lastUnwrapped += int32(diff) - if diff == 0x8000 && value < *w.lastValue { - w.lastUnwrapped -= 0x10000 - } else if diff > 0x8000 { - w.lastUnwrapped -= 0x10000 - } - - *w.lastValue = value - return w.lastUnwrapped -} diff --git a/pkg/sfu/videolayerselector/base.go b/pkg/sfu/videolayerselector/base.go new file mode 100644 index 000000000..29a8cdc42 --- /dev/null +++ b/pkg/sfu/videolayerselector/base.go @@ -0,0 +1,120 @@ +package videolayerselector + +import ( + "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/livekit-server/pkg/sfu/videolayerselector/temporallayerselector" + "github.com/livekit/protocol/logger" +) + +type Base struct { + logger logger.Logger + + tls temporallayerselector.TemporalLayerSelector + + maxLayer buffer.VideoLayer + targetLayer buffer.VideoLayer + requestSpatial int32 + maxSeenLayer buffer.VideoLayer + + parkedLayer buffer.VideoLayer + + currentLayer buffer.VideoLayer +} + +func NewBase(logger logger.Logger) *Base { + return &Base{ + logger: logger, + maxLayer: buffer.InvalidLayers, + targetLayer: buffer.InvalidLayers, // start off with nothing, let streamallocator/opportunistic forwarder set the target + requestSpatial: buffer.InvalidLayerSpatial, + maxSeenLayer: buffer.InvalidLayers, + parkedLayer: buffer.InvalidLayers, + currentLayer: buffer.InvalidLayers, + } +} + +func (b *Base) IsOvershootOkay() bool { + return false +} + +func (b *Base) SetTemporalLayerSelector(tls temporallayerselector.TemporalLayerSelector) { + b.tls = tls +} + +func (b *Base) SetMax(maxLayer buffer.VideoLayer) { + b.maxLayer = maxLayer +} + +func (b *Base) SetMaxSpatial(layer int32) { + b.maxLayer.Spatial = layer +} + +func (b *Base) SetMaxTemporal(layer int32) { + b.maxLayer.Temporal = layer +} + +func (b *Base) GetMax() buffer.VideoLayer { + return b.maxLayer +} + +func (b *Base) SetTarget(targetLayer buffer.VideoLayer) { + b.targetLayer = targetLayer +} + +func (b *Base) GetTarget() buffer.VideoLayer { + return b.targetLayer +} + +func (b *Base) SetRequestSpatial(layer int32) { + b.requestSpatial = layer +} + +func (b *Base) GetRequestSpatial() int32 { + return b.requestSpatial +} + +func (b *Base) SetMaxSeen(maxSeenLayer buffer.VideoLayer) { + b.maxSeenLayer = maxSeenLayer +} + +func (b *Base) SetMaxSeenSpatial(layer int32) { + b.maxSeenLayer.Spatial = layer +} + +func (b *Base) SetMaxSeenTemporal(layer int32) { + b.maxSeenLayer.Temporal = layer +} + +func (b *Base) GetMaxSeen() buffer.VideoLayer { + return b.maxSeenLayer +} + +func (b *Base) SetParked(parkedLayer buffer.VideoLayer) { + b.parkedLayer = parkedLayer +} + +func (b *Base) GetParked() buffer.VideoLayer { + return b.parkedLayer +} + +func (b *Base) SetCurrent(currentLayer buffer.VideoLayer) { + b.currentLayer = currentLayer +} + +func (b *Base) GetCurrent() buffer.VideoLayer { + return b.currentLayer +} + +func (b *Base) Select(_extPkt *buffer.ExtPacket, _layer int32) (result VideoLayerSelectorResult) { + return +} + +func (b *Base) SelectTemporal(extPkt *buffer.ExtPacket) int32 { + if b.tls != nil { + this, next := b.tls.Select(extPkt, b.currentLayer.Temporal, b.targetLayer.Temporal) + b.currentLayer.Temporal = next + return this + } + + return b.currentLayer.Temporal +} diff --git a/pkg/sfu/videolayerselector/dependencydescriptor.go b/pkg/sfu/videolayerselector/dependencydescriptor.go new file mode 100644 index 000000000..8b39049ce --- /dev/null +++ b/pkg/sfu/videolayerselector/dependencydescriptor.go @@ -0,0 +1,278 @@ +package videolayerselector + +import ( + "fmt" + "sort" + + "github.com/livekit/livekit-server/pkg/sfu/buffer" + dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + "github.com/livekit/protocol/logger" +) + +type decodeTarget struct { + Target int + Layer buffer.VideoLayer +} + +type DependencyDescriptor struct { + *Base + + // DD-TODO : fields for frame chain detect + // frameNumberWrapper Uint16Wrapper + // expectKeyFrame bool + + decodeTargets []decodeTarget + activeDecodeTargetsBitmask *uint32 + structure *dd.FrameDependencyStructure +} + +func NewDependencyDescriptor(logger logger.Logger) *DependencyDescriptor { + return &DependencyDescriptor{ + Base: NewBase(logger), + } +} + +func NewDependencyDescriptorFromNull(vls VideoLayerSelector) *DependencyDescriptor { + return &DependencyDescriptor{ + Base: vls.(*Null).Base, + } +} + +func (d *DependencyDescriptor) IsOvershootOkay() bool { + return false +} + +func (d *DependencyDescriptor) Select(extPkt *buffer.ExtPacket, _layer int32) (result VideoLayerSelectorResult) { + if extPkt.DependencyDescriptor == nil { + // packet don't have dependency descriptor + return + } + + if !d.currentLayer.IsValid() && !extPkt.KeyFrame { + return + } + + result.IsRelevant = true + + if extPkt.DependencyDescriptor.AttachedStructure != nil { + // update decode target layer and active decode targets + // DD-TODO : these targets info can be shared by all the downtracks, no need calculate in every selector + d.updateDependencyStructure(extPkt.DependencyDescriptor.AttachedStructure) + } + + // DD-TODO : we don't have a rtp queue to ensure the order of packets now, + // so we don't know packet is lost/out of order, that cause us can't detect + // frame integrity, entire frame is forwareded, whether frame chain is broken. + // So use a simple check here, assume all the reference frame is forwarded and + // only check DTI of the active decode target. + // it is not effeciency, at last we need check frame chain integrity. + + activeDecodeTargets := extPkt.DependencyDescriptor.ActiveDecodeTargetsBitmask + if activeDecodeTargets != nil { + d.logger.Debugw("active decode targets", "activeDecodeTargets", *activeDecodeTargets) + } + + currentTarget := -1 + for _, dt := range d.decodeTargets { + // find target match with selected layer + if dt.Layer.Spatial <= d.targetLayer.Spatial && dt.Layer.Temporal <= d.targetLayer.Temporal { + if activeDecodeTargets == nil || ((*activeDecodeTargets)&(1< maxSpatial { + maxSpatial = dt.Layer.Spatial + } + if dt.Layer.Temporal > maxTemporal { + maxTemporal = dt.Layer.Temporal + } + if dt.Layer.Spatial <= targetLayer.Spatial && dt.Layer.Temporal <= targetLayer.Temporal { + activeBitMask |= 1 << dt.Target + } + } + if targetLayer.Spatial == maxSpatial && targetLayer.Temporal == maxTemporal { + // all the decode targets are selected + d.activeDecodeTargetsBitmask = nil + } else { + d.activeDecodeTargetsBitmask = &activeBitMask + } + d.logger.Debugw("setting target", "targetlayer", targetLayer, "activeDecodeTargetsBitmask", d.activeDecodeTargetsBitmask) +} + +func (d *DependencyDescriptor) updateDependencyStructure(structure *dd.FrameDependencyStructure) { + d.structure = structure + d.decodeTargets = d.decodeTargets[:0] + + for target := 0; target < structure.NumDecodeTargets; target++ { + layer := buffer.VideoLayer{Spatial: 0, Temporal: 0} + for _, t := range structure.Templates { + if t.DecodeTargetIndications[target] != dd.DecodeTargetNotPresent { + if layer.Spatial < int32(t.SpatialId) { + layer.Spatial = int32(t.SpatialId) + } + if layer.Temporal < int32(t.TemporalId) { + layer.Temporal = int32(t.TemporalId) + } + } + } + d.decodeTargets = append(d.decodeTargets, decodeTarget{target, layer}) + } + + // sort decode target layer by spatial and temporal from high to low + sort.Slice(d.decodeTargets, func(i, j int) bool { + return d.decodeTargets[i].Layer.GreaterThan(d.decodeTargets[j].Layer) + }) + d.logger.Debugw(fmt.Sprintf("update decode targets: %v", d.decodeTargets)) +} + +// DD-TODO : use generic wrapper when updated to go 1.18 +type Uint16Wrapper struct { + lastValue *uint16 + lastUnwrapped int32 +} + +func (w *Uint16Wrapper) Unwrap(value uint16) int32 { + if w.lastValue == nil { + w.lastValue = &value + w.lastUnwrapped = int32(value) + return int32(*w.lastValue) + } + + diff := value - *w.lastValue + w.lastUnwrapped += int32(diff) + if diff == 0x8000 && value < *w.lastValue { + w.lastUnwrapped -= 0x10000 + } else if diff > 0x8000 { + w.lastUnwrapped -= 0x10000 + } + + *w.lastValue = value + return w.lastUnwrapped +} diff --git a/pkg/sfu/videolayerselector/null.go b/pkg/sfu/videolayerselector/null.go new file mode 100644 index 000000000..d1b87fb1b --- /dev/null +++ b/pkg/sfu/videolayerselector/null.go @@ -0,0 +1,15 @@ +package videolayerselector + +import ( + "github.com/livekit/protocol/logger" +) + +type Null struct { + *Base +} + +func NewNull(logger logger.Logger) *Null { + return &Null{ + Base: NewBase(logger), + } +} diff --git a/pkg/sfu/videolayerselector/simulcast.go b/pkg/sfu/videolayerselector/simulcast.go new file mode 100644 index 000000000..9bb1d368f --- /dev/null +++ b/pkg/sfu/videolayerselector/simulcast.go @@ -0,0 +1,143 @@ +package videolayerselector + +import ( + "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/protocol/logger" +) + +type Simulcast struct { + *Base +} + +func NewSimulcast(logger logger.Logger) *Simulcast { + return &Simulcast{ + Base: NewBase(logger), + } +} + +func NewSimulcastFromNull(vls VideoLayerSelector) *Simulcast { + return &Simulcast{ + Base: vls.(*Null).Base, + } +} + +func (s *Simulcast) IsOvershootOkay() bool { + return true +} + +func (s *Simulcast) Select(extPkt *buffer.ExtPacket, layer int32) (result VideoLayerSelectorResult) { + if s.currentLayer.Spatial != s.targetLayer.Spatial { + // Three things to check when not locked to target + // 1. Resumable layer - don't need a key frame + // 2. Opportunistic layer upgrade - needs a key frame + // 3. Need to downgrade - needs a key frame + isActive := s.currentLayer.IsValid() + found := false + if s.parkedLayer.IsValid() { + if s.parkedLayer.Spatial == layer { + s.logger.Infow( + "resuming at parked layer", + "current", s.currentLayer, + "target", s.targetLayer, + "max", s.maxLayer, + "parked", s.parkedLayer, + "req", s.requestSpatial, + "maxSeen", s.maxSeenLayer, + "feed", extPkt.Packet.SSRC, + ) + s.currentLayer = s.parkedLayer + found = true + } + } else { + if extPkt.KeyFrame { + if layer > s.currentLayer.Spatial && layer <= s.targetLayer.Spatial { + s.logger.Infow( + "upgrading layer", + "current", s.currentLayer, + "target", s.targetLayer, + "max", s.maxLayer, + "layer", layer, + "req", s.requestSpatial, + "maxSeen", s.maxSeenLayer, + "feed", extPkt.Packet.SSRC, + ) + found = true + } + + if layer < s.currentLayer.Spatial && layer >= s.targetLayer.Spatial { + s.logger.Infow( + "downgrading layer", + "current", s.currentLayer, + "target", s.targetLayer, + "max", s.maxLayer, + "layer", layer, + "req", s.requestSpatial, + "maxSeen", s.maxSeenLayer, + "feed", extPkt.Packet.SSRC, + ) + found = true + } + + if found { + s.currentLayer.Spatial = layer + s.currentLayer.Temporal = extPkt.VideoLayer.Temporal + } + } + } + + if found { + if !isActive { + result.IsResuming = true + } + s.SetParked(buffer.InvalidLayers) + if s.currentLayer.Spatial >= s.maxLayer.Spatial { + result.IsSwitchingToMaxSpatial = true + + s.logger.Infow( + "reached max layer", + "current", s.currentLayer, + "target", s.targetLayer, + "max", s.maxLayer, + "layer", layer, + "req", s.requestSpatial, + "maxSeen", s.maxSeenLayer, + "feed", extPkt.Packet.SSRC, + ) + } + + if s.currentLayer.Spatial >= s.maxLayer.Spatial || s.currentLayer.Spatial == s.maxSeenLayer.Spatial { + s.targetLayer.Spatial = s.currentLayer.Spatial + } + } + } + + // if locked to higher than max layer due to overshoot, check if it can be dialed back + if s.currentLayer.Spatial > s.maxLayer.Spatial { + if layer <= s.maxLayer.Spatial && extPkt.KeyFrame { + s.logger.Infow( + "adjusting overshoot", + "current", s.currentLayer, + "target", s.targetLayer, + "max", s.maxLayer, + "layer", layer, + "req", s.requestSpatial, + "maxSeen", s.maxSeenLayer, + "feed", extPkt.Packet.SSRC, + ) + s.currentLayer.Spatial = layer + + if s.currentLayer.Spatial >= s.maxLayer.Spatial { + result.IsSwitchingToMaxSpatial = true + } + + if s.currentLayer.Spatial >= s.maxLayer.Spatial || s.currentLayer.Spatial == s.maxSeenLayer.Spatial { + s.targetLayer.Spatial = layer + } + } + } + + result.RTPMarker = extPkt.Packet.Marker + result.IsSelected = layer == s.currentLayer.Spatial + result.IsRelevant = false + return +} diff --git a/pkg/sfu/videolayerselector/temporallayerselector/null.go b/pkg/sfu/videolayerselector/temporallayerselector/null.go new file mode 100644 index 000000000..d39cab106 --- /dev/null +++ b/pkg/sfu/videolayerselector/temporallayerselector/null.go @@ -0,0 +1,17 @@ +package temporallayerselector + +import ( + "github.com/livekit/livekit-server/pkg/sfu/buffer" +) + +type Null struct{} + +func NewNull() *Null { + return &Null{} +} + +func Select(_extPkt *buffer.ExtPacket, current int32, _target int32) (this int32, next int32) { + this = current + next = current + return +} diff --git a/pkg/sfu/videolayerselector/temporallayerselector/temporallayerselector.go b/pkg/sfu/videolayerselector/temporallayerselector/temporallayerselector.go new file mode 100644 index 000000000..8219aea2d --- /dev/null +++ b/pkg/sfu/videolayerselector/temporallayerselector/temporallayerselector.go @@ -0,0 +1,7 @@ +package temporallayerselector + +import "github.com/livekit/livekit-server/pkg/sfu/buffer" + +type TemporalLayerSelector interface { + Select(extPkt *buffer.ExtPacket, current int32, target int32) (this int32, next int32) +} diff --git a/pkg/sfu/videolayerselector/temporallayerselector/vp8.go b/pkg/sfu/videolayerselector/temporallayerselector/vp8.go new file mode 100644 index 000000000..fbf697f33 --- /dev/null +++ b/pkg/sfu/videolayerselector/temporallayerselector/vp8.go @@ -0,0 +1,42 @@ +package temporallayerselector + +import ( + "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/protocol/logger" +) + +type VP8 struct { + logger logger.Logger +} + +func NewVP8(logger logger.Logger) *VP8 { + return &VP8{ + logger: logger, + } +} + +func (v *VP8) Select(extPkt *buffer.ExtPacket, current int32, target int32) (this int32, next int32) { + this = current + next = current + if current == target { + return + } + + vp8, ok := extPkt.Payload.(buffer.VP8) + if !ok || !vp8.T { + return + } + + tid := int32(vp8.TID) + if current < target { + if tid > current && tid <= target && vp8.S && vp8.Y { + this = tid + next = tid + } + } else { + if tid < current && tid >= target && extPkt.Packet.Marker { + next = tid + } + } + return +} diff --git a/pkg/sfu/videolayerselector/videolayerselector.go b/pkg/sfu/videolayerselector/videolayerselector.go new file mode 100644 index 000000000..548f736dc --- /dev/null +++ b/pkg/sfu/videolayerselector/videolayerselector.go @@ -0,0 +1,46 @@ +package videolayerselector + +import ( + "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/livekit-server/pkg/sfu/videolayerselector/temporallayerselector" +) + +type VideoLayerSelectorResult struct { + IsSelected bool + IsRelevant bool + IsResuming bool + IsSwitchingToMaxSpatial bool + RTPMarker bool + DependencyDescriptorExtension []byte +} + +type VideoLayerSelector interface { + IsOvershootOkay() bool + + SetTemporalLayerSelector(tls temporallayerselector.TemporalLayerSelector) + + SetMax(maxLayer buffer.VideoLayer) + SetMaxSpatial(layer int32) + SetMaxTemporal(layer int32) + GetMax() buffer.VideoLayer + + SetTarget(targetLayer buffer.VideoLayer) + GetTarget() buffer.VideoLayer + + SetRequestSpatial(layer int32) + GetRequestSpatial() int32 + + SetMaxSeen(maxSeenLayer buffer.VideoLayer) + SetMaxSeenSpatial(layer int32) + SetMaxSeenTemporal(layer int32) + GetMaxSeen() buffer.VideoLayer + + SetParked(parkedLayer buffer.VideoLayer) + GetParked() buffer.VideoLayer + + SetCurrent(currentLayer buffer.VideoLayer) + GetCurrent() buffer.VideoLayer + + Select(extPkt *buffer.ExtPacket, layer int32) VideoLayerSelectorResult + SelectTemporal(extPkt *buffer.ExtPacket) int32 +} diff --git a/pkg/sfu/videolayerselector/vp9.go b/pkg/sfu/videolayerselector/vp9.go new file mode 100644 index 000000000..0e7783d98 --- /dev/null +++ b/pkg/sfu/videolayerselector/vp9.go @@ -0,0 +1,102 @@ +package videolayerselector + +import ( + "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/protocol/logger" + "github.com/pion/rtp/codecs" +) + +type VP9 struct { + *Base +} + +func NewVP9(logger logger.Logger) *VP9 { + return &VP9{ + Base: NewBase(logger), + } +} + +func NewVP9FromNull(vls VideoLayerSelector) *VP9 { + return &VP9{ + Base: vls.(*Null).Base, + } +} + +func (v *VP9) IsOvershootOkay() bool { + return false +} + +func (v *VP9) Select(extPkt *buffer.ExtPacket, _layer int32) (result VideoLayerSelectorResult) { + vp9, ok := extPkt.Payload.(codecs.VP9Packet) + if !ok { + return + } + + currentLayer := v.currentLayer + if v.currentLayer != v.targetLayer { + updatedLayer := v.currentLayer + + if !v.currentLayer.IsValid() { + if !extPkt.KeyFrame { + return + } + + updatedLayer = extPkt.VideoLayer + currentLayer = extPkt.VideoLayer + } else { + // temporal scale up/down + if v.currentLayer.Temporal < v.targetLayer.Temporal { + if extPkt.VideoLayer.Temporal > v.currentLayer.Temporal && extPkt.VideoLayer.Temporal <= v.targetLayer.Temporal && vp9.U && vp9.B { + updatedLayer.Temporal = extPkt.VideoLayer.Temporal + currentLayer.Temporal = extPkt.VideoLayer.Temporal + } + } else { + if extPkt.VideoLayer.Temporal < v.currentLayer.Temporal && extPkt.VideoLayer.Temporal >= v.targetLayer.Temporal && vp9.E { + updatedLayer.Temporal = extPkt.VideoLayer.Temporal + } + } + + // spatial scale up/down + if v.currentLayer.Spatial < v.targetLayer.Spatial { + if extPkt.VideoLayer.Spatial > v.currentLayer.Spatial && extPkt.VideoLayer.Spatial <= v.targetLayer.Spatial && !vp9.P && vp9.B { + updatedLayer.Spatial = extPkt.VideoLayer.Spatial + currentLayer.Spatial = extPkt.VideoLayer.Spatial + } + } else { + if extPkt.VideoLayer.Spatial < v.currentLayer.Spatial && extPkt.VideoLayer.Spatial >= v.targetLayer.Spatial && vp9.E { + updatedLayer.Spatial = extPkt.VideoLayer.Spatial + } + } + } + + if updatedLayer != v.currentLayer { + if !v.currentLayer.IsValid() && updatedLayer.IsValid() { + result.IsResuming = true + } + + if v.currentLayer.Spatial != v.maxLayer.Spatial && updatedLayer.Spatial == v.maxLayer.Spatial { + result.IsSwitchingToMaxSpatial = true + v.logger.Infow( + "reached max layer", + "current", v.currentLayer, + "target", v.targetLayer, + "max", v.maxLayer, + "layer", extPkt.VideoLayer.Spatial, + "req", v.requestSpatial, + "maxSeen", v.maxSeenLayer, + "feed", extPkt.Packet.SSRC, + ) + } + + v.currentLayer = updatedLayer + } + } + + result.RTPMarker = extPkt.Packet.Marker + if extPkt.VideoLayer.Spatial == v.currentLayer.Spatial && vp9.E { + result.RTPMarker = true + } + result.IsSelected = !extPkt.VideoLayer.GreaterThan(currentLayer) + result.IsRelevant = true + return +} From e03f75d6a1bc3ae4096cb2a8c94dd25962ba37fb Mon Sep 17 00:00:00 2001 From: David Zhao Date: Fri, 7 Apr 2023 23:47:49 -0700 Subject: [PATCH 064/324] Implements source-specific permissions and client-driven metadata updates (#1590) Closes #1565 --- go.mod | 2 +- go.sum | 4 +- pkg/rtc/helper_test.go | 2 +- pkg/rtc/participant.go | 69 ++++----- pkg/rtc/participant_internal_test.go | 26 ++++ pkg/rtc/signalhandler.go | 10 ++ pkg/rtc/types/interfaces.go | 2 +- .../typesfakes/fake_local_participant.go | 139 ++++++++++-------- pkg/service/rtcservice.go | 2 +- 9 files changed, 147 insertions(+), 109 deletions(-) diff --git a/go.mod b/go.mod index 55bdf34be..7c052ce66 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 - github.com/livekit/protocol v1.5.2-0.20230405195605-927c9ea2b4c6 + github.com/livekit/protocol v1.5.2 github.com/livekit/psrpc v0.2.11-0.20230405191830-d76f71512630 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.14.0 diff --git a/go.sum b/go.sum index c963460f4..59640e93b 100644 --- a/go.sum +++ b/go.sum @@ -235,8 +235,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 h1:QlQFyMwCDgjyySsrgmrMcVbEBA6KZcyTzvK+z346tUA= github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26/go.mod h1:eDA41kiySZoG+wy4Etsjb3w0jjLx69i/vAmSjG4bteA= -github.com/livekit/protocol v1.5.2-0.20230405195605-927c9ea2b4c6 h1:rvkmoc5s+VJTpShWkY+QWFQ5XhLDMFFyZIZrrr7PgJE= -github.com/livekit/protocol v1.5.2-0.20230405195605-927c9ea2b4c6/go.mod h1:UFgAWejoO4eshaaDe2jynTdQWwSktNO+8Wx19V7bs+o= +github.com/livekit/protocol v1.5.2 h1:mbbkJNxbStvb9sDtB7CFX7NnTObYKFumNU7wWm4UOfY= +github.com/livekit/protocol v1.5.2/go.mod h1:UFgAWejoO4eshaaDe2jynTdQWwSktNO+8Wx19V7bs+o= github.com/livekit/psrpc v0.2.11-0.20230405191830-d76f71512630 h1:Rm5KLZgQxWnTidY+H8MsAV6sk1iiFxeXqPFgSLkMing= github.com/livekit/psrpc v0.2.11-0.20230405191830-d76f71512630/go.mod h1:K0j8f1PgLShR7Lx80KbmwFkDH2BvOnycXGV0OSRURKc= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= diff --git a/pkg/rtc/helper_test.go b/pkg/rtc/helper_test.go index dff552233..dc97f13a0 100644 --- a/pkg/rtc/helper_test.go +++ b/pkg/rtc/helper_test.go @@ -21,7 +21,7 @@ func newMockParticipant(identity livekit.ParticipantIdentity, protocol types.Pro p.StateReturns(livekit.ParticipantInfo_JOINED) p.ProtocolVersionReturns(protocol) p.CanSubscribeReturns(true) - p.CanPublishReturns(!hidden) + p.CanPublishSourceReturns(!hidden) p.CanPublishDataReturns(!hidden) p.HiddenReturns(hidden) p.ToProtoReturns(&livekit.ParticipantInfo{ diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 3e140499c..aa3d4443d 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -339,22 +339,13 @@ func (p *ParticipantImpl) SetPermission(permission *livekit.ParticipantPermissio } p.lock.Lock() video := p.grants.Video - hasChanged := video.GetCanSubscribe() != permission.CanSubscribe || - video.GetCanPublish() != permission.CanPublish || - video.GetCanPublishData() != permission.CanPublishData || - video.Hidden != permission.Hidden || - video.Recorder != permission.Recorder - if !hasChanged { + if video.MatchesPermission(permission) { p.lock.Unlock() return false } - video.SetCanSubscribe(permission.CanSubscribe) - video.SetCanPublish(permission.CanPublish) - video.SetCanPublishData(permission.CanPublishData) - video.Hidden = permission.Hidden - video.Recorder = permission.Recorder + video.UpdateFromPermission(permission) canPublish := video.GetCanPublish() canSubscribe := video.GetCanSubscribe() @@ -362,9 +353,9 @@ func (p *ParticipantImpl) SetPermission(permission *livekit.ParticipantPermissio onClaimsChanged := p.onClaimsChanged p.lock.Unlock() - // publish permission has been revoked then remove all published tracks - if !canPublish { - for _, track := range p.GetPublishedTracks() { + // publish permission has been revoked then remove offending tracks + for _, track := range p.GetPublishedTracks() { + if !video.GetCanPublishSource(track.Source()) { p.RemovePublishedTrack(track, false, false) if p.ProtocolVersion().SupportsUnpublish() { p.sendTrackUnpublished(track.ID()) @@ -374,6 +365,7 @@ func (p *ParticipantImpl) SetPermission(permission *livekit.ParticipantPermissio } } } + if canSubscribe { // reconcile everything p.SubscriptionManager.queueReconcile("") @@ -519,10 +511,6 @@ func (p *ParticipantImpl) onPublisherAnswer(answer webrtc.SessionDescription) er return err } - // received an offer from the client, if publishing is allowed, mark this - // participant as a publisher - p.setIsPublisher(p.CanPublish()) - if p.MigrateState() == types.MigrateStateSync { go p.handleMigrateMutedTrack() } @@ -582,7 +570,7 @@ func (p *ParticipantImpl) AddTrack(req *livekit.AddTrackRequest) { p.lock.Lock() defer p.lock.Unlock() - if !p.grants.Video.GetCanPublish() { + if !p.grants.Video.GetCanPublishSource(req.Source) { p.params.Logger.Warnw("no permission to publish track", nil) return } @@ -872,11 +860,10 @@ func (p *ParticipantImpl) IsPublisher() bool { return p.isPublisher.Load() } -func (p *ParticipantImpl) CanPublish() bool { +func (p *ParticipantImpl) CanPublishSource(source livekit.TrackSource) bool { p.lock.RLock() defer p.lock.RUnlock() - - return p.grants.Video.GetCanPublish() + return p.grants.Video.GetCanPublishSource(source) } func (p *ParticipantImpl) CanSubscribe() bool { @@ -1136,22 +1123,8 @@ func (p *ParticipantImpl) onMediaTrack(track *webrtc.TrackRemote, rtpReceiver *w return } - if !p.CanPublish() { - p.params.Logger.Warnw("no permission to publish mediaTrack", nil) - return - } - publishedTrack, isNewTrack := p.mediaTrackReceived(track, rtpReceiver) - - if publishedTrack != nil { - p.params.Logger.Infow("mediaTrack published", - "kind", track.Kind().String(), - "trackID", publishedTrack.ID(), - "rid", track.RID(), - "SSRC", track.SSRC(), - "mime", track.Codec().MimeType, - ) - } else { + if publishedTrack == nil { p.params.Logger.Warnw("webrtc Track published but can't find MediaTrack", nil, "kind", track.Kind().String(), "webrtcTrackID", track.ID(), @@ -1159,9 +1132,29 @@ func (p *ParticipantImpl) onMediaTrack(track *webrtc.TrackRemote, rtpReceiver *w "SSRC", track.SSRC(), "mime", track.Codec().MimeType, ) + return } - if !isNewTrack && publishedTrack != nil && !publishedTrack.HasPendingCodec() && p.IsReady() { + if !p.CanPublishSource(publishedTrack.Source()) { + p.params.Logger.Warnw("no permission to publish mediaTrack", nil, + "source", publishedTrack.Source(), + ) + return + } + + if !p.IsPublisher() { + p.setIsPublisher(true) + } + + p.params.Logger.Infow("mediaTrack published", + "kind", track.Kind().String(), + "trackID", publishedTrack.ID(), + "rid", track.RID(), + "SSRC", track.SSRC(), + "mime", track.Codec().MimeType, + ) + + if !isNewTrack && !publishedTrack.HasPendingCodec() && p.IsReady() { p.lock.RLock() onTrackUpdated := p.onTrackUpdated p.lock.RUnlock() diff --git a/pkg/rtc/participant_internal_test.go b/pkg/rtc/participant_internal_test.go index 4d294c646..3c585c3b0 100644 --- a/pkg/rtc/participant_internal_test.go +++ b/pkg/rtc/participant_internal_test.go @@ -176,6 +176,32 @@ func TestTrackPublishing(t *testing.T) { // check SID is the same require.Equal(t, p.pendingTracks["cid"].trackInfos[0].Sid, p.pendingTracks["cid"].trackInfos[1].Sid) }) + + t.Run("should not allow adding disallowed sources", func(t *testing.T) { + p := newParticipantForTest("test") + p.SetPermission(&livekit.ParticipantPermission{ + CanPublish: true, + CanPublishSources: []livekit.TrackSource{ + livekit.TrackSource_CAMERA, + }, + }) + sink := p.params.Sink.(*routingfakes.FakeMessageSink) + p.AddTrack(&livekit.AddTrackRequest{ + Cid: "cid", + Name: "webcam", + Source: livekit.TrackSource_CAMERA, + Type: livekit.TrackType_VIDEO, + }) + require.Equal(t, 1, sink.WriteMessageCallCount()) + + p.AddTrack(&livekit.AddTrackRequest{ + Cid: "cid2", + Name: "rejected source", + Type: livekit.TrackType_AUDIO, + Source: livekit.TrackSource_MICROPHONE, + }) + require.Equal(t, 1, sink.WriteMessageCallCount()) + }) } func TestOutOfOrderUpdates(t *testing.T) { diff --git a/pkg/rtc/signalhandler.go b/pkg/rtc/signalhandler.go index dc49ade12..e8749aa64 100644 --- a/pkg/rtc/signalhandler.go +++ b/pkg/rtc/signalhandler.go @@ -73,6 +73,16 @@ func HandleParticipantSignal(room types.Room, participant types.LocalParticipant if msg.PingReq.Rtt > 0 { participant.UpdateSignalingRTT(uint32(msg.PingReq.Rtt)) } + + case *livekit.SignalRequest_UpdateMetadata: + if participant.ClaimGrants().Video.GetCanUpdateOwnMetadata() { + if msg.UpdateMetadata.Metadata != "" { + participant.SetMetadata(msg.UpdateMetadata.Metadata) + } + if msg.UpdateMetadata.Name != "" { + participant.SetName(msg.UpdateMetadata.Name) + } + } } return nil } diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index 6bfc95edf..a83c6206f 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -251,7 +251,7 @@ type LocalParticipant interface { // permissions ClaimGrants() *auth.ClaimGrants SetPermission(permission *livekit.ParticipantPermission) bool - CanPublish() bool + CanPublishSource(source livekit.TrackSource) bool CanSubscribe() bool CanPublishData() bool diff --git a/pkg/rtc/types/typesfakes/fake_local_participant.go b/pkg/rtc/types/typesfakes/fake_local_participant.go index 67ce745dd..7ae08e75c 100644 --- a/pkg/rtc/types/typesfakes/fake_local_participant.go +++ b/pkg/rtc/types/typesfakes/fake_local_participant.go @@ -67,16 +67,6 @@ type FakeLocalParticipant struct { arg2 *webrtc.RTPTransceiver arg3 sfu.DownTrackState } - CanPublishStub func() bool - canPublishMutex sync.RWMutex - canPublishArgsForCall []struct { - } - canPublishReturns struct { - result1 bool - } - canPublishReturnsOnCall map[int]struct { - result1 bool - } CanPublishDataStub func() bool canPublishDataMutex sync.RWMutex canPublishDataArgsForCall []struct { @@ -87,6 +77,17 @@ type FakeLocalParticipant struct { canPublishDataReturnsOnCall map[int]struct { result1 bool } + CanPublishSourceStub func(livekit.TrackSource) bool + canPublishSourceMutex sync.RWMutex + canPublishSourceArgsForCall []struct { + arg1 livekit.TrackSource + } + canPublishSourceReturns struct { + result1 bool + } + canPublishSourceReturnsOnCall map[int]struct { + result1 bool + } CanSubscribeStub func() bool canSubscribeMutex sync.RWMutex canSubscribeArgsForCall []struct { @@ -1060,59 +1061,6 @@ func (fake *FakeLocalParticipant) CacheDownTrackArgsForCall(i int) (livekit.Trac return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 } -func (fake *FakeLocalParticipant) CanPublish() bool { - fake.canPublishMutex.Lock() - ret, specificReturn := fake.canPublishReturnsOnCall[len(fake.canPublishArgsForCall)] - fake.canPublishArgsForCall = append(fake.canPublishArgsForCall, struct { - }{}) - stub := fake.CanPublishStub - fakeReturns := fake.canPublishReturns - fake.recordInvocation("CanPublish", []interface{}{}) - fake.canPublishMutex.Unlock() - if stub != nil { - return stub() - } - if specificReturn { - return ret.result1 - } - return fakeReturns.result1 -} - -func (fake *FakeLocalParticipant) CanPublishCallCount() int { - fake.canPublishMutex.RLock() - defer fake.canPublishMutex.RUnlock() - return len(fake.canPublishArgsForCall) -} - -func (fake *FakeLocalParticipant) CanPublishCalls(stub func() bool) { - fake.canPublishMutex.Lock() - defer fake.canPublishMutex.Unlock() - fake.CanPublishStub = stub -} - -func (fake *FakeLocalParticipant) CanPublishReturns(result1 bool) { - fake.canPublishMutex.Lock() - defer fake.canPublishMutex.Unlock() - fake.CanPublishStub = nil - fake.canPublishReturns = struct { - result1 bool - }{result1} -} - -func (fake *FakeLocalParticipant) CanPublishReturnsOnCall(i int, result1 bool) { - fake.canPublishMutex.Lock() - defer fake.canPublishMutex.Unlock() - fake.CanPublishStub = nil - if fake.canPublishReturnsOnCall == nil { - fake.canPublishReturnsOnCall = make(map[int]struct { - result1 bool - }) - } - fake.canPublishReturnsOnCall[i] = struct { - result1 bool - }{result1} -} - func (fake *FakeLocalParticipant) CanPublishData() bool { fake.canPublishDataMutex.Lock() ret, specificReturn := fake.canPublishDataReturnsOnCall[len(fake.canPublishDataArgsForCall)] @@ -1166,6 +1114,67 @@ func (fake *FakeLocalParticipant) CanPublishDataReturnsOnCall(i int, result1 boo }{result1} } +func (fake *FakeLocalParticipant) CanPublishSource(arg1 livekit.TrackSource) bool { + fake.canPublishSourceMutex.Lock() + ret, specificReturn := fake.canPublishSourceReturnsOnCall[len(fake.canPublishSourceArgsForCall)] + fake.canPublishSourceArgsForCall = append(fake.canPublishSourceArgsForCall, struct { + arg1 livekit.TrackSource + }{arg1}) + stub := fake.CanPublishSourceStub + fakeReturns := fake.canPublishSourceReturns + fake.recordInvocation("CanPublishSource", []interface{}{arg1}) + fake.canPublishSourceMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) CanPublishSourceCallCount() int { + fake.canPublishSourceMutex.RLock() + defer fake.canPublishSourceMutex.RUnlock() + return len(fake.canPublishSourceArgsForCall) +} + +func (fake *FakeLocalParticipant) CanPublishSourceCalls(stub func(livekit.TrackSource) bool) { + fake.canPublishSourceMutex.Lock() + defer fake.canPublishSourceMutex.Unlock() + fake.CanPublishSourceStub = stub +} + +func (fake *FakeLocalParticipant) CanPublishSourceArgsForCall(i int) livekit.TrackSource { + fake.canPublishSourceMutex.RLock() + defer fake.canPublishSourceMutex.RUnlock() + argsForCall := fake.canPublishSourceArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeLocalParticipant) CanPublishSourceReturns(result1 bool) { + fake.canPublishSourceMutex.Lock() + defer fake.canPublishSourceMutex.Unlock() + fake.CanPublishSourceStub = nil + fake.canPublishSourceReturns = struct { + result1 bool + }{result1} +} + +func (fake *FakeLocalParticipant) CanPublishSourceReturnsOnCall(i int, result1 bool) { + fake.canPublishSourceMutex.Lock() + defer fake.canPublishSourceMutex.Unlock() + fake.CanPublishSourceStub = nil + if fake.canPublishSourceReturnsOnCall == nil { + fake.canPublishSourceReturnsOnCall = make(map[int]struct { + result1 bool + }) + } + fake.canPublishSourceReturnsOnCall[i] = struct { + result1 bool + }{result1} +} + func (fake *FakeLocalParticipant) CanSubscribe() bool { fake.canSubscribeMutex.Lock() ret, specificReturn := fake.canSubscribeReturnsOnCall[len(fake.canSubscribeArgsForCall)] @@ -5221,10 +5230,10 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.addTransceiverFromTrackToSubscriberMutex.RUnlock() fake.cacheDownTrackMutex.RLock() defer fake.cacheDownTrackMutex.RUnlock() - fake.canPublishMutex.RLock() - defer fake.canPublishMutex.RUnlock() fake.canPublishDataMutex.RLock() defer fake.canPublishDataMutex.RUnlock() + fake.canPublishSourceMutex.RLock() + defer fake.canPublishSourceMutex.RUnlock() fake.canSubscribeMutex.RLock() defer fake.canSubscribeMutex.RUnlock() fake.claimGrantsMutex.RLock() diff --git a/pkg/service/rtcservice.go b/pkg/service/rtcservice.go index 22592a4c5..52d633423 100644 --- a/pkg/service/rtcservice.go +++ b/pkg/service/rtcservice.go @@ -110,7 +110,7 @@ func (s *RTCService) validate(r *http.Request) (livekit.RoomName, routing.Partic // this is new connection for existing participant - with publish only permissions if publishParam != "" { - // Make sure grant has CanPublish set, + // Make sure grant has GetCanPublish set, if !claims.Video.GetCanPublish() { return "", routing.ParticipantInit{}, http.StatusUnauthorized, rtc.ErrPermissionDenied } From 4969b57c09c5dc9bea213929aaa405ef6f9e09b2 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sat, 8 Apr 2023 12:39:02 +0530 Subject: [PATCH 065/324] Chaging VideoLayers -> VideoLayer (#1591) There was mixed used. It is a struct. So, it is a singular. Change all the places I could find. There may be more, but can be changed when spotted. --- pkg/sfu/buffer/videolayer.go | 4 +- pkg/sfu/downtrack.go | 40 +-- pkg/sfu/forwarder.go | 344 +++++++++--------- pkg/sfu/forwarder_test.go | 384 ++++++++++----------- pkg/sfu/streamallocator/streamallocator.go | 22 +- pkg/sfu/streamallocator/track.go | 20 +- pkg/sfu/streamtrackermanager.go | 16 +- pkg/sfu/videolayerselector/base.go | 10 +- pkg/sfu/videolayerselector/simulcast.go | 2 +- 9 files changed, 420 insertions(+), 422 deletions(-) diff --git a/pkg/sfu/buffer/videolayer.go b/pkg/sfu/buffer/videolayer.go index 15dd655f5..eedf0b2c8 100644 --- a/pkg/sfu/buffer/videolayer.go +++ b/pkg/sfu/buffer/videolayer.go @@ -11,12 +11,12 @@ const ( ) var ( - InvalidLayers = VideoLayer{ + InvalidLayer = VideoLayer{ Spatial: InvalidLayerSpatial, Temporal: InvalidLayerTemporal, } - DefaultMaxLayers = VideoLayer{ + DefaultMaxLayer = VideoLayer{ Spatial: DefaultMaxLayerSpatial, Temporal: DefaultMaxLayerTemporal, } diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 3c75064fc..4043ccd19 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -137,7 +137,7 @@ type DownTrackStreamAllocatorListener interface { OnSubscriptionChanged(dt *DownTrack) // subscribed max video layer changed - OnSubscribedLayersChanged(dt *DownTrack, layers buffer.VideoLayer) + OnSubscribedLayerChanged(dt *DownTrack, layers buffer.VideoLayer) // stream resumed OnResume(dt *DownTrack) @@ -259,7 +259,7 @@ func NewDownTrack( codec: codecs[0].RTPCodecCapability, } d.forwarder = NewForwarder(d.kind, d.logger, d.receiver.GetReferenceLayerRTPTimestamp) - d.forwarder.OnParkedLayersExpired(func() { + d.forwarder.OnParkedLayerExpired(func() { if sal := d.getStreamAllocatorListener(); sal != nil { sal.OnSubscriptionChanged(d) } @@ -599,10 +599,8 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { if extPkt.KeyFrame { d.isNACKThrottled.Store(false) - if extPkt.KeyFrame { - d.rtpStats.UpdateKeyFrame(1) - d.logger.Debugw("forwarding key frame", "layer", layer) - } + d.rtpStats.UpdateKeyFrame(1) + d.logger.Debugw("forwarding key frame", "layer", layer) // SVC-TODO - no need for key frame always when using SVC locked, _ := d.forwarder.CheckSync() @@ -716,17 +714,17 @@ func (d *DownTrack) WritePaddingRTP(bytesToSend int, paddingOnMute bool) int { // Mute enables or disables media forwarding - subscriber triggered func (d *DownTrack) Mute(muted bool) { - changed, maxLayers := d.forwarder.Mute(muted) - d.handleMute(muted, false, changed, maxLayers) + changed, maxLayer := d.forwarder.Mute(muted) + d.handleMute(muted, false, changed, maxLayer) } // PubMute enables or disables media forwarding - publisher side func (d *DownTrack) PubMute(pubMuted bool) { - changed, maxLayers := d.forwarder.PubMute(pubMuted) - d.handleMute(pubMuted, true, changed, maxLayers) + changed, maxLayer := d.forwarder.PubMute(pubMuted) + d.handleMute(pubMuted, true, changed, maxLayer) } -func (d *DownTrack) handleMute(muted bool, isPub bool, changed bool, maxLayers buffer.VideoLayer) { +func (d *DownTrack) handleMute(muted bool, isPub bool, changed bool, maxLayer buffer.VideoLayer) { if !changed { return } @@ -761,7 +759,7 @@ func (d *DownTrack) handleMute(muted bool, isPub bool, changed bool, maxLayers b // client might need to be notified to start layers // before locking can happen in the forwarder. // - notifyLayer = maxLayers.Spatial + notifyLayer = maxLayer.Spatial } d.onMaxSubscribedLayerChanged(d, notifyLayer) } @@ -860,12 +858,12 @@ func (d *DownTrack) CloseWithFlush(flush bool) { } func (d *DownTrack) SetMaxSpatialLayer(spatialLayer int32) { - changed, maxLayers, currentLayers := d.forwarder.SetMaxSpatialLayer(spatialLayer) + changed, maxLayer, currentLayer := d.forwarder.SetMaxSpatialLayer(spatialLayer) if !changed { return } - if d.onMaxSubscribedLayerChanged != nil && d.kind == webrtc.RTPCodecTypeVideo && maxLayers.SpatialGreaterThanOrEqual(currentLayers) { + if d.onMaxSubscribedLayerChanged != nil && d.kind == webrtc.RTPCodecTypeVideo && maxLayer.SpatialGreaterThanOrEqual(currentLayer) { // // Notify when new max is // 1. Equal to current -> already locked to the new max @@ -873,27 +871,27 @@ func (d *DownTrack) SetMaxSpatialLayer(spatialLayer int32) { // a. is higher than previous max -> client may need to start higher layer before forwarder can lock // b. is lower than previous max -> client can stop higher layer(s) // - d.onMaxSubscribedLayerChanged(d, maxLayers.Spatial) + d.onMaxSubscribedLayerChanged(d, maxLayer.Spatial) } if sal := d.getStreamAllocatorListener(); sal != nil { - sal.OnSubscribedLayersChanged(d, maxLayers) + sal.OnSubscribedLayerChanged(d, maxLayer) } } func (d *DownTrack) SetMaxTemporalLayer(temporalLayer int32) { - changed, maxLayers, _ := d.forwarder.SetMaxTemporalLayer(temporalLayer) + changed, maxLayer, _ := d.forwarder.SetMaxTemporalLayer(temporalLayer) if !changed { return } if sal := d.getStreamAllocatorListener(); sal != nil { - sal.OnSubscribedLayersChanged(d, maxLayers) + sal.OnSubscribedLayerChanged(d, maxLayer) } } -func (d *DownTrack) MaxLayers() buffer.VideoLayer { - return d.forwarder.MaxLayers() +func (d *DownTrack) MaxLayer() buffer.VideoLayer { + return d.forwarder.MaxLayer() } func (d *DownTrack) GetState() DownTrackState { @@ -1566,7 +1564,7 @@ func (d *DownTrack) DebugInfo() map[string]interface{} { "Bound": d.bound.Load(), "Muted": d.forwarder.IsMuted(), "PubMuted": d.forwarder.IsPubMuted(), - "CurrentSpatialLayer": d.forwarder.CurrentLayers().Spatial, + "CurrentSpatialLayer": d.forwarder.CurrentLayer().Spatial, "Stats": stats, } } diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index f29ad906f..de689e8dd 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -20,10 +20,10 @@ import ( // Forwarder const ( - FlagPauseOnDowngrade = true - FlagFilterRTX = true - TransitionCostSpatial = 10 - ParkedLayersWaitDuration = 2 * time.Second + FlagPauseOnDowngrade = true + FlagFilterRTX = true + TransitionCostSpatial = 10 + ParkedLayerWaitDuration = 2 * time.Second ) // ------------------------------------------------------------------- @@ -64,9 +64,9 @@ type VideoAllocation struct { BandwidthDelta int64 BandwidthNeeded int64 Bitrates Bitrates - TargetLayers buffer.VideoLayer + TargetLayer buffer.VideoLayer RequestLayerSpatial int32 - MaxLayers buffer.VideoLayer + MaxLayer buffer.VideoLayer DistanceToDesired float64 } @@ -78,9 +78,9 @@ func (v VideoAllocation) String() string { v.BandwidthDelta, v.BandwidthNeeded, v.Bitrates, - v.TargetLayers, + v.TargetLayer, v.RequestLayerSpatial, - v.MaxLayers, + v.MaxLayer, v.DistanceToDesired, ) } @@ -88,9 +88,9 @@ func (v VideoAllocation) String() string { var ( VideoAllocationDefault = VideoAllocation{ PauseReason: VideoPauseReasonFeedDry, // start with no feed till feed is seen - TargetLayers: buffer.InvalidLayers, + TargetLayer: buffer.InvalidLayer, RequestLayerSpatial: buffer.InvalidLayerSpatial, - MaxLayers: buffer.InvalidLayers, + MaxLayer: buffer.InvalidLayer, } ) @@ -102,10 +102,10 @@ type VideoAllocationProvisional struct { maxSeenLayer buffer.VideoLayer availableLayers []int32 Bitrates Bitrates - maxLayers buffer.VideoLayer - currentLayers buffer.VideoLayer - parkedLayers buffer.VideoLayer - allocatedLayers buffer.VideoLayer + maxLayer buffer.VideoLayer + currentLayer buffer.VideoLayer + parkedLayer buffer.VideoLayer + allocatedLayer buffer.VideoLayer } // ------------------------------------------------------------------- @@ -165,7 +165,7 @@ type Forwarder struct { lastSSRC uint32 referenceLayerSpatial int32 - parkedLayersTimer *time.Timer + parkedLayerTimer *time.Timer provisional *VideoAllocationProvisional @@ -177,7 +177,7 @@ type Forwarder struct { codecMunger codecmunger.CodecMunger - onParkedLayersExpired func() + onParkedLayerExpired func() } func NewForwarder( @@ -228,18 +228,18 @@ func (f *Forwarder) SetMaxTemporalLayerSeen(maxTemporalLayerSeen int32) { f.logger.Debugw("setting max temporal layer seen", "maxTemporalLayerSeen", maxTemporalLayerSeen) } -func (f *Forwarder) OnParkedLayersExpired(fn func()) { +func (f *Forwarder) OnParkedLayerExpired(fn func()) { f.lock.Lock() defer f.lock.Unlock() - f.onParkedLayersExpired = fn + f.onParkedLayerExpired = fn } -func (f *Forwarder) getOnParkedLayersExpired() func() { +func (f *Forwarder) getOnParkedLayerExpired() func() { f.lock.RLock() defer f.lock.RUnlock() - return f.onParkedLayersExpired + return f.onParkedLayerExpired } func (f *Forwarder) DetermineCodec(codec webrtc.RTPCodecCapability, extensions []webrtc.RTPHeaderExtensionParameter) { @@ -374,12 +374,12 @@ func (f *Forwarder) PubMute(pubMuted bool) (bool, buffer.VideoLayer) { f.resyncLocked() } } else { - // Do not resync on publisher mute as forwarding can continue on unmute using same layers. + // Do not resync on publisher mute as forwarding can continue on unmute using same layer. // On unmute, park current layers as streaming can continue without a key frame when publisher starts the stream. targetLayer := f.vls.GetTarget() if !pubMuted && targetLayer.IsValid() && f.vls.GetCurrent().Spatial == targetLayer.Spatial { - f.setupParkedLayers(targetLayer) - f.vls.SetCurrent(buffer.InvalidLayers) + f.setupParkedLayer(targetLayer) + f.vls.SetCurrent(buffer.InvalidLayer) } } @@ -405,7 +405,7 @@ func (f *Forwarder) SetMaxSpatialLayer(spatialLayer int32) (bool, buffer.VideoLa defer f.lock.Unlock() if f.kind == webrtc.RTPCodecTypeAudio { - return false, buffer.InvalidLayers, buffer.InvalidLayers + return false, buffer.InvalidLayer, buffer.InvalidLayer } existingMax := f.vls.GetMax() @@ -416,7 +416,7 @@ func (f *Forwarder) SetMaxSpatialLayer(spatialLayer int32) (bool, buffer.VideoLa f.logger.Debugw("setting max spatial layer", "layer", spatialLayer) f.vls.SetMaxSpatial(spatialLayer) - f.clearParkedLayers() + f.clearParkedLayer() return true, f.vls.GetMax(), f.vls.GetCurrent() } @@ -426,7 +426,7 @@ func (f *Forwarder) SetMaxTemporalLayer(temporalLayer int32) (bool, buffer.Video defer f.lock.Unlock() if f.kind == webrtc.RTPCodecTypeAudio { - return false, buffer.InvalidLayers, buffer.InvalidLayers + return false, buffer.InvalidLayer, buffer.InvalidLayer } existingMax := f.vls.GetMax() @@ -437,26 +437,26 @@ func (f *Forwarder) SetMaxTemporalLayer(temporalLayer int32) (bool, buffer.Video f.logger.Debugw("setting max temporal layer", "layer", temporalLayer) f.vls.SetMaxTemporal(temporalLayer) - f.clearParkedLayers() + f.clearParkedLayer() return true, f.vls.GetMax(), f.vls.GetCurrent() } -func (f *Forwarder) MaxLayers() buffer.VideoLayer { +func (f *Forwarder) MaxLayer() buffer.VideoLayer { f.lock.RLock() defer f.lock.RUnlock() return f.vls.GetMax() } -func (f *Forwarder) CurrentLayers() buffer.VideoLayer { +func (f *Forwarder) CurrentLayer() buffer.VideoLayer { f.lock.RLock() defer f.lock.RUnlock() return f.vls.GetCurrent() } -func (f *Forwarder) TargetLayers() buffer.VideoLayer { +func (f *Forwarder) TargetLayer() buffer.VideoLayer { f.lock.RLock() defer f.lock.RUnlock() @@ -526,9 +526,9 @@ func (f *Forwarder) AllocateOptimal(availableLayers []int32, brs Bitrates, allow alloc := VideoAllocation{ PauseReason: VideoPauseReasonNone, Bitrates: brs, - TargetLayers: buffer.InvalidLayers, + TargetLayer: buffer.InvalidLayer, RequestLayerSpatial: requestSpatial, - MaxLayers: maxLayer, + MaxLayer: maxLayer, } optimalBandwidthNeeded := getOptimalBandwidthNeeded(f.muted, f.pubMuted, maxSeenLayer.Spatial, brs, maxLayer) if optimalBandwidthNeeded == 0 { @@ -542,7 +542,7 @@ func (f *Forwarder) AllocateOptimal(availableLayers []int32, brs Bitrates, allow if allowOvershoot && f.vls.IsOvershootOkay() && maxSeenLayer.Spatial > maxSpatial { maxSpatial = maxSeenLayer.Spatial } - alloc.TargetLayers = buffer.VideoLayer{ + alloc.TargetLayer = buffer.VideoLayer{ Spatial: int32(math.Min(float64(maxSeenLayer.Spatial), float64(maxSpatial))), Temporal: maxLayer.Temporal, } @@ -558,13 +558,13 @@ func (f *Forwarder) AllocateOptimal(availableLayers []int32, brs Bitrates, allow case f.pubMuted: alloc.PauseReason = VideoPauseReasonPubMuted // leave it at current layers for opportunistic resume - alloc.TargetLayers = currentLayer - alloc.RequestLayerSpatial = alloc.TargetLayers.Spatial + alloc.TargetLayer = currentLayer + alloc.RequestLayerSpatial = alloc.TargetLayer.Spatial case parkedLayer.IsValid(): // if parked on a layer, let it continue - alloc.TargetLayers = parkedLayer - alloc.RequestLayerSpatial = alloc.TargetLayers.Spatial + alloc.TargetLayer = parkedLayer + alloc.RequestLayerSpatial = alloc.TargetLayer.Spatial case len(availableLayers) == 0: // feed may be dry @@ -573,8 +573,8 @@ func (f *Forwarder) AllocateOptimal(availableLayers []int32, brs Bitrates, allow // Covers the cases of // 1. mis-detection of layer stop - can continue streaming // 2. current layer resuming - can latch on when it starts - alloc.TargetLayers = currentLayer - alloc.RequestLayerSpatial = alloc.TargetLayers.Spatial + alloc.TargetLayer = currentLayer + alloc.RequestLayerSpatial = alloc.TargetLayer.Spatial } else { // opportunistically latch on to anything opportunisticAlloc() @@ -595,18 +595,18 @@ func (f *Forwarder) AllocateOptimal(availableLayers []int32, brs Bitrates, allow if !isCurrentLayerAvailable && currentLayer.IsValid() { // current layer maybe stopped, move to highest available for _, l := range availableLayers { - if l > alloc.TargetLayers.Spatial { - alloc.TargetLayers.Spatial = l + if l > alloc.TargetLayer.Spatial { + alloc.TargetLayer.Spatial = l } } - alloc.TargetLayers.Temporal = maxLayer.Temporal + alloc.TargetLayer.Temporal = maxLayer.Temporal - alloc.RequestLayerSpatial = alloc.TargetLayers.Spatial + alloc.RequestLayerSpatial = alloc.TargetLayer.Spatial } else { requestLayerSpatial := int32(math.Min(float64(maxLayer.Spatial), float64(maxSeenLayer.Spatial))) if currentLayer.IsValid() && requestLayerSpatial == requestSpatial && currentLayer.Spatial == requestSpatial { // current is locked to desired, stay there - alloc.TargetLayers = buffer.VideoLayer{ + alloc.TargetLayer = buffer.VideoLayer{ Spatial: requestSpatial, Temporal: maxLayer.Temporal, } @@ -619,11 +619,11 @@ func (f *Forwarder) AllocateOptimal(availableLayers []int32, brs Bitrates, allow } } - if !alloc.TargetLayers.IsValid() { - alloc.TargetLayers = buffer.InvalidLayers + if !alloc.TargetLayer.IsValid() { + alloc.TargetLayer = buffer.InvalidLayer alloc.RequestLayerSpatial = buffer.InvalidLayerSpatial } - if alloc.TargetLayers.IsValid() { + if alloc.TargetLayer.IsValid() { alloc.BandwidthRequested = optimalBandwidthNeeded } alloc.BandwidthDelta = alloc.BandwidthRequested - f.lastAllocation.BandwidthRequested @@ -633,7 +633,7 @@ func (f *Forwarder) AllocateOptimal(availableLayers []int32, brs Bitrates, allow f.vls.GetMaxSeen(), availableLayers, brs, - alloc.TargetLayers, + alloc.TargetLayer, f.vls.GetMax(), ) @@ -645,45 +645,45 @@ func (f *Forwarder) ProvisionalAllocatePrepare(availableLayers []int32, Bitrates defer f.lock.Unlock() f.provisional = &VideoAllocationProvisional{ - allocatedLayers: buffer.InvalidLayers, - muted: f.muted, - pubMuted: f.pubMuted, - maxSeenLayer: f.vls.GetMaxSeen(), - Bitrates: Bitrates, - maxLayers: f.vls.GetMax(), - currentLayers: f.vls.GetCurrent(), - parkedLayers: f.vls.GetParked(), + allocatedLayer: buffer.InvalidLayer, + muted: f.muted, + pubMuted: f.pubMuted, + maxSeenLayer: f.vls.GetMaxSeen(), + Bitrates: Bitrates, + maxLayer: f.vls.GetMax(), + currentLayer: f.vls.GetCurrent(), + parkedLayer: f.vls.GetParked(), } f.provisional.availableLayers = make([]int32, len(availableLayers)) copy(f.provisional.availableLayers, availableLayers) } -func (f *Forwarder) ProvisionalAllocate(availableChannelCapacity int64, layers buffer.VideoLayer, allowPause bool, allowOvershoot bool) int64 { +func (f *Forwarder) ProvisionalAllocate(availableChannelCapacity int64, layer buffer.VideoLayer, allowPause bool, allowOvershoot bool) int64 { f.lock.Lock() defer f.lock.Unlock() if f.provisional.muted || f.provisional.pubMuted || f.provisional.maxSeenLayer.Spatial == buffer.InvalidLayerSpatial || - !f.provisional.maxLayers.IsValid() || - ((!allowOvershoot || !f.vls.IsOvershootOkay()) && layers.GreaterThan(f.provisional.maxLayers)) { + !f.provisional.maxLayer.IsValid() || + ((!allowOvershoot || !f.vls.IsOvershootOkay()) && layer.GreaterThan(f.provisional.maxLayer)) { return 0 } - requiredBitrate := f.provisional.Bitrates[layers.Spatial][layers.Temporal] + requiredBitrate := f.provisional.Bitrates[layer.Spatial][layer.Temporal] if requiredBitrate == 0 { return 0 } alreadyAllocatedBitrate := int64(0) - if f.provisional.allocatedLayers.IsValid() { - alreadyAllocatedBitrate = f.provisional.Bitrates[f.provisional.allocatedLayers.Spatial][f.provisional.allocatedLayers.Temporal] + if f.provisional.allocatedLayer.IsValid() { + alreadyAllocatedBitrate = f.provisional.Bitrates[f.provisional.allocatedLayer.Spatial][f.provisional.allocatedLayer.Temporal] } // a layer under maximum fits, take it - if !layers.GreaterThan(f.provisional.maxLayers) && requiredBitrate <= (availableChannelCapacity+alreadyAllocatedBitrate) { - f.provisional.allocatedLayers = layers + if !layer.GreaterThan(f.provisional.maxLayer) && requiredBitrate <= (availableChannelCapacity+alreadyAllocatedBitrate) { + f.provisional.allocatedLayer = layer return requiredBitrate - alreadyAllocatedBitrate } @@ -695,8 +695,8 @@ func (f *Forwarder) ProvisionalAllocate(availableChannelCapacity int64, layers b // 2. a layer above maximum which may or may not fit, but overshoot is allowed. // In any of those cases, take the lowest possible layer if pause is not allowed // - if !allowPause && (!f.provisional.allocatedLayers.IsValid() || !layers.GreaterThan(f.provisional.allocatedLayers)) { - f.provisional.allocatedLayers = layers + if !allowPause && (!f.provisional.allocatedLayer.IsValid() || !layer.GreaterThan(f.provisional.allocatedLayer)) { + f.provisional.allocatedLayer = layer return requiredBitrate - alreadyAllocatedBitrate } @@ -727,14 +727,14 @@ func (f *Forwarder) ProvisionalAllocateGetCooperativeTransition(allowOvershoot b defer f.lock.Unlock() if f.provisional.muted || f.provisional.pubMuted { - f.provisional.allocatedLayers = buffer.InvalidLayers + f.provisional.allocatedLayer = buffer.InvalidLayer if f.provisional.pubMuted { // leave it at current for opportunistic forwarding, there is still bandwidth saving with publisher mute - f.provisional.allocatedLayers = f.provisional.currentLayers + f.provisional.allocatedLayer = f.provisional.currentLayer } return VideoTransition{ From: f.vls.GetTarget(), - To: f.provisional.allocatedLayers, + To: f.provisional.allocatedLayer, BandwidthDelta: 0 - f.lastAllocation.BandwidthRequested, } } @@ -743,12 +743,12 @@ func (f *Forwarder) ProvisionalAllocateGetCooperativeTransition(allowOvershoot b targetLayer := f.vls.GetTarget() if targetLayer.IsValid() { // what is the highest that is available - maximalLayers := buffer.InvalidLayers + maximalLayer := buffer.InvalidLayer maximalBandwidthRequired := int64(0) - for s := f.provisional.maxLayers.Spatial; s >= 0; s-- { - for t := f.provisional.maxLayers.Temporal; t >= 0; t-- { + for s := f.provisional.maxLayer.Spatial; s >= 0; s-- { + for t := f.provisional.maxLayer.Temporal; t >= 0; t-- { if f.provisional.Bitrates[s][t] != 0 { - maximalLayers = buffer.VideoLayer{Spatial: s, Temporal: t} + maximalLayer = buffer.VideoLayer{Spatial: s, Temporal: t} maximalBandwidthRequired = f.provisional.Bitrates[s][t] break } @@ -759,11 +759,11 @@ func (f *Forwarder) ProvisionalAllocateGetCooperativeTransition(allowOvershoot b } } - if maximalLayers.IsValid() { - if !targetLayer.GreaterThan(maximalLayers) && f.provisional.Bitrates[targetLayer.Spatial][targetLayer.Temporal] != 0 { - // currently streaming and maybe wanting an upgrade (targetLayer <= maximalLayers), + if maximalLayer.IsValid() { + if !targetLayer.GreaterThan(maximalLayer) && f.provisional.Bitrates[targetLayer.Spatial][targetLayer.Temporal] != 0 { + // currently streaming and maybe wanting an upgrade (targetLayer <= maximalLayer), // just preserve current target in the cooperative scheme of things - f.provisional.allocatedLayers = targetLayer + f.provisional.allocatedLayer = targetLayer return VideoTransition{ From: targetLayer, To: targetLayer, @@ -771,12 +771,12 @@ func (f *Forwarder) ProvisionalAllocateGetCooperativeTransition(allowOvershoot b } } - if targetLayer.GreaterThan(maximalLayers) { - // maximalLayers < targetLayer, make the down move - f.provisional.allocatedLayers = maximalLayers + if targetLayer.GreaterThan(maximalLayer) { + // maximalLayer < targetLayer, make the down move + f.provisional.allocatedLayer = maximalLayer return VideoTransition{ From: targetLayer, - To: maximalLayers, + To: maximalLayer, BandwidthDelta: maximalBandwidthRequired - f.lastAllocation.BandwidthRequested, } } @@ -787,7 +787,7 @@ func (f *Forwarder) ProvisionalAllocateGetCooperativeTransition(allowOvershoot b minSpatial, maxSpatial int32, minTemporal, maxTemporal int32, ) (buffer.VideoLayer, int64) { - layers := buffer.InvalidLayers + layers := buffer.InvalidLayer bw := int64(0) for s := minSpatial; s <= maxSpatial; s++ { for t := minTemporal; t <= maxTemporal; t++ { @@ -812,14 +812,14 @@ func (f *Forwarder) ProvisionalAllocateGetCooperativeTransition(allowOvershoot b // NOTE: a layer in feed could have paused and there could be other options than going back to minimal, // but the cooperative scheme knocks things back to minimal targetLayer, bandwidthRequired = findNextLayer( - 0, f.provisional.maxLayers.Spatial, - 0, f.provisional.maxLayers.Temporal, + 0, f.provisional.maxLayer.Spatial, + 0, f.provisional.maxLayer.Temporal, ) // could not find a minimal layer, overshoot if allowed - if bandwidthRequired == 0 && f.provisional.maxLayers.IsValid() && allowOvershoot && f.vls.IsOvershootOkay() { + if bandwidthRequired == 0 && f.provisional.maxLayer.IsValid() && allowOvershoot && f.vls.IsOvershootOkay() { targetLayer, bandwidthRequired = findNextLayer( - f.provisional.maxLayers.Spatial+1, buffer.DefaultMaxLayerSpatial, + f.provisional.maxLayer.Spatial+1, buffer.DefaultMaxLayerSpatial, 0, buffer.DefaultMaxLayerTemporal, ) } @@ -827,14 +827,14 @@ func (f *Forwarder) ProvisionalAllocateGetCooperativeTransition(allowOvershoot b // if nothing available, just leave target at current to enable opportunistic forwarding in case current resumes if !targetLayer.IsValid() { - if f.provisional.parkedLayers.IsValid() { - targetLayer = f.provisional.parkedLayers + if f.provisional.parkedLayer.IsValid() { + targetLayer = f.provisional.parkedLayer } else { - targetLayer = f.provisional.currentLayers + targetLayer = f.provisional.currentLayer } } - f.provisional.allocatedLayers = targetLayer + f.provisional.allocatedLayer = targetLayer return VideoTransition{ From: f.vls.GetTarget(), To: targetLayer, @@ -863,21 +863,21 @@ func (f *Forwarder) ProvisionalAllocateGetBestWeightedTransition() VideoTransiti targetLayer := f.vls.GetTarget() if f.provisional.muted || f.provisional.pubMuted { - f.provisional.allocatedLayers = buffer.InvalidLayers + f.provisional.allocatedLayer = buffer.InvalidLayer if f.provisional.pubMuted { // leave it at current for opportunistic forwarding, there is still bandwidth saving with publisher mute - f.provisional.allocatedLayers = f.provisional.currentLayers + f.provisional.allocatedLayer = f.provisional.currentLayer } return VideoTransition{ From: targetLayer, - To: f.provisional.allocatedLayers, + To: f.provisional.allocatedLayer, BandwidthDelta: 0 - f.lastAllocation.BandwidthRequested, } } maxReachableLayerTemporal := buffer.InvalidLayerTemporal - for t := f.provisional.maxLayers.Temporal; t >= 0; t-- { - for s := f.provisional.maxLayers.Spatial; s >= 0; s-- { + for t := f.provisional.maxLayer.Temporal; t >= 0; t-- { + for s := f.provisional.maxLayer.Spatial; s >= 0; s-- { if f.provisional.Bitrates[s][t] != 0 { maxReachableLayerTemporal = t break @@ -892,21 +892,21 @@ func (f *Forwarder) ProvisionalAllocateGetBestWeightedTransition() VideoTransiti // feed has gone dry, just leave target at current to enable opportunistic forwarding in case current resumes. // Note that this is giving back bits and opportunistic forwarding resuming might trigger congestion again, // but that should be handled by stream allocator. - if f.provisional.parkedLayers.IsValid() { - f.provisional.allocatedLayers = f.provisional.parkedLayers + if f.provisional.parkedLayer.IsValid() { + f.provisional.allocatedLayer = f.provisional.parkedLayer } else { - f.provisional.allocatedLayers = f.provisional.currentLayers + f.provisional.allocatedLayer = f.provisional.currentLayer } return VideoTransition{ From: targetLayer, - To: f.provisional.allocatedLayers, + To: f.provisional.allocatedLayer, BandwidthDelta: 0 - f.lastAllocation.BandwidthRequested, } } // starting from minimum to target, find transition which gives the best // transition taking into account bits saved vs cost of such a transition - bestLayers := buffer.InvalidLayers + bestLayer := buffer.InvalidLayer bestBandwidthDelta := int64(0) bestValue := float32(0) for s := int32(0); s <= targetLayer.Spatial; s++ { @@ -932,15 +932,15 @@ func (f *Forwarder) ProvisionalAllocateGetBestWeightedTransition() VideoTransiti if value > bestValue || (value == bestValue && BandwidthDelta > bestBandwidthDelta) { bestValue = value bestBandwidthDelta = BandwidthDelta - bestLayers = buffer.VideoLayer{Spatial: s, Temporal: t} + bestLayer = buffer.VideoLayer{Spatial: s, Temporal: t} } } } - f.provisional.allocatedLayers = bestLayers + f.provisional.allocatedLayer = bestLayer return VideoTransition{ From: targetLayer, - To: bestLayers, + To: bestLayer, BandwidthDelta: bestBandwidthDelta, } } @@ -954,24 +954,24 @@ func (f *Forwarder) ProvisionalAllocateCommit() VideoAllocation { f.provisional.pubMuted, f.provisional.maxSeenLayer.Spatial, f.provisional.Bitrates, - f.provisional.maxLayers, + f.provisional.maxLayer, ) alloc := VideoAllocation{ BandwidthRequested: 0, BandwidthDelta: -f.lastAllocation.BandwidthRequested, Bitrates: f.provisional.Bitrates, BandwidthNeeded: optimalBandwidthNeeded, - TargetLayers: f.provisional.allocatedLayers, - RequestLayerSpatial: f.provisional.allocatedLayers.Spatial, - MaxLayers: f.provisional.maxLayers, + TargetLayer: f.provisional.allocatedLayer, + RequestLayerSpatial: f.provisional.allocatedLayer.Spatial, + MaxLayer: f.provisional.maxLayer, DistanceToDesired: getDistanceToDesired( f.provisional.muted, f.provisional.pubMuted, f.provisional.maxSeenLayer, f.provisional.availableLayers, f.provisional.Bitrates, - f.provisional.allocatedLayers, - f.provisional.maxLayers, + f.provisional.allocatedLayer, + f.provisional.maxLayer, ), } @@ -983,46 +983,46 @@ func (f *Forwarder) ProvisionalAllocateCommit() VideoAllocation { alloc.PauseReason = VideoPauseReasonPubMuted case optimalBandwidthNeeded == 0: - if f.provisional.allocatedLayers.IsValid() { + if f.provisional.allocatedLayer.IsValid() { // overshoot - alloc.BandwidthRequested = f.provisional.Bitrates[f.provisional.allocatedLayers.Spatial][f.provisional.allocatedLayers.Temporal] + alloc.BandwidthRequested = f.provisional.Bitrates[f.provisional.allocatedLayer.Spatial][f.provisional.allocatedLayer.Temporal] alloc.BandwidthDelta = alloc.BandwidthRequested - f.lastAllocation.BandwidthRequested } else { alloc.PauseReason = VideoPauseReasonFeedDry // leave target at current for opportunistic forwarding - if f.provisional.currentLayers.IsValid() && f.provisional.currentLayers.Spatial <= f.provisional.maxLayers.Spatial { - f.provisional.allocatedLayers = f.provisional.currentLayers - alloc.TargetLayers = f.provisional.allocatedLayers - alloc.RequestLayerSpatial = alloc.TargetLayers.Spatial + if f.provisional.currentLayer.IsValid() && f.provisional.currentLayer.Spatial <= f.provisional.maxLayer.Spatial { + f.provisional.allocatedLayer = f.provisional.currentLayer + alloc.TargetLayer = f.provisional.allocatedLayer + alloc.RequestLayerSpatial = alloc.TargetLayer.Spatial } } default: - if f.provisional.allocatedLayers.IsValid() { - alloc.BandwidthRequested = f.provisional.Bitrates[f.provisional.allocatedLayers.Spatial][f.provisional.allocatedLayers.Temporal] + if f.provisional.allocatedLayer.IsValid() { + alloc.BandwidthRequested = f.provisional.Bitrates[f.provisional.allocatedLayer.Spatial][f.provisional.allocatedLayer.Temporal] } alloc.BandwidthDelta = alloc.BandwidthRequested - f.lastAllocation.BandwidthRequested - if f.provisional.allocatedLayers.GreaterThan(f.provisional.maxLayers) || + if f.provisional.allocatedLayer.GreaterThan(f.provisional.maxLayer) || alloc.BandwidthRequested >= getOptimalBandwidthNeeded( f.provisional.muted, f.provisional.pubMuted, f.provisional.maxSeenLayer.Spatial, f.provisional.Bitrates, - f.provisional.maxLayers, + f.provisional.maxLayer, ) { // could be greater than optimal if overshooting alloc.IsDeficient = false } else { alloc.IsDeficient = true - if !f.provisional.allocatedLayers.IsValid() { + if !f.provisional.allocatedLayer.IsValid() { alloc.PauseReason = VideoPauseReasonBandwidth } } } - f.clearParkedLayers() + f.clearParkedLayer() return f.updateAllocation(alloc, "cooperative") } @@ -1077,9 +1077,9 @@ func (f *Forwarder) AllocateNextHigher(availableChannelCapacity int64, available BandwidthDelta: bandwidthRequested - alreadyAllocated, BandwidthNeeded: optimalBandwidthNeeded, Bitrates: brs, - TargetLayers: newTargetLayer, + TargetLayer: newTargetLayer, RequestLayerSpatial: newTargetLayer.Spatial, - MaxLayers: maxLayer, + MaxLayer: maxLayer, DistanceToDesired: getDistanceToDesired( f.muted, f.pubMuted, @@ -1236,16 +1236,16 @@ func (f *Forwarder) Pause(availableLayers []int32, brs Bitrates) VideoAllocation BandwidthDelta: 0 - f.lastAllocation.BandwidthRequested, Bitrates: brs, BandwidthNeeded: optimalBandwidthNeeded, - TargetLayers: buffer.InvalidLayers, + TargetLayer: buffer.InvalidLayer, RequestLayerSpatial: buffer.InvalidLayerSpatial, - MaxLayers: maxLayer, + MaxLayer: maxLayer, DistanceToDesired: getDistanceToDesired( f.muted, f.pubMuted, maxSeenLayer, availableLayers, brs, - buffer.InvalidLayers, + buffer.InvalidLayer, maxLayer, ), } @@ -1266,19 +1266,19 @@ func (f *Forwarder) Pause(availableLayers []int32, brs Bitrates) VideoAllocation alloc.PauseReason = VideoPauseReasonBandwidth } - f.clearParkedLayers() + f.clearParkedLayer() return f.updateAllocation(alloc, "pause") } func (f *Forwarder) updateAllocation(alloc VideoAllocation, reason string) VideoAllocation { // restrict target temporal to 0 if codec does not support temporal layers - if alloc.TargetLayers.IsValid() && strings.ToLower(f.codec.MimeType) == "video/h264" { - alloc.TargetLayers.Temporal = 0 + if alloc.TargetLayer.IsValid() && strings.ToLower(f.codec.MimeType) == "video/h264" { + alloc.TargetLayer.Temporal = 0 } if alloc.IsDeficient != f.lastAllocation.IsDeficient || alloc.PauseReason != f.lastAllocation.PauseReason || - alloc.TargetLayers != f.lastAllocation.TargetLayers || + alloc.TargetLayer != f.lastAllocation.TargetLayer || alloc.RequestLayerSpatial != f.lastAllocation.RequestLayerSpatial { if reason == "optimal" { f.logger.Debugw(fmt.Sprintf("stream allocation: %s", reason), "allocation", alloc) @@ -1288,7 +1288,7 @@ func (f *Forwarder) updateAllocation(alloc VideoAllocation, reason string) Video } f.lastAllocation = alloc - f.setTargetLayers(f.lastAllocation.TargetLayers, f.lastAllocation.RequestLayerSpatial) + f.setTargetLayer(f.lastAllocation.TargetLayer, f.lastAllocation.RequestLayerSpatial) if !f.vls.GetTarget().IsValid() { f.resyncLocked() } @@ -1296,8 +1296,8 @@ func (f *Forwarder) updateAllocation(alloc VideoAllocation, reason string) Video return f.lastAllocation } -func (f *Forwarder) setTargetLayers(targetLayers buffer.VideoLayer, requestLayerSpatial int32) { - f.vls.SetTarget(targetLayers) +func (f *Forwarder) setTargetLayer(targetLayer buffer.VideoLayer, requestLayerSpatial int32) { + f.vls.SetTarget(targetLayer) f.vls.SetRequestSpatial(requestLayerSpatial) } @@ -1309,31 +1309,31 @@ func (f *Forwarder) Resync() { } func (f *Forwarder) resyncLocked() { - f.vls.SetCurrent(buffer.InvalidLayers) + f.vls.SetCurrent(buffer.InvalidLayer) f.lastSSRC = 0 - f.clearParkedLayers() + f.clearParkedLayer() } -func (f *Forwarder) clearParkedLayers() { - f.vls.SetParked(buffer.InvalidLayers) - if f.parkedLayersTimer != nil { - f.parkedLayersTimer.Stop() - f.parkedLayersTimer = nil +func (f *Forwarder) clearParkedLayer() { + f.vls.SetParked(buffer.InvalidLayer) + if f.parkedLayerTimer != nil { + f.parkedLayerTimer.Stop() + f.parkedLayerTimer = nil } } -func (f *Forwarder) setupParkedLayers(parkedLayers buffer.VideoLayer) { - f.clearParkedLayers() +func (f *Forwarder) setupParkedLayer(parkedLayer buffer.VideoLayer) { + f.clearParkedLayer() - f.vls.SetParked(parkedLayers) - f.parkedLayersTimer = time.AfterFunc(ParkedLayersWaitDuration, func() { + f.vls.SetParked(parkedLayer) + f.parkedLayerTimer = time.AfterFunc(ParkedLayerWaitDuration, func() { f.lock.Lock() notify := f.vls.GetParked().IsValid() - f.clearParkedLayers() + f.clearParkedLayer() f.lock.Unlock() - if onParkedLayersExpired := f.getOnParkedLayersExpired(); onParkedLayersExpired != nil && notify { - onParkedLayersExpired() + if onParkedLayerExpired := f.getOnParkedLayerExpired(); onParkedLayerExpired != nil && notify { + onParkedLayerExpired() } }) } @@ -1588,13 +1588,13 @@ func (f *Forwarder) GetRTPMungerParams() RTPMungerParams { // ----------------------------------------------------------------------------- -func getOptimalBandwidthNeeded(muted bool, pubMuted bool, maxPublishedLayer int32, brs Bitrates, maxLayers buffer.VideoLayer) int64 { +func getOptimalBandwidthNeeded(muted bool, pubMuted bool, maxPublishedLayer int32, brs Bitrates, maxLayer buffer.VideoLayer) int64 { if muted || pubMuted || maxPublishedLayer == buffer.InvalidLayerSpatial { return 0 } - for i := maxLayers.Spatial; i >= 0; i-- { - for j := maxLayers.Temporal; j >= 0; j-- { + for i := maxLayer.Spatial; i >= 0; i-- { + for j := maxLayer.Temporal; j >= 0; j-- { if brs[i][j] == 0 { continue } @@ -1617,14 +1617,14 @@ func getDistanceToDesired( maxSeenLayer buffer.VideoLayer, availableLayers []int32, brs Bitrates, - targetLayers buffer.VideoLayer, - maxLayers buffer.VideoLayer, + targetLayer buffer.VideoLayer, + maxLayer buffer.VideoLayer, ) float64 { - if muted || pubMuted || !maxSeenLayer.IsValid() || !maxLayers.IsValid() { + if muted || pubMuted || !maxSeenLayer.IsValid() || !maxLayer.IsValid() { return 0.0 } - adjustedMaxLayers := maxLayers + adjustedMaxLayer := maxLayer maxAvailableSpatial := buffer.InvalidLayerSpatial maxAvailableTemporal := buffer.InvalidLayerTemporal @@ -1651,48 +1651,48 @@ done: } } - if maxAvailableSpatial < adjustedMaxLayers.Spatial { - adjustedMaxLayers.Spatial = maxAvailableSpatial + if maxAvailableSpatial < adjustedMaxLayer.Spatial { + adjustedMaxLayer.Spatial = maxAvailableSpatial } - if maxSeenLayer.Spatial < adjustedMaxLayers.Spatial { - adjustedMaxLayers.Spatial = maxSeenLayer.Spatial + if maxSeenLayer.Spatial < adjustedMaxLayer.Spatial { + adjustedMaxLayer.Spatial = maxSeenLayer.Spatial } // max available temporal is min(subscribedMax, temporalLayerSeenMax, availableMax) // subscribedMax = subscriber requested max temporal layer // temporalLayerSeenMax = max temporal layer ever published/seen // availableMax = based on bit rate measurement, available max temporal in the adjusted max spatial layer - if adjustedMaxLayers.Spatial != buffer.InvalidLayerSpatial { + if adjustedMaxLayer.Spatial != buffer.InvalidLayerSpatial { for t := int32(len(brs[0])) - 1; t >= 0; t-- { - if brs[adjustedMaxLayers.Spatial][t] != 0 { + if brs[adjustedMaxLayer.Spatial][t] != 0 { maxAvailableTemporal = t break } } } - if maxAvailableTemporal < adjustedMaxLayers.Temporal { - adjustedMaxLayers.Temporal = maxAvailableTemporal + if maxAvailableTemporal < adjustedMaxLayer.Temporal { + adjustedMaxLayer.Temporal = maxAvailableTemporal } - if maxSeenLayer.Temporal < adjustedMaxLayers.Temporal { - adjustedMaxLayers.Temporal = maxSeenLayer.Temporal + if maxSeenLayer.Temporal < adjustedMaxLayer.Temporal { + adjustedMaxLayer.Temporal = maxSeenLayer.Temporal } - if !adjustedMaxLayers.IsValid() { - adjustedMaxLayers = buffer.VideoLayer{Spatial: 0, Temporal: 0} + if !adjustedMaxLayer.IsValid() { + adjustedMaxLayer = buffer.VideoLayer{Spatial: 0, Temporal: 0} } // adjust target layers if they are invalid, i. e. not streaming - adjustedTargetLayers := targetLayers - if !targetLayers.IsValid() { - adjustedTargetLayers = buffer.VideoLayer{Spatial: 0, Temporal: 0} + adjustedTargetLayer := targetLayer + if !targetLayer.IsValid() { + adjustedTargetLayer = buffer.VideoLayer{Spatial: 0, Temporal: 0} } distance := - ((adjustedMaxLayers.Spatial - adjustedTargetLayers.Spatial) * (maxSeenLayer.Temporal + 1)) + - (adjustedMaxLayers.Temporal - adjustedTargetLayers.Temporal) - if !targetLayers.IsValid() { + ((adjustedMaxLayer.Spatial - adjustedTargetLayer.Spatial) * (maxSeenLayer.Temporal + 1)) + + (adjustedMaxLayer.Temporal - adjustedTargetLayer.Temporal) + if !targetLayer.IsValid() { distance++ } diff --git a/pkg/sfu/forwarder_test.go b/pkg/sfu/forwarder_test.go index 697a0f573..e2f3dff44 100644 --- a/pkg/sfu/forwarder_test.go +++ b/pkg/sfu/forwarder_test.go @@ -13,8 +13,8 @@ import ( ) func disable(f *Forwarder) { - f.vls.SetCurrent(buffer.InvalidLayers) - f.vls.SetTarget(buffer.InvalidLayers) + f.vls.SetCurrent(buffer.InvalidLayer) + f.vls.SetTarget(buffer.InvalidLayer) } func newForwarder(codec webrtc.RTPCodecCapability, kind webrtc.RTPCodecType) *Forwarder { @@ -40,74 +40,74 @@ func TestForwarderMute(t *testing.T) { func TestForwarderLayersAudio(t *testing.T) { f := newForwarder(testutils.TestOpusCodec, webrtc.RTPCodecTypeAudio) - require.Equal(t, buffer.InvalidLayers, f.MaxLayers()) + require.Equal(t, buffer.InvalidLayer, f.MaxLayer()) - require.Equal(t, buffer.InvalidLayers, f.CurrentLayers()) - require.Equal(t, buffer.InvalidLayers, f.TargetLayers()) + require.Equal(t, buffer.InvalidLayer, f.CurrentLayer()) + require.Equal(t, buffer.InvalidLayer, f.TargetLayer()) - changed, maxLayers, currentLayers := f.SetMaxSpatialLayer(1) + changed, maxLayer, currentLayer := f.SetMaxSpatialLayer(1) require.False(t, changed) - require.Equal(t, buffer.InvalidLayers, maxLayers) - require.Equal(t, buffer.InvalidLayers, currentLayers) + require.Equal(t, buffer.InvalidLayer, maxLayer) + require.Equal(t, buffer.InvalidLayer, currentLayer) - changed, maxLayers, currentLayers = f.SetMaxTemporalLayer(1) + changed, maxLayer, currentLayer = f.SetMaxTemporalLayer(1) require.False(t, changed) - require.Equal(t, buffer.InvalidLayers, maxLayers) - require.Equal(t, buffer.InvalidLayers, currentLayers) + require.Equal(t, buffer.InvalidLayer, maxLayer) + require.Equal(t, buffer.InvalidLayer, currentLayer) - require.Equal(t, buffer.InvalidLayers, f.MaxLayers()) + require.Equal(t, buffer.InvalidLayer, f.MaxLayer()) } func TestForwarderLayersVideo(t *testing.T) { f := newForwarder(testutils.TestVP8Codec, webrtc.RTPCodecTypeVideo) - maxLayers := f.MaxLayers() + maxLayer := f.MaxLayer() expectedLayers := buffer.VideoLayer{Spatial: buffer.InvalidLayerSpatial, Temporal: buffer.DefaultMaxLayerTemporal} - require.Equal(t, expectedLayers, maxLayers) + require.Equal(t, expectedLayers, maxLayer) - require.Equal(t, buffer.InvalidLayers, f.CurrentLayers()) - require.Equal(t, buffer.InvalidLayers, f.TargetLayers()) + require.Equal(t, buffer.InvalidLayer, f.CurrentLayer()) + require.Equal(t, buffer.InvalidLayer, f.TargetLayer()) expectedLayers = buffer.VideoLayer{ Spatial: buffer.DefaultMaxLayerSpatial, Temporal: buffer.DefaultMaxLayerTemporal, } - changed, maxLayers, currentLayers := f.SetMaxSpatialLayer(buffer.DefaultMaxLayerSpatial) + changed, maxLayer, currentLayer := f.SetMaxSpatialLayer(buffer.DefaultMaxLayerSpatial) require.True(t, changed) - require.Equal(t, expectedLayers, maxLayers) - require.Equal(t, buffer.InvalidLayers, currentLayers) + require.Equal(t, expectedLayers, maxLayer) + require.Equal(t, buffer.InvalidLayer, currentLayer) - changed, maxLayers, currentLayers = f.SetMaxSpatialLayer(buffer.DefaultMaxLayerSpatial - 1) + changed, maxLayer, currentLayer = f.SetMaxSpatialLayer(buffer.DefaultMaxLayerSpatial - 1) require.True(t, changed) expectedLayers = buffer.VideoLayer{ Spatial: buffer.DefaultMaxLayerSpatial - 1, Temporal: buffer.DefaultMaxLayerTemporal, } - require.Equal(t, expectedLayers, maxLayers) - require.Equal(t, expectedLayers, f.MaxLayers()) - require.Equal(t, buffer.InvalidLayers, currentLayers) + require.Equal(t, expectedLayers, maxLayer) + require.Equal(t, expectedLayers, f.MaxLayer()) + require.Equal(t, buffer.InvalidLayer, currentLayer) f.vls.SetCurrent(buffer.VideoLayer{Spatial: 0, Temporal: 1}) - changed, maxLayers, currentLayers = f.SetMaxSpatialLayer(buffer.DefaultMaxLayerSpatial - 1) + changed, maxLayer, currentLayer = f.SetMaxSpatialLayer(buffer.DefaultMaxLayerSpatial - 1) require.False(t, changed) - require.Equal(t, expectedLayers, maxLayers) - require.Equal(t, expectedLayers, f.MaxLayers()) - require.Equal(t, buffer.VideoLayer{Spatial: 0, Temporal: 1}, currentLayers) + require.Equal(t, expectedLayers, maxLayer) + require.Equal(t, expectedLayers, f.MaxLayer()) + require.Equal(t, buffer.VideoLayer{Spatial: 0, Temporal: 1}, currentLayer) - changed, maxLayers, currentLayers = f.SetMaxTemporalLayer(buffer.DefaultMaxLayerTemporal) + changed, maxLayer, currentLayer = f.SetMaxTemporalLayer(buffer.DefaultMaxLayerTemporal) require.False(t, changed) - require.Equal(t, expectedLayers, maxLayers) - require.Equal(t, buffer.VideoLayer{Spatial: 0, Temporal: 1}, currentLayers) + require.Equal(t, expectedLayers, maxLayer) + require.Equal(t, buffer.VideoLayer{Spatial: 0, Temporal: 1}, currentLayer) - changed, maxLayers, currentLayers = f.SetMaxTemporalLayer(buffer.DefaultMaxLayerTemporal - 1) + changed, maxLayer, currentLayer = f.SetMaxTemporalLayer(buffer.DefaultMaxLayerTemporal - 1) require.True(t, changed) expectedLayers = buffer.VideoLayer{ Spatial: buffer.DefaultMaxLayerSpatial - 1, Temporal: buffer.DefaultMaxLayerTemporal - 1, } - require.Equal(t, expectedLayers, maxLayers) - require.Equal(t, expectedLayers, f.MaxLayers()) - require.Equal(t, buffer.VideoLayer{Spatial: 0, Temporal: 1}, currentLayers) + require.Equal(t, expectedLayers, maxLayer) + require.Equal(t, expectedLayers, f.MaxLayer()) + require.Equal(t, buffer.VideoLayer{Spatial: 0, Temporal: 1}, currentLayer) } func TestForwarderAllocateOptimal(t *testing.T) { @@ -121,15 +121,15 @@ func TestForwarderAllocateOptimal(t *testing.T) { } // invalid max layers - f.vls.SetMax(buffer.InvalidLayers) + f.vls.SetMax(buffer.InvalidLayer) expectedResult := VideoAllocation{ PauseReason: VideoPauseReasonFeedDry, BandwidthRequested: 0, BandwidthDelta: 0, Bitrates: bitrates, - TargetLayers: buffer.InvalidLayers, + TargetLayer: buffer.InvalidLayer, RequestLayerSpatial: buffer.InvalidLayerSpatial, - MaxLayers: buffer.InvalidLayers, + MaxLayer: buffer.InvalidLayer, DistanceToDesired: 0, } result := f.AllocateOptimal(nil, bitrates, true) @@ -139,15 +139,15 @@ func TestForwarderAllocateOptimal(t *testing.T) { f.SetMaxSpatialLayer(buffer.DefaultMaxLayerSpatial) f.SetMaxTemporalLayer(buffer.DefaultMaxLayerTemporal) - // should still have target at buffer.InvalidLayers until max publisher layer is available + // should still have target at buffer.InvalidLayer until max publisher layer is available expectedResult = VideoAllocation{ PauseReason: VideoPauseReasonFeedDry, BandwidthRequested: 0, BandwidthDelta: 0, Bitrates: bitrates, - TargetLayers: buffer.InvalidLayers, + TargetLayer: buffer.InvalidLayer, RequestLayerSpatial: buffer.InvalidLayerSpatial, - MaxLayers: buffer.DefaultMaxLayers, + MaxLayer: buffer.DefaultMaxLayer, DistanceToDesired: 0, } result = f.AllocateOptimal(nil, bitrates, true) @@ -164,9 +164,9 @@ func TestForwarderAllocateOptimal(t *testing.T) { BandwidthRequested: 0, BandwidthDelta: 0, Bitrates: bitrates, - TargetLayers: buffer.InvalidLayers, + TargetLayer: buffer.InvalidLayer, RequestLayerSpatial: buffer.InvalidLayerSpatial, - MaxLayers: buffer.DefaultMaxLayers, + MaxLayer: buffer.DefaultMaxLayer, DistanceToDesired: 0, } result = f.AllocateOptimal(nil, bitrates, true) @@ -183,9 +183,9 @@ func TestForwarderAllocateOptimal(t *testing.T) { BandwidthRequested: 0, BandwidthDelta: 0, Bitrates: bitrates, - TargetLayers: buffer.InvalidLayers, + TargetLayer: buffer.InvalidLayer, RequestLayerSpatial: buffer.InvalidLayerSpatial, - MaxLayers: buffer.DefaultMaxLayers, + MaxLayer: buffer.DefaultMaxLayer, DistanceToDesired: 0, } result = f.AllocateOptimal(nil, bitrates, true) @@ -204,16 +204,16 @@ func TestForwarderAllocateOptimal(t *testing.T) { BandwidthRequested: 0, BandwidthDelta: 0, Bitrates: emptyBitrates, - TargetLayers: f.vls.GetParked(), + TargetLayer: f.vls.GetParked(), RequestLayerSpatial: f.vls.GetParked().Spatial, - MaxLayers: buffer.DefaultMaxLayers, + MaxLayer: buffer.DefaultMaxLayer, DistanceToDesired: 0, } result = f.AllocateOptimal(nil, emptyBitrates, true) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, f.vls.GetParked(), f.TargetLayers()) - f.vls.SetParked(buffer.InvalidLayers) + require.Equal(t, f.vls.GetParked(), f.TargetLayer()) + f.vls.SetParked(buffer.InvalidLayer) // when max layers changes, target is opportunistic, but requested spatial layer should be at max f.SetMaxTemporalLayerSeen(3) @@ -224,23 +224,23 @@ func TestForwarderAllocateOptimal(t *testing.T) { BandwidthDelta: bitrates[1][3], BandwidthNeeded: bitrates[1][3], Bitrates: bitrates, - TargetLayers: buffer.DefaultMaxLayers, + TargetLayer: buffer.DefaultMaxLayer, RequestLayerSpatial: f.vls.GetMax().Spatial, - MaxLayers: f.vls.GetMax(), + MaxLayer: f.vls.GetMax(), DistanceToDesired: -1, } result = f.AllocateOptimal(nil, bitrates, true) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, buffer.DefaultMaxLayers, f.TargetLayers()) + require.Equal(t, buffer.DefaultMaxLayer, f.TargetLayer()) // reset max layers for rest of the tests below - f.vls.SetMax(buffer.DefaultMaxLayers) + f.vls.SetMax(buffer.DefaultMaxLayer) // when feed is dry and current is not valid, should set up for opportunistic forwarding // NOTE: feed is dry due to availableLayers = nil, some valid bitrates may be passed in here for testing purposes only disable(f) - expectedTargetLayers := buffer.VideoLayer{ + expectedTargetLayer := buffer.VideoLayer{ Spatial: 2, Temporal: buffer.DefaultMaxLayerTemporal, } @@ -250,21 +250,21 @@ func TestForwarderAllocateOptimal(t *testing.T) { BandwidthDelta: bitrates[2][1] - bitrates[1][3], BandwidthNeeded: bitrates[2][1], Bitrates: bitrates, - TargetLayers: expectedTargetLayers, - RequestLayerSpatial: expectedTargetLayers.Spatial, - MaxLayers: buffer.DefaultMaxLayers, + TargetLayer: expectedTargetLayer, + RequestLayerSpatial: expectedTargetLayer.Spatial, + MaxLayer: buffer.DefaultMaxLayer, DistanceToDesired: -0.5, } result = f.AllocateOptimal(nil, bitrates, true) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, expectedTargetLayers, f.TargetLayers()) + require.Equal(t, expectedTargetLayer, f.TargetLayer()) f.vls.SetTarget(buffer.VideoLayer{Spatial: 0, Temporal: 0}) // set to valid to trigger paths in tests below f.vls.SetCurrent(buffer.VideoLayer{Spatial: 0, Temporal: 3}) // set to valid to trigger paths in tests below // when feed is dry and current is valid, should stay at current - expectedTargetLayers = buffer.VideoLayer{ + expectedTargetLayer = buffer.VideoLayer{ Spatial: 0, Temporal: 3, } @@ -273,17 +273,17 @@ func TestForwarderAllocateOptimal(t *testing.T) { BandwidthRequested: 0, BandwidthDelta: 0 - bitrates[2][1], Bitrates: emptyBitrates, - TargetLayers: expectedTargetLayers, - RequestLayerSpatial: expectedTargetLayers.Spatial, - MaxLayers: buffer.DefaultMaxLayers, + TargetLayer: expectedTargetLayer, + RequestLayerSpatial: expectedTargetLayer.Spatial, + MaxLayer: buffer.DefaultMaxLayer, DistanceToDesired: -0.75, } result = f.AllocateOptimal(nil, emptyBitrates, true) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, expectedTargetLayers, f.TargetLayers()) + require.Equal(t, expectedTargetLayer, f.TargetLayer()) - f.vls.SetCurrent(buffer.InvalidLayers) + f.vls.SetCurrent(buffer.InvalidLayer) // opportunistic target if feed is not dry and current is not valid, i. e. not forwarding expectedResult = VideoAllocation{ @@ -292,34 +292,34 @@ func TestForwarderAllocateOptimal(t *testing.T) { BandwidthDelta: bitrates[2][1], BandwidthNeeded: bitrates[2][1], Bitrates: bitrates, - TargetLayers: buffer.DefaultMaxLayers, + TargetLayer: buffer.DefaultMaxLayer, RequestLayerSpatial: 2, - MaxLayers: buffer.DefaultMaxLayers, + MaxLayer: buffer.DefaultMaxLayer, DistanceToDesired: -0.5, } result = f.AllocateOptimal([]int32{0, 1}, bitrates, true) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, buffer.DefaultMaxLayers, f.TargetLayers()) + require.Equal(t, buffer.DefaultMaxLayer, f.TargetLayer()) // if feed is not dry and current is not locked, should be opportunistic (with and without overshoot) - f.vls.SetTarget(buffer.InvalidLayers) + f.vls.SetTarget(buffer.InvalidLayer) expectedResult = VideoAllocation{ PauseReason: VideoPauseReasonFeedDry, BandwidthRequested: 0, BandwidthDelta: 0 - bitrates[2][1], Bitrates: emptyBitrates, - TargetLayers: buffer.DefaultMaxLayers, + TargetLayer: buffer.DefaultMaxLayer, RequestLayerSpatial: 2, - MaxLayers: buffer.DefaultMaxLayers, + MaxLayer: buffer.DefaultMaxLayer, DistanceToDesired: -1.0, } result = f.AllocateOptimal([]int32{0, 1}, emptyBitrates, false) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - f.vls.SetTarget(buffer.InvalidLayers) - expectedTargetLayers = buffer.VideoLayer{ + f.vls.SetTarget(buffer.InvalidLayer) + expectedTargetLayer = buffer.VideoLayer{ Spatial: 2, Temporal: buffer.DefaultMaxLayerTemporal, } @@ -329,9 +329,9 @@ func TestForwarderAllocateOptimal(t *testing.T) { BandwidthDelta: bitrates[2][1], BandwidthNeeded: bitrates[2][1], Bitrates: bitrates, - TargetLayers: expectedTargetLayers, + TargetLayer: expectedTargetLayer, RequestLayerSpatial: 2, - MaxLayers: buffer.DefaultMaxLayers, + MaxLayer: buffer.DefaultMaxLayer, DistanceToDesired: -0.5, } result = f.AllocateOptimal([]int32{0, 1}, bitrates, true) @@ -340,7 +340,7 @@ func TestForwarderAllocateOptimal(t *testing.T) { // switches to highest available if feed is not dry and current is valid and current is not available f.vls.SetCurrent(buffer.VideoLayer{Spatial: 0, Temporal: 1}) - expectedTargetLayers = buffer.VideoLayer{ + expectedTargetLayer = buffer.VideoLayer{ Spatial: 1, Temporal: buffer.DefaultMaxLayerTemporal, } @@ -350,9 +350,9 @@ func TestForwarderAllocateOptimal(t *testing.T) { BandwidthDelta: 0, BandwidthNeeded: bitrates[2][1], Bitrates: bitrates, - TargetLayers: expectedTargetLayers, + TargetLayer: expectedTargetLayer, RequestLayerSpatial: 1, - MaxLayers: buffer.DefaultMaxLayers, + MaxLayer: buffer.DefaultMaxLayer, DistanceToDesired: 0.5, } result = f.AllocateOptimal([]int32{1}, bitrates, true) @@ -363,7 +363,7 @@ func TestForwarderAllocateOptimal(t *testing.T) { f.vls.SetMax(buffer.VideoLayer{Spatial: 0, Temporal: 1}) f.vls.SetCurrent(buffer.VideoLayer{Spatial: 0, Temporal: 1}) f.vls.SetRequestSpatial(0) - expectedTargetLayers = buffer.VideoLayer{ + expectedTargetLayer = buffer.VideoLayer{ Spatial: 0, Temporal: 1, } @@ -372,9 +372,9 @@ func TestForwarderAllocateOptimal(t *testing.T) { BandwidthRequested: 0, BandwidthDelta: 0 - bitrates[2][1], Bitrates: emptyBitrates, - TargetLayers: expectedTargetLayers, + TargetLayer: expectedTargetLayer, RequestLayerSpatial: 0, - MaxLayers: f.vls.GetMax(), + MaxLayer: f.vls.GetMax(), DistanceToDesired: 0.0, } result = f.AllocateOptimal([]int32{0, 1}, emptyBitrates, true) @@ -385,7 +385,7 @@ func TestForwarderAllocateOptimal(t *testing.T) { f.vls.SetMax(buffer.VideoLayer{Spatial: 2, Temporal: 1}) f.vls.SetCurrent(buffer.VideoLayer{Spatial: 0, Temporal: 1}) f.vls.SetRequestSpatial(0) - expectedTargetLayers = buffer.VideoLayer{ + expectedTargetLayer = buffer.VideoLayer{ Spatial: 2, Temporal: 1, } @@ -394,9 +394,9 @@ func TestForwarderAllocateOptimal(t *testing.T) { BandwidthRequested: 0, BandwidthDelta: 0, Bitrates: emptyBitrates, - TargetLayers: expectedTargetLayers, + TargetLayer: expectedTargetLayer, RequestLayerSpatial: 2, - MaxLayers: f.vls.GetMax(), + MaxLayer: f.vls.GetMax(), DistanceToDesired: -1, } result = f.AllocateOptimal([]int32{0, 1}, emptyBitrates, true) @@ -436,7 +436,7 @@ func TestForwarderProvisionalAllocate(t *testing.T) { require.Equal(t, int64(0), usedBitrate) // committing should set target to (1, 2) - expectedTargetLayers := buffer.VideoLayer{ + expectedTargetLayer := buffer.VideoLayer{ Spatial: 1, Temporal: 2, } @@ -446,24 +446,24 @@ func TestForwarderProvisionalAllocate(t *testing.T) { BandwidthDelta: bitrates[1][2], BandwidthNeeded: bitrates[2][3], Bitrates: bitrates, - TargetLayers: expectedTargetLayers, - RequestLayerSpatial: expectedTargetLayers.Spatial, - MaxLayers: buffer.DefaultMaxLayers, + TargetLayer: expectedTargetLayer, + RequestLayerSpatial: expectedTargetLayer.Spatial, + MaxLayer: buffer.DefaultMaxLayer, DistanceToDesired: 1.25, } result := f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, expectedTargetLayers, f.TargetLayers()) + require.Equal(t, expectedTargetLayer, f.TargetLayer()) // when nothing fits and pausing disallowed, should allocate (0, 0) - f.vls.SetTarget(buffer.InvalidLayers) + f.vls.SetTarget(buffer.InvalidLayer) f.ProvisionalAllocatePrepare(nil, bitrates) usedBitrate = f.ProvisionalAllocate(0, buffer.VideoLayer{Spatial: 0, Temporal: 0}, false, false) require.Equal(t, int64(1), usedBitrate) // committing should set target to (0, 0) - expectedTargetLayers = buffer.VideoLayer{ + expectedTargetLayer = buffer.VideoLayer{ Spatial: 0, Temporal: 0, } @@ -473,15 +473,15 @@ func TestForwarderProvisionalAllocate(t *testing.T) { BandwidthDelta: bitrates[0][0] - bitrates[1][2], BandwidthNeeded: bitrates[2][3], Bitrates: bitrates, - TargetLayers: expectedTargetLayers, - RequestLayerSpatial: expectedTargetLayers.Spatial, - MaxLayers: buffer.DefaultMaxLayers, + TargetLayer: expectedTargetLayer, + RequestLayerSpatial: expectedTargetLayer.Spatial, + MaxLayer: buffer.DefaultMaxLayer, DistanceToDesired: 2.75, } result = f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, expectedTargetLayers, f.TargetLayers()) + require.Equal(t, expectedTargetLayer, f.TargetLayer()) // // Test allowOvershoot. @@ -508,11 +508,11 @@ func TestForwarderProvisionalAllocate(t *testing.T) { require.Equal(t, bitrates[1][3]-bitrates[2][3], usedBitrate) // committing should set target to (1, 3) - expectedTargetLayers = buffer.VideoLayer{ + expectedTargetLayer = buffer.VideoLayer{ Spatial: 1, Temporal: 3, } - expectedMaxLayers := buffer.VideoLayer{ + expectedMaxLayer := buffer.VideoLayer{ Spatial: 0, Temporal: 3, } @@ -520,15 +520,15 @@ func TestForwarderProvisionalAllocate(t *testing.T) { BandwidthRequested: bitrates[1][3], BandwidthDelta: bitrates[1][3] - 1, // 1 is the last allocation bandwidth requested Bitrates: bitrates, - TargetLayers: expectedTargetLayers, - RequestLayerSpatial: expectedTargetLayers.Spatial, - MaxLayers: expectedMaxLayers, + TargetLayer: expectedTargetLayer, + RequestLayerSpatial: expectedTargetLayer.Spatial, + MaxLayer: expectedMaxLayer, DistanceToDesired: -1.75, } result = f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, expectedTargetLayers, f.TargetLayers()) + require.Equal(t, expectedTargetLayer, f.TargetLayer()) // // Even if overshoot is allowed, but if higher layers do not have bit rates, should continue with current layer. @@ -555,7 +555,7 @@ func TestForwarderProvisionalAllocate(t *testing.T) { require.Equal(t, int64(0), usedBitrate) // committing should set target to (0, 2), i. e. leave it at current for opportunistic forwarding - expectedTargetLayers = buffer.VideoLayer{ + expectedTargetLayer = buffer.VideoLayer{ Spatial: 0, Temporal: 2, } @@ -564,15 +564,15 @@ func TestForwarderProvisionalAllocate(t *testing.T) { BandwidthRequested: bitrates[0][2], BandwidthDelta: bitrates[0][2] - 8, // 8 is the last allocation bandwidth requested Bitrates: bitrates, - TargetLayers: expectedTargetLayers, - RequestLayerSpatial: expectedTargetLayers.Spatial, - MaxLayers: expectedMaxLayers, + TargetLayer: expectedTargetLayer, + RequestLayerSpatial: expectedTargetLayer.Spatial, + MaxLayer: expectedMaxLayer, DistanceToDesired: 0.25, } result = f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, expectedTargetLayers, f.TargetLayers()) + require.Equal(t, expectedTargetLayer, f.TargetLayer()) // // Same case as above, but current is above max, so target should go to invalid @@ -597,16 +597,16 @@ func TestForwarderProvisionalAllocate(t *testing.T) { BandwidthRequested: 0, BandwidthDelta: 0, Bitrates: bitrates, - TargetLayers: buffer.InvalidLayers, + TargetLayer: buffer.InvalidLayer, RequestLayerSpatial: buffer.InvalidLayerSpatial, - MaxLayers: expectedMaxLayers, + MaxLayer: expectedMaxLayer, DistanceToDesired: 0.25, } result = f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, buffer.InvalidLayers, f.TargetLayers()) - require.Equal(t, buffer.InvalidLayers, f.CurrentLayers()) + require.Equal(t, buffer.InvalidLayer, f.TargetLayer()) + require.Equal(t, buffer.InvalidLayer, f.CurrentLayer()) } func TestForwarderProvisionalAllocateMute(t *testing.T) { @@ -629,21 +629,21 @@ func TestForwarderProvisionalAllocateMute(t *testing.T) { usedBitrate = f.ProvisionalAllocate(bitrates[2][3], buffer.VideoLayer{Spatial: 1, Temporal: 2}, true, true) require.Equal(t, int64(0), usedBitrate) - // committing should set target to buffer.InvalidLayers as track is muted + // committing should set target to buffer.InvalidLayer as track is muted expectedResult := VideoAllocation{ PauseReason: VideoPauseReasonMuted, BandwidthRequested: 0, BandwidthDelta: 0, Bitrates: bitrates, - TargetLayers: buffer.InvalidLayers, + TargetLayer: buffer.InvalidLayer, RequestLayerSpatial: buffer.InvalidLayerSpatial, - MaxLayers: buffer.DefaultMaxLayers, + MaxLayer: buffer.DefaultMaxLayer, DistanceToDesired: 0, } result := f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, buffer.InvalidLayers, f.TargetLayers()) + require.Equal(t, buffer.InvalidLayer, f.TargetLayer()) } func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { @@ -661,9 +661,9 @@ func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { f.ProvisionalAllocatePrepare(nil, bitrates) - // from scratch (buffer.InvalidLayers) should give back layer (0, 0) + // from scratch (buffer.InvalidLayer) should give back layer (0, 0) expectedTransition := VideoTransition{ - From: buffer.InvalidLayers, + From: buffer.InvalidLayer, To: buffer.VideoLayer{Spatial: 0, Temporal: 0}, BandwidthDelta: 1, } @@ -678,23 +678,23 @@ func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { BandwidthDelta: 1, BandwidthNeeded: bitrates[2][1], Bitrates: bitrates, - TargetLayers: expectedLayers, + TargetLayer: expectedLayers, RequestLayerSpatial: expectedLayers.Spatial, - MaxLayers: buffer.DefaultMaxLayers, + MaxLayer: buffer.DefaultMaxLayer, DistanceToDesired: 2.25, } result := f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, expectedLayers, f.TargetLayers()) + require.Equal(t, expectedLayers, f.TargetLayer()) // a higher target that is already streaming, just maintain it - targetLayers := buffer.VideoLayer{Spatial: 2, Temporal: 1} - f.vls.SetTarget(targetLayers) + targetLayer := buffer.VideoLayer{Spatial: 2, Temporal: 1} + f.vls.SetTarget(targetLayer) f.lastAllocation.BandwidthRequested = 10 expectedTransition = VideoTransition{ - From: targetLayers, - To: targetLayers, + From: targetLayer, + To: targetLayer, BandwidthDelta: 0, } transition = f.ProvisionalAllocateGetCooperativeTransition(false) @@ -707,21 +707,21 @@ func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { BandwidthDelta: 0, Bitrates: bitrates, BandwidthNeeded: bitrates[2][1], - TargetLayers: expectedLayers, + TargetLayer: expectedLayers, RequestLayerSpatial: expectedLayers.Spatial, - MaxLayers: buffer.DefaultMaxLayers, + MaxLayer: buffer.DefaultMaxLayer, DistanceToDesired: 0.0, } result = f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, expectedLayers, f.TargetLayers()) + require.Equal(t, expectedLayers, f.TargetLayer()) // from a target that has become unavailable, should switch to lower available layer - targetLayers = buffer.VideoLayer{Spatial: 2, Temporal: 2} - f.vls.SetTarget(targetLayers) + targetLayer = buffer.VideoLayer{Spatial: 2, Temporal: 2} + f.vls.SetTarget(targetLayer) expectedTransition = VideoTransition{ - From: targetLayers, + From: targetLayer, To: buffer.VideoLayer{Spatial: 2, Temporal: 1}, BandwidthDelta: 0, } @@ -734,10 +734,10 @@ func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { f.Mute(true) f.ProvisionalAllocatePrepare(nil, bitrates) - // mute should send target to buffer.InvalidLayers + // mute should send target to buffer.InvalidLayer expectedTransition = VideoTransition{ From: buffer.VideoLayer{Spatial: 2, Temporal: 1}, - To: buffer.InvalidLayers, + To: buffer.InvalidLayer, BandwidthDelta: -10, } transition = f.ProvisionalAllocateGetCooperativeTransition(false) @@ -757,12 +757,12 @@ func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { {9, 10, 0, 0}, } - f.vls.SetTarget(buffer.InvalidLayers) + f.vls.SetTarget(buffer.InvalidLayer) f.ProvisionalAllocatePrepare(nil, bitrates) - // from scratch (buffer.InvalidLayers) should go to a layer past maximum as overshoot is allowed + // from scratch (buffer.InvalidLayer) should go to a layer past maximum as overshoot is allowed expectedTransition = VideoTransition{ - From: buffer.InvalidLayers, + From: buffer.InvalidLayer, To: buffer.VideoLayer{Spatial: 1, Temporal: 0}, BandwidthDelta: 5, } @@ -771,20 +771,20 @@ func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { // committing should set target to (1, 0) expectedLayers = buffer.VideoLayer{Spatial: 1, Temporal: 0} - expectedMaxLayers := buffer.VideoLayer{Spatial: 0, Temporal: buffer.DefaultMaxLayerTemporal} + expectedMaxLayer := buffer.VideoLayer{Spatial: 0, Temporal: buffer.DefaultMaxLayerTemporal} expectedResult = VideoAllocation{ BandwidthRequested: 5, BandwidthDelta: 5, Bitrates: bitrates, - TargetLayers: expectedLayers, + TargetLayer: expectedLayers, RequestLayerSpatial: expectedLayers.Spatial, - MaxLayers: expectedMaxLayers, + MaxLayer: expectedMaxLayer, DistanceToDesired: -1.0, } result = f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, expectedLayers, f.TargetLayers()) + require.Equal(t, expectedLayers, f.TargetLayer()) // // Test continuing at current layers when feed is dry @@ -796,13 +796,13 @@ func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { } f.vls.SetCurrent(buffer.VideoLayer{Spatial: 0, Temporal: 2}) - f.vls.SetTarget(buffer.InvalidLayers) + f.vls.SetTarget(buffer.InvalidLayer) f.ProvisionalAllocatePrepare(nil, bitrates) - // from scratch (buffer.InvalidLayers) should go to current layer - // NOTE: targetLayer is set to buffer.InvalidLayers for testing, but in practice current layers valid and target layers invalid should not happen + // from scratch (buffer.InvalidLayer) should go to current layer + // NOTE: targetLayer is set to buffer.InvalidLayer for testing, but in practice current layers valid and target layers invalid should not happen expectedTransition = VideoTransition{ - From: buffer.InvalidLayers, + From: buffer.InvalidLayer, To: buffer.VideoLayer{Spatial: 0, Temporal: 2}, BandwidthDelta: -5, // 5 was the bandwidth needed for the last allocation } @@ -815,30 +815,30 @@ func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { BandwidthRequested: 0, BandwidthDelta: -5, Bitrates: bitrates, - TargetLayers: expectedLayers, + TargetLayer: expectedLayers, RequestLayerSpatial: expectedLayers.Spatial, - MaxLayers: expectedMaxLayers, + MaxLayer: expectedMaxLayer, DistanceToDesired: -0.5, } result = f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, expectedLayers, f.TargetLayers()) + require.Equal(t, expectedLayers, f.TargetLayer()) // committing should set target to current layers to enable opportunistic forwarding expectedResult = VideoAllocation{ BandwidthRequested: 0, BandwidthDelta: 0, Bitrates: bitrates, - TargetLayers: expectedLayers, + TargetLayer: expectedLayers, RequestLayerSpatial: expectedLayers.Spatial, - MaxLayers: expectedMaxLayers, + MaxLayer: expectedMaxLayer, DistanceToDesired: -0.5, } result = f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, expectedLayers, f.TargetLayers()) + require.Equal(t, expectedLayers, f.TargetLayer()) } func TestForwarderProvisionalAllocateGetBestWeightedTransition(t *testing.T) { @@ -857,7 +857,7 @@ func TestForwarderProvisionalAllocateGetBestWeightedTransition(t *testing.T) { f.vls.SetTarget(buffer.VideoLayer{Spatial: 2, Temporal: 2}) f.lastAllocation.BandwidthRequested = bitrates[2][2] expectedTransition := VideoTransition{ - From: f.TargetLayers(), + From: f.TargetLayer(), To: buffer.VideoLayer{Spatial: 2, Temporal: 0}, BandwidthDelta: 2, } @@ -909,7 +909,7 @@ func TestForwarderAllocateNextHigher(t *testing.T) { }) // move from (0, 0) -> (0, 1), i.e. a higher temporal layer is available in the same spatial layer - expectedTargetLayers := buffer.VideoLayer{ + expectedTargetLayer := buffer.VideoLayer{ Spatial: 0, Temporal: 1, } @@ -919,15 +919,15 @@ func TestForwarderAllocateNextHigher(t *testing.T) { BandwidthDelta: 1, BandwidthNeeded: bitrates[2][1], Bitrates: bitrates, - TargetLayers: expectedTargetLayers, - RequestLayerSpatial: expectedTargetLayers.Spatial, - MaxLayers: buffer.DefaultMaxLayers, + TargetLayer: expectedTargetLayer, + RequestLayerSpatial: expectedTargetLayer.Spatial, + MaxLayer: buffer.DefaultMaxLayer, DistanceToDesired: 2.0, } result, boosted = f.AllocateNextHigher(100_000_000, nil, bitrates, false) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, expectedTargetLayers, f.TargetLayers()) + require.Equal(t, expectedTargetLayer, f.TargetLayer()) require.True(t, boosted) // empty bitrates cannot increase layer, i. e. last allocation is left unchanged @@ -937,7 +937,7 @@ func TestForwarderAllocateNextHigher(t *testing.T) { // move from (0, 1) -> (1, 0), i.e. a higher spatial layer is available f.vls.SetCurrent(buffer.VideoLayer{Spatial: f.vls.GetCurrent().Spatial, Temporal: 1}) - expectedTargetLayers = buffer.VideoLayer{ + expectedTargetLayer = buffer.VideoLayer{ Spatial: 1, Temporal: 0, } @@ -947,20 +947,20 @@ func TestForwarderAllocateNextHigher(t *testing.T) { BandwidthDelta: 1, BandwidthNeeded: bitrates[2][1], Bitrates: bitrates, - TargetLayers: expectedTargetLayers, - RequestLayerSpatial: expectedTargetLayers.Spatial, - MaxLayers: buffer.DefaultMaxLayers, + TargetLayer: expectedTargetLayer, + RequestLayerSpatial: expectedTargetLayer.Spatial, + MaxLayer: buffer.DefaultMaxLayer, DistanceToDesired: 1.25, } result, boosted = f.AllocateNextHigher(100_000_000, nil, bitrates, false) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, expectedTargetLayers, f.TargetLayers()) + require.Equal(t, expectedTargetLayer, f.TargetLayer()) require.True(t, boosted) // next higher, move from (1, 0) -> (1, 3), still deficient though f.vls.SetCurrent(buffer.VideoLayer{Spatial: 1, Temporal: 0}) - expectedTargetLayers = buffer.VideoLayer{ + expectedTargetLayer = buffer.VideoLayer{ Spatial: 1, Temporal: 3, } @@ -970,20 +970,20 @@ func TestForwarderAllocateNextHigher(t *testing.T) { BandwidthDelta: 1, BandwidthNeeded: bitrates[2][1], Bitrates: bitrates, - TargetLayers: expectedTargetLayers, - RequestLayerSpatial: expectedTargetLayers.Spatial, - MaxLayers: buffer.DefaultMaxLayers, + TargetLayer: expectedTargetLayer, + RequestLayerSpatial: expectedTargetLayer.Spatial, + MaxLayer: buffer.DefaultMaxLayer, DistanceToDesired: 0.5, } result, boosted = f.AllocateNextHigher(100_000_000, nil, bitrates, false) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, expectedTargetLayers, f.TargetLayers()) + require.Equal(t, expectedTargetLayer, f.TargetLayer()) require.True(t, boosted) // next higher, move from (1, 3) -> (2, 1), optimal allocation f.vls.SetCurrent(buffer.VideoLayer{Spatial: f.vls.GetCurrent().Spatial, Temporal: 3}) - expectedTargetLayers = buffer.VideoLayer{ + expectedTargetLayer = buffer.VideoLayer{ Spatial: 2, Temporal: 1, } @@ -992,15 +992,15 @@ func TestForwarderAllocateNextHigher(t *testing.T) { BandwidthDelta: 2, Bitrates: bitrates, BandwidthNeeded: bitrates[2][1], - TargetLayers: expectedTargetLayers, - RequestLayerSpatial: expectedTargetLayers.Spatial, - MaxLayers: buffer.DefaultMaxLayers, + TargetLayer: expectedTargetLayer, + RequestLayerSpatial: expectedTargetLayer.Spatial, + MaxLayer: buffer.DefaultMaxLayer, DistanceToDesired: 0.0, } result, boosted = f.AllocateNextHigher(100_000_000, nil, bitrates, false) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, expectedTargetLayers, f.TargetLayers()) + require.Equal(t, expectedTargetLayer, f.TargetLayer()) require.True(t, boosted) // ask again, should return not boosted as there is no room to go higher @@ -1008,7 +1008,7 @@ func TestForwarderAllocateNextHigher(t *testing.T) { result, boosted = f.AllocateNextHigher(100_000_000, nil, bitrates, false) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, expectedTargetLayers, f.TargetLayers()) + require.Equal(t, expectedTargetLayer, f.TargetLayer()) require.False(t, boosted) // turn off everything, allocating next layer should result in streaming lowest layers @@ -1016,7 +1016,7 @@ func TestForwarderAllocateNextHigher(t *testing.T) { f.lastAllocation.IsDeficient = true f.lastAllocation.BandwidthRequested = 0 - expectedTargetLayers = buffer.VideoLayer{ + expectedTargetLayer = buffer.VideoLayer{ Spatial: 0, Temporal: 0, } @@ -1026,15 +1026,15 @@ func TestForwarderAllocateNextHigher(t *testing.T) { BandwidthDelta: 2, BandwidthNeeded: bitrates[2][1], Bitrates: bitrates, - TargetLayers: expectedTargetLayers, - RequestLayerSpatial: expectedTargetLayers.Spatial, - MaxLayers: buffer.DefaultMaxLayers, + TargetLayer: expectedTargetLayer, + RequestLayerSpatial: expectedTargetLayer.Spatial, + MaxLayer: buffer.DefaultMaxLayer, DistanceToDesired: 2.25, } result, boosted = f.AllocateNextHigher(100_000_000, nil, bitrates, false) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, expectedTargetLayers, f.TargetLayers()) + require.Equal(t, expectedTargetLayer, f.TargetLayer()) require.True(t, boosted) // no new available capacity cannot bump up layer @@ -1044,15 +1044,15 @@ func TestForwarderAllocateNextHigher(t *testing.T) { BandwidthDelta: 2, BandwidthNeeded: bitrates[2][1], Bitrates: bitrates, - TargetLayers: expectedTargetLayers, - RequestLayerSpatial: expectedTargetLayers.Spatial, - MaxLayers: buffer.DefaultMaxLayers, + TargetLayer: expectedTargetLayer, + RequestLayerSpatial: expectedTargetLayer.Spatial, + MaxLayer: buffer.DefaultMaxLayer, DistanceToDesired: 2.25, } result, boosted = f.AllocateNextHigher(0, nil, bitrates, false) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, expectedTargetLayers, f.TargetLayers()) + require.Equal(t, expectedTargetLayer, f.TargetLayer()) require.False(t, boosted) // test allowOvershoot @@ -1066,11 +1066,11 @@ func TestForwarderAllocateNextHigher(t *testing.T) { f.vls.SetCurrent(f.vls.GetTarget()) - expectedTargetLayers = buffer.VideoLayer{ + expectedTargetLayer = buffer.VideoLayer{ Spatial: 1, Temporal: 0, } - expectedMaxLayers := buffer.VideoLayer{ + expectedMaxLayer := buffer.VideoLayer{ Spatial: 0, Temporal: buffer.DefaultMaxLayerTemporal, } @@ -1078,16 +1078,16 @@ func TestForwarderAllocateNextHigher(t *testing.T) { BandwidthRequested: bitrates[1][0], BandwidthDelta: bitrates[1][0], Bitrates: bitrates, - TargetLayers: expectedTargetLayers, - RequestLayerSpatial: expectedTargetLayers.Spatial, - MaxLayers: expectedMaxLayers, + TargetLayer: expectedTargetLayer, + RequestLayerSpatial: expectedTargetLayer.Spatial, + MaxLayer: expectedMaxLayer, DistanceToDesired: -1.0, } // overshoot should return (1, 0) even if there is not enough capacity result, boosted = f.AllocateNextHigher(bitrates[1][0]-1, nil, bitrates, true) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, expectedTargetLayers, f.TargetLayers()) + require.Equal(t, expectedTargetLayer, f.TargetLayer()) require.True(t, boosted) } @@ -1116,15 +1116,15 @@ func TestForwarderPause(t *testing.T) { BandwidthDelta: 0 - bitrates[0][0], BandwidthNeeded: bitrates[2][3], Bitrates: bitrates, - TargetLayers: buffer.InvalidLayers, + TargetLayer: buffer.InvalidLayer, RequestLayerSpatial: buffer.InvalidLayerSpatial, - MaxLayers: buffer.DefaultMaxLayers, + MaxLayer: buffer.DefaultMaxLayer, DistanceToDesired: 3, } result := f.Pause(nil, bitrates) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, buffer.InvalidLayers, f.TargetLayers()) + require.Equal(t, buffer.InvalidLayer, f.TargetLayer()) } func TestForwarderPauseMute(t *testing.T) { @@ -1150,15 +1150,15 @@ func TestForwarderPauseMute(t *testing.T) { BandwidthRequested: 0, BandwidthDelta: 0 - bitrates[0][0], Bitrates: bitrates, - TargetLayers: buffer.InvalidLayers, + TargetLayer: buffer.InvalidLayer, RequestLayerSpatial: buffer.InvalidLayerSpatial, - MaxLayers: buffer.DefaultMaxLayers, + MaxLayer: buffer.DefaultMaxLayer, DistanceToDesired: 0, } result := f.Pause(nil, bitrates) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - require.Equal(t, buffer.InvalidLayers, f.TargetLayers()) + require.Equal(t, buffer.InvalidLayer, f.TargetLayer()) } func TestForwarderGetTranslationParamsMuted(t *testing.T) { @@ -1765,7 +1765,7 @@ func TestForwardGetSnTsForPadding(t *testing.T) { Spatial: 0, Temporal: 1, }) - f.vls.SetCurrent(buffer.InvalidLayers) + f.vls.SetCurrent(buffer.InvalidLayer) // send it through so that forwarder locks onto stream _, _ = f.GetTranslationParams(extPkt, 0) @@ -1832,7 +1832,7 @@ func TestForwardGetSnTsForBlankFrames(t *testing.T) { Spatial: 0, Temporal: 1, }) - f.vls.SetCurrent(buffer.InvalidLayers) + f.vls.SetCurrent(buffer.InvalidLayer) // send it through so that forwarder locks onto stream _, _ = f.GetTranslationParams(extPkt, 0) @@ -1902,7 +1902,7 @@ func TestForwardGetPaddingVP8(t *testing.T) { Spatial: 0, Temporal: 1, }) - f.vls.SetCurrent(buffer.InvalidLayers) + f.vls.SetCurrent(buffer.InvalidLayer) // send it through so that forwarder locks onto stream _, _ = f.GetTranslationParams(extPkt, 0) diff --git a/pkg/sfu/streamallocator/streamallocator.go b/pkg/sfu/streamallocator/streamallocator.go index cb0e0915f..c1b64fe62 100644 --- a/pkg/sfu/streamallocator/streamallocator.go +++ b/pkg/sfu/streamallocator/streamallocator.go @@ -396,12 +396,12 @@ func (s *StreamAllocator) OnSubscriptionChanged(downTrack *sfu.DownTrack) { s.maybePostEventAllocateTrack(downTrack) } -// called when subscribed layers changes (limiting max layers) -func (s *StreamAllocator) OnSubscribedLayersChanged(downTrack *sfu.DownTrack, layers buffer.VideoLayer) { +// called when subscribed layer changes (limiting max layer) +func (s *StreamAllocator) OnSubscribedLayerChanged(downTrack *sfu.DownTrack, layer buffer.VideoLayer) { shouldPost := false s.videoTracksMu.Lock() if track := s.videoTracks[livekit.TrackID(downTrack.ID())]; track != nil { - if track.SetMaxLayers(layers) && track.SetDirty(true) { + if track.SetMaxLayer(layer) && track.SetDirty(true) { shouldPost = true } } @@ -941,12 +941,12 @@ func (s *StreamAllocator) allocateAllTracks() { // 1. Stream as many tracks as possible, i.e. no pauses. // 2. Try to give fair allocation to all track. // - // Start with the lowest layers and give each track a chance at that layer and keep going up. - // As long as there is enough bandwidth for tracks to stream at the lowest layers, the first goal is achieved. + // Start with the lowest layer and give each track a chance at that layer and keep going up. + // As long as there is enough bandwidth for tracks to stream at the lowest layer, the first goal is achieved. // - // Tracks that have higher subscribed layers can use any additional available bandwidth. This tried to achieve the second goal. + // Tracks that have higher subscribed layer can use any additional available bandwidth. This tried to achieve the second goal. // - // If there is not enough bandwidth even for the lowest layers, tracks at lower priorities will be paused. + // If there is not enough bandwidth even for the lowest layer, tracks at lower priorities will be paused. // update := NewStreamStateUpdate() @@ -1002,13 +1002,13 @@ func (s *StreamAllocator) allocateAllTracks() { for spatial := int32(0); spatial <= buffer.DefaultMaxLayerSpatial; spatial++ { for temporal := int32(0); temporal <= buffer.DefaultMaxLayerTemporal; temporal++ { - layers := buffer.VideoLayer{ + layer := buffer.VideoLayer{ Spatial: spatial, Temporal: temporal, } for _, track := range sorted { - usedChannelCapacity := track.ProvisionalAllocate(availableChannelCapacity, layers, s.params.Config.AllowPause, FlagAllowOvershootWhileDeficient) + usedChannelCapacity := track.ProvisionalAllocate(availableChannelCapacity, layer, s.params.Config.AllowPause, FlagAllowOvershootWhileDeficient) availableChannelCapacity -= usedChannelCapacity if availableChannelCapacity < 0 { availableChannelCapacity = 0 @@ -1164,7 +1164,7 @@ func (s *StreamAllocator) maybeProbe() { } func (s *StreamAllocator) maybeProbeWithMedia() { - // boost deficient track farthest from desired layers + // boost deficient track farthest from desired layer for _, track := range s.getMaxDistanceSortedDeficient() { allocation, boosted := track.AllocateNextHigher(ChannelCapacityInfinity, FlagAllowOvershootInCatchup) if !boosted { @@ -1183,7 +1183,7 @@ func (s *StreamAllocator) maybeProbeWithMedia() { } func (s *StreamAllocator) maybeProbeWithPadding() { - // use deficient track farthest from desired layers to find how much to probe + // use deficient track farthest from desired layer to find how much to probe for _, track := range s.getMaxDistanceSortedDeficient() { transition, available := track.GetNextHigherTransition(FlagAllowOvershootInProbe) if !available || transition.BandwidthDelta < 0 { diff --git a/pkg/sfu/streamallocator/track.go b/pkg/sfu/streamallocator/track.go index 70df54990..e309baec6 100644 --- a/pkg/sfu/streamallocator/track.go +++ b/pkg/sfu/streamallocator/track.go @@ -16,7 +16,7 @@ type Track struct { publisherID livekit.ParticipantID logger logger.Logger - maxLayers buffer.VideoLayer + maxLayer buffer.VideoLayer totalPackets uint32 totalRepeatedNacks uint32 @@ -42,7 +42,7 @@ func NewTrack( isPaused: true, } t.SetPriority(0) - t.SetMaxLayers(downTrack.MaxLayers()) + t.SetMaxLayer(downTrack.MaxLayer()) return t } @@ -103,12 +103,12 @@ func (t *Track) PublisherID() livekit.ParticipantID { return t.publisherID } -func (t *Track) SetMaxLayers(layers buffer.VideoLayer) bool { - if t.maxLayers == layers { +func (t *Track) SetMaxLayer(layer buffer.VideoLayer) bool { + if t.maxLayer == layer { return false } - t.maxLayers = layers + t.maxLayer = layer return true } @@ -124,8 +124,8 @@ func (t *Track) ProvisionalAllocatePrepare() { t.downTrack.ProvisionalAllocatePrepare() } -func (t *Track) ProvisionalAllocate(availableChannelCapacity int64, layers buffer.VideoLayer, allowPause bool, allowOvershoot bool) int64 { - return t.downTrack.ProvisionalAllocate(availableChannelCapacity, layers, allowPause, allowOvershoot) +func (t *Track) ProvisionalAllocate(availableChannelCapacity int64, layer buffer.VideoLayer, allowPause bool, allowOvershoot bool) int64 { + return t.downTrack.ProvisionalAllocate(availableChannelCapacity, layer, allowPause, allowOvershoot) } func (t *Track) ProvisionalAllocateGetCooperativeTransition(allowOvershoot bool) sfu.VideoTransition { @@ -197,11 +197,11 @@ func (t TrackSorter) Less(i, j int) bool { return t[i].priority > t[j].priority } - if t[i].maxLayers.Spatial != t[j].maxLayers.Spatial { - return t[i].maxLayers.Spatial > t[j].maxLayers.Spatial + if t[i].maxLayer.Spatial != t[j].maxLayer.Spatial { + return t[i].maxLayer.Spatial > t[j].maxLayer.Spatial } - return t[i].maxLayers.Temporal > t[j].maxLayers.Temporal + return t[i].maxLayer.Temporal > t[j].maxLayer.Temporal } // ------------------------------------------------ diff --git a/pkg/sfu/streamtrackermanager.go b/pkg/sfu/streamtrackermanager.go index abd4a9554..e4598e675 100644 --- a/pkg/sfu/streamtrackermanager.go +++ b/pkg/sfu/streamtrackermanager.go @@ -295,12 +295,12 @@ func (s *StreamTrackerManager) DistanceToDesired() float64 { al, brs := s.getLayeredBitrateLocked() - maxLayers := buffer.InvalidLayers + maxLayer := buffer.InvalidLayer done: for s := int32(len(brs)) - 1; s >= 0; s-- { for t := int32(len(brs[0])) - 1; t >= 0; t-- { if brs[s][t] != 0 { - maxLayers = buffer.VideoLayer{ + maxLayer = buffer.VideoLayer{ Spatial: s, Temporal: t, } @@ -311,21 +311,21 @@ done: // before bit rate measurement is available, stream tracker could declare layer seen, account for that for _, layer := range al { - if layer > maxLayers.Spatial { - maxLayers.Spatial = layer - maxLayers.Temporal = s.maxTemporalLayerSeen // till bit rate measurement is available, assume max seen as temporal + if layer > maxLayer.Spatial { + maxLayer.Spatial = layer + maxLayer.Temporal = s.maxTemporalLayerSeen // till bit rate measurement is available, assume max seen as temporal } } - adjustedMaxLayers := maxLayers - if !maxLayers.IsValid() { + adjustedMaxLayers := maxLayer + if !maxLayer.IsValid() { adjustedMaxLayers = buffer.VideoLayer{Spatial: 0, Temporal: 0} } distance := ((s.maxExpectedLayer - adjustedMaxLayers.Spatial) * (s.maxTemporalLayerSeen + 1)) + (s.maxTemporalLayerSeen - adjustedMaxLayers.Temporal) - if !maxLayers.IsValid() { + if !maxLayer.IsValid() { distance++ } diff --git a/pkg/sfu/videolayerselector/base.go b/pkg/sfu/videolayerselector/base.go index 29a8cdc42..d902e4a50 100644 --- a/pkg/sfu/videolayerselector/base.go +++ b/pkg/sfu/videolayerselector/base.go @@ -24,12 +24,12 @@ type Base struct { func NewBase(logger logger.Logger) *Base { return &Base{ logger: logger, - maxLayer: buffer.InvalidLayers, - targetLayer: buffer.InvalidLayers, // start off with nothing, let streamallocator/opportunistic forwarder set the target + maxLayer: buffer.InvalidLayer, + targetLayer: buffer.InvalidLayer, // start off with nothing, let streamallocator/opportunistic forwarder set the target requestSpatial: buffer.InvalidLayerSpatial, - maxSeenLayer: buffer.InvalidLayers, - parkedLayer: buffer.InvalidLayers, - currentLayer: buffer.InvalidLayers, + maxSeenLayer: buffer.InvalidLayer, + parkedLayer: buffer.InvalidLayer, + currentLayer: buffer.InvalidLayer, } } diff --git a/pkg/sfu/videolayerselector/simulcast.go b/pkg/sfu/videolayerselector/simulcast.go index 9bb1d368f..ec875e11e 100644 --- a/pkg/sfu/videolayerselector/simulcast.go +++ b/pkg/sfu/videolayerselector/simulcast.go @@ -89,7 +89,7 @@ func (s *Simulcast) Select(extPkt *buffer.ExtPacket, layer int32) (result VideoL if !isActive { result.IsResuming = true } - s.SetParked(buffer.InvalidLayers) + s.SetParked(buffer.InvalidLayer) if s.currentLayer.Spatial >= s.maxLayer.Spatial { result.IsSwitchingToMaxSpatial = true From 48b2ea11c100a81652410e24e205d8ccbe238163 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sat, 8 Apr 2023 13:41:17 +0530 Subject: [PATCH 066/324] Forgot to transfer ddBytes (#1592) --- pkg/sfu/downtrack.go | 2 +- pkg/sfu/forwarder.go | 1 + pkg/sfu/receiver.go | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 4043ccd19..ccd68e5ca 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -492,7 +492,7 @@ func (d *DownTrack) maybeStartKeyFrameRequester() { // d.stopKeyFrameRequester() - // TODO : for svc, don't need pli/lrr when layer comes down + // SVC-TODO : don't need pli/lrr when layer comes down locked, layer := d.forwarder.CheckSync() if !locked { go d.keyFrameRequester(d.keyFrameRequestGeneration.Load(), layer) diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index de689e8dd..8d75db2f9 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -1490,6 +1490,7 @@ func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer in tp.isSwitchingToMaxLayer = result.IsSwitchingToMaxSpatial tp.isResuming = result.IsResuming tp.marker = result.RTPMarker + tp.ddBytes = result.DependencyDescriptorExtension if FlagPauseOnDowngrade && f.isDeficientLocked() && f.vls.GetTarget().Spatial < f.vls.GetCurrent().Spatial { // diff --git a/pkg/sfu/receiver.go b/pkg/sfu/receiver.go index 3d2cc9c31..0868a574e 100644 --- a/pkg/sfu/receiver.go +++ b/pkg/sfu/receiver.go @@ -484,7 +484,7 @@ func (w *WebRTCReceiver) sendRTCP(packets []rtcp.Packet) { } func (w *WebRTCReceiver) SendPLI(layer int32, force bool) { - // TODO : should send LRR (Layer Refresh Request) instead of PLI + // SVC-TODO : should send LRR (Layer Refresh Request) instead of PLI buff := w.getBuffer(layer) if buff == nil { return From a12f467e7adadc8ed2038a523780a41e314d1044 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Sun, 9 Apr 2023 17:51:33 -0700 Subject: [PATCH 067/324] chore: added omitempty to optional config entries (#1594) --- pkg/config/config.go | 82 ++++++++++++++++++++++---------------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 0195543bd..42ab4e735 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -45,7 +45,7 @@ var ( type Config struct { Port uint32 `yaml:"port"` - BindAddresses []string `yaml:"bind_addresses"` + BindAddresses []string `yaml:"bind_addresses,omitempty"` PrometheusPort uint32 `yaml:"prometheus_port,omitempty"` Environment string `yaml:"environment,omitempty"` RTC RTCConfig `yaml:"rtc,omitempty"` @@ -81,11 +81,11 @@ type RTCConfig struct { TURNServers []TURNServer `yaml:"turn_servers,omitempty"` UseExternalIP bool `yaml:"use_external_ip"` UseICELite bool `yaml:"use_ice_lite,omitempty"` - Interfaces InterfacesConfig `yaml:"interfaces"` - IPs IPsConfig `yaml:"ips"` + Interfaces InterfacesConfig `yaml:"interfaces,omitempty"` + IPs IPsConfig `yaml:"ips,omitempty"` EnableLoopbackCandidate bool `yaml:"enable_loopback_candidate"` - UseMDNS bool `yaml:"use_mdns"` - StrictACKs bool `yaml:"strict_acks"` + UseMDNS bool `yaml:"use_mdns,omitempty"` + StrictACKs bool `yaml:"strict_acks,omitempty"` // Number of packets to buffer for NACK PacketBufferSize int `yaml:"packet_buffer_size,omitempty"` @@ -131,34 +131,34 @@ type CongestionControlConfig struct { } type InterfacesConfig struct { - Includes []string `yaml:"includes"` - Excludes []string `yaml:"excludes"` + Includes []string `yaml:"includes,omitempty"` + Excludes []string `yaml:"excludes,omitempty"` } type IPsConfig struct { - Includes []string `yaml:"includes"` - Excludes []string `yaml:"excludes"` + Includes []string `yaml:"includes,omitempty"` + Excludes []string `yaml:"excludes,omitempty"` } type AudioConfig struct { // minimum level to be considered active, 0-127, where 0 is loudest - ActiveLevel uint8 `yaml:"active_level"` + ActiveLevel uint8 `yaml:"active_level,omitempty"` // percentile to measure, a participant is considered active if it has exceeded the ActiveLevel more than // MinPercentile% of the time - MinPercentile uint8 `yaml:"min_percentile"` + MinPercentile uint8 `yaml:"min_percentile,omitempty"` // interval to update clients, in ms - UpdateInterval uint32 `yaml:"update_interval"` + UpdateInterval uint32 `yaml:"update_interval,omitempty"` // smoothing for audioLevel values sent to the client. // audioLevel will be an average of `smooth_intervals`, 0 to disable - SmoothIntervals uint32 `yaml:"smooth_intervals"` + SmoothIntervals uint32 `yaml:"smooth_intervals,omitempty"` // enable red encoding downtrack for opus only audio up track - ActiveREDEncoding bool `yaml:"active_red_encoding"` + ActiveREDEncoding bool `yaml:"active_red_encoding,omitempty"` } type StreamTrackerPacketConfig struct { - SamplesRequired uint32 `yaml:"samples_required"` // number of samples needed per cycle - CyclesRequired uint32 `yaml:"cycles_required"` // number of cycles needed to be active - CycleDuration time.Duration `yaml:"cycle_duration"` + SamplesRequired uint32 `yaml:"samples_required,omitempty"` // number of samples needed per cycle + CyclesRequired uint32 `yaml:"cycles_required,omitempty"` // number of cycles needed to be active + CycleDuration time.Duration `yaml:"cycle_duration,omitempty"` } type StreamTrackerFrameConfig struct { @@ -173,8 +173,8 @@ type StreamTrackerConfig struct { } type StreamTrackersConfig struct { - Video StreamTrackerConfig `yaml:"video"` - Screenshare StreamTrackerConfig `yaml:"screenshare"` + Video StreamTrackerConfig `yaml:"video,omitempty"` + Screenshare StreamTrackerConfig `yaml:"screenshare,omitempty"` } type VideoConfig struct { @@ -184,12 +184,12 @@ type VideoConfig struct { type RoomConfig struct { // enable rooms to be automatically created - AutoCreate bool `yaml:"auto_create"` - EnabledCodecs []CodecSpec `yaml:"enabled_codecs"` - MaxParticipants uint32 `yaml:"max_participants"` - EmptyTimeout uint32 `yaml:"empty_timeout"` - EnableRemoteUnmute bool `yaml:"enable_remote_unmute"` - MaxMetadataSize uint32 `yaml:"max_metadata_size"` + AutoCreate bool `yaml:"auto_create,omitempty"` + EnabledCodecs []CodecSpec `yaml:"enabled_codecs,omitempty"` + MaxParticipants uint32 `yaml:"max_participants,omitempty"` + EmptyTimeout uint32 `yaml:"empty_timeout,omitempty"` + EnableRemoteUnmute bool `yaml:"enable_remote_unmute,omitempty"` + MaxMetadataSize uint32 `yaml:"max_metadata_size,omitempty"` } type CodecSpec struct { @@ -204,14 +204,14 @@ type LoggingConfig struct { type TURNConfig struct { Enabled bool `yaml:"enabled"` - Domain string `yaml:"domain"` - CertFile string `yaml:"cert_file"` - KeyFile string `yaml:"key_file"` - TLSPort int `yaml:"tls_port"` - UDPPort int `yaml:"udp_port"` + Domain string `yaml:"domain,omitempty"` + CertFile string `yaml:"cert_file,omitempty"` + KeyFile string `yaml:"key_file,omitempty"` + TLSPort int `yaml:"tls_port,omitempty"` + UDPPort int `yaml:"udp_port,omitempty"` RelayPortRangeStart uint16 `yaml:"relay_range_start,omitempty"` RelayPortRangeEnd uint16 `yaml:"relay_range_end,omitempty"` - ExternalTLS bool `yaml:"external_tls"` + ExternalTLS bool `yaml:"external_tls,omitempty"` } type WebHookConfig struct { @@ -222,18 +222,18 @@ type WebHookConfig struct { type NodeSelectorConfig struct { Kind string `yaml:"kind"` - SortBy string `yaml:"sort_by"` - CPULoadLimit float32 `yaml:"cpu_load_limit"` - SysloadLimit float32 `yaml:"sysload_limit"` - Regions []RegionConfig `yaml:"regions"` + SortBy string `yaml:"sort_by,omitempty"` + CPULoadLimit float32 `yaml:"cpu_load_limit,omitempty"` + SysloadLimit float32 `yaml:"sysload_limit,omitempty"` + Regions []RegionConfig `yaml:"regions,omitempty"` } type SignalRelayConfig struct { Enabled bool `yaml:"enabled"` - MaxAttempts int `yaml:"max_attempts"` - Timeout time.Duration `yaml:"timeout"` - Backoff time.Duration `yaml:"backoff"` - StreamBufferSize int `yaml:"stream_buffer_size"` + MaxAttempts int `yaml:"max_attempts,omitempty"` + Timeout time.Duration `yaml:"timeout,omitempty"` + Backoff time.Duration `yaml:"backoff,omitempty"` + StreamBufferSize int `yaml:"stream_buffer_size,omitempty"` } // RegionConfig lists available regions and their latitude/longitude, so the selector would prefer @@ -245,8 +245,8 @@ type RegionConfig struct { } type LimitConfig struct { - NumTracks int32 `yaml:"num_tracks"` - BytesPerSec float32 `yaml:"bytes_per_sec"` + NumTracks int32 `yaml:"num_tracks,omitempty"` + BytesPerSec float32 `yaml:"bytes_per_sec,omitempty"` } type EgressConfig struct { From eb095db70a9beb4cc1f31a745d65e26c21abe48e Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Sun, 9 Apr 2023 18:18:21 -0700 Subject: [PATCH 068/324] Batch signal retries (#1593) * batch signal retries * cleanup * update protocol * range check message dedup * update protocol with codegen * block while draining * only log send timeouts * cleanup * cleanup * cleanup * typo * update config yaml options * update protocol --- go.mod | 8 +- go.sum | 16 +-- pkg/config/config.go | 13 +- pkg/routing/signal.go | 264 ++++++++++++++++++++++++++++++++----- pkg/service/signal.go | 119 ++++++----------- pkg/service/signal_test.go | 83 ++++++++++++ 6 files changed, 370 insertions(+), 133 deletions(-) create mode 100644 pkg/service/signal_test.go diff --git a/go.mod b/go.mod index 7c052ce66..b1d53bcec 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 - github.com/livekit/protocol v1.5.2 + github.com/livekit/protocol v1.5.3-0.20230410011118-30f8b4c081aa github.com/livekit/psrpc v0.2.11-0.20230405191830-d76f71512630 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.14.0 @@ -90,12 +90,12 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect go.uber.org/multierr v1.6.0 // indirect - golang.org/x/crypto v0.7.0 // indirect + golang.org/x/crypto v0.8.0 // indirect golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect golang.org/x/mod v0.8.0 // indirect - golang.org/x/net v0.8.0 // indirect + golang.org/x/net v0.9.0 // indirect golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.8.0 // indirect + golang.org/x/text v0.9.0 // indirect golang.org/x/tools v0.6.0 // indirect google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd // indirect google.golang.org/grpc v1.54.0 // indirect diff --git a/go.sum b/go.sum index 59640e93b..e763d9714 100644 --- a/go.sum +++ b/go.sum @@ -235,8 +235,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 h1:QlQFyMwCDgjyySsrgmrMcVbEBA6KZcyTzvK+z346tUA= github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26/go.mod h1:eDA41kiySZoG+wy4Etsjb3w0jjLx69i/vAmSjG4bteA= -github.com/livekit/protocol v1.5.2 h1:mbbkJNxbStvb9sDtB7CFX7NnTObYKFumNU7wWm4UOfY= -github.com/livekit/protocol v1.5.2/go.mod h1:UFgAWejoO4eshaaDe2jynTdQWwSktNO+8Wx19V7bs+o= +github.com/livekit/protocol v1.5.3-0.20230410011118-30f8b4c081aa h1:s7ACG7CGvt12tiBYSsywSavYh3S/JLVZI7Ob3ot0rKs= +github.com/livekit/protocol v1.5.3-0.20230410011118-30f8b4c081aa/go.mod h1:GzQYVsW/eIsI7xdDTNUGed+SD7IpCI1dLdOlIqRmd2U= github.com/livekit/psrpc v0.2.11-0.20230405191830-d76f71512630 h1:Rm5KLZgQxWnTidY+H8MsAV6sk1iiFxeXqPFgSLkMing= github.com/livekit/psrpc v0.2.11-0.20230405191830-d76f71512630/go.mod h1:K0j8f1PgLShR7Lx80KbmwFkDH2BvOnycXGV0OSRURKc= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= @@ -436,8 +436,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= -golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ= +golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -522,8 +522,8 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -626,8 +626,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= diff --git a/pkg/config/config.go b/pkg/config/config.go index 42ab4e735..50cdf5ccc 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -230,10 +230,11 @@ type NodeSelectorConfig struct { type SignalRelayConfig struct { Enabled bool `yaml:"enabled"` - MaxAttempts int `yaml:"max_attempts,omitempty"` - Timeout time.Duration `yaml:"timeout,omitempty"` - Backoff time.Duration `yaml:"backoff,omitempty"` + RetryTimeout time.Duration `yaml:"retry_timeout,omitempty"` + MinRetryInterval time.Duration `yaml:"min_retry_interval,omitempty"` + MaxRetryInterval time.Duration `yaml:"max_retry_interval,omitempty"` StreamBufferSize int `yaml:"stream_buffer_size,omitempty"` + MinVersion int `yaml:"min_version,omitempty"` } // RegionConfig lists available regions and their latitude/longitude, so the selector would prefer @@ -407,9 +408,9 @@ func NewConfig(confString string, strictMode bool, c *cli.Context, baseFlags []c }, SignalRelay: SignalRelayConfig{ Enabled: false, - MaxAttempts: 3, - Timeout: 500 * time.Millisecond, - Backoff: 500 * time.Millisecond, + RetryTimeout: 30 * time.Second, + MinRetryInterval: 500 * time.Millisecond, + MaxRetryInterval: 5 * time.Second, StreamBufferSize: 1000, }, Keys: map[string]string{}, diff --git a/pkg/routing/signal.go b/pkg/routing/signal.go index 418852c3d..5be76e70d 100644 --- a/pkg/routing/signal.go +++ b/pkg/routing/signal.go @@ -2,6 +2,9 @@ package routing import ( "context" + "errors" + "sync" + "time" "go.uber.org/atomic" "google.golang.org/protobuf/proto" @@ -36,11 +39,6 @@ func NewSignalClient(nodeID livekit.NodeID, bus psrpc.MessageBus, config config. nodeID, bus, middleware.WithClientMetrics(prometheus.PSRPCMetricsObserver{}), - middleware.WithStreamRetries(middleware.RetryOptions{ - MaxAttempts: config.MaxAttempts, - Timeout: config.Timeout, - Backoff: config.Backoff, - }), psrpc.WithClientChannelSize(config.StreamBufferSize), ) if err != nil { @@ -75,14 +73,15 @@ func (r *signalClient) StartParticipantSignal( return } - logger.Debugw( - "starting signal connection", + l := logger.GetLogger().WithValues( "room", roomName, "reqNodeID", nodeID, "participant", pi.Identity, "connectionID", connectionID, ) + l.Debugw("starting signal connection") + stream, err := r.client.RelaySignal(ctx, nodeID) if err != nil { return @@ -94,49 +93,248 @@ func (r *signalClient) StartParticipantSignal( return } + sink := NewSignalMessageSink(SignalSinkParams[*rpc.RelaySignalRequest, *rpc.RelaySignalResponse]{ + Logger: l, + Stream: stream, + Config: r.config, + Writer: signalRequestMessageWriter{}, + CloseOnFailure: true, + }) resChan := NewDefaultMessageChannel() go func() { r.active.Inc() defer r.active.Dec() - var err error - for msg := range stream.Channel() { - if err = resChan.WriteMessage(msg.Response); err != nil { - break - } - for _, res := range msg.Responses { - if err = resChan.WriteMessage(res); err != nil { - break - } - } - } - - logger.Debugw("participant signal stream closed", - "error", err, - "room", ss.RoomName, - "participant", ss.Identity, - "connectionID", connectionID, + err = CopySignalStreamToMessageChannel[*rpc.RelaySignalRequest, *rpc.RelaySignalResponse]( + stream, + resChan, + signalResponseMessageReader{}, + r.config, ) + l.Debugw("participant signal stream closed", "error", err) resChan.Close() }() - return connectionID, &relaySignalRequestSink{stream}, resChan, nil + return connectionID, sink, resChan, nil } -type relaySignalRequestSink struct { - psrpc.ClientStream[*rpc.RelaySignalRequest, *rpc.RelaySignalResponse] +type signalRequestMessageWriter struct{} + +func (e signalRequestMessageWriter) WriteOne(seq uint64, msg proto.Message) *rpc.RelaySignalRequest { + return &rpc.RelaySignalRequest{ + Seq: seq, + Request: msg.(*livekit.SignalRequest), + } } -func (s *relaySignalRequestSink) Close() { - s.ClientStream.Close(nil) +func (e signalRequestMessageWriter) WriteMany(seq uint64, msgs []proto.Message) *rpc.RelaySignalRequest { + r := &rpc.RelaySignalRequest{ + Seq: seq, + Requests: make([]*livekit.SignalRequest, 0, len(msgs)), + } + for _, m := range msgs { + r.Requests = append(r.Requests, m.(*livekit.SignalRequest)) + } + return r } -func (s *relaySignalRequestSink) IsClosed() bool { - return s.Context().Err() != nil +type signalResponseMessageReader struct{} + +func (e signalResponseMessageReader) Read(rm *rpc.RelaySignalResponse) ([]proto.Message, error) { + msgs := make([]proto.Message, 0, len(rm.Responses)+1) + if rm.Response != nil { + msgs = append(msgs, rm.Response) + } + for _, m := range rm.Responses { + msgs = append(msgs, m) + } + return msgs, nil } -func (s *relaySignalRequestSink) WriteMessage(msg proto.Message) error { - return s.Send(&rpc.RelaySignalRequest{Request: msg.(*livekit.SignalRequest)}) +type RelaySignalMessage interface { + proto.Message + GetSeq() uint64 +} + +type SignalMessageWriter[SendType RelaySignalMessage] interface { + WriteOne(seq uint64, msg proto.Message) SendType + WriteMany(seq uint64, msgs []proto.Message) SendType +} + +type SignalMessageReader[RecvType RelaySignalMessage] interface { + Read(msg RecvType) ([]proto.Message, error) +} + +func CopySignalStreamToMessageChannel[SendType, RecvType RelaySignalMessage]( + stream psrpc.Stream[SendType, RecvType], + ch *MessageChannel, + reader SignalMessageReader[RecvType], + config config.SignalRelayConfig, +) error { + r := &signalMessageReader[SendType, RecvType]{ + reader: reader, + config: config, + } + for msg := range stream.Channel() { + var res []proto.Message + res, err := r.Read(msg) + if err != nil { + return err + } + for _, r := range res { + if err = ch.WriteMessage(r); err != nil { + return err + } + } + } + return stream.Err() +} + +type signalMessageReader[SendType, RecvType RelaySignalMessage] struct { + seq uint64 + reader SignalMessageReader[RecvType] + config config.SignalRelayConfig +} + +func (r *signalMessageReader[SendType, RecvType]) Read(msg RecvType) ([]proto.Message, error) { + res, err := r.reader.Read(msg) + if err != nil { + return nil, err + } + + if r.config.MinVersion >= 1 { + if r.seq < msg.GetSeq() { + return nil, errors.New("signal message dropped") + } + if r.seq > msg.GetSeq() { + n := int(r.seq - msg.GetSeq()) + if n > len(res) { + n = len(res) + } + res = res[n:] + } + r.seq += uint64(len(res)) + } + return res, nil +} + +type SignalSinkParams[SendType, RecvType RelaySignalMessage] struct { + Stream psrpc.Stream[SendType, RecvType] + Logger logger.Logger + Config config.SignalRelayConfig + Writer SignalMessageWriter[SendType] + CloseOnFailure bool +} + +func NewSignalMessageSink[SendType, RecvType RelaySignalMessage](params SignalSinkParams[SendType, RecvType]) MessageSink { + return &signalMessageSink[SendType, RecvType]{ + SignalSinkParams: params, + } +} + +var ErrSignalFailed = errors.New("signal stream failed") + +type signalMessageSink[SendType, RecvType RelaySignalMessage] struct { + SignalSinkParams[SendType, RecvType] + + mu sync.Mutex + seq uint64 + queue []proto.Message + writing bool + draining bool +} + +func (s *signalMessageSink[SendType, RecvType]) Close() { + s.mu.Lock() + s.draining = true + if !s.writing { + s.Stream.Close(nil) + } + s.mu.Unlock() + + <-s.Stream.Context().Done() +} + +func (s *signalMessageSink[SendType, RecvType]) IsClosed() bool { + return s.Stream.Err() != nil +} + +func (s *signalMessageSink[SendType, RecvType]) nextMessage() (msg SendType, n int) { + if len(s.queue) == 0 { + return + } + if s.Config.MinVersion >= 1 { + return s.Writer.WriteMany(s.seq, s.queue), len(s.queue) + } + return s.Writer.WriteOne(s.seq, s.queue[0]), 1 +} + +func (s *signalMessageSink[SendType, RecvType]) write() { + interval := s.Config.MinRetryInterval + deadline := time.Now().Add(s.Config.RetryTimeout) + + s.mu.Lock() + for { + msg, n := s.nextMessage() + if n == 0 || s.IsClosed() { + if s.draining { + s.Stream.Close(nil) + } + s.writing = false + break + } + s.mu.Unlock() + + err := s.Stream.Send(msg, psrpc.WithTimeout(interval)) + if err != nil { + if time.Now().After(deadline) { + s.Logger.Warnw("could not send signal message", err) + + s.mu.Lock() + s.seq += uint64(len(s.queue)) + s.queue = nil + + if s.CloseOnFailure { + s.Stream.Close(ErrSignalFailed) + } + s.mu.Unlock() + return + } + + interval *= 2 + if interval > s.Config.MaxRetryInterval { + interval = s.Config.MaxRetryInterval + } + } + + s.mu.Lock() + if err == nil { + interval = s.Config.MinRetryInterval + deadline = time.Now().Add(s.Config.RetryTimeout) + + s.seq += uint64(n) + s.queue = s.queue[n:] + } + } + s.mu.Unlock() +} + +func (s *signalMessageSink[SendType, RecvType]) WriteMessage(msg proto.Message) error { + s.mu.Lock() + defer s.mu.Unlock() + + if err := s.Stream.Err(); err != nil { + return err + } else if s.draining { + return psrpc.ErrStreamClosed + } + + s.queue = append(s.queue, msg) + if !s.writing { + s.writing = true + go s.write() + } + return nil } diff --git a/pkg/service/signal.go b/pkg/service/signal.go index 407c173e1..3d9b6c17e 100644 --- a/pkg/service/signal.go +++ b/pkg/service/signal.go @@ -2,8 +2,6 @@ package service import ( "context" - "fmt" - "sync" "github.com/pkg/errors" "google.golang.org/protobuf/proto" @@ -40,14 +38,9 @@ func NewSignalServer( ) (*SignalServer, error) { s, err := rpc.NewTypedSignalServer( nodeID, - &signalService{region, sessionHandler}, + &signalService{region, sessionHandler, config}, bus, middleware.WithServerMetrics(prometheus.PSRPCMetricsObserver{}), - psrpc.WithServerStreamInterceptors(middleware.NewStreamRetryInterceptorFactory(middleware.RetryOptions{ - MaxAttempts: config.MaxAttempts, - Timeout: config.Timeout, - Backoff: config.Backoff, - })), psrpc.WithServerChannelSize(config.StreamBufferSize), ) if err != nil { @@ -101,6 +94,7 @@ func (r *SignalServer) Stop() { type signalService struct { region string sessionHandler SessionHandler + config config.SignalRelayConfig } func (r *signalService) RelaySignal(stream psrpc.ServerStream[*rpc.RelaySignalResponse, *rpc.RelaySignalRequest]) (err error) { @@ -134,92 +128,53 @@ func (r *signalService) RelaySignal(stream psrpc.ServerStream[*rpc.RelaySignalRe reqChan := routing.NewDefaultMessageChannel() defer reqChan.Close() - err = r.sessionHandler( - ctx, - livekit.RoomName(ss.RoomName), - *pi, - livekit.ConnectionID(ss.ConnectionId), - reqChan, - &relaySignalResponseSink{ - ServerStream: stream, - logger: l, - }, - ) + sink := routing.NewSignalMessageSink(routing.SignalSinkParams[*rpc.RelaySignalResponse, *rpc.RelaySignalRequest]{ + Logger: l, + Stream: stream, + Config: r.config, + Writer: signalResponseMessageWriter{}, + }) + + err = r.sessionHandler(ctx, livekit.RoomName(ss.RoomName), *pi, livekit.ConnectionID(ss.ConnectionId), reqChan, sink) if err != nil { l.Errorw("could not handle new participant", err) } - for msg := range stream.Channel() { - if err = reqChan.WriteMessage(msg.Request); err != nil { - break - } - } + err = routing.CopySignalStreamToMessageChannel[*rpc.RelaySignalResponse, *rpc.RelaySignalRequest](stream, reqChan, signalRequestMessageReader{}, r.config) + l.Debugw("participant signal stream closed", "error", err) - l.Debugw("participant signal stream closed") return } -type relaySignalResponseSink struct { - psrpc.ServerStream[*rpc.RelaySignalResponse, *rpc.RelaySignalRequest] - logger logger.Logger +type signalResponseMessageWriter struct{} - mu sync.Mutex - queue []*livekit.SignalResponse - writing bool - draining bool -} - -func (s *relaySignalResponseSink) Close() { - s.mu.Lock() - s.draining = true - if !s.writing { - s.ServerStream.Close(nil) - } - s.mu.Unlock() -} - -func (s *relaySignalResponseSink) IsClosed() bool { - return s.Context().Err() != nil -} - -func (s *relaySignalResponseSink) write() { - for { - s.mu.Lock() - var msg *livekit.SignalResponse - if len(s.queue) != 0 && !s.IsClosed() { - msg = s.queue[0] - s.queue = s.queue[1:] - } else { - if s.draining { - s.ServerStream.Close(nil) - } - s.writing = false - s.mu.Unlock() - return - } - s.mu.Unlock() - - if err := s.Send(&rpc.RelaySignalResponse{Response: msg}); err != nil { - s.logger.Warnw( - "could not send message to participant", err, - "messageType", fmt.Sprintf("%T", msg.Message), - ) - } +func (e signalResponseMessageWriter) WriteOne(seq uint64, msg proto.Message) *rpc.RelaySignalResponse { + return &rpc.RelaySignalResponse{ + Seq: seq, + Response: msg.(*livekit.SignalResponse), } } -func (s *relaySignalResponseSink) WriteMessage(msg proto.Message) error { - s.mu.Lock() - defer s.mu.Unlock() - - if s.draining || s.IsClosed() { - return psrpc.ErrStreamClosed +func (e signalResponseMessageWriter) WriteMany(seq uint64, msgs []proto.Message) *rpc.RelaySignalResponse { + r := &rpc.RelaySignalResponse{ + Seq: seq, + Responses: make([]*livekit.SignalResponse, 0, len(msgs)), } - - s.queue = append(s.queue, msg.(*livekit.SignalResponse)) - if !s.writing { - s.writing = true - go s.write() + for _, m := range msgs { + r.Responses = append(r.Responses, m.(*livekit.SignalResponse)) } - return nil + return r +} + +type signalRequestMessageReader struct{} + +func (e signalRequestMessageReader) Read(rm *rpc.RelaySignalRequest) ([]proto.Message, error) { + msgs := make([]proto.Message, 0, len(rm.Requests)+1) + if rm.Request != nil { + msgs = append(msgs, rm.Request) + } + for _, m := range rm.Requests { + msgs = append(msgs, m) + } + return msgs, nil } diff --git a/pkg/service/signal_test.go b/pkg/service/signal_test.go new file mode 100644 index 000000000..a953fd9dc --- /dev/null +++ b/pkg/service/signal_test.go @@ -0,0 +1,83 @@ +package service + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" + + "github.com/livekit/livekit-server/pkg/config" + "github.com/livekit/livekit-server/pkg/routing" + "github.com/livekit/livekit-server/pkg/telemetry/prometheus" + "github.com/livekit/protocol/livekit" + "github.com/livekit/psrpc" +) + +func init() { + prometheus.Init("node", livekit.NodeType_CONTROLLER, "test") +} + +func TestSignal(t *testing.T) { + bus := psrpc.NewLocalMessageBus() + cfg := config.SignalRelayConfig{ + Enabled: false, + RetryTimeout: 30 * time.Second, + MinRetryInterval: 500 * time.Millisecond, + MaxRetryInterval: 5 * time.Second, + StreamBufferSize: 1000, + MinVersion: 1, + } + + reqMessageIn := &livekit.SignalRequest{ + Message: &livekit.SignalRequest_Ping{Ping: 123}, + } + resMessageIn := &livekit.SignalResponse{ + Message: &livekit.SignalResponse_Pong{Pong: 321}, + } + + var reqMessageOut proto.Message + var resErr error + done := make(chan struct{}) + + client, err := routing.NewSignalClient(livekit.NodeID("node0"), bus, cfg) + require.NoError(t, err) + + _, err = NewSignalServer(livekit.NodeID("node1"), "region", bus, cfg, func( + ctx context.Context, + roomName livekit.RoomName, + pi routing.ParticipantInit, + connectionID livekit.ConnectionID, + requestSource routing.MessageSource, + responseSink routing.MessageSink, + ) error { + go func() { + reqMessageOut = <-requestSource.ReadChan() + resErr = responseSink.WriteMessage(resMessageIn) + responseSink.Close() + close(done) + }() + return nil + }) + require.NoError(t, err) + + _, reqSink, resSource, err := client.StartParticipantSignal( + context.Background(), + livekit.RoomName("room1"), + routing.ParticipantInit{}, + livekit.NodeID("node1"), + ) + require.NoError(t, err) + + err = reqSink.WriteMessage(reqMessageIn) + require.NoError(t, err) + + <-done + require.True(t, proto.Equal(reqMessageIn, reqMessageOut), "req message should match %s %s", protojson.Format(reqMessageIn), protojson.Format(reqMessageOut)) + require.NoError(t, resErr) + + resMessageOut := <-resSource.ReadChan() + require.True(t, proto.Equal(resMessageIn, resMessageOut), "res message should match %s %s", protojson.Format(resMessageIn), protojson.Format(resMessageOut)) +} From 6e5e3bdcf3410a881fa694d2f0df1c1d99de5758 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 10 Apr 2023 11:00:29 +0530 Subject: [PATCH 069/324] Update signal relay config doc (#1596) --- config-sample.yaml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/config-sample.yaml b/config-sample.yaml index 00d072f53..8a6fcf7c5 100644 --- a/config-sample.yaml +++ b/config-sample.yaml @@ -169,12 +169,14 @@ keys: # signal_relay: # # disabled by default. will be enabled by default in future versions # enabled: true -# # number of attempts for each message before giving up -# max_attempts: 3 -# # amount of time to wait for RTC node to ack -# timeout: 100ms -# # amount of time to add to timeout for each retry -# backoff: 500ms +# # amount of time a message delivery is tried before giving up +# retry_timeout: 30s +# # minimum amount of time to wait for RTC node to ack, +# # retries use exponentially increasing wait on every subsequent try +# # with an upper bound of max_retry_interval +# min_retry_interval: 500ms +# # maximum amount of time to wait for RTC node to ack +# max_retry_interval: 5s # # number of messages to buffer before dropping # stream_buffer_size: 1000 From c56e37f3fee3d8c6da11ae2d20ae9e55045e8552 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 10 Apr 2023 12:31:07 +0530 Subject: [PATCH 070/324] Fix VP9 stutter in non-DD and some other misc changes (#1595) * WIP commit * WIP commit * clean up * remove todo * fix test --- pkg/sfu/buffer/buffer.go | 2 +- pkg/sfu/downtrack.go | 26 ++++++---- pkg/sfu/forwarder.go | 45 +++++++++++------ pkg/sfu/forwarder_test.go | 6 +-- pkg/sfu/receiver.go | 2 +- pkg/sfu/streamallocator/streamallocator.go | 9 +++- .../dependencydescriptor.go | 14 ++++-- pkg/sfu/videolayerselector/simulcast.go | 47 ++++++++++-------- .../temporallayerselector/vp8.go | 4 +- .../videolayerselector/videolayerselector.go | 1 + pkg/sfu/videolayerselector/vp9.go | 49 +++++++++++-------- 11 files changed, 127 insertions(+), 78 deletions(-) diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go index 26fb22aca..361162b37 100644 --- a/pkg/sfu/buffer/buffer.go +++ b/pkg/sfu/buffer/buffer.go @@ -567,7 +567,7 @@ func (b *Buffer) getExtPacket(rtpPacket *rtp.Packet, arrivalTime int64) *ExtPack } else { // vp8 with DependencyDescriptor enabled, use the TID from the descriptor vp8Packet.TID = uint8(ep.Temporal) - ep.Spatial = InvalidLayerSpatial // vp8 don't have spatial scalability, reset to -1 + ep.Spatial = InvalidLayerSpatial // vp8 don't have spatial scalability, reset to invalid } ep.Payload = vp8Packet case "video/vp9": diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index ccd68e5ca..256f5defc 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -130,8 +130,11 @@ type DownTrackStreamAllocatorListener interface { // video layer bitrate availability changed OnBitrateAvailabilityChanged(dt *DownTrack) - // max published video layer changed - OnMaxPublishedLayerChanged(dt *DownTrack) + // max published spatial layer changed + OnMaxPublishedSpatialChanged(dt *DownTrack) + + // max published temporal layer changed + OnMaxPublishedTemporalChanged(dt *DownTrack) // subscription changed - mute/unmute OnSubscriptionChanged(dt *DownTrack) @@ -593,7 +596,7 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { d.streamAllocatorBytesCounter.Add(uint32(hdr.MarshalSize() + len(payload))) - if tp.isSwitchingToMaxLayer && d.onMaxSubscribedLayerChanged != nil && d.kind == webrtc.RTPCodecTypeVideo { + if tp.isSwitchingToMaxSpatial && d.onMaxSubscribedLayerChanged != nil && d.kind == webrtc.RTPCodecTypeVideo { d.onMaxSubscribedLayerChanged(d, layer) } @@ -601,8 +604,9 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { d.isNACKThrottled.Store(false) d.rtpStats.UpdateKeyFrame(1) d.logger.Debugw("forwarding key frame", "layer", layer) + } - // SVC-TODO - no need for key frame always when using SVC + if tp.isSwitchingToRequestSpatial { locked, _ := d.forwarder.CheckSync() if locked { d.stopKeyFrameRequester() @@ -921,15 +925,19 @@ func (d *DownTrack) UpTrackBitrateAvailabilityChange() { } func (d *DownTrack) UpTrackMaxPublishedLayerChange(maxPublishedLayer int32) { - d.forwarder.SetMaxPublishedLayer(maxPublishedLayer) - - if sal := d.getStreamAllocatorListener(); sal != nil { - sal.OnMaxPublishedLayerChanged(d) + if d.forwarder.SetMaxPublishedLayer(maxPublishedLayer) { + if sal := d.getStreamAllocatorListener(); sal != nil { + sal.OnMaxPublishedSpatialChanged(d) + } } } func (d *DownTrack) UpTrackMaxTemporalLayerSeenChange(maxTemporalLayerSeen int32) { - d.forwarder.SetMaxTemporalLayerSeen(maxTemporalLayerSeen) + if d.forwarder.SetMaxTemporalLayerSeen(maxTemporalLayerSeen) { + if sal := d.getStreamAllocatorListener(); sal != nil { + sal.OnMaxPublishedTemporalChanged(d) + } + } } func (d *DownTrack) maybeAddTransition(_bitrate int64, distance float64) { diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 8d75db2f9..a0d419ecc 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -123,13 +123,14 @@ func (v VideoTransition) String() string { // ------------------------------------------------------------------- type TranslationParams struct { - shouldDrop bool - isResuming bool - isSwitchingToMaxLayer bool - rtp *TranslationParamsRTP - codecBytes []byte - ddBytes []byte - marker bool + shouldDrop bool + isResuming bool + isSwitchingToRequestSpatial bool + isSwitchingToMaxSpatial bool + rtp *TranslationParamsRTP + codecBytes []byte + ddBytes []byte + marker bool } // ------------------------------------------------------------------- @@ -202,30 +203,32 @@ func NewForwarder( return f } -func (f *Forwarder) SetMaxPublishedLayer(maxPublishedLayer int32) { +func (f *Forwarder) SetMaxPublishedLayer(maxPublishedLayer int32) bool { f.lock.Lock() defer f.lock.Unlock() existingMaxSeen := f.vls.GetMaxSeen() if maxPublishedLayer <= existingMaxSeen.Spatial { - return + return false } f.vls.SetMaxSeenSpatial(maxPublishedLayer) f.logger.Debugw("setting max published layer", "maxPublishedLayer", maxPublishedLayer) + return true } -func (f *Forwarder) SetMaxTemporalLayerSeen(maxTemporalLayerSeen int32) { +func (f *Forwarder) SetMaxTemporalLayerSeen(maxTemporalLayerSeen int32) bool { f.lock.Lock() defer f.lock.Unlock() existingMaxSeen := f.vls.GetMaxSeen() if maxTemporalLayerSeen <= existingMaxSeen.Temporal { - return + return false } f.vls.SetMaxSeenTemporal(maxTemporalLayerSeen) f.logger.Debugw("setting max temporal layer seen", "maxTemporalLayerSeen", maxTemporalLayerSeen) + return true } func (f *Forwarder) OnParkedLayerExpired(fn func()) { @@ -536,15 +539,24 @@ func (f *Forwarder) AllocateOptimal(availableLayers []int32, brs Bitrates, allow } alloc.BandwidthNeeded = optimalBandwidthNeeded + getMaxTemporal := func() int32 { + maxTemporal := maxLayer.Temporal + if maxSeenLayer.Temporal != buffer.InvalidLayerTemporal && maxSeenLayer.Temporal < maxTemporal { + maxTemporal = maxSeenLayer.Temporal + } + return maxTemporal + } + opportunisticAlloc := func() { // opportunistically latch on to anything maxSpatial := maxLayer.Spatial if allowOvershoot && f.vls.IsOvershootOkay() && maxSeenLayer.Spatial > maxSpatial { maxSpatial = maxSeenLayer.Spatial } + alloc.TargetLayer = buffer.VideoLayer{ Spatial: int32(math.Min(float64(maxSeenLayer.Spatial), float64(maxSpatial))), - Temporal: maxLayer.Temporal, + Temporal: getMaxTemporal(), } } @@ -599,7 +611,7 @@ func (f *Forwarder) AllocateOptimal(availableLayers []int32, brs Bitrates, allow alloc.TargetLayer.Spatial = l } } - alloc.TargetLayer.Temporal = maxLayer.Temporal + alloc.TargetLayer.Temporal = getMaxTemporal() alloc.RequestLayerSpatial = alloc.TargetLayer.Spatial } else { @@ -608,7 +620,7 @@ func (f *Forwarder) AllocateOptimal(availableLayers []int32, brs Bitrates, allow // current is locked to desired, stay there alloc.TargetLayer = buffer.VideoLayer{ Spatial: requestSpatial, - Temporal: maxLayer.Temporal, + Temporal: getMaxTemporal(), } alloc.RequestLayerSpatial = requestSpatial } else { @@ -1487,10 +1499,11 @@ func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer in } return tp, nil } - tp.isSwitchingToMaxLayer = result.IsSwitchingToMaxSpatial tp.isResuming = result.IsResuming - tp.marker = result.RTPMarker + tp.isSwitchingToRequestSpatial = result.IsSwitchingToRequestSpatial + tp.isSwitchingToMaxSpatial = result.IsSwitchingToMaxSpatial tp.ddBytes = result.DependencyDescriptorExtension + tp.marker = result.RTPMarker if FlagPauseOnDowngrade && f.isDeficientLocked() && f.vls.GetTarget().Spatial < f.vls.GetCurrent().Spatial { // diff --git a/pkg/sfu/forwarder_test.go b/pkg/sfu/forwarder_test.go index e2f3dff44..46a86edf2 100644 --- a/pkg/sfu/forwarder_test.go +++ b/pkg/sfu/forwarder_test.go @@ -1409,8 +1409,8 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { marshalledVP8, err := expectedVP8.Marshal() require.NoError(t, err) expectedTP = TranslationParams{ - isSwitchingToMaxLayer: true, - isResuming: true, + isSwitchingToMaxSpatial: true, + isResuming: true, rtp: &TranslationParamsRTP{ snOrdering: SequenceNumberOrderingContiguous, sequenceNumber: 23333, @@ -1721,7 +1721,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { marshalledVP8, err = expectedVP8.Marshal() require.NoError(t, err) expectedTP = TranslationParams{ - isSwitchingToMaxLayer: true, + isSwitchingToMaxSpatial: true, rtp: &TranslationParamsRTP{ snOrdering: SequenceNumberOrderingContiguous, sequenceNumber: 23339, diff --git a/pkg/sfu/receiver.go b/pkg/sfu/receiver.go index 0868a574e..2e38540d3 100644 --- a/pkg/sfu/receiver.go +++ b/pkg/sfu/receiver.go @@ -626,10 +626,10 @@ func (w *WebRTCReceiver) forwardRTP(layer int32) { return } - // svc packet, dispatch to correct tracker spatialTracker := tracker spatialLayer := layer if pkt.Spatial >= 0 { + // svc packet, dispatch to correct tracker spatialLayer = pkt.Spatial spatialTracker = w.streamTrackerManager.GetTracker(pkt.Spatial) if spatialTracker == nil { diff --git a/pkg/sfu/streamallocator/streamallocator.go b/pkg/sfu/streamallocator/streamallocator.go index c1b64fe62..bc7b51276 100644 --- a/pkg/sfu/streamallocator/streamallocator.go +++ b/pkg/sfu/streamallocator/streamallocator.go @@ -386,8 +386,13 @@ func (s *StreamAllocator) OnBitrateAvailabilityChanged(downTrack *sfu.DownTrack) s.maybePostEventAllocateTrack(downTrack) } -// called when feeding track's max publisher layer changes -func (s *StreamAllocator) OnMaxPublishedLayerChanged(downTrack *sfu.DownTrack) { +// called when feeding track's max published spatial layer changes +func (s *StreamAllocator) OnMaxPublishedSpatialChanged(downTrack *sfu.DownTrack) { + s.maybePostEventAllocateTrack(downTrack) +} + +// called when feeding track's max published temporal layer changes +func (s *StreamAllocator) OnMaxPublishedTemporalChanged(downTrack *sfu.DownTrack) { s.maybePostEventAllocateTrack(downTrack) } diff --git a/pkg/sfu/videolayerselector/dependencydescriptor.go b/pkg/sfu/videolayerselector/dependencydescriptor.go index 8b39049ce..09d46515e 100644 --- a/pkg/sfu/videolayerselector/dependencydescriptor.go +++ b/pkg/sfu/videolayerselector/dependencydescriptor.go @@ -124,11 +124,13 @@ func (d *DependencyDescriptor) Select(extPkt *buffer.ExtPacket, _layer int32) (r if dti == dd.DecodeTargetSwitch { // dependency descriptor decode target switch is enabled at all potential switch points. // So, setting current layer on every switch point will change current layer a lot. - // currentLayer is not needed for layer selection in this selector. + // + // However `currentLayer` is not needed for layer selection in this selector. // But, it is needed to signal things in the selector checks outside of this selector. + // // The following cases are handled - // 1. To detect resumption - so by setting it to incoming layer only when current lqyer is invalid, that is taken care of - // 2. To detect target achieved - set current to target if it is not the same + // 1. To detect resumption + // 2. To detect target achieved so that key frame requests can be stopped // 3. To detect reaching max spatial layer - checked when current hits target if !d.currentLayer.IsValid() { result.IsResuming = true @@ -153,9 +155,13 @@ func (d *DependencyDescriptor) Select(extPkt *buffer.ExtPacket, _layer int32) (r if d.currentLayer != d.targetLayer { if d.currentLayer.Spatial != d.targetLayer.Spatial && int32(extPkt.DependencyDescriptor.FrameDependencies.SpatialId) == d.targetLayer.Spatial { d.currentLayer.Spatial = d.targetLayer.Spatial + + if d.currentLayer.Spatial == d.requestSpatial { + result.IsSwitchingToRequestSpatial = true + } + if d.currentLayer.Spatial == d.maxLayer.Spatial { result.IsSwitchingToMaxSpatial = true - d.logger.Infow( "reached max layer", "current", d.currentLayer, diff --git a/pkg/sfu/videolayerselector/simulcast.go b/pkg/sfu/videolayerselector/simulcast.go index ec875e11e..fc15c4db0 100644 --- a/pkg/sfu/videolayerselector/simulcast.go +++ b/pkg/sfu/videolayerselector/simulcast.go @@ -89,10 +89,15 @@ func (s *Simulcast) Select(extPkt *buffer.ExtPacket, layer int32) (result VideoL if !isActive { result.IsResuming = true } + s.SetParked(buffer.InvalidLayer) + + if s.currentLayer.Spatial == s.requestSpatial { + result.IsSwitchingToRequestSpatial = true + } + if s.currentLayer.Spatial >= s.maxLayer.Spatial { result.IsSwitchingToMaxSpatial = true - s.logger.Infow( "reached max layer", "current", s.currentLayer, @@ -112,27 +117,29 @@ func (s *Simulcast) Select(extPkt *buffer.ExtPacket, layer int32) (result VideoL } // if locked to higher than max layer due to overshoot, check if it can be dialed back - if s.currentLayer.Spatial > s.maxLayer.Spatial { - if layer <= s.maxLayer.Spatial && extPkt.KeyFrame { - s.logger.Infow( - "adjusting overshoot", - "current", s.currentLayer, - "target", s.targetLayer, - "max", s.maxLayer, - "layer", layer, - "req", s.requestSpatial, - "maxSeen", s.maxSeenLayer, - "feed", extPkt.Packet.SSRC, - ) - s.currentLayer.Spatial = layer + if s.currentLayer.Spatial > s.maxLayer.Spatial && layer <= s.maxLayer.Spatial && extPkt.KeyFrame { + s.logger.Infow( + "adjusting overshoot", + "current", s.currentLayer, + "target", s.targetLayer, + "max", s.maxLayer, + "layer", layer, + "req", s.requestSpatial, + "maxSeen", s.maxSeenLayer, + "feed", extPkt.Packet.SSRC, + ) + s.currentLayer.Spatial = layer - if s.currentLayer.Spatial >= s.maxLayer.Spatial { - result.IsSwitchingToMaxSpatial = true - } + if s.currentLayer.Spatial == s.requestSpatial { + result.IsSwitchingToRequestSpatial = true + } - if s.currentLayer.Spatial >= s.maxLayer.Spatial || s.currentLayer.Spatial == s.maxSeenLayer.Spatial { - s.targetLayer.Spatial = layer - } + if s.currentLayer.Spatial >= s.maxLayer.Spatial { + result.IsSwitchingToMaxSpatial = true + } + + if s.currentLayer.Spatial >= s.maxLayer.Spatial || s.currentLayer.Spatial == s.maxSeenLayer.Spatial { + s.targetLayer.Spatial = layer } } diff --git a/pkg/sfu/videolayerselector/temporallayerselector/vp8.go b/pkg/sfu/videolayerselector/temporallayerselector/vp8.go index fbf697f33..a83765526 100644 --- a/pkg/sfu/videolayerselector/temporallayerselector/vp8.go +++ b/pkg/sfu/videolayerselector/temporallayerselector/vp8.go @@ -34,8 +34,8 @@ func (v *VP8) Select(extPkt *buffer.ExtPacket, current int32, target int32) (thi next = tid } } else { - if tid < current && tid >= target && extPkt.Packet.Marker { - next = tid + if extPkt.Packet.Marker { + next = target } } return diff --git a/pkg/sfu/videolayerselector/videolayerselector.go b/pkg/sfu/videolayerselector/videolayerselector.go index 548f736dc..14229e182 100644 --- a/pkg/sfu/videolayerselector/videolayerselector.go +++ b/pkg/sfu/videolayerselector/videolayerselector.go @@ -9,6 +9,7 @@ type VideoLayerSelectorResult struct { IsSelected bool IsRelevant bool IsResuming bool + IsSwitchingToRequestSpatial bool IsSwitchingToMaxSpatial bool RTPMarker bool DependencyDescriptorExtension []byte diff --git a/pkg/sfu/videolayerselector/vp9.go b/pkg/sfu/videolayerselector/vp9.go index 0e7783d98..9f7533352 100644 --- a/pkg/sfu/videolayerselector/vp9.go +++ b/pkg/sfu/videolayerselector/vp9.go @@ -42,29 +42,34 @@ func (v *VP9) Select(extPkt *buffer.ExtPacket, _layer int32) (result VideoLayerS } updatedLayer = extPkt.VideoLayer - currentLayer = extPkt.VideoLayer } else { - // temporal scale up/down - if v.currentLayer.Temporal < v.targetLayer.Temporal { - if extPkt.VideoLayer.Temporal > v.currentLayer.Temporal && extPkt.VideoLayer.Temporal <= v.targetLayer.Temporal && vp9.U && vp9.B { - updatedLayer.Temporal = extPkt.VideoLayer.Temporal - currentLayer.Temporal = extPkt.VideoLayer.Temporal - } - } else { - if extPkt.VideoLayer.Temporal < v.currentLayer.Temporal && extPkt.VideoLayer.Temporal >= v.targetLayer.Temporal && vp9.E { - updatedLayer.Temporal = extPkt.VideoLayer.Temporal + if v.currentLayer.Temporal != v.targetLayer.Temporal { + if v.currentLayer.Temporal < v.targetLayer.Temporal { + // temporal scale up + if extPkt.VideoLayer.Temporal > v.currentLayer.Temporal && extPkt.VideoLayer.Temporal <= v.targetLayer.Temporal && vp9.U && vp9.B { + currentLayer.Temporal = extPkt.VideoLayer.Temporal + updatedLayer.Temporal = extPkt.VideoLayer.Temporal + } + } else { + // temporal scale down + if vp9.E { + updatedLayer.Temporal = v.targetLayer.Temporal + } } } - // spatial scale up/down - if v.currentLayer.Spatial < v.targetLayer.Spatial { - if extPkt.VideoLayer.Spatial > v.currentLayer.Spatial && extPkt.VideoLayer.Spatial <= v.targetLayer.Spatial && !vp9.P && vp9.B { - updatedLayer.Spatial = extPkt.VideoLayer.Spatial - currentLayer.Spatial = extPkt.VideoLayer.Spatial - } - } else { - if extPkt.VideoLayer.Spatial < v.currentLayer.Spatial && extPkt.VideoLayer.Spatial >= v.targetLayer.Spatial && vp9.E { - updatedLayer.Spatial = extPkt.VideoLayer.Spatial + if v.currentLayer.Spatial != v.targetLayer.Spatial { + if v.currentLayer.Spatial < v.targetLayer.Spatial { + // spatial scale up + if extPkt.VideoLayer.Spatial > v.currentLayer.Spatial && extPkt.VideoLayer.Spatial <= v.targetLayer.Spatial && !vp9.P && vp9.B { + currentLayer.Spatial = extPkt.VideoLayer.Spatial + updatedLayer.Spatial = extPkt.VideoLayer.Spatial + } + } else { + // spatial scale down + if vp9.E { + updatedLayer.Spatial = v.targetLayer.Spatial + } } } } @@ -74,6 +79,10 @@ func (v *VP9) Select(extPkt *buffer.ExtPacket, _layer int32) (result VideoLayerS result.IsResuming = true } + if v.currentLayer.Spatial != v.requestSpatial && updatedLayer.Spatial == v.requestSpatial { + result.IsSwitchingToRequestSpatial = true + } + if v.currentLayer.Spatial != v.maxLayer.Spatial && updatedLayer.Spatial == v.maxLayer.Spatial { result.IsSwitchingToMaxSpatial = true v.logger.Infow( @@ -93,7 +102,7 @@ func (v *VP9) Select(extPkt *buffer.ExtPacket, _layer int32) (result VideoLayerS } result.RTPMarker = extPkt.Packet.Marker - if extPkt.VideoLayer.Spatial == v.currentLayer.Spatial && vp9.E { + if vp9.E && extPkt.VideoLayer.Spatial == currentLayer.Spatial && (vp9.P || v.targetLayer.Spatial <= v.currentLayer.Spatial) { result.RTPMarker = true } result.IsSelected = !extPkt.VideoLayer.GreaterThan(currentLayer) From c70a5c831ff14b5c2911d7fb7f1b4c4378bd9341 Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Mon, 10 Apr 2023 15:12:05 +0800 Subject: [PATCH 071/324] Refine transport fallback for client resuming (#1597) * reset fallback after ice restart * Configure ice for reconnect before send response --- pkg/rtc/participant.go | 4 +- pkg/rtc/participant_signal.go | 4 +- pkg/rtc/room.go | 6 +- pkg/rtc/transportmanager.go | 28 ++- pkg/rtc/types/interfaces.go | 4 +- .../typesfakes/fake_local_participant.go | 168 +++++++++--------- 6 files changed, 113 insertions(+), 101 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index aa3d4443d..7f92c2017 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -774,7 +774,7 @@ func (p *ParticipantImpl) MigrateState() types.MigrateState { } // ICERestart restarts subscriber ICE connections -func (p *ParticipantImpl) ICERestart(iceConfig *livekit.ICEConfig, reason livekit.ReconnectReason) { +func (p *ParticipantImpl) ICERestart(iceConfig *livekit.ICEConfig) { p.clearDisconnectTimer() p.clearMigrationTimer() @@ -782,7 +782,7 @@ func (p *ParticipantImpl) ICERestart(iceConfig *livekit.ICEConfig, reason liveki t.(types.LocalMediaTrack).Restart() } - p.TransportManager.ICERestart(iceConfig, reason) + p.TransportManager.ICERestart(iceConfig) } func (p *ParticipantImpl) OnICEConfigChanged(f func(participant types.LocalParticipant, iceConfig *livekit.ICEConfig)) { diff --git a/pkg/rtc/participant_signal.go b/pkg/rtc/participant_signal.go index 544399a1e..106c8370a 100644 --- a/pkg/rtc/participant_signal.go +++ b/pkg/rtc/participant_signal.go @@ -172,7 +172,9 @@ func (p *ParticipantImpl) SendRefreshToken(token string) error { }) } -func (p *ParticipantImpl) SendReconnectResponse(reconnectResponse *livekit.ReconnectResponse) error { +func (p *ParticipantImpl) HandleReconnectAndSendResponse(reconnectReason livekit.ReconnectReason, reconnectResponse *livekit.ReconnectResponse) error { + p.TransportManager.HandleClientReconnect(reconnectReason) + if !p.params.ClientInfo.CanHandleReconnectResponse() { return nil } diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index ebdd590ef..b1109fab7 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -407,7 +407,7 @@ func (r *Room) ResumeParticipant(p types.LocalParticipant, requestSource routing p.SetSignalSourceValid(true) - if err := p.SendReconnectResponse(&livekit.ReconnectResponse{ + if err := p.HandleReconnectAndSendResponse(reason, &livekit.ReconnectResponse{ IceServers: iceServers, ClientConfiguration: p.GetClientConfiguration(), }); err != nil { @@ -423,7 +423,7 @@ func (r *Room) ResumeParticipant(p types.LocalParticipant, requestSource routing p.SendRoomUpdate(r.protoRoom) r.lock.RUnlock() - p.ICERestart(nil, reason) + p.ICERestart(nil) return nil } @@ -729,7 +729,7 @@ func (r *Room) SimulateScenario(participant types.LocalParticipant, simulateScen participant.ICERestart(&livekit.ICEConfig{ PreferenceSubscriber: livekit.ICECandidateType(scenario.SwitchCandidateProtocol), PreferencePublisher: livekit.ICECandidateType(scenario.SwitchCandidateProtocol), - }, livekit.ReconnectReason_RR_SWITCH_CANDIDATE) + }) } return nil } diff --git a/pkg/rtc/transportmanager.go b/pkg/rtc/transportmanager.go index cc883a1ba..113fd99f0 100644 --- a/pkg/rtc/transportmanager.go +++ b/pkg/rtc/transportmanager.go @@ -457,11 +457,7 @@ func (t *TransportManager) NegotiateSubscriber(force bool) { t.subscriber.Negotiate(force) } -func (t *TransportManager) ICERestart(iceConfig *livekit.ICEConfig, reason livekit.ReconnectReason) { - if iceConfig != nil { - t.SetICEConfig(iceConfig) - } - +func (t *TransportManager) HandleClientReconnect(reason livekit.ReconnectReason) { var ( isShort bool duration time.Duration @@ -478,6 +474,9 @@ func (t *TransportManager) ICERestart(iceConfig *livekit.ICEConfig, reason livek } if isShort { + t.lock.Lock() + t.resetTransportConfigureLocked(false) + t.lock.Unlock() t.params.Logger.Infow("short connection by client ice restart", "duration", duration, "reason", reason) t.handleConnectionFailed(isShort) } @@ -486,6 +485,13 @@ func (t *TransportManager) ICERestart(iceConfig *livekit.ICEConfig, reason livek t.publisher.ResetShortConnOnICERestart() t.subscriber.ResetShortConnOnICERestart() } +} + +func (t *TransportManager) ICERestart(iceConfig *livekit.ICEConfig) { + if iceConfig != nil { + t.SetICEConfig(iceConfig) + } + t.subscriber.ICERestart() } @@ -499,14 +505,18 @@ func (t *TransportManager) SetICEConfig(iceConfig *livekit.ICEConfig) { t.configureICE(iceConfig, true) } +func (t *TransportManager) resetTransportConfigureLocked(reconfigured bool) { + t.failureCount = 0 + t.isTransportReconfigured = reconfigured + t.udpLossUnstableCount = 0 + t.lastFailure = time.Time{} +} + func (t *TransportManager) configureICE(iceConfig *livekit.ICEConfig, reset bool) { t.lock.Lock() isEqual := proto.Equal(t.iceConfig, iceConfig) if reset || !isEqual { - t.failureCount = 0 - t.isTransportReconfigured = !reset - t.udpLossUnstableCount = 0 - t.lastFailure = time.Time{} + t.resetTransportConfigureLocked(!reset) } if isEqual { diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index a83c6206f..0dcd9af88 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -263,7 +263,7 @@ type LocalParticipant interface { HandleAnswer(sdp webrtc.SessionDescription) Negotiate(force bool) - ICERestart(iceConfig *livekit.ICEConfig, reason livekit.ReconnectReason) + ICERestart(iceConfig *livekit.ICEConfig) AddTrackToSubscriber(trackLocal webrtc.TrackLocal, params AddTrackParams) (*webrtc.RTPSender, *webrtc.RTPTransceiver, error) AddTransceiverFromTrackToSubscriber(trackLocal webrtc.TrackLocal, params AddTrackParams) (*webrtc.RTPSender, *webrtc.RTPTransceiver, error) RemoveTrackFromSubscriber(sender *webrtc.RTPSender) error @@ -295,7 +295,7 @@ type LocalParticipant interface { SendConnectionQualityUpdate(update *livekit.ConnectionQualityUpdate) error SubscriptionPermissionUpdate(publisherID livekit.ParticipantID, trackID livekit.TrackID, allowed bool) SendRefreshToken(token string) error - SendReconnectResponse(reconnectResponse *livekit.ReconnectResponse) error + HandleReconnectAndSendResponse(reconnectReason livekit.ReconnectReason, reconnectResponse *livekit.ReconnectResponse) error IssueFullReconnect(reason ParticipantCloseReason) // callbacks diff --git a/pkg/rtc/types/typesfakes/fake_local_participant.go b/pkg/rtc/types/typesfakes/fake_local_participant.go index 7ae08e75c..b63f167c4 100644 --- a/pkg/rtc/types/typesfakes/fake_local_participant.go +++ b/pkg/rtc/types/typesfakes/fake_local_participant.go @@ -280,6 +280,18 @@ type FakeLocalParticipant struct { handleOfferArgsForCall []struct { arg1 webrtc.SessionDescription } + HandleReconnectAndSendResponseStub func(livekit.ReconnectReason, *livekit.ReconnectResponse) error + handleReconnectAndSendResponseMutex sync.RWMutex + handleReconnectAndSendResponseArgsForCall []struct { + arg1 livekit.ReconnectReason + arg2 *livekit.ReconnectResponse + } + handleReconnectAndSendResponseReturns struct { + result1 error + } + handleReconnectAndSendResponseReturnsOnCall map[int]struct { + result1 error + } HasPermissionStub func(livekit.TrackID, livekit.ParticipantIdentity) bool hasPermissionMutex sync.RWMutex hasPermissionArgsForCall []struct { @@ -302,11 +314,10 @@ type FakeLocalParticipant struct { hiddenReturnsOnCall map[int]struct { result1 bool } - ICERestartStub func(*livekit.ICEConfig, livekit.ReconnectReason) + ICERestartStub func(*livekit.ICEConfig) iCERestartMutex sync.RWMutex iCERestartArgsForCall []struct { arg1 *livekit.ICEConfig - arg2 livekit.ReconnectReason } IDStub func() livekit.ParticipantID iDMutex sync.RWMutex @@ -565,17 +576,6 @@ type FakeLocalParticipant struct { sendParticipantUpdateReturnsOnCall map[int]struct { result1 error } - SendReconnectResponseStub func(*livekit.ReconnectResponse) error - sendReconnectResponseMutex sync.RWMutex - sendReconnectResponseArgsForCall []struct { - arg1 *livekit.ReconnectResponse - } - sendReconnectResponseReturns struct { - result1 error - } - sendReconnectResponseReturnsOnCall map[int]struct { - result1 error - } SendRefreshTokenStub func(string) error sendRefreshTokenMutex sync.RWMutex sendRefreshTokenArgsForCall []struct { @@ -2195,6 +2195,68 @@ func (fake *FakeLocalParticipant) HandleOfferArgsForCall(i int) webrtc.SessionDe return argsForCall.arg1 } +func (fake *FakeLocalParticipant) HandleReconnectAndSendResponse(arg1 livekit.ReconnectReason, arg2 *livekit.ReconnectResponse) error { + fake.handleReconnectAndSendResponseMutex.Lock() + ret, specificReturn := fake.handleReconnectAndSendResponseReturnsOnCall[len(fake.handleReconnectAndSendResponseArgsForCall)] + fake.handleReconnectAndSendResponseArgsForCall = append(fake.handleReconnectAndSendResponseArgsForCall, struct { + arg1 livekit.ReconnectReason + arg2 *livekit.ReconnectResponse + }{arg1, arg2}) + stub := fake.HandleReconnectAndSendResponseStub + fakeReturns := fake.handleReconnectAndSendResponseReturns + fake.recordInvocation("HandleReconnectAndSendResponse", []interface{}{arg1, arg2}) + fake.handleReconnectAndSendResponseMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) HandleReconnectAndSendResponseCallCount() int { + fake.handleReconnectAndSendResponseMutex.RLock() + defer fake.handleReconnectAndSendResponseMutex.RUnlock() + return len(fake.handleReconnectAndSendResponseArgsForCall) +} + +func (fake *FakeLocalParticipant) HandleReconnectAndSendResponseCalls(stub func(livekit.ReconnectReason, *livekit.ReconnectResponse) error) { + fake.handleReconnectAndSendResponseMutex.Lock() + defer fake.handleReconnectAndSendResponseMutex.Unlock() + fake.HandleReconnectAndSendResponseStub = stub +} + +func (fake *FakeLocalParticipant) HandleReconnectAndSendResponseArgsForCall(i int) (livekit.ReconnectReason, *livekit.ReconnectResponse) { + fake.handleReconnectAndSendResponseMutex.RLock() + defer fake.handleReconnectAndSendResponseMutex.RUnlock() + argsForCall := fake.handleReconnectAndSendResponseArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeLocalParticipant) HandleReconnectAndSendResponseReturns(result1 error) { + fake.handleReconnectAndSendResponseMutex.Lock() + defer fake.handleReconnectAndSendResponseMutex.Unlock() + fake.HandleReconnectAndSendResponseStub = nil + fake.handleReconnectAndSendResponseReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeLocalParticipant) HandleReconnectAndSendResponseReturnsOnCall(i int, result1 error) { + fake.handleReconnectAndSendResponseMutex.Lock() + defer fake.handleReconnectAndSendResponseMutex.Unlock() + fake.HandleReconnectAndSendResponseStub = nil + if fake.handleReconnectAndSendResponseReturnsOnCall == nil { + fake.handleReconnectAndSendResponseReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.handleReconnectAndSendResponseReturnsOnCall[i] = struct { + result1 error + }{result1} +} + func (fake *FakeLocalParticipant) HasPermission(arg1 livekit.TrackID, arg2 livekit.ParticipantIdentity) bool { fake.hasPermissionMutex.Lock() ret, specificReturn := fake.hasPermissionReturnsOnCall[len(fake.hasPermissionArgsForCall)] @@ -2310,17 +2372,16 @@ func (fake *FakeLocalParticipant) HiddenReturnsOnCall(i int, result1 bool) { }{result1} } -func (fake *FakeLocalParticipant) ICERestart(arg1 *livekit.ICEConfig, arg2 livekit.ReconnectReason) { +func (fake *FakeLocalParticipant) ICERestart(arg1 *livekit.ICEConfig) { fake.iCERestartMutex.Lock() fake.iCERestartArgsForCall = append(fake.iCERestartArgsForCall, struct { arg1 *livekit.ICEConfig - arg2 livekit.ReconnectReason - }{arg1, arg2}) + }{arg1}) stub := fake.ICERestartStub - fake.recordInvocation("ICERestart", []interface{}{arg1, arg2}) + fake.recordInvocation("ICERestart", []interface{}{arg1}) fake.iCERestartMutex.Unlock() if stub != nil { - fake.ICERestartStub(arg1, arg2) + fake.ICERestartStub(arg1) } } @@ -2330,17 +2391,17 @@ func (fake *FakeLocalParticipant) ICERestartCallCount() int { return len(fake.iCERestartArgsForCall) } -func (fake *FakeLocalParticipant) ICERestartCalls(stub func(*livekit.ICEConfig, livekit.ReconnectReason)) { +func (fake *FakeLocalParticipant) ICERestartCalls(stub func(*livekit.ICEConfig)) { fake.iCERestartMutex.Lock() defer fake.iCERestartMutex.Unlock() fake.ICERestartStub = stub } -func (fake *FakeLocalParticipant) ICERestartArgsForCall(i int) (*livekit.ICEConfig, livekit.ReconnectReason) { +func (fake *FakeLocalParticipant) ICERestartArgsForCall(i int) *livekit.ICEConfig { fake.iCERestartMutex.RLock() defer fake.iCERestartMutex.RUnlock() argsForCall := fake.iCERestartArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 + return argsForCall.arg1 } func (fake *FakeLocalParticipant) ID() livekit.ParticipantID { @@ -3795,67 +3856,6 @@ func (fake *FakeLocalParticipant) SendParticipantUpdateReturnsOnCall(i int, resu }{result1} } -func (fake *FakeLocalParticipant) SendReconnectResponse(arg1 *livekit.ReconnectResponse) error { - fake.sendReconnectResponseMutex.Lock() - ret, specificReturn := fake.sendReconnectResponseReturnsOnCall[len(fake.sendReconnectResponseArgsForCall)] - fake.sendReconnectResponseArgsForCall = append(fake.sendReconnectResponseArgsForCall, struct { - arg1 *livekit.ReconnectResponse - }{arg1}) - stub := fake.SendReconnectResponseStub - fakeReturns := fake.sendReconnectResponseReturns - fake.recordInvocation("SendReconnectResponse", []interface{}{arg1}) - fake.sendReconnectResponseMutex.Unlock() - if stub != nil { - return stub(arg1) - } - if specificReturn { - return ret.result1 - } - return fakeReturns.result1 -} - -func (fake *FakeLocalParticipant) SendReconnectResponseCallCount() int { - fake.sendReconnectResponseMutex.RLock() - defer fake.sendReconnectResponseMutex.RUnlock() - return len(fake.sendReconnectResponseArgsForCall) -} - -func (fake *FakeLocalParticipant) SendReconnectResponseCalls(stub func(*livekit.ReconnectResponse) error) { - fake.sendReconnectResponseMutex.Lock() - defer fake.sendReconnectResponseMutex.Unlock() - fake.SendReconnectResponseStub = stub -} - -func (fake *FakeLocalParticipant) SendReconnectResponseArgsForCall(i int) *livekit.ReconnectResponse { - fake.sendReconnectResponseMutex.RLock() - defer fake.sendReconnectResponseMutex.RUnlock() - argsForCall := fake.sendReconnectResponseArgsForCall[i] - return argsForCall.arg1 -} - -func (fake *FakeLocalParticipant) SendReconnectResponseReturns(result1 error) { - fake.sendReconnectResponseMutex.Lock() - defer fake.sendReconnectResponseMutex.Unlock() - fake.SendReconnectResponseStub = nil - fake.sendReconnectResponseReturns = struct { - result1 error - }{result1} -} - -func (fake *FakeLocalParticipant) SendReconnectResponseReturnsOnCall(i int, result1 error) { - fake.sendReconnectResponseMutex.Lock() - defer fake.sendReconnectResponseMutex.Unlock() - fake.SendReconnectResponseStub = nil - if fake.sendReconnectResponseReturnsOnCall == nil { - fake.sendReconnectResponseReturnsOnCall = make(map[int]struct { - result1 error - }) - } - fake.sendReconnectResponseReturnsOnCall[i] = struct { - result1 error - }{result1} -} - func (fake *FakeLocalParticipant) SendRefreshToken(arg1 string) error { fake.sendRefreshTokenMutex.Lock() ret, specificReturn := fake.sendRefreshTokenReturnsOnCall[len(fake.sendRefreshTokenArgsForCall)] @@ -5274,6 +5274,8 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.handleAnswerMutex.RUnlock() fake.handleOfferMutex.RLock() defer fake.handleOfferMutex.RUnlock() + fake.handleReconnectAndSendResponseMutex.RLock() + defer fake.handleReconnectAndSendResponseMutex.RUnlock() fake.hasPermissionMutex.RLock() defer fake.hasPermissionMutex.RUnlock() fake.hiddenMutex.RLock() @@ -5344,8 +5346,6 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.sendJoinResponseMutex.RUnlock() fake.sendParticipantUpdateMutex.RLock() defer fake.sendParticipantUpdateMutex.RUnlock() - fake.sendReconnectResponseMutex.RLock() - defer fake.sendReconnectResponseMutex.RUnlock() fake.sendRefreshTokenMutex.RLock() defer fake.sendRefreshTokenMutex.RUnlock() fake.sendRoomUpdateMutex.RLock() From 6abe3b1aeedbb1760087cd378e08119d6ca35574 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Mon, 10 Apr 2023 21:26:16 -0700 Subject: [PATCH 072/324] Adding logs when clients reconnect (#1598) --- pkg/service/roommanager.go | 3 +++ pkg/service/rtcservice.go | 7 ++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/service/roommanager.go b/pkg/service/roommanager.go index 28254ea62..e44ff78dd 100644 --- a/pkg/service/roommanager.go +++ b/pkg/service/roommanager.go @@ -278,6 +278,9 @@ func (r *RoomManager) StartSession( "sdk", pi.Client.Sdk, "sdkVersion", pi.Client.Version, "protocol", pi.Client.Protocol, + "reconnect", pi.Reconnect, + "reconnectReason", pi.ReconnectReason, + "adaptiveStream", pi.AdaptiveStream, ) clientConf := r.clientConfManager.GetConfiguration(pi.Client) diff --git a/pkg/service/rtcservice.go b/pkg/service/rtcservice.go index 52d633423..b28d480ac 100644 --- a/pkg/service/rtcservice.go +++ b/pkg/service/rtcservice.go @@ -254,7 +254,12 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { signalStats.AddBytes(uint64(count), true) } } - pLogger.Infow("new client WS connected", "connID", cr.ConnectionID) + pLogger.Infow("new client WS connected", + "connID", cr.ConnectionID, + "reconnect", pi.Reconnect, + "reconnectReason", pi.ReconnectReason, + "adaptiveStream", pi.AdaptiveStream, + ) // handle responses go func() { From 29e26931e0dac3d7ddc6524461c352a884a92dee Mon Sep 17 00:00:00 2001 From: Jonas Schell Date: Wed, 12 Apr 2023 09:14:16 +0200 Subject: [PATCH 073/324] readme manager initial setup (#1581) * readme manager initial setup This PR makes the README ready to be managed by the readme manager. It adds custom comment tags to the readme, which are invisible when the README is rendered, but tell the readme manager where to insert and update content. * Update README.md * Update README.md --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 1019ad0a3..c535e62d0 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,11 @@ + + # LiveKit: Real-time video, audio and data for developers + [LiveKit](https://livekit.io) is an open source project that provides scalable, multi-user conferencing based on WebRTC. It's designed to provide everything you need to build real-time video/audio/data capabilities in your applications. + LiveKit's server is written in Go, using the awesome [Pion WebRTC](https://github.com/pion/webrtc) implementation. @@ -281,3 +285,5 @@ We welcome your contributions toward improving LiveKit! Please join us ## License LiveKit server is licensed under Apache License v2.0. + + From 69fb5e51a2a5baaf6b784361def429a963eb1672 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 12 Apr 2023 17:30:54 +0530 Subject: [PATCH 074/324] Fix stutter in forwarding path when using dependency descriptor (#1600) * Decode chains * clean up * clean up * decode targets only on publisher side * comment out supported codecs * fix test compile * fix another test compile * Adding TODO notes * chainID -> chainIdx * do not need to check for switch up point when using chains, as long as chain integrity is good, can switch * more comments * address comments --- pkg/sfu/buffer/buffer.go | 6 +- pkg/sfu/buffer/dependencydescriptorparser.go | 206 ++++++---- pkg/sfu/buffer/fps.go | 8 +- pkg/sfu/buffer/fps_test.go | 10 +- .../dependencydescriptorextension.go | 52 ++- .../dependencydescriptorreader.go | 6 +- pkg/sfu/downtrack.go | 2 +- pkg/sfu/utils/wraparound.go | 121 ++++++ pkg/sfu/utils/wraparound_test.go | 295 ++++++++++++++ .../dependencydescriptor.go | 364 +++++++++--------- .../selectordecisioncache.go | 114 ++++++ 11 files changed, 901 insertions(+), 283 deletions(-) create mode 100644 pkg/sfu/utils/wraparound.go create mode 100644 pkg/sfu/utils/wraparound_test.go create mode 100644 pkg/sfu/videolayerselector/selectordecisioncache.go diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go index 361162b37..8b9a7094d 100644 --- a/pkg/sfu/buffer/buffer.go +++ b/pkg/sfu/buffer/buffer.go @@ -42,7 +42,7 @@ type ExtPacket struct { Payload interface{} KeyFrame bool RawPacket []byte - DependencyDescriptor *dd.DependencyDescriptor + DependencyDescriptor *DependencyDescriptorWithDecodeTarget } // Buffer contains all packets @@ -551,7 +551,7 @@ func (b *Buffer) getExtPacket(rtpPacket *rtp.Packet, arrivalTime int64) *ExtPack if err == nil && ddVal != nil { ep.DependencyDescriptor = ddVal ep.VideoLayer = videoLayer - // TODO : notify active decode target change if changed. + // DD-TODO : notify active decode target change if changed. } } switch b.mime { @@ -779,7 +779,7 @@ func (b *Buffer) GetAudioLevel() (float64, bool) { return b.audioLevel.GetLevel() } -// TODO : now we rely on stream tracker for layer change, dependency still +// DD-TODO : now we rely on stream tracker for layer change, dependency still // work for that too. Do we keep it unchanged or use both methods? func (b *Buffer) OnMaxLayerChanged(fn func(int32, int32)) { b.maxLayerChangedCB = fn diff --git a/pkg/sfu/buffer/dependencydescriptorparser.go b/pkg/sfu/buffer/dependencydescriptorparser.go index 755e2cd4d..c52e86c83 100644 --- a/pkg/sfu/buffer/dependencydescriptorparser.go +++ b/pkg/sfu/buffer/dependencydescriptorparser.go @@ -2,6 +2,7 @@ package buffer import ( "fmt" + "sort" "github.com/pion/rtp" @@ -12,91 +13,142 @@ import ( type DependencyDescriptorParser struct { structure *dd.FrameDependencyStructure - ddExt uint8 + ddExtID uint8 logger logger.Logger onMaxLayerChanged func(int32, int32) - decodeTargetLayer []VideoLayer + decodeTargets []DependencyDescriptorDecodeTarget } -func NewDependencyDescriptorParser(ddExt uint8, logger logger.Logger, onMaxLayerChanged func(int32, int32)) *DependencyDescriptorParser { - logger.Infow("creating dependency descriptor parser", "ddExt", ddExt) +func NewDependencyDescriptorParser(ddExtID uint8, logger logger.Logger, onMaxLayerChanged func(int32, int32)) *DependencyDescriptorParser { + logger.Infow("creating dependency descriptor parser", "ddExtID", ddExtID) return &DependencyDescriptorParser{ - ddExt: ddExt, + ddExtID: ddExtID, logger: logger, onMaxLayerChanged: onMaxLayerChanged, } } -func (r *DependencyDescriptorParser) Parse(pkt *rtp.Packet) (*dd.DependencyDescriptor, VideoLayer, error) { - var videoLayer VideoLayer - if ddBuf := pkt.GetExtension(r.ddExt); ddBuf != nil { - var ddVal dd.DependencyDescriptor - ext := &dd.DependencyDescriptorExtension{ - Descriptor: &ddVal, - Structure: r.structure, - } - _, err := ext.Unmarshal(ddBuf) - if err != nil { - // r.logger.Debugw("failed to parse generic dependency descriptor", "err", err, "payload", pkt.PayloadType, "ddbufLen", len(ddBuf)) - return nil, videoLayer, err - } - - if ddVal.FrameDependencies != nil { - videoLayer.Spatial, videoLayer.Temporal = int32(ddVal.FrameDependencies.SpatialId), int32(ddVal.FrameDependencies.TemporalId) - } - if ddVal.AttachedStructure != nil && !ddVal.FirstPacketInFrame { - // r.logger.Debugw("ignoring non-first packet in frame with attached structure") - return nil, videoLayer, nil - } - - if ddVal.AttachedStructure != nil { - var maxSpatial, maxTemporal int32 - r.structure = ddVal.AttachedStructure - r.decodeTargetLayer = r.decodeTargetLayer[:0] - for target := 0; target < r.structure.NumDecodeTargets; target++ { - layer := VideoLayer{0, 0} - for _, t := range r.structure.Templates { - if t.DecodeTargetIndications[target] != dd.DecodeTargetNotPresent { - if layer.Spatial < int32(t.SpatialId) { - layer.Spatial = int32(t.SpatialId) - } - if layer.Temporal < int32(t.TemporalId) { - layer.Temporal = int32(t.TemporalId) - } - } - } - if layer.Spatial > maxSpatial { - maxSpatial = layer.Spatial - } - if layer.Temporal > maxTemporal { - maxTemporal = layer.Temporal - } - r.decodeTargetLayer = append(r.decodeTargetLayer, layer) - } - r.logger.Debugw("max layer changed", "maxSpatial", maxSpatial, "maxTemporal", maxTemporal) - go r.onMaxLayerChanged(maxSpatial, maxTemporal) - } - - if ddVal.AttachedStructure != nil && ddVal.FirstPacketInFrame { - r.logger.Debugw(fmt.Sprintf("parsed dependency descriptor\n%s", ddVal.String())) - } - - if mask := ddVal.ActiveDecodeTargetsBitmask; mask != nil { - var maxSpatial, maxTemporal int32 - for dt, layer := range r.decodeTargetLayer { - if *mask&(1< maxSpatial { + maxSpatial = dt.Layer.Spatial + } + if dt.Layer.Temporal > maxTemporal { + maxTemporal = dt.Layer.Temporal + } + if dt.Layer.Spatial <= layer.Spatial && dt.Layer.Temporal <= layer.Temporal { + activeBitMask |= 1 << dt.Target + } + } + if layer.Spatial == maxSpatial && layer.Temporal == maxTemporal { + // all the decode targets are selected + return nil + } + + return &activeBitMask +} + +// ------------------------------------------------------------------------------ diff --git a/pkg/sfu/buffer/fps.go b/pkg/sfu/buffer/fps.go index f8f192227..518b8e38c 100644 --- a/pkg/sfu/buffer/fps.go +++ b/pkg/sfu/buffer/fps.go @@ -7,7 +7,7 @@ import ( "github.com/pion/rtp/codecs" ) -var minFramesForCalculation = [DefaultMaxLayerTemporal + 1]int{8, 15, 40} +var minFramesForCalculation = [...]int{8, 15, 40} type frameInfo struct { seq uint16 @@ -357,7 +357,7 @@ func (f *FrameRateCalculatorDD) RecvPacket(ep *ExtPacket) bool { return false } - fn := ep.DependencyDescriptor.FrameNumber + fn := ep.DependencyDescriptor.Descriptor.FrameNumber if f.baseFrame == nil { f.baseFrame = &frameInfo{seq: ep.Packet.SequenceNumber, ts: ep.Packet.Timestamp, fn: fn} f.fnReceived[0] = f.baseFrame @@ -397,7 +397,7 @@ func (f *FrameRateCalculatorDD) RecvPacket(ep *ExtPacket) bool { fn: fn, temporal: temporal, spatial: spatial, - frameDiff: ep.DependencyDescriptor.FrameDependencies.FrameDiffs, + frameDiff: ep.DependencyDescriptor.Descriptor.FrameDependencies.FrameDiffs, } f.fnReceived[baseDiff] = fi @@ -411,7 +411,7 @@ func (f *FrameRateCalculatorDD) RecvPacket(ep *ExtPacket) bool { if chain.Len() == 0 { chain.PushBack(fn) } - for _, fdiff := range ep.DependencyDescriptor.FrameDependencies.FrameDiffs { + for _, fdiff := range ep.DependencyDescriptor.Descriptor.FrameDependencies.FrameDiffs { dependFrame := fn - uint16(fdiff) // frame too old, ignore if dependFrame-f.secondFrames[spatial][temporal].fn > 0x8000 { diff --git a/pkg/sfu/buffer/fps_test.go b/pkg/sfu/buffer/fps_test.go index 5f0ff79ce..206041acf 100644 --- a/pkg/sfu/buffer/fps_test.go +++ b/pkg/sfu/buffer/fps_test.go @@ -31,10 +31,12 @@ func (f *testFrameInfo) toVP8() *ExtPacket { func (f *testFrameInfo) toDD() *ExtPacket { return &ExtPacket{ Packet: &rtp.Packet{Header: f.header}, - DependencyDescriptor: &dependencydescriptor.DependencyDescriptor{ - FrameNumber: f.framenumber, - FrameDependencies: &dependencydescriptor.FrameDependencyTemplate{ - FrameDiffs: f.frameDiff, + DependencyDescriptor: &DependencyDescriptorWithDecodeTarget{ + Descriptor: &dependencydescriptor.DependencyDescriptor{ + FrameNumber: f.framenumber, + FrameDependencies: &dependencydescriptor.FrameDependencyTemplate{ + FrameDiffs: f.frameDiff, + }, }, }, VideoLayer: VideoLayer{Spatial: int32(f.spatial), Temporal: int32(f.temporal)}, diff --git a/pkg/sfu/dependencydescriptor/dependencydescriptorextension.go b/pkg/sfu/dependencydescriptor/dependencydescriptorextension.go index 4d6e339cc..7493f1bfc 100644 --- a/pkg/sfu/dependencydescriptor/dependencydescriptorextension.go +++ b/pkg/sfu/dependencydescriptor/dependencydescriptorextension.go @@ -9,23 +9,20 @@ import ( // DependencyDescriptorExtension is a extension payload format in // https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension +func formatBitmask(b *uint32) string { + if b == nil { + return "-" + } + return strconv.FormatInt(int64(*b), 2) +} + +// ------------------------------------------------------------------------------ + type DependencyDescriptorExtension struct { Descriptor *DependencyDescriptor Structure *FrameDependencyStructure } -func (d *DependencyDescriptor) MarshalSize() (int, error) { - return d.MarshalSizeWithActiveChains(^uint32(0)) -} - -func (d *DependencyDescriptor) MarshalSizeWithActiveChains(activeChains uint32) (int, error) { - writer, err := NewDependencyDescriptorWriter(nil, d.AttachedStructure, activeChains, d) - if err != nil { - return 0, err - } - return int(math.Ceil(float64(writer.ValueSizeBits()) / 8)), nil -} - func (d *DependencyDescriptorExtension) Marshal() ([]byte, error) { return d.MarshalWithActiveChains(^uint32(0)) } @@ -48,6 +45,8 @@ func (d *DependencyDescriptorExtension) Unmarshal(buf []byte) (int, error) { return reader.Parse() } +// ------------------------------------------------------------------------------ + const ( MaxSpatialIds = 4 MaxTemporalIds = 8 @@ -59,9 +58,11 @@ const ( ExtensionUrl = "https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension" ) +// ------------------------------------------------------------------------------ + type DependencyDescriptor struct { - FirstPacketInFrame bool // = true; - LastPacketInFrame bool // = true; + FirstPacketInFrame bool + LastPacketInFrame bool FrameNumber uint16 FrameDependencies *FrameDependencyTemplate Resolution *RenderResolution @@ -69,11 +70,16 @@ type DependencyDescriptor struct { AttachedStructure *FrameDependencyStructure } -func formatBitmask(b *uint32) string { - if b == nil { - return "-" +func (d *DependencyDescriptor) MarshalSize() (int, error) { + return d.MarshalSizeWithActiveChains(^uint32(0)) +} + +func (d *DependencyDescriptor) MarshalSizeWithActiveChains(activeChains uint32) (int, error) { + writer, err := NewDependencyDescriptorWriter(nil, d.AttachedStructure, activeChains, d) + if err != nil { + return 0, err } - return strconv.FormatInt(int64(*b), 2) + return int(math.Ceil(float64(writer.ValueSizeBits()) / 8)), nil } func (d *DependencyDescriptor) String() string { @@ -81,6 +87,8 @@ func (d *DependencyDescriptor) String() string { d.FirstPacketInFrame, d.LastPacketInFrame, d.FrameNumber, *d.FrameDependencies, *d.Resolution, formatBitmask(d.ActiveDecodeTargetsBitmask), d.AttachedStructure) } +// ------------------------------------------------------------------------------ + // Relationship of a frame to a Decode target. type DecodeTargetIndication int @@ -106,6 +114,8 @@ func (i DecodeTargetIndication) String() string { } } +// ------------------------------------------------------------------------------ + type FrameDependencyTemplate struct { SpatialId int TemporalId int @@ -132,6 +142,8 @@ func (t *FrameDependencyTemplate) Clone() *FrameDependencyTemplate { return t2 } +// ------------------------------------------------------------------------------ + type FrameDependencyStructure struct { StructureId int NumDecodeTargets int @@ -156,7 +168,11 @@ func (f *FrameDependencyStructure) String() string { return str } +// ------------------------------------------------------------------------------ + type RenderResolution struct { Width int Height int } + +// ------------------------------------------------------------------------------ diff --git a/pkg/sfu/dependencydescriptor/dependencydescriptorreader.go b/pkg/sfu/dependencydescriptor/dependencydescriptorreader.go index 99f3fd68a..68f00b863 100644 --- a/pkg/sfu/dependencydescriptor/dependencydescriptorreader.go +++ b/pkg/sfu/dependencydescriptor/dependencydescriptorreader.go @@ -1,6 +1,8 @@ package dependencydescriptor -import "errors" +import ( + "errors" +) type DependencyDescriptorReader struct { // Output. @@ -89,7 +91,6 @@ func (r *DependencyDescriptorReader) readMandatoryFields() error { } func (r *DependencyDescriptorReader) readExtendedFields() error { - templateDependencyStructurePresentFlag, err := r.buffer.ReadBool() if err != nil { return err @@ -384,7 +385,6 @@ func (r *DependencyDescriptorReader) readFrameDtis() error { } func (r *DependencyDescriptorReader) readFrameFdiffs() error { - r.descriptor.FrameDependencies.FrameDiffs = r.descriptor.FrameDependencies.FrameDiffs[:0] for { nexFdiffSize, err := r.buffer.ReadBits(2) diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 256f5defc..430bb882f 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -1448,7 +1448,7 @@ func (d *DownTrack) retransmitPackets(nacks []uint16) { } var extraExtensions []extensionData - if len(meta.ddBytes) > 0 { + if d.dependencyDescriptorID != 0 && len(meta.ddBytes) != 0 { extraExtensions = append(extraExtensions, extensionData{ id: uint8(d.dependencyDescriptorID), payload: meta.ddBytes, diff --git a/pkg/sfu/utils/wraparound.go b/pkg/sfu/utils/wraparound.go new file mode 100644 index 000000000..e6c8bbf2c --- /dev/null +++ b/pkg/sfu/utils/wraparound.go @@ -0,0 +1,121 @@ +package utils + +import ( + "unsafe" +) + +type number interface { + uint16 | uint32 +} + +type extendedNumber interface { + uint32 | uint64 +} + +type WrapAround[T number, ET extendedNumber] struct { + fullRange ET + + initialized bool + start T + highest T + cycles int +} + +func NewWrapAround[T number, ET extendedNumber]() *WrapAround[T, ET] { + var t T + return &WrapAround[T, ET]{ + fullRange: 1 << (unsafe.Sizeof(t) * 8), + } +} + +func (w *WrapAround[T, ET]) Seed(from *WrapAround[T, ET]) { + w.initialized = from.initialized + w.start = from.start + w.highest = from.highest + w.cycles = from.cycles +} + +type wrapAroundUpdateResult[ET extendedNumber] struct { + IsRestart bool + PreExtendedStart ET // valid only if IsRestart = true + PreExtendedHighest ET + ExtendedVal ET +} + +func (w *WrapAround[T, ET]) Update(val T) (result wrapAroundUpdateResult[ET]) { + if !w.initialized { + result.PreExtendedHighest = ET(val) - 1 + result.ExtendedVal = ET(val) + + w.start = val + w.highest = val + w.initialized = true + return + } + + result.PreExtendedHighest = w.GetExtendedHighest() + + gap := val - w.highest + if gap == 0 || gap > T(w.fullRange>>1) { + // duplicate OR out-of-order + result.IsRestart, result.PreExtendedStart, result.ExtendedVal = w.maybeAdjustStart(val) + return + } + + // in-order + if val < w.highest { + w.cycles++ + } + w.highest = val + + result.ExtendedVal = ET(w.cycles)*w.fullRange + ET(val) + return +} + +func (w *WrapAround[T, ET]) ResetHighest(val T) { + w.highest = val +} + +func (w *WrapAround[T, ET]) GetStart() T { + return w.start +} + +func (w *WrapAround[T, ET]) GetExtendedStart() ET { + return ET(w.start) +} + +func (w *WrapAround[T, ET]) GetHighest() T { + return w.highest +} + +func (w *WrapAround[T, ET]) GetExtendedHighest() ET { + return ET(w.cycles)*w.fullRange + ET(w.highest) +} + +func (w *WrapAround[T, ET]) maybeAdjustStart(val T) (isRestart bool, preExtendedStart ET, extendedVal ET) { + // re-adjust start if necessary. The conditions are + // 1. Not seen more than half the range yet + // 1. wrap around compared to start and not completed a half cycle, sequences like (10, 65530) in uint16 space + // 2. no wrap around, but out-of-order compared to start and not completed a half cycle , sequences like (10, 9), (65530, 65528) in uint16 space + totalNum := w.GetExtendedHighest() - w.GetExtendedStart() + 1 + if totalNum > (w.fullRange >> 1) { + extendedVal = ET(w.cycles)*w.fullRange + ET(val) + return + } + + cycles := w.cycles + if val-w.start > T(w.fullRange>>1) { + // out-of-order with existing start => a new start + isRestart = true + preExtendedStart = w.GetExtendedStart() + + if val > w.highest { + // wrap around + w.cycles = 1 + cycles = 0 + } + w.start = val + } + extendedVal = ET(cycles)*w.fullRange + ET(val) + return +} diff --git a/pkg/sfu/utils/wraparound_test.go b/pkg/sfu/utils/wraparound_test.go new file mode 100644 index 000000000..828242f87 --- /dev/null +++ b/pkg/sfu/utils/wraparound_test.go @@ -0,0 +1,295 @@ +package utils + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestWrapAroundUint16(t *testing.T) { + w := NewWrapAround[uint16, uint32]() + testCases := []struct { + name string + input uint16 + updated wrapAroundUpdateResult[uint32] + start uint16 + extendedStart uint32 + highest uint16 + extendedHighest uint32 + }{ + // initialize + { + name: "initialize", + input: 10, + updated: wrapAroundUpdateResult[uint32]{ + IsRestart: false, + PreExtendedStart: 0, + PreExtendedHighest: 9, + ExtendedVal: 10, + }, + start: 10, + extendedStart: 10, + highest: 10, + extendedHighest: 10, + }, + // an older number without wrap around should reset start point + { + name: "reset start no wrap around", + input: 8, + updated: wrapAroundUpdateResult[uint32]{ + IsRestart: true, + PreExtendedStart: 10, + PreExtendedHighest: 10, + ExtendedVal: 8, + }, + start: 8, + extendedStart: 8, + highest: 10, + extendedHighest: 10, + }, + // an older number with wrap around should reset start point + { + name: "reset start wrap around", + input: (1 << 16) - 6, + updated: wrapAroundUpdateResult[uint32]{ + IsRestart: true, + PreExtendedStart: 8, + PreExtendedHighest: 10, + ExtendedVal: (1 << 16) - 6, + }, + start: (1 << 16) - 6, + extendedStart: (1 << 16) - 6, + highest: 10, + extendedHighest: (1 << 16) + 10, + }, + // an older number with wrap around should reset start point again + { + name: "reset start again", + input: (1 << 16) - 12, + updated: wrapAroundUpdateResult[uint32]{ + IsRestart: true, + PreExtendedStart: (1 << 16) - 6, + PreExtendedHighest: (1 << 16) + 10, + ExtendedVal: (1 << 16) - 12, + }, + start: (1 << 16) - 12, + extendedStart: (1 << 16) - 12, + highest: 10, + extendedHighest: (1 << 16) + 10, + }, + // duplicate should return same as highest + { + name: "duplicate", + input: 10, + updated: wrapAroundUpdateResult[uint32]{ + IsRestart: false, + PreExtendedStart: 0, + PreExtendedHighest: (1 << 16) + 10, + ExtendedVal: (1 << 16) + 10, + }, + start: (1 << 16) - 12, + extendedStart: (1 << 16) - 12, + highest: 10, + extendedHighest: (1 << 16) + 10, + }, + // a significant jump in order should not reset start + { + name: "big in-order jump", + input: 1 << 15, + updated: wrapAroundUpdateResult[uint32]{ + IsRestart: false, + PreExtendedStart: 0, + PreExtendedHighest: (1 << 16) + 10, + ExtendedVal: (1 << 16) + (1 << 15), + }, + start: (1 << 16) - 12, + extendedStart: (1 << 16) - 12, + highest: 1 << 15, + extendedHighest: (1 << 16) + (1 << 15), + }, + // now out-of-order should not reset start as half the range has been seen + { + name: "out-of-order after half range", + input: (1 << 15) - 1, + updated: wrapAroundUpdateResult[uint32]{ + IsRestart: false, + PreExtendedStart: 0, + PreExtendedHighest: (1 << 16) + (1 << 15), + ExtendedVal: (1 << 16) + (1 << 15) - 1, + }, + start: (1 << 16) - 12, + extendedStart: (1 << 16) - 12, + highest: 1 << 15, + extendedHighest: (1 << 16) + (1 << 15), + }, + // in-order, should update highest + { + name: "in-order", + input: (1 << 15) + 3, + updated: wrapAroundUpdateResult[uint32]{ + IsRestart: false, + PreExtendedStart: 0, + PreExtendedHighest: (1 << 16) + (1 << 15), + ExtendedVal: (1 << 16) + (1 << 15) + 3, + }, + start: (1 << 16) - 12, + extendedStart: (1 << 16) - 12, + highest: (1 << 15) + 3, + extendedHighest: (1 << 16) + (1 << 15) + 3, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.updated, w.Update(tc.input)) + require.Equal(t, tc.start, w.GetStart()) + require.Equal(t, tc.extendedStart, w.GetExtendedStart()) + require.Equal(t, tc.highest, w.GetHighest()) + require.Equal(t, tc.extendedHighest, w.GetExtendedHighest()) + }) + } +} + +func TestWrapAroundUint32(t *testing.T) { + w := NewWrapAround[uint32, uint64]() + testCases := []struct { + name string + input uint32 + updated wrapAroundUpdateResult[uint64] + start uint32 + extendedStart uint64 + highest uint32 + extendedHighest uint64 + }{ + // initialize + { + name: "initialize", + input: 10, + updated: wrapAroundUpdateResult[uint64]{ + IsRestart: false, + PreExtendedStart: 0, + PreExtendedHighest: 9, + ExtendedVal: 10, + }, + start: 10, + extendedStart: 10, + highest: 10, + extendedHighest: 10, + }, + // an older number without wrap around should reset start point + { + name: "reset start no wrap around", + input: 8, + updated: wrapAroundUpdateResult[uint64]{ + IsRestart: true, + PreExtendedStart: 10, + PreExtendedHighest: 10, + ExtendedVal: 8, + }, + start: 8, + extendedStart: 8, + highest: 10, + extendedHighest: 10, + }, + // an older number with wrap around should reset start point + { + name: "reset start wrap around", + input: (1 << 32) - 6, + updated: wrapAroundUpdateResult[uint64]{ + IsRestart: true, + PreExtendedStart: 8, + PreExtendedHighest: 10, + ExtendedVal: (1 << 32) - 6, + }, + start: (1 << 32) - 6, + extendedStart: (1 << 32) - 6, + highest: 10, + extendedHighest: (1 << 32) + 10, + }, + // an older number with wrap around should reset start point again + { + name: "reset start again", + input: (1 << 32) - 12, + updated: wrapAroundUpdateResult[uint64]{ + IsRestart: true, + PreExtendedStart: (1 << 32) - 6, + PreExtendedHighest: (1 << 32) + 10, + ExtendedVal: (1 << 32) - 12, + }, + start: (1 << 32) - 12, + extendedStart: (1 << 32) - 12, + highest: 10, + extendedHighest: (1 << 32) + 10, + }, + // duplicate should return same as highest + { + name: "duplicate", + input: 10, + updated: wrapAroundUpdateResult[uint64]{ + IsRestart: false, + PreExtendedStart: 0, + PreExtendedHighest: (1 << 32) + 10, + ExtendedVal: (1 << 32) + 10, + }, + start: (1 << 32) - 12, + extendedStart: (1 << 32) - 12, + highest: 10, + extendedHighest: (1 << 32) + 10, + }, + // a significant jump in order should not reset start + { + name: "big in-order jump", + input: 1 << 31, + updated: wrapAroundUpdateResult[uint64]{ + IsRestart: false, + PreExtendedStart: 0, + PreExtendedHighest: (1 << 32) + 10, + ExtendedVal: (1 << 32) + (1 << 31), + }, + start: (1 << 32) - 12, + extendedStart: (1 << 32) - 12, + highest: 1 << 31, + extendedHighest: (1 << 32) + (1 << 31), + }, + // now out-of-order should not reset start as half the range has been seen + { + name: "out-of-order after half range", + input: (1 << 31) - 1, + updated: wrapAroundUpdateResult[uint64]{ + IsRestart: false, + PreExtendedStart: 0, + PreExtendedHighest: (1 << 32) + (1 << 31), + ExtendedVal: (1 << 32) + (1 << 31) - 1, + }, + start: (1 << 32) - 12, + extendedStart: (1 << 32) - 12, + highest: 1 << 31, + extendedHighest: (1 << 32) + (1 << 31), + }, + // in-order, should update highest + { + name: "in-order", + input: (1 << 31) + 3, + updated: wrapAroundUpdateResult[uint64]{ + IsRestart: false, + PreExtendedStart: 0, + PreExtendedHighest: (1 << 32) + (1 << 31), + ExtendedVal: (1 << 32) + (1 << 31) + 3, + }, + start: (1 << 32) - 12, + extendedStart: (1 << 32) - 12, + highest: (1 << 31) + 3, + extendedHighest: (1 << 32) + (1 << 31) + 3, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.updated, w.Update(tc.input)) + require.Equal(t, tc.start, w.GetStart()) + require.Equal(t, tc.extendedStart, w.GetExtendedStart()) + require.Equal(t, tc.highest, w.GetHighest()) + require.Equal(t, tc.extendedHighest, w.GetExtendedHighest()) + }) + } +} diff --git a/pkg/sfu/videolayerselector/dependencydescriptor.go b/pkg/sfu/videolayerselector/dependencydescriptor.go index 09d46515e..1fa1fc52a 100644 --- a/pkg/sfu/videolayerselector/dependencydescriptor.go +++ b/pkg/sfu/videolayerselector/dependencydescriptor.go @@ -2,39 +2,37 @@ package videolayerselector import ( "fmt" - "sort" "github.com/livekit/livekit-server/pkg/sfu/buffer" - dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + dede "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + "github.com/livekit/livekit-server/pkg/sfu/utils" "github.com/livekit/protocol/logger" ) -type decodeTarget struct { - Target int - Layer buffer.VideoLayer -} - type DependencyDescriptor struct { *Base - // DD-TODO : fields for frame chain detect - // frameNumberWrapper Uint16Wrapper - // expectKeyFrame bool + frameNum *utils.WrapAround[uint16, uint64] + decisions *SelectorDecisionCache - decodeTargets []decodeTarget + needsDecodeTargetBitmask bool activeDecodeTargetsBitmask *uint32 - structure *dd.FrameDependencyStructure + structure *dede.FrameDependencyStructure } func NewDependencyDescriptor(logger logger.Logger) *DependencyDescriptor { return &DependencyDescriptor{ - Base: NewBase(logger), + Base: NewBase(logger), + frameNum: utils.NewWrapAround[uint16, uint64](), + decisions: NewSelectorDecisionCache(256), } } func NewDependencyDescriptorFromNull(vls VideoLayerSelector) *DependencyDescriptor { return &DependencyDescriptor{ - Base: vls.(*Null).Base, + Base: vls.(*Null).Base, + frameNum: utils.NewWrapAround[uint16, uint64](), + decisions: NewSelectorDecisionCache(256), } } @@ -43,21 +41,80 @@ func (d *DependencyDescriptor) IsOvershootOkay() bool { } func (d *DependencyDescriptor) Select(extPkt *buffer.ExtPacket, _layer int32) (result VideoLayerSelectorResult) { - if extPkt.DependencyDescriptor == nil { - // packet don't have dependency descriptor + ddwdt := extPkt.DependencyDescriptor + if ddwdt == nil { + // packet doesn't have dependency descriptor + return + } + + dd := ddwdt.Descriptor + + // a packet is relevant as long as it has DD extension + result.IsRelevant = true + + frameNum := d.frameNum.Update(dd.FrameNumber) + extFrameNum := frameNum.ExtendedVal + + fd := dd.FrameDependencies + incomingLayer := buffer.VideoLayer{ + Spatial: int32(fd.SpatialId), + Temporal: int32(fd.TemporalId), + } + + // early return if this frame is already forwarded or dropped + sd, err := d.decisions.GetDecision(extFrameNum) + if err != nil { + // do not mark as dropped as only error is an old frame + return + } + switch sd { + case selectorDecisionForwarded: + // a packet of an alreadty forwarded frame, maintain decision + result.RTPMarker = extPkt.Packet.Header.Marker || (dd.LastPacketInFrame && d.currentLayer.Spatial == int32(fd.SpatialId)) + result.IsSelected = true + + case selectorDecisionDropped: + // a packet of an alreadty dropped frame, maintain decision return } if !d.currentLayer.IsValid() && !extPkt.KeyFrame { + d.decisions.AddDropped(extFrameNum) return } - result.IsRelevant = true + // check decodability using reference frames + isDecodable := true + for _, fdiff := range fd.FrameDiffs { + if fdiff == 0 { + continue + } - if extPkt.DependencyDescriptor.AttachedStructure != nil { + if sd, _ := d.decisions.GetDecision(extFrameNum - uint64(fdiff)); sd != selectorDecisionForwarded { + isDecodable = false + break + } + } + if !isDecodable { + // DD-TODO START + // Not decodable could happen due to packet loss or out-of-order packets, + // Need to figure out better ways to handle this. + // + // 1. Should definitely check if this frame is not part of current decode target OR discardable. + // In that case, forwarding can proceed without disruption. + // 2. Add a packet queue and try to de-jitter for some time. Safest is to packet copy to local queue on + // all down tracks. + // 3. Force a PLI and wait for a key frame. + // DD-TODO END + d.decisions.AddDropped(extFrameNum) + return + } + + // DD-TODO should not update for out-of-order RTP packets + if dd.AttachedStructure != nil { // update decode target layer and active decode targets // DD-TODO : these targets info can be shared by all the downtracks, no need calculate in every selector - d.updateDependencyStructure(extPkt.DependencyDescriptor.AttachedStructure) + d.updateDependencyStructure(dd.AttachedStructure) } // DD-TODO : we don't have a rtp queue to ensure the order of packets now, @@ -67,133 +124,144 @@ func (d *DependencyDescriptor) Select(extPkt *buffer.ExtPacket, _layer int32) (r // only check DTI of the active decode target. // it is not effeciency, at last we need check frame chain integrity. - activeDecodeTargets := extPkt.DependencyDescriptor.ActiveDecodeTargetsBitmask + activeDecodeTargets := dd.ActiveDecodeTargetsBitmask if activeDecodeTargets != nil { d.logger.Debugw("active decode targets", "activeDecodeTargets", *activeDecodeTargets) } - currentTarget := -1 - for _, dt := range d.decodeTargets { - // find target match with selected layer - if dt.Layer.Spatial <= d.targetLayer.Spatial && dt.Layer.Temporal <= d.targetLayer.Temporal { - if activeDecodeTargets == nil || ((*activeDecodeTargets)&(1< d.targetLayer.Spatial || dt.Layer.Temporal > d.targetLayer.Temporal { + continue + } + + if activeDecodeTargets != nil && ((*activeDecodeTargets)&(1< maxSpatial { - maxSpatial = dt.Layer.Spatial - } - if dt.Layer.Temporal > maxTemporal { - maxTemporal = dt.Layer.Temporal - } - if dt.Layer.Spatial <= targetLayer.Spatial && dt.Layer.Temporal <= targetLayer.Temporal { - activeBitMask |= 1 << dt.Target - } - } - if targetLayer.Spatial == maxSpatial && targetLayer.Temporal == maxTemporal { - // all the decode targets are selected - d.activeDecodeTargetsBitmask = nil - } else { - d.activeDecodeTargetsBitmask = &activeBitMask - } - d.logger.Debugw("setting target", "targetlayer", targetLayer, "activeDecodeTargetsBitmask", d.activeDecodeTargetsBitmask) + d.needsDecodeTargetBitmask = true } -func (d *DependencyDescriptor) updateDependencyStructure(structure *dd.FrameDependencyStructure) { +func (d *DependencyDescriptor) updateDependencyStructure(structure *dede.FrameDependencyStructure) { d.structure = structure - d.decodeTargets = d.decodeTargets[:0] - - for target := 0; target < structure.NumDecodeTargets; target++ { - layer := buffer.VideoLayer{Spatial: 0, Temporal: 0} - for _, t := range structure.Templates { - if t.DecodeTargetIndications[target] != dd.DecodeTargetNotPresent { - if layer.Spatial < int32(t.SpatialId) { - layer.Spatial = int32(t.SpatialId) - } - if layer.Temporal < int32(t.TemporalId) { - layer.Temporal = int32(t.TemporalId) - } - } - } - d.decodeTargets = append(d.decodeTargets, decodeTarget{target, layer}) - } - - // sort decode target layer by spatial and temporal from high to low - sort.Slice(d.decodeTargets, func(i, j int) bool { - return d.decodeTargets[i].Layer.GreaterThan(d.decodeTargets[j].Layer) - }) - d.logger.Debugw(fmt.Sprintf("update decode targets: %v", d.decodeTargets)) -} - -// DD-TODO : use generic wrapper when updated to go 1.18 -type Uint16Wrapper struct { - lastValue *uint16 - lastUnwrapped int32 -} - -func (w *Uint16Wrapper) Unwrap(value uint16) int32 { - if w.lastValue == nil { - w.lastValue = &value - w.lastUnwrapped = int32(value) - return int32(*w.lastValue) - } - - diff := value - *w.lastValue - w.lastUnwrapped += int32(diff) - if diff == 0x8000 && value < *w.lastValue { - w.lastUnwrapped -= 0x10000 - } else if diff > 0x8000 { - w.lastUnwrapped -= 0x10000 - } - - *w.lastValue = value - return w.lastUnwrapped } diff --git a/pkg/sfu/videolayerselector/selectordecisioncache.go b/pkg/sfu/videolayerselector/selectordecisioncache.go new file mode 100644 index 000000000..b83bf9559 --- /dev/null +++ b/pkg/sfu/videolayerselector/selectordecisioncache.go @@ -0,0 +1,114 @@ +package videolayerselector + +import ( + "fmt" +) + +// ---------------------------------------------------------------------- + +type selectorDecision int + +const ( + selectorDecisionMissing selectorDecision = iota + selectorDecisionDropped + selectorDecisionForwarded + selectorDecisionUnknown +) + +func (s selectorDecision) String() string { + switch s { + case selectorDecisionMissing: + return "MISSING" + case selectorDecisionDropped: + return "DROPPED" + case selectorDecisionForwarded: + return "FORWARDED" + case selectorDecisionUnknown: + return "UNKNOWN" + default: + return fmt.Sprintf("%d", int(s)) + } +} + +// ---------------------------------------------------------------------- + +type SelectorDecisionCache struct { + initialized bool + base uint64 + last uint64 + masks []uint64 + numEntries uint64 +} + +func NewSelectorDecisionCache(maxNumElements uint64) *SelectorDecisionCache { + numElements := (maxNumElements*2 + 63) / 64 + return &SelectorDecisionCache{ + masks: make([]uint64, numElements), + numEntries: numElements * 32, // 2 bits per entry + } +} + +func (s *SelectorDecisionCache) AddForwarded(entity uint64) { + s.addEntity(entity, selectorDecisionForwarded) +} + +func (s *SelectorDecisionCache) AddDropped(entity uint64) { + s.addEntity(entity, selectorDecisionDropped) +} + +func (s *SelectorDecisionCache) GetDecision(entity uint64) (selectorDecision, error) { + if !s.initialized || entity > s.last || entity < s.base { + return selectorDecisionUnknown, nil + } + + offset := s.last - entity + if offset >= s.numEntries { + // asking for something too old + return selectorDecisionUnknown, fmt.Errorf("too old, oldest: %d, asking: %d", s.last-s.numEntries+1, entity) + } + + return s.getEntity(entity), nil +} + +func (s *SelectorDecisionCache) addEntity(entity uint64, sd selectorDecision) { + if !s.initialized { + s.initialized = true + s.base = entity + s.last = entity + s.setEntity(entity, sd) + return + } + + if entity <= s.base { + // before base, too old + return + } + + if entity <= s.last { + s.setEntity(entity, sd) + return + } + + for e := s.last + 1; e != entity; e++ { + s.setEntity(e, selectorDecisionMissing) + } + s.setEntity(entity, sd) + s.last = entity +} + +func (s *SelectorDecisionCache) setEntity(entity uint64, sd selectorDecision) { + index, bitpos := s.getPos(entity) + s.masks[index] &= ^(0x3 << bitpos) // clear before bitwise OR + s.masks[index] |= (uint64(sd) & 0x3) << bitpos +} + +func (s *SelectorDecisionCache) getEntity(entity uint64) selectorDecision { + index, bitpos := s.getPos(entity) + return selectorDecision((s.masks[index] >> bitpos) & 0x3) +} + +func (s *SelectorDecisionCache) getPos(entity uint64) (int, int) { + // 2 bits per entity, a uint64 mask can hold 32 entities + offset := (entity - s.base) % s.numEntries + return int(offset >> 5), int(offset&0x1F) * 2 +} From c2f76b79fa00c92bab0b7009f6922a3080d625e0 Mon Sep 17 00:00:00 2001 From: Benjamin Pracht Date: Wed, 12 Apr 2023 13:27:57 -1000 Subject: [PATCH 075/324] Validate IngressInfo, update the info if an ingress is active (#1605) --- go.mod | 2 +- go.sum | 4 +-- pkg/service/ingress.go | 59 +++++++++++++++++++++++++++++------------- 3 files changed, 44 insertions(+), 21 deletions(-) diff --git a/go.mod b/go.mod index b1d53bcec..e37e5bb7d 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 - github.com/livekit/protocol v1.5.3-0.20230410011118-30f8b4c081aa + github.com/livekit/protocol v1.5.3-0.20230412231617-f70173e98ef5 github.com/livekit/psrpc v0.2.11-0.20230405191830-d76f71512630 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.14.0 diff --git a/go.sum b/go.sum index e763d9714..6a13f4899 100644 --- a/go.sum +++ b/go.sum @@ -235,8 +235,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 h1:QlQFyMwCDgjyySsrgmrMcVbEBA6KZcyTzvK+z346tUA= github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26/go.mod h1:eDA41kiySZoG+wy4Etsjb3w0jjLx69i/vAmSjG4bteA= -github.com/livekit/protocol v1.5.3-0.20230410011118-30f8b4c081aa h1:s7ACG7CGvt12tiBYSsywSavYh3S/JLVZI7Ob3ot0rKs= -github.com/livekit/protocol v1.5.3-0.20230410011118-30f8b4c081aa/go.mod h1:GzQYVsW/eIsI7xdDTNUGed+SD7IpCI1dLdOlIqRmd2U= +github.com/livekit/protocol v1.5.3-0.20230412231617-f70173e98ef5 h1:0BaB2jtGxPDh/8p72MOJnsAUdqRiUvMU99Pm6q++DoM= +github.com/livekit/protocol v1.5.3-0.20230412231617-f70173e98ef5/go.mod h1:GzQYVsW/eIsI7xdDTNUGed+SD7IpCI1dLdOlIqRmd2U= github.com/livekit/psrpc v0.2.11-0.20230405191830-d76f71512630 h1:Rm5KLZgQxWnTidY+H8MsAV6sk1iiFxeXqPFgSLkMing= github.com/livekit/psrpc v0.2.11-0.20230405191830-d76f71512630/go.mod h1:K0j8f1PgLShR7Lx80KbmwFkDH2BvOnycXGV0OSRURKc= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= diff --git a/pkg/service/ingress.go b/pkg/service/ingress.go index 6d66474a9..52ff81e69 100644 --- a/pkg/service/ingress.go +++ b/pkg/service/ingress.go @@ -5,6 +5,7 @@ import ( "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/telemetry" + "github.com/livekit/protocol/ingress" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/rpc" @@ -90,6 +91,10 @@ func (s *IngressService) CreateIngressWithUrlPrefix(ctx context.Context, urlPref State: &livekit.IngressState{}, } + if err := ingress.ValidateForSerialization(info); err != nil { + return nil, err + } + if err = s.store.StoreIngress(ctx, info); err != nil { logger.Errorw("could not write ingress info", err) return nil, err @@ -98,6 +103,33 @@ func (s *IngressService) CreateIngressWithUrlPrefix(ctx context.Context, urlPref return info, nil } +func updateInfoUsingRequest(req *livekit.UpdateIngressRequest, info *livekit.IngressInfo) error { + if req.Name != "" { + info.Name = req.Name + } + if req.RoomName != "" { + info.RoomName = req.RoomName + } + if req.ParticipantIdentity != "" { + info.ParticipantIdentity = req.ParticipantIdentity + } + if req.ParticipantName != "" { + info.ParticipantName = req.ParticipantName + } + if req.Audio != nil { + info.Audio = req.Audio + } + if req.Video != nil { + info.Video = req.Video + } + + if err := ingress.ValidateForSerialization(info); err != nil { + return err + } + + return nil +} + func (s *IngressService) UpdateIngress(ctx context.Context, req *livekit.UpdateIngressRequest) (*livekit.IngressInfo, error) { fields := []interface{}{ "ingress", req.IngressId, @@ -132,28 +164,19 @@ func (s *IngressService) UpdateIngress(ctx context.Context, req *livekit.UpdateI fallthrough case livekit.IngressState_ENDPOINT_INACTIVE: - if req.Name != "" { - info.Name = req.Name - } - if req.RoomName != "" { - info.RoomName = req.RoomName - } - if req.ParticipantIdentity != "" { - info.ParticipantIdentity = req.ParticipantIdentity - } - if req.ParticipantName != "" { - info.ParticipantName = req.ParticipantName - } - if req.Audio != nil { - info.Audio = req.Audio - } - if req.Video != nil { - info.Video = req.Video + err = updateInfoUsingRequest(req, info) + if err != nil { + return nil, err } case livekit.IngressState_ENDPOINT_BUFFERING, livekit.IngressState_ENDPOINT_PUBLISHING: - // Do not update store the returned state as the ingress service will do it + err := updateInfoUsingRequest(req, info) + if err != nil { + return nil, err + } + + // Do not store the returned state as the ingress service will do it if _, err = s.psrpcClient.UpdateIngress(ctx, req.IngressId, req); err != nil { logger.Warnw("could not update active ingress", err) } From 41349659981d1eee8f252cc2926fb3241fb88293 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Wed, 12 Apr 2023 20:42:44 -0700 Subject: [PATCH 076/324] Integrate webhook retries (#1607) --- go.mod | 4 +++- go.sum | 10 ++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e37e5bb7d..6a3fbf6be 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 - github.com/livekit/protocol v1.5.3-0.20230412231617-f70173e98ef5 + github.com/livekit/protocol v1.5.3 github.com/livekit/psrpc v0.2.11-0.20230405191830-d76f71512630 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.14.0 @@ -67,6 +67,8 @@ require ( github.com/google/go-cmp v0.5.9 // indirect github.com/google/subcommands v1.2.0 // indirect github.com/google/uuid v1.3.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-retryablehttp v0.7.2 // indirect github.com/josharian/native v1.1.0 // indirect github.com/lithammer/shortuuid/v3 v3.0.7 // indirect github.com/lithammer/shortuuid/v4 v4.0.0 // indirect diff --git a/go.sum b/go.sum index 6a13f4899..c1ae9002f 100644 --- a/go.sum +++ b/go.sum @@ -181,6 +181,12 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUDDYFRKq/RAd0= +github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -235,8 +241,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 h1:QlQFyMwCDgjyySsrgmrMcVbEBA6KZcyTzvK+z346tUA= github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26/go.mod h1:eDA41kiySZoG+wy4Etsjb3w0jjLx69i/vAmSjG4bteA= -github.com/livekit/protocol v1.5.3-0.20230412231617-f70173e98ef5 h1:0BaB2jtGxPDh/8p72MOJnsAUdqRiUvMU99Pm6q++DoM= -github.com/livekit/protocol v1.5.3-0.20230412231617-f70173e98ef5/go.mod h1:GzQYVsW/eIsI7xdDTNUGed+SD7IpCI1dLdOlIqRmd2U= +github.com/livekit/protocol v1.5.3 h1:xCzKeQss4Fp3tYW21q4E2mvJ2vwQqxxfnymBHmvt9Gg= +github.com/livekit/protocol v1.5.3/go.mod h1:YPmFvsD0cr7KlC7wsoLTLwCAAJun/ovCDBCvUnWvdwo= github.com/livekit/psrpc v0.2.11-0.20230405191830-d76f71512630 h1:Rm5KLZgQxWnTidY+H8MsAV6sk1iiFxeXqPFgSLkMing= github.com/livekit/psrpc v0.2.11-0.20230405191830-d76f71512630/go.mod h1:K0j8f1PgLShR7Lx80KbmwFkDH2BvOnycXGV0OSRURKc= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= From f8a94c21250c16ad3cbd9a762117a0de9bfd8de1 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Wed, 12 Apr 2023 21:01:57 -0700 Subject: [PATCH 077/324] Fixed timing-related failures with tests (#1608) --- pkg/rtc/subscriptionmanager_test.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pkg/rtc/subscriptionmanager_test.go b/pkg/rtc/subscriptionmanager_test.go index 0132abb80..6a0edfdb5 100644 --- a/pkg/rtc/subscriptionmanager_test.go +++ b/pkg/rtc/subscriptionmanager_test.go @@ -105,7 +105,9 @@ func TestSubscribe(t *testing.T) { }, subSettleTimeout, subCheckInterval, "track was not resubscribed") // was subscribed twice, unsubscribed once (due to close) - require.Equal(t, int32(2), numParticipantSubscribed.Load()) + require.Eventually(t, func() bool { + return numParticipantSubscribed.Load() == 2 + }, subSettleTimeout, subCheckInterval, "participant subscribe status was not updated twice") require.Equal(t, int32(1), numParticipantUnsubscribed.Load()) }) @@ -325,7 +327,10 @@ func TestUpdateSettingsBeforeSubscription(t *testing.T) { }, subSettleTimeout, subCheckInterval, "Track should be subscribed") st := s.getSubscribedTrack().(*typesfakes.FakeSubscribedTrack) - require.Equal(t, 1, st.UpdateSubscriberSettingsCallCount()) + require.Eventually(t, func() bool { + return st.UpdateSubscriberSettingsCallCount() == 1 + }, subSettleTimeout, subCheckInterval, "UpdateSubscriberSettings should be called once") + applied := st.UpdateSubscriberSettingsArgsForCall(0) require.Equal(t, settings.Disabled, applied.Disabled) require.Equal(t, settings.Width, applied.Width) From d2bf8f0ba1cf1d309185ceeba1ca9adb9271adcc Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 13 Apr 2023 13:59:24 +0530 Subject: [PATCH 078/324] Support simulating subscriber bandwidth. (#1609) * Support simualting subscriber bandwidth. When non-zero, a full allocation is triggered. Also, probes are stopped. When set to zero, normal probing mechanism should catch up. Adding `allowPause` override which can be a connection option. * fix log * allowPause in participant params --- pkg/rtc/participant.go | 3 + pkg/rtc/room.go | 10 ++- pkg/rtc/transport.go | 16 ++++ pkg/rtc/transportmanager.go | 8 ++ pkg/rtc/types/interfaces.go | 4 + .../typesfakes/fake_local_participant.go | 78 +++++++++++++++++++ pkg/service/roommanager.go | 1 + pkg/sfu/streamallocator/streamallocator.go | 77 ++++++++++++++++-- 8 files changed, 187 insertions(+), 10 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 7f92c2017..a6147c781 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -88,6 +88,7 @@ type ParticipantParams struct { VersionGenerator utils.TimedVersionGenerator TrackResolver types.MediaTrackResolver DisableDynacast bool + SubscriberAllowPause bool } type ParticipantImpl struct { @@ -1035,6 +1036,8 @@ func (p *ParticipantImpl) setupTransportManager() error { tm.OnAnyTransportNegotiationFailed(p.onAnyTransportNegotiationFailed) tm.OnDataMessage(p.onDataMessage) + + tm.SetSubscriberAllowPause(p.params.SubscriberAllowPause) p.TransportManager = tm return nil } diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index b1109fab7..911191027 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -692,7 +692,7 @@ func (r *Room) OnMetadataUpdate(f func(metadata string)) { func (r *Room) SimulateScenario(participant types.LocalParticipant, simulateScenario *livekit.SimulateScenario) error { switch scenario := simulateScenario.Scenario.(type) { case *livekit.SimulateScenario_SpeakerUpdate: - r.Logger.Infow("simulating speaker update", "participant", participant.Identity()) + r.Logger.Infow("simulating speaker update", "participant", participant.Identity(), "duration", scenario.SpeakerUpdate) go func() { <-time.After(time.Duration(scenario.SpeakerUpdate) * time.Second) r.sendSpeakerChanges([]*livekit.SpeakerInfo{{ @@ -723,13 +723,19 @@ func (r *Room) SimulateScenario(participant types.LocalParticipant, simulateScen if err := participant.Close(true, types.ParticipantCloseReasonSimulateServerLeave); err != nil { return err } - case *livekit.SimulateScenario_SwitchCandidateProtocol: r.Logger.Infow("simulating switch candidate protocol", "participant", participant.Identity()) participant.ICERestart(&livekit.ICEConfig{ PreferenceSubscriber: livekit.ICECandidateType(scenario.SwitchCandidateProtocol), PreferencePublisher: livekit.ICECandidateType(scenario.SwitchCandidateProtocol), }) + case *livekit.SimulateScenario_SubscriberBandwidth: + if scenario.SubscriberBandwidth > 0 { + r.Logger.Infow("simulating subscriber bandwidth start", "participant", participant.Identity(), "bandwidth", scenario.SubscriberBandwidth) + } else { + r.Logger.Infow("simulating subscriber bandwidth end", "participant", participant.Identity()) + } + participant.SetSubscriberChannelCapacity(scenario.SubscriberBandwidth) } return nil } diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index c98f6241d..00a7da629 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -1118,6 +1118,22 @@ func (t *PCTransport) RemoveTrackFromStreamAllocator(subTrack types.SubscribedTr t.streamAllocator.RemoveTrack(subTrack.DownTrack()) } +func (t *PCTransport) SetAllowPauseOfStreamAllocator(allowPause bool) { + if t.streamAllocator == nil { + return + } + + t.streamAllocator.SetAllowPause(allowPause) +} + +func (t *PCTransport) SetChannelCapacityOfStreamAllocator(channelCapacity int64) { + if t.streamAllocator == nil { + return + } + + t.streamAllocator.SetChannelCapacity(channelCapacity) +} + func (t *PCTransport) GetICEConnectionType() types.ICEConnectionType { unknown := types.ICEConnectionTypeUnknown if t.pc == nil { diff --git a/pkg/rtc/transportmanager.go b/pkg/rtc/transportmanager.go index 113fd99f0..c7dfcc57d 100644 --- a/pkg/rtc/transportmanager.go +++ b/pkg/rtc/transportmanager.go @@ -782,3 +782,11 @@ func (t *TransportManager) SetSignalSourceValid(valid bool) { t.signalSourceValid.Store(valid) t.params.Logger.Debugw("signal source valid", "valid", valid) } + +func (t *TransportManager) SetSubscriberAllowPause(allowPause bool) { + t.subscriber.SetAllowPauseOfStreamAllocator(allowPause) +} + +func (t *TransportManager) SetSubscriberChannelCapacity(channelCapacity int64) { + t.subscriber.SetChannelCapacityOfStreamAllocator(channelCapacity) +} diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index 0dcd9af88..d2dfe1f3a 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -333,6 +333,10 @@ type LocalParticipant interface { UpdateSubscribedQuality(nodeID livekit.NodeID, trackID livekit.TrackID, maxQualities []SubscribedCodecQuality) error UpdateMediaLoss(nodeID livekit.NodeID, trackID livekit.TrackID, fractionalLoss uint32) error + + // down stream bandwidth management + SetSubscriberAllowPause(allowPause bool) + SetSubscriberChannelCapacity(channelCapacity int64) } // Room is a container of participants, and can provide room-level actions diff --git a/pkg/rtc/types/typesfakes/fake_local_participant.go b/pkg/rtc/types/typesfakes/fake_local_participant.go index b63f167c4..d9e73aa64 100644 --- a/pkg/rtc/types/typesfakes/fake_local_participant.go +++ b/pkg/rtc/types/typesfakes/fake_local_participant.go @@ -659,6 +659,16 @@ type FakeLocalParticipant struct { setSignalSourceValidArgsForCall []struct { arg1 bool } + SetSubscriberAllowPauseStub func(bool) + setSubscriberAllowPauseMutex sync.RWMutex + setSubscriberAllowPauseArgsForCall []struct { + arg1 bool + } + SetSubscriberChannelCapacityStub func(int64) + setSubscriberChannelCapacityMutex sync.RWMutex + setSubscriberChannelCapacityArgsForCall []struct { + arg1 int64 + } SetTrackMutedStub func(livekit.TrackID, bool, bool) setTrackMutedMutex sync.RWMutex setTrackMutedArgsForCall []struct { @@ -4343,6 +4353,70 @@ func (fake *FakeLocalParticipant) SetSignalSourceValidArgsForCall(i int) bool { return argsForCall.arg1 } +func (fake *FakeLocalParticipant) SetSubscriberAllowPause(arg1 bool) { + fake.setSubscriberAllowPauseMutex.Lock() + fake.setSubscriberAllowPauseArgsForCall = append(fake.setSubscriberAllowPauseArgsForCall, struct { + arg1 bool + }{arg1}) + stub := fake.SetSubscriberAllowPauseStub + fake.recordInvocation("SetSubscriberAllowPause", []interface{}{arg1}) + fake.setSubscriberAllowPauseMutex.Unlock() + if stub != nil { + fake.SetSubscriberAllowPauseStub(arg1) + } +} + +func (fake *FakeLocalParticipant) SetSubscriberAllowPauseCallCount() int { + fake.setSubscriberAllowPauseMutex.RLock() + defer fake.setSubscriberAllowPauseMutex.RUnlock() + return len(fake.setSubscriberAllowPauseArgsForCall) +} + +func (fake *FakeLocalParticipant) SetSubscriberAllowPauseCalls(stub func(bool)) { + fake.setSubscriberAllowPauseMutex.Lock() + defer fake.setSubscriberAllowPauseMutex.Unlock() + fake.SetSubscriberAllowPauseStub = stub +} + +func (fake *FakeLocalParticipant) SetSubscriberAllowPauseArgsForCall(i int) bool { + fake.setSubscriberAllowPauseMutex.RLock() + defer fake.setSubscriberAllowPauseMutex.RUnlock() + argsForCall := fake.setSubscriberAllowPauseArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeLocalParticipant) SetSubscriberChannelCapacity(arg1 int64) { + fake.setSubscriberChannelCapacityMutex.Lock() + fake.setSubscriberChannelCapacityArgsForCall = append(fake.setSubscriberChannelCapacityArgsForCall, struct { + arg1 int64 + }{arg1}) + stub := fake.SetSubscriberChannelCapacityStub + fake.recordInvocation("SetSubscriberChannelCapacity", []interface{}{arg1}) + fake.setSubscriberChannelCapacityMutex.Unlock() + if stub != nil { + fake.SetSubscriberChannelCapacityStub(arg1) + } +} + +func (fake *FakeLocalParticipant) SetSubscriberChannelCapacityCallCount() int { + fake.setSubscriberChannelCapacityMutex.RLock() + defer fake.setSubscriberChannelCapacityMutex.RUnlock() + return len(fake.setSubscriberChannelCapacityArgsForCall) +} + +func (fake *FakeLocalParticipant) SetSubscriberChannelCapacityCalls(stub func(int64)) { + fake.setSubscriberChannelCapacityMutex.Lock() + defer fake.setSubscriberChannelCapacityMutex.Unlock() + fake.SetSubscriberChannelCapacityStub = stub +} + +func (fake *FakeLocalParticipant) SetSubscriberChannelCapacityArgsForCall(i int) int64 { + fake.setSubscriberChannelCapacityMutex.RLock() + defer fake.setSubscriberChannelCapacityMutex.RUnlock() + argsForCall := fake.setSubscriberChannelCapacityArgsForCall[i] + return argsForCall.arg1 +} + func (fake *FakeLocalParticipant) SetTrackMuted(arg1 livekit.TrackID, arg2 bool, arg3 bool) { fake.setTrackMutedMutex.Lock() fake.setTrackMutedArgsForCall = append(fake.setTrackMutedArgsForCall, struct { @@ -5368,6 +5442,10 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.setResponseSinkMutex.RUnlock() fake.setSignalSourceValidMutex.RLock() defer fake.setSignalSourceValidMutex.RUnlock() + fake.setSubscriberAllowPauseMutex.RLock() + defer fake.setSubscriberAllowPauseMutex.RUnlock() + fake.setSubscriberChannelCapacityMutex.RLock() + defer fake.setSubscriberChannelCapacityMutex.RUnlock() fake.setTrackMutedMutex.RLock() defer fake.setTrackMutedMutex.RUnlock() fake.startMutex.RLock() diff --git a/pkg/service/roommanager.go b/pkg/service/roommanager.go index e44ff78dd..52778c916 100644 --- a/pkg/service/roommanager.go +++ b/pkg/service/roommanager.go @@ -336,6 +336,7 @@ func (r *RoomManager) StartSession( ReconnectOnSubscriptionError: reconnectOnSubscriptionError, VersionGenerator: r.versionGenerator, TrackResolver: room.ResolveMediaTrackForSubscriber, + SubscriberAllowPause: r.config.RTC.CongestionControl.AllowPause, }) if err != nil { return err diff --git a/pkg/sfu/streamallocator/streamallocator.go b/pkg/sfu/streamallocator/streamallocator.go index bc7b51276..a65834ed7 100644 --- a/pkg/sfu/streamallocator/streamallocator.go +++ b/pkg/sfu/streamallocator/streamallocator.go @@ -48,6 +48,8 @@ const ( FlagAllowOvershootInCatchup = true ) +// --------------------------------------------------------------------------- + var ( ChannelObserverParamsProbe = ChannelObserverParams{ Name: "probe", @@ -70,6 +72,8 @@ var ( } ) +// --------------------------------------------------------------------------- + type streamAllocatorState int const ( @@ -88,6 +92,8 @@ func (s streamAllocatorState) String() string { } } +// --------------------------------------------------------------------------- + type streamAllocatorSignal int const ( @@ -99,6 +105,8 @@ const ( streamAllocatorSignalSendProbe streamAllocatorSignalProbeClusterDone streamAllocatorSignalResume + streamAllocatorSignalSetAllowPause + streamAllocatorSignalSetChannelCapacity ) func (s streamAllocatorSignal) String() string { @@ -119,11 +127,17 @@ func (s streamAllocatorSignal) String() string { return "PROBE_CLUSTER_DONE" case streamAllocatorSignalResume: return "RESUME" + case streamAllocatorSignalSetAllowPause: + return "SET_ALLOW_PAUSE" + case streamAllocatorSignalSetChannelCapacity: + return "SET_CHANNEL_CAPACITY" default: return fmt.Sprintf("%d", int(s)) } } +// --------------------------------------------------------------------------- + type Event struct { Signal streamAllocatorSignal TrackID livekit.TrackID @@ -134,6 +148,8 @@ func (e Event) String() string { return fmt.Sprintf("StreamAllocator:Event{signal: %s, trackID: %s, data: %+v}", e.Signal, e.TrackID, e.Data) } +// --------------------------------------------------------------------------- + type StreamAllocatorParams struct { Config config.CongestionControlConfig Logger logger.Logger @@ -146,8 +162,11 @@ type StreamAllocator struct { bwe cc.BandwidthEstimator - lastReceivedEstimate int64 - committedChannelCapacity int64 + allowPause bool + + lastReceivedEstimate int64 + committedChannelCapacity int64 + overriddenChannelCapacity int64 probeInterval time.Duration lastProbeStartTime time.Time @@ -176,7 +195,8 @@ type StreamAllocator struct { func NewStreamAllocator(params StreamAllocatorParams) *StreamAllocator { s := &StreamAllocator{ - params: params, + params: params, + allowPause: params.Config.AllowPause, prober: NewProber(ProberParams{ Logger: params.Logger, }), @@ -274,6 +294,20 @@ func (s *StreamAllocator) SetTrackPriority(downTrack *sfu.DownTrack, priority ui s.videoTracksMu.Unlock() } +func (s *StreamAllocator) SetAllowPause(allowPause bool) { + s.postEvent(Event{ + Signal: streamAllocatorSignalSetAllowPause, + Data: allowPause, + }) +} + +func (s *StreamAllocator) SetChannelCapacity(channelCapacity int64) { + s.postEvent(Event{ + Signal: streamAllocatorSignalSetChannelCapacity, + Data: channelCapacity, + }) +} + func (s *StreamAllocator) resetState() { s.channelObserver = s.newChannelObserverNonProbe() s.resetProbe() @@ -536,6 +570,10 @@ func (s *StreamAllocator) handleEvent(event *Event) { s.handleSignalProbeClusterDone(event) case streamAllocatorSignalResume: s.handleSignalResume(event) + case streamAllocatorSignalSetAllowPause: + s.handleSignalSetAllowPause(event) + case streamAllocatorSignalSetChannelCapacity: + s.handleSignalSetChannelCapacity(event) } } @@ -649,6 +687,20 @@ func (s *StreamAllocator) handleSignalResume(event *Event) { } } +func (s *StreamAllocator) handleSignalSetAllowPause(event *Event) { + s.allowPause = event.Data.(bool) +} + +func (s *StreamAllocator) handleSignalSetChannelCapacity(event *Event) { + s.overriddenChannelCapacity = event.Data.(int64) + if s.overriddenChannelCapacity > 0 { + s.params.Logger.Infow("allocating on override channel capacity", "override", s.overriddenChannelCapacity) + s.allocateAllTracks() + } else { + s.params.Logger.Infow("clearing override channel capacity") + } +} + func (s *StreamAllocator) setState(state streamAllocatorState) { if s.state == state { return @@ -846,7 +898,7 @@ func (s *StreamAllocator) allocateTrack(track *Track) { } // commit the track that needs change if enough could be acquired or pause not allowed - if !s.params.Config.AllowPause || bandwidthAcquired >= transition.BandwidthDelta { + if !s.allowPause || bandwidthAcquired >= transition.BandwidthDelta { allocation := track.ProvisionalAllocateCommit() if allocation.PauseReason == sfu.VideoPauseReasonBandwidth && track.SetPaused(true) { update.HandleStreamingChange(true, track) @@ -958,6 +1010,14 @@ func (s *StreamAllocator) allocateAllTracks() { availableChannelCapacity := s.committedChannelCapacity if s.params.Config.MinChannelCapacity > availableChannelCapacity { availableChannelCapacity = s.params.Config.MinChannelCapacity + s.params.Logger.Debugw( + "stream allocator: overriding channel capacity with min channel capacity", + "actual", s.committedChannelCapacity, + "override", availableChannelCapacity, + ) + } + if s.overriddenChannelCapacity > 0 { + availableChannelCapacity = s.overriddenChannelCapacity s.params.Logger.Debugw( "stream allocator: overriding channel capacity", "actual", s.committedChannelCapacity, @@ -987,7 +1047,7 @@ func (s *StreamAllocator) allocateAllTracks() { if availableChannelCapacity < 0 { availableChannelCapacity = 0 } - if availableChannelCapacity == 0 && s.params.Config.AllowPause { + if availableChannelCapacity == 0 && s.allowPause { // nothing left for managed tracks, pause them all for _, track := range videoTracks { if !track.IsManaged() { @@ -1013,7 +1073,7 @@ func (s *StreamAllocator) allocateAllTracks() { } for _, track := range sorted { - usedChannelCapacity := track.ProvisionalAllocate(availableChannelCapacity, layer, s.params.Config.AllowPause, FlagAllowOvershootWhileDeficient) + usedChannelCapacity := track.ProvisionalAllocate(availableChannelCapacity, layer, s.allowPause, FlagAllowOvershootWhileDeficient) availableChannelCapacity -= usedChannelCapacity if availableChannelCapacity < 0 { availableChannelCapacity = 0 @@ -1155,7 +1215,8 @@ func (s *StreamAllocator) isInProbe() bool { } func (s *StreamAllocator) maybeProbe() { - if time.Since(s.lastProbeStartTime) < s.probeInterval || s.probeClusterId != ProbeClusterIdInvalid { + if time.Since(s.lastProbeStartTime) < s.probeInterval || s.probeClusterId != ProbeClusterIdInvalid || s.overriddenChannelCapacity > 0 { + // do not probe if channel capacity is overridden return } @@ -1207,7 +1268,7 @@ func (s *StreamAllocator) maybeProbeWithPadding() { func (s *StreamAllocator) getTracks() []*Track { s.videoTracksMu.RLock() - var tracks []*Track + tracks := make([]*Track, 0, len(s.videoTracks)) for _, track := range s.videoTracks { tracks = append(tracks, track) } From ac266fbcd6eac726c29772157c563c4cf88e176b Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 13 Apr 2023 17:00:32 +0530 Subject: [PATCH 079/324] Support subscriber_allow_pause connect option (#1612) * Support subscriber_allow_pause connect option * optional subscriber_allow_pause field --- go.mod | 2 +- go.sum | 4 ++-- pkg/routing/interfaces.go | 41 +++++++++++++++++++++++++------------- pkg/service/roommanager.go | 6 +++++- pkg/service/rtcservice.go | 5 +++++ 5 files changed, 40 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index 6a3fbf6be..657207fa6 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 - github.com/livekit/protocol v1.5.3 + github.com/livekit/protocol v1.5.4-0.20230413111958-5fea69067bbc github.com/livekit/psrpc v0.2.11-0.20230405191830-d76f71512630 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.14.0 diff --git a/go.sum b/go.sum index c1ae9002f..69bad2832 100644 --- a/go.sum +++ b/go.sum @@ -241,8 +241,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 h1:QlQFyMwCDgjyySsrgmrMcVbEBA6KZcyTzvK+z346tUA= github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26/go.mod h1:eDA41kiySZoG+wy4Etsjb3w0jjLx69i/vAmSjG4bteA= -github.com/livekit/protocol v1.5.3 h1:xCzKeQss4Fp3tYW21q4E2mvJ2vwQqxxfnymBHmvt9Gg= -github.com/livekit/protocol v1.5.3/go.mod h1:YPmFvsD0cr7KlC7wsoLTLwCAAJun/ovCDBCvUnWvdwo= +github.com/livekit/protocol v1.5.4-0.20230413111958-5fea69067bbc h1:15IrYsN4PRgrH2MldkYgnTqqNxgRgjVGLjEtwurphCQ= +github.com/livekit/protocol v1.5.4-0.20230413111958-5fea69067bbc/go.mod h1:YPmFvsD0cr7KlC7wsoLTLwCAAJun/ovCDBCvUnWvdwo= github.com/livekit/psrpc v0.2.11-0.20230405191830-d76f71512630 h1:Rm5KLZgQxWnTidY+H8MsAV6sk1iiFxeXqPFgSLkMing= github.com/livekit/psrpc v0.2.11-0.20230405191830-d76f71512630/go.mod h1:K0j8f1PgLShR7Lx80KbmwFkDH2BvOnycXGV0OSRURKc= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= diff --git a/pkg/routing/interfaces.go b/pkg/routing/interfaces.go index 20c090809..566d2a2b4 100644 --- a/pkg/routing/interfaces.go +++ b/pkg/routing/interfaces.go @@ -34,16 +34,17 @@ type MessageSource interface { } type ParticipantInit struct { - Identity livekit.ParticipantIdentity - Name livekit.ParticipantName - Reconnect bool - ReconnectReason livekit.ReconnectReason - AutoSubscribe bool - Client *livekit.ClientInfo - Grants *auth.ClaimGrants - Region string - AdaptiveStream bool - ID livekit.ParticipantID + Identity livekit.ParticipantIdentity + Name livekit.ParticipantName + Reconnect bool + ReconnectReason livekit.ReconnectReason + AutoSubscribe bool + Client *livekit.ClientInfo + Grants *auth.ClaimGrants + Region string + AdaptiveStream bool + ID livekit.ParticipantID + SubscriberAllowPause *bool } type NewParticipantCallback func( @@ -117,7 +118,7 @@ func (pi *ParticipantInit) ToStartSession(roomName livekit.RoomName, connectionI return nil, err } - return &livekit.StartSession{ + ss := &livekit.StartSession{ RoomName: string(roomName), Identity: string(pi.Identity), Name: string(pi.Name), @@ -130,7 +131,13 @@ func (pi *ParticipantInit) ToStartSession(roomName livekit.RoomName, connectionI GrantsJson: string(claims), AdaptiveStream: pi.AdaptiveStream, ParticipantId: string(pi.ID), - }, nil + } + if pi.SubscriberAllowPause != nil { + subscriberAllowPause := *pi.SubscriberAllowPause + ss.SubscriberAllowPause = &subscriberAllowPause + } + + return ss, nil } func ParticipantInitFromStartSession(ss *livekit.StartSession, region string) (*ParticipantInit, error) { @@ -139,7 +146,7 @@ func ParticipantInitFromStartSession(ss *livekit.StartSession, region string) (* return nil, err } - return &ParticipantInit{ + pi := &ParticipantInit{ Identity: livekit.ParticipantIdentity(ss.Identity), Name: livekit.ParticipantName(ss.Name), Reconnect: ss.Reconnect, @@ -150,5 +157,11 @@ func ParticipantInitFromStartSession(ss *livekit.StartSession, region string) (* Region: region, AdaptiveStream: ss.AdaptiveStream, ID: livekit.ParticipantID(ss.ParticipantId), - }, nil + } + if ss.SubscriberAllowPause != nil { + subscriberAllowPause := *ss.SubscriberAllowPause + pi.SubscriberAllowPause = &subscriberAllowPause + } + + return pi, nil } diff --git a/pkg/service/roommanager.go b/pkg/service/roommanager.go index 52778c916..9f4df01fb 100644 --- a/pkg/service/roommanager.go +++ b/pkg/service/roommanager.go @@ -305,6 +305,10 @@ func (r *RoomManager) StartSession( if r.config.RTC.ReconnectOnSubscriptionError != nil { reconnectOnSubscriptionError = *r.config.RTC.ReconnectOnSubscriptionError } + subscriberAllowPause := r.config.RTC.CongestionControl.AllowPause + if pi.SubscriberAllowPause != nil { + subscriberAllowPause = *pi.SubscriberAllowPause + } participant, err = rtc.NewParticipant(rtc.ParticipantParams{ Identity: pi.Identity, Name: pi.Name, @@ -336,7 +340,7 @@ func (r *RoomManager) StartSession( ReconnectOnSubscriptionError: reconnectOnSubscriptionError, VersionGenerator: r.versionGenerator, TrackResolver: room.ResolveMediaTrackForSubscriber, - SubscriberAllowPause: r.config.RTC.CongestionControl.AllowPause, + SubscriberAllowPause: subscriberAllowPause, }) if err != nil { return err diff --git a/pkg/service/rtcservice.go b/pkg/service/rtcservice.go index b28d480ac..d6f622239 100644 --- a/pkg/service/rtcservice.go +++ b/pkg/service/rtcservice.go @@ -103,6 +103,7 @@ func (s *RTCService) validate(r *http.Request) (livekit.RoomName, routing.Partic publishParam := r.FormValue("publish") adaptiveStreamParam := r.FormValue("adaptive_stream") participantID := r.FormValue("sid") + subscriberAllowPauseParam := r.FormValue("subscriber_allow_pause") if onlyName != "" { roomName = onlyName @@ -159,6 +160,10 @@ func (s *RTCService) validate(r *http.Request) (livekit.RoomName, routing.Partic if adaptiveStreamParam != "" { pi.AdaptiveStream = boolValue(adaptiveStreamParam) } + if subscriberAllowPauseParam != "" { + subscriberAllowPause := boolValue(subscriberAllowPauseParam) + pi.SubscriberAllowPause = &subscriberAllowPause + } return roomName, pi, http.StatusOK, nil } From 843328125e8d268e2dda16d4edad647781abef82 Mon Sep 17 00:00:00 2001 From: Benjamin Pracht Date: Thu, 13 Apr 2023 08:09:00 -1000 Subject: [PATCH 080/324] Do not use redis transactions for the egress APIs. Make sure all ingress related keys are on the same cluster slot. #1606 This will break existing ingress in redis. --- pkg/service/redisstore.go | 41 +++++++++++++++++---------------------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/pkg/service/redisstore.go b/pkg/service/redisstore.go index bd3f835d2..e7a615ba4 100644 --- a/pkg/service/redisstore.go +++ b/pkg/service/redisstore.go @@ -33,9 +33,9 @@ const ( // IngressKey is a hash of ingressID => ingress info IngressKey = "ingress" - StreamKeyKey = "stream_key" - IngressStatePrefix = "ingress_state:" - RoomIngressPrefix = "room_ingress:" + StreamKeyKey = "{ingress}_stream_key" + IngressStatePrefix = "{ingress}_state:" + RoomIngressPrefix = "room_{ingress}:" // RoomParticipantsPrefix is hash of participant_name => ParticipantInfo RoomParticipantsPrefix = "room_participants:" @@ -320,10 +320,10 @@ func (s *RedisStore) StoreEgress(_ context.Context, info *livekit.EgressInfo) er return err } - tx := s.rc.TxPipeline() - tx.HSet(s.ctx, EgressKey, info.EgressId, data) - tx.SAdd(s.ctx, RoomEgressPrefix+info.RoomName, info.EgressId) - if _, err = tx.Exec(s.ctx); err != nil { + pp := s.rc.Pipeline() + pp.HSet(s.ctx, EgressKey, info.EgressId, data) + pp.SAdd(s.ctx, RoomEgressPrefix+info.RoomName, info.EgressId) + if _, err = pp.Exec(s.ctx); err != nil { return errors.Wrap(err, "could not store egress info") } @@ -410,10 +410,10 @@ func (s *RedisStore) UpdateEgress(_ context.Context, info *livekit.EgressInfo) e } if info.EndedAt != 0 { - tx := s.rc.TxPipeline() - tx.HSet(s.ctx, EgressKey, info.EgressId, data) - tx.HSet(s.ctx, EndedEgressKey, info.EgressId, egressEndedValue(info.RoomName, info.EndedAt)) - _, err = tx.Exec(s.ctx) + pp := s.rc.Pipeline() + pp.HSet(s.ctx, EgressKey, info.EgressId, data) + pp.HSet(s.ctx, EndedEgressKey, info.EgressId, egressEndedValue(info.RoomName, info.EndedAt)) + _, err = pp.Exec(s.ctx) } else { err = s.rc.HSet(s.ctx, EgressKey, info.EgressId, data).Err() } @@ -457,11 +457,12 @@ func (s *RedisStore) CleanEndedEgress() error { } if endedAt < expiry { - tx := s.rc.TxPipeline() - tx.HDel(s.ctx, EndedEgressKey, egressID) - tx.SRem(s.ctx, RoomEgressPrefix+roomName, egressID) - tx.HDel(s.ctx, EgressKey, egressID) - if _, err := tx.Exec(s.ctx); err != nil { + pp := s.rc.Pipeline() + pp.SRem(s.ctx, RoomEgressPrefix+roomName, egressID) + pp.HDel(s.ctx, EgressKey, egressID) + // Delete the EndedEgressKey entry last so that future sweeper runs get another chance to delete dangling data is the deletion partially failed. + pp.HDel(s.ctx, EndedEgressKey, egressID) + if _, err := pp.Exec(s.ctx); err != nil { return err } } @@ -589,11 +590,6 @@ func (s *RedisStore) storeIngressState(_ context.Context, ingressId string, stat txf := func(tx *redis.Tx) error { var oldStartedAt int64 - info, err := s.loadIngress(tx, ingressId) - if err != nil { - return err - } - oldState, err := s.loadIngressState(tx, ingressId) switch err { case ErrIngressNotFound: @@ -611,7 +607,6 @@ func (s *RedisStore) storeIngressState(_ context.Context, ingressId string, stat } p.Set(s.ctx, IngressStatePrefix+ingressId, data, 0) - p.HSet(s.ctx, StreamKeyKey, info.StreamKey, info.IngressId) return nil }) @@ -631,7 +626,7 @@ func (s *RedisStore) storeIngressState(_ context.Context, ingressId string, stat // Retry if the key has been changed. for i := 0; i < maxRetries; i++ { - err := s.rc.Watch(s.ctx, txf, IngressKey, IngressStatePrefix+ingressId) + err := s.rc.Watch(s.ctx, txf, IngressStatePrefix+ingressId) switch err { case redis.TxFailedErr: // Optimistic lock lost. Retry. From df963c356139543048d5b2989d40bfb25a29ba1a Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 13 Apr 2023 15:07:28 -0700 Subject: [PATCH 081/324] fix(deps): update go deps (#1613) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 11 +- go.sum | 403 ++------------------------------------------------------- 2 files changed, 19 insertions(+), 395 deletions(-) diff --git a/go.mod b/go.mod index 657207fa6..39d22ba18 100644 --- a/go.mod +++ b/go.mod @@ -37,9 +37,9 @@ require ( github.com/pion/turn/v2 v2.1.0 github.com/pion/webrtc/v3 v3.1.59 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.14.0 + github.com/prometheus/client_golang v1.15.0 github.com/redis/go-redis/v9 v9.0.3 - github.com/rs/cors v1.8.3 + github.com/rs/cors v1.9.0 github.com/stretchr/testify v1.8.2 github.com/thoas/go-funk v0.9.3 github.com/twitchtv/twirp v8.1.3+incompatible @@ -73,7 +73,7 @@ require ( github.com/lithammer/shortuuid/v3 v3.0.7 // indirect github.com/lithammer/shortuuid/v4 v4.0.0 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mdlayher/netlink v1.7.1 // indirect github.com/mdlayher/socket v0.4.0 // indirect github.com/nats-io/nats.go v1.25.0 // indirect @@ -87,8 +87,8 @@ require ( github.com/pion/udp/v2 v2.0.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect - github.com/prometheus/common v0.37.0 // indirect - github.com/prometheus/procfs v0.8.0 // indirect + github.com/prometheus/common v0.42.0 // indirect + github.com/prometheus/procfs v0.9.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect go.uber.org/multierr v1.6.0 // indirect @@ -98,6 +98,7 @@ require ( golang.org/x/net v0.9.0 // indirect golang.org/x/sys v0.7.0 // indirect golang.org/x/text v0.9.0 // indirect + golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.6.0 // indirect google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd // indirect google.golang.org/grpc v1.54.0 // indirect diff --git a/go.sum b/go.sum index 69bad2832..2f0c06ec5 100644 --- a/go.sum +++ b/go.sum @@ -1,66 +1,16 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bep/debounce v1.2.1 h1:v67fRdBA9UQu2NhLFXrSg0Brw7CexQekrBwDMM8bzeY= github.com/bep/debounce v1.2.1/go.mod h1:H8yggRPQKLUhUoqrJC1bO2xNya7vanpDl7xR3ISbCJ0= github.com/bsm/ginkgo/v2 v2.7.0 h1:ItPMPH90RbmZJt5GtkcNvIRuGEdwlBItdNVoyzaNQao= github.com/bsm/gomega v1.26.0 h1:LhQm+AFcgV2M0WyKroMASzAzCAJVpAxQXv4SaI9a69Y= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/cilium/ebpf v0.5.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= github.com/cilium/ebpf v0.8.1 h1:bLSSEbBLqGPXxls55pGr5qWZaTqcmfDJHhou7t254ao= github.com/cilium/ebpf v0.8.1/go.mod h1:f5zLIM0FSNuAkSyLAN7X+Hy6yznlF1mNiWUMfxMtrgk= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -79,10 +29,6 @@ github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/elliotchance/orderedmap/v2 v2.2.0 h1:7/2iwO98kYT4XkOjA9mBEIwvi4KpGB4cyHeOFOnj4Vk= github.com/elliotchance/orderedmap/v2 v2.2.0/go.mod h1:85lZyVbpGaGvHvnKa7Qhx7zncAdBIBq6u56Hb1PRU5Q= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/florianl/go-tc v0.4.2 h1:jan5zcOWCLhA9SRBHZhQ0SSAq7cmDUagiRPngAi5AOQ= github.com/florianl/go-tc v0.4.2/go.mod h1:2W1jSMFryiYlpQigr4ZpSSpE9XNze+bW7cTsCXWbMwo= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= @@ -95,62 +41,28 @@ github.com/gammazero/deque v0.2.1 h1:qSdsbG6pgp6nL7A0+K/B7s12mcCY/5l5SIUpMOl+dC0 github.com/gammazero/deque v0.2.1/go.mod h1:LFroj8x4cMYCukHJDbxFCkT+r9AndaJnFMuZDV34tuU= github.com/gammazero/workerpool v1.1.3 h1:WixN4xzukFoN0XSeXF6puqEqFTl2mECI9S6W44HWy9Q= github.com/gammazero/workerpool v1.1.3/go.mod h1:wPjyBLDbyKnUn2XwwyD3EEwo9dHutia9/fwNmSHWACc= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-jose/go-jose/v3 v3.0.0 h1:s6rrhirfEP/CGIoc6p+PZAeogN2SxKav6Wp7+dyMWVo= github.com/go-jose/go-jose/v3 v3.0.0/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -158,17 +70,6 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/subcommands v1.2.0 h1:vWQspBTo2nEqTUFita5/KeEWlUL8kQObDFbub/EN9oE= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= @@ -177,8 +78,6 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/wire v0.5.0 h1:I7ELFeVBr3yfPIcc8+MWvrjk+3VjbcSzoXm3JVa+jD8= github.com/google/wire v0.5.0/go.mod h1:ngWDr9Qvq3yZA10YrxfyGELY/AFWGVpy9c1LTRi1EoU= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= @@ -189,17 +88,13 @@ github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUD github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru/v2 v2.0.2 h1:Dwmkdr5Nc/oBiXgJS3CDHNhJtIHkuZ3DZF5twqnfBdU= github.com/hashicorp/golang-lru/v2 v2.0.2/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/josharian/native v0.0.0-20200817173448-b6b71def0850/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= github.com/josharian/native v1.0.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA= github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw= github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4/go.mod h1:WGuG/smIU4J/54PblvSbh+xvCZmpJnFgr3ds6Z55XMQ= github.com/jsimonetti/rtnetlink v0.0.0-20201009170750-9c6f07d100c1/go.mod h1:hqoO/u39cqLeBLebZ8fWdE96O7FxrAsRYhnVOdgHxok= @@ -210,25 +105,13 @@ github.com/jsimonetti/rtnetlink v0.0.0-20210212075122-66c871082f2b/go.mod h1:8w9 github.com/jsimonetti/rtnetlink v0.0.0-20210525051524-4cc836578190/go.mod h1:NmKSdU4VGSiv1bMsdqNALI4RSvvjtz65tTMCnD05qLo= github.com/jsimonetti/rtnetlink v0.0.0-20211022192332-93da33804786 h1:N527AHMa793TP5z5GNAn/VLPzlc0ewzWdeP/25gDfgQ= github.com/jsimonetti/rtnetlink v0.0.0-20211022192332-93da33804786/go.mod h1:v4hqbTdfQngbVSZJVWUhGE/lbTFf9jb+ygmNUDQMuOs= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/jxskiss/base62 v1.1.0 h1:A5zbF8v8WXx2xixnAKD2w+abC+sIzYJX+nxmhA6HWFw= github.com/jxskiss/base62 v1.1.0/go.mod h1:HhWAlUXvxKThfOlZbcuFzsqwtF5TcqS9ru3y5GfjWAc= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -251,8 +134,8 @@ github.com/magefile/mage v1.14.0 h1:6QDX3g6z1YvJ4olPhT1wksUcSa/V0a1B+pJb73fBjyo= github.com/magefile/mage v1.14.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/maxbrunsfeld/counterfeiter/v6 v6.6.1 h1:9XE5ykDiC8eNSqIPkxx0EsV3kMX1oe4kQWRZjIgytUA= github.com/maxbrunsfeld/counterfeiter/v6 v6.6.1/go.mod h1:qbKwBR+qQODzH2WD/s53mdgp/xVcXMlJb59GRFOp6Z4= github.com/mdlayher/ethtool v0.0.0-20210210192532-2b88debcdd43/go.mod h1:+t7E0lkKfbBsebllff1xdTmyJt8lH37niI6kwFk9OTo= @@ -277,13 +160,6 @@ github.com/mdlayher/socket v0.4.0/go.mod h1:xxFqz5GRCUN3UEOm9CZqEJsAbe1C8OwSK46N github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nats-io/jwt/v2 v2.3.0 h1:z2mA1a7tIf5ShggOFlR1oBPgd6hGqcDYsISxZByUzdI= github.com/nats-io/nats-server/v2 v2.9.8 h1:jgxZsv+A3Reb3MgwxaINcNq/za8xZInKhDg9Q0cGN1o= github.com/nats-io/nats.go v1.25.0 h1:t5/wCPGciR7X3Mu8QOi4jiJaXaWM8qtkLu4lzGZvYHE= @@ -342,54 +218,29 @@ github.com/pion/udp/v2 v2.0.1 h1:xP0z6WNux1zWEjhC7onRA3EwwSliXqu1ElUZAQhUP54= github.com/pion/udp/v2 v2.0.1/go.mod h1:B7uvTMP00lzWdyMr/1PVZXtV3wpPIxBRd4Wl6AksXn8= github.com/pion/webrtc/v3 v3.1.59 h1:B3YFo8q6dwBYKA2LUjWRChP59Qtt+xvv1Ul7UPDp6Zc= github.com/pion/webrtc/v3 v3.1.59/go.mod h1:rJGgStRoFyFOWJULHLayaimsG+jIEoenhJ5MB5gIFqw= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= -github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_golang v1.15.0 h1:5fCgGYogn0hFdhyhLbw7hEsWxufKtY9klyvdNfFlFhM= +github.com/prometheus/client_golang v1.15.0/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= -github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= -github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= +github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= +github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= +github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= +github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/redis/go-redis/v9 v9.0.3 h1:+7mmR26M0IvyLxGZUHxu4GiBkJkVDid0Un+j4ScYu4k= github.com/redis/go-redis/v9 v9.0.3/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rs/cors v1.8.3 h1:O+qNyWn7Z+F9M0ILBHgMVPuB1xTOucVd5gtaYyXBpRo= -github.com/rs/cors v1.8.3/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= +github.com/rs/cors v1.9.0 h1:l9HGsTsHJcvW14Nk7J9KFz8bzeAWXn3CG6bgt7LsrAE= +github.com/rs/cors v1.9.0/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/sclevine/spec v1.4.0 h1:z/Q9idDcay5m5irkZ28M7PtQM4aOISzOpj4bUPkDee8= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -414,16 +265,8 @@ github.com/urfave/negroni/v3 v3.0.0 h1:Vo8CeZfu1lFR9gW8GnAb6dOGCJyijfil9j/jKKc/J github.com/urfave/negroni/v3 v3.0.0/go.mod h1:jWvnX03kcSjDBl/ShB0iHvx5uOs7mAzZXW+JvJ5XYAs= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= @@ -432,10 +275,7 @@ go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -444,73 +284,20 @@ golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ= golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug= golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -522,7 +309,6 @@ golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= @@ -530,65 +316,26 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201118182958-a01c418693c7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -602,12 +349,10 @@ golang.org/x/sys v0.0.0-20210216163648-f7da38b97c65/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -622,10 +367,7 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= @@ -634,51 +376,11 @@ golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190422233926-fe54fb35175b/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= @@ -687,71 +389,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd h1:sLpv7bNL1AsX3fdnWh9WVh7ejIzXdOc1RRHGeAmeStU= google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -759,16 +398,11 @@ google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -779,20 +413,9 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= From 7b347d6f4f50bf52a187c38b45b92d9dca8b1cd7 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 13 Apr 2023 15:13:23 -0700 Subject: [PATCH 082/324] fix(deps): update module github.com/pion/transport/v2 to v2.1.0 (#1602) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 39d22ba18..e6549ab34 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/pion/rtp v1.7.13 github.com/pion/sdp/v3 v3.0.6 github.com/pion/stun v0.4.0 - github.com/pion/transport/v2 v2.0.2 + github.com/pion/transport/v2 v2.1.0 github.com/pion/turn/v2 v2.1.0 github.com/pion/webrtc/v3 v3.1.59 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index 2f0c06ec5..f6e8e47c1 100644 --- a/go.sum +++ b/go.sum @@ -210,8 +210,9 @@ github.com/pion/stun v0.4.0/go.mod h1:QPsh1/SbXASntw3zkkrIk3ZJVKz4saBY2G7S10P3wC github.com/pion/transport v0.14.1 h1:XSM6olwW+o8J4SCmOBb/BpwZypkHeyM0PGFCxNQBr40= github.com/pion/transport v0.14.1/go.mod h1:4tGmbk00NeYA3rUa9+n+dzCCoKkcy3YlYb99Jn2fNnI= github.com/pion/transport/v2 v2.0.0/go.mod h1:HS2MEBJTwD+1ZI2eSXSvHJx/HnzQqRy2/LXxt6eVMHc= -github.com/pion/transport/v2 v2.0.2 h1:St+8o+1PEzPT51O9bv+tH/KYYLMNR5Vwm5Z3Qkjsywg= github.com/pion/transport/v2 v2.0.2/go.mod h1:vrz6bUbFr/cjdwbnxq8OdDDzHf7JJfGsIRkxfpZoTA0= +github.com/pion/transport/v2 v2.1.0 h1:tLBmDy/sfPu4UG9QsiKiI7Zav+i9zhUYvg7VlCUpIV8= +github.com/pion/transport/v2 v2.1.0/go.mod h1:AdSw4YBZVDkZm8fpoz+fclXyQwANWmZAlDuQdctTThQ= github.com/pion/turn/v2 v2.1.0 h1:5wGHSgGhJhP/RpabkUb/T9PdsAjkGLS6toYz5HNzoSI= github.com/pion/turn/v2 v2.1.0/go.mod h1:yrT5XbXSGX1VFSF31A3c1kCNB5bBZgk/uu5LET162qs= github.com/pion/udp/v2 v2.0.1 h1:xP0z6WNux1zWEjhC7onRA3EwwSliXqu1ElUZAQhUP54= @@ -314,6 +315,7 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -360,6 +362,7 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -367,6 +370,7 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -374,6 +378,7 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= From c223255b215687e941980ac7bd6f3c0f828c69d1 Mon Sep 17 00:00:00 2001 From: Jonas Schell Date: Fri, 14 Apr 2023 00:31:50 +0200 Subject: [PATCH 083/324] update readme (#1610) --- .github/banner_dark.png | Bin 0 -> 157432 bytes .github/banner_light.png | Bin 0 -> 51437 bytes README.md | 24 ++++++++++++++++++------ 3 files changed, 18 insertions(+), 6 deletions(-) create mode 100644 .github/banner_dark.png create mode 100644 .github/banner_light.png diff --git a/.github/banner_dark.png b/.github/banner_dark.png new file mode 100644 index 0000000000000000000000000000000000000000..f96a81f14ff17b4c72d3068d537a45d0ccb5c139 GIT binary patch literal 157432 zcmeEuXH-+$_pKK}K~#!V3rGn_4MnAeB7$^K5FxaHG?gZugeC&gdv5_zK&3=LIsrj? z4WJb1HGu#D5=sb=m-~C;eS6;f`+t06yfIEj#>j`W&)IwJwdR_0?nFJ-*SK=&`lWN{ z&Rx;cR5Ltx?jq{kxxWas7bw5j|Ejg6e9^tu1bd!4#|HTO^Ou$(|IWE{x6f&*J$UMu zyNNW91z7qg9?s**ii)$OfK9wjv14%w8ZCHCB3m2s6xSD37M zuU=*3S92D;+VRf%T9mc8X6nps^*a}%Vk`<6HC=fD4bq1VUoe(6Ccn&#OTRqJ^Vxz! z#jW%V4(Do0pDC8U!kv~K2DWcHInHyu`Rl*CLQ3_R zG3GKCscrx3VI&v>UfI4;nSYb=KX^I`Ou%JE%#GK~v|Z;eSpRp|g}j8nr~=RHQ?+q? zKX>`Ry1p0SB~=RCzH9vt=77UX=>jd`iY(p#U=Biuy3d8Q{3RG-efht98Eb14k>fT> zW`UnE_OJi$WzJo`EYE$O`uIXF^;^mRdO;tc|Nr9r|Hb)#dgoY=adHGvkL+`}jy7z- zxT6L$eYuyRm}B`L4H)=el)VsAo=pkzi@f6ee@u<9%(7$c|om~l3ZpW0D&|zhV1AXa!UfrO=!TzwpJaMmb zIA=e`q5?dams5e}_}}-F__zDLMslj?1~>9nDhA!RN&2y$>Ltpx9%S{kz|nKfX#$@_ zNh#LY>+%1V`Ye<$+}H5%VgHc6)ENCG0dm{5_;D-TRx4m&;_G02m&-T^K&bTUCO2{#fo?5D9r5-whnSlvvoL)fIi=IY~(`Ri}z)ps5G0XyH9z_ zL&N{9!WFK6OHcJ&m+u0wjXlm~f~mC%@dxV06MJsgWM7$~t{^HU+#b9(iLDv_5C`Y} z)s4hDO$y}QP=ih20JL4!Uw8lY!E=UHZn@XBO(op89e@APQB@8ev95P= z5>}AGT={I=9yZSKQ&POMOV7XM`|oQ~ zj~`*|`wJ>MO>zv^9eOA45}#^aA7|TGReR2UvrMPBu6CyY@4jg^RN*WBtNhJZy5%6x zE0yt{Pt}Y0Mw)^U4BcXKRa@J5e5{yoa@e@5Z_*HJ(h0;KI+pcCPl=J}|c zGM+}Zohz62uMF3g!)6BU7iDtnTjC`*^?eY&S6$Lg-LTDN%dqh|sEEH+fVw90~zwrE4$k^C&Ohfp;SLXw-7pcMg zM+0VKbY?tXURz61L`b2ZoyQl&n!0Ts*SWuKUUjm5nH8^wMqb7F+E`QaB$YVzAM+&2 z@BsS$rwCtU_R!ZkLRtU4=zTUC8*^}7oa6ksg$gQC)AvA!+go(7bU1%7b!l+0{C@SX z83~jk1oH!rTTfWe{ zeuLGQK?#yGDZ<3Q`CxS=v5}c@%EgT6Y-^Z!zYvefsEfZPxpiT2YJ8_lwrb z&pTPGzVpEXX#P50z+fYtm?G_UCY$V_{!?o?DahvOod98x7lRlj9b}`Z+tP27LyOd= z^n0FX!g(2H%y~hW*;WP~j?n)C-sSvIA&zdnR|!txLzU7NK4umZtmAtSfLrKv=-aaj z$`#E{)YqdYNEClaS+Itggwta7o`1ao-o6A3vHIK`Sf9X+l zDmTzxEy)n(w{3(z!3zEC*W7QvbRAYPXT)<-BhXilotTME=?q<%n)0X2i`A$3{n*wV z<)$B#K2EBNkdG577|Z#5Itucy9luKne(UA8=l|H$B!Pg;81V){W|Iu({L2Y%mx+N7 z3tD?v)Rm@EDNo(^yq6~p26u1tV+1MtBUE)tV9Y{nTHZ9roo9FqF{0Z4UbywcNTTbLBV3#~V!k>~?&IbI+y<1j+UYL0Wm1nr&Xr)$_}s-(csfDt}iz{ehx?))o4J zO}*TFZfz+4ZBLUxBl^`K-e@n|!K!}+w&{OmAZEiuk!q;GZ%3uzOS z{I`1lb&5(6@zcv}Q>+X9GQ{f1k{0fj(5yJm^Ps@O=3IURUtTU}Hk>4a9JbG4_B~J* zv8-s+>?<#_@D6vjnD_HLiF;V!Z&x87k9XeHjH?W@yN^B$akI+x@5jIU2oSY1{x(RI#pEQbHDMRkR7m2{IcN4X%&hIW7(=qiJ~?$3Pc|I zXgk{Wr^ZTUb<;CrQiqP9Cw{|)wg}elxEQ%H`uWRaph$K!O=GLIt6{1MQ|vwsENfXe zW^=7-1RH~|(Zw9=$nVyzc6$)z52tKe(3zp>o*{Ckx*Kd198)BwFeivZ6!RXFYh4 z4#SR%8rlcSQY&>EyFq&b^~gir{Tf$D!7xkH;j^vtg_8^q`8}nlFyMqiHZ-rN{mrmz z_lEVC8CyP5sm)c=r5w*(ax`OvY$_Tpe3wcQNB4gqo7zKKESPl1i`xaSIH!=K9-!*^E}A2r;LQBfI^!09Tx1O~tMD;%|_i zWDF6ESts!XqM+m+Qup8cJ@xfJb00Tm$<1F>l&{b@vB8V3$DaeV_|5cg!9LQRN3k_Y*8y(eZ?fM$b2$COH#hz*( zG)g;Wq)b<6Qp0f~E7_T5j6V~Worw~DL1z!bMHeK=0z8~6uOvG+ zB&*!UMxsvZfB6Jfl=J>NEb*1nX;GFxyDzUyN>O;_pN+NaIZJ28G$2t;Es$$}!|57I zqvmycS@qAABb;isE}+T0fIY@j?ETe}!?H4MCgTRG*&gwXy@7BzJyMSaPzy95{#fxn2~3K(Lf}wl^ad%kO#6He&Jw z1PWq}Qi8gEL3kGOduPs)r7jueMuSrDX|c%O@Dc+3YQ1?T8d#VZsN;|u6PYrRu^F7V zdb59Ld0ZMbc-fjbm5O;8G8d~hoPtjFhNq$ruR?^A#ON8qXadWUPxJ0gsJ)n$I`Zc2 zKTi9do?ay$BEFgFsW-eIgG38oo5_}rfY^(QSJE_YM%an#T&TR6Fc65xCCkBQ-q}5k zKnqc!f&oa{GSrGVa0l!$ku(d6BJ?q1w5ipX)|?#Ql|`ymTaW_0 zvU_T07%JjhV$~8OyS|BP+qqn)vg9vwo(PEhYUkeA;r}f1@;&?fS;6{x-^NI7#TNG| zl7Q5U{{2NII0Rg%^YJUz{o+sRHL_c3TEJjQg@pgTN*>L{ zAoiES&S5buOMxs8?ifl6p$4JpEEsvUmTXTQdV8B0@#!E}EeG%d3M*Dlb0@je<17mQ|#C$^WVcFIvIa3tG{AeVB{mJ&2zd{*xCA&Rb=hG%JWSfa=2w>A= z@%lFBygIVJw8sHtDACJ?q|t#sSCf(j0uTfJIq^B&?$%knJl*)=bVVP zdb&JRWvlH-2)s7XUFuZ&0_8e?p#gB zte4m3Kp#t6v6Q!ol%|v-zR2hfO1&csoIaZ3M?qUjfveQ8DwEggn!faegJXHeG?qH; zF2^3vTT;a^?idb3&M&3QDbDsJsQVF}7nA!tl{z&G#VEMoFrY99E$H^NxFT;cKi5S1 z-Xk!60Dw|Q39dp9X6TBFUi&+e@lRJTm|&@x3Mw`(E4j~`e)MQLls(O4rc&fwyV*2J z7R}c2h+Ui?^VIsHl4&c05xbFsfr3hC8uLL04*A2$wpG`=n=a;a8j|wrRPtlv)0)Ry zC-8haaPFaPaFvqiZi{Z-nBRkk&9r<_#h+A)jdQpmqhIG%pvS*?6Eq5ZfzP50Y%kZ! z2SZ|1Z~=^eBi#Qx9lUTXjY?Muo{Cjb_AgoSgo83r;BvDsqFs(cpYQ6WIGM};NNbbac;9_IHkDC7KR zwfBdXRi|4{g<`1H%607<>ooGTgzZk@-IuX#e}^CwCXa#l&-A`U@P$T3Om#BVDSbj& ze*?3g*_X%yE$t%n{*o-kUT=_Gpi3Gb+j`37S#ERfI&62eY5z4$m6)`-5sWD&jL@6t zt>py?nJc}Gs$R(_#K?p{ol2azTj$&8$8oB6%tngps-SM%Ea5>pX%d{>U55pk*31g% zouE@_W9mld)=yQTmWw=A0lf5j$u|3ZGGUtm0qI$3qA_OrYYacBScD9Kc+Ch4jhVo@ z943rSZS??O6L+1(jWP*XoDRZm*4Ml*ziG{gBgj*=jx*8lpoMBykknkle!e_jhre0} zj+MN}4iXipEAzaoNlSR0YYRP_n!4!i(vysqAHWM`ZErg{^%$6LS$peWZPm5BrWe9h zb6ax|n{{to0SS&*1Bx$lWw^VMTV`s`0XRbE)e0`okJQXG1%J^02i@q6iVXEJST4l% z;6-hb#~n+!nSBWEW4KwcO;oGY;zp?JCBoC(z{jggf&IE}Z&gf?(@wMmJ8*EXaC~1K zkSmZtb?QI|rSvb49rLk!TP=!fmfhJbn7!cEpx^ifuiYNz!G=YRDOQ3j1a1!fkiGyEKg!hhtQVWgl(gYhDLg(j2=rRo)2j zb&@$b?9$({+y}OCcy)e{@8svi1o$nU7yYB)5}?Y7DHSEeUM6gL6I6ggD1evG#|{q< zg=Fn7x|Qj66gU>-mX-W+c`*d=n*YY>;;xL9`{9E6%MfvrBRiwm%EXi5vN>T$s< z&GgC!#dTcyYPx2VlOQUOdDbMAo!%>H+^k?;?x^h9L4!^~>Ae3X!dC;op7K;s@ZX3% zleOE2Upiak$3sA2il=Iq>Dp-MvWc+nXAxHJ-;9{5$WF=*wzIPMd?Z{if1=&%tXOL} ztJ;^Dv#0=U-$O^lITUn@iM21wXU_GqV%qDuoE9s}R5C=pvK+yC0aBoE)Qu8(fr^%u z)k|7mj8^Cf4PQP=xak(SxfQ4+bgZU#3~fN(N@Aiz@^s+SK zMSkVfr*UCwLBs5rGh`0-U24a;2i9))yL9#M1a9RJrP&uD;%}b~3uc|yb0`pb|ALlK zNVGrr$d;A&iwr{O)?Z~pt*It|AO`+I$P?>RIv*;BSbYdyUzH|*O{T32B4LVc%lty3^2c}YN_|!f{uZV8U zWW!V^m=&0FVnp19l&{6MHF6%gzEzh(5v|)JjQ-e&9z}ARB2?yu_)x}(nGCCB1J#Z! zq4bCAyjkm&->l+?rNgtc_F)oDIU-d3w1kV@5Di4-&H37od3K2OB(xG$po|9Gk)E*M z>luY#P0@7K8b01@msC)LVfQBAtN1FrI_d#tN@)q57*MuBf_{sb5UM1IYF3(lcHk%5 z)GG&hN^tpaXU1rM)DAN*5D%r4j&~|#UTHj~`^Y%uzH+sEZ%gWtJ^P2df~bL2Q1ZQ{ZOIi0PHu9-WxnoU-62 zHPalo>ELw1biH9eJ?w*FOrpl=pHCotYBA13>8}j4!8(Gyjo#V{z>pT&p>+2PSr3*4Eq~SVkhcL~SR3W?%3a`O-DSAjv8I;u z5+M=meTiU0zMXBrY^X|haH3L#6!y==`-cQS7Gt~wXngq5K%@K70g)6-WPJyIHp7eo z$3XPw)bLs#U%_Q%eryJcmz61y41RYe{vR(uvu5G6=YF4U+e1GKhpE&4P#~qv>EoAS zyQixm*bo!48ARw`IIwC)@#uVyNW0h)&*j;2_%Rh!H#|+beYF(TtfT=VLM1_ZcyM%z z+7?B3-u$37^)>nj9rt6KKOzS{`+_W~ZC*_3j=7ey^2-$=lul}nQrhUSNP`D%p{Q@+ zCpp8}%?J(v3=(<8jl~rERx|gcovN30w<6d~YgO3)BARLyFeFW~+nDhWrBGSIwE8{E zH%%FO1vx)29<>-}E$nyf^YiWH%W!gm-(sy{2uV30+QQ95+fYUv<=aO`*wSVWY}`n? z)ly44A`ix7QO!Oq)Vbh33A>%EF)FCw#%14YmQ=*$r;l;TYjzH`>=Ga;Kn>IUEiQ zSXj;G&pPdOF-3SD@Nitc+T77lDy*ShpwL%G0YF<$0;oaeZOR04o;SZA^LNVX_@|Wh zO=!gJZelnW9U(=gZFi|p#CAN*dz}SCH z(=WV3Zfju)ni->8BSE<>bZpC>=w-FD=z2;6)qJpJ7i4c=*AI1p6gcX*9|i~wsj@RU zJG|!}#wm0^&17s42A_gGMNA6(H3^Sa?BX~guxi?(6)i!QS&}P*QJ)>=^wnmwM5q%9V7$asR=B$y#hSBl2>1c>0Pkipae8FQdy1%~E z32UwVFt>Z+nOyt|bd1WfBBXjGDnca~&qU`XloJPXxGZmH?Q}6@=%SQT;vad$?NbJ z|7t_3&oGv=Z23FpQ1rdDvN|U6E{j|#i5YG0n@wf8lBToN>z+gMtMW_yt&xQ_;1rRRVF|p;5){ClV zte;EB0x?$us6WQGHdqqD)TL{WYqJ94bFay9_oDX9C|$5Aou0H^estAv%AyDg@Ui?& z0|T!_0y^VWz(I-c(|1*(%(T^8v#Uq@a>J|`F2#|-0Z3H;kBn%ob$fCAqb`lK4J&ux zYPXRK^V2S0Q0q!SPxJ45i!gBCJHp^Kj5_{cS1YpyxMva}TH$^xSKeYZ z7Gi7^r{M(!9Z{{Gv#e$KWhSrUyFihlK>Nm<2)6I)f6u#`INC*oc(nUnTnHg;lFz@~ zag`~&g@V-i8|{|08L%rs_3=VF1v+7zy>2lyFx8JaQkGIX++3}FWjl$v>=ZU-c{6#% zLDQV-r5K~}YuS@_kq^H2Uxhy22pJ>z1MR@6CkGs)h191qS4pdU)hzlhbV`y zCDT|wgX?HMdzCbE?fo5+09r`yfOnF<2U-xyHx|)7|H<%_!mU`*KThVsueK#w_*+r#eDuuKDXeoP{r6#+ z902eWU~8qF-^5n*jWhYZn3TlAk;)PRU(pg)GS?fJE)l>sL?7*XzD%&43f-Xy1-zZ> zy{P#|@b)P`91a+{#w-EQFHMscA44~3(x#`ZcME?2ujFbviskk>{{U>3arMg!-!hj~ z+E<_b2)~OE8xzzfM4zyS2$Q2Htx7)qTwVyC`LzO_GTesUR9Dd%e!1u_*HAPm+Vr(x zG}oiYtM}@-7>AmVcGvEw4pmmBRo-{2MEnSNOU^j(>TgejxZNE!F|)+Pz-w10vb?R< z16VDlS2zv?*hl;rj&;yf5kdV)7P^(n$kN{)M;7M4h#oinw?RGw69Z@Ha7ENKE>xev zOYQ7Y%!wGR0RLAs!{a{+y*s^M){bHRg6-uF2u9L!1%ptZ}VG zgmg1xuHhr#zVGE)iotKkA@3h7an@s1sGL-cp7!^y)kY(>Vp6#b-`ejXQQ!XN@o=bK zR!dOQHb=(H@+a|Otlj&%d%?oPIkASgt$z3JTxXBD2{!cB?v{6jnKE!td#`Et8s&5NZBgy33kAWGF&fDfiFiNU&EWT?N56a%yLNLlw#NU=sEX(5Gl`_3|$( zN&iNlsKcvWB}JR~;wqGk2b1=;Y6{g~+ssQy?0v!0Qn8=*0b^`(cbiOk$us_Ot&Rs! zxsW+mPG9YMw>SI(RMf{`IAMoweShYG&Rw{#OpnY}J7imzWA?V>>?yaaV0X}}6$#5p zT2oK<4U|ImN=;fV|EvNbL{Yxub>9nSp_Nf1OSvL|v3~f6shYI)A$3-o?nD``Z6Y>1 zf0g>Uk-yQr;PCCd&jM>QHKN%0 zs@pJ7nvboxsAI;=4*HOHR>8c^IW;j)VM3z-;jbSMV2I3uf}@Mrq(-p^RWi2aRn1Ss zLtffLtuh0x%nD?9M#rXgE)j4-?k&mRwF@8@p}DQf^X3n4&mlgjaMyAKkqWL2w?4ce z&;9*~@#-J-UogaO?#1K6iR5tj=;zN>J=}HtkPHbolfsFN^}&ApHVw?T3^&e(9c!Ol zA(}ZU-Or5U%8!YTp@SKJP>VVnXOq|6u+R6WB{)Y}DH9H2sA1Jq4VK`Rt54>T0rPLl zcASzzltdNTzo=RE#;c$oGI7WLIWs%39cvewFs@I#q>(GctfyI*?AgyfWnUho#~(}8 zw?oUN9m?bO9WD{RRHb1|;r=^T`^`ZhV=8TUL@e=lstS;A^9!v?h6>q=CS)$i;h>vu zAcwo#09?Rep0ezs=0Rq}1jsI(m-=JB{f&17uYUaLFXHbv7C7zE`vUDb*Ot9k+>o9P zA?T8woA(sn)pXHPtf+gf;ZV592sV;{y)!0~8<9)Q1`J~_5sE*!gv@<=iZiowGYnvs z3Ur##DVVP>4x!W}Cho*Pzxn$hjIlbbRu>^SK5={2JgzvQLGj64_RU8HCaGw<0$=gr z9Md~vuLX|x`*eYj3bMoCCjOfWcc2Fql7y*a=^-4CHah|@L{h_!$ovRP`q$L*+^$D& z=>NFxPIlIlRhmHH_%!QQ4mUP&VN09fuu2@d*RO&A-RCX`4P#AOvq-;R6-A{3V4|Z< z>FeI4X>=S(;1c19jV#S^T+oiC4y{|dXa%6BZnn~xab6_)3iH+kPz0HAOBio)b|s;_ zQDr#mq5^BXt!okmw+bqOe*97Y?q*ipU43DnG#6_zPfct=j*JXJKLJpRVbG};#REY| zd>}q04GkE6*xPvxdn>ep23FnOXq9ATl1X8=aeG@dR7lCFZ9y7KQnGLqk{P9ay?+?W zEN8-qF#1+PLma=Les6K^o7%iu5*$)YZc3wu?KeEW_oesXb+FLl(8W7SYJYsT@7ZdN z3L?mLpl(z8&aCF9UgSO<;T5MvJQ7g*k&Jxz827B0q=Uz0J>f^)$f3ttdVM`PlhR?Wu3!xI z+S>Z5KHZC6r38wjf&ac*HC8Dz5d}?_1rX?AnI&NxgHQA?8?Vss?^XKZt#({+8llRv zCpX$4xo5s@`8g+J3k-2*%L{ts?zgiyjDT$|NX4sM+d;HJFLtEc4D`;dcId}~xlWFb zaMa2u^0hYNP>gEjS?DP@L(u#V@SP-T7cU(2i9FUOgXF#%@2VuqvQCo0U2OH;!O!jD z4uTsgJ|F`i^<;c7jtw(75biM&_Ee()znI7n@kb!_)gh^+LG`q;OF(l!dN1IEnj@W@ z1&RI)dpl^FtE7=L+`g+4TiL*sBsP!}U28{iq0GgrxfOV;C#jeu`@A^8GKomudQ{sS z-|1be31q`KbdemP+ z)>gNp24`YU!{jM+xXQ|T-anQb=di~bU+V7sG-qAkUxgY!$sO|0bL_4ah{$1CkBe8o z_ZfOh-3VbL@0*3~NNVG28?f4i6VBkpC*ikCsT-3!rafE?bZpJ#rwtoTt(=m$`fSOu z_-`PY!w0SUW}ggVQKXcNqHT7C>68V*?$!;lYy=FKn?>~XZM_=gy-W6US#}#Z%dmFG zw}ZE5O$u~Qj+g4*`YX)K+xxIm%9!lDRR0(|Ex|z;v0}UwNq%PVtqCpmZ^0YKx}iQ2 zcyQRwGuo7lwqJGdEJQrevM?@|Np+bkUU0SxFiSyqN?m}$0sT#kYg;jd=LdahuU(&J zn`GiX$IflJe?{KXe$IqZk^+Z$iI(Wu7&9C?lfzdAh?>}uT@SdyVR9@am5veQVm6>ixgo9-w? z1PdJD<1ZhsAAsMhkk1xMo^le){2EA2H`3P3d%18j&y?9O5e~M4YqqCp4h1OHH~#3& z3VSD7NL+)DZK|w3~<%=ZY-)U=l zJ%o4~Sm)^0tOHKwMoV(QPi5Lip`q53^YiDDtkSF3CZ{VHUGR4n`zGh&J3I{UPFGE( z_r1@Zb5}F8(qKkbsTerGu@m~-!`nm7tVX~rcs$|xo4G;gP7LLk73@mV1n*$VaNrE- zffFR?aAz32e5gl`C~qahM#I8tvnZO?yuu0bTjbpK#d+?N zk2QJol1EdjA#+WVh$I^;%UrkhJ1m$SGfsYz6|zYI(s?;#?ldRppJ1^;PKbBpen8Y* zJP09}tleK{)kKA($UkjkV|Bs;Klrfye!wg%zGZzYd{(q?o8N1UG8cHHaX~bnZswcH zA?Lggn`0T|i;3LTBy@X9$5=|;joZUH*Sk09Wxr_Qk0<0wpspmy-i*fm=_Ly_Z;p5M z&-Q-jMQ!t;mdiY=z5G?-2rKCZ#waN+n1Fioi#KQwwt43*10y_=w zeFh(eL-|9{PY5?FbE2!3WRY$OQZ%q*dZOri_-y|@H(iG$%w?M6C#&uwwqvDoN=hV4 zL3%~Vv-^DFG7xHZx^Wn(hDe2+;m~F~Z6m;=r!aO`j2SoVCysL;`u2Jz4e-8sIAXz{ z<9j(|)<}|}$lK6DLAy*)`ZL}ZBRVTi`eNnpTMZNx{){essVpDY*NbbvL<0rv!&nrFO2Px0WRUTmHZ=7b$NS5R15lU4hGDH9tkZ+GMBg~wF-=wB{zwBur#bXlLP%x>2r3HVxTci* zC%ugFyTy$Q8v=DN=4Qz=($@m3#=l&&pVG+yp!|P`oo3t{+kiog2P*zo_YIRz4$pPj)Q}@Hg3us`v-ZP>w!9(5hVMFv(s0S z-(Nj=12TY<*&W=Kck^@3K6KP>ZoQ<-XW2aTfq+T-1&8;q8(`&1k9^-L+sAC>c#3?O zXTt+q+6iaI}a1dYp`3CC-4Wuz2$bFGwvo_&T?47=bUYD$- zKHmAX?=B>*MC`5(3bAr}{|H{zd#$Hw0(g9C|GJWF(Mg=M#a+5EWy}>_*s!~I4MTVK zDuy7x)Ep^um{~%NA2PJ>y{Z;~E3@$Vol1CJjjj7+QC#Tv)X_ z=rS{PDnD>lh972Fz~1>cqIs!BDMM}?^nd{g;tmt0VPc`9rnWpBVk3J{E|ajm>tV@% zK9Au}fJ(@pK=)r9_34+N_NuuV708GTpSH+CR@6Fy3pxRdbG5Q{)W=yJBx?{zT=IUe z`;PN|Jq$Wc46dr{jbK`go#yD5?Qmw3VxlEclM zTa|ZBrlrH0oeyQw$@hQHCFq^OKvpBWzo+%0y`yjmL8HGB$ho{vZcRg83HeRz0ZK== z#+Px25Oc$`3~gEhH|MG9YkZe%AF~Vq1`atZ=_X4oTM)x|`rI}=ONSBLyW-@&`Szf{ zRdVcag_}32f1JU)E$&$H3r=uD=O3TWQK*wRS?;d?yWQzOVy2Q!loh0a{nB9ao<$2S zB;M^JJ1PHj%Lm>lZ_Ty+AIl~iZf0)}rI+>=?00rBY0-ObW{Z{(p}K4(WPArz#}M<)Y>+8piM&mdeg%0E_T7oTRgce513DF5&@wx ziADfJHM-}`PGcT!Z`RiY^q_N9eFhBLSvALIiL31H!|Curtlq01>ZPQoe~B* zKPFnlx(WrA{I~{)v6Z{2z4(BC^oM4jr=`5&Rk1IJxm1d&A4Q_zo{%XYIzry#f_@)+ z(w^D-6{Tk+@$Q0fXg%_E4s&qenQ}K^z%e*@KXYey+&yPH z{v%A5O^YYUZAX~xF#afHgu9xnb-vZseuO;A%Z6#KJSksL&LZouwziyzVWB2XxY#)i zT}VqF8wRTnVyX@kz}2ik-CsPw;R7a)#5#8NQX@ z$u%naLm3vHD`!5?o$zDQ)ROzo&QT6VNnkp8qQ~roL$sHTU|?|Y`wANm1mAfiUZ(CJ zPji3$Ooh}xXvmjH2M&7sc*`qsygkfI&IKFFvOpJY-GKRK!+Kr4OE#`x&vgbdl3gE2 zT5U3t6>#u>yZ|g|!7keHLk}95-(`(z5$uup0o*=*tb8?PrZS0f>JGTNcb6_-WO+}? z@AZkdjX!}g7wjA_sBs;ZT${ouT1Dj-5^X&PX~OmpqcuwllG z>7Yy)dD6+9WONndZ6eG=w_S}NDK_QinC-}DMX86n<}m~lEm;(XGQWf~@8Jh?j3_<|bg7whiX``dKXOKT1=O$>F3KOh zZA5WwHIee|=YH5d$Q(#(4p->!6+@p|RtS1*J)(5>VGM$Q?Cj5-pMZz#(-OLLULL9y zDC^r7YNfB0BTIx+bX)Gov~oc7heCsblW3XdaBtCQ?y|&rt@~MFLX!ZB+1!KwSanN- zCB00V2IkKFYZNejOVMVQN^#~eh4>I#ujb+CxE^eyczb4B~ay4c~uxPFNW{B&hTTIG^e=RDM48mq!CX{U2E zq5UdnDs!JH(bPgAyDCc#&9rN&l|3zkP-i5g!x?!A##nSYsS#zD)M zJE?*jUCr3Vf}w=U@pb;R+kF&2(virnn;sh< z+o-9+`BP3wHxenkWe0e(d>Jv?FKgud1{Fq$wit#lZ~{` zrm0SW8mAlfu|clK5OaigEeS+Xw`h(%_GROSDo_Lv%$aQKW_&vzTz zqdDH4cvf@?{1Gs5B@63;sH_kvS*(%BzE(;)vGcNt%|yS&{hq)^;91Jim;rw*l* zx@vUUxNion=awFur4ON$jBS;SmX36_2w!tal}3dTS!nllA0xFK@SM#7%(M$Bi1&5e z5bfRCh}r5>s-tcsC_~cReC>bs-*X5KVA_mx6ZxwmK_nTYo!M>wd-c#PH)8a}=c`Oo z{o`Hhy)IZBcm5greI~?yD`Hxa?jXW96mq}gAV2?9rVD%XqO#Z*)YW*nc)X_dTfca((QmB-VV(| z{r_6_7st^@n&ZK}(_!p^l5f=Wx*45A zrMv`#a^PSf_3`6gPrM|3nxqT;XiS*eu=t_yRAncn)MJ{AR~s@^l%J4<_kM%Y$B5@k zV#k+PIYsdBKuXC1d#aARAvPtwD49KP5yg|%^I`{W(%>>@Ma#d_3O$_y8uYZ*W z_E3!Drl$;4FJ%uqBKyT@iWhH2ISN!-hW#4W6uBm0J3TO7>mSh__=D2qe)tG2jBV01 zOsY`0M|~X6hG{BRQPy@bSRu9vub@o}u~Pf;o*+Qf-3YFoO-GBotf=VbZSgLnKH4BFO8abLaFfGK zN>9sRO$fykarxTFq97t|Lghm6ErM7$%f0MIDT7^|wa2j*fv3GRU&l8$P z^N(0^MjD!8fsvYIq3*6GU4?RS*F_3?p75v0$%)HfI4!Di0cbL--W%A7S#>mY-Zt1) zV>Xv6(Yv}~|9*P-$<$hm#58CMXQsA6IdJ#0CGuupdJd1TR){#D%aY1c%5Jbb9K7rr z<$B5Mv)_V99e}@jlyX*JU)J08iqa@@YW6uur+}17Zk_aZI2CKy`YQIYnUP2Cz71|^ z$+o&$ZC_558G|!p$2767He8)|0ylOv%;Kg8%7#)fy=RNUrT&S2%{(ae=5$t!1M%Q- z!LaK3?`e9PmWKu0y4#;*pOUXL2Y%~Qn*hujkDqG86KtUs;SF=~@%||YBgDQ)3|abU zeM*^@qQGL?5MpEq)-Cwe!J*Nz@UDhKut-f_D~RmaW}r)U-H}0)dH3slmzACE`>;o= zovoGBjWyObj$AVvigmSk6Gw7rhDyfKey3Y!<%!s@(Aii@pJnV zk2O&J=P#!Hu`>TO_gxZ8mYKr_uo$-)yq&7IM^l&Dk$(Aw zP?PYUk@iiU&uC2;LKT+Lm$BkNIWNr~;6piyms;yXPbd=RMAUf-70wD#jX}PaCNDS! zQCW5YL2GvW+R+NyvC3NeAMVo}M?3{?<2#ovpyKWlO-^z1tBV<*6Gx||)|efF!Ax2d zluHv-Z%}2*lNByLjavOy#NsAG73j+hk|*Bp%(V-r{D(#AHK5V%cNR>eqyMwaWnk3+ zAc)hL^55<(y92 z&jfaT&@4ol6g085@+0hhSxD?_HO}6OI#&ICmk5u=^AW0!@25{u6jN`5Wy@?&yp90P@c;ZGpy4QZRxDFAX>2eb#yX-g+Y+r%wEgrg86p=pzP{Ce%^pEC=tFM|%+7b*l zlXrpst^GO_3dbGymrr-L2*Vxl5t2`AqhW z#;@7z!$WhW4c2qZB_EbCTtFWB2UXknHPwccsk8}2uD_`>(|@MUT2)L87mud{*kDJwsOX!i8#)3OS)3bRi_AC2kd&qgG#X9taD5C=9dzmR^ zE=*ZxtPC?$ioNPm16)nXbXYbFJ>_WHf*2p_ht*V2vRo>Kni(xY{x8PfJRIuxZyWz4 zvQ>(ZgzUS?mYqptC;LuiOGM0|8QW)C2u1cK#*iiJgzSco$TpZkO!j1%u`e@>z2Ce0 zKJMo@?&tUY_I>`tAKuq-U9anUZRdHuV!=4InT33W+?)i&-}Yw1qK3#R&55oj_p7=^ z*fZ(SRbH0v>&bOACvSZ0tH4?{bWMHQgx@a90D+VO@fuMZSm0v#4o%X7H>pB6wiOu! zGOT5-nBj5aLQ+8GVj~A>!(zj0NLI84$C6DGUi-FJ_o;fsrQ_{!keY6j{`#g_WSH=P ziAtmqrgers#x@z~4n&iGxx@rQxRZKMy}W*XCnfkXQ?|l#&b58spS@0V@(;i``tgXE zSG(S(a>OZ)IE7N9B>)RcVR`#W2rh(A_lC7fCYUi$P9tubGEl~U2-%{rU(`yR07ixm ziiHVa4#^t?H!tIy2ZsA3T!vj}l)J4jsQ@8gL78{HiGoA^mW_eVZ^8 zu>u-fj3dJkV9ufZ$lT+HCr^WM`$?zHlECFw3Von$n%!#S=6n~WCtM4mJQBP?&9)SPuwO=8z3RJujw|AeLl)C{i zWd!B4xQ|RI-#?6dc#1J#e50QO`2Io4`I8zntJ2*%W zT(@ufXryg;+6VN;&dAa>>7;=Z78}vvHVi8%go&<6yn)kv4^{^zcuX(KdWsy)ks`Jp z9;-d~`mkBXODaLG2wEbeGq>gzD^wdCvW zDHn~w1{th1NWu1BH4IvjFOvk!s_70rC|{cxR&UWZji1%vp~FE^>Xly2*NjlxeW4sc z&B|U*dlxoviBy`qw~|r#`H9V#OEq7UQBl{I1Gx#XHeW2L?cSXP54uaK%WR0tO5 zl%03Mor^%N>GYmQXd3X92=dDvO8_?PbCwU`a`#GfOE|ZNEl2o*SJ`TJgq=s3k|bWY zIzUCpEFa%??rM zYP<)Mi*;HU?=Mn~yMxdKmMNMDCM*)}#PDJ_HHyKUFuyea5ULh%s40jxo?dc;S=h4{ z&)h&RH7+wtNs$c0O!2jb*~tHq(;CIN~MYF3ILZxU0vxh#WdZB|PVg75N@ zu-0p-n~2IXxqd!7$eq4=tfGnvXhB_37{7Tfu1jVwR}c>y8K4LKTUhAd2(xsA2?xp2 zYYfQzMTa<7lkzfGI$91Nh6{0h5|+aQfv;SfU3zP0mGNnH5W+9vb~u8UsoNUlUBh>{ z94g%B#Cp=>ps-bR8NnM5YjG9W)RFOy5odLbF9zs)I~;PNZ+h}+dM_pzk-GVr>2-9l zvK#>Mb)j{!KPX|}rGv(qgN{UGs(~fEHOh~?_w$oCqLSmnRgYd>8{5l7)rWJgEy_1I z0DdA|o;92xc+}EYTiyb8^?2yD3&vd{RV}>z8j#J7>C<*iWhZz{g$mqfvnTbQRLSE` zXb;tsMjif+W>rrbD=Xi$I(KO2g^CuQ3SyzSKY~>j)U%Sh0{ZsLyTsCfqV;rt6ifFw z#~eI+(bY1)EZ`P=K#%B=y38kIb+d0#kC}ai0g8FYFFw>^Ou1K3{K2a$o+muUS$ZVs z7Tjr1BV{GvTxc3tY0~_)x*0C+sb9}^z~PH_bo3vEeLH({{Gur$32<-RR)2JFB$xcz zN5=UHC&F+#L6->M6e8It9Ia5u}#s|fSDPT0d`@-m6t-6>wjEcP)b_d|Og zE3u^CyDa-iJ4VQI!hq}4*n5K?w;NF+*I0JKiVzbnukCEXgk{N|TW6psd$tFrrutFS z6S}uHPKjgMFM*K!-OPHW?4`?|nxLAc3zSVMg9!_K-=vEe^)&)E4v5S_i-OW=&{iqOpF8cfh~P&0i~<||$! zDCWWmL}<2~%gycO;1$AThzIJu@RV0f617C`Ks?dUy%Y@}e!nx78s3`2j{_Jhz!$vo zxQT9haTIFhUABve%tQ0yXsTmVbyC=Tv@KrG7(kfRPpnPlV-0 zN}wAj#@hQ`w(c;7Xnx$fP4{nD%|My|N3t6V5Vg@{){s`<5q@X;+^+M!RL+-~flAPr z^)D;4Wg-4aNO+az)iuY(iKyi1#!;vg2daYskRTi!px%d4rs!Q%fTU_?>&y6`G1;L59cCVtdG3X zzeIYyb+bT;j^p{~-(SJm=QqTIwUD#*d$uoIR|5iN2D%UWd@WINzJ>#s?D;fZ?zz+b zSV*X8PCr^ML#9U+_m0RapsW~LTKG>E9h{p`oqryBpdw@D=cNMl54dTC(YMqIc5;(&cBjTtB>H>hKHJVlx$h^-RKynE_qJ!EqWq?%zR z_VoPjsSs_`&{>Tu+j}&0Dl$9*-N|wqr)}zY)74l)-!>oiA9?0R(`Xpw1KdBTpWMUU zkX7}&vb6?OMyQHtWuvSWGeR?kl>{^~p3V?ClO4yd8ySYu!ioM?e?lDJ zzu!T3=um?n+38J*6!>^#aq-snViWw$wnsx}IT&T6f6Gx$a$Q@lXgAnYw|W!SOLmsV zn*q4lv*D>c%olb%<^>u6rI-f^E1y`_SH9zBqBTb3-Ak_Lx+lLc^sng>b+Ws}g>ael<+C`_DRW{tWHPKD zsF!Cy;b4aH$cVUCW^@BLTsK~a1uJXB!6mh&>-@P;!zf2c#~swF6qL-Xb$yK<=rSozd=6X0==mA-X{WnpEeg?mj zckbWc%)icQ?rD;wfsTXJyv)NdDIE#%uJq9(p@#0)aT|`|#>vCteG+E+qT~R9gNt72 zrS$tJA3zmt!q|J=7!1`as+_BvELn!Gdu=$bC9mUoOJ><-o>G3pv3_>e64PdKAjy?r zz<9X7Un?A=>(H}pu(vcvsl3LvJX`)NqKevbPxbl$+G3n2@S!H2>M2Px9XS-WlG++p zAcrt%%)UYb=n_dru%`!vD?PYxG;dC(sq+tTTnj`bL+gzAa6nri|Z9Z$Z_bH{CK?4l*lB`PNvwz8zjijuXc-NiS0W zHuEg#4v-i8*(Qo5Ctzrs0R}J5f*{)n+BDhtSf!bxb$-*iw=rOkUlHU)`$;IOj{tkdLiRg9CJYnMtsl> zrWxBwwLya0=Hd%vnZE%Y(z z8CSkp=SwMuo}^uyCCY{rpe7(Up>?_S)i@C-kW^c z>?@TM09K1JtYK^Jd1=`uwN zFSGWW7M?N5w>kf+(^GC;_K^kT^WyU9n42xRNtyWdqB`#*-Uk6ci;z1ps3;vpe6 zpWNl*I+e4hEyq9aEH>2eL1xbb+B5N{-)24DH@&Rxxg@Nqev}uL4{{wFWnW5NbdLW= zGJd}y`sA$ef(|~`H#Y&7ikF(WD+CVldVJkv`P0>0KnbsJrzJ*iVrEO7#1B{ zT?iX{rf+`$$XTgsUd{+?Ii0se&%$a|NW95g4#7nedTVBz5>a854|Hk5OWi96qv~n~ z2a}va~4QaLbA9L$A6+dgL549+LAO7P4%&&Fn z@#NgCpVfO7tg{kyd>+-&lX8oV(!I_e>ouLQ>>yq&1duRr+Jkut!9RK#>^|(({QPk8 zHTUGUtgnAUHC>aeZGm2+RV<#sTyiflJOL1tIHU$TUqs)gehF=x55F^mahb&?{bsl@ z@|pgj(xRz)uGA}U_R*)5(39bbx%8PzQ0EiIT770pIk8Z#8q(<%ukK@nOiz1O>kdEA zkAM66k~B_uUiIi%R*$xWmBasF0W|$3m^Cz9>}fD#R~3Z@u{VEU5mF4VL(3vZ1teoE zcHi=a*8FT4*OiL1AbeV=l@b@K-MUx^v&epL|5Ftz@=QQbL!vn1K7(g7CDa1yW#}*6 zYKenT&k~Nfj3R6onD?Q#LghY8_b7*Km^FP4|K_R@+ivgUXSo(uj#*GsYd{W$RZ>l6}1t?nHeS4|pXQ=kzp5R_l74@f6MUGpD(e)+x;I z0wec-FkL-Vz2Fd8Zs?@I(~WhSp~knm=bgk)^-~KC*Y6c~khYxgQMsft{RfH8W@b>> zL49dhkP2)DK)^bEntPXP!T>e8iZ;sTy!q}c!v6Wm;HlSzn4MilaufNU;oV$Nu6vhs z?)OmEH4_k1S1~xB=!8wu4lM7ie@#9y4nRr?S4Q%sTZ3|C{g}z_w;B@p{q&zn3eHHJ zaaCL!-ddj5kBzXLDb5u=QZ{wZTb)_)blq3(&=SWzdBBU^Q4i3RdkN_59J%LG0f~$P zPAoKGcyrPqZ-bSzktSWZu^n{-P<=X&IIX;w9STarHjLfCA3XgM&Gv`-Ktd8?AdO_7 z*={Xtg#?>X&rs7l-`~JV_EN6rISAS2XhUl?#A`s93ja+Fd3a55&Kdd$4Yd&lMSd# zaSZ0Y9PCpEkeG5iE#M-h8^iVa^9;XCcCs+fN+z@$d$+`zy0O={If|Ei;Jqu;@+zQB{*nd<HA9d6#&%HEzFhlBRQdH=Gpir_`DmQmn4- zsH>k#SLY*oW)piM=o#W$W@t3PP8zCb{pHC>>Gwf*ABzd%r^zJ~8m@a>zu%BQHz^*u zq6mqkQNzXrBg)50iJ|$7c+=+x`^K-(%Km0ZQ8~1vj-@u{jqh$EP`W`}^NA@>S{H}8 zSfKptMesG<_wlahYPL&ahqlCO7QzWXTh;<%Zf3tVt+;fXR6F}N^~0u1M!lw&p%KM5 zQ9C;hoRQy@IUr^i+>lp&_V=;fi;4S^W!eX=Wq-!2I|Dr%$$q?()o?D-=NS)2; zMG8+QvudbTenhu4{&nA4z*X9&XHebD5MqS%FZhjtrOyNe2tp*L=SSGN0Z|Y3;a~yB zEKU&e4vO6Vf`#`Fc%oeOvp{~neUP(ml}>!J zdmERcY&w(G-{n$Id144Bq%RR1vu&?pU3e0a@hR`Ds_ooEf*75}K1wGd*K+*!^h0#2 zLCuN6Xpcx2+?I0Im9;rp`|er?J60acpUO^|ejTb@P6`o(CUh1GB(}qk2(lKR5J5}kQf;|hkPrBUl z{Pey-rbt-7v{dS3`qE3dVY!}wMjEMdemnUW5L7%1r-fU`M0|(^`xki_S;Kzgc`u0n zv4|cf z|It+WV>+OJzLM0^@-f)M|LE05H&Pn8syBK*G&JCE;gRI-ccjq`)hIJt6jhRuavU0# zc`yg+$1+3VzuDBbe~k6^t!nUPyQO>5$!jS~g>tXCuaHczVwa`jja+0K*VgP6KiYDq$IR)iBR2q-<$O|=S(~0s5 ze=zg%>Y5!&$>L&!g>=Wvk=^8=rXGsOJa6WRte3$`I-yq>De#@CH$_IpoCf4_yJ%wq zW})IH2a`g86u}UcoB4$ude)^S!evH|)icOWx*@s>*~yRSS$pqdEUJH*2`y z0<6-wUN8sYVk?lI{@q!x1`vBIQj$dy^5}thXyZg5rROm%FNcql9DoYq!-Z6TC+U@c zOwvE`!e$~EL(C8}cR3Eb7%2D7r=<@*=_N;fg$oX4D}zLFgaHeA8V%{xRYOW;g0w3# zT?73dwy_!w6u|6%CdIT?{MwC8l=pK#o?$Y&gmzKNeC7PWWoGZqs^B-H7QYMa6IM1)4{n_Su3A&(xQWrissJASK|O>KD(zFN ztc0yjXSA(@9(+6sYkOj=(z+x+MmGA%`L-nI6|Wa+a-1}tIWwDjZ@8u6-8$OEi~W>~ zbT)x?<2|6u0tj~ypuY>BE`Kh3zMwlC6k(vON(}J`U>Fc=pV zigNV=f)u=UT@7pJT}GU?xWKe43No~8&H|#FyAs%;V97sSX1^9@I$d1Hnvr; ze`Y;M4qsE*>rv3?pVC|@2!wEF{~W?`OW=)q$vfjj!X1>7@Nk(CCG7G>ea4(aYMB2G zwnk}hm%8&$)mSmH#xp>?1kis58a)*x@r+$RRAqr84Vc=e2@)JE>xBaJL(~1IH2;=< zcyr5K4iBV!_AnPyRCAjhABFmS?!K!oLBL466}m}HlL&T9B?0=G)RvUwl^g>~l;JbZ z*paBOFvUcd5ZSwNa*cs#s;3ToWGC$VEDOLy6f4ESXyF9yv<4Tqrz0PxqmZsAGhdp{ zrKS?Y13v42ySOR!OVr;%xv}?&UE(NeES0; z&B@SqeS6JF2*3%Pzc&0yKfhiaf1-0ZAq0PcL7#kmN*-3XEeMes!Ysftk((6;l%SpX97z<#J7?Blp0}m;REGQZd@ivp zsudl2&q?|!fKf!ae5W=&P+1f9a7ZI%W(P^(j6CO$;wXNdjV+Zu^6AQG0el=$orb43 z-u>5r69TE>3W(+dv3DX7W^w!!j&np2d!cfGF$w@zQtKl>K-X?E30+RD%r7w8LKup%vKgX(`q5?u)&XgPdv$($BYYCxaYhKaXgs zh59ulD21PdI8&zy8{Cx*ncHYVhi2U7Aw?6%O4?n3wY9O5h+wv+Kx@Im$I3~0i5~;GT9KQo; zEs{dFToOBKvX;Ut>E+vcE8$ZV&`Eht%Cndy?L8WaS)bFlOjStTz>%rdhBvuLS73q! zwpWwN_FEdU2>eTqL`{#`Vq5ub~V$_}Q+=+*wGBp$nXy(-cQY&301lLmE zm4$|nZamk&C6gK<`Qh6V!V|SZX?^Tpt5v=7jBy`c!gU|W%=1}~it+bEzc?GDfo zjPG?sg-8w!oV=?t@Hh^2B9As)gH)6%pf!o}J_$1{mienHvrusWQ9m>Dywsp@4MMCf zc>`7|pgw!%&bSSLvEsM4ZeG2(_bs>=2zm$pj2uyr!~vr@Y!({TB&Cf|8)lodKvVF| z&y_7nNge(D*ekF#U>eGgA%Lw+uA!RiYq^WFaWn1yeDf91rE3QuoY}HT$Td1`05it5 z{_Au>`-yGeuq#h^;UUHl5LITI(%5%mL*PZ+1WWxu_^w zaJe|t!6BWy<}P4=&vkaht5BuDkW`JKgY zUE=~WyY&Jx2V?snP@3$i(zvM3cQn%6J~~8*wI64BFq~~vS(R8dWq)TAo+$}$owZJ- zDKgypJ=>x?b;@i`VKoahfAoXovumeu%gl6!_V<{HnxV{k<%FeU@4@U%58MI03M4{K zJ$BC=2Rm>cDV%C<z;xdEzIf2vtUBUZ+VtG$Mh$SRqK z_C||d!0F0zJXq()={jZZd}H4Uxhw}%xm)dUlU=^hpI4%Lk<0L3x;wA4hr2W0bc>>2 zxI_+Bf%$4>@HK;N*V}lRFTc(6HH4r5xZpM6NZr!cVUS!ji}}>_FDDHpqx={3HO0Syg-=c(Wvu#Rwj($h`d7!z;>| zX`~8B6)I*#I>t{A+u-Q!doIQJs@y7rgCPQYiBdW}_dB_~00G1OZ{lUb@~%cJq^Ya2 zJUmCcLbEw^^drr8#ShZQewn~{yN-VBokBug7aVSBVln5I->o+ec8u>NhD#ycgove2 z5Hb+a)>90lNQYbP5B}Bc-d%p!0H>@pA*SU(CO|Ioo#fyN@t7z~JP!346vZK~*I&2f z1{6)2S&R)ma80q=33~l?@7L;RtokyrdNIq%&f!MMfMCcKnmhWTYcxaIpIT@C_&7 z@q`I#nWZ1)lfnP1LU;(r!J<^n01e8E&!NIfk~EjRJ{v5h@b2W@EEa}8iU>XUc1=*YB40KvbCP53)I0+NLQ9iT zIdZ!rz%Ux+CKr7MNx1rA%lk;nfPFcov(6qq+z*&Y%cM>-6xA0YL!N3q*8*@oBYD&{ z4{lAg0t(|QCkxs)xFS_txXgOLk46H1x%&0(ct92Nhb&c4@s`9F->o1fXy$4{!nz?_ zQ|t?Id~&y!CpOqR}Pn(Ga+#DC^*ypw0SRk7Mn$m2_4#&4BF^FA9}gZgsQ!1 zjNH7!3~eN4v9o;BE*Y)uHA1Q32#&Q4b30quA2l`ivfd{G69$mQd`Gntt=W;|qW?K; zjbWyE$|sX)o7(=ma%Kq-5gr~)`RJf8QZq6j*S1p_8X!(p)^5xI0Z6q79~=isAgsVR zAgo1RIIKT#W3jIxGMmPkStQZKf@#WK`{hW-yno8ySO;bPK-uxQsgyQW3%Ti0J;D`v zi;K$r%5SkJr2t0DO{X*~(w62#U3#U^sy2-j7&do0VD0@#It%9$yEN^D;FXcUyNTF0 zdUkC(*II~mNODAB_=#bx^6 z`JEp3@-R`#y8#KyXZ$@r*3Ru~A(dFYxT{Lt_Xgj?q_GZbf=MkT)gr3-piuw|#h#u< zgIAENe&IkhYu1)k0XDUiozBF!6qfFmx;<9DqT=O|P_Il39m-CS)aco~b8ADlTau`u za%(y*03De-+uJxON-VQFJCrKxi`W>{v?XvrZohC^{a~?Xq-f?|euw|`tb{xNo9q$y zo)fz`2rQl)pzI2*El}OzmGf?C!{3XP(?AZSz^_Ik&M@l@Ht?So61B}U(jio(-A zuA8DJS;Ipv0GiD;%szYb!vb0=QLOeF#PInh252U;CRrmhTt?%wGufHL+1{Q+#s(9Q zGc|}EkgGCTLlBpE-NV0E-CCP}WdgBD(yN(o;SpK#US1aw8;^B=3~Hu_mX7Dnq!3lz z6I0WO_B75CtQiT3M~#w3JpZ|}9ZNIa92WW(G-l+HYs(3U7!|$D%=3*BXAl2?1XPh=n-6!~W20Ty?E3?bzjkO5$%7;N*zu zdoocp&heAiMH4Y_*JPS$Cx#pI8(E+wb(L)AyW#DVmui8KQw$%1Wuo}MTqJlW(i0xfoQ8E5xaUxAjw|HsyPTG7UWCV}bsQo_VA!uKd7JfmqQ{WwtY@hq*ud9EZz5mzwzt8;Nzm#&4zCF)e5UQ=)byJM( zC<*9ddHmie@?Nplt(r}bE{7||8F;?(pwk8!Cd%jCJ_F_T#W4-7l4sXv9hBbvy^49k zgqW$3z|YGiX!#wU)xw&fYh7Z&#fCp95a8BH={tO)7Nf~INAeM;P})O>khVJG4?iY% zEO+w^i_DLFMMlJaCcXJ(--fxfpXs~qG`_keS1~)o+pEw#>q?NXv%zlL?N7Jo<5$f{ElKvv=%5A4> z=S*6`bgB{d?I6j2c{KlXXuf9;Fcb~v69r!@qYn|I`BF$ z)v@{F=_Oo!X&NJC;MIKIUjV?avjWbCfEbeI9{5<3YxwRYm6 zMPMgU(D7i`N1NZ$yp6#%s9Si6%&~_%Avf;7F1hzE`JPQV?l}vkogQ^G=`7FPiA z1%=kH{SOvEveWLubDb+}dqQb|VI(aXmu^W{d(eb`h*ZAvx0Sa|8hEyykus`d9aKw` zGb4C%T@-#3j*87hFqqm>~xS1bdo`kBB`(%shGWlJ(63 z7syLrpoCVH4rISiFlVNi0Jzbf0SA9AQ9S)<{VMT9_O=9GX@N$=PKN)P-wjND7RwWt zY5S$li%1FytAgH%Q(ogf8Pkr-rMR$_f=m0*!OmdNf@Uot5~n9 zWYehdap5?2OC^L^1`pUu;=MEJPO3CjHbxrr2%EKaN58ZlJoiwuD7GikKe{JMyQ^YHO9V*1_PMvSrPhVjWHY(Ah41}6x_BORm3q;V0tv+AIHGCB8UhZBa~9n z(S)=1BUXeVYDD7&CrLr}p2(c|4SY`IN8KCre|$TXFOjl!KNkmqv^n232SQMCw!&)_gLzEDuKQgpZH3qxUdLsA)5!xoFe6kOUI@wG9+Nosn zk$Liykwe2s6S3UmXn1PlrLq^A9ue_@QU?|5)7k2a!J$XQewBVo>K z#lx+>*V1O{lcpREM%$q3&(o=DRXr$dwiO>qQ*C}VHs7NELAfQ^L(J=uc9da}v^n!~ z`@H_~mKGL-Kbx6mi!$?MpnR}EM94UzlEqRgMcb~J+_fuye9!#x;4O~F{P-G?tKSS! zK1RDYK8{3zcnP?Jkb(0jK97< z-{4zwDxp2xJI#B<6~9ZK@pUtY&zJ*7W3gq#^dF1{Z2x4+f(mWoGdxVWP^x#^Z45Wya$kvvUEbR(55<} za^n?>gi z@>UDZ2UG&oWE$sSzIV>l&m!vcZ{uBWU4BgbqWeX0HrrnRrZGXRt^BOB;KTP+Wc}1M zTk}!~1NPdz=BF;G8rkT|v*swh?#tQrl58~7Pj(w$8B)Y+Bmj18O(nx{=wqRwS02hA zUl@m&l_&SK)Te9GKPASU=6TgO+x4QXNxm$(Ic{(QnQxwGJVueEfP4}Y<>HDt$wI{fwYTH_T;)-mp46#F~# zPLHh8Yt~mtL-3w$HwE?{Z<=sxaXiSN<|Ir6FTQKQ10*f9054*9MBysvK}X z^+)F)pzDXU)*}61F=y$s#q5hc3Oo}!Q~5RSZGpzub4tN$4h=o&%O_*Ok#9Z}GF+5$ zm9a`5Y5D;i=lqjgE!GWAx|lm~(8G>|jR=i^N{8y(0R!FW6Db5TXnmyiE<5&EbxUDJ z272oRas{e)UrTNC$);s0JXn-&Y$YeqxXQ?F?Fafm#T7+QxC+7(4(VT>czv;xz)M@_ zCfMFOt*;eQo$fgBiOVQDlE=wPw^y+?qm#Zbb!Uegr85y;gkXL++~j3SI? z%gH*%LkHHqpZC_PL%!!ey3Ba3RXdf=B$%XNYw6+Op!oP(#ow+hDl+&1$xrivx2}*n z@B6zx8xW#{TP-)b^5cuyhIY9)E6WZ9=^W}g%L0q+kdy1V+?I%$$5PA>?Ulxkq6^T) ztfUplm^o11K#eVU4`)ZBGnZQ`QHV?gB_a%EUO6}LpIPk?kZtzp5S1zAerz0e( z@c{Cf)mdiPKsz}lvG+=3PQVo0Z5YBya$B^;vsILUSA&^BS>Qub8-f5Ac z!JLmV^+}JW!Bn{m4PRvuvc1I_FX?g3IxR^#9-YAJ>Xkmh)GcW?&tCfZ`rpR?hBHz+ z#CXvZQK$a;dgkhE-3gzXn7h!KgBZzj=DR<`7TAg6#5enH#F#GFNDRM}rx!;HV6z5a zCBqY(eRh7JPY!g7WA_!&Nm~luI(47voyq0RIxS}zu}V2-!(lUh(t~kWip_%J#6oxyIT9MjM?dArhnujsNL)@9$y!urpA(j} zU4F8;+kn%&SX$(jY_NyU?hBAgE6tX!u_?aMXvW6=u{0;6=+=fp+IT3-NH4PrIr%2Fb)D2rcZgy`RjT(VvFGxM?P+qcsWESrntM#MTA3&`@hK zU4(hYTcm++#%x{Vf!@CX^R@`8L@HQl7gQF`j6YBGdII4IK;HRCCqRu zO?+Zp4iTddMm1~EGc5e8y2%<4ucVR&EQC~RTRfy)yfrK#-Z^^JlKJ?fYkgAdE96qz zrbP*8+-Um}u{A8FNcg*98gb&FSGw@I-73#{N2`$_Lv3fO@^Imk%jP~Zz9tWcZ+^p- z#VFvi!|u;JEpT%iE}LF{Z)fRld`ZNvuw~YraYx-eK<4EVfI}JaksdCNHGLz;%mBsm zRg--$k@j`z+tx0ogCo0w2%%=`+b`2YLKhPCT8vDQyoY3<)OULQ#=k5DT{Vp1O&0p> z@)R!@lHO*zvh+pv!P{@C8=8;@4skvFwNd&gVx@My(`8>FoDfaLwE(2_#IU=gCZZBF z6tn|gy=WQg*2GIiXI~kUoFVZHt4CFTg-Jwl$* z9$K44lv2ODCBR@x@C{4k;paqU;w91s?##c;*%9XNNYR8rhA-I(cgl(}B_)*_vKHI{ zyk*+oF2xE8Tod-OXdSQLrB{32LL)vD)@1-yTvjj#ld^FQL(QE45w5kz9 z?V#okemqDuVEE{i+P1kk{qgb1gqOOH`=AnS(-GwFfBD{{t|K%ZhxvvUBf%`RaEts+ zC*=*ar9-qs0oS#$EI?Z&=u z3jAzoG2lCR1<#Z<{G$HrKz7ii*X?eVKKJaZMGz7#IM6WWb>>bY^4)S>mu8T=#{ad= zyTHJ@vPsNQ+kpwXXD>>r6}B)a|L^zsA1jotWsHjnT^6@&>920;ap2bi8qAsmC11ee z&W=YVqjQ~8u<+QErwcz1Z{AmXwY<_SZe`u*GTWmG0NfC#jlGIR%2)67ckSAofw|Nw z1EXlzUTLy?eyk=-^S{r3mz}h}@tisuy0IPWyLp76CM>8#fkPrMUg|mXFLDc#{Jnc9 z&UYm1lPY!l2R5!lC2givF|^#;Fo2}Ade7P+-W?79RW|OG6YIzmK(d$`<6n`%&ZqV) zIUD-+Lf;06KzwC+-13Pew2G?^V3p+kmyHkENPTR}6LK0s4|VLRQ9o+=@zRk!gEhBN z^PD`#)sAKe)93r>(?kCjfl0pE!{tPcwm;k{!!mq@P0o0QHh~Q#B3ISFa-VHG-><{T zPuN>y0j5~pDdM{9%v_Yc?l_)5+p3I9`n<^K#667h8=k3_NH~;4%?Xr>BEKN3Xr^g^*nXboQ zUEz~Oj)I{Pe=tTtLA+5KX;~ZdxZPW?$G1ji$k)1SBHiXdLbpD6A)P2fxQz^6&#Jy< zG<=O|DzNbI(aTIJLW8TFmeQ@#y~(tW&&=Sv8;XmtrYlNE>i6}ElpwDGB6!8#amiWs z|9wqz!N0zDYt6Bbx9+4yZqbp&avd3;EFT{+sJ;hamo}w?mG5xsBgM+2elD z$iSCWWm_X?SM=I!h2&&e;!^NrZJ9@fa3+0;%e}P8$43<*e*ItDDz?j>Eo8R1>t%DE zNVz+HRDvK0a)Y@Cm=rp|&#;WcZZSZQ^Iy0IuL|R5cj`ibOB>p{r-+_(Hm7r)RS)i2>ZosCXNY{%(pGwYAuN(ReUX+obGd7E=eA&m{S z(mv#v-O`88O8X==K{kQk0w|;z>IU7Qp&HaqBiIZ*B1u*l=F0a72 zN>#$;n9(?Lb|9uy%xIWkn!?GsvM0)$BG~)HQD<^U|Df5u7(QwmaEeX8t>Z@xdXaA)0@%ti$v7FHnZkmk$aL>zFzlF?}1X3ypKM ze>nSe9yRqGF` z53egV2@co3bIGR{%55wuWK^hxu8$z(I3EW2c#XZ;frchnZFi5cQRRW5L)Hls^(o_BetATA3A6P2aHn1dw-?pXWpV%akc$R(c;5K8c}oAM6&4p5e>N21r^HYLXRw9mhkIug;;p!84YCj&H#>_ zZ05-R-@2dbxw~IHh zYPXrqrSrl%mO`#Gr5|uY`i)ME)gsR{QHw|XDXOVB>(JS?_yob08Nw;BrcqiQc0S%3fm z*f37+wm@gk5DG*)x8H}SuZcP>V_z%pI0J;cK*}VAlUkDiMN9}{W6#ZmG25*BCQ zVd+}wx?RrnXP%%G3mNf~ZSSxhR9%(5uY5Iu7em5*)55yu#YBK$A7Uo}v{xhf`y5(RZRl@Dx_W;-Btd?kVGLHcQ3NCW8vVV(qxL%hdq2F5Xjr z4P8R#$X4^iFav`#%NDf*RQC7+`I`zDXL}gD5<1K`R}K+Q_E~lPqD2b@8j-u{=E&l- zAq+=f$8>Eg{R{0x>L>boFf#H|mxgNSA{;`&=b;5vGC(na4jAT$xHNa>U=RY_;fCB^ zem1&tEO(k+eKvwI0{;2z4ZuH-tr_%-hXa=@%kKu3o>>DzzH@C|-ClpCDa6^n(xPl= z#F|~r^0lfUFw*cq5-a%rRgwP@gGFWCQkeKTw{vcWORT@in?D;|n%j!OC+OqpGEndb zk%8r!z$~ZM^c~z#rJXs<$uNU}f9CO&)yFcrv6_bYeaatv2I)1_bKY?R>(ZcRFF1Sw zdJw05UVoyONM1ep;VlLVjE{VU@TJMk)$2p0tz&H(8&NT-ZFV(nz@G~ampyQK7M^UIGi;|AftM-L7r zGBA9Dy-ip9)T;QCs(h76fP&zIJKDPM`zvO5)-UPbb})p2KzKgYk&+$GU<7A->4s)g z--mez3inYSH)uiQX05p7;7zquvQ@RF=pnVoiLMoF=Qdw=XqC$#f&;4QgdSsVRdRE6 z8XkJAO-yAH9o24LW6ibHOHJP&*{X^>eXk!He!f>8;Jq?Wuiz|C7#dOOw`cXKLO7Py z#*&_oA8-597E0CSon0=^-9YDWb+ulai-${1$!;{Wqat@;WXx+G%aT718!2D_Cz5#Q zciHT~wLZLlD>&NMJ6_1*`x}fb@8^$xt~D7 za`zn^hSPMJmCO)WhKPr5QETQoklQaW9Pm8+7=X(gIblqCQ7 zNx@V0_Al|!wIRuu3Y9+fr%2<~2$QUz9p|D9D7+Mpcv6d4-5VX}<-iJ>RsHC*N%#He zi>&z@L?hkU?z}e+hB|JX`Dk2+=ISbX62qFw*Q)|Y z=UAvSP&{)-(1MHUA*%|hG0Ad^?uoqf4)2?uoV4UVcwXvWWSsA@V~6&yEEhP3^IEcy zVGY{aUb64KCY!Bl=uUk7F_wa$c!BHMT2u9y^bJ`7>V_nO%0ny&O-XJ=U1xlN^-4>n zTe#IVc`g`Cm-j?NdkaZBcdYpCM^(C&%ql;2S-+W}@aCaDMjrLqdDL$VK@!d1NeuAH zv$YD@Y7Fa#8+hk}8o+;O)1PxIbN2MrK1Ifl_*5uQ6OL1$AGxu^vNLM?UK;At_iEf3 zy~s5MK75~th+9O%{gy=?II*qo=Wdfqm7ARa5HjUpm)UnyXLX|oZx6P5tpGHDw_pDq z(RG1_*hUXgdEa4F>xtNN=n~Uu+csGoB073-qZ7L~scTkQlXOjbHj$Va|G^&`UA|kbn*VEM6v%b?eTPrh;cKbqwne_WudQ-NdN{0KpCVsG}%u<&B@Hu`` zDc5ln(3(h5l30>$Bp+)-(whe8=8)uU$OVEE*!5x4LI`W z{BBR6xJe~Ht;v|{&*TAE8#!|tW5Og){oPF9DS8S0KF{7Hn7f%K9D4}C*>m49$A@$5 z&#oMm)69(?mw2wCEwR34WfuEAg?qUsqsc#=n`qE)o!d8sJtA{;9(z5+!w06W79(O= z2f7d~ws-NldQnWl18|n!vq?M=gV65v4=Ki*l)GOf-vx0{zCz5CZ$8ih^fp(eij*d$ z%~L}U(4+Bf^>RHNFHu=Hj&V@33?lRA_bgwax8J93^(V-1OBN)NA=74e5)`F_O^<~+ zy!M*Pp2k0nP`Wb5mm^v~-JAR=PCQR5yS&c)oEtL7b|bmNu4G?^`bSxjM)X5&5)&5q zlS0%2)3^kSa+5+Tji?WNzI=BPn7;wI?*BAW`eBOX0msxKdd1Mva*!1KW6fpi2tabj zox8rO8lYSm|FwED^sv)O{!WWwU85y895l)?JMZo6HI;;}Sv|HbYF1t>88)FAeWT#) zF^d-T-{XB9F>j2Muk;XqdKwgQ7k;NJ{QODif#!jSX&*knK}=sq)TOd_ke_z0cPyny z*WmkX{+f;TbJ=H1!5x4iDx7P1yqyGUwja6?^DS%l!w<-AObS!w( zvKWS4)s`ptU#+7Gp0Axu{#F&-%Z2j8gX*SdM&*2mZ2{p%rDkQVwA=|Pkk^D59^d76 zX1g?ZX|C|C#s&SS&HyiRHV6S*7a$j419rg&(WMH5b63*2ZWU*)RQY-~;f zwR(Ez*@QOJ5{pl`(_)qmq*B~+;b$Zu21(Ybe10nqvM((hof^1aj@(_!>IUPG_923lmr*S<~k zO3<7wSLTM?mKGKCvAp)mJ=^YR-b>xl-m;xS8CfzrtkQF`w*ZcAtYyq9hueHfP*b%h z))skz_WV37>E9N=s^{Js0P3!H=)05J%1!j3L2}rTx&x_p#!ai6L@I@Xdc-e3fiwjx zLLw#0Z4&8sZ%@L%z-N!9fLGKi>3bK$DTL$qe(e?0tweSm+&-)i&KZSg9G$Rq)4pkK<_v<$PLI z9DIrYi%SoC1L!fG>8R&IQ${f|4jp_~sOM@xf?DU4kutU$PBuS-C^yQcx+VmYcttb$*kbxIRTCwMVj)zjMlAsM0n*Hqvq5 z!Am7FGsK<=CO1CTdufgz7zo{G2OF=H4dt#)A93k)C0hT^QmZgt>0@l0%LX{dS^Ydl}~{{ZSN_`Nlgers|FqY=hTJGDp{ zCQvaRPf8BH6N2T-khA!}*lE^7J*S5fi2n=>R`*2_fni$apa8YA^}?IO!=w>_A?f>*V9wb1R`G^K;G6&vTqFq?#|rDZygWw{tW$(G@?(FyGKBB zCms%jKw&w|xbOHu_&B~_9@So@zyG|(a-rCNzhXFiI~dugY0~;|t8!jBZ8%HD_74A` zW+eqkKu@4E)X8&XofMDLOChPYYr9p`c(Ao6I5IDT_R1@0f2xlrqgCNS-@nm@g1HIX5I3tgvcN;YJ;QhrvZS1elwqI6fxdM!rdN zAHwVmxctpeGpOxrb6pT&6XFz-{rdpd=6d%_vV0MLT|w1C@e}X;*`<<6BdGv|Y`VF- z6t0@E!lqh{RBq#J-!w3N3EHpU(n3L|(VBMO-WhSdN%IvKDv-#ROFb1k15DP?1RTBD z-yJ>p)X{%AJ+9n@-6eeOXNgGptsY$L?Q;+G52^gth|`bu^6HM0y7RE12EM~%EXZya zF4(as*0=t_h{S&(UDy7FTu}##}uuZ)oiZT5?^^C+?E* z=eQE6XH#gwYxX|4ciTh-8Uv`XC(<92-r~-@Xj5&H%FVgIF@g!3imp_QKks(Yr4T;d z*#(;0^c4`7(x2&Ny0axArLyGm-v8tS$APdNU+2~|ijHu;BiD*ZUN|t0{T-fztFwlb z6#9w;x@H1Q$j_HE1iEy_fR#rQej+CQcc+_$y#1#M2Il#pk@Ck?I?QUjaqb^wmJ<;o zaz#ac?Sc+&;3`DzB9Vh0!4-B>5Y`mXNH!In$l~s=q40usqV|lalpkP_JE1$bFmlS%v6AMLe zpXCmkisJ9?A+5gAb+FvNR`rYttX~76?i7I)a%21OT+2S0=iTUT_RVqB3?3J{?eKuL zX@@o72aCnL5h}oJ*F3EL#z_8o*_NJ-5et9$XPWCTbepep^F$;lSS~Gv#iQbx695-0qKi znHS&Hog7nLL*UxY57f;qk7A4bH0Q+wW%TDb#`2H~CzbvOo~zqfpAOy<-&OB%LIk6< z&SDbpU)LZXIZ$j)Qk!R=_l5EpH6?vFiXP?2AFS@`jWgji&V|ZS&yl@PlQbeWw@ek? zt8rLQdt}+)vEOZaGhHkYQ2Iv(+Lsn{G-LO+Dd(;1!X1yRa(sQSZkTcPJS2sNs8V$p z1294h%|ivS3vXgT+XCju_jp&BMeY8GVwz|NNUG+`QR=`P)BS07$H~A?T+rvfZgeTL zAysFgR>u3us;OkCIDG^epnVhQX-k$i7pEIXSxVrlL58et7&2?nMY+TmPZS+ z%0>Qw+^RrBFCI+?wJ1iYqyl8h_}cK|!y+~C@u9E-nmE8(i)JorlX99(pQ`jo zvKHDHA6@Mb*z|gz(h=67NN7lYh8eM>Nj`xTaT*|Eo;zU+-}7|r0X*zi9*0Z??dddz zvI{Irf(_?2HG*dRT$~IgTwrOqp$9F53<>D|;4I1f@SjZoYL9h&}XOQPJ^IT+^J`m{CHaIbeB7sI5|$&5jdH`9OPSaIPbSt zUPJVR3QA@k?c)({PuCf*NIU!piFG@=Q`XnzSKX{qP# znCP6_hNJ}z{4oc7*4T;6n*0)Yuv>o?Tt9(355L15!HFD`TU0kXMq3k&R!A6ocZAQFiPf)&x!!?HwIxdFC>HDtpMi{hXYV@b^lsvmVu48UaWr^gZR0_6Zp_xL zbU+)yn0f=LU?-rvc|WVgkJYE0D{Q*^lg(0KH#{XCEMu8YaiEd;uZr29 z=L|k_b3*ML(kQ6S`MYzGiJ)-3;QX4x?S<4nsF_I5V!;;WekAAjYXUH<01R!iKPeHgd#R^oDd>r?Q}u{gy<1xspV z!OZerpNKCM$?ZHUA7oAe+M~ru72xNO5!KMut<>_Cj2;V%F&pR;QB#Yu2s zq~L*Kee-58{RtaJ>%q`btHES=`pmJk=>t52b6upEQc=~oiKl0$b|88c7>Z(i&Ip% z;b`YcU%Lf=8F?d9ahdCYm3C^K@ZNRo`T4E1k8)>|08i>;5zM^yNF*vPiTe9@Ip+BF23dhaFtN|3zx^{31+_k!=(c=)vwA%AyY65xp{7J)xSl7({TL zhA;<@c}Uz*j3KvzL~JjwG(k>}RO9_WuMFA{iw_6ye*vL0)=*2Ebm-GE^qjB!MpvZa z7mZPIB+`?=^JdViwBc5#s;<3@^h8W7f0+FR2i5jR9kYvYutlsY7CE5bsv>N5-^KgM zXyf_^y~Mnswd`c+a31%XyRwT$-P4fUH8QJH6&%$Q-J~AY)%7HSZAf2q|8mzpzOYI5 z*V#{h@bl~|0%w17sa9_7GqJKH`bZSP?<0V4KLUqgXL^dzh15?MFnvm?4{NvG>n!Gs za?CYG_XWiReAS#SEsBo%fFW(6kNdY} zWo;+1TSM)55Ps@fly(}3W`64##F+=n+UbIAlx4VZ>y6xLETl;U=um9O;(`1cYkG3` zKJH<-)<^Ij8Df`xYUlYW|4@wbt`iS2O(BtL@E$wB+-)M9>i!H*2E;nf+Mjb1hn(*1 zKmKEdLeL9sactd9yqD(t&NpqAV?q@nO`R2N?cBH0@_Dw}I`b37?Qrl*Gcr>su88+_ zJ}}zChpr#uq?^GwD~X@cg?8o-qEXkor)tVU$kDn3+$kC7;J1+tMyQxHCHIR8)+`(y z#uRQVyW}M)WbaOAJMp_|M9ORgdCupG0>=~(VLtc~ApN_N0=>=IkdRF_{dp;oe(My+ zhwC47`poM0f6Vs{kdoFtcGxBl)lto1P-%rrXrCjb<%3KypCcaNQXYJ?EJjqU{5&M{ zKM#rVuL|fX7g|cV-#1MqfA~qqLVow7sVkx>4}PdOMcAzc)i&??K7d)ID2(48-B{T@ z1OX_*-?Hk*vb0y(Mv^d4^giDG;KbIZ z*b!`8A232kEnV=QnSoPM3#mVbDZ zEm!X&tPED7G9Dp`Sd=+#^4*CQ2n!W@xU+oG*$2!|Dy0OJ{6@W3aTH7}vIvW8WRfU} zlpKj(!?oX0@6X-OKG)A|9ku{P70pHu74BB_G}h)=3u+6+GkBObfE;=(+yYcDM>$Cu zfAgcq3$iV18qe0*!;EF|K9S}y4&toZUj_tqBS0Ca+GT9^W9AwFtG#b~1#OEk)INXfi~yCqoCmAFt=0Uboa6g;uqzcblO zSV8Mc%>=k@!6r{$LTi3+Ax^H96Ab&r)0adSnf=_wDUWd1IGKJ{ThtQ}FErWwdWHY8 zrV@m|(!A@z>dA4da?y#l=csv5!YPlvF3cqF;yUegNuR&of$HH8K7yuYmb^h zoq)WU5d6Cm0N_Uk=oYr3incD)=*pXaFr{k>A=nc}xGZWuujhQIX!O$D>v)(eI7_la zda444H8%hA&F$}*8;{@WSQR{+EP7Vw=kL<)JsHd$W^uCR?DdfK=`-|;p5=L*T+iMQ z;qw4uSYWAX{3-Eb8$R`Gx?p113S9g5oBh#eO?Rl{YpaVp+S&q!=CK9b{o(09gl#Cm zV&h+>9%sx25(j}t$Xr-Rh^2Wt2o@Y3b|p?^{fv~r zI(UF}oEE@kT-Y!_ivuSA)i!fN5582-a~g<(Xet^SUfevWYcCfPyxw2VKJJZFLaDtM zdiOUrqvqydaL(2vHJ2FC_`mPRm{>WiDdF49+efviC6y-gYlzXvT z9gj?*GuVMzzi=QT6jZI~!;pdJ!)o&eOTI+F@=&!sbM{958 zHw*W^I~3s05{=6K1zYx6f$8;rj?$K^ndi~!Mn8Gcq!*IDZXuS;d++N&w_5Warj;$?r>V00a3$?l?9ejqdcH#rz}@Me6WXB`Zwms`ulj#eO!gSjf8)y(7rS`BOLJ; zN3ItJ+c&sH1%A)nx}0oUGm$v4Un=W^y7$XBCjT=E&qCzf_mf@V{)dU#*%&Tzmzd>y zEXP6m9!`7BP4@b){f^ZP8CJ1JJN}|O2d8+Z(>kw9E$LR5smmB3; z7B2m}wMSS)RX^%e*79y}OnI)HAlv35N9WCJFz=m#HI9=J^I*}m;R;}ryu_Ek39p$nANTBj5!y~>^oz&mX(XP z*1l5NRs>>cJpH=NOvZs|(Qk^v8^@$n)Gi`$I;jue9deOdShJIBi;VR@Dqs+-nV=o3 z8}lyGIi&$x#MZxZz>kvg46C4ZqglxB8+QKqxA5}WsO|e9)i&vm8)sGE;fg}~E7cM4 zqc5yx*O?^!g4frNRu0B2&l_q35GrhErsSb*M$uMz(NQQcNPVBXGS*cClGnLfz3AjX zExRf=;zsa3k9KAP+QrKsi!Xaz8qZyx44ggkFh)ztW6z338rHP zb}PTv(EnTgtue0sXYi*acwl8M8F*k?ccXdgLqn60-WGoFrCZ=Wbr*ya8g>3f z2)=Hu0X23|PsYN>x3`?uwZ~&`G*cfQPO$j4JhQ~?_jywnt+yDSq;)yS9Pdt9WLFvu2}cECPL6#Xzn-cn z&_LZoByNynjF1oMxJ|vP6j{>&ocqmKLrA3DevYB_0U$Wn89cuG|FaTcL$uoUgaZs1 zZud)y(P>1-^qRdO*qIy(GK};DW5OBR~xPmGm$C_`jULPWpvk^)?~r zuRNUxp$>jZMc=niiyTRSBbMK-rwW7bVGroy`-Pc5HRWK4~-v>jx#LxaLaEsjXf;%OI$Bz1n85} z(AbPE$4dWxFlrPy*!QI|kFU%D!pbow#L3-{$;UgdDaJ6bR0>vs@F)GhTK5;^0<8NK zcC21^D3$ID-JN)g9zQsuqL;;E4puz<>UCv+Xtd7{Y@XBDP}@Fwpe|bo{P&gij@q5b z6F0+;hklEuGaxYSDFISfUo-E-_w)|8g;WwEvZZaCD!ELAIiV1V?=m}6n&r9x|HloO!@ZG+> z0|knFukVJT3FhL)!w6fmo##^yv2kKIq_shoP}I5~dfGq=&@~IOt0M-?gAV1B7|MOS>rB(M&UpimFiO+3bOD6f7-=?QL>t~(j zRV8=!e{$r2dhMLhrC)!iKpVgqz?A$r|G&K7XV`prF+;OP?A>Iq!W|LGPg{s|sIzVE~l)?677yRI= zs2F*Q@6kzPA8fJZ;;TZ`%@|3X)gS-El8RZez-H16Or@A#+NZznKSK3~fTSxwn>tN0 zN8`f`N$Y9`%Fqi2JM}0(NiWlg{#tp4g7B zJTk306)FR*Fw-|r9j?lYbllZW%uBlvY+>D&9xs`xL25kr+0bT5=_eXSwu8BwfotG)h;Jd)&gAf$&uxIc zJ89={7CBEOUNeZYcq4)R`t_S)tBD?e&)el>_WzVnx7eme+?4sW7oIWu*HvE=BY~i>MZr>^ZW=X4k+&38VLFKgQd`SAq?}NCE_W$a9c^c)&;zap+EZKJ19D+|+JeJ`j}k?*Mw?1f98fV_(Bk6PS!fTcKZjYn@!R#1 z3>4oq8~jMd{F^$$p-j03qi6`UA=^|JKB#2HI;w$nd_3<5@#Q*kbrZt!KD?Vg7CrwT zN2!7yxcK@#YZW(d1S<`}R8uvMx2IB>GH<%*7{zC$t5q!g2OAS2Wo?lBLYV%$S5ct3 z+rxj<gcH3r{Bh$%8Q zN&}w&GKxDh)z&_$EWB-TFK_U4w;S>Cbo2zG?t5X?taL9UoA5LXO2C6wFJjsMBMQB5 zc&WLrmqj1quRSFuqPQJ6m80BT6Zh7ADktb!8+)(0+L%bDqc1S~I{G`3ds?;WrHs!M z`0VLKCWWu|l*%Tl!}&;e*7nkGLHS>9nTF*KE7Zs_GfPWcfGNQ3SH!|@8Vjb+VBXCw3(>wyJ6OA;NHErs!e!e zA7v8ai`-a_1WYw|?)a_EuEkr8z5@1fJc&(E~?qu9DS+GZmi@ij*)cjpAyn zxvSwQO0_Q59atTqfvfRUNUJ&F3;2&6YkXlk3Oe}2?nzF%jpQ56LfpMLLw4FwCa_qy z&u&pAG-PW*(hct2&X|6U34vNmptulb+*aRwev@Y!QoP*_w48fn+UdeeFTM75k0A~T zbyMAxG6$4 zKRDP$xPtsLXcCFAk=1763GVnN$)b^3dZOa+DO;)a(P3xo+^o9b+U0osLdcHHhKy-) zYX++D24bvV{JeQFFgI$S&D7xA^msZCro5!Pbj$7b;iqh;(it_C~;;##ftdbvGe46 z1^PDEUO(rzu}F&-y>lG_Jc;1R{wGdssaF@x?6!5)x=Anjx-4)*1lz7LpwY9cdG&)O z+mAoi4bMB+)MQjfBRp}!d*A(|NlMJ@dX93MgImN8Yls;*9>OcD*=}+qYvu>v9pje2$p7{{9-tze`21>PF+#ICqqs zwR~NRNI0gI33bOX|$XzQbYSiwwbs2mF?LLGtLehzDx>m z^wjOIW?92D^HNsG=WU-P>H1vqnJbQ%`fkLOR;PS@JZ$;rK-BvR#s>_um_x<=h@0Nk zgFV6)1zQ*0weMGt8kPys7PY+fVMfGPhV|PmoA}A%E^I)jGs=;!T4Xf;n(r-AXhO+#b$C;z%?-iw zp~F_MD0Kp#P9ZnOyZ?#n-LRWOWOnav;L{g1!*{;C^G9HweyNRT9VY9Ww*rrSGg1k1 zv|3gSqL#h1d}phA;H+FNhLeBNowK^5glhjT(2c1SCE%LVnwlVCXqaeAzV_zM^yLL( zHd&N1R5)0Gfr8pfS?Nh>nD9sghH6Uoncj-0yM*mm%*me}IlUH4AOu}I`8_m0rEnvB(ih0AMiS?j0e)K!cL7G}x#c6}S1dT@;Vh^x_iv`0A&*fpHpPV%jyr$5W7 zLHJ`{@!tTN6$oy2PW=H=s5hl1S!5eB(Q_Lf?NOINGZ5VnC9KV3s}%cE@-ud9=KW8l z2JctvFw9$p8W-%eTotj)Wyjjkh-~Kh^*Lx}x4M?E-FA#h8Itq98>W!jnugdNyXEVg zti;hBGaicJf(f*^Zy()J+BdpFgHfdlgkDN9D!W$?jO@L>*>B`*ntNCL^1`}Nz2rrDAtG2TDk!hDcw$}705$msxD}qGU zZ^`v-0T+(5R$h7!iGFa_MftGwlMw^F1%IKP1}YTidb zSFrkPkh+_3(oP0F4Fc2*ri1h&<5a3OY@5DlpKrFvMvVb+L#pcCSt;5&4U>&Ui@a$` z`S}3NT2&GD8UelTkwOk}#2H%4(%8Fuc%A4YvkwBalx)PZ z=(3TFiSRFCh?$ErvQ3lCH_$G79k0IWh7!=~R@17w`YkX)Ra>j`gXWA7ic!gj=&eN` zBvArKGYh+f;HS4QhPOMIWh@$|tL$|%Y0og7nQWDL53&i*wFxgWK47Y2>03Y6Ydq?h zKGdu4GkBu4bNFYX?9nd$4IZo}iC(aXmbww@{WP7;hpmw9wAlKFw2kjqVnf{2=!3y2 zK7(}@$lM?g>=V}6Wb%ywE7dGjT>RX#nU(!x0H$8Mm+&g;%G^vd_@E#s#zC;vM#@qj zNoSrGXrLb-a+mMsOlcql{{0V{SMDh>Q3Bnes+-m1=b7wuk&Hs`@K-i?rZZ$B2d?VL z+8P{ly@ps^b+Tbv2)^-@qn=`O&V?d+)dhPsUdeJ-ih7PPHf@9mRY=yp_{24zSMhqg zQZBPTl5Cv9w7!W0x_M;n4mEi#CEvl(7-O^()RswKwJ3hGrH`?N>xH#pN>VU5;TDK= zH=0FLjw`OzYHh%nby64b#G@J+q13VtW}5I&H<(4NqIOIm7xhqZSbG&qc(n#j@-_%> z^FE<<$iLZbr!Mb4gh3}fF}~J0plDyAzJO)O1KMC@2U)iKS-X*xE{(SLpSS0!Zq6rF1p!O%q%76 z`JBuLjGHXT3{R;#rK*|Fk4A zO+%!|zc5pfp})~yu-O%Ru0*Ukw5|@?ZZ{e?2+T>9Tfe&$__@Qztq`~sS;yF^5$_@u-8p%vf0-`a+mKv#^eY>r+evag}MZ^ogZ zwGjK`TCBF~sFy=YpAP87_!?;ESOyqcFz#L7z%zZq^-Js-DKE6_myl1Rj?5@ zdY&f=&e^p`;9lK`63a@|`-ruF^IF*8o`Q5#Y>3v4AtqgWA+?V!2j=Tj#9$?Cyx!qd zy!xNW8T$GK0uw<6{S_F)^fk|xy4YbN&5SyP^m`T1IQb69tPp*!#TZH9)|69%Zq#)L|JV zXO}(c({y;*$=}1q!qCQ0VP{zSE~TfSMe>x`#{I1j6tB*QVMb4lrnoZu%|q(Zt)mM7 ze1R3Sey>;6OO0p6mYV7=R;KqyW~fF|%;h|}C4d=5v3W6Be3;kAwMNJejC>bcU!N!|YPmo7BLUlCIh>`xapxjFalc5G$$;4)`z zJlQW3j_W3^=w1i;JHMgWmJ*!`#VAlc=}MwI#!l~jxju`w)5;=58Dui6dT@0SM+p>` zc91|no%OC7lS*2;6uRqvlft5HgC;^xM0W6vxd(r%e~0J|J|-dhgUJmu0OoUbK7%~uw0ipXka1h3M4WH#)ZvMbAfB@}UmK1PGoJ1(e#JGFfq zr9G^e+RA)-i0r#Kk8GOX&-a zTr;WxaqO)WLAmowouIkBXXzo%Oj@xz6|SftV& zWGNPv5sfOqGbl)F0(i{fO$|04t^RNZi_eWSd((isU?sdB6&kp2_ ztjymnX&}j>Wlbz)_Y=uCRr6IuQX)yBA!ekU~UODI;4yoeY;KVHd2nIZcwu2P<5-^q@jPoTe9u z##dc$qr!rmvLiR?xuTp_{cKO*;vsFZ0Mw6)u)MbZ_8;T@*?TbNtb0$h4j#99P*?Ge zqLUTssvhX`p(`|DaCI~CyS38K5(-%y&_`p%Ti~%S-~&u4B`Xs4lBJ-|4^pt4Te^Ty zk6{v-v0;VTUH>lj8d57J!6dIq^5_oRPV>-J`8fPG2eUJq9ptnd&$NqYL@@$S1Kct$ z&vV|biT~R&h1!tY9@Uk3mQH*ohnQ%Nxy&IEoK4i z3n{c86ae9YDJ*dswz`4Amr-7n7)nL8OFXDnm~9Z5FS({hw?Zl zJwi|q)e!>F@;PRaNF4DzbMVd&m$V1A!Z(BSwW!xK>37{LjXw#CjmgPC_?t*;2JFpl z!`M;JUz))k43pI}ARKM%m&YirMuh_K4Vl%tS-eWXOt+pb+HjN(C)D-1zM9+C{+tL* zAnfW{E1f6LMd*5=R!EJuq3BZMNi(@ScKwN}p8V~lE0EhOo=U5b^hPuNyG)LwqbG8& z{+SBDXYGuMF2W7Hqu68=2~q3>-sjgxt00hVA3W zC+>h_oGy}NVe-BVbg>AuzK>7%VrFUKw8}yr=?rmlX1yalKDOxr^a`gBo3$D_*JwbP zm#LV>xA)jBYEZgH$)JL;mA11qbT@Jq1MDZ?3M9q0x>3|>_CFg@ix7QEJAa!)YN%=o z43hJNof6R2ecEv{+UMQ2602|byBO$SDfyA9%7JLGw=BM|XD-~*FCG!h@R+Bz+Mvwfgpsz)7I4V? z2Y^Pv%)03&+MaXuGq($O&#_t55uG1pjQ{q-_e+z;w>s{X_}!}5`jdIf?Q;4k1*(9e z`-mt#gC|(J*-K zsi}YWRx(TT6lu~MG?@=p6Oa3r+}j2EyB2^;P@;6EY^=)uVF%Z$=-^JuW;2H61LHmY z#{@rabGlT;iZ|gXwwN;^=iGzb4vB*0+5XNU&>KK3R8uK*kxu*rj;eWZwr)~r_1gUy ziw0iY7C_cdisYqKzYeh%g%wIDx8L`crZ(4x<55{7m6SMq$vait<8BeKGGrzY<)~w! zn8C{leGH2z?;IT;zubH$eOr<~;k;YV0%5lrzfiIK{rvN!ZZiN6vC3Yh>5O8W7_K23 zpW=j;!Ch*ZM7iM-kGjoen;!KbRNbj zfN%&9r;TnZ;YqAK0w!TAe2*5NEJ1aQ?Z1M%G#{-3)?{KIXkVVc>)>e+&TMw?FysC7H_md!$jDqf8hUGL^xOw) zHe^p;_1?PvF_^*CK$6Zj5Q=F3=55YV?bm=lJ}M0J9=joASf~dig-3r5sXq$$WiHJP z1A`B8%ge;hd@!{j)vX;g+5kW%xv?A##r9G8*qPOVeTck=AiVh|dZ~5Ohe-CGnbr(w zrljJy3{j?4bAK(Cz0+N(IzrKT5QI0=axcuycgPYR-AQ*6N$IGeO@P0u+4MQ9v5(8D z{~#*cvvchgo;D#_R8({o2-8^_L&03D%}<_wVp(6eh9Vau9lDd#&NCr+*9NICJO>22 zr!7Npi}jeXYV)E6GtSTH{k=%{lvf6|d>Pa!d3+bx1iF;g89{Tk^zlcQW--Y$LpL1^ zxnD~PDkt#ihi=-lloR`X$Mk4gEDo~ zhPN?SCvi?C)sp92B+uTq_;h-OUJAkk*Qd#oNEKWd(S^1alQ*NY^a zaS(A4Fi2_*gy=?iRxYbvL_Gt&hH~`Gu%#3!Q^k=QZ5DBQtPId6C+}WDJm*Aw;FvYJ z-54^Z+bQ{y!Sq%m+x)KejaM5a)HGVk*sea0)@5?GQ5O;7sZ zYIa%<)~l9s^G#!!>apGSPoZPOb1d82f{2ZX!W0Ud-Kaj%W505SR&q#36Z9_7c?c@E z$i_W6NJr`+5?3HK@mgNqH|S!zfI-IXmzl^~FE@OQ;6_`9jg(p7%{+ zlrDeYJtT;}T1Zn{-VzbLb&s{Qcc}|@nGRwwZs^>poxJNCsqYqp?>OaD^VFnB{^#`~ zk|_-f6&etqrUJGQ`C@W?=$^BtZ8w8ZJj8335p=_r0~-y4m2U|%jGHpd+A=ejkxA;vY!LO-xJ4dCgpw z#P<)}+a)g##h2UpqPPlmwcpnZQ<;M4mdj?@l5TSuQak<_NsSPg>zfKAV6J<5|Mf0A za8Hp-UN2l3qMNd>SC3h`=D^&>{(bu>rdzVDK;q`f(o-2j??9G${A1x5b9seFA9cX& zlSw*=qY_Bag0qMz-UIV{h)WB^m;UJe>hY9k!133wia~>`;>rEsD+!d6J3=obb~Q6; zBPlcJ^mUlD$cu@jm(njrd-e%ed{OLvZqCm+1^YR;j*^l%g!{&=`v8-EvNE(aujE~G zVI>#315ff2H5RIVl4BO2Y29dTD>Q}z{k zn=#h;ymh>7zgcoz6%60~)4@?cp;w%Gbnz3fIVB^uHK-x1YG$yToHw9IRVe zSH3;y5VO7k-E^a@s;Z4jSD__ey-(xcpt7u@+q}?Hs_GY7JU{I%-*&?vvtM2bw?y(2 zg`?}dB5vfSm_OTzXLbl-_D2=aY;ENP@qZX!Zpf($qeC6R2?z?Ki*s68mWw2?;F->=WP3fBD>v%EL zTVvziBQ|L%5f{Es_dXD7_0%^4+ziJAp^%S>rYkS#baNkF^!R)M#jN{ET@sDQCFr?l z?Od{x)V}ljttrlL`plOp=EQvAun3s``J^bN(P__Ka!0+-$hM9{zs%p6^?gW^IeD&h z)yhNLt4jmUTfYUzcyf7)-T8;EyLb-lUnSlWre=R`_*M;?e5lZ+_fzas*i}ros%5&i zl+tl@JKv5{e!XoL`)I*3UJP7(NV^~Q#))AQ$BK%#R{ZQX`b}2%9h(Wi)3eX8N9)Ig z4E_1on^XgM-IA7hHBNM;h_~)Aj-h8lt*;dSMmAQwk59#pVfTaFTT_`p4=KJCwu}p< z47Z7WVsFw$$0^KMB^zqm_UoM0JBYBpsFWJs-enDDJo8{TxG~FrBie@Q=ft;W%;rw> z-S_SK2i@Dg4tU9!g1+pb3O2^fL;?R%q6mHk2L7ro|8&-%3eWuWV}G!MfM%3kNDEhj zvy2Oq-cGbZQ7#SR0BoIw^}#N9_}7lVn&*>n=l5vI+gM^<4fr z@vuT55GXe?;t;aO;_W+RAGZcuM^?B-P{9}%SCefD3U`Io6a#Djl7atN5#lO6}LT+U-qA!XdIzo)(u5yLSiegTwBW%C4aT$QDJpp zcH4RQf!QZ6W~LCV!SiVHEu#=oge)nZ5EM>&(A$etcm_V1OB{t?YO_3!0U|20aT3y#UEIB(9! zhh5optrMba$+9TbZoJXd{u4Xr6YEk)#jzyEpk(omx^1}}UxmUDyNy3v45B-VqT^!A z>^k@md>)**3BJEj$|di<7JGZx^8OFGwSWD+L?EM1i|Cc|nViuj2c2pWzN}SPph3c( zD(qDU)8})p_j%QxQ{gpJGkLfli>gnk1BW7<9<2(OGR5Dz(1Z&mh3ORFuS)Wd=$PRr ziLI8;z7_eYBzyZCYefG?GRo2$JIDji%$-RviDT#{Xjij)KRNq_wO=lGmg^pq8#SMJ zI|mepk@!;0FT4eV4Y(j zk-q6yQB2_7;3r@#4x3I)?8wRF4Sr`I#?`tmzLfZ*wcu>RbL!=%j@W^QO$Voi({$DN z4ezRIorGtR&#(!F*m(Fk?{0|(k|B@JU`WyehmN!$cFV4LG>Av5cfO`9}(Up+mQtGdDjtT!i?*>{m&5R)vb>=_J+H zUDpu%eD$1b;3pW(oD~Jy;JDd}eYmIOB=5ED=$EvpQF9L-j`6G1f241Pxy)LI);C{s zim%B_A3bP5V|A7TY=i`mKWpcSpLFO|e*vnMc?_kZZ*c%sUt~%Wb@fyW>KG?0wv~(D zjHeLVkG?TwWHO64PF^}9@-T}~XIC&;pr_lEeykC<;)#h-z|bzG=|7T5_01@| zeRGF=_0Oaerv8b{D!qW!;0d>pd9w6)M+l+%ykv8;`&&QBOUEpW(}$y+eX@?0A3D~G zGJd>e8GezNvL9aJX{fi5wsGot7<{zjQ$~t&qpV-W6m!7>43#N@FqbtUxGNQ|xZnI% z{trQ|mp|}a!YVWkt(r%?w-9+&k4oO_zQTa)V=N3j%ax*QGR3)JNJAZ^?kV*GVWat1 zF0Q6e{p;V~8z=ANs;24W;ZsZJsu8N@;!njX=W_N<*af7-mmWQrS*3PZEx+hF5(&Ij zimgx9Q`9WWh%fa^O6=`p&()EDEsen@QZK zQ1!;quy12R(^l{BBs4vD68$~>3(lmI6)Qzzc;ELoUsWK6J7SN}*!D^{Z~GqU=2XDW z4Zrog@%Lu@jB(~4Ihe`i*%F;PJjVn*nI@#{5cwj)JpP>vRjM;>$Bv!4SesCiDW zXi*mLoM~aRQG=2BD|N?_1}BlO{V;P}iH4K!GL3=_ zy-6KfozLfMheBVI=ME5_F8g;3KWpF*lJ5I_=5pCAN5Xy$UBZK=q01zsiwtcib~`R6 zJ9Q8FWP$EX55X_ft=0dXnaj(M^UD~j1Zf0Hr+2sOy{?M@_mqsMX-4=V?d2y|vwGi~ z!NGeAKW*5$_whceocjnv0$vcs4Nd1Uo5HF+7oO4D92@I;%9wwpMmgBG_RXbB?Pq1Q zBAObHP8P@LD^G$xojWha+6GVg+w4#;IvQo41T_R^UVNEUY@rig+L5+B-&?GyZri-` zS<_kE7<<4ZvNmfcY`y%>q?lgGKGHi8c^B* zm{$A6zjccKoqDN4cnO%J(vEMl9#kj{{gFItES5om3ebpkKi5wzGXQ4Imfh?v;U5Is zZ4k8`ziK6Kq>@&FV+g?-P^DUKl8;SpVtS6%uwPu9R6LHKTL2IE=284u61qcT8Z%@V zrA8Z2nbhq2W=O>opXO_aB1&=v$i;Jrf@#!)bY3~`wfHqsg{rNne@#$5o`%ZO!n$-4 zcWE%ITLa`$HYG5Tgkqm3#blqSpuTsagje)>g7;s!uOEkOgzq6qv`=D zXo)5$n?vKxfS45QWlS$Y@QcWzZ%w43OiGxt5c|)Zyw?F5p7RLZ$-J*zBBP#$ z`q(2bFqHlBMDxIR$a{m#0e@&+=Xat}^32h;{6~pl%${+9ytS0gTSuJAR|4~C; zP>9ws|NULpC%sL#!SSm1MVh$-krMu{M?S0NTRJpJC9>w3@A2+*TRq(2oGRKWFb9k0 zx=Ve}9DXn=9p^ntB0jlXkA=_9SBlv_M4>oWHJz}SbfAiRGvO>2K_lZBw?k%N-No%p zrlouJMc7Q{U9bW2!3;^V8W}Do=xN%}3&|#)uxgy+davo?*CKcMaA6~HM^A9tGKcUJ zNWa@NXIA~`tHnmkaqh0O`*FbMS|&Rh_57GE9!0vUq$O}P+v--x?Nv+W-~7DwvFa~; zs#Iy1S$0<1?#Mc9rhENyko-3;WZ-9}gT#){kdKq^%wicSE{V3*yP17$Nj}1E7`WHP zOC@ZnUv+;u0lB=g53<|tUmSZ{dF*1huzKf(GHW)viKqQS?%B3?XLu9rKWyJRCMkIm z;Ocvvz%qK(Pp_I={h4k=+zMrfY|UAx*vsYShT^WE>! z0ha2ucAIHC%Nr0)u>n|VYJL?8d>jzqgdjhYDS0--_}n+3|F*xUxunFz6w4&w+u_T7 z@BR}_#GT9IAJmd!BpkS^d66&;0XEEt9fNDB{|na|g@$Hew1u%KUSG8pm1h-!9>y@t zptAma&o;k2iM%q4wnXBB*Cc}=$l2*!-(d^Jfed8rtITTkLR4nxm4}u+8^OeK%I7*_ z@;Hm~yv74OE9{Zileaq~jk~+Hvmf*+%MAB(6t4KnKzL5Nz7-c`1kaAVS2aQw|0qcHoZ6SFY^jE!M6?wFj? zK(qf*zwjd0CQw`Db4kZiy6&ycGycvu)sfxGu5$>GE~(*($jiz4lmSYPwjTf+n(oj? z-zNEKOJ{t2nFz)r1re&!g-Ca4&ubc?{*nrpjkziOgW-~u9@_5jhac-8X?kfkbsti6 z;)+XmR#PAzFWB~tjGO8jTXTq8Xj?1?JE-JD);j*W}F$kQm&<3DUJOE6pagOw)4l}^6(#|D@# zOF55)*Rf1Vy&?5 zcwo%^*7%bjSwzmQas8cMw|-81O_hIil<+Obft8QmXd+SMdJ%{4q-xphTsYEcsk-T&zk)Dbg6M^<+mvn1Ek{%h5aW2Y@2K;T?sU~MsmE|d5mUrUC` z`k}^j5w3J*>Y@B}1NXCgxOp~ELI*jdoxz z=1^2UiTtl1?{iJyTO5hHR{(yX_ zS|6u2Ji7TOkTF~6P@J0NH4D-cr?ENEzxmp@($l|0in&Zc^`kcYqHgcltS^n_LDt`O zmPy59{vrrFhYlY4&T1(%d=DT7K(uE6ctzZAAVy#OW5ta=v8!kGC6rZsb>1#6yq)c52^9+`otOZ7(m*=(_Kpi`x3|^UX5bOPO)SyDc*4F>L;xM#@WRTdd!RuY58&^0)Ot4GD>%!J{L0TNhba2>0RfHb0J6`>aj#) zF%2F{5KPxOBpG(T0hs(9!GjH$qKv<~yIfkFsh;S?;bcUwHYEjDnlAY$@#(3vz9|&J zW{ueoh5S04LUEL5xA3a{eyf?&$>YnP*hB8Ie}rP@2N6d~iZ+0MEX7JYon(KyaAm#; znKYqJOh~m5@3Cgf;M?%u@U!b})`u)tbI+S|x%DS?ej zZ&5?Ub7C&3sr0WU*X$a@z%j}2Q@ahOhmp7kyFP{(aSAZYCTK8}o){ia3q)rI0?-ry z`zMxIkZOSAY?UiMy$!R2up)}#FKJ&@ozHWv_9HK1q+F!LqJBNbyoS7BY<=Ylh)K=H z2Mlu+bJpTB!I`~q&@+BHg`wvZQp5Pjy#{;@-W)N&xYtyCpup*(X@7F`TfjAdyBJ%I74zz-WP6dASf7t%c5E%NkAq zgs@~X}o@l6qH~?>9z_t;Q!MfDK;~z5U}T&I@9|h7bnwmf`vkv&O5Gj5xm+ zTv<#w*9qM6Dh|rlnOhQF@RKI_cbB=RRO&!?&*>je{pPTv?#7r(dAoH?kV8%b#tS3Z z0MgnEiuExppI8o4(#gBYI!A4`0y6-7XOJN0wa89DV67K4H$&+14lq_a=U|9$>_{=j zwkUI5GYUf2aXjccPx2S*+AQq+bm41Evvac?!^n1x5gIkJ> zV4u6wuI!(QkNgSh3E_K8QZ z$fhhrjs$u zX|_iC^w=5CoO1n{8|n5i#UGatLvLZH&rr^Q5V8_X2)(#-@rRo*={VOBCN$)OZIbWI%_z%4Rp|;Uk8u789L-dP~B$O!- zoWnxfL#*5Luf){9qHG&_ZExhJb~s^}LM?@VsrML8E?c~b?Dhr!j~1Xr(sZ{V;-QsM zI~`16&RIluwEP!eB{JLAfxP7NzbLFt969G~Pao%ej|!x>l;nBfLK~yyP();T-=Ep1 zyd5i5iy2IFWif?)x+_jMe+DcGrX=R~l3a+)OEiz1{PPAFAjD!2CP$6^5_l5g$@wd1 z6|A;zfJkRXKwpwJj9Mxh{GhQWt{rqipB}wRPaRy&PtrEI9`7l*4DNCP4iF&wu{!G( zP-k6;Qe%cL7REYDlS5)B`PAE-21Qy}e?lZRD?SNbhH`Wmgkn!78)_s;!(jm5!geN9 zX9c#4ILn+*KA#JVAZpt^Io7{D8~7%uuWpq6+b9fWaF7U}j~rMzQ~P6a$Ut@rtX#k3 z=+mbK)(*7D4K^f%D_s1$ehpt#N?~I4YkTniOxgE;?$mT_A$&`+M?;~J|Mp5wo1wAsHqOpgRyQvHFVZ>3=tbi*tGW9!UKrJuJxdiG&#i{DC`YF*ypY71zmFyF)OX zQzz;21ic2G+LA^wFy6&w2H~mDh|SH}bW((o@h)qu zB-J)JE1uP-Radf5X6!n=RpFTTD6Y!2Xe)`MTjdKChWn>iwV)%wd-k#v|9lt1u`DlUI+$+eIz8n(S~s&^m^s`Rp2GJOe3^94Gf z{shRQOP27nv-9=yNkW)@yQ<@(%b($(=S;95VgdoiMqt$GiBb$1Kj3)Q^&6iAc%XOJ zUea|iTRqDv!+cnPf#1+dg%Qi|88*nakDQ5Gbm}Ki`reuLK_pfj$>mAnVuyRJz79k~ zo>gJ~N28r{ufqW2iy1y5q-{wb#;kRKg2as6hXoslXWjv|>EuU%}buNhT*yz-a;mX)>DQDYq^l1z{_%noZ2bR!Krg3k$(lSs&tbFXB;c{ zK=*)GC-8P1O;=Rtha1)*E?wY0t)Zom>EL0fRZeEYMS|RtoZPeBDGxo@@rgUfhhHT} zZD~n7-acvgP2{Cl>_W3WA3VoB8aM$~7^Zj0iO>9*W1P0fQRW|YklbDZ&TJxyS?DC= z--ZX^_{h9izS7+*x!&%Mzai^wB6JfAWuuU2t3_hboXdYE13Ui)*?<0GAyr83G6aew z*uNIsuB295x;mT>uhd&07&1~~d5VZ-e_vxRG2SO_{Wd$~HpRd;?mUO9=4F+y-Eq~= zaD2Mr1}hulHYbbBq$R9&hOl3IlQk~8?Xl(FN~PEA#^(sR%rk_*`1?#w`5lwPKx`;@ z4;0vx8}g@?KncFc_y*M`c~;%RbG~`Q0@%pr+4p^5J56vZIWmXy=6+Sjk*6)dRq@rb-da$nJ09qtzZx0SxqcU5c2Ff7UQg!A=HN^>sDHBOT zKS=Pj@JZQ*1hw2_PTM^R7L1j#LuP;^;C*O(P6yv3frz-VXqvchzRCw$OKidxnNPo- z-nS|3nEGoiL<>^oPG>$GHxK5^AFK~+%I(X$7e4L`|5i4A=KrJVeD~bgtwI5IQX9Dv z29EM|(nFZ1wjV`hR5+C=(lfu13k{OzoBZ7u78fDW^1E=7+jb(&=pp}g`_;;(Ysn1J zss0k1oy;2}F)t?k5Y*{ylws*3Y&0_{1d&a;q+oQZ7IuE!R7y{9W}|9>4txpE?kt-6 zb|%#|YgOWyN7H}%y<2NyNmyH$ShzMW{`Kzay>3urLMTAI7^#gnlP^r2*IwH z4TtPL<26s!S>9()UUXrGT$@H20dV@*-(0X<1~^i8Jkx)6Q(1@wxnFeuGx#Y1@9_I> zS@oL*!0FCoxaH6&J=Cg9u4TGom7$joaQhJhk-{7LF15kJmog#W;k~#M&Z)zZYv0|N z<|&V{Ps6_gS`1i@xsa~vSm^y4A@SN>V{8>4()ug*SXCB`ht`h_nRB=%20Z}gq?!Iz6keeKgT0N10S-tOuGSC zp@6_1T=e1Hf7%rP)g@_6n_}7CyG)CP_T8OJ+CK3*Y}NsSvfqY~iS)P1X96fokv78? zKyBJD*U~P5v5Po|6wC9`s1KK|IX#_#y9oj$F8WDWA4xs$Zq8AJ{!6Q&Uw!BA(tdH` ztR@Od4{^}AN_b=3$}4aG9t-E#C5cWjny+@HzNX;sNHLP42s5t5{}YrglZIwzxmHkF z=Hjbt;B2XBD_911pm1{<5NAi*PhFl=TnDxiX1tnS(0nz)cV@TtGx8XXSScKr_cY_1 zWFeElX>kqx=?BCH+{$~Z^XZ;XDE0+wKb^j{+)z}3{x$5zkjW-8o)>q- zT1S8vYZhe=vz(}oKtsDM4pTN^Bk$WqGA~4eU${EQ5^(P)xDXUIRyr58_%*~~-^HlM zJ@CVrTPO3LW~HDtir%bsprT^U%Sv(Ttbawm@^Y>MN12$-XD>7;B2>nYL)oOfl>pm7 z6wX)CN~JU!I^{75)NrLt5e+{9a925Gu|0jHL-xdRezto-bg4t3w&RyHYOmg1l1l5( zW{t#QY^M*OirE}6a9SaCg~Z^M!DsKfMd8^p-+MpK;mT}9%nuB{{Iz3vc_uf|kdUM> zI@IwFSccEy_f+*l&W|crCE@?{-`W3t@mrFe&h|=^j9?)!-GlO^RW&E2s$2x%h5l3Q zvM~K_u{i07Cos8oaL`wgPo(X~781o$7$8_8Z}RIoHoRUt=4E?0%C7(RuN zX*`10d@bIb1ppy*x#6Qmn@^yQV!8x>Na!l}_)OIzmyng2;P2QnvB7j-ByOjQ?J!lb z3v~-v32+Fy2DD6ahQLU3$9%e)pZ8IHaa=ZpI+w3OA)4(ce|)e_0E9fU)A zGdrxmlR1D1hxtIkDws#63=3^XNeWm1K|+300POh6^4W07=*}Ymjir_+G@tfpOmk$4 zSuxGvevNDs1#&)wuDu&ZeDU~}jeyRyVcYGdI>O8adBuJKP&B9*=3J-1DY~CIp2bag z#D@>9EF%2AgVcH&e&Zzh1*|LUTKpseH8%}|lZ3SDOSkKnFnW;?j?a-LRl}WF!#mOp}*VLGw)HWYc(IT zb9>spU=^vTyZPSxgSV{@@^>HC|C|Z z>-IE5c8MLuWKS$GvB%Lp+xeCzNR2&MEqZKU>gn&F>J{Y8`we@_IFG&MY1uy0viUI; zu5AYXadiK^)a?30pf74FBJ%v-S+a1M5e<0cj6ZMC>tceh_at6mui1nDZ=3z^bz`mz zw*~rk=uRF~-Fx$Jjt#j~m<96tJ1cGS4Hkj+Et|IzIK3qIyTzCO4`29yUmudjZ%sI8 zI2A6%GggT%MZhbSq$WZ;T)1+WT2wd{Jle9cfi^@R=Qn&<93Vkh4PdL)>ocq;y?G(0 zS9mD*IH1=92#;=~ey9FTZyIa6H z?g^NX(x&_s({C&={2BWLEbIu%!I9)teFiEtj>?Tf0Pej0@EmZ0IAa!{(^$N&Y zs;MI}y#L_+*DlhyenB^0>VZtV{`;LLpQ!*CRuf|X-Dz*$MAEeJyVFXPH}wzK-EEof z^McCC!_kA0_7p&AB`sp?LGCD94d5XyA7PEc!~jNGC+|GRdsW1B;d#Bg5( z@)5kIfth;6+Td6k9Dt(vYFH{e*ZVM@(hAR%723y!?_Xf^{fWjAyhzyqCdQNV#onJ< zm&d;)E%+7ME{?k)K5c<5rP$;ekuhS0&&`i`QQDUZV0^gx6+jy22OZ_#D1ZoqKdl|Vzn2kuj!`w*3Rlw6 z>>C2>2EXB%|0`QUKpo(%T*n+E=zs(f4}|NT03qs6NU`|v18N#b{(c0M?Nxl{b(_ZO zkpS?X1cgXT(S-rn)#ie^ELYYdEED$4>$9V0JT!rj^fW5pS;E{MceC{QkI|io1-;9z z`NYi-Ryi>QE6gElt$US**NXt_a>kt)X~A%H+i_;Td)^^qwtIoN^(98`K(g{nhnsNG zl(`qdf&YgIO|Wts!#XWH-&-geenRk7@({XL_Jo^>rvg;D%lZ`RnDR?EC)x$j zB!b#%CvrWGY7fCit~BY6pIsXfGftI8%fe~?S)WMb9@Z$cFxCQWeN_Y(6{`e){ zTxMj>1ndT^N#HOV11xTqseA|wiVDu&f0(`CaP}|XMxMK&*hpkB9O+mflJQ!6COCsx zz=%dLIx2Q@ah5!4vox;dJ|Yz5czvq@tInlsXS|q#QHiV&AcX(E)i7G<@F^1n1xFi} zz&iCSV7fdtZoE@!pOJzsewuu`>LmkboXU~u7>bE}MLfhFdxR9Uq*?70i*N|gjnb=~ z@Y~SLIk<11meqQd!pMLUCmqH5vQ{51>>t^iWi=_am|wJusiN}ZCkwBl2y*9EvrlOs zQ5A_cJN66QAH}N8JN6pZsho)q2_%==?!=!m`lK-~2mI0&&-b6Mei~@~n%V7SBUB=CFj zDg?Unk|tKjc#;(nEpFRU6%I92HaM{6uO&CV!Fz%7;UQ*9ihFtP`gjSp_=p#six|Oq zpY2RMS}k`frMv$z5>q&l|6MpnqWH%}$T1c_?~>U;culO#QdWqiw_qS8%}=;8F1;h3 z-a^1TLw$XT1jMW8cVCyKA3EoTjbS(Y3<{xfaL3?gw63)$x~COVb1^y*lgPf^Y30prr#^G zNabawF#I4h554?N?upBHl33JuTM%;G(q z_wyS(y3eiG979(c5EkD{vz_lg79U>wYdydh$8O{NejPy@t=65l2M@y?N`rLDCx=UF zX2omt+Eh#)-z#}u{FR)r`k9T%(Yv}4hdsS3u_tU-hNNp9`Sl3tQ@>T-Z+0-29h6w3 zbylKFKAi75Nj3Q?ztZJf2gl!(x|&V&>Pa}%uKhs}YESPGbfz;u zgsyfNv#1IGCarzyt7tk?!)q|pAkjpKzkOrc0X?urB4Dul0_+4B*;rE~+NKow^PB&} zgW{L}8X34`F>ul6`nw5YRT;KK>YIf1ybSTtgEzCk=l?lrYqv;jJ|pS|BZQ0klOPw| zlUjC)hXZ?e+U>y!PW}l3vjLwF0W#YRe2JM0L&6J)CtA3YNrZbcsgvnmz1nrR z_=b-P6NM+y7LS|#V6)EJ4kPl#p2PQrOQ^_8KfIHa{WM@rvj1Th1R(y!yI$ZHyg6@P zhgnQ3WR1;QKo~Tfp#|?iQp2onP%rwPhY&Gntdztua~L?AQhoswB5FQd8k4qPF?AZo z_0eZ@B`a1w_DwUyb&GSSwIU6%x#JB}p~sov)0Zt5DM_0cD3=u=H4gH*tvkGYKh#qA zsnp-0pfMIi50EJ85Q;M2!G>slan+m+XCs!@c3@ZxX3sPO z5r+Y7_a&o<8_vFm^2WBKud_uanvQ2+XZg+N8>>H7qQ|vr#`Iim2H6b#I*f@|YnTzY!P9Y4GCY^(3{>~S+vnHC>AhqzM19A?Do>;Q=N z@e1GZe_>eSQX(2U_`Ad{(ZCDmzm48*H^i^%tVYtZF7lm&{u=buaFNTMdF>cAlGbls z!O%ll#s^e)yVQ|9{8HXilGl>`(saKs`Tk5sPD#q5oEH0QR9mwrrQ+oJ#qo|>e=1D( zvo647++~mxy`7My*_M`kNrlswW%HP0-1Z>wzmD+&0~&xF2thNk@M0Gscdo_9UJNKrev zM+;M*oQ%aj-6kCAN@bN;cwH-+E}T3ndY8{vq8&jA;@mm>)B1C=B6Y&&`c=&})V!l) zwz5?@&}OxATrd1+EzZ68KK&zi^@OPOq=rf97*_YkVaq7#M*zRxC%MErt>kF*C(nA( z5^P4uu=nl_?AM1g|JvE5n(QaFxx%6Y*rHQ2A~Um`bM&{z3@UbSf6Z-T%i4k!T3ZVY z75X&8{GYwzD$MRb{-Ir3X1dWg_PbIX_DseVFjyQd>SQYffK|V(*M&|&STKTwrb;H) zfZxcoqkrFnBeoZHD6?2NUW5&o(DeluzBa$|*csNql{EunNpS7=YnqgHMzc(tGA@f> z@I<4$CM-8J0;|pnw;icv5j5(rO+7-c0yc1DqV$WP?SS{K%Uvh4gf)PVn8abES@aZ! z8PX`jq6lP53Xcj7B*Mq94Lf(5YtZQNbTwc8R7sA9PG|#^$HKtr5B#LE1sy(x&BQ2H z5X4Pwy39l;`|-oV=<(6FpV?hIlnTS8#_)(Qkiej#JK$jP>9o_3lsxz=)Jz_%?%}d} z!LD#_UAH_&3utiR@wmrUBq5c@irLO3nHkFhc>YZ_t-E&%sU>dn4YblwlLdEtzj++8 z-Hqgq;?-pHLEO|AaGc6$?hs2Zjad=d7rGjCu&+24i4Ed!_lH$ke`jd?_4kawYO_Jv z)>pc5$8OqVL2m|Z5#ld%>W6Ps$khc4&Sc)&S$M;OvTkpFRWa3!Ip9~*-d1ip=c8X1 zXAuhQU%0+Xx5{|Se|M4dvwiNT%X4A3w)pT6_}^ViI<~!d0v@uQNSgS~f!&s=q+yMR zMbToK9Vx${I`%*DmHdZ#!CU^PHTOUWdW4L25BxaO!NzYX6g78(YPsF_u3!=t8zvYA zuUilAEfCix|3w9TXM}N1p!Tr*3T4nrF-#qfm{lTZvWPlKSA{xWJLVR-Tz7@O55!1k z=Dxf@Nvu#Qn>#ri{~ z4dNjtPGkj#`Snxp)>c-Q5{fBy{5p89p$3BbbB;C4zm9*B0nWkB=8Ql+g;pO!{h(Ue zC%)Lj7pj6q&qMBCVdlr`#a1;Zj~&_KaOHu6-d_V-Uh7|}Bn20GZBI==Wn}QsW$&_! z6NnfeZGsgfRO3B#3zB*ob>gw@mb9uBnjFZi6o+>DNcs#+$l`5>z?_1-Vw`rZ8Z&+R z5$@4_V)r;BbWdPY=rcBqQgMCDuXgW=IV?oH3t=)!YYdWaE=VMMT0rm@BOf|UoZS0; z&`{%A=!N}D-Q4kEp}$Q40u`vaZ<#GIbNAp*2Jd-b=wRxzJ4!^)u(t91}?Sp4ZVr3z1mfKR_BqO zAK6+ndO@?h=QkQXRi7ts(s{zRlU15dR=O)$?p?q6e6qpwyAc$;qb8GI?ya`CRYL}j zw*q*m0k89(h3j-RIg`uZE%b=n?ga0)xzKDW3Hy1)m=+L8&(>c&1n2*k9%p&7KXfFF zqD4j65ASkT<>o;1(GD)w)d=Vq3V11?zc0MJ&-40#RcgwrPTcz)BkxiHnUITm_a(aX z!$+1xQSW#*+4w(0Z^%O$*P=#xE;;*t&)|aMDO#9!JWT}`chHz=F22TvKS?xEysZ^{ ztpJEtI`e%Dx2LmIits}e|BJ5lmASd zJRh7J`?pIcd8N_-St&l6-$Yrp2gl94wcb#{wGM2Z^)TW6FX9|yT(DuHEQ~TLJmZWy zJPORD{<}ji!>v*Pi~GAP$FJ#ojEY zch+z6B1NDaZ=Q}bnKxIkfPz!-&^t^;#^g9{=g7a{{0?x8N>_Yt*c>e(6sdcrEd|hE zV3Fu_!`s4xcY*U{>NJd9qtBr%A^b~SL|_7+{Rphh6Ouu9e>g1@G+_aG7^VjgG)`EH8SmF@#eIQr2bZ#K7($S+ZBNXR-(`>n!IxcF@V;FyHJp)9MWk?CtRgVj*N^Wiryi}6T*2L_&t+e4) zn;qYyEBlEvT#5Iq?uUO^hvt|yn>YWVU&F=_0`d-R#6sWGeJ7a3`)e=*{&=l*DRKZB#g z!5_Xa{tI2hzvE|HG4!Qv(Y=YXY4PfxA7SdA5RvE;v1~C8C_Yt0^pvFOiTn;Ds(jGI z_KqP03MO8kr{x~4SmaA2p~3yK+`4mnVFLPwp~wr*6yinp@9xe1_e@RWY7rK9O+HRUtHB2;BM|89(HouvVDSbcmn=_5p?Ez zzR^QAmrn#mtwy+D`Qg#{c~R)|U&D zMy`|8ZTL{1*8U4DUrc+8zj{&N`QEbucWu0h!6MU*M6idvpuPjv^yH6S;Yq)_&mumG zU|4CviN$$OgHMNi!sE>P1t74+D>T<{r2E}aly8fd+=9q$uja~CV3&I7+dwA@oY5!n zu2h{3eo*$*7ckMbp4vYT&Gfak9BPL}Z1lM_r&I_chh`H9Rf%cMBc@B`j)pJ^k_j%` zigCWhYSh9B2V_|!|36&4cTiJp*zLVj=v}IG^-&b0SP%#;6uVNTNEbwu4xvLJfD}cF z1wm>Mlnx@&tDt~V5~M^r2qd9{Ap`<}Z$Iapcg}gfznRHohMj$1*IK`|Znyk9yy(-$ zzyjiN4^&z`0!+yWVJF;v$=(7#{BTnH@0~oSu!q}w(i(C_MZv)-;SqCjp{s<7sBRol-k)YvpQ>O+}o{tJ$ff{Z@10}QF=Pg zrx%iG!ETVV?RVfGcf)5g>=19`{dKk9I!x+%m!DKwb`C<@-sScXL2@Ur;l5;VOUmrv zCc4iVU%u)F)pM@><$_;@juwQBRXmwfR=^t22+jH@Zgo)&CNkni)$4AwOXk}HBq5up zqv}8J?$t4oTp#z(t927AxauC4X{(W%5W8!^lUL3kD~q#<)9^>^Wbu>^f@@_(YEz|d zG8(_nh%f}@ihot<|4mY2|A|*AjpQ|xxdQ-9@a&5Hy=@J+^fX%M4i$>|-b%hR%(DqWAI z+Cz^1?t=K4_8=3o@(e(Kcpa1TXeNh@R+!!MV*@10-rlr5ZARuyutoN%)UaXqRq&^g z1lQA*7PNF`;wR@1?_x*~?Qdf&<WM}$f*7E7Y;%2JDXBF8F*prFUy4G8XdC$gN%tKul z3sHKxQY-UsOT1xqG3;6UDc1PAN?|3?2ROFUpZyeQ$xC%0QB$4iH>x{^XT8CxMs1?T zsnTP$w1ss4DNKWauMIio)DUt@b8vS|m(Z=*xnV2@^!b-rD8Bn$Wb2EhFVRa!6z?8Z z{+ivJxE7Jr{)Gi^{KN}D{4V7`%Yreu<->-jZR0rc@&YaxEp0ztc?TmYl*zk|%P<)OEPfU)WJUplAm0}laXV7Dsq zOQ{9{um!4BlEYi^7j*4z2CYLW1`lT1%s&8_)>s{0iD5F0v#riMJiL`E<0@punyxk~ zZ}k3;$W*LO=XH*ea*i9H1mm7`z6=v*Pj@>*-B8P&S-&tq8y$Q~l!t7^{P~Rx=5h}9 zaL5da-QCs@JZGH15&Nf`H4GKXA#(+OId1J^JeP7zRq+`mGP?fEb2KHdqGljXhz{Ab z;=ASHGgaTx+z?T3o4?w)Sp4?8Xs%03zW+c)U{ZO5-G)(KV#+&$x@WEfLA@>&(DAU^ z5P22nTR{ow_IOZ#gR@TIY?vxZ4Kb@Do6cnuhGL$8$^eg_w<<@y2GKF`|{nxggC$IqxGEe7heWV~{+ zWa=APUT7)_=kS7HxcyIMQ}3cR99w4s3HjBEHlV$+*<78tkM<1By`&+7X}u3cYHx;& z{%p-&;f{C2x~+YSVYr$>ivoabOv3~vY$TFo?$JFPc8LwAj6H$zEN@}^w(f{U2m_}) zm;6gzNAYhV%?%IMOL0BpQ#EolFWvq2$dSrnSQ>q{V!T@t>22l!=P9F>Oc2sMLb>#n zN|b#}ZA%od>Hss}Ttu9KNaH9e6{Sr1&ETw6Tuw)BM7P`{45?7l0=PffuH8iH&ruKN z=A++w%&laheI|1gUGA%OuU8pLk{cOvZ3wW^ykNzQY%92O!ZmFqHom_xY5a7JX;X!r zC*%H;&4WE6&+t$DP3Z7A)E2tU#D#xg^N25&|3)q?YHhs1ey^DY56Ke4aO0bAK`2p6 zZgy(CZ`aBY+Lx_iI3{E_24$>X{MNOwZ$9)%*S-FF= zZUlOSAX^7D3xHqjsyVt<6i)VI+{YY(VG3WV8G_7Vqr65MlUVq!G$WmvETW}fnPX<} z&UiAXVOyLtNIBnMb%~e}+b{z4H&t@N6r6SF*gH8E)#=qe*B*RC*>}y&6CqhXOtXoByl@>AW&Tn0x-6gLX+dlJs!%4|~%2edo zP&4kTL$jbcJ?Yf5dkPbWvjL2o6<#;@y|_lHqAJ9)z_rn4e27eCxxtEredr$>vQNs2 zIBe5fG?V+e zTuBzUUO3`gj!2)^c2kZ0jQ%OP&M`+3elF0vn0X9aw-PY`Xx=`eT;308qwV<>15;*N zKc=(!A(-{w4VN4){`@D;=+{nnfhuKfoGmB%s$4OySmnI+@6Wd7HTP15zM5CP*qN@8 zQ09o-d}QH8yOEtRApv=~bYW+xVpV$K#EtWjFDiq+EGvgn)ZoMAj!E>Hag5WvBZP36 zy=Hb?vYv@^l&^hyHD^@za(BsZi0t?)JIldFm7O)Na5(I@+S@)#%56Q8x2YZK@k)Z)>N6F_*#YSXfOqGG89XWQ+y(F%S} z6&(`$+Hbce*NWQGM{WlhYa5BlkloMY;&q}#kX|>R>)RC153p)!l}5G&E}&Tl>h9PW zXCi8s(=N0|eb(V3!3pGHp!!NPZK3;qu!PCh>T%t+M(Y(3eq6mimyrb}u?jqM{$Ys{W~!BMbe z(0cT?lUY2)KuO=_Hm$d_($q_g!QN$kJ;LByntvEuo!8ghW8Z)GF5i?$zHHYH<<4+( z_TjAIo9#FQ8_o{47WgIoIPL1?=@q(5kNs~_oS2lkmnb7Eg!a2`n9@IE0zz^y>c}%G zr*^0GPU)$`!<{~W{4bXYKB;SZkNrnujD;Puz{ z@$I`jWAfT>!OcePTOYm_ARdH=-_@ub*{M#I@UK1m6ymsjntDj24PWnyDA1arJ5k3* zrPQQHt%yy_?u9PRLv_|d&+cQ_Wz(iQ6cwIU4+ghX=RP4c{dhwiFmv8FU(q<@rx4Uq zz0%0?4Wv+BH%@=W;dB1{+~i-L>!PUW4>SyLk3P6IKD@6vy$4=zudeyg5ovn>Xp-c* zhaWKPsK3cF!}}RC#)>bFYUwsB$-^#I5#V_)^xg%WfL!J!At6Uc_WzMQ`^9@g^ezWS zH<$n)Nh~YTkV%#+bjZLEGQ=L(G1l>K@Y?%ZYIAntt{?Mai{i=S(wH2Sgt zv^~9^_>`=}s~R%&eUSFYlgF?3=>j8W^*XP_&PJDH@8GH4rRHF9NFzXAqvzh<2V+)} z8oA(tkSqWX1b+7Q`$O=@k4=b?%;jUHe;U87aU)GTE_5`yp&te>I zPdKF}ET8I^GJatic3I@;qDbaQFZA2Rg)y_tk<#T(@ZT!s@tijAwEynFR%$DM@6^-e zXRU>e7R8n1KTdxn>j!Y}Z+w+KTHTn@JTRa-QsT~NbzC^qYth!`T$lQ$-W_gNx_#~+ zeTTE(F~9?zey9gG+b6@34bx;5t-SC@2QxxHYd||mojSSeRn!!ltE==?vrpJH)U|@S zrO%&Cc{Gd{T~&CFv75jK)n1u{{&EfCzBNHe4lluL&8YmjOPmUk*IvP48asQv$7lAL)`pGg;9VKyS-XuE2MmRF^&Sh0!SB#`)NQe2x`mwP!6Kr_DejW99F*; z72KzkZFONc=JMYvpPZuglC^P<<&jfTM%q&c!pmTkFgNthT;K_sq-d}~xf$-+d)d5n zm+QwMHjI=t?H&)k+Qv`#ovJRGR}JqjtV;xEh-<{w@@3#uy|cx&14;uOBTdHB!`%LE z_=Uc@OLdr6Bhhb^i8betm9JH(e^UI#JsGod>`UWHrqpNb=`YhdAUDvfcqL+?BDlWI zcZ#DdcwXBCjQt?yz2@udrcVlO8z2;Rj3(%6A#I6F)ZHE%~fe`cf9Ciks?JCRgEp-V8JC zf5VKw?<`M$B8pj;RYPMu%O?B7QI`IQW|mpC?n|?JOgf8Ve!%WGR*4)?{uTbTjiF7tC@n#(dlRjsX7|n{K<~wyaSSDnsqIRyUJWWI$*?01YL{|?+^hl&s5kdSDpUzvdb_Hp^&y> zD6K(*AOzUXhXCkE2p!+byr_kg=?wtA-( z(f70$;CI)55GfWo_nr?omT3!N_Kb|a{d^deT}x&0NGzHD!u|4RNC>CvLi7F(7;h+-i}!p9 z*$fMKZ0&KcqKM}5m=6Qk!U!k#|GwLrF+7`!p70uSnYcfXs0{S&SGNuQgx`A_JZ4K; z2$Au9fvR{!Ntr)JoNQ}4jau)|x4)svuXKZD_o$Vq=+e{B_Wg=}MQSLV%s7-f^B=Za zDW12*Lgj!0+ljna(=FodJv~pE)En9BKdypl(q(S9;rhrTQ`oO8$-&Y*%~2c;bZ}&s=PEF0Jvc16oc4AslO@JfI!A?E}HHW$-_BnFzp; zCl!T4I^RO@@=s6Wz%s1AJ+F^8F`I7WJKz03&jahxhk&G1}0Xx3VtG0$4Bj+D8a>!dxH^DxJ|y z_KyWM8^eD9L_Y+Ohg=h!qH$0O@VN%qKpq;X<-HN*$4nNhZ{Y>V;&A-A^T_LRhsYBw zUv69aRhUkhK%$9+;2~GY-z1^{Y&d2G-ml&me;udkvdn`$xl`o35JNo53vdgJSGzD~ zpldapDI~4YX6|4xb+c$S0kWMo!f7>&LN>jeU-o5e^$`>RzgpQgmj%t3uEz}_epI5hcW3aqC4edS!OMbwFKE5(;EG#~?o6?dS z!_uDcH`dqStjhI#|y1vfOv%KYG#b* zX!OaxC8rGXyVeh9KR%dxfF?Mw}$y|K-g+rv~-={PIl79ciBSiyh7w5&}5nLuw~%W|fzK zW-cxN!-fx>RQFERG{O=c`Zppi-~b;dT2A6~=rET$C52im5Xod*0kW~DnILzW$-h-v zu}}thE-?f2<6xL0VYHpmlRW@ryH5rm{^^in!*?0=8v=?c3n|5MQyMua^rxcHZUp%8yMEPw4oW7Yx;~1mBQGKjX0V4nJMUI=MXg zPkie}J1R^(J=Ea{TZ3%#Wg@cmxN+<#KUV%3^Hlxxm~!%@jh!fNW@ol|Fn4GTD%}QG z3S__XN{xkZtxs((1k*y;oAh+pf8{w44Pu92KOs#g8E82=;{X01UF0(nIX#^=wx2wG zRt4u}$>f)deLzd8w(%aSuG;u4g}#?bGfLFz!7zFRmV)JX)e|jL2-Cms@-4!s+x|)$ zh>=|X0t;`n(x0LPo{|rV&PmM*oI?}=%w;bLjGY??-bDfIb*O)88ZRsH(`|H0U`RO5ZtM%OQ*t0Bvn zzi%A|AkfH@xm?RmZq;BL3g<7j5<|0EC8O+$=v7xL)E|1r__PxM3_IR$8-Uj_>CnY(v4fdBCPk91q?$UIBtA9^@-h&@gJASIAbwi&h8K+gB6Q~vLer_|(*~qJoz+)h`?b9u zrtc8`MQCl$!Rti27LqX(%7mxvRBdQnyIRpbjcb^?s)NlbZVTZPe(Op6B-uA|eL?b9 zZg6-CVdl|Ir+e^^ahQ<(=JhjaUZNNUujLQ0?H~6r+$zF4ABNh%f2O=!KHia)DPWzSIm7ghv_{JNB`bY87U#H-Rp*-I_vFqG*ow z{s!#n)vH?jhc8`TC?lf%IEKy@86w7-Q7KbR;ga21(fjW@Na7a%N=@tN(Dz&C?uA=% zIB!{YS~w+FocQgOXt*QsPY=N1@~QG#LI87+=zCNyQ4oV>#ec0}a6%|0++0i$ULhtj zEm0#HV5nZF22jH&EFUCR(yWe-hZg7i+I8NR!kX$t!pSTO$jSnUUlg%GgC%B@?aMdC zkH{k+F(L*;tX64Q$Tp8kgsA}J1H?~u^Ck*EYc9A3AaJRi%z>`o#|@wU`Mxz{PIFR^ zoDjEc6@-$NqsRis-t*rmY$XdPvV(aZB}EhN9JAhI@8_v5kqEm4)j*^Vi(q4lDZE}E zZ^IFDE5+@1RODRayn@aqV|+7{`Mr1Cyc_?KV5C-E@ZE=#|HA^jRI<+d%tByk-T8z> zHf4&$Cn!+}?BKTwA2W4JxC^s1At&~m70Mm zbsFO+?LUuQdq3+?6MXfFitnX%TDTt7*K?DHm2=}<^LgQthhnt#?!`yOZ*Fg92)ABK zpJq@~RRTkt7YCTwgLOiV6P{EO2Tw&W$Bbs5@BYo0%=XH8)kfg>J$?K4R5x|Ihdn66 zb|F|NU2=E7A83-qmdnFx=+b zaNJFQ1D!VYbD#o*(`hjWb=3(V>jQhI1XS}AV|Tah2V+di{V=1Jx~SE_D^w7@sPF>x zrJX0>+A4|j{Tqcj4U^Ww4z4ffwM-*=wog5(?*!a#TICbC`C>Jv4m|TVR?LumUjwKa zOJ3Z($&oorV=urM$%H(zZc|)3%!i47NhpWlY2hWsuzJ$XdSxn(x?dz*6Pf06x%uIu zB2zIQ2DJF{!pXmC!OY)scrYNbQ`G?>Ic}bxw8bcxkrj|$ivanx2Y@&m9#KIR)>Qkr zahCxb*h@|dzmr|A{S>f`g^+7{U5BE{ocO&9wx_<-fDQJe;|=pQg>{W82x7rm*Oyg~ zt>p0<2@WlPN=DBal*C;NZ`@EjCO-co!icnz%$gaZ8R2*V5PaZsLFn9Ca88Xw3o9~W zg`BojExW(mK=fLmb3(~w-=@ukUA6B&zGE=Feq|+<%lh(_=#`1uI`EWK^k|9REw1uf zoURnk(3Xxnj#j95;3lB65L#P)=6Z$BoH@}NG+qt#GbzSFW8m+MsHr9;tuNFwR z3*qYtf|H*BEl=p16`+RWFvG2qp8(xn{CRHmwd;s8h%Z3>+N2O)Pcf5ApQQlklY-c4 ze-QLpI$>T0LAEu`avfQSnEm6$w#oZD_{-MTWG(}QQDkDUvHX)RsSoJx&KCXB`h}8T zu8`GOzmP^Y>ZHRBr&Rv>tuvsH|5%@sfe2Z?H^uy_>JgzKusx8n{=2y-Aei;t56=vG zmZ`L=%FeJbAE&MW6B!?xKD*~n_5RV$Gh7Gkp)m%sp--~80EvO@(mz+dS)N;RnrHNu zvkWRb$hyw0L$|A4YjjBtn;^yrAD?Pj}TC5Z)b9G z`EsfNW}s#8^qoStZ?c08!U&q4cIjM$XX@<285ng)H4QI*J4Q0WX7ejId(A_e%C-+1 zdxHwCYJ1vSjVkT-XIG0=F+>XvFZucyw`x_Swo&Er;0rXD80fcM zL0b>HU62@xK$1WRizD!}PqNTDpQnDi;Qww*`d?1dw77_V`t<3Cih?CHXaSGHSw$Xm z$51L;!6*Ib5(l%@G4Wt8O-!pYs+~I_w$=k;BU*L(zMnmF8pJeJ}IZ; zHRo%%YGL=M4Yv|A&5y~^Anm6q;~jp!SVIcBLih_bg1r;c2)TcF+-QRI0}7Rpo>O4H zRHzf;%yR;KKyXnoiR}Z0aZ00(ra{8?1@zJHz!=0YNp&c^T=eScz_m9a-#69wxj8?7 z108YCH^j0lD!1-GB~5njX@GBavo+?f)6|dPnCI^g2F8oKsUWm+?KDoa=;odnlsp_N zib;W!R}}zRztYPt4i%vbqEt1x%RILMGAEraM^9Yr9!B!9-q$PZ*$}#`EYizmj+^q= z zv@E4b0Jc-q>fsCxD#YO#RN}3jfU-|UUk*YYc)BR!h)-{HY z!hWmMlV+`t!k;v$mHbMr+!i$FKe?Yd{(UMlNJR)@d3X$5yk0 zjWK2gui4Re?|#*S8vVkCnRq>zkw+Fzcgk}OcvxbRmQh@Ioge1n*ftYAr%+gi;45+M z-2A%p-7D9%J<5Ld3*{~5WJryhqV&FGON|SD#l`L4Q{)ni_N){QuWPOHjK(%F+49P` z(bBm2XDO{}TiSymEvvZLXir@{R+wJl|&*<9lpf*N~BPKWJ=yT-*=N2+bV=(k-~s~q&AQLcM@D1{ZvD6#&|Q&I}TSTQl0 z5;D+Z@)UA|c7(X_N92DxuKq1y<0;{hbk|0t*okUL-AI){Z|welFP4ABwZ>cu2%+x^ z9?#7@9>qf_9JX}g&cC2Bg{lu1_*5YxFyK>81jsGDA6}iXJ1+Yt|G7A+hQy?S?n`_* z_XOnu#u_CaNKUCjoGy>2e=p)Oe3X9FX)G5ksg6jQ&K}oz)60hi>jK7(vLY`-wx2-o zW33X>ca1TvQDBb6G!N`lQAR|bh$SDx8k!{?$8gVvT?FtlB|=sWKz14Ed~02T|3t0l zH=2qf?fLN0=VKZQx6e`6tL%R=_Htbrl4Bo2kwB`y2zeP1>V*=+Er$m+PgA>^EO^Sr!~dS6z{mv;tzk2hOHKx8;v6FXquU{d1IVF z#c8y{7K8facD*-%tKepEI3GwmZR3RKo}WC9bXT-feZ_gDo*Wu^+*3FW2`Se^y4-dg z>b9GpSz!iv8TwG{Yjl(UnX~M83oiXsfN_9<`2`GM8(3%SIPex7fUVxP&vo=m z+=j%y!MW~D&<`jIy*^z!Ff3`(Eh`*DJy7#EYBJPB(0#I0e(qM>rgkg8h?)O_ayxms zK8675Pr0VK5=Z&RfX1~!ZYH~lRJRx{%JSyQ>y>u_Ha9B*t(kOAw@hPLKp*HO)HS* zyoUY{6*OxOS!-e1bjEC-1T?k7MB$55mQLm%xB2$P>F1A}4&&#{)HV+uJzZFHtf(nA zP_Xa*ou!F-Iy2RKaGpJa@|9Bd^G|+n)z?pJ>n*PJ6R4@+a0QLDzzej8TDhGU&qQB1 zFDaF&`N5!KGS8sG#v4Nm2xj{@ z1W#j!Zo?q>HLk}KrlNn=qplFW`KNZoWNxs1!swRtt~qgBKF9miRfJA|X($N2wbRvD z3rPb?14O?P43okNHv1Wsh6go`PP^ss$r6F#h^+J265>ySL1ax=Jchz%6_n%{k zq>(Dbgn^rwtA}C#Y*=;3eXzN&R3$OIVlyyJP_gdNX6@caJNh-d0~=mv!~;TK0+8DG z5`Ifrz_!6=IiW0=zObv|>->j%)|}Xg8>K17EpseNQ>%kGlU2gaHHQug&*X4*(Kr(4 zy1R~QLK^pMsi0I{T9|pH-{RHBjJf!NST_=yF=Ni7q9qQH#UPkgP@GE+%+c4 zzLn>IP6BUwd#i-9H*Wrz?-v05zI5f+fU|W`%Eq;q zcA6dj+hDs0za~JgK!>UUj2FygEqxxC-?*>#w!gA169(?O?LTe-!?9tVu#UD9XY-x+ zV>9%3_ygb{I2T^&njyDjQ5ckUauPMfMWTx{^{Yw*YpG;rF?)_2`39g)87QZ zFMaqW4n;z#N*}s=M_|C84a9PZ$oc`K`vT0UI9vr%f0^q+MytuPq_G z_?6kNCfBZ*jv>vd1#z9v&#Uwr6x#;fcmJ79imLeVY%3u5!%7ZM^$F_6G;S5n$_u)yV_bTDUG_G zyh@cKm?U0Xy?}B{m2%+{mytMjKr242E5cA>bX|~=#HGHGC9FNv?02+zDQ4o<bGei<<4iMP9^&?D1*}Yk=f4gV|(BW!Ji+gYoT`8EnuNW0zx6QDjvi7e69D zd!9>+3c*&|SY)DHpxpu{jgsFlka}-oV&i)2ocaviskOL31#0kx7$3Fv>LXP5;ME*d z?u>i^qrnSxNL9me^iqKBfR{bh^#0?lc#Fq}dYI`CKikp>iy@8*y@fY$X1uQ+a4m+l zegFwyN*Ew<;tM%qnjd3#8slk3R(Gv!%r_(R!ee^6(583Y1QGWe-GVAQp?+KgFhC*# z2ly?clK_c!%`Ss;0J<$CnK@-LmjLz4-z?8D{x_;KIkUB1<@HRm9$+>oiDAdwg)YH+ zd9v3`7I|UPv!CM=mt%ChLt786;N@Lb)<}7yB+D!kBw3>SOM=*ub9>(>aQ}I%z+M*Hl<=(Tbd6T1 zXf$9FFyi*nN#Xv;_&H%c{5eR@Pw3KzPRhDq_t&b8ul3X7#=rJ9d@BsHy)Noeijyw< z@i;M?p;hB6)AehiA}B&jTB@&F3{p||t~VKHuXL4}gTMQ>yo1O98eN;)i=f@uU@joZ zGgs|{cSQO{k#sP2CxWz+YSWHkr9kB$k^{q@W5>^3JFj!rJn!`@ejw%E`&0VYndoox z^;x{(n_-P5kOmgqoPVqZFDHJi5iJ^$n~&a40?18^;3400_~xF?!v3*rBfe31G6}LJiAMf$w3(>{D~-Fd<`UHzC2;lNs<%i z9N>AiVyoORU4D}RpDp%E7kI< z{CA1r`tK5x4pWKX_T%_P8E*)L;k~W}Ndn~F>umU;Hz?bVGAM6#C|*3OFOT*G^6Q;e zeoNG6cgtZxhYh|vhH(a`p~LXUvJPWYu*IrNqfFiPcj^^9T6*0wjZWUFOk3gn#hn!h z>$~pT6$^gm>n7sfj#6u%8>47aRu`)=VKeQ9(XyEp_0UdpW?DKNp9nT^vk!YaiL;i( zs_dAqTl-md0e z$C@LCF@WrwAdqol{lC`DN=FBVrDV4?WlCxB)ANPHCNc8`tQ-Sra)nCgjj+4LgI>hx zIG2pJKa(~zvpnF-x>&e<-h9}@h2Wa|Ev^nw_qq~`%S!c*xs^7$y#X8}r@h+>?2Hv3 zxuB1FhJuE`a^iC)52UukvEW^80OFh5H9I7xb&MGc+n$56Vor_$2thZ?cxMO#>hF+u06FXT00d+ zcZ}2;W;{<$J`#ghvku7FNJB{C(hURlF#Lz?qrDE)9!o}>EKwpOG{`g3EhoXmVf{q= zP0H7g!b%pxk=?VZCplm}135&?*ep)v-q?5GdT|O)+(a|Sch)2mpfFco-TMMKex&vj zQo?v1-H1X!ZFE;V9Ss9M%w!{_u3IQE$ClDFPK6=Xd0VPwGuNF4Vl=WM5IeVjsM&4! z-8*}hG)Wy;uw5knXaJRj!kGf6E{9XbQ5e@b$^F&9Hi{4_()Y_nd+nl{DhQp1in| z23?L_d0lV+Y3XHi@L&|q#jI#XK-=or(jZ#zzEl^T+q?!$ncj7zJwejns z$T$ox(&i1$pJO;U;xo~4yqSn|4vyBAqQ~ex97`JjTB_-7Qu|RC}#37O&kz`7G4BG*37K@4K}kt&8;0;6Mu1E`IO>G4VjV6CpGu- z9E@@}gAyIoH>5NU>E9yc;i)(C&In?K+#|Dh_Pblt_9;5bt!51$BRkCCB_PG)4>vUAM``-u`eUAf)nedt~igRqiz7H_8_fo%&$g3qx;r=dR^URl4}Mh$%9 zFJ1v8oXE2`z-e>I(ofo<>iHaq_+1|H_(`f-0*p$ z^RXx%c$u3__N?wjtCh10A%RCd`y5u86K^5MvQYc+@6`f@P)P)JtCd}s%!GiJd1aja zhUairbd+Mcb>5AkL$0$#)7=RL=yvN-#IgVQ3-M}7B5L@=?9e*i7HK4ob>6&?XL#oz zn6v&|E^}WUY2bmq1uG`t$)_j2?pN3P?7vy&jDMer$svMiVNT2faP~NT;1IUGx{P82 z0HJ{-uh}ZE{#=v;^Na869j9L`Z8<&mX;lss>W7NvvlBQyMU0Z8nQe^YOv)V8|HY^L0q_x9e%9Q38 z4Wrk>z~K8F8>|+o>UhA!W#a|hHkShwYRQg z?8MGv4%dDhl@FLp4RGQuG(mZA(*_*ij$sv-J^>6PU>nq#5MyU@X#iafBs>fqxGeb! zXr|zNj&9+=w7!SXZ=JyyP-caqKK_H8xdceuYxkA}!C(t1$)o;Ol5-t$BaBf4wqvR4 zHt{$NAOASC7b_kVuxf#dn7KWFWY?!@(GNDbaKP@WZ`j;@7y39{)wZb(zZD&$Y*V?i zx;|$)YE4M~>_C)T*)WkKTl$W}$TI5CZM++R4|EL_7?<%j-juL7(7j-N<64X}Z$Q6# zImlt8PR<-vRN3R_`&K0SG#&d(?Tm0q=r*GP54DvkWvh)%waOIEXvrqrOubqlVbjzV za>!k_9t@zHz%J;WHPaIFXF}_Ywxxe-RsOG+7_Vj|j-_3T$h>98Tm!sL^F{k$|q!ogPe5A?@IBe<9UG-xfl^}wd4$A z)A5Xz=T-7l>ROxS7LeTB=y1__8TIs9IB*us;hd|{LS3W=s$Ow1tecYB%?fDa!N|yc zfwD?OJwB{@2>AvqlCviBa0wiSYCs=7*xjCoCS8~iZb@L^Z{pY9)B zl9@ic|HA@QjlF(`17Z*_9(ZS6wF9s??!oHrUGjTK_I?5Zx~Z`icBgo0%b(@p{!ia@ z6zGgRo97VrNm>G(8EUZMzBo{uBDHpYZcaRgf+wE*_UB=vV90&cJMBO}x(25Igz7rX z?CBxu2?@L}zJ0L1Eiz*I0I2O$f$OgU%#R!mZo^sd*{mG%P-@f2YbSf7nXH4%Uw_Newh~@P8fR4em4Ht!*LOmosNIHkY4P>f5$TjNk!(W+tpl zofoaWA-3fvFBbB#5LpLI#YkcgSfrVZEL)jzHPwGp!6Jb8Rx4}SdOcu@#XzYQ&^>X0 zi)MJSWC2$?E}+^Q458aQ=HC-utN++~r(T%UpQ3wzooB&7ox|zVnYSBp6lgH~I|px5 zLK>%06mEx--ZhuSEy7a@T2cxaJ#C<|hRJ{_kN+(0w1$hb3erY2pX1}MG#mH`l=93X zw}Rb?|D4l}c_rfL=>9>zzF^WH$F%_1Or*@DSa5D)@tsw_r4%68P|Xq_?qA;G0((Ml zJ6mnLH8AK$za!jt-Zn6r`W~=$h_crQCIni<&*UlBrTa>xm`G3W1YEEvxG*i zODnQw@gKxePC!&%uw+rFV?#lb29?uKpu{WnT?Z+EZ3zL==+&2<7sgxpHBgizh=*G- zn4n?fqBI~kw7;u~ttS#3!f#}%|IF$=XowahDS5SPeWl{|;aH7_FL`2MkJ>abS|c|8 zvucSbgod_Yz4=^F-6gwH@9?>h$zLb3}E`*$tJJq2t(< zX_8*|2-p?-@YIYtrXuvI6Yd@}*~PfrW}&HKWF|iMHdefrUS;&|6jG`3Nqi-Z)H%N= z-#?>459Z=4IJVIYyC~%r_Bq8h!{H4F|DATX5jR&cSUW_H%6gV}^Vf6#un6itf& zQ_L{GPrh-q#(!<2J|NqS?59wLd>?)dY$}`GZSh?w91hK#F}Ba$UHSF5TzKmpi0UOi zSFPj)aNu<=LE(8jufHb*LQ%=@?7ua`PCyVXGXt8O>Ip@;_Top`Pygg3q&`&KeXqn{ zF-ZLx10ZSV$28;DcdnA$?1bIhL;be8th-y^gwANG9eVj_>IpVN7DwM_+vQ_w1_a8N zsO!3nm%2_c^Cb8x7JOYv-BK1uWuQ&Hj}vinrg&r-bSP%npni2M zjh=y#n^^$e(q9L*-B1A_4kYeEB1~1X+p71xyx3_QzK7NlM|M(}BZ>c(;ehem7V(}! z-3?|eO&1LUjhKedBe_K8y+6uu$PQY5L?lNyVW!}hd~-50Hu({ZJaTMpVo??P4XzY# zq=W@xp^#7LopLnf)(t#l7IyFBvF4uX4qm5^w7_Ep`(R3EHkro=*o;i{QU`c2G49cw zCBdw!J7u?g+42B9C`~!w+{>Rq?${dTe^hB^Zp*)&bc}yMkK6>gc9VJ*xZKnBptA15 zi;XR&&v3(Dm!!lbO}yUAa+ZKmH^B z4bt)7Wr?N5G2+06b5cPz|EH#FNNx%yQ1TQ0nx5w;7%fsY4nB9vBg8sd`EsYvI~bNu zO{IKlQ>r)+KhSEoW1 z%v=WJN<#a`++k$}T6024I74>?NC`g$FEf($+>;cMJNqI<)*sXVw`q0deQx6xCN8oU{{d#Vm+LL7TKJue}MrhnZ~;Jv$FR zUkGKUoP2{*BFuosurC%ue-9@^1VIeK;+a10)Xw(>u~Nd zb+;wBz)L88T$2PsE?85d$6H2QC1$TcJzL*Jj6%Tc#+d$w6RgGTPa-j{P20d`ebiUY z0$j0EDo2qcZhtCmCD|?1qW-G^GE&mo;eG*WSGem)Kf7$YklT$`{2D>tVu-|TT2Zya za6gt?5bM>r`Ou9I?SysXxRKBiO25>?bRkdTQ6n>%*8+}=>jno`yS@$yI@8AJ`aKY8 z2+YH6%}?de`~~PtT<^vqe#_BGgEE-I*`7EJO!xrc$MM;H1;Yg!VDnDp!Bev?e3^h? zQAhvk@zX;R$iiSz;@xz@tN%3s53%A2%Pu2U*@7QQBgJ&|a!sP}h@nbEz7a>P-FK%` zQk|3pZrH4X1DJcJI%^P;mBT;c1G6forLsb~=k~reqblD8>Y{LYNdy(G?9d1huTaVP zV;5~RPdDcN_XHji%>hqdNH++NDzV}HxB5-i-ZWl3Xw=_UF}~NdM*~SPyIq!MTdw&1Gb@30^wLSO!e7~Rk!5~i8w0gpv)73bx zKSIr<@(={i2)#=Wt^BSgl8)J4`gaYy3{tz-4MrT;Fshw}(Ue{P6L3Pb3!-`B6gxQ{ zr}_0d0zY{bp!o-*-m;NE?ZV;VjsFY#*{LA@=93X{LU}=@AjmrKk5i2s)ili)MbJaD zENYuGHNe(cgOsbz>g{t2+MEro_)dhna#y8f@N>M-z`HZeNG5XYSt-%=5wmmYTd4d= zCZE9r%*sKRv4`d>TDFVOl{4bN#T@Qtbu!)h*@?}InOC({ADgV*5zHmoT=bVkBdGlf zXeGnS@5e)fVK3c>5Sso*&gm4pwCyrToQy!u(uK@2{k;Yq?gHke}=r$@QPVbA$LkS2HDvs zeZB?LXQT_WebCA^+e{XDIie_(bq)e&%ClPgg5SN|$UFmfC5)UhNCm?v5AEfGp)JC&Fj|^Md8PE+qyhu zuK5zsj239O&|ACbq6le65~d$QG?@06p)_T>%A%?G=aASnduR~!7QaN)*P~md*R#IC z$+t3JhP+h`nm%AD+>yLvxQQgX4s`B%ZG8u57N0Y-Hk{RKLLbsQJBOL??#dD)a9ei@ zI2$91>-I9B|HL~)K^|>Qtvsar??EWF7?dd9A@Nh0$fpck`8DwV%U2Y=a=Rb!YskI; z@FJDcpxO!dZlIvw=6NHyM|G%x1qMLIG(+#Mlh{ipveg*iUsqWYZ3I+W0l8?15%LbSbPQ(HCHdTgQKStO4Vc` zubF68E-fz1Z(r3G08EP{E8qgI=K)$WE4~rX22(Md&I1g>9ehdYac%vtYiCmG9s~?pRpfy8I2^t zyB#TXUTnf$eCE$RDL@R=mSmS71L{oPF1b%2sGrs9EVBD%DQ}yehe}MTY$aBH0kise zwT7086pz})Aa!B>j%&_dis!fT+)Isy-_*lhk}!uAYe>Hn^W$c7cbM@dD`SO4C}XHU z-EMG82h73gWd>U&{uwU+v&@DEL$KzmKu`gC^AAy%C%*`uT_Lb6O|bdlMvv=W-N2mlmu6XHTs|ffQ!P4-db*ZLbqU zH}yqz+{0!qn)sA@uu#9JZJ|sm?Sw*Hz3`JgD*8d)UHlpxOSkW!7-x2FO{c@0&lIlCY)mW-38VvBZ5x{B z-wT|s=JbyV|9#@j;4-;>EK%$TsI^lW0v}%ob>`#X8Kju$&GMXV5C8QckuK&Ia8F_= z+snC9zA-XDbp;o$<9irDRbCry2&$F+2}@sJs9&NgGzHh5&~oj--^>9ZMGH6l zYbk!OgDa)FHX9-SB&jqpt=+qe0zJ07c%|ZwFL|I=6oM(yMl}XIdnSh*b-3y@ZlkGnM?AsA=M8Cp7@q3!~oe%VZ~%B7qEZYT5^G z4H#|rZH2E64-O2YG9Imw{&#wA^vf6xNtkfQhnax>io2UBPnxK^0Xr#A4((*f0jBH+ z97DhFZuZPKE$`Wz296{5&4GK4@|^enB`vXZr9v{f&!Khh0(3K3G}uL(0g@juusXqK z^6-bd@fS>#g6Qln(2S&PWVEh*V`MmaGE&OCgCKPtB0LyHr*1}B=FMI2vj?Gx0R1{R z((G-mLNr=8KoBi}b)$NDC?-jZt*{_G6<(4D+IYD?b{Uf9_Z&FB%aH zZmwaCF*ETtrRk#0e?o!pZFepUj^>e#Skk@m-BC*hyd8?q3ybP5>1%HN4m5VT7i}KS$tX z?~C=7X+q@KT{~ig?22{c$lh8{&ms|gGF~y0b+5c-BVP$Yap1j1kETC`_``lOp-W{7 zm%wzULQ+1Na=+s24Ag%>RSH4~!T_`c7@5<61S4y!t|l+2u3nM#=Xwo0y>=u0P1sG^ zbiZp3$^Q}Jv(gO|Rb6jAz2PWvc=u^pDgWJ$C1GQ3enLbEBi0dbAw?PYmT+O0H^1(?*$8!>2RZwBzJZI!`ACZZiR;_P^jcTno+y*hH& zu8{#d!A0IuwSMFt+_)usSb5_3^k_2Zym5K~Xgp-1nvBoD+SrMnKPeZ`Aax_a zdyGo-NYr;Z?oh0Vk}F<2&NIKUSGYa@h=POA>7H7~@$L3fTCulbySu!FYRgTW^q&I^ z2$@mqP#V3_%9Gda-K{*XErsU$C0c z-$ZJiH^_5bC^D}u9y>-*`V_?nq5B4GY+19i(Gc?Z#(BWMVL?e!1E3)mjpq&;tv(!H zZ1H8DzXD`{e}n_)LYme(Cv-YiY5^QCG~)DYpSrDfz1bQk7`^f++RDueAIm~6eaS_J zyuF6!^1b^vTTlDp#Eu>%BA&}a@d5aRgO!h3E_kDC2>s!5%uj*UHh?Z7B4M?074TQQ zs@r+b&i6+*dm8^HcJr4Bpn-N1B#?q3^bgcqprDxlA({gmy@JfZ{VMqGoH&eXZo-5k z-@+{oAtqdBGc96DvMZq!EyOdC)=H|uW+c@!K1-%@2lh+hW|k2Xb@o$=L}0_+#hdv6 z@jf@rxZ*(U%Z3G>!kd?q^5dApz8>cF%nob73hK&+Lvz5}6ZYz`5a#uCXeOAazG+S~ z+b;E6qoj%GC=Nz$umTR1UM7i7YYd0gwR-Qr->(Abz~APj z!Foe%SvF9YcNn7Leu!ZuYd!(Aui(T1`p%&t+mUw3-{}864a2E8GGa9nr3LoobzJoR zX>|(i^qrY3cWM)j*a2RBJA3JmIH({Haux#;bpblFonolPi>F*XZ`at#obe zwNDLblSQIS;zK~l>T%|)F}Xg&qFN4j!@iq4JOCDCF^Q)P;bc#K(6grYuR=i2>%E@S z9lWE<<8XsWOw9t=P6Hgx{~15GC#nY~dOR&cb$Iq|rwqU{Kmk&;5x{7GlZW8wseDH{ zS_Z}4d+?K%7=hO$_`_!O_|#^}{Iz~NjU$9)D5Ljxt4q_a&NEQ1n9NL#ed!673ju=|GJx0&PAEq!qawa;?TOgNiU-@hu^fcLc!Tf<=?x1MY0zy;a zMo;}@AwMYI+SHZ!+CE{CeGRm#({D+jOU>~5tzL7sv{&tUSLJ|^s?BSTcrHxYPpM?a zQE#t(p^|z#=sj`SBlp5{H?OgfIMF)A^Jn$_qh^^%-ve%sEJSjZwo-;CaW>#V9LHBq zKbh~W-QM6`nX0gTRkRJtK&7A7{tzv(2Q7s9-E(hRb()*%InX+?{S*!|_q$GJsK51w zIXBIBONV=FW#$nMulheXJddB`@8`Uz+0h zM)SAy1_^X(3=46PP0yT_d{@*_U9c-q^*qFX*y!j^b7PD6>*c814{svD!SDn`^b{_E zFO&J_i&NNIK-=*`Qi;@!J;%@f-Ik(fS$HPAL1yKpjkY`b!gloJvv{%JicQ=Y6(y}$ z$$$z|77Bve{$bpjtQjL}DHk~d@vlHv;{&q7&WPk_dI|VtwyK-{d1`8+FX1727*pPJ z-BllEMNL)!5p=UEpIA{D1fg{tMTw!TPfQG6of4-C@Co<`IdA`Yk|1H438l^-i7J|m z|CNr-O8y&mM8W~zJ@}^$1SlFIR9Ema2WuGmVPfbhie|*iJxz(X^nElyyiD3Gtc+EL z&^9Wo`Gh+d!pW-g$5d(7D&dm57YQM1nGnM0KP=Ni=pUW*P^?%m;1>xY_ zwTht1iVoa$W=p$G8~3{rOo>fTyuisvlo0UN1_zRKGe(G8)7i)`P-d5-s)Q#pEWLa>{0 zk0`YCaFpF%Hg#@wN}zj0Dd-R*g+8359cpe><}3$~^;6{W;oqQCK{go8RRygRbJbsj zP_A9iMC~|27~5Jg=ZpO$2842g|E~0UqxN<<^yB15`*o}9^tE<1eC#hkTLR<}H0@Xaa3wX#Z`MX!XMmiE*D}l>E#k@A6SJaB zi~bPab|6lzb1jp09&5k7cL{nnQ;~z*vuMLWeLRtjLQrnn+ID`ygGl6Gf&~8R%}cJk zDAuPeNBdB%!5^YGJluZiWAX-S8;>b$1{yMv%cCus7UFGg>4uA>WNwBtxlP>qA1#2h zlC+j#(4)!B(Ltj1^b0}4yNOB-Cqm5@j(MH|K4pu~Ogo~?>b<*PvXVE!B{r&TKy_8C z^ktiRsUV=At-=Hw;=j+EJ@B(!J?}aXq7SA-Y-bRFoSZNmf{9MTTTn401rpsi9k_<7k1|UN^DTBWgZ*0jcL-NI~lRKcaZdC_=E{`*M9x0A1<>pb=>vp97o> zzqtZ*%@2;?zL&MKAAS!X9{x8L6TuCT^}&Kl?6tx)D(I55Zqa+l5YCeZ*Da0aiq}(^{U3oYezNd<~F8jUEh`C9WLM_s#?0yIz+RIBZCve3D$rBo$%r$&KJ%C>@a|)XKNfUwa>g9rR>K~?zCUx&M{JVVvo!^S zclxE}H9b*EpD97U_;6X<4E1BhmbqXktebvI^c)1$Hc?vIH;w@s4jbWh_#68ZZ!t{dc=|Y)j+NE=maSX(Q`K{v zrT`CeaU*~19Cg~Rv&F~0S`q4yq2g_CohHY{#@ zfBEN@HOCUGY-MdJaWOfg-&+)$n`Hwkqwqz=!fS}`E>|!;G`V-o9y&eo{@VBH1}ES6 z&-u~=I8o5*EAeaR@8;K>v=i5?XcR&saZQ;pP9Xu;W5XvHgPFPzjp3i2E>W9`s^al2 z6gO>{R;+cJwh@5a6c7kaVI`N>+`EkD>Q?-#@udUWfgO?|7b3w587z9F6-QNYHkBj2u3AXlJ>+T+!?Y3;9VDfTqD=3sLwq$r)zQU ze!XA7vf5$nk^UPM$Fp;hW)e)oEMrqd;Lo8Rg9U#S3nP6&+%pw|q= zeV!eN=Pn4?(yIU^Vw6AQ9&tMcU@iz|=Osvw#E*)ujO@~jYzvzMY@9U3@FkNc9(QNU zPYei~-EoreYy>~C;CKv372Z8!5>dAWrV@eY>>M+>X`Hc9M){`Oi3ICUPd$EncLNK{S2h5D^w_C;_K`%OIm@$LX8quBUR6?U{l37Z%JY4c+08R*M=*5XTj zx;mQTil9>bBK28G$6tlk3lm_=F(S*YFskq4RS|twHj|0R1+>B^0OJ#kT*h7Ia{RiR z_r|f$mfNuBIiL~v7pDx<+&vD1s3`NW(_W@oqF}~FCW}i~*|nV|=H`@?g?B6hxGB!X zh}x;7Sl}RR@z$`h$(^m}G~ljpLeXS@H_#eG@#TTLoKI9d62`IR8|^@Rcr$mtN>`%V zW!m?V7QEg==Ve-3I@9pwW(}@|ljAWf-GQn^3ey%RUREYv9?(xVOGJmwcR~JTYG2OHrXB1;mWpX|)OhdLNEi~sVzY-(TP(*59% zPeG@aN7PkQi2q~5s{j@F&YI(N)7JvA{TgK&dV{tydFU|uN0)1L!0U`PXN%>OG%ujB zspt5$7OMyI4>Mm=9#us zqNS4#hAAhavAJ7v1(G+K0|W`>a&00@q`Ao76Er*uaK4d-qMd%;Q z^lu6Ie>d&F08RHe9ZwDdKoZKjZ)oqd><;=0n-9GIfua_Pi|j3RL0nXPbdRgUEGvBw z$oQoZl^Q>iESr!A{MCZ9G9cLN$BU+cXlWK!J9~V$<*FP}Thx8T_${!PVy=?SO$Z$r zm?Ufmr7F6SS;{JX1V|DOm*Jeg{>QeYH6OlQ7n3v>;NLyIZ-hh*eQI_oDUVr!H#?@i z+^?9JmHYl9WLZTXFK0bi+9e)8Uzg@)XISu(>EaRzb!CeZow$Gew#T;KL0+lJ= zC_QgK)|KNxL8~xyEXq(w`klJNrhwKrt%sP(3&n$kTHaAFFmrx1cfVcbGH6j+ zVcQVklX}dETv-nWFCs$|M5ij(l@%ps1oh9C$6TYYuf?*fOKb?Mn?a&9vREpVq_+mz z2RWf0YYGluLsX`;`4l#*e7kh&!fc`gtr7O2f%YO_RVAJ4&mss`+w=QpmWVELR>Z5A zPtVm>WJ_j)VAhY$J^I5G#Qi2O^r;{(=Ito(n)C(B9>W(hc)m6;X;ES@!A1R&i4Z`# zC1NfU&|^!%0b=NS<1#L;6J9OmEWwuIH!#?oWi*7d&l3yTLsmuKiT&zMLzj@ zo10`i#Wgw*B&RVMItGtL6I~a|RuhK?wVQ56*mk%tNhCN%CmI`;dTNOgKm5t2s>E{_ z8pa_c7mZDu-2>>mjr6@TLHWYf&9Z{o7)ebKKE^U0aCURM`-ERa4BsjLwRM#-W0w6r z6+CHRGQ(-W-z@2H8dKdLWszvv2Od z^Q&UQcVAQk%a4}FctL#5@~Je$|KVH>^UY}toV?J(O!ZQtkt&{6_66UiXYr%g^7In* z{bA(e+jl?7W)NyaybAl9ED3@dGEZ3v`nv4-ku$FNH{dj4ySj1^w&wI=D3e~k;GS52 zFzs$!xE;cYJ&jc;LGsNbOSSoG#PccZbgvc5>x}wrnDcxsSki$2Q4y}>G^Q#OQ?h-v zX3LiJ{OI>YCK9sd?8@r_us0QKC3vw^MyrDPa7Xf#CFgY$3Y^YzHZ-@s$`j+d|0&V$6w8CiduTr@D;Y;6@8 z&-h%~JH|}S82mgCUGD^>2UH&iftTOK=;=`9GKUzDD~1lL*FSDe2*ey2iVojRy{iyC z?q|n*GKQ0Uqj~AiAoCU~x>tA*Tf1tU|K;%kW^?kZ52c{gA?CrAnM-Ar^6cnX+zT&- zlKLpc*EMPU)FM3+8#$YFN0g9;qP7B50rhH+@ngA9(>6Fr|0Ot)^=DgsqxcW1Fne)S zQvE~ITCgsjtBIuA$I{!wC0O)5czNg3_FlP+U=Gw7_%^Q7CEJwU)ItcR?p_EFlq`RwJ5&LR%UQD z1X*l4s#^Bi4HVvSiYF##ry|E}cBHthuH&xhnHnP=Sn#9K!(7Ny!e<{bJPLXq+%npY=0 z&JZrnya3vPb0a;u*4#7+^^H^r{ai!Uc~JF$;!cDT-0zo+-@b6&3YdtZ7Oc<9O*H)= zHktI_nXZ)}G?90c$HyWDeuwg-)yA}POEB}c9dL89_#AtT-|DAtwVrYmbH^uP(dtD} zW1*s7(_#38h^z+`NiflvEi;*(4$&yWi4`91{%zh`iOwe_L+CY!hAjU9pMzu)2f1@C z2BjqzR8z}oa*`QtCDWBp2MT?Y})*U zJxMJ1S)$=Dqw@?T>>o)Ha<50HT-vQCM4pg&u>q~8|9-YGyV^dgFz#)(XOzl8PKHe@ zMb$E)nQGw$_o&45qKcEwOj%`dOzC)>{>L#yCI-LA8;NW;WG^XK+}QFQ9qrbsI)U~X zW}d$R1-0juC{WP+9Zse4WuvOkmkTP~ShLn9g7P0&oPRGi5LBNp+nF#eMo?!a)euW; zt9ifu-V#F^&fd}H{}COPKI*Xp)FGXmTe4_kpHNqVIb4=7SnBWeLPhmd$dH>l^J|x* znSwT*pxot~r%dMX(QYu=Qo^Y=axZY`UnCbuCF|Des~tauMu=?3-yelgF(&{jLC@*= ztJPnR8kwjp#nEszvmz)W}gatKC8hD5WfiYkquu- zW?6oh#V6p>`Kd1T4{FXhQtJ!1^Psft3lXQct0dgRPmO+*2@OW1k0mk*Pw!0!#mW3- z5xp$wwfvJ+91zWO9`-2eW|cA#TB0g|z1dY~g1fUV9;l#VIGfLv&#j&4 zQsUQ0B{02QbYO$YDEK_t3qObTUu@HxT42W8FwGAi+Cylz=h4#qb#Kdu5n}T%7or`_ zBaPr^@&azLbrtvRuz;Bl%q-6!7?Wn&7dDz&b&!ShjVHVLeyKCH;9W4vjlTxZk32?@ zd+bOQok*8gJuT?Q9S0Q`AuI)4DbfLzZuRhOhFtjT?^PQDKXZSHiCZ~_sxtjF7&;0$ ze|&nkW30&(idHL8>i_kyq&PU|H%TVL#SSA-<;DLunU48#>5;sr@JaB_x2B2-5L4 z`3kuJz@?kr_CC}!AicsBx!B5^F^9e%wpEoh5|AF_Vkr)wEOrQD0&#{=FXBG*WhdP2 zLfLrve|W`%(gT0y4+UPYOe@a1kbgSmef26v(;6s()o`B8Tq%c~e6NquIr{~*)Wz>< z$~~4*K)TCJnSFdx2!4^YF8g&|b9yb#3 zQa9}EtM9qJ=5wLX#!#ht4=H!FYQp`CZIwt@rP?1YIzGg#nF2X|lWR4{L8GEoh$*-e zzdu*YMUl2odlP#pcvdhK%odC9F9ACQr7 z7uHf&HP^%osfz^j!Ggfw%h6nFV#)s$2wamJj5xW9P}5ALSwL*LD_^Z<^DR4hWbv4u zO_@k##=?r%bgjGH8cda~pxISUFem60Kz<;>IX>iLwwyN-L|>J1LpwFp9q2xszh7^! zsS8qQuIgNv4uCexLN1%WI8vj}2cdf3*K$2W%GObi#IW|H6o+!6HMF68%pbWhh3aUE z6qxGT+c*KvvE64XR?fEDf=@o^*fMs4gZp9Pwtn8qmk=dv=65beT*nV>t{#V^l05^v z7Qi7YpyvcJ@QjA`?jyZD4(;lz-rEv5|4Qc};MJcIT7z=uy@be35^1d||*aeNtq({DAb zkWspAC)*d`rKN1ScBZf)p_+SgNhKvKR&Kjv#$n~#Q*fEJHMN- zh|Eh`zFlEQG}Y@iJ@=?#@_U%3Xr~&+Q|mFZgAK&I$%pCvlvs;%M~mL|mp}&_RG~DB zu|^!Q8Dg`wGJ9|+X&@B+ztW5UCgvVdA27&$$LUn5v_&mef?zboru|%De5fku03st` zFJ}=?=68OXI6!q47mPaCwWDJpFB%w=7zryI`0*1^)7r?Tv+WVp=<9;mm%-#QjHNCWo9Rt2LVRkwOJALI{tgFw@)5N&Ra;!jx=Qt5?7^zAAd&KK% ziJ=2Kh@;}h7NZ@R(N0Eis`NepMyTr!d;s$_^#xT|>!j<<$PZq7>H5DzyYgefI9$JK8-C=tD$uT>PGS6IB z*U~-ZI^omABUap4K(Y z$H#qc3t~h3Jh$=M=UhujMIDaP&mNP&h-|+p?DLw3+{^o^39t<>*lQA`d#Nd2KlsE< zz0-6Zn8<7b#xB>^ z@ah-m=Mt=7{v4ngQy~%v(b!h7S{DKH$bYJwT!WFvSOH=*4^EkWo1(*yA{kdU6|!+Z zO9>S=7{XkHs0DhSUE#KWZN!tou9xY7D7*L^ zX4G|?^ym_Sb=x+ln$|W=DmJoeWdm2Z%QVo3v`2gAamx; zq+j-sZZ;+c2v;pvT|qVg&cr%F1=%2GyXQtjZa0^ihrAbyxiA88YJXQZ7Dh$B3VUEN z`d(n^IB|Ro_vw$8!$!eh+q}k(+Q~PY+hMH8j#`(Jpx0IwQz}YCYp{1r6x8+U9&JyE zqJTQlX(ZYoO7)n?%td99a#5svQ>#{4iF*;zJhwRhTRimtp*nY={*a!+Aa;W(?sYA0 zH0h(>Hae;S!1<_TN47jL8D{)Mf2!iX3>}ae2Fh{UVcR~6_?O4R`SJ6uKYTyQIb~^^ zs-t~%c925imz7GLvwsSip(rdSx4eI5%81JKg_eb+Fw)zzH-nvb)pbLj;2Vm*0V8{c zrMa`mCT(B6>>IRwbt^+73DD;1-2-F7sYO*+pwv6Fjju-$6U@{=2^uIOuJX%E+w08X z2IHenB*fR7bg_ooJcRJ}HfDrogCU{$O!)AMz$U@J=PnPO6$?cs4``mbs~aD#ssXZ~ z<5Qe?KQV85x$X6d^^#0O;7w4ygeOB*mIM%H!*2_TSRE+Bh9I&@wkuEJSaNvbwz1Kg zAMER?3pz}RDV=;j{@Ra%g@{un_VHV8)j1QR#rHen(qz4c69+_kjNqA5Xn1i&G3d&4 zigmek3Ja!~M6(j-`HP-sPoLIH7~Q4r{4RV4s)1_B+nspNl^c_BmNl_u9;K2L#nGBV zq{c004WyQ6VxuObw00ZvH?>(CPEtB}6BYglDj^W@1u3z>_Rz_HI zF%5p%@!ThSPzttX|Agz7{g%LLv+I`I_N96u@PoKw8yi@>NjW{5pucSH-8*`d7`b?h zxu#+=(_*RaOtb7YP7Vw|J?$nK$~?PRXEPE%e~u-c^y>s*jGwDtEWe;u(*|kw4}3VI zrv@XpTzphgE8#siMXA1y8?{Xh;d~fygl{pVI<#&JTQ~pUeWm?g*QQa^hqyd(@9;|J zZldXLE{8oCiox^Ojgx=e5o(l7OCVzgX;UqSFKMnd&zhyD=Ag2!nrFGfs37ogaEQ}; zc<%q8nX96YC%JRiU_K~Gt2OtJAEg14Gm>kkO44~@bx`*?1os;_2}lt)k;iH z;)U=Aa_|AUzr*yL*EY2286nPvSHd&x{z&K>r+-*Ad?kwWdh=|XbCNTZKGK9)d>WIm zH^4&H{t=YVa>x3iNpicy1c#gJJGsqRHe8d4($)bqQH(Cw+VHhj1SHDzbS`a}!~6%N zvi*wAqNN4dSl;6W%W|FG<%p)c=Xh&2Z%cI?f-)htO#S5~v!f7=77Pa`*>fq-GV267 z%UX z6ViaKj<7S9X)@_UHdb=p)k0!Uz&Y|eRv{yF?Hf9 z-!~>T-mr{x7;5rBEe1sY)$-g=oax@q*<}SNn%@eRgZ@Vgu#-N$Tf(Jyx}`ZQznP1m zA=`IRGo|e|?_(`=U`ftbiX;LEaYel+#|NOub6WROTGYHtQmCn4y zds1Sc&cw)T6)_b3oulKzVx>GUI>5-OphVEJ08@ZSKmww|pe0uEZ5Kc}<)O*aUTxFi z%1tH7;S`YC^eaiiZ8>hfS{O@MRDMKteaX*2UVC$CJss`W)A$J}2`U&W4xb)IG~B*@ zQ~TFa&i3cU(WmyQ-hSl@7>-{QADOt5H4Fg4%YECW&;{yg{2*UhOQ5!TN_j*m|Sb!n0pvb zt?;b!ji_TheV?&Zt=5D$VpBBO&hII{&1Kwle?$smK6_EV9*OEdX!M*Y##(sW;tEKH zN`Y~e$~Ee{Cu+8QhR<#s{Q^cFIg4<&XiwEXM)VUe4m#~P;1mm7{n>fg{FG zFq7kNXaawbpb&Gh>@#ndMc}6Z2;wJPUdD9o!Rr#VMvK2?_QvKLh=z95FXS3ZU$JHC zXzwxjpn=H;%EUavRwu7CAdG02Sj;d-&&g%d^P`Xm1?AtH2^a-&bD2OY02V*dJeNa3 zYC1##jyGC*VAYJVN3;pse4uLqG(Zaa`2Sw1f%wd7B|1KE^+XIc1II?1bxGwUP@+NS zlJ_n7tw6S@90nlT2cbVn5$W*=-DuW(m($}}Y|vL+`2}J*qWV!p?O7-4U>1eAOKHG~ z??YDDnYBBVg_bP+)8t0lM|_2a-g{#6KkY02==w|XtM=|p@GZLQ(U0?U8m|2{;_oA2d?wzu)z6iY0!`lIYPHBkd}Su01Qcfja6?*rZ%% z8zHT2)TrO4*><8$g)yng^FpZ|tr6Kc3+JaA7L=YRW+}Xv2elmS!h{6NHI=HjmZsYA zY-BJrgEvFROl_yD>i|0lcer6U;@ukofIOhV7}B z*=Vq{8)H8Fr85)EOiLggqn*%-75~}sE{l3#n(fkL8ewk3$-8_j*yu>Lx3mu!03#d* zf-23spTn7gyWP*hK<~3czFU_&E_P4VhhVe4tHoy?{&%&6c!PKYDnT{m1VBWIR#2cL^+J{(*%Jm#JuM7>Y zejvx+qFdD+8m=-L9V5Q_U5eZ(>pn~m4c#_sv@2cn;XGl&0V3_T-lx!3L-h!eEX7ll zpOp!IU7X#-=yZ&6bz|uaa=6;qeX}-_a`z(`bK5%A=D-N@6Au>Wqs|^q$^6HP=e)y1 zRPlR^5>3iPWF{UZaF?J-g?ip90y&QE{4NEkRFPNDL{8TAd?Z~W4#u&e8c~GbTyU4x z1O{hMhYkv?PeXO=0Uy`6(>66j6+VjLW$G*^$^h+2M1~iiFnCb7>puH#eE;&8_4 zmVDOI@1_>+KV6d%eI0Y+<;CsMq@;JvV`*7Rw2bYcEWS2Y4lj@U#?zC4_^9-p2SL2$ z$9~7y?yVO}=!?&q)3cE%X`hm@<6sVu1@$68Us@y;tTbz#k~IDi7&Y%9NaD%bW|hx5 zM$nl!0E1T<84D5dUV{=(?;YWL5un++G5K&|h~3S{Bu3;Z7(Suk)13t3T5TVnl}ohVb_czYVn$La44ayuYuIYQc&c^jHe;N+aGFV@X@or1d&xDYbo-#BpRv0z zsq?Y2t*l>Qg-<9aT4Uce^w27MZSPNaM00pQpsZ%l!d0XeOVc>%DEMlV+eq7KCpgi# z(Y92(VO9QVZKIirCwp8)KA>`T6FH^dhD2|SJn`g@INboE%;wi~?6TltQ)i~3uGql?}zULGAm)PSrR}rV&LmAzz4k+6aC{F0W@%~VLd>5ym z;Z@P}K&A;f6w??eW25gwU6zApO(RF?LwBzKv4fJO=w-Jo|n& zb+(1FHUn~xjr3$yLh96UJx#&&T~kKmF&{KOWz(uY!eLRgPJP$XEYXrr3t?Rhp_^9t zsI454i31&_?a(S)!gLk;4h}l0BaIIzimsno4O z4Qr9{E0Cex)Ak*~ujQH+@Lo78NgKr0h&JGw+1?GVm_Ukkj8XH0Vn0QCH|-LtHV0M`v_*yDk_pexE6d0}j;6 zj542;@ZBH!v(UYE-~BH?q|2&pr}XQfv#I)roM&6dl>84bXxP@1ejpVEqQlC^PKy3a z-}6}teW`gaxO)*Ji;^1R>Y3DQ-G~v=+J@0qfvuH)58h$se?EA)yK`WS;kj#nY2nm4 zJ(|{!4NVAv8|E>Om>Uo^RW+A*b@$H%SHmUlUPI*SgEsM2x&4|wLLKK|c}H;QOKu(R zjpD51hV_~DoPahjx>N(u9%v{28m$3=s$_F23pt*R(U&o$81^nO9S|Gv(FhFiro96@ z%s_Sa(Z9zYhhNwA&+dj7E(lxSkPtk}xI?ST&-zB>Uh@vuv@K5mGF)UY-;dz;CR|Ht zz?ai5le*#qsB2tTv24sg4Bbm*=f)xFp*Lq|d!COHb05W^iiJ-n(E~~Y<**gIx3k4R zi8`$u2%7&b{abeY0>Z}`*o+I#)qBj_14`>D+`lX~w;$Xudm*60o31mQCE7Y%*t#PK z8U8cAW2J>)>VcT)EB0-~umYo>wV~ReTAVbk`cX|;OWcC^_WS9{nk&PMl-(o0F3pzp zCNw#X!8!w94vRg z%S5S6Utqks@sxGoQcfVV!lcvUfyK-kv&BaJTc>!V8;wSgjSUv?1r`lG(9&?+H<{(V z-|Eo*N4#(%FYcasC%1h>aMyce0>%NU8&<^gbHC9t19k!7>?K|;BYXNoVJtOdK%z*# z9sM+@D%|xk7X*xgZ8&}4Oh4=-u4xV|)iqNcZYs@9cH4bG-7U_ivb`&Lpta>cG5f15 zx$M^&{901%+}!vz4a&4_9){aNfD7A=Q{vT9^Hlt`xN)!^#ShTRwLev9Hb=o6pzlPlfT&`Y0?GQ=wVFwoS^UYev(>tNg7LD`?Ok+;<)<7SW1p zn*b0<@0VB3C5}77YELqX!Uv1aEh-%(hzNP9)Zx#)=yk)I*5ninX^|^cT_%w8Z?qVq z&J-L;#k6wFXpig8T|OO~_NgnV;kA7d|NTevPjqPIoSaCKfHv6%eDPt(#?<03g;7gF zz{+r^Rk!hH!F<=y1oqGJO1cbj=c&&LDyE~=Ft?uLqK;^wa*$ccNv-uvp6n-(-%&jp z#fDXSQUP6j9FI?~b-tFT@6svTTqI29>2hjozRwEMc^-^uZc-nPhx0qS4~CtcMk5FHqY)D`g;Er>>F;G$@vK5at)*T*PalIVY3dbLif{eAB*9G zije1_my+SC|B-D}sNVw(5$ZA4{1+SH))&Y5AbEau6afkcsE5QCl!y~Sb7nO?H9AWc zrmr1YcIV;D+iUb8fOqg`E~Egy81PUSRd1YL)A;%^`6hcwwvhfJY|yOv-!xG0u>v-U zyyJ7BN>fN4c$UOy4u7A zK*a2xWEMquwP+gK*Ah~4)}pc!)^_btd*xuwO^xZJ8sQ?6c;%J+S5iX3c#Ao#?yBn{ zzaQ~T257^75hMH!h@crHP^$j!O}2)!mVVL$*WEMI3p7wuA_yJ@%u&mI zgQF#qwJsnT#+jZKV$fJm>1$Ey$u+(58E_#?JtTl&v)VJ1=cE&O77|(J=6MD#yI=b5 zcf~_4U zzvg%36E1?ca?ahgm+Z4QtD!qB8IM-}sebVeJ$0)azokIDJ-5~Z&^QXf!NzO>ofB9f zS6@2bmP8(QP7T89G=+Z*?r`<$EI>e%x~j(;1In6cKTjSPskQ43(ODLn>XVxhR6j3J zBgE3{G<0T#C@xP0QXAPn4#4qyG`X6nPmI_yt_gt5rccbv$9Xh`h3AAQk)Q>$WTri!udK{73G+?FA5gei~UN)TT!b&ld^IBOn*fUYwXA+ zLH<$pjU%k~>sZ8Fe2`YI75%mFd%H>wFDSsfelBiICumj=iv`6lIHy>?yq z!s6^jo39sbA|m$O<>t=Q`K;Gh@r(3vhYdQazL?&Fn5y&a1^Qb``+u-ufTFs7&GkR5@0Q5N6QUA2LAF?0d3oa9+ls8tMqclq zmt-$`jz9Po&K7kSswBNC@*5i-wOIhcI5K&q-R^_zG>BkA*rT1IHURNP85G7Wu}4wp zLC@z)HgQG_7#e{ll0oYV=7kBlg&|}A{Cm^(9jyUnlDCd-{Z2ums0vcWjVGVy_s>r_ z=RHfVF6-hw2@h?@vrSmzU+=Bl=%1jxxHM^*AIQ3qhgYq6+y@`uoxKmDN%ktZ1~Tug zA)Gu!v!PK&Vx=uFIy#WuQk<}U`PU9M+JHGBH?$YhH9EU~+E!z!OtnQcCjVXfyEcc9 zhxaejqs_4h0l$&u@zWCBHxwT#!6(|v@N2upWy%x1nUx+`rcK=7uCukWG#u;3pp`5; zYZHFTxR<|j7X8)T$>=N}GQQbRN`^17NdYaveT^Ll(f}?D3G+0Oqm$pg^N$mE_jdImrr>XG^tQCi+TT2e1@RMrg86GA)z_1*5dQJ?m)q z_}m$=qK&sF)UX=@Vu^Glu8I@@T0`fHViR_Sb|$P?at7*)*KXaD-pl~jMOd13U2`lnOrv==)GLUm8cx0eM=P|Ifx-rBw4TP^t zddfjSVQ-Y!x|>>t?C{OEcNqu)NhNDSpjRGp>&22#)bjIL?5yTJ_3DZBV7yyi)-Sc4 z)|I)?JLYe$x{;heLGBnM#myjwpRHM|1ndbAPTKBCZ;?zX%Z?=ZuY)-h9-!HkPwg*a zD7P{n%@SLJUHq2SgWSV6U}ufMjAeX9=G3-vfu9ZOo6R(Mn@?5L8;erf|62{zy-(G) zSw@t@F@#@#EJbb@--2I%K9BPXSycUG+p<|Ya+T(EDh~d`EzkgUWaD6sEmz`g);h2( z8v~5^A41FOkOZJ}0u|q@`e2tGE8!pgNOpYNG z1*8gJrfbcwhEW{PNlh3i!+z%Sk2eKndUssEwZ5nIk*->`2JnU&s+qJJflugG9)Xcu z`XAU=rK50o;9lYM6P7r*g$!zHEh7ME1&K-#yuP%B)ldr$g3TjR|0T+wKW4%-RDH2& z*Uq^9#fzKOsp0~Y;2WL26`OK|K3cW~@fWYHydF>Z!+$N+J7rBpO33F;IxjB;p1cbN z+iRO|%vrD=H-EOx+6NPOFa$yk>*tcd|Hr-22q$(^axJdhRwL4tXziJ|SEtSDnJ_r` zoh>hXsBJv9BRWXIJ}w$G9@k;KD+%xHivLdrx(2G)HS%PdKlaDB5{DgquSvcm-%$lFFdksdav{?MkP%ml z7y3{Cgf$*Nds8dE{Fuz^{(JG`Bf7hjAp}n!zO70-o-UYpjaND9-{VS_=aT9+0w2Ev zzx*k3Ps&y*k+!FsFnBRxSq2}P7uI4Hk#Q}=0#nh;p*Q_{haOE!_)kD3qA#Rj!PHrA z=Cr3V$=GxI7rl_~T8j}oaTqCS>ZC76(s_JFjy;a0wCBr*6Xi2%#K;YsAL%*{*4 zrOqqfO0X#`S#B|l31>&knqHI!j#|bAEn=o-hYy|Jx-n_9NU~UwlBhZCOzBIr%FkWS z)newahg$Ya@O;#$pu|aN;pEAp=MtjnBnTV9$*l&b|9_V<-8os1`=Fy+C8+1+Q|z5D zy@Ug6fv*Ee+rh4D$f!=^x>9FrBmqhjez%=0pevE&_|M9x+F~~K4%&pFNyu75dr#TwKL5r7 zjTBFQKlZNWt2XD3;0K>St+qJk10~1E>r~?kIsua{BaS0hq8=&9*K2xU|L{=2$NMuR zAEGp)iRWJJep#)uG~w991Q5e>@OGT!1m$&yEOyNDRgQuPj*n) z2YhApB0mwrspa_iu`@sMq^#=WSFG;Zi$*}on!H1{nOuH)D2Mx;mqrSJA$FP9%H$cA z1E1pVmJ!7ki!JYYXT0!BGssIVPs|cFaCiD1S1#~);x_FLs}S#AGNPSu_G4}E$t}r`F*CQSOa9P0BWnQmH{vN2lbcnr*doD^nl+HfJrf812H8q+m z^wkEql3k|7%1fM{j?6R`w6MyzD(9|U_dZeoN(2L(V$I4wqn#jRN zKZj1kiwOPdq|y8J@AmN9@i zVeba^Uvt87?s@ypGUkE-fr;dDgs=J1e)Q8EsQigbTF_0Pj%Q-$Np|>Y${9NCd2QX( z2j04uI~aq(*@D8x?%~J2x39hs{Dxk_(h0vj{iEsL6z8!>cqi^T zAig%a8s%9vcazH!@RC+^431z%h^FHxAV=Hy=j&T8XD~r&QdWamJ~x7)2b$i4DV09E z6)WiT#vehT%`{a&6(lKGCOpyInC(96Pal89Kt-!f1o8`~US-B`hH`dGUnxAYBm2E; z?D;fVJM8|s`AJtQ!FY>P%MvEL=d3;ao078v*Yxx~;;v#v+2T8`wf;6o4=1ReCO54J z(}oc%pC4k~9SAmUHm~o?xVfRLD5k3>|LNv)w##(mY*>gfrp{s{^05+eXIhCM{XbrS z`=%Wp<+FFRr#Rm#%$}cpGZ;8I!73(` z!{Ql$h)=C}EuHw`$K3|9;9m}Y(ypalw`*3l3H&);P41TcOuP$-zBZGm{|xX(JAwqSV?lHX9Vw?jP}8dgLA(0ygncNcd$m=B#4D9nKjH zSiNqt|10R~fsPJ)V3>h6zXEw0LUsuyKG!Qt%^peX>7dR-xwoJWxlGFkaDLA9xZmrb zjN?m=b|yb_1T_tU#^OrxtGAr)%XPhtR|;?i)<6VRVRGwR7ay+dlQwdsQ@!_oFgTcz zp_L{-SykRhNQtC@Puat`vKKe~u0E!*Ltnb6hh0^npvEDvc$E`$oZ0_Xuc{u8C;43c z-ZvBo)2B|iW2GzM{E=)8s3ao^i&qx86CK4xIZa!MqsX3Mo^G!~;2&F_K_Oq(SF8uu zUSv2|0?uj|JRz_1U2yy(*AIg`y?p%a0C^{ohH)NkkWxK4|a`!2Klxhvx*Nk=5M~M#P;vm{9c}!{t*X!V5uQ`~F&(5+<{(+lRDc#oC^zFBa&BnDN^!*#k|8&OlbzSj7b_HP{;?S1Pz_;f zU3Sjx+b}%`dup(ZvTrZ|p>bFrtJW$72nDY|sgc;aD`MrNMCPZtrS5iJ8yff~wdAki zKF->-pv`}ucG!;_8UL;l*vADET%0*sOWdZ=jdx8eS;zy=Erl(OYGQX;N7x*NA;*bZ zXV|<_lp3HGyl@$$R@2@02$z?UJ$v9?p)!%JTfvjyGa^F?rL0u7xp+#Gayu@3s?HogY7mau z$w$@9sFNU^nmhpfhxZN5fA2!!PWIWjFceJri+68fzOcM(=x~(`%B$UvW%6$J$xF7b za0f$1rN@J#wwaA51;kqyB$EQ6rg;L{MShZC^$F(-FusZ)BT~UZLDE!oZ1JxSX{ZA zgd}WNVXPMQ7V(QRq)5Cqx|NpnHdtF$>0@LgN0y%FFWP4TZx8{~xV>-0e0I`bDlrF~ z5CzCN<&i_$(jCL0{6Aj_a|oR0yJPPZ%McN8CV>>r;w)pY80&~}tvBd|YaPvD{8Sd=HjpPY>QlI@DjGcTD?_{QQC)YD@MHHUV z8%EQ##9M)~SP%^o5**!hq81Wb6W-UOc5|(vX<3W9tf5I=|FE4k);>RlD~&=)OH75P z^x&wLqd;ENt5o({^1Md;%2EXVIa|aEPsGw2C(dEiV!Y)p)Ujaz_Ldtj*wBP*xK%lS z>&bk_SW7?)Melufg226JstR$ahvQ?wOfxw*vjkXB^}k%jMq^!|jb_!#pTdi-kFm8g z2@}A}jx((4f@DHtscUQdE@(fZS3E-^U#Dy zFgQ`ikZ@r;@2gk<<~)0lOm&Y2;!h@#QO-NT01OZbq&GHuS6V9~ATP51hvST^;)%t& zb5(b`tfRiwnjOqk{7A<|n;L|Hx2X{ZdJlWO%*{)hcxNVJa|Gw4Nz-xb|AbFTSHiO5 z#Y)5@heEzLl`Uy2_4(x)lYAJ9KWZ#^$(!jBTr??Cg4biBau2%sf~?iA6l4dvF{wyb z-8tVU!Zkr`3b3pXssgt5N!|?BN&TSLISldbd{s_{v+CUD_|ws{A!wpgl6~=Lq@|kU zAiJ*}Hltl2|doBm$K6WD%_PWmg2hAIT+@}dti0>SMfVsRUJLxkO zK!I7VBWk0iXC>m|xwpi|!)FZc-YQGJm5(u2+OxoipV>BRvK<>c+;Pu!mC0QQU;=;$JsPe}{mddpc)at!xX)Rg3_4(!^H!s7duPe(o9 z1aA;G6Nj=Me+nzg>ZH6Nm2yRX$wLsVkHK*Bt$686GcZ_xhz?tyR+Xg}*OT8&=8lh= zP9`^Ol}B)KS;<_vsL2fB}&zxn1YTU78Az1u3z=#)yj-03cn93lM^#j^P5)W@+(z>3gtXX z17C|58Op(`UMRi#`*@knl+h{^_2W>RdK;IdRd>}&Svr&Mj~V#8D06b&A}@4+n4=-a zj)}y#I6{0gRkxYNsslYLpL5(=8Y+ty2al(Te>Kkt47(e%-Ad$tS`Q=##E1{kN zb@;hLFH>gL`x|_V?B(@JlRNveb-uY3=+EDe^qxq!f_RN~bNDcT4-DMZOcWoQtV(01exzmnfVmcy+1yAaTE|!`$f0}@qL6htOyAD% zzz$hM`qKOK;I^{?;bNVvJbO*_1uAV=4HM@QD!xr@9uT6wj(SettYZSCewYuE&jx*hY?TJ zUz$6;ITyfP)`6s|UX<%G&F~iK0`CJD+ail&MP+L0YBECuUl|>JYX{JHdmEIY^qsKV zhiufT5<^^4j*8z^kNH9|!Sd>F@F?w)&1ou8;YH3~X8Mt7!p2 zBZaxZ354p@4Gr0S8|i9SN}djV4|y_uVyBmA(C!#6L8S0kei6-lnF>vl+V}>|*ggX4 zJKC*{cA)7IyLSeO!mO9hkPItgP9?e0-9B|LPP!g_e~Iw(DpQ%J!qwY%Z(&-5xXpf6 zH`puXARnpv8f4mUQhRk60s3S zQakn(JP};fcs$k|1)=XQp7F3#(x&-xv|DgmnauemDn2}~EpY-`NzIh6vq$5Mm1km^ zX-aHKfdCQA7hAB?>u(JetEQ9%x(`u)cSZL1Na4#YWAn0w+p|dY*V1!##=yli5OC0U z6nj2>VA0|fiFMH(s7)Umub%5XtE}kB2_MxL{}YuvL*+#R|D3f-e8~#`0XK-0pgN-z z%h08HbKcK~EDo1|qhdlYou|wwWz9MqqcoeI*RN1#+sT%1aKA4@1C*4*^fCvsw)<1O zf8-QM?h9X5#6NJpoEecaE($wtZF<;aWIFr?9Lt_gpg2Qf2_`9KR_)r8qVc-sVW4d8 zClDEi!TYyN*9Qe68x7?&yxN3p-7PoW+i6jY8(=+`o^fnF(ln1 z^^m!yDu>CCb9Vi16imq@3zJ4MzS}|sdp;)k*`P36yob-2*qfl&sK+{Ny1?fT3OKKX zQN}M>y@Ynl8R!YOT`3zmB|3(vQ!cq6C?fDEhx0ELUf(CW#*0t(yrR)pLXs z^%Zk-3AvZHOL|cjJ-DA0+lS&g7NoP=(hOgjF_0Dn3>>%H%WclE3PYfuSqUV>zLtAw z$L99gp0ne|$Zz+D@B7a6R9cf(-;#WW+T$aAG(w!C-GBDX15@|)6>m{X-I!`eH& zfYdhv8*7R^w+fv)8>}Sa(Ug?fucPhX$@-Ml?*HS>oeOr+spF=Kn(Cm&l-O;$yVU{4 zZ44b_hk<>wlTulSl4=@tfkPch7u5k-EmZ->4STk~vulq}7}#07mYG@Pbclf+nuX$j z{E+~g@*y}Fx^o~$VaqBg_Je#pB{GEqK^&*`I>LV3g%J6O!dHFa=C1KxpDu9PQ;p>l zau)N+Iuw?DFD{swviEh}wWj>O>GSuM>Cnpcflp^!5eBn%R=+7+Px_P#gy7v+{IKCE zQM8K$EzwNix?%g~5-M-aC3Lw+^NTAY*NFg*j`iOB0{`K>g2P-zhV*lwlfAVbbY5eK ztx9WKpjmRlaGFTG?phm#$L>^Xea~8v;E|Benw{{G&1y_v{pCiIJiyK^XF%v@%n+ss(aK^sfca6L&)|=UXWGFC60@ zCt;07KQD9Wkgk3my10LOGYmT)kS@;;*;tV6mQ}3}zB(aJ#)IP6^|n-rLp?bA!7qb* z;8z`%!XSxX`A8M6f{>A_UuM0YWUh&PI_AuS5lIj`{-aQhWaIv%(?-n*&oOzk-*?<6 z4y9#0>Ad zYFqyi3yET{ziA|lIu=8h4OBoMgwP>_V&GD$e?P}=i2ibl-6Sr9Vuh4iFR3Tla#kJr z4b}|KKj##lHuMQPzhip#k#1JX`Mda%Ro5Sx*G*gpOkXqcOZ0~|{DchZXA6&-x4-zF zzKzr(lGNIFBE}t~4xt$o0!#j6{vT6~O7auZenDxn6#-O2;e3GjoeZ@Y|&259wNYWRl$ z-9jH|y$wN4hWxwA*}HN1?|FNi=68|lwh%`)hI?bjRbi1U&&gpel>iP4t$$OX+()t` z=4;-@(2xZJ8iA-fACkzt=gg<6g6~v^vlo#E&UVXD*M?nCNwReS6F87~~+v ztn?V1Pcs{Q=S(h8%Nh5?%p>rGj^3gQ z@A8+*cFtnX_`{2a3O%i2CEbaW%!_XW%FA_uViR?%2&yw$N^{Bd``x1CeijVd22=~W zR_2SXj7#$pA#d)E(k&7n6BoK_7`v-IUaKiMYPef!o~mqFQrf zS_3)x;+x1(%EqlPOcJdWrhGm`>GDWcpWNgYjwrIK&(bV259VSlYD^-SP%Xb};C0v) z_q$c?5}0^9NrASf^g&$WJj9Xq_ z_?>fDuxyfSAhZ8`Xq+yoGFoMie{Y-kDp07WQ7`HC>1()9Pcs-hd72%LFy5blvVQ5? zx6j<2uUUp`&q{JV)iJN1L_>AJsSvaX9es~bi3H4?vGzM7!M49L1*6xiN6vPB6+R9% zGvB1X`7SViqi|ikO0ucnfo9w=FZJ@CNG;hHc~qFX+_&4!6GvC@K4@A<@{{3@_~8?# zr|JX39I>yq0NCQu(nR>?3wQFT*syCk`QqG`52`A@>q$zqIaYMeWAR=575MR+fS9YM zm>*`HduP?}I=6E$p9ejd&t-VY^Rm4q*SP{(S78~x9ot{5_a*cJT8=md+YS@T#Q%)J zv}?0^AqY_{NCR(Y?7z7+ga3ZC)P?7viaodyVaHQ1DSnD6?+%@Tv7}?Q3|3s=D zd-D@c43dvOHz^dM%S7a-a!E$B@16o)4+^Ct@M@6abda~L=}fD{XYtCetY;Gqo!Q$; zt_W(=k&l_MPwtB(ahS6^!6Nt|L3C@@m8Im90VGCoxJ7>a-<_ox9AOoM7#&)om>_{a zIm}urq<+6bD(ZyCO)BHuPB zu%h{3JTAHBD)X*{hX_Z+7*g>Eg3X#_`f((2Fk)(duH=Kbn4_9ld31w{IUl4SR_S{G z&*LRi9evPXPpk9;MYp#M6oIQPfaOMt-&5Mt>+eM`oGizF zK4c{R$#SnViAM>hb1Snt^IdX%ichwGUi{Z@68$d@?OB?XczOiye7U@DBpnxtp~t_I zjs{@Sj@k}Y?@$M@Woo$$WKJ%0IDp~{jQQ;tT3zzFJBF$>TEaG`I_wbo0@!rN`M-Zx zUbeJtrC|dJMau+nG*SG%M9a<{_r-2cK@=)&cWnP;{~+O#%TRmviRm{e{_=?@j_-cEC^$y8k2*1zNR3 z);=}qb0*I$7k4&&ROJ{oY84{LPePZ(iWpoH`xzg3xgBOM)($meAu{f-qKaB* z-iN^zJ~=M!9$N+2EjdeU*kf8h-m{ghk-u31oox(qK*kfw$5Sd6>s5R$5)p)sb!CH= z3_k3kpR>A;x7|cL(m|G?<5Ix97v;W3HGGhydYt%Yxt8`zvVhsTL`uGW#4{CrGRq=; zv*X(hpe9h(3;&ghYy3js!oo}s?esivEV8|;F`six5d#PJ3?o+F@t0dAgba70BdILu zS53I}wXweU2Gv9itCiH--xl?f zPhzUWq9S96XFtubKtI912yA?eVP!Y`b`Sl+T9(DNDk$%@&z)p&j(d>i7=E}N!|%EU z%O2wOhG47C zFESeYlK_#|O1K$3TK)x7TAf2=Bl}NP!18ut=W>4n@>S%=mSw@gffXys^^T?8?dC8l zoz`XiP1YvzPd3GP7&S-3OUCIS|3E~;zq@weEcE{GSvUm80BrlY13CVwXtu@roKfuf zCCiR1`G-u1((uF|{!>o)$FFQySqe-+frg3jyg|Z*{fU?Ft4$x@;9ZG<(&>_#|aqzw0NLew$zN=rr59 zgDJu@DERR$Irx&~`_1K_f1ImeC6Aw)V>y%jnI1oDOJK@xm~*(Vbkt+`p|du}qgpMF zX{`S|9Mvd!Fv~sjI_d5QFx$#)el&g-VE)8bJI3_eejHtRT7%9^GUdU^&cW^k4j>VR8 z%QMNDK+Qr#WyGEb^;O+EHupDY(B^^kg+u3Dpn}^pWNVgZ=sFz4yqU>np|07b+fu5| zSMB+7gPUXU+EzWH>MY)CEF5AF~~U9XBZ|Cj$oq~_DmA683_7X9|&Ht$d%^gu;}Eg z!ZqdR#8BupwfvGk8IyLeTJ^WhT@0!fk2uyQ!I}l7Er*VWRbzA#As!8Q;c zxJICR5$iSLGp`pu$M=h8)+u$(8#yR%51#sgsJ&!WR$}}835D}#J;_9=7p2?Wf%2zz z#M>1IxKX~P;P+fmSu9X_&E~E_CFq$tdmc-CXyuyIM(% z@p6~o;2ZHBeIk$J+sPeUim$mc9i{m1d<56`(k&JP__4h3g&z|yNx2G}R8?qm36=$q zywVP}LkOChk!1tVMoS;*ybSfe-cFxL}~~OSq)TiDM72z*nU<=;|uVJ^W= zZ$a|%XeA;_VJh1U!q0!RALIRxYQa1yx&u_8&G0xv*;mS>APJD-wOlb$Bv*T}x)8p5 z95v&zqSa40XV?j%OV%)bXXi;X;NKZiqL}I?RPJruRjjIPEWToUW4|@tuQ8MTTTX)l zUi8jZlp)(`8`mxY8_@^M(hUowl*8u3$ZllrJmZGz&tPKcV9Ba9fv<>=0=!c!x}jn^ zGdb#H>)e#i)0{uJM^G6=FZ?J(B{V8ILPaK~YSk zD7Vs#@RSa|I!*gorNUpu*(g{XwiXxmOzYW&=jw_6lsX}n`1g<87yZw@s`Ji=RXoj3 z<)ecM4tg$#56&XX4FxVdux;rpBB$H6hmFW~!@IKM3$bMqZfPCRqM?k`E-y`ucY}XC zhKXr}oZs?;)@#0v<^`^JHraBs$$6*!VjtjJoQ*jlY4le5CNc1%M?RV@-AYOg*8O3H zt%)b#WTpBouy3Oi@D_=o$h2G~fRV!#0h+Fo$Hpi5CdQrR(G!qTZt|8?(}+{xC7#*b zK@t6&!^MK4?O>2(EU5BZX9oc>i`SU#BGAV0{Vd2YzdU>|7%&dH@lOU_49w)*?KpU} z@Hs~eCBvdxwfVK`V#84cxykSRacg#sfwu)MRDbMy4!01ZeOsP!b@93Jcy^nnji+$# zwT7%e1a&Jl;gf5{!T<3BNPq_pyD#0kQ0_#U`~;Ax`?gIC12b}TVD6=z0`qdZzhY!1 z#n$^3&}E+?ag<-VFxt+X%dwmkHk!gMy-eKOxjctHjNWc~_(tP%954*Dy9*M^HT998 zWK|SFGofx$x z1&3hCsY#K>Y1I&^?c(b+5Jx=O?^zC;mLy$^G7*3T-S}H5leppc+L@>bH}y&yF-R|l z@}&j?^7OX@7@5U@Xj_Ni9AJC&V@9({q19+}jYTfW4AZqPfCJn_vJkT?q8Wgr>=p^- zC4mQ4Dgm6l_%KP3*Tq!N3+@gssQ|k?Vl1>t4$AU7uRgs2?ZCJ2t-(z&NYG-(V{oI4 zl2gZe7(j%71zrmPwCTKiIG=zCS7NsDz)rm`L6_kDWx?z0IDh4N{~>@f)0LP>#4fH%;S zvmd7m0qODHQ_~P8nd*+2m%wX-j(CL1f2uhFbbi1c93g&gE7=ydC35n*WJpQYw< z8BP6>0lVDFFL#}bxtsJqEUbOgqVfj8}HdHI3a(>C(C=6>IQmYTYX z>rn$)73=ru)5@HURTj8s8=Cz+(?vl=z3|`bb-Tf%$G--DzkyEPmV~1Ap9cQY`*E@M z&Pv$UB4l{+!=M|yat8-J0qxO1mxPJ<&w7nR&wGt&TFG~t?U#fKh5PCCW2&b%>@PLF z$uJ32--?RFP>*r-I}@6)@^Dmk{FyUdpBqJ=1DjhP2j_Yy$f_!l6=5PTtlkZS3{eSJ zJDV=!qtCK~Jo*9&kC-WU*S8zo`aU|1Zs6xvZqIR@;~2A#M~4F`2t^gwTX->FA~Wo% zbRPZsVVwP{I(JXJzlpJBJL!bkAUTC1;1Z_E*v!mJFP?EUe~-d5boI&J2X1hSuVnX| z#E``qu!}nhbD5~FNhPtAm_UQz+LH7DP6SbW*Tk{&wY$|?E9S=(l5E|Von zacr}V_S2?pO{^5psM@K!_w08DWTGi@ln#j-YblFB+Aobn7Qg94X}d*?04PN7aJYh$ zIv3%;mDeDDI>v64f*19Al&h?W0vn8XYW5RKkh?~FA#0yKKJk{>i^)#V>RSF~Mm_e5&?hfLst<)Y8?XHui5MM=jkrQ}xT>fmhfm)lo+Ylish zg?~VUZhJfg(iUVbku5IgU;}k5^~G-4az=4LrTgRKMu!;g7ZV8czXV5LCCKPB2@3Zb zj4R!H*&)IMxsg{otZdO%J}ce=yDcQzw8GXv^Ux78)*oE_1h1r%;1-W7#8V;_;8I=Z zoCD7*97+8jo5RGy1hXWebEy%6!(S0v$6g-uAo++px?lVhAWA1yj%3Em!Zznsjvay; zpGKgcC?E2z?aQ`?ihB#GddzuHZOjWBnd+F59`GYo6XHqtHfy>$ z)z%TxM5jQE4(aEiaKTVV=kapE(?rXS|6M~&6FC#%tmNtLzDwZrPdeiA=OPMzdbX>w zLTq<6h#EU+_70no!x=hXn1T-N!Qom-4Fb#`V(4qxYe5x}WItZEw^+JjlnYi)T4?7? z2-lKcR3N(zEV(K8nOoIOIde@GHRiZdPDaY4`;%dMRG4(K*xd(|v;h~<7h`cK^I0OL z7psfWkZ6HG56ItmV(i}r>ew4m;x8I+J-Aaltes%AKe}E!3A#?e7koM&vN;dEVQk7A zBoR-c2>O{2^b4xvwsiQ$g6pZ4;#g*#B%p4|HBl)23KHlk+D@`XNSCa(P5a~4{6zXM1 zrz*|jr~A5Ud7Spg5s5YdHm4VqGre({Q^n^CyH15m9)`le!T+46Rb-Ez^5Dr%z`k3L zb3LFQ0Ozk_B_>;=K3#(E$jx|1NJ$5RF_iO~W#yXXI#cju;!ZWp?_-Az5?iMX68Ces zIDoI#9Yr(%mp%a6MluP!d0{qrWIn)my$GGFFdLp(hnn5s1;~&Ykco>*Xs18@_6z8z z=Q%VV+tYyo{Yj;M=d?+OO(LtJx?fkb6m`{k5_plMkB1jC zT}Pn&CqU^H4f#Y0I9TzmkaDn0>JQ4-e;*FoOU7Cy9EM*NbKFmZT)3U)dnDZ>1iU?p zXeIUv7owW-rF#*v7*X;Ay>X{g`Pk6#ZYGi{Ux-3gzMo);JDX8;V!(9hgQ}<6`oHE^ z(XDzyhCRgQbhL?48W4dn-A{s{?i%hSt1{g@wU{OYf$yN=n@8fMl#Azeo7gpeW?&z zuVr_CSC&_wA&UsH(47rjtYqx&4ALY@KC$b>nK;}jlyZW~w8?1mC3r- z2)_H~1^q#AJ~N705$QRRY2c{I1L&V$c0sDXbN)(r6CE?$FX1o+0}YD$XM#MI`x~)7 z9XPAO3#&ey^**)h*r!D=AB`y7leHk-f8l!R|8-nOQIB7!jIwXxDkNv_@aVcqX6WbL zp2eJdRP^~Pkqvggx;@=JH{}AdJ%=2<`k?UXHevyEB|+c|vZcF(6Z|yFUMQ1m1A$XV zI%^|Ax_5dO4Z~(w@1e(-SBohmlv@NDGJV*G2(ch2xV1X~FxMV?2OJXYX@u!N`fT7c zMufE$*@OHLtrXveAq5go=5pSFeRav4VlcGGid6Zr6ooj9@o$pu4Q23oDJ96t{B<$D z$Um(E^n7WbnBrVu@HJEIDOpppt{Loot4gTgS|DV;ZzPHNxWjqL&4^y}aU7r{?1re^fj7)(YU@R#!(h?2XO#y1#Nvc|H#bpkC@65q(VTxsTt zf6xJ5cKHP~}^FL@cg#Cwal6r-IjIJ4xVGJl8!g zr}l^a5!o*WU89D7%*yO+ge-nudN`~F5(Y@(V32%mky2MWWQl&9(^Fn}^DF0vZEJlI z-S*Rh8@$l2Op5=g=(eJ&{buSK|I9lk?S3?>5PG_bZUg8zPZYhV5|nXA;TXpr-1r4> zickLc6sKfE0@{_XNJ!|JOh0-5yQ$_p$WF&vtPDu?td*LGq2%;U46`k+(lD80HZ6Xl+1-&~!l5cOER229}v7A;iAo z(jMGpi~45T#*m}S*z*FkwA(lTqP6#Q#`2~v+=@qGt0Gy4|e#x>*-u~Zs^M=K~Nq>(qK+&mFO3*h%Zprb&( z-?gsV>Id}|`9q|2n$SS0-xwUk;bWde1M-txZCCA<>C27D_UW=XR{gR(X$B5{@{~(Z z=|tH+KyaaVQIQAzqyfR`lgl1g&j5Xrknv7?a{0(}oILRR1zH<6O7g5(6;xGflM+l-~fgIviDfm)7mg zzw4G~UV85D5o7e7nzb+r~~g zSRH#8GbJ!a#)M3{Ydj_>S{!N@EM6{R9G2?v=Hs{h?k zPKlu^f5>+j%sbOR+bVP~ydwqe-KPG$s%R4}>tK_66&&yZR&^K0!;qapf>PZFdJJ#q z{uAmEb_M$Mz9oQso9|Fd&R8U#sQ`5<$>o|BpQP>Q6t+!INJ?lYK{B&87anEzM&O+j zKM5rI68#9UX#6NX9JIt<&vpft!53%))gRr`YH5K4askQUGGt=^J6)OA&Pf z(7h0FA4<;5E>;=b`uL+)hJC2F|Q3&F(U3yT$LZc~l7Qi7J?&y;CN{4d5fdu6^&oml5XO zVUcHI-^H&BVPOznaL{tz0d+Qi-B`jT3rNOI6eudO%h*f_zdO0u4Mop)1SlwRaXx)a zQiw4~dLgw}7FTbkePL)rm%f1PH>Orq_baD3oZXl?dT-R&^uHMU@@T04@Bb1dDj`%v zWGOZNtUc*U&ahYc4h`K*~>P=U}A=8tiO5xe&5gg z^ZtIn=X}oZoYx0VaSK?jBX+@$QZej;rwoRE^UelNgD+JAq0qt02``N^a}%Z=Oi z zwvhZ_PGb=KJo#mcjfjpgiQ7IYuX(~~rsfDr$DaRLg3QPTJ&6j!D;VuhRxajw=NalQ z<3&;Ay&@gDz{Veh+n64_-0Q)`3=1#OM-|3cp}+*G*6lxIc@eU#-Ue!{9f8Y1i3=|i z`F$s0&&fw^Kdx7XEl26TV%hSJy0fh$-#V5}&P6XVVuPo=^e2>%@5s})v1)?wT?AOU ze;fT<^E0mdu+8j$%!d4O^@#e_nA;mCINaX3#(pYcF1&i7*HnF`>fEE#FK)`%Ts=R~ zpQy5S8*?3&@<1_LaOK%C!AE^bi~SHCS&7FYBvzX$W0f)2NNWKh=A!S%;k9>8ahDPN z%TF97O;~Cez*_5{jlG=PMDKJg9C=sh_pTuJ{JGgv%C<%HTlz;AeS{hom;ah&TE!*z z?dX?d^G6N%?aE&>*9?7VU-tAqR-O*gV0!C78`E2FKimP{S1g9|V=qu}9MaP%U%|}i zf8TsbaA9SrP$rW_NU9uqJ8ba)GaYx77U*2Ln41RiCJvJ7Qty!!(Sk)wUsLq*N#;EdLVS2L41$w(S0e< zLJv@yN9R`&`LXgtuf{dtD>>hOy$_qI$w?7^1ozfC^>a#p@)(XOd7o9>IBcZLVf@8` zuV(gMfJb@cJH_H$HtfZN1td&9?1+`LjCz(h{rH2femoGsuZ+Iw)&+{kO_1&M~sG!{e~_9RJc+J+9Q&mQK!|G zkj;6ao$hcl11geH8%KT8skWKJMc}@E%Fq>Ao_Rv?L&r^?QPny zF&U$?ktk?>z-RH*mDz>1HE({3oph$FRZ@9eks=;xp@SUj+9$E>KmM+W-_P4fWqgd^ zo%HvJwTh<48UR{jyHRm^;JY|)q>3*@Gotf^jX7+ z!`0Vu#cW{E2T~`cpA8~6BoRJFmLqtR?blUJF3%hnZmVP|qR7S#&OwI4r*HaMO3xO1 zytg)RZC(G4)ao2t?o?idd~j{%SyB?z_P)g1*L`+@2Q#7capKHO-TJulF(r-Rb{e{0 zO{NmO0S{daoTc8p-dIV<1N*>($$gHR{`VHuo*b?I+x?#DTp-yX2oYaqbP=XrfPKxW z*i{E;m3At}58A6>uH~SH!}gu8DD@F#-Wz%9e59{6y>}2I>)P0|FI!Z+e5-t@RPo)7 zAe$ezek;Z5S_HtI(yi@dZsazZ$Y>1hsEZ)y!4I63UDjS{;#Z)G{H4hjwE90))OVHLi8Qhof&M&{3#l4 z+^bh}!|zvwAx<|zc4#t|CMVtP&G)TR`iX;}P1vr#E#LS`$K#3#Kz)`a7+fL-j&9g%N79FUY0pOIXu|8 z*&EedwqGU3AE*X;cS2{iTrZuZaeT=4Y=#m!cz>z*`Dp9d*3|PH-510*}Dt zw(sAYltKcZqSnt*kSH|-3ZAXJ+(fBIi*|nttG|f8+8ZZUE8V^(s2x;f-hJjni==ge z;#HhtiP5m1By$isr3GPixp_S7D>{prZE;3g~B#<{;B&N+^WmMEt?W)LvXnckJb zy5pw3-g|+%to}LJ_3GehOv{S|T^+bqy*ANrQOgzynECCTEt0Z(8NZuK3KqkF7c{9H4V^LOFkC{j! zc@rJ()J4sVZ?BQ|C)=%<{N-Uy%BM2AhbicvPr??S$V{$i#)$wvI^y8_vxC{E9I3`Q zTi43-L_n~wR$^qCTEaN}lsgrSTfBKQ+Ahns@P0FD5B2~~=FvJlG#CPv-{-PJ?#n>4 z{p_m^w^Wagpl{)N=j=V^hU#L9`$B_9;s7;`j*+o+q9@pA#%$Dg?OG!c*#MIF{c<&? zKWk7}k{^naBpGs@(~!y-L!;gc{lcN!moAY}Y4CZ_A-e1!Y?|t8vx{y)4ls-YTT~6< zT>GKyx6eE}a{T{Y>u_Bzzj?Wr-{~E}+^@EVnR8`AUIZl*6rrEBGlc{uq~bJ7BdUDU z?wKH*{;<=u3#>r5E8>JDRJd#dtWxf16z-;(7!|A1&yf_)*l8@KJ~z5ksc43nTmT1V zUI}n09p0jT+u9zIgMr+xobE*$Yx)`RlZYpf90VD?SsFB*nynY+qUQI*5J5T45wzcG z5F|;MzcN@x_4I${^LRoOduh{APyJ}Y>U7TGSc|TFrK=yBP=iv$FK#tdc#YFnuRqHr z1fL^y)tgmV8N{qNrbThm0y@EBP-<+@q$bpMJw$XfILS7f+^mO)v_)qxb-kZp|BygF z!3C2WJt*=|tzF+zD>4luOf@yKU@G+e#D;+i)0AV6YoFbOZ+LrC9ektKoq&tHfNQ=f z;VmYs%iNOmy4u^PMZS=I{6^S)=Yb_g4Yowo!l$)zhsn0=^)maYwUI7O?-bn()ti3& zMx8L=TgvesR$B?UmaxH@-KGqC%7k?DG_^+@XtrsPXWP&$ad`0a>#c0t(U2iGJ1Z4o zpDyvb{VHQso5kjD1)F}#kL{p1)Ry{K*0kC0<-ocK&)k-bwDc02U~AVI-bcd2W_R7y2(UV z9au{jjS!m8_C_qo=I*gVSg*OCwFnu{aHQ7G=dHS1BUy*$e%DJ%wU(`DeY%yd2n)pm zl$tHblLZQGSMjqs`<3foui;gmoD6C^El3@;Tb6{ix}GI@@_tz;e|G2*w?hll$1;7$ z0VZcgPR%Yd@lgwPID~goLud3H1W1jAgRDh~&H1yt{UZryV&HYoY07;8B8%iSLW*L` z2stfnJxs&>u0QjrYYvBQJNOWeC#6ERDBh^ISMff9-_c38rPEwxWlZ%Z?HhdT`n6RF zHC&lbzcU>x*eswlH1^p^T0C6kSnrmON`-4-ZX!cpo=Wi962<*?}6as42^PS2fv%554zbtWLY$JxPlTyNs9EV}n@52ur%H zif~L$EbIWRGfUr2yFZvLEgRr_*T5AEPy$JI7TfLJkKOYM>2((fUyjBvas4#c>p_k- zS{MX8kVI*pSFuU4qYTQ*ZiU6l-YfO#pN!$9KF=&8msZ4GYL*$SXl=h5@6|DtPjDA* z-UPsH7n=bG_F&&}?BkHe6ypRU6ncYi+ zz-;;EBLwYf3=jV2k|6_v5_?M?Te|}R+UpZ3enASsVZSFoyQHPA*LR=0Mplk+EMMW7 zM*sSBvB5LA)+pGA-!lMh@PmroAE?!-ohG)nz(X6{g$FuI)m7?!Y}*7Ae$sc1}CP!?#FH0=x5kHY;5Icxm_W0`5j`url}plm_N3L{-1o=dy$gEF0!N~Q!I7OJbz zN5_lXqp6F^-Wf%_db$1;@tAkI5=ApnMcujaJ^uyoGeXT+}-j@CnP%`NhUzbL8i*53(Q7O2Ax#%veOCOVf0#f zPPtIuzJ5FQbd)FoljHjhk$RC$)k$+-tY4MxvftJloHQwHh`M99%i%SCAX6_Y>KA%g zU6a-esk(`GyKEQfXYOU^k&@tTiKkv6x4f%nA062ji0vm3+uWSe2d%p92~O-Zwbgzu zKbkB;c=MWEYY}^MIV5QY890x3%hjX{khlz`@a|n|mZ0v}i=Qy={fzaL#m=wUd*iO$ zB#%{dTzsIB^6MUy=)AljibhCexF-{9gwyk=0#ubK;*VT@l8?+#b}TKiuNq%^(-r`z zKD}}sfs!t7FO}XYRlwP9X5wOkISHH5dpptlgZ2@V4)+CSJH#)px!-8Peo>mvmXoo1 zJRZzR^<=}C4>vCwLw`(OA^_%sdCetI75VtcKv>>y(*U<@hptFMfQPT>n*6!7?2GLk zjmehjM1z>hermqHd<5lIJ7jt-%ZCEtmcEtk_w%OT+)ZRzr;^e(%6aDbTyn$q?#Xmw zmkqfU&{lX_Wc&Te(bFQ8*pu4j6D@wP-@$Oy%JocvhHZit1r@PWp%J(>Cj{1}%?keG zL_IHcY?^}>0w4jZ#Y7?mCcSKBj)>s=c&`z!<@N;>?tMk~lwUI4YPP;0U=K}Ss8Tzy zmJQ3cZ9=hqC|z&hpw(Su)Fdj7&NPHa(6O1x?}~)7#rQ~a#@E5u8~xHldK$uf##*Pb zcI;km4}g56Sfz^G+H3kpnm3>DlCqDArcOb;o;es{4bDXOfPi5H*)$dX6xVi?mb!>XeZZm|Slbh6$3b_0RCFxf&W% zRpD&DdCm|)4BSBY%1&xKuAAF>W%}gGM|rP))`08fqqR*8n&X|6EHE;3Xc?|s9Pga- z3OF#nOSI-^nmtf=m8~!x}&rdIm&lCLkZ}^@6eX0L` zCO*NMP=ExOpzJrtOeJ*uK?4rpGy8*TNN?E!sJ-#zc)V=CkKX62cq!=t7UsYsgQgQm zLaVtzo#%+waIV9~lmAGwx+L($uVRa_qjlh| z$B?4NQWG=Wt%{_3VukR$J@38yKxC;hv`Q|&_A<_^z)G4}z)@YIi%hv+x&xKC5E_aV z;_DYvp@yZF2Be6idNzI1!iPZ)rS`nb4zv6_J2a~_u9AzDsYW+hA)g1X)hElvX#f#% zb=n!AYt@c}gSPc{UsF@e5XgxSZBD)I?ckZC{zdAS#j~%{d|7!cjpYTe=#@>zvFzgz zd8u?$)SVxl&Ci}aN!o#Ahb*&Se)(Owt-#)1{m2hCAO8UfoS3OWWhxuWF)*D)i#u?s znc2T7r^f^;N(N?XiLn|d`WG9^ zUE7QioFgN>%bRW=ee&_BS4KoOTl%?WX{$!#br3u4JdajdhKG~dQfYaBE$WTU!)61s zt&X{x1=A-|ah?)z#*}j&*T=#3=1iC09y&g0tVLwSX$L6hEX1yv;6Zv>dHvnA*f&Yf zj`bE}2@v6N3lbInJtinLxJjZr6|4k2a3-S_25%mQ3`5_=m>yr=oRS=hbHXh)Qy^Nj zET6I?+uT{fr<8v+M3}DyFsEV$a!dRApj8&p_Z;7N|1oM(ilOf?hKoMkJ;Z6yYp3eA zw-^Hsp-Xz-)HcKav}ygL)~mw$OwBh2&<|e#v8q}bpwHVamgX`d?*(x#a=Fo7$@hlu z0bAvenGN^TO7E7utMoDkNg1ukoSuxbC|g16jq@#WJkd%Y4pALUV3$_&dno>f;XbIA ztVtfe4Z30`7R63$$X4@p z&{uC<1%_8ICH@Xs3Qyn5dpwcRe_>HUxfimbs&TwSU2X!MSb3~j*Xd-4(ss9JO1w~J zB32;JJ$zcInOV9z=Bw~7`$%^SE4`bGZZ8@e2_6N6{37l?_O}7j9~s7IX{|^&{Qy@N zaph~2=xv`ZHHbMI;ukDZx(jsI^nZe^EK8Bvsu>G*+d97p7K+i{$}o&jGo{*OI$-mv z^8`UE!oYXNZ_i3|(oiHf%yzbuC^=|^;ouJe6o^{G*7Yma4p7Hh3Zg2Di0P;p^kGo3hE# zFesq7#d`LrtuV${@*Jj-`v*#og=Gcy-%^!Xrf*U7^eaIRDmZUDT~l4S*l;0ujYDAB z{0>M*t#uxm#R0bUb@!6RYn18DmQ#+*DRwCcLkv%}5yPmbCFv#zl@!uor&(yK^W#*& z!6@Jtz@{}B+1j}#{_r=$r1h-FSuyLuwJ;z0QvtQ=^+GQHxiqQu>OBX<7kk^Jb@-KE zbLgCwO?0V_{8U0mq77-|b+ZHevgM`-vi&-4zMQaMIsTYBjD0*E{;t+PKd#=e%_2{2 z2`;ro7c_*osL7fJ?G`2YjBHdW_=YGb|M@%&MC4@>++zeSTf&S|s zmLH!Y$^OW@boj$EIb9#@@)&-1pjkrp6LN{-IbSXFenvLIZCkM>+l01h-=OqDHT@oo zHlw6JGe>DUV!Fofw};H&R+ILwQZ}!GjvjTgh`hJ2+vOLYpp;;rg`lJfx zA-tv04kRZHS7wIKeq`A5M)3p$>1EtVxYpD0&9^);u(RL2AX^W6JT&*x0qqZz=IIHQ zo$O~t;r@J2i1UkBxOzHo@B8A~8bd^Q>P`9WgK5Q;f@UF|%iW5O*x6F&Q#3WvwayOs z>5qcZV@?R20L{q?FN)_t_PdQrNqh=*c|VW*aWj0rl_MxypU%77Vg0_dRLmDNYAZEA zkc@~~sAcYDg0fH7YgR>>yX-8M930yc?KFlQ%E2bn;y)IGQ=W0Ig;0B>{9GCQ>TRpY zN!v5ooF89mCe(rG7`JZ61T#VDEZ>~-lN0kQ-4o?n(@{RP(X(N@gP(3{QT5=VmBGiC zvJ|6LMGpKp{!y?@X~DF4Vk)&CaS1$dTRyjaboV|HZpgQs^&wVvWN{0W3$=*NI z-7Sb-p-TEp?TbUKU^5l>eXk6sB+j3jaf)c{qNe1X9$iQ)-8}P}aX+v$61FZMX>g{f z;+w%{gF|gr1<;2LlGS1l13hZ!MSb6U9vjExFm73N1Z{(<@qp0V2#Rv49x{dWaURnW zm=&M&dvt-ZsA#La-l9vfFGyIt^Vl{RaOxLP=32OZ-fqtP(r{(T_1}E3V7g7-r^kiH zK)Ds^yj{MB&=`prh zt}6@MT1MUA2m8*|i4BE0l2PGHTU+Q?%}c{MrPN=C4VQb4{;R6{qI#jJjA#FgXl@^i zmM>u9VY_KTx(h_`BYcTCI^O+K7p-=PxeNCiCBm6>2Ek|QG>?=%LZ$3mWqK#1C4jO~ zk(6A!3~v4POM#2~zCg5JEMb-*`Qq1Wuv$^d$l_Pp@P%mU${bNk`4HlX3)y=@>!rGM zcO*EXp^;e6*f{CICer2oPH2ancC52a4l6-nqEkJ%<2taJHhmS0)1R!}&vy|TgxN1Q z{iN}C)L)YbPzoE$G{2i)@lJ0h~z!`x7QEc?#<*a`M3}%FU<$>6@TF5DMCGyzG9wiC4n9o?z5p8#nmMHVg82M_wkXN_~6heEE|f4r_R97n-LT(i+B zrv?j6rZE-Ks^cDL<{fJxonCVcgyfHCB3GmzW!LLaL_=kzuXR_s5h|vkmB#+Wxu!8; zf~E&SSxdNB(#@+433#`gsWP=c<+t>YoQVmPFV7=2?#Hz$@vxmgGcPnGub%i-Msyve;+k z-kUwWa-^I8_Fl+`5ji+lH7m-F`DHIR(Aigas<~~V*0qYuz5KB+3I4pW>04%mI9-Z$ z->LnNRlBd|&L$DBEa-h&uNN*i{xjv|=%H}N-%`o~#b-DlUqrpyl~Z$*B;j-%F7M-V zUxlu;cX9d_a=Nmt1F02B_CM3fli{583OMkq)7Uzqb>$TI>g7#hfja>%EI$P8WS{ZK@-WwKDt{WNWT@?96$%+e`Z zm!m;^lF5uxQ}IpxPdWCbZY}l+JNBfK8W;Tr=8LHExhyUXNJfJu8r=y)*IZn&wU#(FAqCkrn(#!PLSw2 zFx&+We$7}KKR-tNc~fwFzp}&eV!%N#1MQU+mLUk2|3{=@-m^Y8=`>c4&Rkkp-}jHvvU z7@#RVF(*`hsKcJzc)pBZOw0F{$YH$EX1F&@w=G(PW44nkmhwdV(UU(nFRvS(c?0)y zc1T#C+^=+=s?X_|`t7Xy@1&7=H}a?6-A1TX=p|eoGL*RJ980_Jf8JD_(B?EU*7zvZrVruu7%kTI3q0`4&=k=_aLI!wIB4-_RJRwKEnqX!7D~ftX~#Q@UOcVfGwDl7 zuAV1Gdp@#Y5(4opsiZg6LQk&fzqlS5WbqArk)8H65-joWVEZ?g`nT}~SfWiaK^T4@ zq1a<2N1d%Vl?9eNPSRS%A((~b!`5g4(i-aIw^Hqdq1zt?MFkwPLZv{^hby&Dnj>S@ znDrj zI(>5?Bjgq`aj{>OEX@LCZA0_?x0B|drw-Hpx!z0t%8GiBDDpbjR;XG5Q=)q^>8faZ zX?Eb3un(lu6igWv7oAAjZ}^$e@st3lh*v&WSFXUtQyEXZH=VP*627{`gUQsTUno95 zJjK7!UB53Br$Tt*`(>iZ`LTn>XI)|f?|~}pe?RQM-f5~mb&H#1z6chCty7PNDvG*k zAb(63HQudM;aMI0X6$`(JXP0EeCUn=q%5Af?~%GR0m$vM&cv(wNsa6qK%Bff;qJ6XU!QlXj;o^h6NhYR|Hv5rt~LKQ=w|Rx zQM|^}ud7nycM2hjqQhQW;5Zw4EB&1Ku)|E_yUCzk7qwjRo8#&nJe89UfqV|xCtkg+ z{xhf9RnH}bo2-9IG0f`bCjHu+GR$I@hzEhUg1j5+)W+_?1q}WK1)!!T)vT&9!K8N) z29P{G44Pk(zSL%eTX$LY7LhMbao%cUx(WNQgtPy_O~N{LKkHH~;VOQ!aCXH=wJ~?s z5RC0ZWYWf72)N}BDm-$d;w+&!4=q6$ni*Lo zg5Z0@ai=C%9kT?GEgqq-}0xm>u-I;=7ho^|cn3p+@r>i!=NBC`x^&PMPh@;nH4Q z@)cL`Os=kTQ2#m zkaj_qTphDN9jyPw4gY3D~;ateNGwLM0T^zT#z5P+^VlVB> zmd^d6Hiy0HzWzMVT2m30X^GA}JHq?t3s*s@U5DmR6)xLB&DE$}UMyB%F()E3BsaouYE*1HN; z?p|t10K5^rM9R z&?^WP#kuG9QEz&mz1lrVmCw999vEA!aTaw$KF7(Af0?O7Gk>}zj(hbDtkSM&U74kl zeT$dmE4t1c;VFgBE+-4mr^ozmPW}|ZGA;UD`JwtuNBvX^f6-?Nf`CR0rNlqkL>jp$#73vEb9PpP=V)ob}Kx7Yuiopx@{^~T1f z_{GdUw6Su3G{Ye~Ix1}K4xI9oV1-SeDz*bvMC%i9+5*1E)2H6tH}lE6-l#l1y1RJR zW`VdMG+TO0#oz&U)U@j1ld?;V$pXz5y(#83zGGO%o4Ifai((C_B@WkwSXVG|dEq+* zI>?Ih+XDaCnSLtkCyLMZS@BQED@S^+4eLJ-glXH3PKYW@OAHn6LwGZVZ4r$L?hjZY z@7x_nRs!R631bf~P)qD}&XV??U`5>j_c!tPJM=nem=IXFY{!wcyKKZlnm-yj8a>_H z1|85|y*EGJo6tX40Nbx_5ubD@l1o4cDGi%!zM*u(oQm$`olw}urDq}OZj;D36vJ%)TNdVspYY1=Yktt-V|74{cs893l z(yc8Lw68N?%R9ltt=%EnmBfcYoQNH^5*Gp3v=g|4v!QS8tlDnv2lMO)ZR=r>-$wU< zQasRxK)C-EQSf)WB(gbDv@NYs!Xw?ZY#!sjQXXi<1R5W=FBh+BrCy&^cLWKcL9Gtq zvX1kE41rXcN%Gr)lT6C) zOq&4u_~Pfi4*|-)(Jcw1ZMg$*S*B0`)Pq}_!!0iUGFQ+!m0Yh*Ofe#&z4o78Ps~5`1M?_8&n$D(8N)=3 z?>S$lRCR0H;{9yd&4!p_APDv^EdcG+a^c0WlXk=fdmL9KA3*_oEx`{P>3}WboM#Ts zkRg}qt2@<#BR0$jPmVaI=G9gR`VbWCOp*2U((+4XN(NE(?%K9vH^Fa;XRlBk8a-SB zsk&O!k>IaPF&K4wO&Qjz8vdzE>XHm2RGJyjqyBR|C*Ow(8^R~W6gE*`_>JB^yiUXgX)Wep)b}0Fk`joGskN?hE1*i88jdKt3bQn_oLN699Bja>_>y`8I>u4KPBqF^I7zZe{e-ZA}4 zBbT$*_VD29{KEO&P=UzG(jR4f%VMP?)%-E#%J)^rBAd52H3!!X}@ zR|NUI1A9?v9$t_Arx|WBbC3dMDG#J*Hi$;o$^og#47t9tMM?JDMQ)F-Tni0>$&v4? z(Ruzq^4<>J+7+q@#Ag~LkLPR;-4boIUMi`rkYxcHmnZ9|ZlLjkHOo4k^d9>R_%<*#gdVCsl=&2GHCAA7vRI z@Z(8?N0-}*{r!J;N#iZ(PA;P#ER{;~if8U3(lgTtd~$^-RN|F2d$$ql%g@Q+%V5My7R`a+7eLOM7&0tW(QU z_(!%dOQ?`GQI=(Y1-ZC3=N01T3frNP_BVG{t&~;tC7BuTNc~xoCD^IN8?@MMh%f~N zfX!UcDeFg#*}&3RGV4y~FY+9{H2zNBcdEhIi_PP)X!CGvq8xrJQR$GeVNq7EL)3jJ zkF&DtPL|%&3`&xtr!#jJED-yEw}$Z=FPN&I_KO*fDYoRV9v*8CfeAsaSDlcrgB-6l zeo7ZApvWAqLC*AaMF*KDdLBIVia&s7>dN~_reJv$);$PJms=JPxx*YT{j1Ts)TA(5?y|m_Yx}RcmePTd&N%DVLZ4J#yS>_9n)A&(A0SH0 z4@%~&r2(T}<*kO|uV%1zVjfZOy*;MBWmL&X0uI{RzuB_wE~TR;%`w^gGGF(w5?h%{ z|KK$|$r*RcgeuW&j68f!@wg2#&Yq)rStj{v2=0$>M3+M>16+6pZn~DYdVC?DqRA%w zn6l+dVp7HzEpzR)si0DeLW4ktV;~?2!($^BWbQuFk9!-0)(oEy?y8}vbwN34K?AQl zHZyvQ__@6l1l9tzGc{1pf^m%SdyUyt0IU*=v#G$ybIe(%cTnU4lo!G#nfMY|*aOIQ zj?_?#D>OjaA+pPGOnu9S`VUNBmmag{Uoe(spHgZC)6}4y~V6lsgx>=e&z)74GRuYh^`wOUw8^eRvtq=*-c< z{yWS4A0^|F}D2~I-%y|L*1{r?7F3fZHhrOT6^^q;zF8L*UXI@f({AT#AU@Zg-GcWi2&w2R^UB3s=nR_(@ zXjW7XZ2iixc)=5E0aD7E^C#VO>O+9RipguE7T-_ocAfW(KP==geZHS0t@q_B9=Tw@ z$xo7hKhn2OS-}jGb!?toDaq#pPv5%o%+jeJc zoh(XU#_?z|7%rL%t3LxITmzpitE<+5*VMnU`7M>HoR&p9(xUM^Fh%n&N$fg_4I08`uj@fVbjjo5e)5iw~DB z34q^zp>mU2#mwjSex`;T-}D#Xdw!N3b z$FWH0I63OyhwN|8dbCiPjWt~?(2;BSlsytSg$fGhhJ%WY=F#T6lO|{Kgco(_n+whc ztc$5@vj7cO3C}w6kVNI%k;yL^_R=9EyL5h1_I~}dHr3~suH&O*fIOx%1(yOWhc3$D zvN?jr2YA%)LVmbp$l;L&BsJ=M7PakeHdPrWEt-Aa5+>(paf(IDPGyrmR`z~tP{!8K zuh#!;v%??`Bma+hOCj3k+#Kod@>%6UK_WSFIUXa=YAK7!PFL6pA{9O4I^V7 zUA4&-xD&B$*iSykijo{jR?Oywi4j4?1o^|Rpw(Y7SH_dS2 zt$U_Wix6Mh3VBx}t{vQJRk35+t8czjS~^$96A-M`t}6(WYwt*X)x|RXd+Q;~{&R|5 zm2P^e=Y9@EOaD*rg=Mkt`-Ky&#k%rK)};-T>_H{J%Xvu(DtfFCf9#=<54MFBqE3_m z;hfwg?X}5Sxz^_#v{L=_Eg7hMwo79LWv8kC zFhQ32#F)VV+2qOkyDhA!1;;~(aHQQ8sMGj!=Bea|ji_aMg-|N8j6zA(o&E6sVsV?x zfYabCpK2n}tbt!B@~t#(jp-8(i-%Y6%$+Pc`u`~ml;{e;E~jQ$eyo_&Oi;o80Y+IH z$CZ!IXj>j0c3pHTt?s$=%{Esz(zWbvJ$n%M)8!MpmY3F&JxV!lqyY0c#qhrql(=Kg1595|PJw(}lEU{hJ_x;jpCqi% zNM=RdI46dW3v9i5WpV(Vu1g^9d94LA<85uRZY1wje8)s%|HTy$q0B4%VgQ$iOlV2# z8N2zY(V!^<$wsYX7|k@C#pd8yF&Cg$-NH1JV^>@Y^oP&JF`A2rhA93Y_Dy9Q%+Bl+ zmpFsMIO*DKJK+@nu03YS=DpXb%*7!n?=na>MVQK!5=_~4*dspvN5}pG)i?%4+8}gv z7-~jE5~1eV&rxfHRXkx6E|rhZ1y}W$`cn!;wOz~jW08Zz}%>HDFrAi z1gr@l9yLu*+-%w>oh2zCgJJ6vs=w6xK3&GkudzV;rSV+$CxfWX#8E1asi|{7x+DBN zm}L)Vz2jr4U@4=J8Z`CsIF8T#^^w6%)8r);@e_D$3i9Fd6sft397<&+_;(wNp1 z;7Yh=D-7lgip*^IY^^ovnw2x2%W*#GCW<|&)@KBX(LNl#YHC>`#s_DRry#kobsiEW zmj%jy8xa{Njb}nG96nR-8gYvUGzGyE+~=0PSTvDy_n$MzU4efxx6R1xAqnvvwMVaT6yhHg&H5M zwJ%CY@h`MV;NgEnn}XzrA|G~0(`t0Nl00I*7-|1V&kOCXn~XAw(f^Vof|l;7OW7C5 z%z2RNaRw|#Nr^y)lXQ;5e0i-)bR#KeNzrmJ-;yPZ*CwC`C8c27`9$l7!l&f$25Apf zoZp!v!g^_r5TA;Nri-7RJ^kP@LGI%|v8)vH&RhG;X#CAlwN`>IVs|i%nxlf-WmZ22 zh2NZ7Ma8i>^6az`M?{|zDO+fhasOP50zE4?QrHS^m`H*PUX^CqCqWy&7GK4e@c@^c zQ4#&PA6+Z=bNxa5q(#MHD$+E}KhhuGA1HyR53pdKcY&qxBU{YgRwN3{o<2M_b+zYG zJ32(R)_b@bh*R`|BHvqKG}9B`U(Q6^kmndHZ7IRJ0h2Ck8@wb%WrW5FMJZ1rKrt)wRpjps3CFU_*eEqMPP?3tUA8`)emKs|D-RDoxzk}xRD$reMwh8xrD?kiFgq5v#*yPw{0A$n;IjIGp^Ybsm;=Tn zg4`JV)kqpkt;Nr#!*|UIB6;Q-e6WZloB1nvjs3---JzjPovb}V)E`%}j>sVSzq8)| zy|7qUKL%{@XNuLbSs@s+ew`^MnD+tNB+uedkS6C;uxXB6xn~glw!-Y?(j4QyYz!bixD?S<6m)26Mz_@{r0rHPUrwInI%Ij8k16 zAjm|2IuOb{1I)#D5?O#QjX7NC=1e0(fLWu(cda6dD8ZhdRa`4z*IArnc-{F&p=M}I zP_`~Tm9(vv_j-UKIGIDxD{$9lB5R?c_c$yY)QOSu zfLtkep4oTJ!J+{?R^X-kZmalSemWaU{qAj_`#ngfE;tA6ZClPk$2Jr2X<;1NzC8>g zI=x@o`tM4SzkT}u8lkyIonW9;k`F5+g`;ybdRiFn5&t=_mia?1!mLIse-}K08oul@IHC?p#HkU2X6;V0w_LvuG;#hZV5V|@#&zZCarZ)^L1zCPttT`Sf-IXXfNWDb{Ja5pS#_GK?S>}{uot`LFo%R@iYFcqH&a%Wcf4`v$B%2y(GE-1<2onwiOkABHZQ;G9xFubaF7KPuRlfwoFjqByDMAb1TaYyztwjTP|1v~Vt_n~ zT7E7w-@O*Y1m;{u$Nx`zXC4mq{{H{6lvW`mMfO655?O{xglx%{WoEQWvTw~;rYRzl z1*_{PmveGS|G{^V**G z^M2g-D8D_ksqs2IM`eG47*@u4ZT7~H>}?y|S06S)n+z>kXEzo^Rvs_X_myrXK7Ys*JA6GgECzEvFgT%AYg#i0lwp%6;vOKXX9O&B%DM+2^YR zsL9-`mkzCJ*T>>sOv4-9vV)tR4jwL9NE?F>MbXw3a?b~hm}C7qq(!7DfjqSZKOY06sy6>^FO>7 zX{Va67^idkqJFsAqE1dwvvvW@6 zaV7MwNX)CXX1>m+Zn8P`dj9^KO8j@(?^fPCsO_DS?Sh*y)@{~Fu_*H8K@r*XAX{1C zwj-YR{2xRJj!J)G=?BWVKi1U~BdSwbRp-+f2hvV%xxCZ3D`XC#i1L1Zj(2bIsp8Uj z{{_(I{^>p`cOJJ%F3SQZv-+&Pem3+YWprq@vAf=*-moYPSIeSVq?YgpAx92bbar%P z+EB`WufWQ=0y`qOjN@3$La)E2ZTf#Hs=ebz4qed3oBtD;`Cp9V&tU}wO^gsb(2`B~H0XA-r15mYt+Y?J z1((I8NDlh4UCuY%iZ!5(-2*a{M)q{MYSUY;+CAiWUx%#Sh?{<&Qj6VM9E*MK(o-i< zr&F(G8h<5JE+wt?rF52fUA$$#eY!65<{zSK5BEpq0{bj1%|>#3^BJq#8NP3X#GSFh z(09w>TF@^2lP2~-iouwZ4_MJJOToVq%F5C*nbx+qzE;08TK_xekEfy4PIo7KxOqlq z06uF#6g{rKUUnlrje8)B2S|ElmYBY~xlqQxHe)@ZiTQ;oJEC$<$u_4K+s1j?Y-Y1k`#*3CH1 zBE+k;5YP7~1RJ;an58CZH4iW5y@}u35~V~aknaTeJpv!uv9#lCpYwg)zOM&pTZ}Uf z`H+^YX^SQ?p$7w}sz}=Bk|m}@qNRACB#@IZ9bLM}X|9p^0jThWq(w#KaJok;->&aN z%&Ln^CkZCHKd)9TZIGw<1h<(SZ*-+wW^he&kQQ9pseAM-g_~0ta@K66&l%y5r{U{s zI{4ocUPw9Qu5%G%)}_#o9hrLBZlKy)Eio3|cdWi_Uv9CwECQXZB+)%1*wTUw`dTYf zj&zSS=-WeeUy(ora1bco zWLq^Qvj1^`9w&OX*CSNs71YWJoP~K#<+2hx+jiB)?8@p(o}Kv&dR23g-h?^Ct-UpqZM`Cm zNM3%siFJMM>fCSihA*-g{!24@Mk%6uN}H=Cy!sN@JH&5k#UxJjGSiBg(BR;x`GSB^ zlq!WDVYjo9wb!dpzWIIE*r|uQ>ElLDV-X7a9eN=e-@f$XI+V)+@QY5+7pk3A(OX2} zblI`xk#<{K#`#YnSk#LeB?{#lyuYWI3eu^n2#C}#AX)CiEvYLE-wQ^7O zYUTA+uSLedycCF{>6u7 zO>hm7>$DHo=Q;JPXw)qU380>bHlNMR+|j@rp|L&cLV`pUKTp(E^&iVp zt_|Kpm0oFC{D*ppDBDptp~T0--15MZ|M%Oa5$XS|()i1%no5Ns4Q|p*o?dZL(mxld z7Kl)w#Hc>a#d}gpgbPP9DOEya4PhyW_uxP zloG>`CmwxZ^tG3j)F+d7w2RcdY*alSIgr6GfPRnEH}>VFeu|r4ZDV7kejL+yzENyG zyCGHf7$TNtolBR8nm(l)4=JQo7a000Q>BMhPWGHNZfr4GxMlKqtKyVfVlc+i2_g2- z_m#I=tIz#sS!xkP7cO_WI_WUPKl4&$%BR(?ZTb~9IR2S>eDq~v`xpD;Gg?jaeOsTP z+~;Q_0RO+Nw1i~xxCzsU%e_C*_?p{@8;Q**#!HIfRx!CF^daZMl0=g}nZS`CcP#Cu zsc?J__mX19JHe&CJ9Ha0#L!LEB25ThEcwi*Ib{Rl09x!O4JP0(cx*jDU$L~q#@C&)byF|BZ(;_1WGUwsI{)UPh@aqg%WTj}+Ozho zV{=78;pEZG4k&-M=(vrJo7q<#s`qBOdTvTXRO5VH%2Zpqo=QFX8q9yeif!D(LxO{S zauj3u<8AfLRjd1KHEOsyP}vrPp02dV2dej4%g!lNhb-Aiwu0!=OM?R=;k&RhNZfiM z)gtxe6Xm^b&pNE$4isI=-<`;+pwx68AmU4u-q%kbsF~=T)O=S`S2Piu$2DU>f#m>c zS=JK#f#m?%5f7otZk4CLA2}=O`>Fcb?^_EG-&UjyjW#fO@46M&J7-q(Ue=<;@1sDK zoI<>5q4|l2XCb#Ww`{wXREILAB%VG`SUo%s`OpEs-Eg~)Q5Cp}b#;sWmd>L9(z8fes`Ek7Z8JmwZWfW5cMZd*sj-GkcxJ@$7kf_X<745Qha9-ZE1vQfusWpGumb#C(c>qq?gqO_ej zQHgV)aL^pb#cvUI`>^c{(ML<#YF)i?f8_5JNz5a?vQxf<`RDTyrIAT0 z>j7yjDe*%)j;(TdXVm?g=2Syjw~hX&V|YH|-j+}-LF>krbxF*F!-mW&#oY}pDLGlE z!bgg+F#q1;y)Lu2EuJ?>U`5QvmTm^uzMML@`K)EH0i>=v!8@i?Ds&HZJ<|F|@x3NB zn|hJG7UhUCIuM8u8D5`eQQLm(Swo?TpKM>ku!Jb_y7S!7-O<1WAxTDP8~x~~K2i)? zecuiylzBs&Bik|asdw%U;e)L|9I}mZ9kS;Kfz8?-4D&M~Y7}Y5EOWiu>SAq*(qfuI z@0q?;BD3PKe^54mVle)**rR5FHBABS8i`#w#Hs?fmHy^`=!$WCS(o?Y0Q(~VVkKLKA8B@M~!HILaLy{TE zaWm51+I@h!MS-MuS%dnmGG7JVVYRm+6%(%4es*~Em-N(KZH@@8wQ(jWobG7$dhh;?EyOGqFqhx$NG(SBNRi4zpUGdMT`GK8S(%*H znk6RQ;~tmSt@(w}(KsKe2Tz)qpRWN0n2`3U$LMF@np<_)&Cf%Gcr+hI&2y_;R^t`v z)Kmk5aGA1Q>49_z3IouK6F(*|c8{PqgEdvlzKGMyVW8Ul{YbcY=~!;n_P$;uO=HCa zK)M%hv3L`0yhEvV082}&l*AMs?!kz|Gkd0H8k;9)dEgz1?>q9I5jX`pn6PCz=?>QQ ziH9!@FK-ixMSC716m9I=N8LTy(;)YX0GAx9r*c9&@`?h_M#jk|H}2)f8eKc`kK(q! zG5fzS-bCG@H`k%8TxHyI(uK zE4l8Qn)n2AWg@=m8{d522C~ydvT1J8CUJi+61P0KjdA(_T#`ZO;@zCa<9gFIsK5gI zV!HA^{hZXr;(VCyrckKu1xgykHriRMKdRTg zj%1mWF;$*;iLE?nD9JFsFr>=Yw~tuKuaMvTgU%1&sGeyGeh*_dXTxgl8pWGL2V?PoQ@6sI&F+OY4vf} z7aQ5vHfNhBXf`4mQfo~C+w#EFJcqS;kOMGw?E0TRa?xgiPb_m@Q!r9aKPxjtwJAyv zy|X&iA?^fTa!fuuLX3gi-yN>pph{_ss+x5NsDVtAPpn0VgBcp(4XSe5e9|U5?Y|E1 zGVBms8kpE~np7D~>ZKW7QfyU{y*u>Ey1&Z^VOyEJ`#_(1^N0@sY<-k!K?&1gya?kc zz71?$*!2pZ*NQU~b_o1%tG-paRO60XSzgE=nH!^z%yoGwL7yC|S`JtBHq$|7N4d#N zLd!P>izM46(~7cF@86s*6Pg8S@w-d|-gDD)SumH5;3d@JfV`fu`w9ny&P}adEYF)O z!vOT+jp&L$0{GW-vHsD%?XeP5hXy^S0s{kxk<|-#M@~`ZKzFC6R$h9$&4wUh0xbJ7 zCEbNp*#sg6aRj}t5z^E+Z=Yi!+_UFI+NeCopIm!xNQM4>6Y|x}kqzvkT1Ry-QH5Bg zBcO0LjVXaq2OOZLtv4OM@WG#ow0 zWNMH!gpW%f`ee9)(o%>-*v$mb*z{!t?9EuKv$2g~)BPGKmxK4cG{Lgt7tr7KV%tA6 zb^p5V_4<=jO#Boq(tgu%g%oLsaY-Sd6bD@$4Jr0=%O5RHT% zrMbu5OAw!rb4z7kY>z6|_U3eYlBN{Zcawa0SDh)wZ$z~T51u8~dlDo+7GW|$0@TQFb2 zz}EJ5fj7oYSW`<}aP~Y=HxhnG=jkPODKEXwW^2)wjVMKVxN7$%Cx#GP9f6pYT_W5`n-HlU-Y-BM{q0TyTVjNwk>_`$FlFZmNs z9>Nn8wwy(%l-?i5HQ1C#Z4*R`qSCLHU2?|=7ua&Dj8`kz zZQSF90XW%E^5yfDVr^Uj46vJ(?9$67QpC> zpIG9c+-NP|ZlO~`{}hz}x}H+w9{?p=?UU`S=apN3TW*ala@0=jwLZ^2$GzeN&vX`B zwHIgf#Knen-Gu2@(zl8OtO0|Zvc6qbT9ljy5QnA`bAb!xbMucJaxfnS(9sp;ht%a4 z8w0zy8WmjN=&2#_I_VNaPn{P|5+MhmP$qGgofRK^R-?bDHhH;5BkDcH6!dfYmQMi_ zW@c_8tEt0^59$leZb%`l7a1+sBY9e5e@EyMjjEI_yT>^30w9T3s}_h`slM#^QDA8# zyzb76u(0Xl6JgV&fMaNsL;SF7OjMZ6(kQ9W)~<+pUZ(MWdu3pLEO&ZB8))q>Z$AdZ z01dDX`vZ?`4L=bSE|f3zs#7XA^TA`DYJ@16`i-|ZlaE4N?n??TIcZ}xS`I9_tsp?D z^1HeCkL(l%0fyWzfCfK}KGZm~UWt<0Q7Fl1xeKoyeRY_V1Tc!=!6xRtTP{XV@6y{v z`%XsTGKn~kM8{4ScMip=XfebP^IV8+8r0-ERxzRVcz?^OBpmm{4!Q+6?c{051L>6wt?N( zXxZKlovgX@YW}e@#arXNhXYnaju{7&MekPz(S+eC)~*vjl^L)Oz9N$#Dpa#b ziCP@+h`tnj`7Nv;bkMpK4_p1RpE}Y#Du~rM8(y*csk^YxJSS1JVgvi}0mCXI@4S*t zA&XT9B*~}B3+<5p{!U3mwKE>o`Hk&0C!mXL+t_R(EhaJ;OwX=v1>OG|Kh*H@Vf`!ph@0i;` zaYynt2JDT=_j$y>Yd-$EIP{H(@>at}>qk5iAe2G;6{twu%E@h50!*^`k{H8lJ>=la zsM3WNx17~e7b$)cjFV=U#Tfae^ao#R;E|K?9OEq@k?X<=7aYihuhwNZrK`xM137j# z6Yx-0*E?+##&+DfJ<)m%mVU4W;8iIfW@X~4T^6G}B6TbL?S?>62cSbgD-mGA8&XEW zU>DO`F~$#|loj4gv#&B$sl`V$I?)i*Ob*2W9`QhS>!R`?4UWZSUoRn?cy@bxUAa8c z>Xgf0`VujEbt7A3Gv5zGW&!|$TCJS+@s_?;Dv~h{_bTQWpqsp1qslm@maiM`=gLc*ajVo88-Ahq<+a*O8FM#gl2j^tupE`t@mH`|rQ_D|` zfsTD`ie)~eBAcvoD1aW&#ErTVj9&{7exGp6$(F6j<;$XPsC57*wxP*SR_dC~+K2gh zr}GX{IqF_H8D!&ZJ&G}8Q%K6~V}aFxWH(c;Fw329e#g5w|Hod6lIf*!@m@1*^a(X8 zSzsCe9{9NW`*`nKg6uW3=xPY}Nm>mxS)p3pjOIU}TT8i6iUBlIykgbFj85XfY z$^b+HI4ibG5H#%|_58sY4Jw?TU7cG1TRkDdh`ZDZd*~&ytbx^#_5gGvvrdX(D7=sb zC<9Lu!JjCSH?Y^R)ki6?{z>bc#N|>aqeQlKUByS&bQr|>HeKu9!BT#KrLMPXd4L$H z=q~sY84UVp&H=>4Y_FC2R{ZB6!_>N|ZP*nfiyTQMf=qSl!8SJ!(0}^;9M`5+*EC0k zv$S7?kud|%;#k}(h11(zpwf4adbeK8+J%=Ql9b1`W4V-Ea&zu2gB2nICuvQ>_nh+}ZJIU$qD7 z#nY{&Z+U_Q_ovz38GJNkq;9mGJT-L&oF$D!a=3y}2){vL9xO~}3E2d6*WPBcGeumt zq>0=;RELx-uwSjrfSX~4VoSnJKMJCQKI>aEdAot`5+x z=oTNZz@pI&v?5)V39VSHkw35E9;|eqb?`mX;Cv|Bud3{kWY@kKaGZe0l{XOo<8E?L zXqeao4L&!#iLO(dvHiVq&{wMB$u}US&qo{UcZU&l3|+By=wr>=qyDSe8`+b0VVK|* zcjc;)+94tKaQTxyjBAd+v>=)`2GcQPoR!P%{euLrhv8xAnAECM5{yNUV>zsv5!5Dr z&FM8i8&ys$_KnU|@(rIMlJ_=-sC;dcD0{3q&O;dyiL_NVuWKI%C6o>3%mY&UoVw2I zAxezw1eO9vLII!t;fc^tkXkUYuS)1lO1>Wo*UMR9%cp)4ZSkwV2SXCci_xH`yGEl| zYwV|H8u`Io#9i+3`T-D`)H4B&`3^Oe!7UW5s)gCIfVNBvg$w|Aj;X$uJfkbH}YSwB19H5ww(JKL*!aY>q41*x-$`k6O zrx(|v=kZL`qDiRUUgvB|7c03T&R>++noCpeBK%**FP0G(e znpoC3=?Hq(N9ReNA_7zacF4<3md*LxOBmD-ENKAY-8ECmW_4+SmivC$j?u?xE~YPK z@RaB##L*Hv9{0%3j;6e>(sy&9l(@0PwYw;B*$4-97(BJY85x7ba_{yo3b9`urZ8KT zsGlcSd~^{z7E-iYrS5X7GyBz(=W+Y&agjFCc1l#Fw88dg6=OSPPl{{cmG*ZgdwmP+ zGDbkS_drpyo*niTAzC5z}etrVLrUR+4qsRZ@(hzR_K zkg>Ece*L70E>f1nlwm4L`TZN-g`YbhFhc-;0F10-#Qj4l)kV}XOOGTP0|HCXZpi*z z;=;X89;vYOy-o8$@geG!bzy+O8ArD;A;(u7O%o00MCjgME!(CCDc4Lpj;cOuGER+e z@joKx`_ZAX##NLN3RYHX`EkHUI4%^Gb_gTgnWd_rC<3+3VCT^m4LB_y4Ou)jF{HYm z)5>aQ-G`{84t)z!Z)NPamh0P?U@ibUZCA!RPmUdW!Uy-yfS7m`_LhELTeO7_EhHVZ zJIF7h9Mrrm(uG2BjfSbGzeFFr3X^@i=Z9{cC$xy#V-=EuVY*y9QTQ?2O8mug z_8KxfDaz^29p?iza_v>?LQg*c61aW>ujbS+)-hropk%h?uIox)JE=y)lwh?fETtE{ z$!~*qIilo4+HtP}ZQFNtIacMbG#=!qiiLh=f>2uEirz49v*Fvr-Fyx4ixi{7ecwa}zgY zU7h0kOCAhg4o@)ZxqQJ4tJz zX!%U&x1!WMV*WTOEQwC6);+yl=iFu40>j4FRu;H(spfr@#TW<%ZO$*HTQCaVnIthA z3WX@6{0fcmcp$iW@_(KZ=cqt&lNHa2zWY0D_^*pz%XI|Mz`TPT<-DCyl;9?U^X{1` z7Kue*Cn4$Flt$n4Xf9*bh&a`ek+<|&d~)Op=;=hrTVq`nQ0bQ+1)?*pKclmqzv3aj z;Degx+T~fW9j{6VEDb-Ghv~*ijQYr@PF~ZiaP*IW>q(c~VJ21tdd6fNco^VSy}AuW zUZflPEJX1GeR2)9o=<}B2Q44CO>b=DK>9GZ=lruwJsfTlzKwBg!$YgNZE|jg4b|w9 za8||0d#l+%^Q65UuKToF9iMSOv9*LkS4-B(ESlkxJ&E5~bHmpPdZ5xs;QDvnUjAu6 ze!Ir{Nip1Wxh{E$Fkpf>#Ad>2m`$^RP6)2k1!i%&X8%Pgcw@-Wx}@=ANu#H)^sO27 z;9MY2mUvBs^>ooQG33(wURuB814*u{w@*SyjGf2QPF+2=eI!t|iC+y5ReRZ>(9Ut& z7~J}F6Tj}Es+-;8QPV=hpYGU2X|xcc7>Ck9s66dy^DA*Pr+6pyUN&fe-qaQH=uFgzw>yMAp;U%<*6xba`bh zzozDqpK9UF_DOzsazuu`YMY&ZGykn&x8qCW@v#*d^oKhpQHBFM`Q2@cf<;l}gUMbp z;LC~G_!A@eDpt-6oh39Ul6|z1QX$@5axGE><7Bdca9 zlP>sY$9nAgvVb4YB#?_KEv|MkBbd*zk6k|%moPCeihBL5cFMacUhxlT;cTD!qUCh` zvS7fd5}7!x$Cu?LM=|-fgJk|prf5=;hX~0Q!q0O{Qt#MPWRv&ZorsxztrWq@tx!W_Fs`?K(-X)4`zNC=*1tn#0IaRdR~Bva=ju6?x@*r*wdw07d0t$z^GS}%N1yXS z+JT0m_AAdc3>OMSzJ4wVzgRM;$vJv9kod6Y0 zoCWdWD03cB`11)L+RkBIA?s0xCGf)ETtD>r7jBQM&@y>|2pNzvmlxijbk>sd2#-Gq z^JK4CBUo=~dc={=^{aZBbI1Ye$W|c7Ate7A9>ZT59%Mwg@IY79{Te$HROc0{CZYQ2 zSbT2cE@7hbpGUmss|vlJKXO1g0QIBgYnf?#@N};oFG)rsX)b?x{Xh4s-@Msjx)ddX z6J8fT)8|s_(WgSb(WLih(67u<{RNpCP$_i^+12IY7oJf)aOPrK_lP&oRa&3&r(=A- zckx%I6>>9MO`l0JR6e7ri}N)6W-!Hz@x4}ObRp0ac%Gc>Xm!99B+ScoSVJUq7p!YZ9KxF+aauoe@7LjZcsw7^$8(P_+E|K<$%%nL zAn|i&PhSFoMDQRGA6--!c*nE1+5`Bp>DJjRcR(QN{p)}E&RyEK00Qj>ojYyr5dLO% z1cq1layuBwH8^$2!Ats}?;fZimD?Y*m8o92;v%ErO0JrmoP2HM zWNW@D?IIOAs+&Egov-x002`jdGy#^>$E)p&c78s}%9$0=$0 z4^_{4SIy<+v}y+Wv6BtGmaG(q5#(|0+5Xz1o&d{7y+nowaqwBmO|OnO>WXn)Hb{dJ zUv`EPqP!yD^AWj8?3SzPqJOS2=AAe`N%6{dl#@P!8&w42{C;ddFxISsmFA-#k%z`K zES9PB(|2uSKH;Wip|9-9A9I#M2^7VG{#Xk+M2sl_lMusu?K2}{;nZfPrPlG8rJm}b z@bF|1JSAfMIl6OoU>D-hjlohdpTT?QzfZNv>nW`%F-s~O6V1%A2tSr^W|x6QYyb|i z_i9G-#lV-fZ$jbJQjQFpHT;p5Ij1tR9hH;_QoO9bVX&Wm3X^G9(@!1bGoYP`X)vIA zg50)m*mq_+6J%VNlu=w_^y6($r3ksby&b!se;jep?XJm>iN(g)&6xq3)w))(aMv9H zYBUe|;D(`MH`G3FKiCejSNc!X!Xj*sY8&IT;n1o9jd#K7^h` zi$;!naX63Xq)*&oS^4m02dlk)X)|kxL28Ml=hmrfMuX2!Sylu)8-&&qWaO2HZU+;y z;a;2;e`X>UYa$DcQ6kkBBXg{8i!t@N`h*Nz1=ND~<>PR*r&nkc<@6-mqj|Qzc%cTm^yS@!4zRz+OiMsj>1WXo6RfXE&D>=vwhX}s4Ef#aSNoKl zldWy+eT2?dA+|0OwNnsId#E2Uy=06&q^o9?2E9uCd{<&qNQCCh_^2GIfOi70thp*< zLuQF=lDyLB%Q@nm_qPo}5&VBb%T-@~I=oJzShTpiPlDZcI{c-u)&1PD{_(rx?=|mL zM?K33l3Y8Y=q6_n694J&NUCUjyQ-IpLdjr~d0}Esar22?apXABAJen;BU(}YAY=Y+ z4HKN72vfap8d<)mu=k}^tJQ--?WwkEsR`U!LF9dgO6fqk>Mv2-nZ(cGS5b$CpMnr+ z+HT9wK9{LnNPVzs0q_28o_H+;UXssjEoc1Ru{q@}G?iJdF<(OQYK};FU+?d-IfE3W zxASVqKNAxqOD^R~rTBb0ddRFZE(^TJ%Rx0Cqn_TTKML8|cv{F;5W9+B{Q1rP=_&Ne za8%U|t7~ot7>DPOqV?VHeV(nXRlYDVekhueP*|dpCRZhDNxSAuw}^_Shphz}=rRig z=>5pTwyAT1WXjG_4h^fmE6FEK)}z!WMoGQoOk%ZX(@6|ih^(FH=s)q?8vD$!5RsF9 zf^)^8nXn1(!Skq-4nIh2ea2yy{=6rpb9``@ z=l;l4f^Velys}fhNsB|I1O?fC)NQ1_wzg)k%4=tbn@w7R6AjEXUp%}&u@!Hd>AFk5B~3k&tfE@(p`g17&o1#Uq>H|!rPOKX z!B`Cir#qeENCHD?uW)A{WJ;!`K4tf#^CdL4)CotrcKZC1BZM|2sf62N1`1(TghAIL zAH2eZK9RU8BGQ(Bk2>w%*NfFvMzxycN$CDQ#WBF0${%16Yf+T@uXhPl*lc94@76I& zJ)C^0XP(^`^A71-nI3m`sFi;F{y@pABVu9+C(#AptP<-Uex{BS1rdJvK zbkU35s@(-&6?NdZx^%un)dpz2vvyAn4~4{E)D(ZXS5QvxK}HIifht0MLr~|MIVu5p*VXi&P(t@kM1y<6+DdII0 ztZ5|UG^N4M{;o-wY+BWkoK;#PLo{md$G= z4|GWs>N5S=xz}qyXTb!EXyc_qlZ4%REs!U%vdY0kat&Rfh1H)UidUgqGj;iw3MbC1 zmG)4UJcvWC@?f^1e@vEe-+k2;5#y+z&Di30m>-Ll{rYsg(r?+x+U1-BX2>Z0V)e}% z*yN_RS<)}Gs)J1vro8HPocO2s{6b)r+0w20_I#10Vjq3lk*zwvhHzdBJ4cGt&1?y1 zB^@|hi!U-cPhiQf_qdM*!TCvE6yCje+UTmbRZJH0753bPqJo|>Bda1grzT-?S#LE20a8q77e4YXBK5zd+l&lbV*Zee!62 zw=G5G?Ue~W7WP&WVwu-Xi6qDe+8gr~A(E0OlkmPvFOpch5v)pvwN0BW%P$a@+;B8k*+EXp;k1D%rrSJ?8 zyxCyfGC^5(I$`2)Vez_;EMs-W@e_L<9p$Hgiz^(QrrclN2-q={E%+Y1cIHu|#-T#W zNJ9Qvz|m&t^C8P8Uwn-pxbUm&;%V99Sn{`j4$narrz z0y=0*vD4i@SoYStw4>MnPv{VEJ6RfvZq8*8?Pm6e!t5UWVN;)Y$pqxf!9cZGyiVhHuEFBTJf+esZ2*c*6*fE*2)Uf8?a@I^ydq9%ghe~ z^nz-urmV_)#~QP0sZTw;?VmeK8CVc=_gr-NYI(?{6i9}}Ty{=(*!I1@t?t)osEvhb z%$BXp-D8Wifng4gVTNUSJ0NcXjThYU`A186abb-N)>B zCjIqY^J@<8&`39F`j^F)=rd$#s^4~l$)ey48yHc~ED4&}^U3d$y;G~fG9~yWIH9#^ zS6Ao#4v5`l8Pc(qFU3OSRhom8YwJov{Di2rr6elult@x>I0AfDk)+l*y4JLL6?WQ6 z96}$h?3@`D=q{_o?-uPe{c5*-%gptlosbezSEI?jXf;kLe%?HImg3izf0a7!-Z%Mw z*w*qavEsG?QC(g-QtDS7w@gYXcQH)ze4e^^(_X7~o>60cZEt}!A<(9Y9qldEDuqyH z%kA-ekm3WqWw%Y|P)M7thr6D|od+$e6P->^rw~ddsyZE}H@59@euLKU=nt(7w(?Sx z;LqiAuheIDX?777ag?2XP=Q%}WlHEor)2+f>oxxhvatmxE%4yUv9(UyOof1gEE`MW z=D6bl)Tbq6q$tY6F_z2f2KO=o1n5z>1|-ZUA|JsYImKgi-BTcIyx#zFy%))S5p zTw3I=g%`-9NSL7yL0?GB);pEuetR?Ljhq!o!qq+@8?@iM3;ikg%*qO+pWuy@W5D3BGy0AUR-vQA z>&wzpbpA|X`v}XGo*R&d;`oF_Q|0Z)M^bD35G3W$uQld9BVqd+anT^70^N6@q+1_- zT`VM2xAb)8%%epw0uEg?2;iK8t(uYgdBsvf13Kk{@l=c?h@32OK^K1O@L=@hDYoYB(#2;!DeZCZJ+;GP@27+bH8FBgA1ScArPo$_YkY^jDd_3> z`9ke18@BS0k4j7EKm%Q1>VwD^U59y$t?FPA`-#1wW&~OkhOAmVPwI95%(Tgs}9A9AyyLG z*JC8y_cv)=cH6s<08(rGC_0es{89CjSGXiU6CIR9smP&T*>&kmjJ$_7e~ak0gHl_k znESa3=5RO3+?ErGcNFc%Y}<9EIJ-D<6|$!je!(w(HBuuu(!<_h$W$NElo1z^9VlB9 zg}zt+Xo63_H{pf)E4KZ@{Edtt(35l!d}-``^^t!BzyaGBjZayzAO!Xwd2scoAaE^l zRi6Lo0xbai^D&MHuKbG{AkcrLWl+qw9jMLkz4m;9{KXa^kgV={B)k@Iaf)vtXcSAjrRm;NKefg{OPc#l^>Bq`jvqvhE~4W4;)9Z z&oWYw1F3Lso?!(px`{jcmN(;Fy}--bwZe@X4bx#~KnxxFx$~Kpk?a}{reo`HM~7yn z+K7!^0tb9|bhzcIJZx?k$Mi3W(sL@t#5J5Np==cTO=rocjutyqevnJQF{#H`OQH=x zxWDJy=AxFLij?1%Fag&-U3pW;WDX&5H@rNbaaUSxeh%8;;4Gx3C_ZlY-z$natoIIt zozeZ?T6KW+HttmgX7~dx6kor13@uhuF?k1F!DHFh^lio=Zdg{cvuGv!bZ^Ac8})&F zL)*>xgJ=Jpzr9t==n*prZ!GVEYS{!e6uul4be;*N8V_zRWSb^Ei=y)CZ=Dn6ee?@0 z?(e_`nLG<%eShaq<>@2gBwHIs{(W17(pwG+#3`L&GV+tCH6jSkt!uIDELf*Ru^A6S zHF!}1<8^g2$jXXNcTEKU*^iYrUnFNCY7tSgr5FgDZ%7aszjX99Ih!`UT(Xfcr7_q5@QcslMMnq36d;)Pk2~761*k+gTT3>I! znYjV4L7@CJ5H|XeJb1+C>)Ha(-aR(ch89i2^}EYOS_;7&a%BQkRGWJ`*)#GC2`QB2 z1$2UD;Y}G-QrzDU5DuSyhc4eNqd_11kZBlZVhCfnV$7Xb9;z)eJwdK~tPg#MlqX4? zB#6<6>%NTUpjWui9<+@of`JRd^#kv$u;EO|LCsV0%m(&K$Ri3@b2? zaW@4p?0y^@3X2mPtBD@_9uI=kkeQ`XoKsZ8+VN%WDp(K5nD^S^Emn-)_2^eU!ZvEw zMgA}Sr}ed^wc5%eCfYH^Mr2CeENS9IeRwXxM#1#;50&*xF%&Q2k$Azk3P7Carmi2k ztED&-r&X3EFMM+Fw;|QUsn(Xso4&4jm0A^Kk}}6y`Z%l1rFy942PIWcR-i?M$@d4H zqNlo58Ftmn$7+RHt2ENz=!1U%%8WP~#K-DtYSmEcOf0D@kJx91*)`7PwE2uF*H=YO zFPAgIs7B@G+4a}uR=n-rW;Jjh+rCj1$In!~7#Kv?0wwxm-TM&P<@ z$pj4hohdT&+SwyT9c-u}Fy>6OXo^S7sHOMT@hmTyRS za%a9tVUj5QapHI?d-7DO6`!epOf(Z`ch}vr7&gjN#KG9^_8-#5b~n6d_)pH@IB)Z$Fhu0$Vc-V%xrV%EhWmvkuD85tMfq*` z#hB<{chP5}6SrdS9Ui9X1rGQ7b&`1-r0oIY6yH?v=tQY3G7liN=5JyVE!s*s$l9Gl z2|hnlQ&Z(}nhdY>FDsNn#6!iT9ywH4gyykW7-!q@KF++zcf?}3TMFhWiams@2`QW7 zWl)8QhPaSGi;J9oOB;tkQc(09K0d~XN7FImwIB>a7O6E+fs;R}5ctGhup|m;7}jun z#~GP3!H}IBKu}A2v6`{P^C`rN0tO z7_VwgJmNW|N4;ED68pn25~+LkLHZDK_{v~@uozQx8@p9`?#p6w-jHF$xP#^mD*`l$ zJ5U?!5=#Cx*3E1_F&%`UkbgxR-6oTp3AXJzV_llsj!QGq$`FI#V2H%L%dC(gR(q8^ zl>o1#EFOe;lm%X{_Z!NR@;EvwGc>xe&=#yTWDMAGh@qDal9GuDfv`w3pGNOjom{zm zZ?~edkDr`=D+wU3P|VT(B(6y_*Mg{rH315kdAE$!=*M%-i&8=!+F9KmUzyfj!7U#% zCSkAF14zchfv##EIAVFFmN}L0%VUKJ(9=%Bn;poQA(z^xrzF(9kX+Jc8Hl{p0#tzR z&Ev4JRYfCS+GeHz$4X_IuW_Hif+7RHAX-#Fc!A`_egW*Fy+!iij}5dWF{gFb^Zy5{y8CA8eSI?;`6Ke`Vpt)I|CoZB(r-z2K{FH zyrOzeh~X`bn_2q#kBEyzC&wfrd$Y)9h|Jg>Ie+4Ey0KbxG&Ms4$PpcNol=3ME>n|+ zacEg|BtQ($3?5S(MG}*)NZ@M%haG1IAF<5`ut#pro~3SDM4j(7&Y6~*G1UPtNB?!` z9mx-EB#&5VNkg7k-rAXR!zb3SvLH+21;+iUG(slj1}A=k?qyfzcA)UrQg0F92(3*Z zN@tE~`)Ncz=4s4v*1^K-G``kqbwkYv9uPvE>JM=1zQ0RGi#%+~k>)$cO=~|nP>!mF ze0)PiicVw*lGn-AX12$;R*PLY`(CT!|`?ewxRkI< zFvT5R>xk=P;}#tlc?dS$f!d|SnMmoWq_U=0rA&EWXfhh_ZW0$3Q~dHQFDmuUSOW z<^jjKxMU8Ct{slL>Y_6Om#d!#XkI$4d-{{8X6yb!nH=RMdvEk-s6!5^X*wHZj98-> z3}KNHX)O)j%6qlim@q+(B>!X)x96fh!w~X%SI-=JDAA)pO+06vTHJO7q4DZPNCX==~BhO%XU?QSoIbP>vGK%QETaSqIwnzEJ-`l4B~6Rp)~ts)bh*Ku6)$ zf~gg9C=xnBNk<)Sq=iS!mCnH~0QP>{dB91^;gt|ZavS>E+2m*?#|I;lKV@R4$}}Z& z9l)na0`zjsTRD!E+#)`RGzXId`01D9>N;ixf=BZB9J@}* zE{0i`HWjLN`v4vW*W7_RE^(&@zUUtcV3p`|zg%}dQeGK)aef{L=}=l*ZW5@GG+)j^ zI4EQR#BdHAK#i(u(SMO)eJITT`LE{38_w{BU}K5y2_!5!M8O8==R^RRpe`^s2Fz! zOp|9=NZymR_bPzCgOWqH+$F_GeIA(fYOKF6Lw`Q%0wLZ8;u2CE99_uLf}g$i za>hnRh$QJ?iTo-4BB3|4kCjFYOGYog{usuBHFO+0R0%A9S~xI;B*bWy0M zqPx#Yki59yIP5o(C4KB|F5wKTu7sTRQUYHzzeu5?YMHVd0rQjUONi!2pBY=pEX>Wo z!T7$WM4zS}AEZcCg}_O4;o-MEsLFH8e#rontdmh49^~FTqf)Hka4i#}hCQcNI9gQb z-D#T)@nX%D0-Us3k!oX+&Mp~ftFs)+a(~ngZR4mB&>C)Xo+ zKAZ5C5rDn^DKF#1@DpLDS-*Nwjt9*!EUb*gv@ZP7g^ft`Ud$p!=I-eG66NpBsr>=f zJ0RKU_y&h0%}$l@g!sy6`AVe_IRmiJaFdPZ=svR~P-s8*u$Y2KtoZ%>pEiVVi3>+n zV`i$$8%H}Rb+H_AOOq94hcH4=P=~Fvm@=pr@woZiy_R?26$4oW3ASU-ewRyA|j%h znm}g;t-*PmMXy56{cW+QiLEYYsH~`FY&R@%R@`ZjsUKbLh?h41 zRa|mciL(uf#HeDNNxG^F(+``P1Ur1lNQQk+;Y|MCw8ym968qWesudoEzYKalqFe2& zrotP>>r{tr1nBwZ!3LfzlnvtF%Ob&o18?TksLUd=gT=;1_l{?UL8iP5^P9|u+$jWT7Z`rz| zMJM0r@kI`%g6}T8_v<(gq(?KHs$h{yA{ti$^kgea1?V3z-zR`uHd(!O*(Fc`C6ONd z);s^D0Uv8IZ%5TU7V33U@-Og}>fT+V8sb|UcS~-!()W-UrM*XN5biqz zr;=b7E$+CLm4;n8JJLJY7jXL5_P17c;7AkHHxaU@hYG7EaAEiWK#fk$2(l<#oe2GG zpO`F-UEAPJ)uzzj=n##J>H5byX*Q`l)6nau7Gpfa$D+I-W(J#{gE$>D(}cZ0q?9N( z#r>}%%N>JQsk`swP7+w9I}9umepMD09&bQtW^}+_@zdX{y(`poabn>=tEeSneSgqJ zRkO3#esNDd?VHZJ5p&|KLXfkCF_0A&Ut*>b8*H0a%q8{$H%gN*-WhA5y|srhOSH!w ztp{9&o$T&m7wjgT-3nCzBegf1b?Hi*+7g~8EpE48NEMo}H7UcB!`n*lUcsgQbqoav%`1EXbYiRQLAn> zwI`GfmeFyEa~ACgbBLZe^uawl^YkE}i@U~db4kHR0DfCV=d8dtazgg4V{Wio<$GuH z?mXBg8GZ(tYmX$nerPN{#a6FAQF6VFrr!~DC_@6z=E%PFdO%&o`GmWZ_?`e}ad{Zw z-OTkL`A`z(_Qmn7rG#Gr-cX&znf&!p7y2;ylI;Y?W>N#wew5Zx@yu%f@LK+o z0*f@TbaYF#$eis!>I$%NcJJ5KXW55h9&M|h*Y4efSK0johxtJtO@1Lv*6v#x3uyp@ z7XY5qQtFP}-qfnmC;XNoRNEmC)|<6#UgCS?^n2GZL+I~Qt5HSv-%Nf0S?3S?_Axj{ zY3XyQ-G$VWf*mM{!JA!2bgbt+{>F_1s*tfvVHoH9Vr66Pq0>MoO*Ai*l&_1C7i+D~ z5PSe_C=ODa%UG)%f0uUKcm!RrMPr7V4gCznw1^yHk%?N0O_xix#@rDha_|;AJ6DHI z3KQ?=cXIEy_`69SNfBa6iV=)$-2}-T<`iM6qlVm6z8Lnc`g~ypyVc3d5FQ*jK4^Pt zIyxcF^5!alpJlFg`o=_$HO2zMh;iaKySqvf`||7yWJZ#gn8o3612>plgG743Uh$c~ zB!g8+foKxG{CF&_YXbYOI^*mqbo3!9se0|$+YfTkHd(5!so}Z_7juZFl9yVLDlfew zM+^IMGkZZ;&6csS*58*Vvppol0@0G4QA-^9#{i*zM#^5m);*WvFTv0JoPD-0HZfq5Gqxl&h zf1MaM_}vf4h>u&#mR*XwG9fsle0fN~xPpYUz(*VT<|dIpl&;8sQM#W(wxN<*PhR@? zruy>z&{seFVzNr1?mt5{W;+~J08_f}7+}Kw*Utv!-v_|~1Dm4yXRairC`*g(i zBhj%~JsRN1`_%h9=hMZ2R^%suZq7M8{D<{Oi$iJU%Oq9T*BenrUb_G~6KsEHg^?%B zUEu=n!x_nVoAa)lITO=0cSs$!ku|bF*9S8zH81wT$wMfT43PH@*v{L{K=HUDFe|8|Q(F?RpoG2)a>cwM;GW{<0V0mC242w7dczcM{7EPZet(UqoU zAQ{NSw6``?y{(v2GAs`Xt^+un6ZFu(r)ibxvl)Nfq}n28tNr}fw>hz&SGD+9Ms75_ z6j=D;S&X7_lz(@fz`*|?{1m=GN)E{A(kN*gA%nVfFlUm1s^3<@WS%iqxK92e%q+TS7f3(L>VW;zhg7NQR{OE(7LZJ1 zSn1-4Cs#JtrhP=sg^pu`w?r(4C8u3^J9{Sw;e~B8$oHm-kn6%ioVo_9$fRqfylcec ze)b<)R}^j@dFAF;#7Renk#2kB0UDE`X=L61^t@n$+hp(R_dYNmzwL4sW@TQa27g*u zwfow3W9PN72fom1scYAx7Tx=%9~c6-!gZLpa?-TnWs(mC8FhVZ+{BnsB91*j(5qAk zh{j0QACJ1PFDDho;DizmgeE1RE^Tr1~P$5QtXEW!YZm8HZag zSg9a!TeF*q&5Hn8XSXsci>3slJ<3d#5(f2^^y1lWJQ|D7iyow4JG_I&-o?3cV19?o z@j#cwhJ#8Y!*v+FkEK(S>ByS^Z+95mFmB*I4qzB8EsLx1bnXkZ4=4u~m2jyOu9L0L zHatD%alGbx8lT-@8-ITGr}u~a&$~R>3%+ah=wg|#JhU-Mw1I_5+NKUbqak-eMA65+ z#^NRu!@1Y&?b?~(2fG*Xr+Z{=}S|;>0JIo)<-7iJc zS!z27Ax}!WgV7rToS1X*@numK&H#Tb?#WW;e^uZ+TpiK6-!#GJ^Nb^Y>V7%^TI9B$ zw(t-H*^Ry89#PtRxuiShuyXvFPm+6r|d`S-N}M!J*YS8}XeZ65DpKwV=Qz$q->= z0Y6p>ouDzN*Q%tz!#?}N>|)&Yo4zuzSdXtxWh5_$G?HI*dZx)T{fTVXQbpO%;BTvu zA$O=@QC$%jG2nAGR+)`cir70lNt&gme_28MgCzieg+fD$TV*J#sIk5>gu7Nch8tvgMCI?fsR~?;v1j|%-)!x|1TUyKJ)aN- zv1^VHg!_OIk4&CuCnLfSDh6a*;3Xi(EzLM{{FS;JRP3E zmMB??FuIy7JWJ_EGX0`oy)nhzK5*{ z-5x2N2P(l@j_B?7dqvTcckGmNTivUr;l=0Nq#H4S?x-a!nNr)cc**O>Dj>AUF!Fx> zaKaNzTkSj~Mj|byBS|1QqWtHDH8#q|Gr|tY_|M&!shj`a!7)V?S}LQft+B9x->L|o za5BXXFe7k4=E8?_XOG)>17_*loPC|#I$b-T9|{EW=dxy}d!<2%mKbQ?lusro4R zqO&v#-COFD7xDecv-4<$&I_kLmuqLbAN1VA_T7vx%HO8BUE)~?){&tjLbkps@C^e- zB7OE%1+;gxEovnZ){I2RF3E+9g|w8>rg(mO$W&_K{sVs#RFN4gmaA2w{Lmt8BRxSSM-|)3&GvXy*bXI3Ve~^wA6?EInFz3yyNM$>Lv6UUqdjP@cYdJ`rSu`TI!J3 z`td*zH5xhNyqG3D;3F$LarGr1>n>Jg^ZlZzZCRzp#@_jH1^L!_DRW&*pM#v3s4JWC z8Ev+y%FN2}vjzTxYj(oqoDWkpKGwG#XkV(WLm^>qD8y(d$33mp&%|^xs;3W^ZwW8i z3fMC5Kmp6`byhOKri%A_L|LM;f)-8wkvh*Fes8K2Rf*?+l3(I)-3^p6f;*w;=BHN! z(w>!A$7DTBjgXwBLuB?09LhDf>v<(C?-qbo%}w~_)9?GUF!+T8LgmX$q#@;mEuqfF z>+IL0SUa1rEP#|U=%Or6-EXGfP0Vg1(fMAk$#qbwz;u+=`ewxN865=|LzKFu!98`= z)VUIYEC>d+laF<%bXD(y)9G8zO#SX{h6tsp<#gMkqBtK;u*tL^6lk{2zvNMlZnxl# z7QnXrW8!uXp>ybpIaRYHd_8Z_e8`qy_w;adt0W3NHo1?VUT{>2 z5oK|eupC+l=r140?K72cM2U@LqYB#BFNDMP5ygARO&jn<8Ym_3)5}4}a7HKXz`+}f zt_e^IOrMv%PiYdfP-o!+ml^N>EIBG>^t6Lss=BFG!3$Ge3?XXPHp8!&R~Pz|;M)Vj zPVaeGUm0nlj3U+S@HnL8c-Z$Je6j)PvXiu0`!DFbmdRCNUMI|up*}J0;45jVr3V$% zmF8uqs@wv+bOp()eif!Siwfbbo`6`c_E_M{$BRW<#vO4jFJR7mxP)o3R2*^5uC{y`(;r4-yHXPD_%vR*k$pxO|X0~ z8M+CN)-;pwk-Y@QD*3VVF+Xe(Bh8_J4nnE*nMA>Ly}0a7IdsGpJk2K42zt&lGLzk3 zFGvQ@JSXM#Jd-IoZUQ*YkrTvQVocc@=sjx4xzy|#O%8mA#EqB#IFBB_NN0}0A%*zq zey*QKA)yw~_wGWWN&ERL4Bp=ClBHZXEZT3@Y5Q4`S!U5qRnE@!m8r>vR%*WgV*o z_M^;9mnti4Jnfj6LrF!JaFa#F&XBUq-t8!+d+GxRA5U@t|Ac{~eN2|46RGps^R=J7 z_xskA@~W>z8;p*ptG!5E99<#Sd{F)K)UbmRr(Z6+amTWNO}3n5ee@*U``^b(yX!6p zSyk;{d0|v?6}I&F=2$EUZU8j(B2S20?LGY?;k9r#K>=c*{bH^-A7t#7P}|X@Hld}- z7g=wbn0KTB2>QyDo`sO>QlwP)X z?%o0DUnA}VE1CDBRBgi?mQ((B1}1>ge2-+?8T@whGY{f?WF4imAFk{*zV|XJL35?2 zuyw7I=10!|e16vQwa)uDH&*nE_#(MukG<=v=P0qyw5t%#Rr=)5kE@5SSo1|TY!8>~ z>r>J*qfVz&RRMv(t;Bb&6BXmpQs%)A9@?q9g>mK`qPMhuOq__<-WA~bHWH9Rws=Mc zX0NOHkB>@dDqYcz4dZzg0btA#0AnA2vmQr(GyU`|NZaw!P<62pWO3o_n<9NG1?z`j zPSs#{sSGKwTcVALe7N@Y5^2Iz54rl-s{fM2-WD zqThjN&k9J;<8ST|I1yl?R*y3>IHq~f=mHO@Cu$bs9GgD^y^|eG-f^%`0 zn{Fa$4p>{lx2l2MN#&4)vSZ|^ykkEc+z|H}=l=G*Tm{K{3UzwS^a9F$l(*5!eXhre zmv?VFBCc7ay4xv0h?SF}$uI|ha@&6o7uD=NnEfK`;_l&4L(gjfluPi~<1}eF`+^^u zyT{$Y7z#9i2q6G1Yktkw)z62xg|^UC&?kg3EQv@ZMAkalZI+?R|gyG}F0|8zgjaiJ|%(K|Ad#zo<<}$d1faDf7SwhwL^fAe8iW&iWeC zR5R~I=*Hfi*g2sw57CCgDnFnt$gOQ=y7^^E+{bZZgY+c7=;{vk`U*+L@bGJw7kB^Y z1~`$qX5E8k)PB>|gS^~e*veR3-Kcol3B;ldqHASv^{q1N?c;eGo5)Ifp0jMX%fg<&Lj3}-Lq?o3$sAX=T82-QpL^~NXs?BFmoCxA1 z8qRx5Wnr#V9DuDY4scBoe@x`6LQ07?_P7?7;BA%Z>J=Fkb7hYiH4UUVWoibg3k@h* zSc_LH{i+m6e(T@&hi#A1__A|=LwW2`yX-c^C@wI;)FrBG4PM!EU6p7L9!N+k43<7muHvSHI)sFl9;0~Dt5qV+n@T^ zUiPRZ%enPej_8VgSKRpTI6hVljx#q4Yaqd#!AM>Wk3lSt*z@OZ&g?=_UHeChB8Bob z&5~yHO6}}x1ZZdfZ27xa4EJ}k?M){)C>YqpgfcH&&-M>4F*``qeq)fCX%swt-4Hfa zoEUq!qP{+AQO@`d>w9jP$kkjyvf>U@)0z;O&6sOKneq&niyZpumOaY+q1rym@N^V-ae^Iw8zjf+wjH**TT65hp=u$NA}roAB36_~{veweP^;+@(}p zy^L1pvH$p>SrIZdnpx)(OEP8Un5HY9xV~Ac%3asAnfLYwYRy_5?M55!^s>IqS zi5qMtwbN1HKR-v9>_;dKkjgUUt#)=+2aeSGK8*Q~#pnspF$m7gz+@`KBqTg~GR5g; zk|}qk(17!w58R9=l(?kAJDE|P__0+fC)el3E5Gg+&HwyM0{uJAn`&lB8NbqX{T^=Z z)U2B=p9O5iEd}lNA17+a$C@9htB|SxB~;R9%v&Rw;(TpC{^#oW=_;0qHwdnGtT(QI zrO;&LKNd!jz^kmlo7LRARE-cs_GrztZPlNTNpk{-NsRzte66Hfk&3%Db`k@ntgX@p zfKkH8fbQ{r7JK{LJ9Jh$U8BU8L%TIUyn2o*_Q`4+DhwFb>`_FunvOy?E;FcM^(Okt zADyojy5r4>4C5vIMBW_zT~A3h4xl0uhMbLVeW>6>194cU1~x^1?Z2H<4Z!Q!vY>zu zxt&|*UA{YJg#RIkW`Q26KXG0FOH(4j0A69VX{#GO>`8NU5z9o!SO&D^z`9jto~z?= z*%e7qzf}fAOcw6W>S7{(G>6Vr5lUC6$Lq($v4k&v1PlZR42u1=rk3LkT{|jxXE&Ca zBuM^|skngu>??-1&C*Yq8*QjrFuzv`q?fB?>aLu0BOJE;wFux$KQ`f=B4W4+xRnH~ z+513PtKq2j3r;w*j+p5dHUfP**xh|e?_3aqJtR-1y*(9YKuo&aZg_G*w0TId$*-@N z5<49>8sQa`L6b${Xqi<1UdyglHzUQtpRd$iE{{vKzBbt$O|)c@(3I*+q;>+*Z*at@A@*25X6fj&q$aO5|(6!X%#|K+bn6 zmb^J53~=EyXJZ=L1n5-GC^euG!(%~sGtXyRElpWnEgb2)Gr4b!B8~#)*Dg0l#jU0C z%6s4+s15AMyI~n-;dq_N&6_t50B{aLw?zCbG<64jk=eoCXGF)Jddg(>xda8QozTAU zxuC^9gE7i%2;0Y5pP|0!B$(-DJr?~N-{s%h=Z2NGkLqmNIq!4G=rLsit!qYH%Phgh zZ5n^m`g3kk>+_BzBaYUY$^yrraezyTy;d(45;G6@ub!3uKC{kpV;DnczNR1x`LfE7 zpf^fn){~4gqe87}Mym(w&m}yIRxoEB5}tONeMInTG(X+$ zwL^^3F}?g=QV~Es07|j1fa35r$32v zIfo0Izk?iNi!|-qQjy50h0Zv`Zwh|Eac6KQ)xV60*Rk{~TkzD6q^y<-q1-u<=wBk` zBXwb3$@|PUD8;6@I#Nd_vJPb2AoIhPWBd+eY*NJgSSI17Fm$Ma`IH-vGT@)JD?GLz z7X4!d7_;>YfDx*VWr%lXzg{}X7wM{;GrwbdlB355 zI;lJ^HOLKWBHrU?Ims?iEdyp8T2F0uat&_Ax59eCPM$6`fjeh>6n>mefSMvq7ir(&;R?fB^KhQsLoaJgSWn%gj#$W z=6%=i)II<>wrkl#IAU&5Kk?jowXYJhocR;Q+ok3>gjRi%2DlPSB#L zSmxH{D?q#;IK+~2wsZlq29LJo_3(a5nOKpvTs-?(M0u9JGPv!(Hu5&iVYk!c_<>6P z8ebRe^rZC=te4CVRVE94Uj-SQqy!6+%P>B=q0Y-L2hEbsPe*;-1(zBcDSxZKTgekB z^bB&r^s=ci)__F{2#|nrVHo1LKTrd`u|&39ZsYl1x`5`l+WaT$Pe`DZ;q)&5H|Y=8 zno4_>_?C0;FPZOQH8wW>1jb66*K`T4$LPV$krW6JJ}P|nZg|J1`tyX&QnC5#s`KV! zVOIAQ`Jh)W%MaP8y{HKo(0;B9%!z%l0iGDudJ2TLDK%mg$SQdOXkCuHDk8AgyVL14 z+Ao@!=x2t1gRC`mrbD9Z>|z!9*tz${fm*0k9}Q(6Z=p8hf+SJO!-#^Tc#PpO{!jJW z*VEQ$$MJ^6r!DZ6Ivl%h;Ddkv`?rf+PNU}9$%)R#S8J^&S7}+y`q>uwpznb=4r8`~ z4d|ysEVYyoVM75mo|=6e%kNe8Y$ zS$InW1~3ZKuB7mIKu&`}OPr1tP7C`3j)oo| z=yBhO=EsS5sK^Fw_qF^G4Vap^qCRd+Jm_ow9-Qt96J zoL?HMUlUG4Z}W7u!7zDXNEnRoQ9!jCj8eE~_D4Sl0+g4AnXU4h0s}NGVaEnM{RnZtE`lA=_lxR3F_4K?+#n z)|XBed?zmh85rFU!pbIAAfTb`J|QC$bpslIP5C4J)>H* zl9s(xf5gG&)b9f4((S!%KCx7k-k~Q*K>RUN>$L1r2-Mb1)A=Re5RSKFKz zeXj$%(+yw@f5xnrX`P)i_opq#LC+V-rK9DMLx31;$5RuPdkgtU$-7}rGtZc()~=~` z<=k#~x}Jr(-(_$s%>(P%2LSPiDX-}Kkm(|yJsYlVCbReAlbGOsX?vqM(dj||gZ#e_ zr1TBxnJVr(W9EJ^&88?yWy_P824dm8DWI*_1ARwJVMkEz3#?AWn1qO&mT3SCLA8fm~RJgcMH(+4j@1faJ|hfQmEt5 z^#YSEHS=@5E&AK-s5RTvv8S)Zq{Ehm^_VH`o?fbygAEGMFUtcABB1I~gBX4fhF$~6 z$JwYFcz_%#sY0S~cBBf?%w;N{-ZkkjsqGep1dzbFn6+w5PG2pF6DUka2oD}tDJ zoiXy2jw_WwV1sXbxa0g%FjpozO}^iL~oDAWZj2)gC$!G91^ckVpkE| z4f}NQL)B>?f87;P`kKEn-f~&|RiYB^#^7{XWIZoT`v)v@0jv7bBzW!jMe{qg%J=h$ z^GD5W(MySk2is4BjQ#jnz|6-_u~#|Q>ui**T!62Z&@(>Bi90!c*`W5P{2j15yZ}%Z z1LP0zbuWXGe}Xf>P+X6kJHS~TCYjVMQJVCkC!M167C(+u2e{15Sx$!b%n_s6a{~q9 z_OVKz*&kH#2y$HXp}Mh$^3JhjC-(?XM!!drXV1oW$rKAOj<_3$uXdNA1(wZcZnVCf zgPx>ZBLJbWXQsGm6zIO#_o3PH@T1-(^ee!0@Eymhj>yQ4>VQ%09ej~Sv(K}V()PUd z>oXIP-Kf(?gviS5{#H3AX+4Yh?<`%9#O%cvsr9*CjG+3(mNKE<9QV!e{;P;I5C;IH z!9kInG!2~fBQBiCgbyU!n!L7F`%{I!cb*+>I4TOjTP=(sIMiWWT59mL(5>h?=&3$B z7Wl#o;ClmL1VGb-e}j1T3xF%9lnMKQE?aMu9pv|YvUxABXB!HA?C|LXnz0yvNKDtp z3A)8J0A2eQ&T0m3HNy?r?LSfreaf8RP!WinPVZVyY6CrQ;DZr0qSLQ3hyau?mj&qW zD=UA(yx#(gY?Nux8mbA>;iUy^s3?o2R}c4TI=U;Xm)2`+_E7Ooe1}tUO@w}t(W4$3 z4jg%=dvd1Iz=W&tmJ3LgVgHM`caLZKfB(lvp{R5qMW|OrQS_=%w4stt(up~ZN^&MS zn{7Jkgh~=(i*g=0XN-=7G>2x{jF2{mZL|}cZQlpg`}_U8f4|%J-*5jEWj&sc$Mv|b z`*pwW*Y(_P@aEn@$hxPrK`@VQA#s~E{Yw~mV^a3`lX5Vy$X$%;k>@?IG@c_A zGiy9;pBNPsCZV-10%kYt*};1SW5drJsz4FmOuA0a{0(zHo97r=sTw1D+cf#3WVlCW zI19x=dJJ<#_+V6#K)|WvG&Io~S=F@ikULiOEe&|*ic10hxMnprY6V*MT~&InXgPHM z^rGRsg!#nR$?iht`0lv-R^?y15`4WPooS7X?;RWb39BPd6|Ktc@pUzI(^PA&oO0mY zd0;9j2DU-Uf=0Hu^xJH&MhZUnrY!-TLDsvrsR17Qe&Rf04O+k(n8_UQ?aR|L4Pf$~ zNsBQ2QcaXph>;1+sOTGmc1i#47ly38B#{TD28ba;dp)I=I_3?{5#TXC$$L;Xzw%^hetAL`^E2ZaDntLU2QJ?40 znMIxP&=S+3imvsI1yOtc@d(VScEU(O?p|1hH)T{CzJJDjnhXAFdOiJ6Z@GMDKR_&pFuVCsOuqAM~1uMss2lT5~2_Vv#65s{I@B#_e%z<%*bN$d7LN;p>R+aD&o zz~X^{B7@nN;|9u|9Xkw~HiiXNxXlkWZ1NNd_B2d~W~M1Ya6ml-=$^Wz9S4rL^}q1I4C$k@E4VYAN$AR{y%Iq9nRadM z%0Yc!pu5JkZsuo3^Uja4aJiT$;FQX}q2tc@{k)n6KEAi|H_ZlmO5^8`g1vwj*}%Z30#luJJIC z158GkTbpW!Ob{%q4rxUHO2!7DjFPl1o*;l9O@Ikg(W9xFm@hBhvrkp#7tV!h7{9>2 z3iiZ;{d!}@?mtx1o2cIg3g(9oT-cm6}#?iw3EI5;=iE09f`@?&7v}(uu z?KOQ}V^Kzpv%e;?kn9#cAe#YU4FdV|M7AIrjTn0P_SKXr3SM2%<~osLzdCS)^vD75 z0p#Vl=530>G2Iz1+TC*u7T&eSDC2Ll!k=f+1zwu`V=n2|;&WUFL_NuQ|g^##6teeVfuUAoeE5hl!57gjnR#r zeES?WK2}t!6Me4F4D72Ka0)>gtq|xsz!BE{O!s%u_Nce4-evTMBa54wC$nG#h~!(D zCy0-Tp8TW%eo3T>^jWhGvUn=GULv5y!%A|o>VxRwOj!H&+*~ZEXdhIFKKrO5Dt8X>a?QK|o!7hN?@+d;19LcBvU((?M6VH;kLs z3oJe}-0r^)P8S!S|0bx5Vkxe$w78tHBGwd&!Entm-7Qmn0C*%=TJ)UIty(n7X_QG1 z1%faSqSobZJF_a~&fyLBm2{$T-KhUEsXMhUHvo?nXPK4b6HR!T4lkWjpF1^|ydi+{!Q=Ji4gUpi!mI87%X+gQ5{{J3 z(G>FG4Zl*lST@Ubu0|!ZB=NFIU2|w|P5sFBGY&V=11OL_xN zJK5l%tbrqLm}t?*PF%$NQ8hq@=PM+reR855SB0h0vC=0&oQ#3lOT(eUm&PdHp{wDQ zb2iw08r84Yfdxg&ZQHkd8ArDq14zs`c2K1PB^FfmYy8)p2acg16c9|tIE(t)0G-G2 zt;BmPiO>;r*W6DpayeuOy+^p$S66rj7Fs@6DchBJs^?xzMxh69jrlv-W}{ zPN%J|_Sp2H!6FZd-!RG<+?~N{^bcF~i_Qo%n}H76)Vw=DT?7y?&G|NL0@fEZj>p3y z=ho2u$}E$$e&B-n^r?N9PmOr$ik*ffvvY{)mqhbYEKx}0XwRU3x798LSA}{bAUSBR zK4A_90xf)WYH`2mZE*ixhdo6Vz`O#^{&QQ$^kDEAIUQh)q$T`b60)D`qBIR!3*+aw z{j>La2+G?d5jOJ!&+2>^nn=!M^3DqIMgLyS4 zYHRiWYqMOEtwHq34}A^nYGQ%zwAJtqi)+I3J%B!2)K5$Q34!jLe6yf;j(KvVe6us7 z@2-}B!F%(2w|lNs_u{}Lc{Q&2akrgN-{8v`yX(W_rHwGHDsbw;LP9(=b;XN;SZ;f7 zW@Nv!OA1;h5qj&+e6o#684ql-E2j0|1PtB#xmyMW&lf=Um@C;dmzVyv-MQ>2BWzIv z6{s*4Cf%LX6NidfH8nAqg~`J^Cc?&s1LnvU3s)ox%$=Dmutyb!H_jd6#sT|3Hf0qG zg0!7^hg%>31(I=(0Uu2EP(3FUD|QBmmOKgp4q0z9K5bOdv09nS{K1O zEIjq++IKtyfFXIA3EVSII+cWpWtx1R3ndMmNDo1Is@Je-!iJdQPMz-0k-nha;WF@t zm1FOLbKF-4yyT(k-^E2huKs?OSGB-S=3J|o*God1)`5bG;2C3IUZ)t%h(zmW+z_+t zO3CR=M70yQzO9w(ytD&!SXY=4NA2*UUvmY>9=xmBBgx;{kH1gheLHy#E*?952kTR5 ze<~Ucctlg!lV)AazKq9<@N-GWoy3PAdWK(YHuIN@*#jDY1pqr$J77!Z!tO@)U#&83 zFR)5I?Q1}gB~g`rSJh;lIh)nU3Jmt=WdXL*JVOzXlki|=en?`%oXhA1pyF_n|LO(2 z%R?XTOf;h>nq7b@%YE`%fRIhkWT*bAMg1S2i)wqaeIo}uAi@u{<{l|dfQ_w(8uWXL zrbC_rrF-N3H*^1Q_1L(7No{UTX33vABP4wlE^=E>d=@!^+ipftE(ErWsXujZY%(N@ zn=!DjOFAy!XlWVZzs~J?^j2?EzbbG|aw}%LfgpY6f z7A*Z498qmKSKN!mHoJKOR}s4{Ql>lbyLPyDcZdFNw2hcCLJ*bo2;k!d(wQKHp3Ns``$>#`)G)Ct`6V{>CMRnBckmXzXwJfb8f9*y=r?&ZN?u-sgOQGuAj+G_fsCl z=8Cr)q|8sU^w^{vjywoXWZY%_Wbn+hoKBB<%k6*XxswWqNu zv6AV`g3qOAURCa>TZ5rj!Gp38G`!iNoPx5ncO3=C<4UZKVN#zZC1!*hISXy^Ot+uj zY!BP6u%#`vqtuCk&2}5Yj01xLJ__HlStSv}RkW#4a_G2CT*HpK$I)0M;vHh7vpdVK zt1v66{}ex(eg=tYKBPZfxX_C^Ud|?6xwDc@&4KO35IxGvD>bn|0Klf}B$YO%euX~@ zJ%c%363J^Ks-R9YvP|TxQ4XkH6e-N^Ug)ux`nSG96N$~#YgQz^88o6z&~sv4KFyM1 zGU_jfOZ}gZZ;e};A8_W}V_U-%7DeM`3)N!ZjEllyU$wqk>mX`0FruDi9qJ9e=T6A= zyp>vym8nIHWF9-!k-vUBN8C-NyLz57X2N`itkOaawDLs@nFMw*iNp63)C>0qhDYPS zhNx9ik&noi^nNP8N7yxe5=o1NEvI*6#uBd~6*{9CFJ9Ha!z2P{|Di+uwSmQmSeg3% z)aaVKO?ny6i3+$ibSh#rnAR}4c?YJ7Ex2)1zpBgr|)sr z?%RY&wK!!h+>b34<7LF~@E3Vut($Da{OzH^E`23;k z*APg|1h+^NN6WQ>%T-%keGXr>w+})`gqNTTLMtVm+LjY*p58|g^nEEWhc8Y+6SZ*q z>YMwJCH`GV(<>5oFV^?k!vFC|tFNerDSUR24@-%wehYt+$)blg+qnk%!!H@Qg&lCk zKXt4f{o&8R)g9#>#(}6+9oI2dUhg zrv7B)=GExtQ%Az266)@Zrg^Bl3k|F&7awQusPl2X zl`GGy4^;o9XOMr_W_9GfMUmmrw}-z$=qHYs8Md25HHFOIu@Kh{A(yL)xReB z`jrYbWZ($(DkZ$V@fmE@()V718E7fGGNNb}qbBl zcX*M#gys*tSEl5THhQ3+5J@I&EoSumXlV2b`H&*5v*;y7lB#4T)iUs)YNO{)Cy!75_R_A{eP34Q z&LtLazAG}7UvQ7=(mJ;Lhey+`i~|bgHI)!Vo!f=b5^U(og693^}Er3#;kvxxAC}-pKtL?^T>jk{d78f~! zl%8jN$!^mgxG5UFxA2|g(+;Pn&9}%w(W2$eWpbj^Ef9nZ%oN5^fc^brrEg^`Q)E(` z&Cf0`+(~ymGb7Un!y_yz&7w2jM!O(KlN_BM&eMDgR^w=&r|8;;j{ z_}5@AH=2G30da_@BF~Fk)Umju6Ducb$IupOqWw09u8`*mGIeY_`Nn}C3>#_7K6g_a zWuKjPi`}X`cFsek)`iTE)-J2$2;bsjBE3ty8r~_>+e)OxD{(Y(7B!x#m=fNWDn!PW zocwyi$j87c5fX z_FpT*$cg$?kk$uC598Y;x=!-E2-bFO+thw2Ip(l?E>zg^GV0#^ky@c5_%)2&rdv3X@2QCORd*K)zjXV)_yQd98{9HJww`JC@X!Y`U?yCpfOeZ_2-#@i|h^Tifc4|tH@(zG{?%)GRLBUSk#Zcv58{HF-pZlDS# zE%Xdam$CQ9m6U%EcRh>AM*H_1T4qr<5Zx>IhRoZ_n74nqquxdI+P$ke!B9!^lhvQM zsyXC)XVek8BuJB#9Y#ooT-B4Aewx9CtaF4K3e&ep>&?-IUzB(wH z{o3VxChtI1G`p4;JpKhwI&X8A!DASOg(Nmw)?vv(jobKRlNjhtrzP>5=0mN>9c(m7{z=bU`` zFZOdQCtd8GKRvd=wTHr}s&^$c* z`2j&VYz=6cEzU(hF`IfP+yfASNI@>fY9VtWlZlQ)k3Vn6o&Wi6@45`u@Qr#~p@o;R{+SA#^&|>|mTxL{dmqp<$uAITiK1JAJ+&u(9}({T53l~+|`RdBUeehwJNI6Sxe#*NC;Gd6~O zRqv=Q6+d4?-Z=QL(|x|L!s&Bf)=~RR`JLHfSG})x{tN*F^g#7jD@3SAkmSQcF}sb( zFFEaF$Ht6($*TKxn)^Ln$@@vdiGGV@jR_|PyKNc7ZiAKTZ3rlu(?ME=uGHB5VO{hK z^z9>Y{T5u~nTN6`of?1+d7Aa zBm1{CyVraM&82#1rg*fiUEsw(pP-fg>QuP=VdgNzit^XzuoXdvnr#Skjlr9*cQ5IJ z=i!ng?G%=lRwaoluugXm>Ga>dgnq4sPtrLQPs{UuITfjqlrv(sEW3rnIuslm0wwU+ zudvuk4On=H*TOkNfmpc)Y!y`QE_?eC^O|w-5G{_Xbp?Sp4tI{pc3X?E6!C;^l78(+jpT=08cF7-j3g3g^j*M z8P}9ayqmHm4kQ=J-jZ0f$>By`Z~y#WK-QlHGaq}L!d~g--UYxf@Si!W(Po~AYq;!X z=@SKu@%6{)aCR(Yn#nx7$2EhYmA%Xv5OG7IqXrsg-f^~@#bo{dq#mi%x}Fi;RXH^K z<3+8_B5+=s^HiK^OB* zwuYU==IR`LLC#x%&T^P7s_hCYBwcj##dV)lv%ydbvTWQP+yC6%Io8V(9Y0>}FRNd{ z?`k=()()%DK*+idh0P?vbG*@^|Hcem5AA#zY5#GR>Q^2~GpRu+YqurIzMOVjhT%6^ zyH%r?XRfp9d^aLtHe(2yP=4o4rdDGVW}9@ZE|&_cx@v_zR;$n3`}=8d|AH8_)Gw|9 z8oDlCE}X``5Oblt?((`)HXhlRL+|e^weX-7q7q4{D_fbhTjF9^!IbRgepkyiXU{}) z^)bFMP0R&->E>D7LT0KZ171!e+8QVfY`S?bTtQZ>PBUCX6w__;W~K5B$D%s~a6<$bGH+CY@;*p5drpGl-NS`^W1`LH4E@!8K-q1hvIqSsr*n z3UgBmfssp=L=ToYrfX=wzm*1KWUEVvxDXN}`qtC1gIE>H|GCKJaxqm9#96&sw(*$9 zEbq$K%37k!m*7s*JHr|YQx&e6@u~A-T*Gvs2Xh!ij^4qr!$94C z8Zt)H=n2w3^xg;LY(i3HVe+{XfN#g~N&CKyEEZT7f zN4m)YFZ-JRH8$_ap_0_v=;xn88;M`3#^Y@#(*m$>hR+e*zn8^ucesaqE1GPrRzk;Y z12cosZ!yLT!tkx!#o0y0=1UVGL}zLwt;UkMww>6~r#xEH`E+EJxwslGjWsdfyN4AJ z=U!#syW%qR~KAG*5}ISJlZ#AOD|gAjZt0Wc`yJ(h0lM>rV>%bR@N-F4zL( zP(}0Qe8EI>>X77e-8!5uE2{|^5>NOUjJa%Nlr~@pwjn(_!q{mb>Zj+ugyIM6M+tTD zod<%)5g+dr=Pr*79(fTeC+ZqkN=X_7Co-HOI+v_=QM}KL80-L+@t>_I1li$j$nUM_ z?o9N!!=rih-%sZOSmuwV6=A9zXSkJ&E8T-`_FBd`Y{LX z7e~e@Fw(?}q2X*2KbQ2ox(3NloO8N7p2yGCcxvn{YF8EJyo@M=>Y8|Ye<|%_|I$)= zX?d(D3oGS+JhY&IzZb@=dox}N3k%fA${7x2-+y8P z1d>*stEwoK5Im-1?6KW4(R3}&a^@I6arjo6N~(~^ZV~0nUR>K|Ig7U`g-#UVNc@uj zpEcb)7Cif&0DseUwuSXDzQzv=pH-PqRym7u+$_jO4QqH4+iK%M`vA0BnS!xjh6apC2(BT9`zp^>Zu)8Tr5gU!OJ;)~XA<-wjeAh@QA^A|f88@y(Iul= zme|@AR8aT&n#lrlc#IB44DIjzIcpoI1MUwIf&i_u@=rd(GZtH-uBGbEy37#$P{OMm z2aq#<;j~j7<4V*EHkA7Fx*WfPGTTpW4mO6)AlPjvDD-y&9?54+TxVUl=SH3~P*UZq*VO z$0A3J3ppn42a9Y``_1qVj68?kry<9hy9@17+*MH%but8-kgnVj`7>$QUnLw`?G^+` zVB|2l(M3h1^EJ>wzZ9=|wt>KM7)y7WoC7zFhxon;&@V(@ETBtevi9X!S>12np6^RcLN=l{#$95f zPV}>qliAzd#^l)#E$1WIxDr#@c1&76-HFXe+|Naf#J+Oh=i_fP@}BdQCwlf`4jM$I zjNe4n+vh|XM@UY?J}{UXTmY}|AQ5Y5&_={WzJ6mbhQqg33@nuA^>>KBeN5>w&jWFE zIo$f2E1QL3Ld8=e&k_r1BFfXEWYd+QC45>p$f_~DiyY-}C#qI}L^bmB5tUsIkNO!; zD@ZJx=X2qc%HR%hj8jERL~cmb989(f7mJlT$!(NfwEm!JS@5mN9>*S1R`1tuOKR4v zSzd95>&+4T8uFv%bsrgy%mrNk*p+FU{P%J5S4kvC2wuy5j|ZdiiZH~V4neP%u-1-; zw)+8Owl5v|ybtrLlx}_>eoKV@IYPo$uEfO$5L9g~SR}O|U<c0^^LP(yp>=zO~_*XW@45?p)YrAg8 zlK6{L?!7yXE!z_|vc&G^uiBMy{uvye0ndi&c8aNM$IK1?vz8dz*uPNga`wRq1`o=yfI#rh<6hFJZpSZJVS>0-! zrGQIM_dOcdpGcABN>?upG~uh;)lY3_Nx#}k9I>TtT;l@n594-WfH5yuJ}lwtfb_fD zJZP&+3bGBgq~6oF5A1{97oy=QDpi@iJe-)f=a)-@WlT${{7pVNk~|PTc zISrTtoorq-zy&PlhM4bFs^PA7mfix&*h{8M)daVfN`l>>fAlP!bn*;yl_LlW;9Et* zP}yBb;il8VeGM{0l~aq5ZTjv)Pk#d$!w6O4I)ClnWGqun%FfC0Ydp4&2SEsb@ODqg zgnMAH#3WaS_^Tpcn$K}yVGzqY!abHPTXuclE;<8LAaM+Jf=7*s;vIHKCTH9n$~Ew` zaR=?$2r|lXV2X9`d2_jer){^K=y`J;m0Wh5auEv%caSuL=bSFS+b~u7(QvS}!%sk9 zT;`Mwot#Na+TH_eU?*{uzn+v!bbQD6j4R1aK_aZ;`i&f}NAXL=!6t;Jt0`J!__cxB zN^(V6{ig&a(^-Ra>-aFMMgd97y4l0-r*Ap+$l6G$e_bbNyRHr5&lxkW?Psn=HsCK_ zZ>{{aGbGYLs;RIbTCN4ut9H-T;Xc-iSCvIpN0Jnjk7Vdz69ht4MLjDD%GrZ{yA&a8 zb8&InTW1VPiBsXn`!f=Q>LbJxRCd4Sy~dIy3QJq29+6H%{@t}#1A!d*St~9s?F?-x zuWeAb)I}PJyrIDe`?W2#FREV7jO5?_VSXsM&mvM=6v?L;ynV<{ZeaAYrc{JBLf4Uz z?nH+BbxFS~-1c;71^nfhP$)3wWxg0iq+y-!H|8G4Af6)Xqfo*3o5Vfv%($}JGIcD4 z? z0)n3dCIlQ21K*G=NGu#BnHW7;TzcTXSA(Z*F9cz^;7daCUQby8IJ}Q#$4RHL&!wY@ zo>O8I^CNLOE9D3nW;PMH5N2*?t!TEq?sNYeSn|S>*(2Z;6os+x6AWJds8hok#wt6M zy-|{jG4n~fG*CW73{rkb3G$LP$7(u!$`?l`5hKl+P+Q2ad8_Ge1hNSp+*^_uu_7&T9)1bA0X29|r{! zC9zUS$A#;Q&0e2AnL3Mb`4pIO4WI4sO!;`gzN zI-#k`yWjQOx@aa~<7GZ* z{CdY5UA4<{Th<3FEJ#=9um+*9WwKRqpv8EGe=k%^k>?**^6G#gvvOtG@mCi>>BW*4 zuKbmbd}(P2Dvld6Bti?b=AkDVxVo#!jvEaYIfN$VagCMVhHXDmAt?-qD~bC|v@|c# z(|>(F_Z`0eoYD5(A9PG z99zfM&M+(-OHcVAY`!@$aA3>*u=h}WPmiQyiq|*>kLCnZn%46TkzZQGFG|mxn{W^A zEnMM{LUd0R9dKMec8g3nL8K8(&AuK-8gJU3U+tOZy7!+_V*;n1jTa@9ST~n59s|ki zy0+Nn#B2znNER4XLhgwkoN*vUR^9VVLzTq92p)u7v8P0WoQI|a3;>?(Fze>oV`$mb zScP)caa^dx!*uqj!1Rs&2ZA7J9PF`4?FG;e>(9AZGnxqv46)cO&kO20xQ<`LKqFhm zB}|)p`s~MB7CtABA=2xI>oRS6?`~|6BlLIb39}kzYg4EU0pDYbWhktDEMY+@GW5y= zj0ZE6J~0PSt>7kW-FrPA)HR=I9riQ1XsN&y#ojb=E_&f0bril?daw_I*pTB|;xc$& zl3xGlV^CaHW#!I~%2{fJAE?Itt0KK}kzc%8S^B$caYy4yq?e7(5_W|TimJx?Zk->q z^bO+#9)7_&GHymtI&x$Z{msmY;=h1cnNmrJz@H`=W^iu7*~HpLQOuZI)H55br}%Ks zu1;t-mlYo8?!BbkurmCM@A+)8iQPm(N!g~EC3pklV^;%ypW<@xGwO-O4TV3YD5}XNyWB2h;T?!CV`a=X6_bdI3QYE{!xog>ezH-5*H})B zo^x{O?w(b1*F@*iKwtBK*%`5h2rlIm!hL?O`eNjUzbt^zZ(WJ$ zcYgjzSC zExttizSDmwPPOh){n3IbixnUH^(_+pHltSK^X(f>kaEF$S(U3s>(^lL zZqr@=w#IWl;0y9vSqj*nT8(>X%%?}NBQEji} z2=c_8hA~{nRRn8~Q_e&6j4Zgqn_IpPf(s=;39%mUGZ0wkXA=0&we5|F+;`S%!A=Ft z1meH*podN|{TK9AihTycp^XM!O?Bpqy^n@`2g_bYS_YSfctli{*LC&6Y%TN*E>^8N z^+g&R+{lx3Z&DIhv0o^Gcg897r2jaz`)98ZR_eFEw5j~0XS^1WfQZ|UA5xwvi3r?g zczquFf)Sq7+a3jj5^_w%kAqxt1wibz!VYRctk1PUUo=wWfT$Ub?p$)2o1X$S4V-S%5m(l3bs(F%Qslj z&m)D4+#<#6z>U9^u~com@`nc9bbUd0AFgV2(%^S;pX^mi${pY5vEI@Sf-8Q}@_=0JyN$F7X}tjkTP3U1l6s{imK3TyPhQ&c z2>k+Vv%EP6aoUO5?~N>FeS6?oNPY6Jv5Q?s-Z9Sybp&&c^xK2oU!A;t?;lCT3r)r) zRYl#bD*s>h@0MQXmKIpKk)^QW@+f~9rdF*_Uh zA_R--^rQn>j8%_e7v8-rBcAr1Jpwwa7IlovvgsqMtmw$%S@(6%9peWioGaMgi0@wy zb}P4nCqU> z{@|TGV|8Qj-31qZ4C1rS%9Z6}`vYfpjfT=$b1rmEP=uFf=99+(5eM!R+BM?X4p9zX zAFo(0WWvs0U)rH@DfK8O$Z-#-CpietRljxX?Ee{*#@;#drAUz(W6n&Ny z&Fx^;&7~@VXOtUG1Dj@pNmOV@N@q*9BFYB-oO5WBa zD%GvlE(%h(ZVujvW_XeU8>3nf3L-;TjsgtN2YIi$`EG}u7cXN;hc3n=#J6U7ghnEf zeO=@@xpS%SUoC)xu~d46MM6$>k2n&$8H9PlW=CHow!rZrIq=AuY>#^JI@`v|xt*-} z9Hs%rdP1AXl_F=uVD7c_Y@k~*$*j?|F!Tu_l`7ZVwLUfJ-B>*#DP4)HLrM&m7}0kB zyV>wTD4Zo??q_Zy$ONdv{>o>&^~_((Whb5y!15gII#w@eDeGgLv`KARUuR$4&o@Ao z`8nL(Ig!=ZLW0!&8X)Ql+^J>ngKVA)vs=S!lXnJ8I-iXoe*X5HJQbnJsm$ftt5gC< zxS=_*9)hsroN%|&1xDEgz*hu(M4yu(bu;9V=QDCu4V--}GEZPE)SVukKt-3|M|su> z7{01Mc2?&lxu}92&hWIwD}Z|AFbuhhYOy~uozK9Y-PA$pWsC>O&K(eDuN<%m zh=W;HIK@nSDsq1w3Xj8k(xm5HseZ!WHn_CI>+$FH7Q6&RXLhahsZ3Oy3{;0r8y@?d|u z4<)FMwl-w?cog&k68cbwg@avb%dDNLn*u|(BlKjBAjcyUS)yzA= z4wDFV-yY*pI{EVZl{i;|nd!pFjQtlBtX>MN%#f?Qgz(1Pvj@2VcpR;&K*1Fui&Mz)va>#r|vJiz#z<1(@)kNJ3p}g zzBSc-!;E_-T;tn%liKPZnf6iys=s7{ye1`57Ap{fxvBkVS_HqoHIy=VYtnaB{{A_P zu}qv*yP--Fol81K(}MBl_GVi_I2ZrQCQXTCBfDRx0~e7|Yu;TCoYsFGZ-kVlXg#X1HU zH>68+V>WQ^RO@WH2cxilqcOX*t|8MMz8kIJ5Z6nxN%w9)+q!g;sgO!1kGm{&R?{OC z9X_DQY-uHDFaMlnpCWky+%dx}J6{8thwHt9*xk`8BADEnTVR zcf+7f`z||e|4P^Wbz7|9K0~*5MSQ$;YVQTW;G#-=4Oav%au4TYfSoP%Fpt z>SPx>CY+-1&j(9SIr#qLUw&`PIk+rnr%#QcBsrJy$Z@4b@V<`1FtLe& z7;8g1Ht0vavDwTDzJmqZrA2Hu@f?u8?N88-mLt$?E1K#$IDYV^ zofZ$RUG-jvNePbge7FQNs8e1k`dl$0_~-NyDghfJY%EIg_p`X{mz_ceLQCgSPIqu$ zt|x1|$*wr?QmOZmn}ug|eZ3Vqg92Yq&Tvh?2j6n%ct$3 zp24~o&q?8pzi;J`qN2-SKF+4`dku13&05?zH(*CTrl*YG7Fksf{fM-)e)Xl9Zn-y5 z7MAkM;dM5yn@F)4$w4g$TdS41Pnc$I3$;}ExB7GNl@f=uU>3r)y>^G4+@0mr^WIJKi7 zSA4O}etfq4T0~^_tp?-84E1e=OR5utlT!I#{1qxuX36LmpoMSfK_~BJrFtGh>w2Fz zK3Du4hWGE16=He=*X?JYLR3c9ASDqtA*U3SEL!wyEo1r#Mo^6Qv4P;yIwsWN%)r-> zuBP(hvIEuys8q5Ne}bno8rw%|LE6uS+orkHdcx_6$$7tx-|MI;Wyd{JXi6px&BNb@ z#+)}R%%9MEB_;_STOcqN&r!p}Mel@^6EUV6tR%-aTuG(!R3j`jaBZy#q7A3~_VBYt z4>@iIr!)){kx@_%pgX9twJI&%#$otvU$W1A0aaUQ&x@+}`0lIoRT-ivFiku)=7A@} zr1!8Tj*h@t{3hqa5)dJ2Jrl~40A&ha`{Ks~;@G#Gg1WPs**arDwCv)QUX@y>wSENu z4!BLjlcDcR>H*hbxbK9DsLLe{;F_(OJ!|*>p*S*EYj3Gr!*sRJB9Dfp|uoDpAh9`yh`~D)OwonIlw7@<8GK?S6TM z#2QdS7lh>_jRkd2#=Wk zvmRsh%PG#A3)b&8b~>ZmZkWP3eXLpI15rk`fg-AwZXJp=871&Q%y;?(z(H83}XEk1} zlsV7hX!tsWp3Z(fcGvT)O2;bdDi@*adAm%U`KHW!@@w$l$(G(J5aahH>DwxrqcrKN zH{Y7;n$UA9txCU?skv2^B}FfaF(RzKd3@|x3;*%rwG~Vh9QWk$R$;WXWzWiB`&VTS zzS~B})`YRsd(4K#1ErGYE|Yu?6Fp8Bj@q_3<%-Yp`!_M3!=48CKS#Pn9$&KeNqFGI zo<-9o)pjVm;TV_3`64JWE#;Yq?51essjYFKNj23T+2Uxzmmo?Zp8QKh*&eMP2;I!; zn7?NdoF;^UPU~IO-m^!~>yLD_J^sM%inipIO-!BwOqep9N9kC|% zwD`3$Z2a-=e%At7nxCb#o1E`arH<3(WF7*#*%5y420i;FZgJA&U1}JUs3y*Y&Mp~{plni(HOAWv*OogI{9W-ul@BA z=Uy*46IbGJqxwaKW8g9^JI#}$z5}`BmrPxQQSG7HqCV)3&z@`N;V|g8^RHQ^wV1$( z4J<;x`c=diz8CafcQjn09(!wve`Ra+i6WOTH9#yD@iTFFzH7F~VNS)GC#Oel6WQFin~8 zP-HS}>BB)Mz9I#GjSF(C8R*kMSv6#a{$guDEx(Cb3oWJ@*$vz(CcY`EHb83Tps1_% zeF&vkEL_5MIy7(gyFi8J2G)?$1y>o@AXgozvK^22tI?tIl*`y^k6)G!0pZhkxY2Ya z(6&0ql7m%!YEXQ{nUrhwB?`=#2pFZ5m|hnU`L^*sva1O@IgYeZJ0UtT*-ePp{< z_qp3fgS@YWj}U2X&%O%i(29b0Ia#a z|Mf7u!*ZOezI@n{bbs(7B=9ohT8N}KiDC>b`YyP3rg8ELrb>GMr{~9?mS!&Mfa>SQj!VeGgKU$)!R2GlMY61Qw=8Y( zn!>HUh^oGNPK4rkj?~N==_%%+P^E?5U?mN^FcW4E{~77;o2?P4uXUgPT2A42FbKcp zNlTK>t8{>}>B_*Mu)4b!k|U}kb69w7PVSXyrmn56t#y{TpY2Xdm9I2^QYKG4ASe1z z6PXR_#MNDA^`LE{OaTzzK=9%jaMkW1g(dy@x;p;BBc{iAx11Ii{-VQP{vdx17Hzjm zZNqYD-sV4!PE>=g5q}o{cmsun>Zm%p_eVv~jq61oPDyRI#p%fN#xb)fv~a-G!D-8Z z%?&7+Ez$+3w*RQ_^ zqXM;FoIHE={1t>BbwSL75>-eC5;Q+6chCZ&kui}QtStLrd~+pyRHfQAGPK7K(=djD zf;zeItR3ZK^Jtl@wXbl51}Z}1^;pm-(;@V(yyPqaqWwG!QEIB_XD*wroo2KobY4t; zKr(k8DY~=Dz#b8?nK#w%yDp%CmzKgb48|^sfjJxvDEcgJl->8Jjk;N8Gcr^$U3}vN zwdZT+8zrVOrdqn`dK)HI>it-hZ=bSY;I96EZUSI?`T<)EG$o;OpeH3dKcyUIXqoD6 zKX8VaaE>!G;9Pjt(Z%|DVReSE8*pN_Ca%dkK-GG&-qV zVJY9h%IG7^sz&E~q+>6C1#%H@$Lcf)B$QFN+To(?>+ebpG3ADu=GGqDQ&T096AN`*Ef6soON6dU?=DzRG zygM;%{$5l3(mS+~-ZoYT)ulNAd&wzCJ7)MGY^7iC!?-bVoG11Jg+Njyde` zURJ}dzoNMG9^E*`U}=?nbueeJnR|F0zBSSldC%#Jhuo+8aGTnI`2&AsA20J>Tgbso zvXN%jK|D3!QpoCcH?O5iL6+XOpZJC5=~#o&k> z^B5KQu%*uYwZ4nZ#mS<5l!6WMK5ir5={yEe@i&5{HjpUR>6{p?5q|A`GgZJr*t=%aRg2SHECq3*QJ+YiQ(q~c9R zI-2O{gCz$(8`^O)^@tNde)YkmrKN69lcCEC8eiim53%~749m~Tspqb*aIa;Wy)LY+ z57OPp@C3H~fxg{0vF-Vh z{oniN@c2Py9~G$=9D`ZSL&feaGsf~(u`A2oeJ{!JuGsRIMfq0^4@4Ul@->1p?>nWt zCKMb{<#7Kr@^7rZ0rGb3!dLfmaW*wxk;LM!L~B`m$pqaadqFG^#JdbeeG4J1C#hqQ zaO9WEfiHzo7~H=HU{DGpFdYC~40sNat1|B1oB0h&n$n#(B>#HxvTwFUtXh+jmnhn5 z6-AZ6Q!+|VLY`YVefJW_qIIMmIV8i^*u<{$F_A=f^l`%C44BHK2}7r(I;3a&%y}vO zbsYJC|N3Xg*0|DYCIFh^URanmSNjtnahk7}VA>HGznH9;RtzQS8vf9rVeP=#hr_ zTilj6S)A5iw6EFyTz@&YVLCvy^ZLywAc3~RB{fF?za%0w>8Bh5)cyBw*Bq=Eba}&x zR>HkQU9P^-kkmiz5K(_af5n>zy4=JE0hf*pAo9&PAxygsfo;(;e=cK)*OTHyK;wdK zM(pYYjfGB4=VryA`SJWddB5vm2XTaJ?$Wp?IOA zlt(K9vj7eZ;J$b##R#x8zo_k83DDkBliB>Cm7*)3F6L(nJeuF;RMKK8&hl7ie%3qg zv3EAn$lDoyR|~GOA1U72iwpWcum>*bCE2c#*pV)X9nV-sD9Zc@jx^V+B>Ya02qXjAS3C)8Z~6m+~z z6qdKZ1e2H;LgBc`?78Uj(eKEzqg9NP8*5*o7}zAr%|zSN1QzB=$3>{RFNJZRZpdf1(5#gskoj^YS8hfHR0qkL8RF&eE5oVN8>4;? zZ#iQcz-xEd{z0S~f<0IGo11JlnET5<^hTBPr0R0Oe*ua@N}_u7~%@lsvaU=|-xmbhnkSMi2My*ReCagLRfZ zRP7BL|KqKv_Kz$U`7XUb+@7aia8}l4AC#?@ChVQM!a~zI%R-t z=(?-%>-6+*NJGp)VdBxhg$@}xJCje~vfGfo@Er33fP#|wiGu>N%V~!_xF~*59(#{>WtF38}U)m17-jtWE!hGNPZnx5Z*&IO=PHPye z>;+SE^i?&JminIqWMjN+T<~-&iH3IEpy~(Sf#9BETvUr8emc3)LUAVi;EN~(!eN;g zYyf+{CONy-M^A=H5O;NSxvH5MGaC-N+#E6)|sXznm0GAy1fZ;?t_k^T5Q^N zP7VSE)QO=xysvUEJUE_Tx#W8FG>N+ftsaD>^QfHrsKuql)hn$qm{W&d*BT{irD?8V zAHgTkugF>$dq~)Ky}Zo(b1A{`3G7}Abn4-=lr9TS?JTdNsb01vlY#192u69me!@+` z3vnnfVFlUb@+7xPfItA}*@|08^$!}Wup5y}6MrIcNdN5K>94~%CDa;A(H81%dHMH- zTl=Ptvqz%P(a`n7vK_(VVVynU#himD(>`olg=O4vmxWeQ#FEIrm=Q4p`BS4@2VkA!CW4%_Fo z`@_4ZaEc{)0V>P7@h8Llns>F=o81h{crcVuIE&MoKJVjoszM)Inm@iH%Yx#8OA+~2 z$Rw3(l+955F#bFCTtJ;<8hGfdTK;3}J{rL61vNcGXzt`}1p=a1q$pVT%bjM4C*%|u z7xN5M@l4U+qUpeg%Euo^XdmBmoor|`SMC~%QucB^OVLdm*gVQq`3by$RDVd=E1D*C zZ$@TZ^nUfDPE3GJZ{pkIj9DAa)ePH5N~-tiRIgOm9c8gWP%H@GhCv1zLAf2#p%v4 zy&arQm6i9ccEiYw9&a0@K6`Ceug5ku&E8AT(=}rs=a?6p2aX$@8)`w!ASXfLRlRaT zm-&`JRf(qMj0x3E;&I5y49~f~W8L++`O%@R4b>tNT1Itd%x20lauA2f($6avy?61M z+~r@m97vhfw$PS+;oL-i9b)`s_-bAd^R_QRn-$Pii`T@X!p z-XHFFVj`_0M1WKc`6z@vD3SFZv&A}mSk(bv$a^5|J9E-ovr*2r0BBp5NT^!oX0K&F z`;n9TkKAD5Vx!`X7^e1eqO>M*YQ{kmcygJ~{*DAoHxS~xDTpeN)gtk>`swkzv7G1I z?3kqts6!0py*`oWrgr(`Xbl9P1SESU7en|{i39|cbunZsSw!P}Z&ly21h1I2(A^du zYclXsHuZ8{ic}BR%^`-dCf*Y`x9QI^FBr8*>aZ3tUb=?@1x zkh;h~%We4&4lR$&+-U&Zl6qe#dqObP{&flNT9ph>Ky~lsOF28P&0LXs`L*nyPytqH zKQ{E5k(1H09q1`~TjUsGfGSS!lMJPAs)y?Pa%>{DF4{f} zJz&I4uP6mPC`Z=} z&NZQm$f=P4Oq=zR{^2jMU7?^Lz%R*o!rx?h@{j?vx7AGl*?X*e`}tpYQHGH zOBv_xYnCbgRcP8CHDTPg`Y4a}tCL+7cB3y`sZ~P0%csMxZ3(Y_ zYR7lFC(&43PG^EH;(I|5Dl$?Gia7DMqq_Rl@|FE_bFJCX*%KwtDljB?DAto#<2C3l zP+yJoG$~T%WSNU~Nn6vhtq*1RQj?#jj`|7PkYysXw&xw?V{9BpqJ|FE_;eVguPQ~`PCtfr4G9U@*tXb;5UWhTr+Inhnob5;fpretngaIIm~JJu}?w4ooSxu*MT(9YRJ=k6%&_SP~)9*g4fb z!)A{;Od$8(>S_<=XC1!h_NZ{1BJJj1eu0Q$6kkFtZaR_EHU)8&Y6*d?3&E=N^jYpi!yBoxzb`m+M-jDx z+r{c>X)-|JpxRlCjPBuYeO{hh__CI);+UIcYn2AKSA80ig4F0HTnX|3uSAK0HbyDkM?Wyu#=pIPcX9l%&{?C$qwGw+1jSB$0$?8;Q?5ik}0E}}g zhlzRDNt&`zoo`Qg8^Z4)$0hNsL;6E(MBQ0>F z)F_M0&oAsms>)PwtDqI%{D+Q5Av> zR;V;C2|}VO-;FtsyXl&Oj@n620WS5w5?UesE(tdz72z$iUb~8A#Q}C zT|eq62r+-Y&9}TbFmI6iU#Wp;fvsRRUp}$07>j^^LI)lxJB_>{uM?`SA{p|jCN@#b zt6vbzg+TwF_kmKG3gT;jM6p4#uEte2?JbyXer*f+*Qlr0;nxu7^~5jg}q283uH z1s^1NA#VJv(mQ9K?zZuW{4ev^M#~1rN~~A6s~=hk;0VXaHGQ0CQuAtp2D9r3<*}k=`<)%^imIVunSAblz_740Fe!A2VM9qn2oqjzD@9q} zH0^ik$&MXG3L{wiCx==zn!IB$ixtc#wpTpjr_Z1CvEln0*~;RCDnu-$mzG<;u4muF z+|ajY8l`MBIT)dAJ9p-Vj5bw9l~wfMz8DuLxYKd3(}cmiW*^lPkAb__HkW26tNy43 zdVUo|9sX+W&zP&W5Atu+iJDbE*YGhM*2ukVwa20La!~ZC9Ge6OmpjHvb;D?@oA-E} zM$i#d;)S#x^x^*&L(Nfo&kGl)c8MxVh0kUW^-Q_78#|Ly%KI@h<-z| zsJ=9DCK?{@q+AWwB)15ciP2@?sNB;cGoosjTkNRS*j_we4AeE118tlcJ=WB$34$?u zjBVhSmC{U+=wvy~!1`AQblspK>0{=TM2?Tu&Fngd!qovVHC^D==-Lj)Sf-ecws6Sf z&CabF`eK~sDR|CpofL??@NMglaRXtQ4ip=bq++GxA8G|R1}%Py*dl z=6Ym7^vhNdmK`)P(X@^|1>)L|!yGeG=i-j1owqbJ_u+QB?CyAHet17G`^6JE-vUh> z{0vC`GyTLXRj&A+>+hq2ZG&nPyga=WQ(dyXH&#YY=fLBqfu?-v5>~-;#Wyc^tJ6@! z$dtMsOSeS&NiQY$fmL&ojGmW6TM5r8)F$sxw`29SDqstO#o0=Mz&u++HODtCg@m-c zs|QqtZ9YRimVChDTXqUl_ph7TGJBbkago{Ta1(P!wDuNa1RvI_hQjNPoiam}7LMGKpjFBKTBgw< z4rhvp8i&W4nx)f6FmDpf&+pHZZaSD-8W0@jk&gGeY@GCUZ9lrlQ1x*kck~Ad0EiL;Fg?_zBxPpG`er}-Ucs1d)(oY~SILkBh1gAZ-VQmE-L~JEL>r@_E zr@<*cVS3Al%FudJSUEPL?z{D;!nmIdF$j?(oQ1GZUH4tSo@&T|zTJUN&a9j{Sf=M& z?GWOJc#SL4c)j<&TXDd)FKa+L(^6hS$gu9w-^ii%<6=2a9~ZZlwI6$73=NB(I5-iG zbH5}IoGLk3qhy9Wi{quMs>y31KG<-CEPTu^8ce^jvvp6N201zKVf}z|pqJ*A91M3o zBJ5}>UMT?3Ha{+|da$gJtNTPHd?c+-tndaV1%pGE|2alXcb?k}?zS30ke z3KFD#m&Y3L63t@_5aUkN+f#3MqDv4)hUuf6yz^QlM_o$U9aJUmNSeI7ykLXjY&6)P z(wcoVbsA@4e$hK2)-TX!C*Jo1OHBhxB2C|%5oDO|8#d$ci01vL6#OnF{%hl-(D3j+ zwwK~&S9_+i$hlXv;=3;(x27{eEOPeq^D+D?)cpSj$Bf8fQJLzpnoZR zi*-zM6<)6FF!nt>dL2#ga^OS(r03wz>@M} zB0e=dq-13B!vIle`j}*p$KZ>-;+}G<#9nfv&J8wZ&nDqgRBL4h-!9$X>&h!45FLrz zZbtyx=TiXx)D~P^^}A$zMErhD>QYshuTPNfP#_O~3pouUdDSv^J*fCSq9Ec~QCqv7 z9^bFH^T_nL5x8Tp^M95u=4N1cxYYU=+(A?_`yG1%xjQRbvQ_@^qR7mMsx~1uTL+ad zQ(dqCBAF+UW!OQEyn@2kVBvh4wL)1HCuoqmTxxg@g-{r6{vwH{?TVM(Of&EYfykSu zXHD7&_du?XR{Ehf!=G(I^a0(zd+6io(H&E1UkpjZl%x;Li&-Rs&Tucn(Dh7qtQ5US z7cow0V0O0tHZGnBk z&ouHfQm{EW_P$?7Lc`cy27h_6BglBo&@tv>mm4bNFN(s6GB2Rn@3TlDTzvBxSi7bt zY5o{B8@YxkZS79m?*)||0Hul=gAqBih)RBA&ItW0|B?WQ+0Pn9oSIl7aC4*@L2?7)$#oJ^;WSsz_m_v*Xa+dCw+I`W#yU>#4lw0N=DA=etb$BXGbuwZ;| zLzD&hk5w&lAcpOG#?$Jn$%F6Oc_2PyyfVsm=;-a=GpEjKrQz~KP1I}ub^WrucneXD z(SRN}_kH~vu3mF%pvlge1w4I^ivT|AXOju2>c}t2gi>xqB z|2H?8b~~=kod~5QJr~)|o@&lMFWG+yghZ_$ddTgx|3x|W%v^onVB0_9e4Um)>VWrv z{*41t%x(FCd7=}*0II%{WviYq61ZE=jy75oYdMv(<6C{MOV*?QGsaoTwg8LYQOy5f z(<8P|=D|FS^b^LcDX#(H&Y-GDZ%MTMR5ENZ1puNcJA<$23*i17#-D4{;^}i8^cAvN zxGk`;r_yK|fnGx*^di%0bu&(W<*GW~{ zX8fb!`2~5Ea+IXfv1!_iaDf?nA35cYIFTbA<&#&>t2)cPmz`sK5yx~M`Tk7v&bd`6 zYBcM}19qoo+GLVzqXl^Ily?wng<9XxQBAENHi|o?$42!Y^Na-sjb6=}tig~%U>pBA zEjy&wez|h$ajczoX^;9qidBOMF7aZ^N+@)B=0s>}Zo5yxM%xG}E5U0Xbw%Y!90|Vn zm5^dJpct>G;OKDTKtWLM@Y#i469E=(%)^i+3uOh z>+(SD__d0nL;Q?vl>QDCxL5j_ED^etVYa(fXn?(Q$~F|+((u%FeYehj_-thRt#PeF z%L5E2+O-%Ar0AerrMDX!ILd5dzaGXfYW0P@J%PhEN;tv89o;p*?JWTz7bO{0+3E6c z0X`M*QU(_V;EoY*%Pw82-8#J1TwT6h;&M!LQ@K?={N|yf)UQJu7OR|}*;pVD6Qi5e zt9gO_X)BM0$(wQB9S2{48rck?e5HfGRv*N79Gxc# zO-D$&9p9nsTrg#Lk&pXO(g8O(j``+)!%={9g+)CM5Gp@TS679H(1qp~VkHFsFlcS# zzQ;=tOE3d9E`5DN7a4;iM7#CTcY&4sKDaQf#HYF9c8%AjHo1gWx(KAnHpwPIqr#E{$Smrm(!?DF(Ua@t!WUaV?$iOPu%gj9$bCt)D|LHWIux-&GU0J ztMoZ&L>Ni=FR$6g;+GH;6@hX7I0!%Ac1`fPL93;ml@uC!2wL`8b(vB+ULXh^QK`E# z{#c|GK0h1}!Ot#xE_sYJcROSOA^c1bihF4w@%3SV3z?fGUgx6|p2pT(9kJ57F<;AVKo?TUX>(@M?1J9|POmLwUY&8jh8X zUB$F*jXTjGbnQxfHsQ@&PXqo5JcvVK)aetUX#-q#7V}yR#R6DBL${k)zK7^;sb}Wk z(i^w(v%dKrh$L%=-79lIDFf(gjgrvsv4j-}#?3Z(E;SxfU@!>svnSqeL;EB<$Qgr! zi}%b|sHNc*)7dP|2$w(d%RkPj7rPyrSgtw0taMyzU&!Hw#7^->IZ90Rwva$-cY=-#g{ z2QJiGaFgX3?*#Q_c+BSUkSqu)NFQ(Y3tA09gG$o3A{jDY3=c@qtp0u^^YoTZS2-|S z{B0v;x6}^)ViZqn@cto5quq>~FhbKX#FPjT31YYTd(Xf*xg9{F8l`RE#WmzBZh_Mq8(6sn~PT9K*AOnhovlu^$LLNX#h-BDtR^s-7-m z`^EX5kF$-Ss~h|7mcVB&0H+e!=<=?;#~TPL5ow}rzEz58*?1l&;x*PkCi_;cXo-}D z_qvC^oaWMb9HMG2dyygPgvyEajbCzbDW1IC)tC;%5tImimbYRlZ;O|)@eThPg<2km z@ot;;_kn%cJmo%nQo5g`hkJel}BdOf2;>?UVz_Pomdp9iqG-QXFXaRN?K zZ`O&oQfSir5idB#n=L}@XSVBw)p^xiOW{Lgq)BfnXcOI)ITr_%Td)Z5@M-u=w@gZ$ z7{SLyiG!5dMk2H$)>(|k36RnS5W(s?BU3pBEV~_uIS@6*q)miw_XDWZfM+z5A3Jb# zHZmFRER4%p{ch#)t2{7wD5>bQKTI?h9^gg#GfLwoz1sD{IcDdN9<+A<#(*ePDlY;%ISoj}?`M*@R#3L`tDE zKlc=PpCU1vz_V;K)Oyw-db#9c)XonO{SsC)H(+Ezs@|ujx@`9)*4oUj`cxs`3&ln+ z%gSrP_KxV$-OkpzSz$}}*7V(I+gt1XJ>)kB{NmZMsZ(c!g z`r=<9QLOUc-N)s*bE4{%OTt9~FBwilVBRXMzSf>(*kYS zva~k*dpN}?drp+Y^28uUlv{go2$<>|4uH&Jm@D5_K*1)5@&EHy+qvPrr`?M9Ap5Jh zSCsnhvSI#!?p#pT_Cm~s1E^1PoeMwDO2TLI-r9VzQZ1OzoeNJQ0;kmU9h%hN-+13M$E>}Llk{;0^F?tSe*(;qx z;26F^a5YmoXUSq39t%~Rr#PORdAIPjfw~39pEVx1KCRxQaBuUyg1GOCKX<}0ASIF+ zXAh}){sIrc8Y_PFhK--GHi>cS1>{DUaMPfleCXtlyek`T#dr%7eeCL+q7~=XCfA?R zf44t)v;PnD;C$b?9fvnjdM%j$`*X9uml1I1|KBM^Hi`xjF|GeuJ@DWEu*L*TPEY?U zoFblK9_lsVTn-@VEY@*@1ZH5d&dHM}afaNfs9auO@{|N*TK~6c;x^4L?SOXs&5>6L)GdIsbdiG$WU*Cp^D-snNH!0K{B?_qX0N3ER>>p!3UnR1%iJ2 zyV8G0c>%-LJRd?`1q8oyxBgngArfFE6X5`iX{nLF2zVK-_S=ji3z&FnxaQi*J!Sya zLmHS2?NcoU@&<98-}i9UjKHAKfm|j2wm<32%+D8I20GW};G;%{Iy#A18jIDZYh-xx zz-SM*}xiFAxA#`E^ry>m;cFU&^KJ+$7B_<^u%i z<08!WrbOUYu*Dx1@Q?$$o%h*??%DCy!KOH0@?090P=_KkC>VxdKLuUUC%m@gJPX@} zZi$K*(dQ9sJ_&OoC!090@dZ$V)u0ZiBm8>1ynMDJWS zePPuDN?4fZ#Pu(9pNB$8Kvz$r;crZ=FYYn98Qv7l8!Jw`jJ4Pk>725n_L}1?6J1tj zGYjv18Fv>eLq-^^aK>viquA2{OOs!NrvemWTz<4Od-8S57L4(0Xsdsea0pFPI&z?V zait@CuRT*+YJLEM>RsG-qPYZuPr#5$FQk61ODMk#H71$yA>NQSy>;dM9fOh~_e3OX z&0(O;y}sg4s+@**cKfG2X`~XleKorpzumJ>$xRY-&5hJL=F7!c$?={HK<3v?|MRE4 z`V$~3+6hj(o97yPnd@v%dYA%rJ;H+z!P9FPnL>qe*Ub4@D{w0LSqY|N6q$Fo!2rof zG2*^adA1D zOm&ONJ~~Wa{OX~3h1$AnnnY1`ZKh3`p3vRrlm0oZ^hJZjokdjN!{vw^**+B(-fUN@ zQ;v#jUkM?nxl@#QSxl4IHMCZXG{_X46}nmaerV*n1X457VxE$G1xub7{&>Wux2S`y z>m{&W)Xjr*{*d`x1^E<-`)QkQFKL>P8|$ReTk*<Cxh4_P%;3`W`5c%x8FH&+2 z8?Ap^jx6AmDF(iw9bGDA<~~?eu7)j{$f^Fxx0VmVhvuj>Wm^$Y7PooCkv@;2tjT3c zg4VT{7x2ZL_IGhd?Md9>XYpGCK)R8QsV&CYNMa$hf!jSKljjqfULw)v6JyZdhT_iB zua~3b_z=2Vnh;^!G358yym^#w7q`X5b+QL>e-m=#hTW>OUHr2 zRe%+~`t8%_ut$Cin`%Y+w38gMiVN1|j<{-4Na@)8VJNlRiJ(ml9zbMWC1RkuX{gMS z>^P^v{DG_b1$bE1+LXnf(!t=h=&rN(3#Tdx`sR?Bz+bJxPu-M!cS<8YeLFuX7~2v+ftZq5pCfLt^knd|=`yo5oOE z7@sMDLK0 zC4fnB8DxOgK)INft(sv4uO9oc7~B;m_KN5!=X$I+zEz>QekMH}bj=#yWzlRFV<3Ij zy)<;a@z#4%0%edc)!cT(;IiNituR<-?$PB7V#J^w+|kdvIUc_F8(UWL8|!Hhea$Ev zRV<~;Kmg?}DGFKqF}2%)Gvv0gPN19Wr;Qw)H{4YF*I!sd%QJsz^x$Gy0sma^4S%H2 zGA(_4>NYhb#vr(BWyp@JBsfiNMmh2f)SI2en;9TY=Ar9*9I%L-qVCl&@JXh&$A@IE zx9h$Mkk5!8#(cFujTGeGW}&Nv&@*d(O7HfF9k^}Wb`lr?j*Za{-<->jK@{GNqVjQ{ z-DiOBw>(EohQBHtUvC{KF<_33Pa(GwgKD|YFyGl1dASI|tGYktogIIm*LyQo-8IzA z1G0=z#W>wlk-VqR?mAIDz!JNoeIRhkm2m7ODuH&7MA0&`2n)?d7MnCgVLXzzL<30W z{T@P?WgIQaHyYW+BO)f4vp)GB@(cWKB!_-`1>#xh`44jsvzpGKK7|y>Lgs%i)ptdR zL90{A<6t@+?CxcHoU#Z>cRo(;6)~e`MNKm-<37O!SO!gVIV$TEsd9_+M_}s)rxw9% zP~x-yv-uQ)pM>Z?hVT7ES!0&*kN5@u&);NCXg)eZ|ujUxrq?e4ZP(uXA~sFm)L= z3-LYod%d{zkI5;H%{=e>^7XnfIsQ7R@i?5HC2O_SfbhC|jb{|Di+ukWq5*H-Cp0T>O2(>E*-8IXRt91*9#+s*`AP`O)BssyaDk$k308 zU}F;8bu23K`0lIuy5T9C?k}ma)=aOGlM}Fp3ZgBNi33w!xJNwIQ#6c*tzLA_mMM1g zuEi)xurRJ4%%VpSa_`SInv0|tbai7~DXoote`U%u4jM5D{#Yt3Iyjj4?o+Nl=c$V8n~ZCL;!gbBd6vOcP4f| zDl&S&H^(KUxh}G{c}rbZ6yz1ylW)w%kR2O|Ip^;FSO)ML`>qQ=WK>uELGOR{=*M6i zk^@7#eBY0cu>mC$qV`{6aWIc8Bi2HeCKe;YU`|e4W$?n!Otp|#KbC#)Q}XgZJ!9p+ z-Znb7S?Aw!Kp>^R9B0JtUtWO!GTLw45gh!}_Wl39@_)4@*jU2`j3f&AG|@tRF|wBg zy#J5?>{)%|v)9P^Rr=>jq5t#UzrOV5Iiz#kI-f+=j&19nGqjDTa{i3@X^ipJ*#8d< C&9X57 literal 0 HcmV?d00001 diff --git a/README.md b/README.md index c535e62d0..6e004dc4d 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,14 @@ - + + + + + The LiveKit icon, the name of the repository and some sample code in the background. + + # LiveKit: Real-time video, audio and data for developers - -[LiveKit](https://livekit.io) is an open source project that provides scalable, multi-user conferencing based on WebRTC. -It's designed to provide everything you need to build real-time video/audio/data capabilities in your applications. - +[LiveKit](https://livekit.io) is an open source project that provides scalable, multi-user conferencing based on WebRTC. It's designed to provide everything you need to build real-time video audio data capabilities in your applications. LiveKit's server is written in Go, using the awesome [Pion WebRTC](https://github.com/pion/webrtc) implementation. @@ -286,4 +289,13 @@ We welcome your contributions toward improving LiveKit! Please join us LiveKit server is licensed under Apache License v2.0. - + +
+ + + + + + +
LiveKit Ecosystem
Core Infralivekit · egress · ingress · livekit-cli
Client SDKsComponents · JavaScript · Rust · iOS/macOS · Android · Flutter · Unity (web) · React Native (beta)
Server SDKsNode.js · Golang · Ruby · Java/Kotlin
+ From f54a10304962a29ce9d21b1f80d0a9c0c25f6ac4 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sat, 15 Apr 2023 11:51:28 +0530 Subject: [PATCH 084/324] Do not let request layer overshoot available. (#1614) * Do not let request layer overshoot available. After a layer stopped on publisher side, an optimal allocation side while initially adjusted to not request the stopped layer, a subsequent allocation went back to the higher layer although it was stopped. Prevent that. * simplify --- pkg/sfu/forwarder.go | 63 ++++++++++++++------------------------- pkg/sfu/forwarder_test.go | 16 +++++----- 2 files changed, 30 insertions(+), 49 deletions(-) diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index a0d419ecc..b203c27da 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -578,54 +578,35 @@ func (f *Forwarder) AllocateOptimal(availableLayers []int32, brs Bitrates, allow alloc.TargetLayer = parkedLayer alloc.RequestLayerSpatial = alloc.TargetLayer.Spatial - case len(availableLayers) == 0: - // feed may be dry - if currentLayer.IsValid() { - // let it continue at current layer if valid. - // Covers the cases of + default: + requestLayerSpatial := buffer.InvalidLayerSpatial + for _, al := range availableLayers { + if al > requestLayerSpatial { + requestLayerSpatial = al + } + } + maxLayerSpatialLimit := int32(math.Min(float64(maxLayer.Spatial), float64(maxSeenLayer.Spatial))) + if requestLayerSpatial > maxLayerSpatialLimit { + requestLayerSpatial = maxLayerSpatialLimit + } + if currentLayer.IsValid() && ((requestLayerSpatial == requestSpatial && currentLayer.Spatial == requestSpatial) || requestLayerSpatial == buffer.InvalidLayerSpatial) { + // 1. current is locked to desired, stay there + // OR + // 2. feed may be dry, let it continue at current layer if valid. + // covers the cases of // 1. mis-detection of layer stop - can continue streaming // 2. current layer resuming - can latch on when it starts - alloc.TargetLayer = currentLayer + alloc.TargetLayer = buffer.VideoLayer{ + Spatial: currentLayer.Spatial, + Temporal: getMaxTemporal(), + } alloc.RequestLayerSpatial = alloc.TargetLayer.Spatial } else { // opportunistically latch on to anything opportunisticAlloc() - alloc.RequestLayerSpatial = int32(math.Min(float64(maxLayer.Spatial), float64(maxSeenLayer.Spatial))) - } - - default: - isCurrentLayerAvailable := false - if currentLayer.IsValid() { - for _, l := range availableLayers { - if l == currentLayer.Spatial { - isCurrentLayerAvailable = true - break - } - } - } - - if !isCurrentLayerAvailable && currentLayer.IsValid() { - // current layer maybe stopped, move to highest available - for _, l := range availableLayers { - if l > alloc.TargetLayer.Spatial { - alloc.TargetLayer.Spatial = l - } - } - alloc.TargetLayer.Temporal = getMaxTemporal() - - alloc.RequestLayerSpatial = alloc.TargetLayer.Spatial - } else { - requestLayerSpatial := int32(math.Min(float64(maxLayer.Spatial), float64(maxSeenLayer.Spatial))) - if currentLayer.IsValid() && requestLayerSpatial == requestSpatial && currentLayer.Spatial == requestSpatial { - // current is locked to desired, stay there - alloc.TargetLayer = buffer.VideoLayer{ - Spatial: requestSpatial, - Temporal: getMaxTemporal(), - } - alloc.RequestLayerSpatial = requestSpatial + if requestLayerSpatial == buffer.InvalidLayerSpatial { + alloc.RequestLayerSpatial = maxLayerSpatialLimit } else { - // opportunistically latch on to anything - opportunisticAlloc() alloc.RequestLayerSpatial = requestLayerSpatial } } diff --git a/pkg/sfu/forwarder_test.go b/pkg/sfu/forwarder_test.go index 46a86edf2..86657e270 100644 --- a/pkg/sfu/forwarder_test.go +++ b/pkg/sfu/forwarder_test.go @@ -293,7 +293,7 @@ func TestForwarderAllocateOptimal(t *testing.T) { BandwidthNeeded: bitrates[2][1], Bitrates: bitrates, TargetLayer: buffer.DefaultMaxLayer, - RequestLayerSpatial: 2, + RequestLayerSpatial: 1, MaxLayer: buffer.DefaultMaxLayer, DistanceToDesired: -0.5, } @@ -310,7 +310,7 @@ func TestForwarderAllocateOptimal(t *testing.T) { BandwidthDelta: 0 - bitrates[2][1], Bitrates: emptyBitrates, TargetLayer: buffer.DefaultMaxLayer, - RequestLayerSpatial: 2, + RequestLayerSpatial: 1, MaxLayer: buffer.DefaultMaxLayer, DistanceToDesired: -1.0, } @@ -330,7 +330,7 @@ func TestForwarderAllocateOptimal(t *testing.T) { BandwidthNeeded: bitrates[2][1], Bitrates: bitrates, TargetLayer: expectedTargetLayer, - RequestLayerSpatial: 2, + RequestLayerSpatial: 1, MaxLayer: buffer.DefaultMaxLayer, DistanceToDesired: -0.5, } @@ -338,10 +338,10 @@ func TestForwarderAllocateOptimal(t *testing.T) { require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - // switches to highest available if feed is not dry and current is valid and current is not available + // switches request layer to highest available if feed is not dry and current is valid and current is not available f.vls.SetCurrent(buffer.VideoLayer{Spatial: 0, Temporal: 1}) expectedTargetLayer = buffer.VideoLayer{ - Spatial: 1, + Spatial: 2, Temporal: buffer.DefaultMaxLayerTemporal, } expectedResult = VideoAllocation{ @@ -353,7 +353,7 @@ func TestForwarderAllocateOptimal(t *testing.T) { TargetLayer: expectedTargetLayer, RequestLayerSpatial: 1, MaxLayer: buffer.DefaultMaxLayer, - DistanceToDesired: 0.5, + DistanceToDesired: -0.5, } result = f.AllocateOptimal([]int32{1}, bitrates, true) require.Equal(t, expectedResult, result) @@ -377,7 +377,7 @@ func TestForwarderAllocateOptimal(t *testing.T) { MaxLayer: f.vls.GetMax(), DistanceToDesired: 0.0, } - result = f.AllocateOptimal([]int32{0, 1}, emptyBitrates, true) + result = f.AllocateOptimal([]int32{0}, emptyBitrates, true) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) @@ -395,7 +395,7 @@ func TestForwarderAllocateOptimal(t *testing.T) { BandwidthDelta: 0, Bitrates: emptyBitrates, TargetLayer: expectedTargetLayer, - RequestLayerSpatial: 2, + RequestLayerSpatial: 1, MaxLayer: f.vls.GetMax(), DistanceToDesired: -1, } From e75b73af52c9c89d7be98a5fc4a2dda35ae36534 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sat, 15 Apr 2023 13:07:47 +0530 Subject: [PATCH 085/324] Use last received if lower. (#1616) When detecting congestion based on loss, it is possible that the loss based signal triggers earlier and the estimate based signal is lagging. In those cases, check against last received estimate and if that is lower than loss based throttling, use that. Without this, it was possible that the current usage high. Loss based throttling may not dial things back far enough to pause the stream. Ideally, congestion should hit again and it should be dialled down further and eventually pause, but there are situations it never dials back far enough to pause. --- pkg/sfu/streamallocator/channelobserver.go | 27 ++++++++-------------- pkg/sfu/streamallocator/streamallocator.go | 8 +++++++ 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/pkg/sfu/streamallocator/channelobserver.go b/pkg/sfu/streamallocator/channelobserver.go index f19bbb692..dcef36f3e 100644 --- a/pkg/sfu/streamallocator/channelobserver.go +++ b/pkg/sfu/streamallocator/channelobserver.go @@ -148,29 +148,17 @@ func (c *ChannelObserver) GetNackRatio() (uint32, uint32, float64) { func (c *ChannelObserver) GetTrend() (ChannelTrend, ChannelCongestionReason) { estimateDirection := c.estimateTrend.GetDirection() - packets, repeatedNacks, nackRatio := c.GetNackRatio() + _, _, nackRatio := c.GetNackRatio() switch { case estimateDirection == TrendDirectionDownward: - c.logger.Debugw( - "stream allocator: channel observer: estimate is trending downward", - "name", c.params.Name, - "estimate", c.estimateTrend.ToString(), - "packets", packets, - "repeatedNacks", repeatedNacks, - "ratio", nackRatio, - ) + c.logger.Debugw( "stream allocator: channel observer: estimate is trending downward", "channel", c.ToString()) return ChannelTrendCongesting, ChannelCongestionReasonEstimate + case c.params.NackWindowMinDuration != 0 && !c.nackWindowStartTime.IsZero() && time.Since(c.nackWindowStartTime) > c.params.NackWindowMinDuration && nackRatio > c.params.NackRatioThreshold: - c.logger.Debugw( - "stream allocator: channel observer: high rate of repeated NACKs", - "name", c.params.Name, - "estimate", c.estimateTrend.ToString(), - "packets", packets, - "repeatedNacks", repeatedNacks, - "ratio", nackRatio, - ) + c.logger.Debugw( "stream allocator: channel observer: high rate of repeated NACKs", "channel", c.ToString()) return ChannelTrendCongesting, ChannelCongestionReasonLoss + case estimateDirection == TrendDirectionUpward: return ChannelTrendClearing, ChannelCongestionReasonNone } @@ -178,4 +166,9 @@ func (c *ChannelObserver) GetTrend() (ChannelTrend, ChannelCongestionReason) { return ChannelTrendNeutral, ChannelCongestionReasonNone } +func (c *ChannelObserver) ToString() string { + packets, repeatedNacks, nackRatio := c.GetNackRatio() + return fmt.Sprintf("name: %s, estimate: %s, packets: %d, repeatedNacks: %d, nackRatio: %f", c.params.Name, c.estimateTrend.ToString(), packets, repeatedNacks, nackRatio) +} + // ------------------------------------------------ diff --git a/pkg/sfu/streamallocator/streamallocator.go b/pkg/sfu/streamallocator/streamallocator.go index a65834ed7..fdf6c3353 100644 --- a/pkg/sfu/streamallocator/streamallocator.go +++ b/pkg/sfu/streamallocator/streamallocator.go @@ -785,6 +785,9 @@ func (s *StreamAllocator) handleNewEstimateInNonProbe() { switch reason { case ChannelCongestionReasonLoss: estimateToCommit = int64(float64(expectedBandwidthUsage) * (1.0 - NackRatioAttenuator*nackRatio)) + if estimateToCommit > s.lastReceivedEstimate { + estimateToCommit = s.lastReceivedEstimate + } default: estimateToCommit = s.lastReceivedEstimate } @@ -1156,6 +1159,10 @@ func (s *StreamAllocator) initProbe(probeRateBps int64) { s.probeEndTime = time.Time{} + channelState := "" + if s.channelObserver != nil { + channelState = s.channelObserver.ToString() + } s.channelObserver = s.newChannelObserverProbe() s.channelObserver.SeedEstimate(s.lastReceivedEstimate) @@ -1172,6 +1179,7 @@ func (s *StreamAllocator) initProbe(probeRateBps int64) { "current usage", expectedBandwidthUsage, "committed", s.committedChannelCapacity, "lastReceived", s.lastReceivedEstimate, + "channel", channelState, "probeRateBps", probeRateBps, "goalBps", s.probeGoalBps, ) From 40ceddd18b54c35ea1466271ed30f3383adeee79 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Sat, 15 Apr 2023 01:20:23 -0700 Subject: [PATCH 086/324] Integrate QueuedNotifier, fixes out-of-order delivery (#1615) --- go.mod | 3 +-- go.sum | 7 +++---- pkg/rtc/room_test.go | 2 +- pkg/rtc/subscriptionmanager_test.go | 5 ++++- pkg/service/wire.go | 4 ++-- pkg/service/wire_gen.go | 8 ++++---- pkg/telemetry/events.go | 8 +++----- pkg/telemetry/telemetryservice.go | 16 ++++++---------- 8 files changed, 24 insertions(+), 29 deletions(-) diff --git a/go.mod b/go.mod index e6549ab34..4717dd5a7 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 - github.com/livekit/protocol v1.5.4-0.20230413111958-5fea69067bbc + github.com/livekit/protocol v1.5.4 github.com/livekit/psrpc v0.2.11-0.20230405191830-d76f71512630 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.14.0 @@ -98,7 +98,6 @@ require ( golang.org/x/net v0.9.0 // indirect golang.org/x/sys v0.7.0 // indirect golang.org/x/text v0.9.0 // indirect - golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.6.0 // indirect google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd // indirect google.golang.org/grpc v1.54.0 // indirect diff --git a/go.sum b/go.sum index f6e8e47c1..3b56e50e4 100644 --- a/go.sum +++ b/go.sum @@ -124,8 +124,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 h1:QlQFyMwCDgjyySsrgmrMcVbEBA6KZcyTzvK+z346tUA= github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26/go.mod h1:eDA41kiySZoG+wy4Etsjb3w0jjLx69i/vAmSjG4bteA= -github.com/livekit/protocol v1.5.4-0.20230413111958-5fea69067bbc h1:15IrYsN4PRgrH2MldkYgnTqqNxgRgjVGLjEtwurphCQ= -github.com/livekit/protocol v1.5.4-0.20230413111958-5fea69067bbc/go.mod h1:YPmFvsD0cr7KlC7wsoLTLwCAAJun/ovCDBCvUnWvdwo= +github.com/livekit/protocol v1.5.4 h1:lfEUqsE9AV1ZI/w8oZUKSAoi708V8RYwraOjeY83KVo= +github.com/livekit/protocol v1.5.4/go.mod h1:KJJVGHiNR6abdJIpoxB1kqQH2s902wM3cMt+P4p6jao= github.com/livekit/psrpc v0.2.11-0.20230405191830-d76f71512630 h1:Rm5KLZgQxWnTidY+H8MsAV6sk1iiFxeXqPFgSLkMing= github.com/livekit/psrpc v0.2.11-0.20230405191830-d76f71512630/go.mod h1:K0j8f1PgLShR7Lx80KbmwFkDH2BvOnycXGV0OSRURKc= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= @@ -233,8 +233,8 @@ github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJf github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/redis/go-redis/v9 v9.0.3 h1:+7mmR26M0IvyLxGZUHxu4GiBkJkVDid0Un+j4ScYu4k= github.com/redis/go-redis/v9 v9.0.3/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= -github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rs/cors v1.9.0 h1:l9HGsTsHJcvW14Nk7J9KFz8bzeAWXn3CG6bgt7LsrAE= github.com/rs/cors v1.9.0/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= @@ -382,7 +382,6 @@ golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190422233926-fe54fb35175b/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= diff --git a/pkg/rtc/room_test.go b/pkg/rtc/room_test.go index 9f4a4fd10..8d93367bf 100644 --- a/pkg/rtc/room_test.go +++ b/pkg/rtc/room_test.go @@ -701,7 +701,7 @@ func newRoomWithParticipants(t *testing.T, opts testRoomOpts) *Room { NodeId: "testnode", Region: "testregion", }, - telemetry.NewTelemetryService(webhook.NewNotifier("", "", nil), &telemetryfakes.FakeAnalyticsService{}), + telemetry.NewTelemetryService(webhook.NewDefaultNotifier("", "", nil), &telemetryfakes.FakeAnalyticsService{}), nil, ) for i := 0; i < opts.num+opts.numHidden; i++ { diff --git a/pkg/rtc/subscriptionmanager_test.go b/pkg/rtc/subscriptionmanager_test.go index 6a0edfdb5..05fea3d8c 100644 --- a/pkg/rtc/subscriptionmanager_test.go +++ b/pkg/rtc/subscriptionmanager_test.go @@ -76,7 +76,10 @@ func TestSubscribe(t *testing.T) { require.NotNil(t, s.getSubscribedTrack()) require.Len(t, sm.GetSubscribedTracks(), 1) - require.Len(t, sm.GetSubscribedParticipants(), 1) + + require.Eventually(t, func() bool { + return len(sm.GetSubscribedParticipants()) == 1 + }, subSettleTimeout, subCheckInterval, "GetSubscribedParticipants should have returned one item") require.Equal(t, "pubID", string(sm.GetSubscribedParticipants()[0])) // ensure telemetry events are sent diff --git a/pkg/service/wire.go b/pkg/service/wire.go index 67310da29..5da0e2c79 100644 --- a/pkg/service/wire.go +++ b/pkg/service/wire.go @@ -114,7 +114,7 @@ func createKeyProvider(conf *config.Config) (auth.KeyProvider, error) { return auth.NewFileBasedKeyProviderFromMap(conf.Keys), nil } -func createWebhookNotifier(conf *config.Config, provider auth.KeyProvider) (webhook.Notifier, error) { +func createWebhookNotifier(conf *config.Config, provider auth.KeyProvider) (webhook.QueuedNotifier, error) { wc := conf.WebHook if len(wc.URLs) == 0 { return nil, nil @@ -124,7 +124,7 @@ func createWebhookNotifier(conf *config.Config, provider auth.KeyProvider) (webh return nil, ErrWebHookMissingAPIKey } - return webhook.NewNotifier(wc.APIKey, secret, wc.URLs), nil + return webhook.NewDefaultNotifier(wc.APIKey, secret, wc.URLs), nil } func createRedisClient(conf *config.Config) (redis.UniversalClient, error) { diff --git a/pkg/service/wire_gen.go b/pkg/service/wire_gen.go index 080858485..bce220b63 100644 --- a/pkg/service/wire_gen.go +++ b/pkg/service/wire_gen.go @@ -63,12 +63,12 @@ func InitializeServer(conf *config.Config, currentNode routing.LocalNode) (*Live if err != nil { return nil, err } - notifier, err := createWebhookNotifier(conf, keyProvider) + queuedNotifier, err := createWebhookNotifier(conf, keyProvider) if err != nil { return nil, err } analyticsService := telemetry.NewAnalyticsService(conf, currentNode) - telemetryService := telemetry.NewTelemetryService(notifier, analyticsService) + telemetryService := telemetry.NewTelemetryService(queuedNotifier, analyticsService) rtcEgressLauncher := NewEgressLauncher(egressClient, rpcClient, egressStore, telemetryService) roomService, err := NewRoomService(roomConfig, apiConfig, router, roomAllocator, objectStore, rtcEgressLauncher) if err != nil { @@ -159,7 +159,7 @@ func createKeyProvider(conf *config.Config) (auth.KeyProvider, error) { return auth.NewFileBasedKeyProviderFromMap(conf.Keys), nil } -func createWebhookNotifier(conf *config.Config, provider auth.KeyProvider) (webhook.Notifier, error) { +func createWebhookNotifier(conf *config.Config, provider auth.KeyProvider) (webhook.QueuedNotifier, error) { wc := conf.WebHook if len(wc.URLs) == 0 { return nil, nil @@ -169,7 +169,7 @@ func createWebhookNotifier(conf *config.Config, provider auth.KeyProvider) (webh return nil, ErrWebHookMissingAPIKey } - return webhook.NewNotifier(wc.APIKey, secret, wc.URLs), nil + return webhook.NewDefaultNotifier(wc.APIKey, secret, wc.URLs), nil } func createRedisClient(conf *config.Config) (redis.UniversalClient, error) { diff --git a/pkg/telemetry/events.go b/pkg/telemetry/events.go index 75f7c8ce2..c75e4d828 100644 --- a/pkg/telemetry/events.go +++ b/pkg/telemetry/events.go @@ -21,11 +21,9 @@ func (t *telemetryService) NotifyEvent(ctx context.Context, event *livekit.Webho event.CreatedAt = time.Now().Unix() event.Id = utils.NewGuid("EV_") - t.webhookPool.Submit(func() { - if err := t.notifier.Notify(ctx, event); err != nil { - logger.Warnw("failed to notify webhook", err, "event", event.Event) - } - }) + if err := t.notifier.QueueNotify(ctx, event); err != nil { + logger.Warnw("failed to notify webhook", err, "event", event.Event) + } } func (t *telemetryService) RoomStarted(ctx context.Context, room *livekit.Room) { diff --git a/pkg/telemetry/telemetryservice.go b/pkg/telemetry/telemetryservice.go index d78664bf6..9e6ef65ae 100644 --- a/pkg/telemetry/telemetryservice.go +++ b/pkg/telemetry/telemetryservice.go @@ -5,8 +5,6 @@ import ( "sync" "time" - "github.com/gammazero/workerpool" - "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" @@ -72,22 +70,20 @@ const ( type telemetryService struct { AnalyticsService - notifier webhook.Notifier - webhookPool *workerpool.WorkerPool - jobsChan chan func() + notifier webhook.QueuedNotifier + jobsChan chan func() lock sync.RWMutex workers map[livekit.ParticipantID]*StatsWorker } -func NewTelemetryService(notifier webhook.Notifier, analytics AnalyticsService) TelemetryService { +func NewTelemetryService(notifier webhook.QueuedNotifier, analytics AnalyticsService) TelemetryService { t := &telemetryService{ AnalyticsService: analytics, - notifier: notifier, - webhookPool: workerpool.New(maxWebhookWorkers), - jobsChan: make(chan func(), jobQueueBufferSize), - workers: make(map[livekit.ParticipantID]*StatsWorker), + notifier: notifier, + jobsChan: make(chan func(), jobQueueBufferSize), + workers: make(map[livekit.ParticipantID]*StatsWorker), } go t.run() From ce33d38b4f1e722253e5019e7b474416b1bddcf7 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sat, 15 Apr 2023 18:50:18 +0530 Subject: [PATCH 087/324] Use current bit rate when calculating bandwidth needed. (#1617) --- pkg/sfu/downtrack.go | 3 +- pkg/sfu/forwarder.go | 77 +++++++++++++++++++++++---------------- pkg/sfu/forwarder_test.go | 2 +- 3 files changed, 49 insertions(+), 33 deletions(-) diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 430bb882f..8eb026205 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -988,7 +988,8 @@ func (d *DownTrack) IsDeficient() bool { } func (d *DownTrack) BandwidthRequested() int64 { - return d.forwarder.BandwidthRequested() + _, brs := d.receiver.GetLayeredBitrate() + return d.forwarder.BandwidthRequested(brs) } func (d *DownTrack) DistanceToDesired() float64 { diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index b203c27da..94d764bdd 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -484,11 +484,11 @@ func (f *Forwarder) IsDeficient() bool { return f.isDeficientLocked() } -func (f *Forwarder) BandwidthRequested() int64 { +func (f *Forwarder) BandwidthRequested(brs Bitrates) int64 { f.lock.RLock() defer f.lock.RUnlock() - return f.lastAllocation.BandwidthRequested + return getBandwidthNeeded(brs, f.vls.GetTarget(), f.lastAllocation.BandwidthRequested) } func (f *Forwarder) DistanceToDesired(availableLayers []int32, brs Bitrates) float64 { @@ -619,7 +619,7 @@ func (f *Forwarder) AllocateOptimal(availableLayers []int32, brs Bitrates, allow if alloc.TargetLayer.IsValid() { alloc.BandwidthRequested = optimalBandwidthNeeded } - alloc.BandwidthDelta = alloc.BandwidthRequested - f.lastAllocation.BandwidthRequested + alloc.BandwidthDelta = alloc.BandwidthRequested - getBandwidthNeeded(brs, f.vls.GetTarget(), f.lastAllocation.BandwidthRequested) alloc.DistanceToDesired = getDistanceToDesired( f.muted, f.pubMuted, @@ -719,22 +719,26 @@ func (f *Forwarder) ProvisionalAllocateGetCooperativeTransition(allowOvershoot b f.lock.Lock() defer f.lock.Unlock() + existingTargetLayer := f.vls.GetTarget() if f.provisional.muted || f.provisional.pubMuted { + bandwidthRequired := int64(0) f.provisional.allocatedLayer = buffer.InvalidLayer if f.provisional.pubMuted { // leave it at current for opportunistic forwarding, there is still bandwidth saving with publisher mute f.provisional.allocatedLayer = f.provisional.currentLayer + if f.provisional.allocatedLayer.IsValid() { + bandwidthRequired = f.provisional.Bitrates[f.provisional.allocatedLayer.Spatial][f.provisional.allocatedLayer.Temporal] + } } return VideoTransition{ From: f.vls.GetTarget(), To: f.provisional.allocatedLayer, - BandwidthDelta: 0 - f.lastAllocation.BandwidthRequested, + BandwidthDelta: bandwidthRequired - getBandwidthNeeded(f.provisional.Bitrates, existingTargetLayer, f.lastAllocation.BandwidthRequested), } } // check if we should preserve current target - targetLayer := f.vls.GetTarget() - if targetLayer.IsValid() { + if existingTargetLayer.IsValid() { // what is the highest that is available maximalLayer := buffer.InvalidLayer maximalBandwidthRequired := int64(0) @@ -753,24 +757,24 @@ func (f *Forwarder) ProvisionalAllocateGetCooperativeTransition(allowOvershoot b } if maximalLayer.IsValid() { - if !targetLayer.GreaterThan(maximalLayer) && f.provisional.Bitrates[targetLayer.Spatial][targetLayer.Temporal] != 0 { - // currently streaming and maybe wanting an upgrade (targetLayer <= maximalLayer), + if !existingTargetLayer.GreaterThan(maximalLayer) && f.provisional.Bitrates[existingTargetLayer.Spatial][existingTargetLayer.Temporal] != 0 { + // currently streaming and maybe wanting an upgrade (existingTargetLayer <= maximalLayer), // just preserve current target in the cooperative scheme of things - f.provisional.allocatedLayer = targetLayer + f.provisional.allocatedLayer = existingTargetLayer return VideoTransition{ - From: targetLayer, - To: targetLayer, + From: existingTargetLayer, + To: existingTargetLayer, BandwidthDelta: 0, } } - if targetLayer.GreaterThan(maximalLayer) { - // maximalLayer < targetLayer, make the down move + if existingTargetLayer.GreaterThan(maximalLayer) { + // maximalLayer < existingTargetLayer, make the down move f.provisional.allocatedLayer = maximalLayer return VideoTransition{ - From: targetLayer, + From: existingTargetLayer, To: maximalLayer, - BandwidthDelta: maximalBandwidthRequired - f.lastAllocation.BandwidthRequested, + BandwidthDelta: maximalBandwidthRequired - getBandwidthNeeded(f.provisional.Bitrates, existingTargetLayer, f.lastAllocation.BandwidthRequested), } } } @@ -799,8 +803,9 @@ func (f *Forwarder) ProvisionalAllocateGetCooperativeTransition(allowOvershoot b return layers, bw } + targetLayer := buffer.InvalidLayer bandwidthRequired := int64(0) - if !targetLayer.IsValid() { + if !existingTargetLayer.IsValid() { // currently not streaming, find minimal // NOTE: a layer in feed could have paused and there could be other options than going back to minimal, // but the cooperative scheme knocks things back to minimal @@ -825,13 +830,17 @@ func (f *Forwarder) ProvisionalAllocateGetCooperativeTransition(allowOvershoot b } else { targetLayer = f.provisional.currentLayer } + + if targetLayer.IsValid() { + bandwidthRequired = f.provisional.Bitrates[targetLayer.Spatial][targetLayer.Temporal] + } } f.provisional.allocatedLayer = targetLayer return VideoTransition{ From: f.vls.GetTarget(), To: targetLayer, - BandwidthDelta: bandwidthRequired - f.lastAllocation.BandwidthRequested, + BandwidthDelta: bandwidthRequired - getBandwidthNeeded(f.provisional.Bitrates, existingTargetLayer, f.lastAllocation.BandwidthRequested), } } @@ -856,15 +865,12 @@ func (f *Forwarder) ProvisionalAllocateGetBestWeightedTransition() VideoTransiti targetLayer := f.vls.GetTarget() if f.provisional.muted || f.provisional.pubMuted { + // if publisher muted, give up opportunistic resume and give back the bandwidth f.provisional.allocatedLayer = buffer.InvalidLayer - if f.provisional.pubMuted { - // leave it at current for opportunistic forwarding, there is still bandwidth saving with publisher mute - f.provisional.allocatedLayer = f.provisional.currentLayer - } return VideoTransition{ From: targetLayer, To: f.provisional.allocatedLayer, - BandwidthDelta: 0 - f.lastAllocation.BandwidthRequested, + BandwidthDelta: 0 - getBandwidthNeeded(f.provisional.Bitrates, targetLayer, f.lastAllocation.BandwidthRequested), } } @@ -893,12 +899,13 @@ func (f *Forwarder) ProvisionalAllocateGetBestWeightedTransition() VideoTransiti return VideoTransition{ From: targetLayer, To: f.provisional.allocatedLayer, - BandwidthDelta: 0 - f.lastAllocation.BandwidthRequested, + BandwidthDelta: 0 - getBandwidthNeeded(f.provisional.Bitrates, targetLayer, f.lastAllocation.BandwidthRequested), } } // starting from minimum to target, find transition which gives the best // transition taking into account bits saved vs cost of such a transition + existingBandwidthNeeded := getBandwidthNeeded(f.provisional.Bitrates, targetLayer, f.lastAllocation.BandwidthRequested) bestLayer := buffer.InvalidLayer bestBandwidthDelta := int64(0) bestValue := float32(0) @@ -908,7 +915,7 @@ func (f *Forwarder) ProvisionalAllocateGetBestWeightedTransition() VideoTransiti break } - BandwidthDelta := int64(math.Max(float64(0), float64(f.lastAllocation.BandwidthRequested-f.provisional.Bitrates[s][t]))) + bandwidthDelta := int64(math.Max(float64(0), float64(existingBandwidthNeeded-f.provisional.Bitrates[s][t]))) transitionCost := int32(0) // LK-TODO: SVC will need a different cost transition @@ -920,11 +927,11 @@ func (f *Forwarder) ProvisionalAllocateGetBestWeightedTransition() VideoTransiti value := float32(0) if (transitionCost + qualityCost) != 0 { - value = float32(BandwidthDelta) / float32(transitionCost+qualityCost) + value = float32(bandwidthDelta) / float32(transitionCost+qualityCost) } - if value > bestValue || (value == bestValue && BandwidthDelta > bestBandwidthDelta) { + if value > bestValue || (value == bestValue && bandwidthDelta > bestBandwidthDelta) { bestValue = value - bestBandwidthDelta = BandwidthDelta + bestBandwidthDelta = bandwidthDelta bestLayer = buffer.VideoLayer{Spatial: s, Temporal: t} } } @@ -951,7 +958,7 @@ func (f *Forwarder) ProvisionalAllocateCommit() VideoAllocation { ) alloc := VideoAllocation{ BandwidthRequested: 0, - BandwidthDelta: -f.lastAllocation.BandwidthRequested, + BandwidthDelta: 0 - getBandwidthNeeded(f.provisional.Bitrates, f.vls.GetTarget(), f.lastAllocation.BandwidthRequested), Bitrates: f.provisional.Bitrates, BandwidthNeeded: optimalBandwidthNeeded, TargetLayer: f.provisional.allocatedLayer, @@ -979,7 +986,7 @@ func (f *Forwarder) ProvisionalAllocateCommit() VideoAllocation { if f.provisional.allocatedLayer.IsValid() { // overshoot alloc.BandwidthRequested = f.provisional.Bitrates[f.provisional.allocatedLayer.Spatial][f.provisional.allocatedLayer.Temporal] - alloc.BandwidthDelta = alloc.BandwidthRequested - f.lastAllocation.BandwidthRequested + alloc.BandwidthDelta = alloc.BandwidthRequested - getBandwidthNeeded(f.provisional.Bitrates, f.vls.GetTarget(), f.lastAllocation.BandwidthRequested) } else { alloc.PauseReason = VideoPauseReasonFeedDry @@ -995,7 +1002,7 @@ func (f *Forwarder) ProvisionalAllocateCommit() VideoAllocation { if f.provisional.allocatedLayer.IsValid() { alloc.BandwidthRequested = f.provisional.Bitrates[f.provisional.allocatedLayer.Spatial][f.provisional.allocatedLayer.Temporal] } - alloc.BandwidthDelta = alloc.BandwidthRequested - f.lastAllocation.BandwidthRequested + alloc.BandwidthDelta = alloc.BandwidthRequested - getBandwidthNeeded(f.provisional.Bitrates, f.vls.GetTarget(), f.lastAllocation.BandwidthRequested) if f.provisional.allocatedLayer.GreaterThan(f.provisional.maxLayer) || alloc.BandwidthRequested >= getOptimalBandwidthNeeded( @@ -1226,7 +1233,7 @@ func (f *Forwarder) Pause(availableLayers []int32, brs Bitrates) VideoAllocation optimalBandwidthNeeded := getOptimalBandwidthNeeded(f.muted, f.pubMuted, maxSeenLayer.Spatial, brs, maxLayer) alloc := VideoAllocation{ BandwidthRequested: 0, - BandwidthDelta: 0 - f.lastAllocation.BandwidthRequested, + BandwidthDelta: 0 - getBandwidthNeeded(brs, f.vls.GetTarget(), f.lastAllocation.BandwidthRequested), Bitrates: brs, BandwidthNeeded: optimalBandwidthNeeded, TargetLayer: buffer.InvalidLayer, @@ -1606,6 +1613,14 @@ func getOptimalBandwidthNeeded(muted bool, pubMuted bool, maxPublishedLayer int3 return 0 } +func getBandwidthNeeded(brs Bitrates, layer buffer.VideoLayer, fallback int64) int64 { + if layer.IsValid() && brs[layer.Spatial][layer.Temporal] > 0 { + return brs[layer.Spatial][layer.Temporal] + } + + return fallback +} + func getDistanceToDesired( muted bool, pubMuted bool, diff --git a/pkg/sfu/forwarder_test.go b/pkg/sfu/forwarder_test.go index 86657e270..bbb079258 100644 --- a/pkg/sfu/forwarder_test.go +++ b/pkg/sfu/forwarder_test.go @@ -221,7 +221,7 @@ func TestForwarderAllocateOptimal(t *testing.T) { expectedResult = VideoAllocation{ PauseReason: VideoPauseReasonNone, BandwidthRequested: bitrates[1][3], - BandwidthDelta: bitrates[1][3], + BandwidthDelta: bitrates[1][3] - bitrates[0][1], BandwidthNeeded: bitrates[1][3], Bitrates: bitrates, TargetLayer: buffer.DefaultMaxLayer, From 9c64d71e912369081315a8f05f27df849e60e6d9 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Sun, 16 Apr 2023 08:40:23 -0700 Subject: [PATCH 088/324] update message counter in signal relay (#1620) --- pkg/routing/signal.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/routing/signal.go b/pkg/routing/signal.go index 5be76e70d..d1a759199 100644 --- a/pkg/routing/signal.go +++ b/pkg/routing/signal.go @@ -84,12 +84,14 @@ func (r *signalClient) StartParticipantSignal( stream, err := r.client.RelaySignal(ctx, nodeID) if err != nil { + prometheus.MessageCounter.WithLabelValues("signal", "failure").Add(1) return } err = stream.Send(&rpc.RelaySignalRequest{StartSession: ss}) if err != nil { stream.Close(err) + prometheus.MessageCounter.WithLabelValues("signal", "failure").Add(1) return } @@ -183,8 +185,12 @@ func CopySignalStreamToMessageChannel[SendType, RecvType RelaySignalMessage]( if err != nil { return err } + + prometheus.MessageCounter.WithLabelValues("signal", "success").Add(float64(len(res))) + for _, r := range res { if err = ch.WriteMessage(r); err != nil { + prometheus.MessageCounter.WithLabelValues("signal", "failure").Add(1) return err } } From 96f3aaa587170daebfaa2690bba94f0d1f25e4c1 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Sun, 16 Apr 2023 17:38:09 -0700 Subject: [PATCH 089/324] free signal join response to gc after forwarding (#1619) --- pkg/routing/signal.go | 23 +++++++++++------------ pkg/service/rtcservice.go | 38 +++++++++++++++----------------------- 2 files changed, 26 insertions(+), 35 deletions(-) diff --git a/pkg/routing/signal.go b/pkg/routing/signal.go index d1a759199..20077b233 100644 --- a/pkg/routing/signal.go +++ b/pkg/routing/signal.go @@ -180,7 +180,6 @@ func CopySignalStreamToMessageChannel[SendType, RecvType RelaySignalMessage]( config: config, } for msg := range stream.Channel() { - var res []proto.Message res, err := r.Read(msg) if err != nil { return err @@ -280,20 +279,17 @@ func (s *signalMessageSink[SendType, RecvType]) nextMessage() (msg SendType, n i func (s *signalMessageSink[SendType, RecvType]) write() { interval := s.Config.MinRetryInterval deadline := time.Now().Add(s.Config.RetryTimeout) + var err error s.mu.Lock() for { msg, n := s.nextMessage() if n == 0 || s.IsClosed() { - if s.draining { - s.Stream.Close(nil) - } - s.writing = false break } s.mu.Unlock() - err := s.Stream.Send(msg, psrpc.WithTimeout(interval)) + err = s.Stream.Send(msg, psrpc.WithTimeout(interval)) if err != nil { if time.Now().After(deadline) { s.Logger.Warnw("could not send signal message", err) @@ -301,12 +297,7 @@ func (s *signalMessageSink[SendType, RecvType]) write() { s.mu.Lock() s.seq += uint64(len(s.queue)) s.queue = nil - - if s.CloseOnFailure { - s.Stream.Close(ErrSignalFailed) - } - s.mu.Unlock() - return + break } interval *= 2 @@ -324,6 +315,14 @@ func (s *signalMessageSink[SendType, RecvType]) write() { s.queue = s.queue[n:] } } + + s.writing = false + if s.draining { + s.Stream.Close(nil) + } + if err != nil && s.CloseOnFailure { + s.Stream.Close(ErrSignalFailed) + } s.mu.Unlock() } diff --git a/pkg/service/rtcservice.go b/pkg/service/rtcservice.go index d6f622239..16639c9d5 100644 --- a/pkg/service/rtcservice.go +++ b/pkg/service/rtcservice.go @@ -190,10 +190,11 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { // give it a few attempts to start session var cr connectionResult + var initialResponse *livekit.SignalResponse for i := 0; i < 3; i++ { connectionTimeout := 3 * time.Second * time.Duration(i+1) ctx := utils.ContextWithAttempt(r.Context(), i) - cr, err = s.startConnection(ctx, roomName, pi, connectionTimeout) + cr, initialResponse, err = s.startConnection(ctx, roomName, pi, connectionTimeout) if err == nil { break } @@ -210,8 +211,8 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { prometheus.IncrementParticipantJoin(1) - if !pi.Reconnect && cr.InitialResponse.GetJoin() != nil { - pi.ID = livekit.ParticipantID(cr.InitialResponse.GetJoin().GetParticipant().GetSid()) + if !pi.Reconnect && initialResponse.GetJoin() != nil { + pi.ID = livekit.ParticipantID(initialResponse.GetJoin().GetParticipant().GetSid()) } var signalStats *telemetry.BytesTrackStats @@ -251,7 +252,7 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { // websocket established sigConn := NewWSSignalConnection(conn) - if count, err := sigConn.WriteResponse(cr.InitialResponse); err != nil { + if count, err := sigConn.WriteResponse(initialResponse); err != nil { pLogger.Warnw("could not write initial response", err) return } else { @@ -301,14 +302,6 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { pLogger.Debugw("sending answer", "answer", m) } - if pi.ID == "" && cr.InitialResponse.GetJoin() != nil { - pi.ID = livekit.ParticipantID(cr.InitialResponse.GetJoin().GetParticipant().GetSid()) - signalStats = telemetry.NewBytesTrackStats( - telemetry.BytesTrackIDForParticipantID(telemetry.BytesTrackTypeSignal, pi.ID), - pi.ID, - s.telemetry) - } - if count, err := sigConn.WriteResponse(res); err != nil { pLogger.Warnw("error writing to websocket", err) return @@ -443,38 +436,37 @@ func (s *RTCService) ParseClientInfo(r *http.Request) *livekit.ClientInfo { } type connectionResult struct { - Room *livekit.Room - ConnectionID livekit.ConnectionID - RequestSink routing.MessageSink - ResponseSource routing.MessageSource - InitialResponse *livekit.SignalResponse + Room *livekit.Room + ConnectionID livekit.ConnectionID + RequestSink routing.MessageSink + ResponseSource routing.MessageSource } -func (s *RTCService) startConnection(ctx context.Context, roomName livekit.RoomName, pi routing.ParticipantInit, timeout time.Duration) (connectionResult, error) { +func (s *RTCService) startConnection(ctx context.Context, roomName livekit.RoomName, pi routing.ParticipantInit, timeout time.Duration) (connectionResult, *livekit.SignalResponse, error) { var cr connectionResult var err error cr.Room, err = s.roomAllocator.CreateRoom(ctx, &livekit.CreateRoomRequest{Name: string(roomName)}) if err != nil { - return cr, err + return cr, nil, err } // this needs to be started first *before* using router functions on this node cr.ConnectionID, cr.RequestSink, cr.ResponseSource, err = s.router.StartParticipantSignal(ctx, roomName, pi) if err != nil { - return cr, err + return cr, nil, err } // wait for the first message before upgrading to websocket. If no one is // responding to our connection attempt, we should terminate the connection // instead of waiting forever on the WebSocket - cr.InitialResponse, err = readInitialResponse(cr.ResponseSource, timeout) + initialResponse, err := readInitialResponse(cr.ResponseSource, timeout) if err != nil { // close the connection to avoid leaking cr.RequestSink.Close() cr.ResponseSource.Close() - return cr, err + return cr, nil, err } - return cr, nil + return cr, initialResponse, nil } func readInitialResponse(source routing.MessageSource, timeout time.Duration) (*livekit.SignalResponse, error) { From 0ce3ba418f824eb0d98c8e1e8fe1617996c6be96 Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Mon, 17 Apr 2023 10:41:07 +0800 Subject: [PATCH 090/324] Update pion to parse multiple simulcast sdp correctly (#1621) --- go.mod | 2 +- go.sum | 4 ++-- pkg/rtc/participant.go | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 4717dd5a7..3f93906d0 100644 --- a/go.mod +++ b/go.mod @@ -35,7 +35,7 @@ require ( github.com/pion/stun v0.4.0 github.com/pion/transport/v2 v2.1.0 github.com/pion/turn/v2 v2.1.0 - github.com/pion/webrtc/v3 v3.1.59 + github.com/pion/webrtc/v3 v3.1.60 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.15.0 github.com/redis/go-redis/v9 v9.0.3 diff --git a/go.sum b/go.sum index 3b56e50e4..bedd8cfa5 100644 --- a/go.sum +++ b/go.sum @@ -217,8 +217,8 @@ github.com/pion/turn/v2 v2.1.0 h1:5wGHSgGhJhP/RpabkUb/T9PdsAjkGLS6toYz5HNzoSI= github.com/pion/turn/v2 v2.1.0/go.mod h1:yrT5XbXSGX1VFSF31A3c1kCNB5bBZgk/uu5LET162qs= github.com/pion/udp/v2 v2.0.1 h1:xP0z6WNux1zWEjhC7onRA3EwwSliXqu1ElUZAQhUP54= github.com/pion/udp/v2 v2.0.1/go.mod h1:B7uvTMP00lzWdyMr/1PVZXtV3wpPIxBRd4Wl6AksXn8= -github.com/pion/webrtc/v3 v3.1.59 h1:B3YFo8q6dwBYKA2LUjWRChP59Qtt+xvv1Ul7UPDp6Zc= -github.com/pion/webrtc/v3 v3.1.59/go.mod h1:rJGgStRoFyFOWJULHLayaimsG+jIEoenhJ5MB5gIFqw= +github.com/pion/webrtc/v3 v3.1.60 h1:FLF6HT3x3CMHtPz5JbdAARfIUpMZu2YeOSzkVxaeF+k= +github.com/pion/webrtc/v3 v3.1.60/go.mod h1:65gfOgxrmszb6ec7kEiZp32QwnmDNIrJK8hgo/0niWY= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index a6147c781..30a8ad14c 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -1152,6 +1152,7 @@ func (p *ParticipantImpl) onMediaTrack(track *webrtc.TrackRemote, rtpReceiver *w p.params.Logger.Infow("mediaTrack published", "kind", track.Kind().String(), "trackID", publishedTrack.ID(), + "webrtcTrackID", track.ID(), "rid", track.RID(), "SSRC", track.SSRC(), "mime", track.Codec().MimeType, From 5d187523faa355077701bc26a9a7a931e80627a9 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 17 Apr 2023 10:16:12 +0530 Subject: [PATCH 091/324] Refactor NACK tracking in stream allocator (#1623) * log some NACKs * split out NACK tracker * remove debug --- pkg/sfu/streamallocator/channelobserver.go | 56 ++++---------- pkg/sfu/streamallocator/nacktracker.go | 85 ++++++++++++++++++++++ pkg/sfu/streamallocator/streamallocator.go | 7 +- 3 files changed, 102 insertions(+), 46 deletions(-) create mode 100644 pkg/sfu/streamallocator/nacktracker.go diff --git a/pkg/sfu/streamallocator/channelobserver.go b/pkg/sfu/streamallocator/channelobserver.go index dcef36f3e..20fa0edf7 100644 --- a/pkg/sfu/streamallocator/channelobserver.go +++ b/pkg/sfu/streamallocator/channelobserver.go @@ -70,6 +70,7 @@ type ChannelObserver struct { logger logger.Logger estimateTrend *TrendDetector + nackTracker *NackTracker nackWindowStartTime time.Time packets uint32 @@ -87,6 +88,13 @@ func NewChannelObserver(params ChannelObserverParams, logger logger.Logger) *Cha DownwardTrendThreshold: params.EstimateDownwardTrendThreshold, CollapseValues: params.EstimateCollapseValues, }), + nackTracker: NewNackTracker(NackTrackerParams{ + Name: params.Name + "-estimate", + Logger: logger, + WindowMinDuration: params.NackWindowMinDuration, + WindowMaxDuration: params.NackWindowMaxDuration, + RatioThreshold: params.NackRatioThreshold, + }), } } @@ -94,36 +102,12 @@ func (c *ChannelObserver) SeedEstimate(estimate int64) { c.estimateTrend.Seed(estimate) } -func (c *ChannelObserver) SeedNack(packets uint32, repeatedNacks uint32) { - c.packets = packets - c.repeatedNacks = repeatedNacks -} - func (c *ChannelObserver) AddEstimate(estimate int64) { c.estimateTrend.AddValue(estimate) } func (c *ChannelObserver) AddNack(packets uint32, repeatedNacks uint32) { - if c.params.NackWindowMaxDuration != 0 && !c.nackWindowStartTime.IsZero() && time.Since(c.nackWindowStartTime) > c.params.NackWindowMaxDuration { - c.nackWindowStartTime = time.Time{} - c.packets = 0 - c.repeatedNacks = 0 - } - - // - // Start NACK monitoring window only when a repeated NACK happens. - // This allows locking tightly to when NACKs start happening and - // check if the NACKs keep adding up (potentially a sign of congestion) - // or isolated losses - // - if c.repeatedNacks == 0 && repeatedNacks != 0 { - c.nackWindowStartTime = time.Now() - } - - if !c.nackWindowStartTime.IsZero() { - c.packets += packets - c.repeatedNacks += repeatedNacks - } + c.nackTracker.Add(packets, repeatedNacks) } func (c *ChannelObserver) GetLowestEstimate() int64 { @@ -134,29 +118,20 @@ func (c *ChannelObserver) GetHighestEstimate() int64 { return c.estimateTrend.GetHighest() } -func (c *ChannelObserver) GetNackRatio() (uint32, uint32, float64) { - ratio := 0.0 - if c.packets != 0 { - ratio = float64(c.repeatedNacks) / float64(c.packets) - if ratio > 1.0 { - ratio = 1.0 - } - } - - return c.packets, c.repeatedNacks, ratio +func (c *ChannelObserver) GetNackRatio() float64 { + return c.nackTracker.GetRatio() } func (c *ChannelObserver) GetTrend() (ChannelTrend, ChannelCongestionReason) { estimateDirection := c.estimateTrend.GetDirection() - _, _, nackRatio := c.GetNackRatio() switch { case estimateDirection == TrendDirectionDownward: - c.logger.Debugw( "stream allocator: channel observer: estimate is trending downward", "channel", c.ToString()) + c.logger.Debugw("stream allocator: channel observer: estimate is trending downward", "channel", c.ToString()) return ChannelTrendCongesting, ChannelCongestionReasonEstimate - case c.params.NackWindowMinDuration != 0 && !c.nackWindowStartTime.IsZero() && time.Since(c.nackWindowStartTime) > c.params.NackWindowMinDuration && nackRatio > c.params.NackRatioThreshold: - c.logger.Debugw( "stream allocator: channel observer: high rate of repeated NACKs", "channel", c.ToString()) + case c.nackTracker.IsTriggered(): + c.logger.Debugw("stream allocator: channel observer: high rate of repeated NACKs", "channel", c.ToString()) return ChannelTrendCongesting, ChannelCongestionReasonLoss case estimateDirection == TrendDirectionUpward: @@ -167,8 +142,7 @@ func (c *ChannelObserver) GetTrend() (ChannelTrend, ChannelCongestionReason) { } func (c *ChannelObserver) ToString() string { - packets, repeatedNacks, nackRatio := c.GetNackRatio() - return fmt.Sprintf("name: %s, estimate: %s, packets: %d, repeatedNacks: %d, nackRatio: %f", c.params.Name, c.estimateTrend.ToString(), packets, repeatedNacks, nackRatio) + return fmt.Sprintf("name: %s, estimate: {%s}, nack {%s}", c.params.Name, c.estimateTrend.ToString(), c.nackTracker.ToString()) } // ------------------------------------------------ diff --git a/pkg/sfu/streamallocator/nacktracker.go b/pkg/sfu/streamallocator/nacktracker.go new file mode 100644 index 000000000..13a53b887 --- /dev/null +++ b/pkg/sfu/streamallocator/nacktracker.go @@ -0,0 +1,85 @@ +package streamallocator + +import ( + "fmt" + "time" + + "github.com/livekit/protocol/logger" +) + +// ------------------------------------------------ + +type NackTrackerParams struct { + Name string + Logger logger.Logger + WindowMinDuration time.Duration + WindowMaxDuration time.Duration + RatioThreshold float64 +} + +type NackTracker struct { + params NackTrackerParams + + windowStartTime time.Time + packets uint32 + repeatedNacks uint32 +} + +func NewNackTracker(params NackTrackerParams) *NackTracker { + return &NackTracker{ + params: params, + } +} + +func (n *NackTracker) Add(packets uint32, repeatedNacks uint32) { + if n.params.WindowMaxDuration != 0 && !n.windowStartTime.IsZero() && time.Since(n.windowStartTime) > n.params.WindowMaxDuration { + n.windowStartTime = time.Time{} + n.packets = 0 + n.repeatedNacks = 0 + } + + // + // Start NACK monitoring window only when a repeated NACK happens. + // This allows locking tightly to when NACKs start happening and + // check if the NACKs keep adding up (potentially a sign of congestion) + // or isolated losses + // + if n.repeatedNacks == 0 && repeatedNacks != 0 { + n.windowStartTime = time.Now() + } + + if !n.windowStartTime.IsZero() { + n.packets += packets + n.repeatedNacks += repeatedNacks + } +} + +func (n *NackTracker) GetRatio() float64 { + ratio := 0.0 + if n.packets != 0 { + ratio = float64(n.repeatedNacks) / float64(n.packets) + if ratio > 1.0 { + ratio = 1.0 + } + } + + return ratio +} + +func (n *NackTracker) IsTriggered() bool { + if n.params.WindowMinDuration != 0 && !n.windowStartTime.IsZero() && time.Since(n.windowStartTime) > n.params.WindowMinDuration { + return n.GetRatio() > n.params.RatioThreshold + } + + return false +} + +func (n *NackTracker) ToString() string { + window := "" + if !n.windowStartTime.IsZero() { + window = fmt.Sprintf("t: %+v|%+v", n.windowStartTime, time.Since(n.windowStartTime)) + } + return fmt.Sprintf("n: %s, t: %s, p: %d, rn: %d, rn/p: %f", n.params.Name, window, n.packets, n.repeatedNacks, n.GetRatio()) +} + +// ------------------------------------------------ diff --git a/pkg/sfu/streamallocator/streamallocator.go b/pkg/sfu/streamallocator/streamallocator.go index fdf6c3353..d8731512b 100644 --- a/pkg/sfu/streamallocator/streamallocator.go +++ b/pkg/sfu/streamallocator/streamallocator.go @@ -781,10 +781,9 @@ func (s *StreamAllocator) handleNewEstimateInNonProbe() { var estimateToCommit int64 expectedBandwidthUsage := s.getExpectedBandwidthUsage() - packets, repeatedNacks, nackRatio := s.channelObserver.GetNackRatio() switch reason { case ChannelCongestionReasonLoss: - estimateToCommit = int64(float64(expectedBandwidthUsage) * (1.0 - NackRatioAttenuator*nackRatio)) + estimateToCommit = int64(float64(expectedBandwidthUsage) * (1.0 - NackRatioAttenuator*s.channelObserver.GetNackRatio())) if estimateToCommit > s.lastReceivedEstimate { estimateToCommit = s.lastReceivedEstimate } @@ -799,9 +798,7 @@ func (s *StreamAllocator) handleNewEstimateInNonProbe() { "new(bps)", estimateToCommit, "lastReceived(bps)", s.lastReceivedEstimate, "expectedUsage(bps)", expectedBandwidthUsage, - "packets", packets, - "repeatedNacks", repeatedNacks, - "nackRatio", nackRatio, + "channel", s.channelObserver.ToString(), ) s.committedChannelCapacity = estimateToCommit From d9cd07c4b1edbf991332fcd04a609448dde0a34a Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Mon, 17 Apr 2023 15:31:28 +0800 Subject: [PATCH 092/324] Return chosen codec for simulcast codecs (#1624) * Return chosen codec for simulcast codecs * Increase timeout for subscription test --- pkg/rtc/mediaengine.go | 8 ++++---- pkg/rtc/mediaengine_test.go | 12 ++++++------ pkg/rtc/participant.go | 10 ++++++---- pkg/rtc/subscriptionmanager_test.go | 2 +- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/pkg/rtc/mediaengine.go b/pkg/rtc/mediaengine.go index 1b01b8874..4178dfb15 100644 --- a/pkg/rtc/mediaengine.go +++ b/pkg/rtc/mediaengine.go @@ -16,7 +16,7 @@ func registerCodecs(me *webrtc.MediaEngine, codecs []*livekit.Codec, rtcpFeedbac opusCodec := opusCodecCapability opusCodec.RTCPFeedback = rtcpFeedback.Audio var opusPayload webrtc.PayloadType - if isCodecEnabled(codecs, opusCodec) { + if IsCodecEnabled(codecs, opusCodec) { opusPayload = 111 if err := me.RegisterCodec(webrtc.RTPCodecParameters{ RTPCodecCapability: opusCodec, @@ -25,7 +25,7 @@ func registerCodecs(me *webrtc.MediaEngine, codecs []*livekit.Codec, rtcpFeedbac return err } - if isCodecEnabled(codecs, redCodecCapability) { + if IsCodecEnabled(codecs, redCodecCapability) { if err := me.RegisterCodec(webrtc.RTPCodecParameters{ RTPCodecCapability: redCodecCapability, PayloadType: 63, @@ -65,7 +65,7 @@ func registerCodecs(me *webrtc.MediaEngine, codecs []*livekit.Codec, rtcpFeedbac PayloadType: 35, }, } { - if isCodecEnabled(codecs, codec.RTPCodecCapability) { + if IsCodecEnabled(codecs, codec.RTPCodecCapability) { if err := me.RegisterCodec(codec, webrtc.RTPCodecTypeVideo); err != nil { return err } @@ -103,7 +103,7 @@ func createMediaEngine(codecs []*livekit.Codec, config DirectionConfig) (*webrtc return me, nil } -func isCodecEnabled(codecs []*livekit.Codec, cap webrtc.RTPCodecCapability) bool { +func IsCodecEnabled(codecs []*livekit.Codec, cap webrtc.RTPCodecCapability) bool { for _, codec := range codecs { if !strings.EqualFold(codec.Mime, cap.MimeType) { continue diff --git a/pkg/rtc/mediaengine_test.go b/pkg/rtc/mediaengine_test.go index b774f49fa..19f81529c 100644 --- a/pkg/rtc/mediaengine_test.go +++ b/pkg/rtc/mediaengine_test.go @@ -12,15 +12,15 @@ import ( func TestIsCodecEnabled(t *testing.T) { t.Run("empty fmtp requirement should match all", func(t *testing.T) { enabledCodecs := []*livekit.Codec{{Mime: "video/h264"}} - require.True(t, isCodecEnabled(enabledCodecs, webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264, SDPFmtpLine: "special"})) - require.True(t, isCodecEnabled(enabledCodecs, webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264})) - require.False(t, isCodecEnabled(enabledCodecs, webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP8})) + require.True(t, IsCodecEnabled(enabledCodecs, webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264, SDPFmtpLine: "special"})) + require.True(t, IsCodecEnabled(enabledCodecs, webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264})) + require.False(t, IsCodecEnabled(enabledCodecs, webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP8})) }) t.Run("when fmtp is provided, require match", func(t *testing.T) { enabledCodecs := []*livekit.Codec{{Mime: "video/h264", FmtpLine: "special"}} - require.True(t, isCodecEnabled(enabledCodecs, webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264, SDPFmtpLine: "special"})) - require.False(t, isCodecEnabled(enabledCodecs, webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264})) - require.False(t, isCodecEnabled(enabledCodecs, webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP8})) + require.True(t, IsCodecEnabled(enabledCodecs, webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264, SDPFmtpLine: "special"})) + require.False(t, IsCodecEnabled(enabledCodecs, webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264})) + require.False(t, IsCodecEnabled(enabledCodecs, webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP8})) }) } diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 30a8ad14c..55fd04df4 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -1461,10 +1461,12 @@ func (p *ParticipantImpl) addPendingTrackLocked(req *livekit.AddTrackRequest) *l } else if req.Type == livekit.TrackType_AUDIO && !strings.HasPrefix(mime, "audio/") { mime = "audio/" + mime } - ti.Codecs = append(ti.Codecs, &livekit.SimulcastCodecInfo{ - MimeType: mime, - Cid: codec.Cid, - }) + if IsCodecEnabled(p.params.EnabledCodecs, webrtc.RTPCodecCapability{MimeType: mime}) { + ti.Codecs = append(ti.Codecs, &livekit.SimulcastCodecInfo{ + MimeType: mime, + Cid: codec.Cid, + }) + } } p.params.Telemetry.TrackPublishRequested(context.Background(), p.ID(), p.Identity(), ti) diff --git a/pkg/rtc/subscriptionmanager_test.go b/pkg/rtc/subscriptionmanager_test.go index 05fea3d8c..d5a9d0f4a 100644 --- a/pkg/rtc/subscriptionmanager_test.go +++ b/pkg/rtc/subscriptionmanager_test.go @@ -39,7 +39,7 @@ func init() { } const ( - subSettleTimeout = 300 * time.Millisecond + subSettleTimeout = 600 * time.Millisecond subCheckInterval = 10 * time.Millisecond ) From 5b34d754e05015ab3741828c7d1bb1a907fe52e3 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 17 Apr 2023 13:02:24 +0530 Subject: [PATCH 093/324] Safe access of sequencer (#1625) * log some NACKs * split out NACK tracker * remove debug * debug * Sequencer safety * Clean up --- pkg/sfu/downtrack.go | 26 ++++++++++---------- pkg/sfu/sequencer.go | 18 ++++++++------ pkg/sfu/sequencer_test.go | 31 +++++++++++++++++++----- pkg/sfu/streamallocator/nacktracker.go | 6 +++-- pkg/sfu/streamallocator/trenddetector.go | 8 +++--- 5 files changed, 57 insertions(+), 32 deletions(-) diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 8eb026205..8a091c9d8 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -563,12 +563,15 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { payload = d.translateVP8PacketTo(extPkt.Packet, &incomingVP8, tp.codecBytes, pool) } - var meta *packetMeta if d.sequencer != nil { - meta = d.sequencer.push(extPkt.Packet.SequenceNumber, tp.rtp.sequenceNumber, tp.rtp.timestamp, int8(layer)) - if meta != nil { - meta.codecBytes = append(meta.codecBytes, tp.codecBytes...) - } + d.sequencer.push( + extPkt.Packet.SequenceNumber, + tp.rtp.sequenceNumber, + tp.rtp.timestamp, + int8(layer), + tp.codecBytes, + tp.ddBytes, + ) } hdr, err := d.getTranslatedRTPHeader(extPkt, tp) @@ -577,10 +580,6 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { return err } - if meta != nil && d.dependencyDescriptorID != 0 { - meta.ddBytes = append(meta.ddBytes, tp.ddBytes...) - } - _, err = d.writeStream.WriteRTP(hdr, payload) if err != nil { if errors.Is(err, io.ErrClosedPipe) { @@ -603,7 +602,7 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { if extPkt.KeyFrame { d.isNACKThrottled.Store(false) d.rtpStats.UpdateKeyFrame(1) - d.logger.Debugw("forwarding key frame", "layer", layer) + d.logger.Debugw("forwarding key frame", "layer", layer, "rtpsn", hdr.SequenceNumber, "rtpts", hdr.Timestamp) } if tp.isSwitchingToRequestSpatial { @@ -1443,9 +1442,10 @@ func (d *DownTrack) retransmitPackets(nacks []uint16) { continue } - translatedVP8 := meta.codecBytes - pool = PacketFactory.Get().(*[]byte) - payload = d.translateVP8PacketTo(&pkt, &incomingVP8, translatedVP8, pool) + if len(meta.codecBytes) != 0 { + pool = PacketFactory.Get().(*[]byte) + payload = d.translateVP8PacketTo(&pkt, &incomingVP8, meta.codecBytes, pool) + } } var extraExtensions []extensionData diff --git a/pkg/sfu/sequencer.go b/pkg/sfu/sequencer.go index 4f8558512..c3c2e4302 100644 --- a/pkg/sfu/sequencer.go +++ b/pkg/sfu/sequencer.go @@ -92,13 +92,13 @@ func (s *sequencer) setRTT(rtt uint32) { } } -func (s *sequencer) push(sn, offSn uint16, timeStamp uint32, layer int8) *packetMeta { +func (s *sequencer) push(sn, offSn uint16, timeStamp uint32, layer int8, codecBytes []byte, ddBytes []byte) { s.Lock() defer s.Unlock() slot, isValid := s.getSlot(offSn) if !isValid { - return nil + return } s.meta[s.metaWritePtr] = packetMeta{ @@ -106,6 +106,8 @@ func (s *sequencer) push(sn, offSn uint16, timeStamp uint32, layer int8) *packet targetSeqNo: offSn, timestamp: timeStamp, layer: layer, + codecBytes: append([]byte{}, codecBytes...), + ddBytes: append([]byte{}, ddBytes...), } s.seq[slot] = &s.meta[s.metaWritePtr] @@ -114,8 +116,6 @@ func (s *sequencer) push(sn, offSn uint16, timeStamp uint32, layer int8) *packet if s.metaWritePtr >= len(s.meta) { s.metaWritePtr -= len(s.meta) } - - return s.seq[slot] } func (s *sequencer) pushPadding(offSn uint16) { @@ -168,11 +168,11 @@ func (s *sequencer) getSlot(offSn uint16) (int, bool) { return s.wrap(slot), true } -func (s *sequencer) getPacketsMeta(seqNo []uint16) []*packetMeta { +func (s *sequencer) getPacketsMeta(seqNo []uint16) []packetMeta { s.Lock() defer s.Unlock() - meta := make([]*packetMeta, 0, len(seqNo)) + meta := make([]packetMeta, 0, len(seqNo)) refTime := uint32(time.Now().UnixNano()/1e6 - s.startTime) for _, sn := range seqNo { diff := s.headSN - sn @@ -190,7 +190,11 @@ func (s *sequencer) getPacketsMeta(seqNo []uint16) []*packetMeta { if seq.lastNack == 0 || refTime-seq.lastNack > uint32(math.Min(float64(ignoreRetransmission), float64(2*s.rtt))) { seq.nacked++ seq.lastNack = refTime - meta = append(meta, seq) + + pm := *seq + pm.codecBytes = append([]byte{}, seq.codecBytes...) + pm.ddBytes = append([]byte{}, seq.ddBytes...) + meta = append(meta, pm) } } diff --git a/pkg/sfu/sequencer_test.go b/pkg/sfu/sequencer_test.go index 776434004..94a74f791 100644 --- a/pkg/sfu/sequencer_test.go +++ b/pkg/sfu/sequencer_test.go @@ -15,11 +15,11 @@ func Test_sequencer(t *testing.T) { off := uint16(15) for i := uint16(1); i < 518; i++ { - seq.push(i, i+off, 123, 2) + seq.push(i, i+off, 123, 2, nil, nil) } // send the last two out-of-order - seq.push(519, 519+off, 123, 2) - seq.push(518, 518+off, 123, 2) + seq.push(519, 519+off, 123, 2, nil, nil) + seq.push(518, 518+off, 123, 2, nil, nil) time.Sleep(60 * time.Millisecond) req := []uint16{57, 58, 62, 63, 513, 514, 515, 516, 517} @@ -41,11 +41,11 @@ func Test_sequencer(t *testing.T) { require.Equal(t, val.layer, int8(2)) } - seq.push(521, 521+off, 123, 1) + seq.push(521, 521+off, 123, 1, nil, nil) m := seq.getPacketsMeta([]uint16{521 + off}) require.Equal(t, 1, len(m)) - seq.push(505, 505+off, 123, 1) + seq.push(505, 505+off, 123, 1, nil, nil) m = seq.getPacketsMeta([]uint16{505 + off}) require.Equal(t, 1, len(m)) } @@ -58,6 +58,10 @@ func Test_sequencer_getNACKSeqNo(t *testing.T) { input []uint16 padding []uint16 offset uint16 + codecBytesOdd []byte + codecBytesEven []byte + ddBytesOdd []byte + ddBytesEven []byte } tests := []struct { @@ -72,6 +76,10 @@ func Test_sequencer_getNACKSeqNo(t *testing.T) { input: []uint16{2, 3, 4, 7, 8, 11}, padding: []uint16{9, 10}, offset: 5, + codecBytesOdd: []byte{1, 2, 3, 4}, + codecBytesEven: []byte{5, 6, 7}, + ddBytesOdd: []byte{8, 9, 10}, + ddBytesEven: []byte{11, 12}, }, args: args{ seqNo: []uint16{4 + 5, 5 + 5, 8 + 5, 9 + 5, 10 + 5, 11 + 5}, @@ -85,7 +93,11 @@ func Test_sequencer_getNACKSeqNo(t *testing.T) { n := newSequencer(5, 10, logger.GetLogger()) for _, i := range tt.fields.input { - n.push(i, i+tt.fields.offset, 123, 3) + if i % 2 == 0 { + n.push(i, i+tt.fields.offset, 123, 3, tt.fields.codecBytesEven, tt.fields.ddBytesEven) + } else { + n.push(i, i+tt.fields.offset, 123, 3, tt.fields.codecBytesOdd, tt.fields.ddBytesOdd) + } } for _, i := range tt.fields.padding { n.pushPadding(i + tt.fields.offset) @@ -95,6 +107,13 @@ func Test_sequencer_getNACKSeqNo(t *testing.T) { var got []uint16 for _, sn := range g { got = append(got, sn.sourceSeqNo) + if sn.sourceSeqNo % 2 == 0 { + require.Equal(t, tt.fields.codecBytesEven, sn.codecBytes) + require.Equal(t, tt.fields.ddBytesEven, sn.ddBytes) + } else { + require.Equal(t, tt.fields.codecBytesOdd, sn.codecBytes) + require.Equal(t, tt.fields.ddBytesOdd, sn.ddBytes) + } } if !reflect.DeepEqual(got, tt.want) { t.Errorf("getPacketsMeta() = %v, want %v", got, tt.want) diff --git a/pkg/sfu/streamallocator/nacktracker.go b/pkg/sfu/streamallocator/nacktracker.go index 13a53b887..cc91bfe98 100644 --- a/pkg/sfu/streamallocator/nacktracker.go +++ b/pkg/sfu/streamallocator/nacktracker.go @@ -77,9 +77,11 @@ func (n *NackTracker) IsTriggered() bool { func (n *NackTracker) ToString() string { window := "" if !n.windowStartTime.IsZero() { - window = fmt.Sprintf("t: %+v|%+v", n.windowStartTime, time.Since(n.windowStartTime)) + now := time.Now() + elapsed := now.Sub(n.windowStartTime).Seconds() + window = fmt.Sprintf("t: %+v|%+v|%.2fs", n.windowStartTime.Format(time.UnixDate), now.Format(time.UnixDate), elapsed) } - return fmt.Sprintf("n: %s, t: %s, p: %d, rn: %d, rn/p: %f", n.params.Name, window, n.packets, n.repeatedNacks, n.GetRatio()) + return fmt.Sprintf("n: %s, t: %s, p: %d, rn: %d, rn/p: %.2f", n.params.Name, window, n.packets, n.repeatedNacks, n.GetRatio()) } // ------------------------------------------------ diff --git a/pkg/sfu/streamallocator/trenddetector.go b/pkg/sfu/streamallocator/trenddetector.go index 959b7d9de..232e8878d 100644 --- a/pkg/sfu/streamallocator/trenddetector.go +++ b/pkg/sfu/streamallocator/trenddetector.go @@ -109,10 +109,10 @@ func (t *TrendDetector) GetDirection() TrendDirection { func (t *TrendDetector) ToString() string { now := time.Now() elapsed := now.Sub(t.startTime).Seconds() - str := fmt.Sprintf("n: %s", t.params.Name) - str += fmt.Sprintf(", t: %+v|%+v|%.2fs", t.startTime.Format(time.UnixDate), now.Format(time.UnixDate), elapsed) - str += fmt.Sprintf(", v: %d|%d|%d|%+v|%.2f", t.numSamples, t.lowestValue, t.highestValue, t.values, kendallsTau(t.values)) - return str + return fmt.Sprintf("n: %s, t: %+v|%+v|%.2fs, v: %d|%d|%d|%+v|%.2f", + t.params.Name, + t.startTime.Format(time.UnixDate), now.Format(time.UnixDate), elapsed, + t.numSamples, t.lowestValue, t.highestValue, t.values, kendallsTau(t.values)) } func (t *TrendDetector) updateDirection() { From 93604d241506fe3634e9d7646c0c4ed8af745842 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 18 Apr 2023 11:49:51 +0530 Subject: [PATCH 094/324] A couple of stream allocator tweaks (#1628) * A coupke of stream allocator tweaks - Do not overshoot on catch up. It so happens that during probe the next higher layer is at some bit rate which is much lower than normal bit rate for that layer. But, by the time the probe ends, publisher has climbed up to normal bit rate. So, the probe goal although achieved is not enough. Allowing overshoot latches on the next layer which might be more than the channel capacity. - Use a collapse window to record values in case of a only one or two changes in an evaluation window. Some times it happens that the estimate falls once or twice and stays there. By collapsing repeated values, it could be a long time before that fall in estimate is processed. Introduce a collapse window and record duplicate value if a value was not recorded for collapse window duration. This allows delayed processing of those isolated falls in estimate. * minor clean up * add a probe max rate * fix max * use max of committed, expected for max limiting * have to probe at goal --- pkg/sfu/streamallocator/channelobserver.go | 4 +- pkg/sfu/streamallocator/prober.go | 2 +- pkg/sfu/streamallocator/streamallocator.go | 61 +++++++++++++--------- pkg/sfu/streamallocator/trenddetector.go | 35 +++++++++++-- 4 files changed, 69 insertions(+), 33 deletions(-) diff --git a/pkg/sfu/streamallocator/channelobserver.go b/pkg/sfu/streamallocator/channelobserver.go index 20fa0edf7..25dfe7cac 100644 --- a/pkg/sfu/streamallocator/channelobserver.go +++ b/pkg/sfu/streamallocator/channelobserver.go @@ -59,7 +59,7 @@ type ChannelObserverParams struct { Name string EstimateRequiredSamples int EstimateDownwardTrendThreshold float64 - EstimateCollapseValues bool + EstimateCollapseThreshold time.Duration NackWindowMinDuration time.Duration NackWindowMaxDuration time.Duration NackRatioThreshold float64 @@ -86,7 +86,7 @@ func NewChannelObserver(params ChannelObserverParams, logger logger.Logger) *Cha Logger: logger, RequiredSamples: params.EstimateRequiredSamples, DownwardTrendThreshold: params.EstimateDownwardTrendThreshold, - CollapseValues: params.EstimateCollapseValues, + CollapseThreshold: params.EstimateCollapseThreshold, }), nackTracker: NewNackTracker(NackTrackerParams{ Name: params.Name + "-estimate", diff --git a/pkg/sfu/streamallocator/prober.go b/pkg/sfu/streamallocator/prober.go index 0e3661bf0..dd18401bd 100644 --- a/pkg/sfu/streamallocator/prober.go +++ b/pkg/sfu/streamallocator/prober.go @@ -488,7 +488,7 @@ func (c *Cluster) Process(pl ProberListener) { pl.OnSendProbe(bytesShortFall) } - // LK-TODO look at adapting sleep time based on how many bytes and how much time is left + // STREAM-ALLOCATOR-TODO look at adapting sleep time based on how many bytes and how much time is left } func (c *Cluster) String() string { diff --git a/pkg/sfu/streamallocator/streamallocator.go b/pkg/sfu/streamallocator/streamallocator.go index d8731512b..cb73976d3 100644 --- a/pkg/sfu/streamallocator/streamallocator.go +++ b/pkg/sfu/streamallocator/streamallocator.go @@ -45,7 +45,8 @@ const ( FlagAllowOvershootWhileDeficient = false FlagAllowOvershootExemptTrackWhileDeficient = true FlagAllowOvershootInProbe = true - FlagAllowOvershootInCatchup = true + FlagAllowOvershootInCatchup = false + FlagAllowOvershootInBoost = true ) // --------------------------------------------------------------------------- @@ -55,7 +56,7 @@ var ( Name: "probe", EstimateRequiredSamples: 3, EstimateDownwardTrendThreshold: 0.0, - EstimateCollapseValues: false, + EstimateCollapseThreshold: 0, NackWindowMinDuration: 500 * time.Millisecond, NackWindowMaxDuration: 1 * time.Second, NackRatioThreshold: 0.04, @@ -65,7 +66,7 @@ var ( Name: "non-probe", EstimateRequiredSamples: 8, EstimateDownwardTrendThreshold: -0.5, - EstimateCollapseValues: true, + EstimateCollapseThreshold: 500 * time.Millisecond, NackWindowMinDuration: 1 * time.Second, NackWindowMaxDuration: 2 * time.Second, NackRatioThreshold: 0.08, @@ -259,7 +260,7 @@ func (s *StreamAllocator) AddTrack(downTrack *sfu.DownTrack, params AddTrackPara downTrack.SetStreamAllocatorListener(s) if s.prober.IsRunning() { - // LK-TODO: this can be changed to adapt to probe rate + // STREAM-ALLOCATOR-TODO: this can be changed to adapt to probe rate downTrack.SetStreamAllocatorReportInterval(50 * time.Millisecond) } @@ -273,7 +274,7 @@ func (s *StreamAllocator) RemoveTrack(downTrack *sfu.DownTrack) { } s.videoTracksMu.Unlock() - // LK-TODO: use any saved bandwidth to re-distribute + // STREAM-ALLOCATOR-TODO: use any saved bandwidth to re-distribute s.postEvent(Event{ Signal: streamAllocatorSignalAdjustState, }) @@ -333,11 +334,11 @@ func (s *StreamAllocator) OnREMB(downTrack *sfu.DownTrack, remb *rtcp.ReceiverEs // callback from previous REMB comes after another down track's callback // from the new REMB. REMBs could fire very quickly especially when // the network is entering congestion. - // LK-TODO-START + // STREAM-ALLOCATOR-TODO-START // Need to check if the same SSRC reports can somehow race, i.e. does pion send // RTCP dispatch for same SSRC on different threads? If not, the tracking SSRC // should prevent racing - // LK-TODO-END + // STREAM-ALLOCATOR-TODO-END // // if there are no video tracks, ignore any straggler REMB @@ -487,7 +488,7 @@ func (s *StreamAllocator) OnProbeClusterDone(info ProbeClusterInfo) { func (s *StreamAllocator) OnActiveChanged(isActive bool) { for _, t := range s.getTracks() { if isActive { - // LK-TODO: this can be changed to adapt to probe rate + // STREAM-ALLOCATOR-TODO: this can be changed to adapt to probe rate t.DownTrack().SetStreamAllocatorReportInterval(50 * time.Millisecond) } else { t.DownTrack().ClearStreamAllocatorReportInterval() @@ -662,7 +663,7 @@ func (s *StreamAllocator) handleSignalProbeClusterDone(event *Event) { } // ensure probe queue is flushed - // LK-TODO: ProbeSettleWait should actually be a certain number of RTTs. + // STREAM-ALLOCATOR-TODO: ProbeSettleWait should actually be a certain number of RTTs. lowestEstimate := int64(math.Min(float64(s.committedChannelCapacity), float64(s.channelObserver.GetLowestEstimate()))) expectedDuration := float64(info.BytesSent*8*1000) / float64(lowestEstimate) queueTime := expectedDuration - float64(info.Duration.Milliseconds()) @@ -746,7 +747,7 @@ func (s *StreamAllocator) handleNewEstimateInProbe() { // // More of a safety net. // In rare cases, the estimate gets stuck. Prevent from probe running amok - // LK-TODO: Need more testing here to ensure that probe does not cause a lot of damage + // STREAM-ALLOCATOR-TODO: Need more testing here to ensure that probe does not cause a lot of damage // s.params.Logger.Infow("stream allocator: probe: aborting, no trend", "cluster", s.probeClusterId) s.abortProbe() @@ -853,11 +854,11 @@ func (s *StreamAllocator) allocateTrack(track *Track) { s.adjustState() return - // LK-TODO-START + // STREAM-ALLOCATOR-TODO-START // Should use the bits given back to start any paused track. // Note layer downgrade may actually have positive delta (i.e. consume more bits) // because of when the measurement is done. Watch for that. - // LK-TODO-END + // STREAM_ALLOCATOR-TODO-END } // @@ -894,7 +895,7 @@ func (s *StreamAllocator) allocateTrack(track *Track) { } } - // LK-TODO if got too much extra, can potentially give it to some deficient track + // STREAM-ALLOCATOR-TODO if got too much extra, can potentially give it to some deficient track } // commit the track that needs change if enough could be acquired or pause not allowed @@ -1040,7 +1041,7 @@ func (s *StreamAllocator) allocateAllTracks() { update.HandleStreamingChange(true, track) } - // LK-TODO: optimistic allocation before bitrate is available will return 0. How to account for that? + // STREAM-ALLOCATOR-TODO: optimistic allocation before bitrate is available will return 0. How to account for that? availableChannelCapacity -= allocation.BandwidthRequested } @@ -1144,11 +1145,25 @@ func (s *StreamAllocator) newChannelObserverNonProbe() *ChannelObserver { return NewChannelObserver(ChannelObserverParamsNonProbe, s.params.Logger) } -func (s *StreamAllocator) initProbe(probeRateBps int64) { +func (s *StreamAllocator) initProbe(probeGoalDeltaBps int64) { s.lastProbeStartTime = time.Now() expectedBandwidthUsage := s.getExpectedBandwidthUsage() - s.probeGoalBps = expectedBandwidthUsage + probeRateBps + if float64(expectedBandwidthUsage) > 1.5*float64(s.committedChannelCapacity) { + // STREAM-ALLOCATOR-TODO-START + // Should probably skip probing if the expected usage is much higher than committed channel capacity. + // But, give that bandwidth estimate is volatile at times and can drop down to small values, + // not probing means streaming stuck in a well for long. + // Observe this and figure out if there is a threshold from practical use cases that can be used to + // skip probing safely + // STREAM-ALLOCATOR-TODO-END + s.params.Logger.Warnw( + "stream allocator: starting probe alarm", + fmt.Errorf("expected too high, expected: %d, committed: %d", expectedBandwidthUsage, s.committedChannelCapacity), + ) + } + // overshoot a bit to account for noise (in measurement/estimate etc) + s.probeGoalBps = expectedBandwidthUsage + ((probeGoalDeltaBps * ProbePct) / 100) s.abortedProbeClusterId = ProbeClusterIdInvalid @@ -1163,9 +1178,8 @@ func (s *StreamAllocator) initProbe(probeRateBps int64) { s.channelObserver = s.newChannelObserverProbe() s.channelObserver.SeedEstimate(s.lastReceivedEstimate) - desiredRateBps := int(probeRateBps) + int(math.Max(float64(s.committedChannelCapacity), float64(expectedBandwidthUsage))) s.probeClusterId = s.prober.AddCluster( - desiredRateBps, + int(s.probeGoalBps), int(expectedBandwidthUsage), ProbeMinDuration, ProbeMaxDuration, @@ -1177,7 +1191,7 @@ func (s *StreamAllocator) initProbe(probeRateBps int64) { "committed", s.committedChannelCapacity, "lastReceived", s.lastReceivedEstimate, "channel", channelState, - "probeRateBps", probeRateBps, + "probeGoalDeltaBps", probeGoalDeltaBps, "goalBps", s.probeGoalBps, ) } @@ -1237,7 +1251,7 @@ func (s *StreamAllocator) maybeProbe() { func (s *StreamAllocator) maybeProbeWithMedia() { // boost deficient track farthest from desired layer for _, track := range s.getMaxDistanceSortedDeficient() { - allocation, boosted := track.AllocateNextHigher(ChannelCapacityInfinity, FlagAllowOvershootInCatchup) + allocation, boosted := track.AllocateNextHigher(ChannelCapacityInfinity, FlagAllowOvershootInBoost) if !boosted { continue } @@ -1261,12 +1275,7 @@ func (s *StreamAllocator) maybeProbeWithPadding() { continue } - probeRateBps := (transition.BandwidthDelta * ProbePct) / 100 - if probeRateBps < ProbeMinBps { - probeRateBps = ProbeMinBps - } - - s.initProbe(probeRateBps) + s.initProbe(transition.BandwidthDelta) break } } diff --git a/pkg/sfu/streamallocator/trenddetector.go b/pkg/sfu/streamallocator/trenddetector.go index 232e8878d..2c5bf4850 100644 --- a/pkg/sfu/streamallocator/trenddetector.go +++ b/pkg/sfu/streamallocator/trenddetector.go @@ -37,7 +37,7 @@ type TrendDetectorParams struct { Logger logger.Logger RequiredSamples int DownwardTrendThreshold float64 - CollapseValues bool + CollapseThreshold time.Duration } type TrendDetector struct { @@ -49,6 +49,9 @@ type TrendDetector struct { lowestValue int64 highestValue int64 + hasFallen bool + lastSampleAt time.Time + direction TrendDirection } @@ -66,6 +69,8 @@ func (t *TrendDetector) Seed(value int64) { } t.values = append(t.values, value) + t.lastSampleAt = time.Now() + t.hasFallen = false } func (t *TrendDetector) AddValue(value int64) { @@ -77,10 +82,32 @@ func (t *TrendDetector) AddValue(value int64) { t.highestValue = value } - // ignore duplicate values - if t.params.CollapseValues && len(t.values) != 0 && t.values[len(t.values)-1] == value { - return + // Ignore duplicate values in collapse window. + // + // Bandwidth estimate is received periodically. If the estimate does not change, it will be repeated. + // When there is congestion, there are several estimates received with decreasing values. + // + // Using a sliding window, collapsing repeated values and waiting for falling trend is to ensure that + // the reaction is not too fast, i. e. reacting to falling values too quick could mean a lot of re-allocation + // resulting in layer switches, key frames and more congestion. + // + // But, on the flip side, estimate could fall once or twice withing a sliding window and stay there. + // In those cases, using a collapse window to record value even if it is duplicate. By doing that, + // a trend could be detected eventually. If will be delayed, but that is fine with slow changing estimates. + lastValue := int64(0) + if len(t.values) != 0 { + lastValue = t.values[len(t.values)-1] } + if lastValue == value && t.params.CollapseThreshold > 0 { + if !t.hasFallen || (!t.lastSampleAt.IsZero() && time.Since(t.lastSampleAt) < t.params.CollapseThreshold) { + return + } + } + + if lastValue > value { + t.hasFallen = true + } + t.lastSampleAt = time.Now() if len(t.values) == t.params.RequiredSamples { t.values = t.values[1:] From 10b70c929946314cf424124c3ede95dbd857befe Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 19 Apr 2023 10:40:29 +0530 Subject: [PATCH 095/324] Choose max available layer when current becomes unavailable. (#1631) When current became unavailable, it was possible for target to be set to opportunistic. Because of that, the downgrade did not happen and PLI layer lock was requested continuously. --- pkg/sfu/forwarder.go | 53 ++++++++++++++++++++++++++++----------- pkg/sfu/forwarder_test.go | 43 ++++++++++++++----------------- 2 files changed, 58 insertions(+), 38 deletions(-) diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 94d764bdd..ab4e83dfb 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -579,26 +579,51 @@ func (f *Forwarder) AllocateOptimal(availableLayers []int32, brs Bitrates, allow alloc.RequestLayerSpatial = alloc.TargetLayer.Spatial default: + // lots of different events could end up here + // 1. Publisher side layer resuming/stopping + // 2. Bitrate becoming available + // 3. New max published spatial layer or max temporal layer seen + // 4. Subscriber layer changes + // + // to handle all of the above + // 1. Find highest that can be requested - takes into account available layers and overshoot. + // This should catch scenarios like layers resuming/stopping. + // 2. If current is a valid layer, check against currently available layers and continue at current + // if possible. Else, choose the highest available layer as the next target. + // 3. If current is not valid, set next target to be opportunistic. + maxLayerSpatialLimit := int32(math.Min(float64(maxLayer.Spatial), float64(maxSeenLayer.Spatial))) + highestAvailableLayer := buffer.InvalidLayerSpatial requestLayerSpatial := buffer.InvalidLayerSpatial for _, al := range availableLayers { - if al > requestLayerSpatial { + if al > requestLayerSpatial && al <= maxLayerSpatialLimit { requestLayerSpatial = al } + if al > highestAvailableLayer { + highestAvailableLayer = al + } } - maxLayerSpatialLimit := int32(math.Min(float64(maxLayer.Spatial), float64(maxSeenLayer.Spatial))) - if requestLayerSpatial > maxLayerSpatialLimit { - requestLayerSpatial = maxLayerSpatialLimit + if requestLayerSpatial == buffer.InvalidLayerSpatial && highestAvailableLayer != buffer.InvalidLayerSpatial && allowOvershoot && f.vls.IsOvershootOkay() { + requestLayerSpatial = highestAvailableLayer } - if currentLayer.IsValid() && ((requestLayerSpatial == requestSpatial && currentLayer.Spatial == requestSpatial) || requestLayerSpatial == buffer.InvalidLayerSpatial) { - // 1. current is locked to desired, stay there - // OR - // 2. feed may be dry, let it continue at current layer if valid. - // covers the cases of - // 1. mis-detection of layer stop - can continue streaming - // 2. current layer resuming - can latch on when it starts - alloc.TargetLayer = buffer.VideoLayer{ - Spatial: currentLayer.Spatial, - Temporal: getMaxTemporal(), + + if currentLayer.IsValid() { + if (requestLayerSpatial == requestSpatial && currentLayer.Spatial == requestSpatial) || requestLayerSpatial == buffer.InvalidLayerSpatial { + // 1. current is locked to desired, stay there + // OR + // 2. feed may be dry, let it continue at current layer if valid. + // covers the cases of + // 1. mis-detection of layer stop - can continue streaming + // 2. current layer resuming - can latch on when it starts + alloc.TargetLayer = buffer.VideoLayer{ + Spatial: currentLayer.Spatial, + Temporal: getMaxTemporal(), + } + } else { + // current layer has stopped, switch to highest available + alloc.TargetLayer = buffer.VideoLayer{ + Spatial: requestLayerSpatial, + Temporal: getMaxTemporal(), + } } alloc.RequestLayerSpatial = alloc.TargetLayer.Spatial } else { diff --git a/pkg/sfu/forwarder_test.go b/pkg/sfu/forwarder_test.go index bbb079258..ac3166f70 100644 --- a/pkg/sfu/forwarder_test.go +++ b/pkg/sfu/forwarder_test.go @@ -302,6 +302,23 @@ func TestForwarderAllocateOptimal(t *testing.T) { require.Equal(t, expectedResult, f.lastAllocation) require.Equal(t, buffer.DefaultMaxLayer, f.TargetLayer()) + // opportunistic target if feed is dry and current is not valid, i. e. not forwarding + expectedResult = VideoAllocation{ + PauseReason: VideoPauseReasonNone, + BandwidthRequested: bitrates[2][1], + BandwidthDelta: 0, + BandwidthNeeded: bitrates[2][1], + Bitrates: bitrates, + TargetLayer: buffer.DefaultMaxLayer, + RequestLayerSpatial: 2, + MaxLayer: buffer.DefaultMaxLayer, + DistanceToDesired: -0.5, + } + result = f.AllocateOptimal(nil, bitrates, true) + require.Equal(t, expectedResult, result) + require.Equal(t, expectedResult, f.lastAllocation) + require.Equal(t, buffer.DefaultMaxLayer, f.TargetLayer()) + // if feed is not dry and current is not locked, should be opportunistic (with and without overshoot) f.vls.SetTarget(buffer.InvalidLayer) expectedResult = VideoAllocation{ @@ -341,7 +358,7 @@ func TestForwarderAllocateOptimal(t *testing.T) { // switches request layer to highest available if feed is not dry and current is valid and current is not available f.vls.SetCurrent(buffer.VideoLayer{Spatial: 0, Temporal: 1}) expectedTargetLayer = buffer.VideoLayer{ - Spatial: 2, + Spatial: 1, Temporal: buffer.DefaultMaxLayerTemporal, } expectedResult = VideoAllocation{ @@ -353,7 +370,7 @@ func TestForwarderAllocateOptimal(t *testing.T) { TargetLayer: expectedTargetLayer, RequestLayerSpatial: 1, MaxLayer: buffer.DefaultMaxLayer, - DistanceToDesired: -0.5, + DistanceToDesired: 0.5, } result = f.AllocateOptimal([]int32{1}, bitrates, true) require.Equal(t, expectedResult, result) @@ -380,28 +397,6 @@ func TestForwarderAllocateOptimal(t *testing.T) { result = f.AllocateOptimal([]int32{0}, emptyBitrates, true) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) - - // opportunistic if feed is not dry and current is valid, but request layer has changed - f.vls.SetMax(buffer.VideoLayer{Spatial: 2, Temporal: 1}) - f.vls.SetCurrent(buffer.VideoLayer{Spatial: 0, Temporal: 1}) - f.vls.SetRequestSpatial(0) - expectedTargetLayer = buffer.VideoLayer{ - Spatial: 2, - Temporal: 1, - } - expectedResult = VideoAllocation{ - PauseReason: VideoPauseReasonFeedDry, - BandwidthRequested: 0, - BandwidthDelta: 0, - Bitrates: emptyBitrates, - TargetLayer: expectedTargetLayer, - RequestLayerSpatial: 1, - MaxLayer: f.vls.GetMax(), - DistanceToDesired: -1, - } - result = f.AllocateOptimal([]int32{0, 1}, emptyBitrates, true) - require.Equal(t, expectedResult, result) - require.Equal(t, expectedResult, f.lastAllocation) } func TestForwarderProvisionalAllocate(t *testing.T) { From 4bd0646fdced1dd46b1078582e896ed033a22e21 Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Wed, 19 Apr 2023 13:57:18 +0800 Subject: [PATCH 096/324] Don't close rtcpreader if downtrack will be resumed (#1632) --- pkg/sfu/downtrack.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 8a091c9d8..87498d987 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -836,7 +836,7 @@ func (d *DownTrack) CloseWithFlush(flush bool) { d.logger.Debugw("closing sender", "kind", d.kind) d.receiver.DeleteDownTrack(d.subscriberID) - if d.rtcpReader != nil { + if d.rtcpReader != nil && flush { d.logger.Debugw("downtrack close rtcp reader") d.rtcpReader.Close() d.rtcpReader.OnPacket(nil) From ab42aed3604ba0455ee1f802d1ee9dc42abd6e0b Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Wed, 19 Apr 2023 14:56:16 +0800 Subject: [PATCH 097/324] Add flag to control candidate fallback when udp unstable (#1630) * Add flag to control candidate fallback when udp unstable * Don't close rtcpreader if downtrack will be resumed --- pkg/rtc/participant.go | 30 ++++++++++++++++-------------- pkg/rtc/transportmanager.go | 35 ++++++++++++++++++----------------- 2 files changed, 34 insertions(+), 31 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 55fd04df4..8e64941af 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -81,6 +81,7 @@ type ParticipantParams struct { AdaptiveStream bool AllowTCPFallback bool TCPFallbackRTTThreshold int + AllowUDPUnstableFallback bool TURNSEnabled bool GetParticipantInfo func(pID livekit.ParticipantID) *livekit.ParticipantInfo ReconnectOnPublicationError bool @@ -977,20 +978,21 @@ func (p *ParticipantImpl) setupTransportManager() error { SID: p.params.SID, // primary connection does not change, canSubscribe can change if permission was updated // after the participant has joined - SubscriberAsPrimary: p.ProtocolVersion().SubscriberAsPrimary() && p.CanSubscribe(), - Config: p.params.Config, - ProtocolVersion: p.params.ProtocolVersion, - Telemetry: p.params.Telemetry, - CongestionControlConfig: p.params.CongestionControlConfig, - EnabledCodecs: p.params.EnabledCodecs, - SimTracks: p.params.SimTracks, - ClientConf: p.params.ClientConf, - ClientInfo: p.params.ClientInfo, - Migration: p.params.Migration, - AllowTCPFallback: p.params.AllowTCPFallback, - TCPFallbackRTTThreshold: p.params.TCPFallbackRTTThreshold, - TURNSEnabled: p.params.TURNSEnabled, - Logger: p.params.Logger, + SubscriberAsPrimary: p.ProtocolVersion().SubscriberAsPrimary() && p.CanSubscribe(), + Config: p.params.Config, + ProtocolVersion: p.params.ProtocolVersion, + Telemetry: p.params.Telemetry, + CongestionControlConfig: p.params.CongestionControlConfig, + EnabledCodecs: p.params.EnabledCodecs, + SimTracks: p.params.SimTracks, + ClientConf: p.params.ClientConf, + ClientInfo: p.params.ClientInfo, + Migration: p.params.Migration, + AllowTCPFallback: p.params.AllowTCPFallback, + TCPFallbackRTTThreshold: p.params.TCPFallbackRTTThreshold, + AllowUDPUnstableFallback: p.params.AllowUDPUnstableFallback, + TURNSEnabled: p.params.TURNSEnabled, + Logger: p.params.Logger, }) if err != nil { return err diff --git a/pkg/rtc/transportmanager.go b/pkg/rtc/transportmanager.go index c7dfcc57d..de7c0a32b 100644 --- a/pkg/rtc/transportmanager.go +++ b/pkg/rtc/transportmanager.go @@ -34,22 +34,23 @@ const ( ) type TransportManagerParams struct { - Identity livekit.ParticipantIdentity - SID livekit.ParticipantID - SubscriberAsPrimary bool - Config *WebRTCConfig - ProtocolVersion types.ProtocolVersion - Telemetry telemetry.TelemetryService - CongestionControlConfig config.CongestionControlConfig - EnabledCodecs []*livekit.Codec - SimTracks map[uint32]SimulcastTrackInfo - ClientConf *livekit.ClientConfiguration - ClientInfo ClientInfo - Migration bool - AllowTCPFallback bool - TCPFallbackRTTThreshold int - TURNSEnabled bool - Logger logger.Logger + Identity livekit.ParticipantIdentity + SID livekit.ParticipantID + SubscriberAsPrimary bool + Config *WebRTCConfig + ProtocolVersion types.ProtocolVersion + Telemetry telemetry.TelemetryService + CongestionControlConfig config.CongestionControlConfig + EnabledCodecs []*livekit.Codec + SimTracks map[uint32]SimulcastTrackInfo + ClientConf *livekit.ClientConfiguration + ClientInfo ClientInfo + Migration bool + AllowTCPFallback bool + TCPFallbackRTTThreshold int + AllowUDPUnstableFallback bool + TURNSEnabled bool + Logger logger.Logger } type TransportManager struct { @@ -710,7 +711,7 @@ func (t *TransportManager) OnReceiverReport(dt *sfu.DownTrack, report *rtcp.Rece } func (t *TransportManager) onMediaLossUpdate(loss uint8) { - if t.params.TCPFallbackRTTThreshold == 0 { + if t.params.TCPFallbackRTTThreshold == 0 || !t.params.AllowUDPUnstableFallback { return } t.lock.Lock() From a9fe9f331c0d2cb4749d7c426a77a60d68d38493 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 19 Apr 2023 13:05:43 +0530 Subject: [PATCH 098/324] Run quality scorer when there are no streams. (#1633) * Run quality scorer when there are no streams. In the down stream direction, receiver report is used for scoring. If there are no receiver reports, it should go to `dry` state and report poor quality. Update scorer on dry condition only when update score has not happened for longer than some multiple of update interval. Cannot update on every interval when there are no streams as receiver report might be just missed. Waiting for longer to ensure that report is definitely not received. * update last stats time --- pkg/sfu/connectionquality/connectionstats.go | 20 ++++++++++++++++++-- pkg/sfu/connectionquality/scorer.go | 4 ++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/pkg/sfu/connectionquality/connectionstats.go b/pkg/sfu/connectionquality/connectionstats.go index 185acc6e6..fb6caa91f 100644 --- a/pkg/sfu/connectionquality/connectionstats.go +++ b/pkg/sfu/connectionquality/connectionstats.go @@ -15,8 +15,9 @@ import ( ) const ( - UpdateInterval = 5 * time.Second - processThreshold = 0.95 + UpdateInterval = 5 * time.Second + processThreshold = 0.95 + noStatsTooLongMultiplier = 2 ) type ConnectionStatsParams struct { @@ -158,6 +159,17 @@ func (cs *ConnectionStats) updateLastStatsAt(at time.Time) { cs.lastStatsAt = at } +func (cs *ConnectionStats) isTooLongSinceLastStats() bool { + cs.lock.Lock() + defer cs.lock.Unlock() + + interval := cs.params.UpdateInterval + if interval == 0 { + interval = UpdateInterval + } + return !cs.lastStatsAt.IsZero() && time.Since(cs.lastStatsAt) > interval*noStatsTooLongMultiplier +} + func (cs *ConnectionStats) clearInProcess() { cs.lock.Lock() defer cs.lock.Unlock() @@ -177,6 +189,10 @@ func (cs *ConnectionStats) getStat(at time.Time) { streams := cs.params.GetDeltaStats() if len(streams) == 0 { + if cs.isTooLongSinceLastStats() { + cs.updateLastStatsAt(at) + cs.updateScore(streams, at) + } cs.clearInProcess() return } diff --git a/pkg/sfu/connectionquality/scorer.go b/pkg/sfu/connectionquality/scorer.go index af7861d15..e13575a32 100644 --- a/pkg/sfu/connectionquality/scorer.go +++ b/pkg/sfu/connectionquality/scorer.go @@ -363,6 +363,10 @@ func (q *qualityScorer) getPacketLossWeight(stat *windowStat) float64 { q.maxPPS = pps } + if q.maxPPS == 0 { + return q.params.PacketLossWeight + } + return math.Sqrt(pps/q.maxPPS) * q.params.PacketLossWeight } From a11944f84d681adaef32ed3c7ae45527c538f330 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 19 Apr 2023 16:21:16 +0530 Subject: [PATCH 099/324] Restore VP8 munger state properly. (#1634) * Restore VP8 munger state properly. * clean up --- pkg/sfu/codecmunger/null.go | 8 +++++++- pkg/sfu/codecmunger/vp8.go | 6 ++++++ pkg/sfu/forwarder.go | 2 +- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/pkg/sfu/codecmunger/null.go b/pkg/sfu/codecmunger/null.go index e6b3f00cb..f9c327a2d 100644 --- a/pkg/sfu/codecmunger/null.go +++ b/pkg/sfu/codecmunger/null.go @@ -6,6 +6,7 @@ import ( ) type Null struct { + seededState interface{} } func NewNull(_logger logger.Logger) *Null { @@ -16,7 +17,12 @@ func (n *Null) GetState() interface{} { return nil } -func (n *Null) SeedState(_state interface{}) { +func (n *Null) SeedState(state interface{}) { + n.seededState = state +} + +func (n *Null) GetSeededState() interface{} { + return n.seededState } func (n *Null) SetLast(_extPkt *buffer.ExtPacket) { diff --git a/pkg/sfu/codecmunger/vp8.go b/pkg/sfu/codecmunger/vp8.go index 271dac71e..dbe8f0665 100644 --- a/pkg/sfu/codecmunger/vp8.go +++ b/pkg/sfu/codecmunger/vp8.go @@ -64,6 +64,12 @@ func NewVP8(logger logger.Logger) *VP8 { } } +func NewVP8FromNull(cm CodecMunger, logger logger.Logger) *VP8 { + v := NewVP8(logger) + v.SeedState(cm.(*Null).GetSeededState()) + return v +} + func (v *VP8) GetState() interface{} { return VP8State{ ExtLastPictureId: v.extLastPictureId, diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index ab4e83dfb..069132850 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -256,7 +256,7 @@ func (f *Forwarder) DetermineCodec(codec webrtc.RTPCodecCapability, extensions [ switch strings.ToLower(codec.MimeType) { case "video/vp8": - f.codecMunger = codecmunger.NewVP8(f.logger) + f.codecMunger = codecmunger.NewVP8FromNull(f.codecMunger, f.logger) if f.vls != nil { f.vls = videolayerselector.NewSimulcastFromNull(f.vls) } else { From 1cd8e45fdf8d0ffacd993af29faf75b18b3bff90 Mon Sep 17 00:00:00 2001 From: Jonas Schell Date: Wed, 19 Apr 2023 22:48:45 +0200 Subject: [PATCH 100/324] =?UTF-8?q?[=F0=9F=A4=96=20readme-manager]=20Updat?= =?UTF-8?q?e=20README=20(#1637)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * update readme * update readme --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6e004dc4d..16fd9dd8b 100644 --- a/README.md +++ b/README.md @@ -293,9 +293,10 @@ LiveKit server is licensed under Apache License v2.0.
- - + + +
LiveKit Ecosystem
Core Infralivekit · egress · ingress · livekit-cli
Client SDKsComponents · JavaScript · Rust · iOS/macOS · Android · Flutter · Unity (web) · React Native (beta)
Server SDKsNode.js · Golang · Ruby · Java/Kotlin
Server SDKsNode.js · Golang · Ruby · Java/Kotlin · PHP (community) · Python (community)
ServicesLivekit server · Egress · Ingress
ResourcesDocs · Example apps · Cloud · Self-hosting · CLI
From 422a28551eb6b4895671a580de5ca3b4f72da0af Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Wed, 19 Apr 2023 15:33:42 -0700 Subject: [PATCH 101/324] record signal read failure metrics (#1639) --- pkg/routing/signal.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/pkg/routing/signal.go b/pkg/routing/signal.go index 20077b233..0c1ac576b 100644 --- a/pkg/routing/signal.go +++ b/pkg/routing/signal.go @@ -19,6 +19,9 @@ import ( "github.com/livekit/psrpc/middleware" ) +var ErrSignalWriteFailed = errors.New("signal write failed") +var ErrSignalMessageDropped = errors.New("signal message dropped") + //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate //counterfeiter:generate . SignalClient @@ -182,16 +185,18 @@ func CopySignalStreamToMessageChannel[SendType, RecvType RelaySignalMessage]( for msg := range stream.Channel() { res, err := r.Read(msg) if err != nil { + if errors.Is(err, ErrSignalMessageDropped) { + prometheus.MessageCounter.WithLabelValues("signal", "failure").Add(1) + } return err } - prometheus.MessageCounter.WithLabelValues("signal", "success").Add(float64(len(res))) - for _, r := range res { if err = ch.WriteMessage(r); err != nil { prometheus.MessageCounter.WithLabelValues("signal", "failure").Add(1) return err } + prometheus.MessageCounter.WithLabelValues("signal", "success").Add(1) } } return stream.Err() @@ -211,7 +216,7 @@ func (r *signalMessageReader[SendType, RecvType]) Read(msg RecvType) ([]proto.Me if r.config.MinVersion >= 1 { if r.seq < msg.GetSeq() { - return nil, errors.New("signal message dropped") + return nil, ErrSignalMessageDropped } if r.seq > msg.GetSeq() { n := int(r.seq - msg.GetSeq()) @@ -239,8 +244,6 @@ func NewSignalMessageSink[SendType, RecvType RelaySignalMessage](params SignalSi } } -var ErrSignalFailed = errors.New("signal stream failed") - type signalMessageSink[SendType, RecvType RelaySignalMessage] struct { SignalSinkParams[SendType, RecvType] @@ -321,7 +324,7 @@ func (s *signalMessageSink[SendType, RecvType]) write() { s.Stream.Close(nil) } if err != nil && s.CloseOnFailure { - s.Stream.Close(ErrSignalFailed) + s.Stream.Close(ErrSignalWriteFailed) } s.mu.Unlock() } From 09af509edbf89d63b77d9ea3f7a8d14149e03666 Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Thu, 20 Apr 2023 14:15:30 +0800 Subject: [PATCH 102/324] Add subscription limits (#1629) * Add subscription limits * Add limit to ParticipantParams * Don't change desired of subscription when reaching limits * Add subscription limits config * Revert comment * solve comments --- config-sample.yaml | 6 ++ pkg/config/config.go | 6 +- pkg/rtc/errors.go | 11 +-- pkg/rtc/participant.go | 23 +++--- pkg/rtc/subscriptionmanager.go | 104 ++++++++++++++++++++------ pkg/rtc/subscriptionmanager_test.go | 112 +++++++++++++++++++++++++++- pkg/service/roommanager.go | 2 + 7 files changed, 220 insertions(+), 44 deletions(-) diff --git a/config-sample.yaml b/config-sample.yaml index 8a6fcf7c5..7375b2398 100644 --- a/config-sample.yaml +++ b/config-sample.yaml @@ -261,3 +261,9 @@ keys: # num_tracks: -1 # # defaults to 1 GB/s, or just under 10 Gbps # bytes_per_sec: 1_000_000_000 +# # how many tracks (audio / video) that a single participant can subscribe at same time. +# # if the limit is exceeded, subscriptions will be pending until any subscribed track has been unsubscribed. +# # value less or equal than 0 means no limit. +# subscription_limit_video: 0 +# subscription_limit_audio: 0 + diff --git a/pkg/config/config.go b/pkg/config/config.go index 50cdf5ccc..bff9d19e1 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -246,8 +246,10 @@ type RegionConfig struct { } type LimitConfig struct { - NumTracks int32 `yaml:"num_tracks,omitempty"` - BytesPerSec float32 `yaml:"bytes_per_sec,omitempty"` + NumTracks int32 `yaml:"num_tracks,omitempty"` + BytesPerSec float32 `yaml:"bytes_per_sec,omitempty"` + SubscriptionLimitVideo int32 `yaml:"subscription_limit_video,omitempty"` + SubscriptionLimitAudio int32 `yaml:"subscription_limit_audio,omitempty"` } type EgressConfig struct { diff --git a/pkg/rtc/errors.go b/pkg/rtc/errors.go index dcd31c979..20c41acb9 100644 --- a/pkg/rtc/errors.go +++ b/pkg/rtc/errors.go @@ -14,9 +14,10 @@ var ( ErrMissingGrants = errors.New("VideoGrant is missing") // Track subscription related - ErrNoTrackPermission = errors.New("participant is not allowed to subscribe to this track") - ErrNoSubscribePermission = errors.New("participant is not given permission to subscribe to tracks") - ErrTrackNotFound = errors.New("track cannot be found") - ErrTrackNotAttached = errors.New("track is not yet attached") - ErrTrackNotBound = errors.New("track not bound") + ErrNoTrackPermission = errors.New("participant is not allowed to subscribe to this track") + ErrNoSubscribePermission = errors.New("participant is not given permission to subscribe to tracks") + ErrTrackNotFound = errors.New("track cannot be found") + ErrTrackNotAttached = errors.New("track is not yet attached") + ErrTrackNotBound = errors.New("track not bound") + ErrSubscriptionLimitExceeded = errors.New("participant has exceeded its subscription limit") ) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 8e64941af..86c721ac7 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -90,6 +90,8 @@ type ParticipantParams struct { TrackResolver types.MediaTrackResolver DisableDynacast bool SubscriberAllowPause bool + SubscriptionLimitAudio int32 + SubscriptionLimitVideo int32 } type ParticipantImpl struct { @@ -1065,13 +1067,15 @@ func (p *ParticipantImpl) setupUpTrackManager() { func (p *ParticipantImpl) setupSubscriptionManager() { p.SubscriptionManager = NewSubscriptionManager(SubscriptionManagerParams{ - Participant: p, - Logger: p.params.Logger.WithoutSampler(), - TrackResolver: p.params.TrackResolver, - Telemetry: p.params.Telemetry, - OnTrackSubscribed: p.onTrackSubscribed, - OnTrackUnsubscribed: p.onTrackUnsubscribed, - OnSubscriptionError: p.onSubscriptionError, + Participant: p, + Logger: p.params.Logger.WithoutSampler(), + TrackResolver: p.params.TrackResolver, + Telemetry: p.params.Telemetry, + OnTrackSubscribed: p.onTrackSubscribed, + OnTrackUnsubscribed: p.onTrackUnsubscribed, + OnSubscriptionError: p.onSubscriptionError, + SubscriptionLimitVideo: p.params.SubscriptionLimitVideo, + SubscriptionLimitAudio: p.params.SubscriptionLimitAudio, }) } @@ -1765,11 +1769,6 @@ func (p *ParticipantImpl) getPendingTrack(clientId string, kind livekit.TrackTyp if pendingInfo == nil { track_loop: for cid, pti := range p.pendingTracks { - if cid == clientId { - pendingInfo = pti - signalCid = cid - break - } ti := pti.trackInfos[0] for _, c := range ti.Codecs { diff --git a/pkg/rtc/subscriptionmanager.go b/pkg/rtc/subscriptionmanager.go index 9e880aeb3..640e85d19 100644 --- a/pkg/rtc/subscriptionmanager.go +++ b/pkg/rtc/subscriptionmanager.go @@ -42,6 +42,10 @@ var ( trackRemoveGracePeriod = time.Second ) +const ( + trackIDForReconcileSubscriptions = livekit.TrackID("subscriptions_reconcile") +) + type SubscriptionManagerParams struct { Logger logger.Logger Participant types.LocalParticipant @@ -50,6 +54,8 @@ type SubscriptionManagerParams struct { OnTrackUnsubscribed func(subTrack types.SubscribedTrack) OnSubscriptionError func(trackID livekit.TrackID) Telemetry telemetry.TelemetryService + + SubscriptionLimitVideo, SubscriptionLimitAudio int32 } // SubscriptionManager manages a participant's subscriptions @@ -57,25 +63,25 @@ type SubscriptionManager struct { params SubscriptionManagerParams lock sync.RWMutex subscriptions map[livekit.TrackID]*trackSubscription - subscribedTo map[livekit.ParticipantID]map[livekit.TrackID]struct{} - // keeps track of tracks that are already queued for reconcile to avoid duplicating reconcile requests - pendingReconcile map[livekit.TrackID]struct{} - reconcileCh chan livekit.TrackID - closeCh chan struct{} - doneCh chan struct{} + + subscribedVideoCount, subscribedAudioCount atomic.Int32 + + subscribedTo map[livekit.ParticipantID]map[livekit.TrackID]struct{} + reconcileCh chan livekit.TrackID + closeCh chan struct{} + doneCh chan struct{} onSubscribeStatusChanged func(publisherID livekit.ParticipantID, subscribed bool) } func NewSubscriptionManager(params SubscriptionManagerParams) *SubscriptionManager { m := &SubscriptionManager{ - params: params, - subscriptions: make(map[livekit.TrackID]*trackSubscription), - subscribedTo: make(map[livekit.ParticipantID]map[livekit.TrackID]struct{}), - pendingReconcile: make(map[livekit.TrackID]struct{}), - reconcileCh: make(chan livekit.TrackID, 50), - closeCh: make(chan struct{}), - doneCh: make(chan struct{}), + params: params, + subscriptions: make(map[livekit.TrackID]*trackSubscription), + subscribedTo: make(map[livekit.ParticipantID]map[livekit.TrackID]struct{}), + reconcileCh: make(chan livekit.TrackID, 50), + closeCh: make(chan struct{}), + doneCh: make(chan struct{}), } go m.reconcileWorker() @@ -282,20 +288,21 @@ func (m *SubscriptionManager) reconcileSubscription(s *trackSubscription) { s.recordAttempt(false) switch err { - case ErrNoTrackPermission, ErrNoSubscribePermission, ErrNoReceiver, ErrNotOpen, ErrTrackNotAttached: + case ErrNoTrackPermission, ErrNoSubscribePermission, ErrNoReceiver, ErrNotOpen, ErrTrackNotAttached, ErrSubscriptionLimitExceeded: // these are errors that are outside of our control, so we'll keep trying // - ErrNoTrackPermission: publisher did not grant subscriber permission, may change any moment // - ErrNoSubscribePermission: participant was not granted canSubscribe, may change any moment // - ErrNoReceiver: Track is in the process of closing (another local track published to the same instance) // - ErrTrackNotAttached: Remote Track that is not attached, but may be attached later // - ErrNotOpen: Track is closing or already closed + // - ErrSubscriptionLimitExceeded: the participant have reached the limit of subscriptions, wait for the other subscription to be unsubscribed // We'll still log an event to reflect this in telemetry since it's been too long if s.durationSinceStart() > subscriptionTimeout { s.maybeRecordError(m.params.Telemetry, m.params.Participant.ID(), err, true) } case ErrTrackNotFound: // source track was never published or closed - // if after timeout, we'd unsubscribe from it. + // if after timeout we'd unsubscribe from it. // this is the *only* case we'd change desired state if s.durationSinceStart() > notFoundTimeout { s.maybeRecordError(m.params.Telemetry, m.params.Participant.ID(), err, true) @@ -353,13 +360,6 @@ func (m *SubscriptionManager) reconcileSubscription(s *trackSubscription) { // trigger an immediate reconciliation, when trackID is empty, will reconcile all subscriptions func (m *SubscriptionManager) queueReconcile(trackID livekit.TrackID) { - m.lock.Lock() - if _, ok := m.pendingReconcile[trackID]; ok { - // already reconciled - m.lock.Unlock() - return - } - m.lock.Unlock() select { case m.reconcileCh <- trackID: default: @@ -381,7 +381,6 @@ func (m *SubscriptionManager) reconcileWorker() { case trackID := <-m.reconcileCh: m.lock.Lock() s := m.subscriptions[trackID] - delete(m.pendingReconcile, trackID) m.lock.Unlock() if s != nil { m.reconcileSubscription(s) @@ -392,6 +391,21 @@ func (m *SubscriptionManager) reconcileWorker() { } } +func (m *SubscriptionManager) hasCapcityForSubscription(kind livekit.TrackType) bool { + switch kind { + case livekit.TrackType_VIDEO: + if m.params.SubscriptionLimitVideo > 0 && m.subscribedVideoCount.Load() >= m.params.SubscriptionLimitVideo { + return false + } + + case livekit.TrackType_AUDIO: + if m.params.SubscriptionLimitAudio > 0 && m.subscribedAudioCount.Load() >= m.params.SubscriptionLimitAudio { + return false + } + } + return true +} + func (m *SubscriptionManager) subscribe(s *trackSubscription) error { s.logger.Debugw("executing subscribe") @@ -399,6 +413,10 @@ func (m *SubscriptionManager) subscribe(s *trackSubscription) error { return ErrNoSubscribePermission } + if kind, ok := s.getKind(); ok && !m.hasCapcityForSubscription(kind) { + return ErrSubscriptionLimitExceeded + } + res := m.params.TrackResolver(m.params.Participant.Identity(), s.trackID) s.logger.Debugw("resolved track", "result", res) @@ -426,6 +444,10 @@ func (m *SubscriptionManager) subscribe(s *trackSubscription) error { if track == nil { return ErrTrackNotFound } + s.trySetKind(track.Kind()) + if !m.hasCapcityForSubscription(track.Kind()) { + return ErrSubscriptionLimitExceeded + } // since hasPermission defaults to true, we will want to send a message to the client the first time // that we discover permissions were denied @@ -453,6 +475,13 @@ func (m *SubscriptionManager) subscribe(s *trackSubscription) error { }) s.setSubscribedTrack(subTrack) + switch track.Kind() { + case livekit.TrackType_VIDEO: + m.subscribedVideoCount.Inc() + case livekit.TrackType_AUDIO: + m.subscribedAudioCount.Inc() + } + if subTrack.NeedsNegotiation() { m.params.Participant.Negotiate(false) } @@ -460,6 +489,8 @@ func (m *SubscriptionManager) subscribe(s *trackSubscription) error { go m.params.OnTrackSubscribed(subTrack) } + m.params.Logger.Debugw("subscribed to track", "track", s.trackID, "subscribedAudioCount", m.subscribedAudioCount.Load(), "subscribedVideoCount", m.subscribedVideoCount.Load()) + // add mark the participant as someone we've subscribed to firstSubscribe := false publisherID := s.getPublisherID() @@ -512,6 +543,14 @@ func (m *SubscriptionManager) handleSubscribedTrackClose(s *trackSubscription, w } s.setSubscribedTrack(nil) + var relieveFromLimits bool + switch subTrack.MediaTrack().Kind() { + case livekit.TrackType_VIDEO: + relieveFromLimits = m.params.SubscriptionLimitVideo > 0 && m.subscribedVideoCount.Dec() == m.params.SubscriptionLimitVideo-1 + case livekit.TrackType_AUDIO: + relieveFromLimits = m.params.SubscriptionLimitAudio > 0 && m.subscribedAudioCount.Dec() == m.params.SubscriptionLimitAudio-1 + } + // remove from subscribedTo publisherID := s.getPublisherID() lastSubscription := false @@ -581,7 +620,11 @@ func (m *SubscriptionManager) handleSubscribedTrackClose(s *trackSubscription, w m.params.Participant.Negotiate(false) } - m.queueReconcile(s.trackID) + if relieveFromLimits { + m.queueReconcile(trackIDForReconcileSubscriptions) + } else { + m.queueReconcile(s.trackID) + } } // -------------------------------------------------------------------------------------- @@ -603,6 +646,7 @@ type trackSubscription struct { eventSent atomic.Bool numAttempts atomic.Int32 bound bool + kind atomic.Pointer[livekit.TrackType] // the later of when subscription was requested OR when the first failure was encountered OR when permission is granted // this timestamp determines when failures are reported @@ -705,6 +749,18 @@ func (s *trackSubscription) setSubscribedTrack(track types.SubscribedTrack) { } } +func (s *trackSubscription) trySetKind(kind livekit.TrackType) { + s.kind.CompareAndSwap(nil, &kind) +} + +func (s *trackSubscription) getKind() (livekit.TrackType, bool) { + kind := s.kind.Load() + if kind == nil { + return livekit.TrackType_AUDIO, false + } + return *kind, true +} + func (s *trackSubscription) getSubscribedTrack() types.SubscribedTrack { s.lock.RLock() defer s.lock.RUnlock() diff --git a/pkg/rtc/subscriptionmanager_test.go b/pkg/rtc/subscriptionmanager_test.go index d5a9d0f4a..0c3de6955 100644 --- a/pkg/rtc/subscriptionmanager_test.go +++ b/pkg/rtc/subscriptionmanager_test.go @@ -340,7 +340,115 @@ func TestUpdateSettingsBeforeSubscription(t *testing.T) { require.Equal(t, settings.Height, applied.Height) } +func TestSubscriptionLimits(t *testing.T) { + sm := newTestSubscriptionManagerWithParams(t, testSubscriptionParams{ + SubscriptionLimitAudio: 1, + SubscriptionLimitVideo: 1, + }) + defer sm.Close(false) + resolver := newTestResolver(true, true, "pub", "pubID") + sm.params.TrackResolver = resolver.Resolve + subCount := atomic.Int32{} + failed := atomic.Bool{} + sm.params.OnTrackSubscribed = func(subTrack types.SubscribedTrack) { + subCount.Add(1) + } + sm.params.OnSubscriptionError = func(trackID livekit.TrackID) { + failed.Store(true) + } + numParticipantSubscribed := atomic.Int32{} + numParticipantUnsubscribed := atomic.Int32{} + sm.OnSubscribeStatusChanged(func(pubID livekit.ParticipantID, subscribed bool) { + if subscribed { + numParticipantSubscribed.Add(1) + } else { + numParticipantUnsubscribed.Add(1) + } + }) + + sm.SubscribeToTrack("track") + s := sm.subscriptions["track"] + require.True(t, s.isDesired()) + require.Eventually(t, func() bool { + return subCount.Load() == 1 + }, subSettleTimeout, subCheckInterval, "track was not subscribed") + + require.NotNil(t, s.getSubscribedTrack()) + require.Len(t, sm.GetSubscribedTracks(), 1) + + require.Eventually(t, func() bool { + return len(sm.GetSubscribedParticipants()) == 1 + }, subSettleTimeout, subCheckInterval, "GetSubscribedParticipants should have returned one item") + require.Equal(t, "pubID", string(sm.GetSubscribedParticipants()[0])) + + // ensure telemetry events are sent + tm := sm.params.Telemetry.(*telemetryfakes.FakeTelemetryService) + require.Equal(t, 1, tm.TrackSubscribeRequestedCallCount()) + + // ensure bound + setTestSubscribedTrackBound(t, s.getSubscribedTrack()) + + require.Eventually(t, func() bool { + return !s.needsBind() + }, subSettleTimeout, subCheckInterval, "track was not bound") + + // telemetry event should have been sent + require.Equal(t, 1, tm.TrackSubscribedCallCount()) + + // reach subscription limit, subscribe pending + sm.SubscribeToTrack("track2") + s2 := sm.subscriptions["track2"] + time.Sleep(subscriptionTimeout * 2) + require.True(t, s2.needsSubscribe()) + require.Equal(t, 2, tm.TrackSubscribeRequestedCallCount()) + require.Equal(t, 1, tm.TrackSubscribeFailedCallCount()) + require.Len(t, sm.GetSubscribedTracks(), 1) + + // unsubscribe track1, then track2 should be subscribed + sm.UnsubscribeFromTrack("track") + require.False(t, s.isDesired()) + require.True(t, s.needsUnsubscribe()) + // wait for unsubscribe to take effect + time.Sleep(reconcileInterval) + setTestSubscribedTrackClosed(t, s.getSubscribedTrack(), false) + require.Nil(t, s.getSubscribedTrack()) + + time.Sleep(reconcileInterval) + require.True(t, s2.isDesired()) + require.False(t, s2.needsSubscribe()) + require.EqualValues(t, 2, subCount.Load()) + require.NotNil(t, s2.getSubscribedTrack()) + require.Equal(t, 2, tm.TrackSubscribeRequestedCallCount()) + require.Len(t, sm.GetSubscribedTracks(), 1) + + // ensure bound + setTestSubscribedTrackBound(t, s2.getSubscribedTrack()) + + require.Eventually(t, func() bool { + return !s2.needsBind() + }, subSettleTimeout, subCheckInterval, "track was not bound") + + // subscribe to track1 again, which should pending + sm.SubscribeToTrack("track") + s = sm.subscriptions["track"] + require.True(t, s.isDesired()) + time.Sleep(subscriptionTimeout * 2) + require.True(t, s.needsSubscribe()) + require.Equal(t, 3, tm.TrackSubscribeRequestedCallCount()) + require.Equal(t, 2, tm.TrackSubscribeFailedCallCount()) + require.Len(t, sm.GetSubscribedTracks(), 1) +} + +type testSubscriptionParams struct { + SubscriptionLimitAudio int32 + SubscriptionLimitVideo int32 +} + func newTestSubscriptionManager(t *testing.T) *SubscriptionManager { + return newTestSubscriptionManagerWithParams(t, testSubscriptionParams{}) +} + +func newTestSubscriptionManagerWithParams(t *testing.T, params testSubscriptionParams) *SubscriptionManager { p := &typesfakes.FakeLocalParticipant{} p.CanSubscribeReturns(true) p.IDReturns("subID") @@ -354,7 +462,9 @@ func newTestSubscriptionManager(t *testing.T) *SubscriptionManager { TrackResolver: func(identity livekit.ParticipantIdentity, trackID livekit.TrackID) types.MediaResolverResult { return types.MediaResolverResult{} }, - Telemetry: &telemetryfakes.FakeTelemetryService{}, + Telemetry: &telemetryfakes.FakeTelemetryService{}, + SubscriptionLimitAudio: params.SubscriptionLimitAudio, + SubscriptionLimitVideo: params.SubscriptionLimitVideo, }) } diff --git a/pkg/service/roommanager.go b/pkg/service/roommanager.go index 9f4df01fb..0244abb84 100644 --- a/pkg/service/roommanager.go +++ b/pkg/service/roommanager.go @@ -341,6 +341,8 @@ func (r *RoomManager) StartSession( VersionGenerator: r.versionGenerator, TrackResolver: room.ResolveMediaTrackForSubscriber, SubscriberAllowPause: subscriberAllowPause, + SubscriptionLimitAudio: r.config.Limit.SubscriptionLimitAudio, + SubscriptionLimitVideo: r.config.Limit.SubscriptionLimitVideo, }) if err != nil { return err From 70041f004f6e50aa87a19a35cf9fd421ac7cb01b Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Thu, 20 Apr 2023 03:27:41 -0700 Subject: [PATCH 103/324] create signalStats from out of order join (#1640) --- pkg/routing/signal.go | 4 +--- pkg/service/rtcservice.go | 8 ++++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/pkg/routing/signal.go b/pkg/routing/signal.go index 0c1ac576b..3338f7235 100644 --- a/pkg/routing/signal.go +++ b/pkg/routing/signal.go @@ -185,9 +185,7 @@ func CopySignalStreamToMessageChannel[SendType, RecvType RelaySignalMessage]( for msg := range stream.Channel() { res, err := r.Read(msg) if err != nil { - if errors.Is(err, ErrSignalMessageDropped) { - prometheus.MessageCounter.WithLabelValues("signal", "failure").Add(1) - } + prometheus.MessageCounter.WithLabelValues("signal", "failure").Add(1) return err } diff --git a/pkg/service/rtcservice.go b/pkg/service/rtcservice.go index 16639c9d5..cd6bc93da 100644 --- a/pkg/service/rtcservice.go +++ b/pkg/service/rtcservice.go @@ -302,6 +302,14 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { pLogger.Debugw("sending answer", "answer", m) } + if pi.ID == "" && res.GetJoin() != nil { + pi.ID = livekit.ParticipantID(res.GetJoin().GetParticipant().GetSid()) + signalStats = telemetry.NewBytesTrackStats( + telemetry.BytesTrackIDForParticipantID(telemetry.BytesTrackTypeSignal, pi.ID), + pi.ID, + s.telemetry) + } + if count, err := sigConn.WriteResponse(res); err != nil { pLogger.Warnw("error writing to websocket", err) return From ab6c994db4398a43c928b58f2a349fbba9aad7dc Mon Sep 17 00:00:00 2001 From: David Colburn Date: Fri, 21 Apr 2023 12:43:20 -0700 Subject: [PATCH 104/324] update protocol/psrpc (#1643) * update protocol/psrpc * metadata references --- go.mod | 9 ++++----- go.sum | 18 +++++++----------- pkg/routing/signal.go | 2 +- pkg/service/signal.go | 5 +++-- pkg/telemetry/prometheus/psrpc.go | 2 +- 5 files changed, 16 insertions(+), 20 deletions(-) diff --git a/go.mod b/go.mod index 3f93906d0..3230c86e0 100644 --- a/go.mod +++ b/go.mod @@ -18,8 +18,8 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 - github.com/livekit/protocol v1.5.4 - github.com/livekit/psrpc v0.2.11-0.20230405191830-d76f71512630 + github.com/livekit/protocol v1.5.5-0.20230421192204-0975cb52f603 + github.com/livekit/psrpc v0.3.0 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.14.0 github.com/maxbrunsfeld/counterfeiter/v6 v6.6.1 @@ -33,13 +33,13 @@ require ( github.com/pion/rtp v1.7.13 github.com/pion/sdp/v3 v3.0.6 github.com/pion/stun v0.4.0 - github.com/pion/transport/v2 v2.1.0 + github.com/pion/transport/v2 v2.0.2 github.com/pion/turn/v2 v2.1.0 github.com/pion/webrtc/v3 v3.1.60 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.15.0 github.com/redis/go-redis/v9 v9.0.3 - github.com/rs/cors v1.9.0 + github.com/rs/cors v1.8.3 github.com/stretchr/testify v1.8.2 github.com/thoas/go-funk v0.9.3 github.com/twitchtv/twirp v8.1.3+incompatible @@ -70,7 +70,6 @@ require ( github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.7.2 // indirect github.com/josharian/native v1.1.0 // indirect - github.com/lithammer/shortuuid/v3 v3.0.7 // indirect github.com/lithammer/shortuuid/v4 v4.0.0 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect diff --git a/go.sum b/go.sum index bedd8cfa5..7a928ad54 100644 --- a/go.sum +++ b/go.sum @@ -73,7 +73,6 @@ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/subcommands v1.2.0 h1:vWQspBTo2nEqTUFita5/KeEWlUL8kQObDFbub/EN9oE= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= -github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/wire v0.5.0 h1:I7ELFeVBr3yfPIcc8+MWvrjk+3VjbcSzoXm3JVa+jD8= @@ -116,18 +115,16 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/lithammer/shortuuid/v3 v3.0.7 h1:trX0KTHy4Pbwo/6ia8fscyHoGA+mf1jWbPJVuvyJQQ8= -github.com/lithammer/shortuuid/v3 v3.0.7/go.mod h1:vMk8ke37EmiewwolSO1NLW8vP4ZaKlRuDIi8tWWmAts= github.com/lithammer/shortuuid/v4 v4.0.0 h1:QRbbVkfgNippHOS8PXDkti4NaWeyYfcBTHtw7k08o4c= github.com/lithammer/shortuuid/v4 v4.0.0/go.mod h1:Zs8puNcrvf2rV9rTH51ZLLcj7ZXqQI3lv67aw4KiB1Y= github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkDaKb5iXdynYrzB84ErPPO4LbRASk58= github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 h1:QlQFyMwCDgjyySsrgmrMcVbEBA6KZcyTzvK+z346tUA= github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26/go.mod h1:eDA41kiySZoG+wy4Etsjb3w0jjLx69i/vAmSjG4bteA= -github.com/livekit/protocol v1.5.4 h1:lfEUqsE9AV1ZI/w8oZUKSAoi708V8RYwraOjeY83KVo= -github.com/livekit/protocol v1.5.4/go.mod h1:KJJVGHiNR6abdJIpoxB1kqQH2s902wM3cMt+P4p6jao= -github.com/livekit/psrpc v0.2.11-0.20230405191830-d76f71512630 h1:Rm5KLZgQxWnTidY+H8MsAV6sk1iiFxeXqPFgSLkMing= -github.com/livekit/psrpc v0.2.11-0.20230405191830-d76f71512630/go.mod h1:K0j8f1PgLShR7Lx80KbmwFkDH2BvOnycXGV0OSRURKc= +github.com/livekit/protocol v1.5.5-0.20230421192204-0975cb52f603 h1:1O+cSgXWFWZDF+6cYOmJhu4w2HAsQcpLJI1YJfuixR8= +github.com/livekit/protocol v1.5.5-0.20230421192204-0975cb52f603/go.mod h1:/LpoK6XF8IeJDs/d5BPVmicp8pDPXmgTN1GVGhdpEX0= +github.com/livekit/psrpc v0.3.0 h1:giBZsfM3CWA0oIYXofsMITbVQtyW7u/ES9sQmVspHPM= +github.com/livekit/psrpc v0.3.0/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= github.com/mackerelio/go-osstat v0.2.4/go.mod h1:Zy+qzGdZs3A9cuIqmgbJvwbmLQH9dJvtio5ZjJTbdlQ= github.com/magefile/mage v1.14.0 h1:6QDX3g6z1YvJ4olPhT1wksUcSa/V0a1B+pJb73fBjyo= @@ -210,9 +207,8 @@ github.com/pion/stun v0.4.0/go.mod h1:QPsh1/SbXASntw3zkkrIk3ZJVKz4saBY2G7S10P3wC github.com/pion/transport v0.14.1 h1:XSM6olwW+o8J4SCmOBb/BpwZypkHeyM0PGFCxNQBr40= github.com/pion/transport v0.14.1/go.mod h1:4tGmbk00NeYA3rUa9+n+dzCCoKkcy3YlYb99Jn2fNnI= github.com/pion/transport/v2 v2.0.0/go.mod h1:HS2MEBJTwD+1ZI2eSXSvHJx/HnzQqRy2/LXxt6eVMHc= +github.com/pion/transport/v2 v2.0.2 h1:St+8o+1PEzPT51O9bv+tH/KYYLMNR5Vwm5Z3Qkjsywg= github.com/pion/transport/v2 v2.0.2/go.mod h1:vrz6bUbFr/cjdwbnxq8OdDDzHf7JJfGsIRkxfpZoTA0= -github.com/pion/transport/v2 v2.1.0 h1:tLBmDy/sfPu4UG9QsiKiI7Zav+i9zhUYvg7VlCUpIV8= -github.com/pion/transport/v2 v2.1.0/go.mod h1:AdSw4YBZVDkZm8fpoz+fclXyQwANWmZAlDuQdctTThQ= github.com/pion/turn/v2 v2.1.0 h1:5wGHSgGhJhP/RpabkUb/T9PdsAjkGLS6toYz5HNzoSI= github.com/pion/turn/v2 v2.1.0/go.mod h1:yrT5XbXSGX1VFSF31A3c1kCNB5bBZgk/uu5LET162qs= github.com/pion/udp/v2 v2.0.1 h1:xP0z6WNux1zWEjhC7onRA3EwwSliXqu1ElUZAQhUP54= @@ -235,8 +231,8 @@ github.com/redis/go-redis/v9 v9.0.3 h1:+7mmR26M0IvyLxGZUHxu4GiBkJkVDid0Un+j4ScYu github.com/redis/go-redis/v9 v9.0.3/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= -github.com/rs/cors v1.9.0 h1:l9HGsTsHJcvW14Nk7J9KFz8bzeAWXn3CG6bgt7LsrAE= -github.com/rs/cors v1.9.0/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= +github.com/rs/cors v1.8.3 h1:O+qNyWn7Z+F9M0ILBHgMVPuB1xTOucVd5gtaYyXBpRo= +github.com/rs/cors v1.8.3/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= diff --git a/pkg/routing/signal.go b/pkg/routing/signal.go index 3338f7235..dfef77dec 100644 --- a/pkg/routing/signal.go +++ b/pkg/routing/signal.go @@ -16,7 +16,7 @@ import ( "github.com/livekit/protocol/rpc" "github.com/livekit/protocol/utils" "github.com/livekit/psrpc" - "github.com/livekit/psrpc/middleware" + "github.com/livekit/psrpc/pkg/middleware" ) var ErrSignalWriteFailed = errors.New("signal write failed") diff --git a/pkg/service/signal.go b/pkg/service/signal.go index 3d9b6c17e..f7ef7c7cc 100644 --- a/pkg/service/signal.go +++ b/pkg/service/signal.go @@ -13,7 +13,8 @@ import ( "github.com/livekit/protocol/logger" "github.com/livekit/protocol/rpc" "github.com/livekit/psrpc" - "github.com/livekit/psrpc/middleware" + "github.com/livekit/psrpc/pkg/metadata" + "github.com/livekit/psrpc/pkg/middleware" ) type SessionHandler func( @@ -101,7 +102,7 @@ func (r *signalService) RelaySignal(stream psrpc.ServerStream[*rpc.RelaySignalRe // copy the context to prevent a race between the session handler closing // and the delivery of any parting messages from the client. take care to // copy the incoming rpc headers to avoid dropping any session vars. - ctx, cancel := context.WithCancel(psrpc.NewContextWithIncomingHeader(context.Background(), psrpc.IncomingHeader(stream.Context()))) + ctx, cancel := context.WithCancel(metadata.NewContextWithIncomingHeader(context.Background(), metadata.IncomingHeader(stream.Context()))) defer cancel() req, ok := <-stream.Channel() diff --git a/pkg/telemetry/prometheus/psrpc.go b/pkg/telemetry/prometheus/psrpc.go index 1ac3d9a72..958704605 100644 --- a/pkg/telemetry/prometheus/psrpc.go +++ b/pkg/telemetry/prometheus/psrpc.go @@ -7,7 +7,7 @@ import ( "github.com/livekit/protocol/livekit" "github.com/livekit/psrpc" - "github.com/livekit/psrpc/middleware" + "github.com/livekit/psrpc/pkg/middleware" ) var ( From a77eb2a07d213b12abd59afe32dd88792a2c9725 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Sat, 22 Apr 2023 07:05:07 -0700 Subject: [PATCH 105/324] add room node assignment check to signal relay (#1645) --- pkg/service/signal.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pkg/service/signal.go b/pkg/service/signal.go index f7ef7c7cc..87b29d527 100644 --- a/pkg/service/signal.go +++ b/pkg/service/signal.go @@ -73,6 +73,19 @@ func NewDefaultSignalServer( prometheus.IncrementParticipantRtcInit(1) if rr, ok := router.(*routing.RedisRouter); ok { + rtcNode, err := router.GetNodeForRoom(ctx, roomName) + if err != nil { + return err + } + + if rtcNode.Id != currentNode.Id { + err = routing.ErrIncorrectRTCNode + logger.Errorw("called participant on incorrect node", err, + "rtcNode", rtcNode, + ) + return err + } + pKey := routing.ParticipantKeyLegacy(roomName, pi.Identity) pKeyB62 := routing.ParticipantKey(roomName, pi.Identity) From 745410bd699de7aa97a6e4625d3a4944dd8d880b Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Sat, 22 Apr 2023 17:48:10 -0700 Subject: [PATCH 106/324] only increment participant version after updates (#1646) * only increment participant version after updates * fix test util * cleanup * test uptrackmanager permission update version check --- go.mod | 4 +- go.sum | 8 +- pkg/rtc/participant.go | 77 +++++++--- pkg/rtc/participant_internal_test.go | 1 + pkg/rtc/room.go | 2 +- pkg/rtc/types/interfaces.go | 7 +- .../typesfakes/fake_local_participant.go | 133 +++++++++++++++--- pkg/rtc/types/typesfakes/fake_participant.go | 33 ++--- pkg/rtc/uptrackmanager.go | 57 +++----- pkg/rtc/uptrackmanager_test.go | 42 ++++-- 10 files changed, 260 insertions(+), 104 deletions(-) diff --git a/go.mod b/go.mod index 3230c86e0..ec383e2e8 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 - github.com/livekit/protocol v1.5.5-0.20230421192204-0975cb52f603 + github.com/livekit/protocol v1.5.5-0.20230422131440-eb3ef0f4bf36 github.com/livekit/psrpc v0.3.0 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.14.0 @@ -92,7 +92,7 @@ require ( github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect go.uber.org/multierr v1.6.0 // indirect golang.org/x/crypto v0.8.0 // indirect - golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect + golang.org/x/exp v0.0.0-20230420155640-133eef4313cb // indirect golang.org/x/mod v0.8.0 // indirect golang.org/x/net v0.9.0 // indirect golang.org/x/sys v0.7.0 // indirect diff --git a/go.sum b/go.sum index 7a928ad54..fa24f3bd0 100644 --- a/go.sum +++ b/go.sum @@ -121,8 +121,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 h1:QlQFyMwCDgjyySsrgmrMcVbEBA6KZcyTzvK+z346tUA= github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26/go.mod h1:eDA41kiySZoG+wy4Etsjb3w0jjLx69i/vAmSjG4bteA= -github.com/livekit/protocol v1.5.5-0.20230421192204-0975cb52f603 h1:1O+cSgXWFWZDF+6cYOmJhu4w2HAsQcpLJI1YJfuixR8= -github.com/livekit/protocol v1.5.5-0.20230421192204-0975cb52f603/go.mod h1:/LpoK6XF8IeJDs/d5BPVmicp8pDPXmgTN1GVGhdpEX0= +github.com/livekit/protocol v1.5.5-0.20230422131440-eb3ef0f4bf36 h1:Qrl0N7dAeR2iLOk6u4WNelFhTh2OetyZzd/h8kbxLSI= +github.com/livekit/protocol v1.5.5-0.20230422131440-eb3ef0f4bf36/go.mod h1:iZ289+6H5xn/9kP2iqpRvVWxuc8GXBMqN0qI7LdN9HI= github.com/livekit/psrpc v0.3.0 h1:giBZsfM3CWA0oIYXofsMITbVQtyW7u/ES9sQmVspHPM= github.com/livekit/psrpc v0.3.0/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= @@ -281,8 +281,8 @@ golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ= golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= -golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug= -golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20230420155640-133eef4313cb h1:rhjz/8Mbfa8xROFiH+MQphmAmgqRM0bOMnytznhWEXk= +golang.org/x/exp v0.0.0-20230420155640-133eef4313cb/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 86c721ac7..532030aac 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -144,9 +144,12 @@ type ParticipantImpl struct { rttUpdatedAt time.Time lastRTT uint32 - lock utils.RWMutex - once sync.Once - version atomic.Uint32 + lock utils.RWMutex + once sync.Once + + dirty atomic.Bool + version atomic.Uint32 + timedVersion utils.TimedVersion // callbacks & handlers onTrackPublished func(types.LocalParticipant, types.MediaTrack) @@ -194,6 +197,7 @@ func NewParticipant(params ParticipantParams) (*ParticipantImpl, error) { supervisor: supervisor.NewParticipantSupervisor(supervisor.ParticipantSupervisorParams{Logger: params.Logger}), } p.version.Store(params.InitialVersion) + p.timedVersion.Update(params.VersionGenerator.New()) p.migrateState.Store(types.MigrateStateInit) p.state.Store(livekit.ParticipantInfo_JOINING) p.grants = params.Grants @@ -291,16 +295,18 @@ func (p *ParticipantImpl) GetBufferFactory() *buffer.Factory { // SetName attaches name to the participant func (p *ParticipantImpl) SetName(name string) { p.lock.Lock() - changed := p.grants.Name != name + if p.grants.Name == name { + p.lock.Unlock() + return + } + p.grants.Name = name + p.dirty.Store(true) + onParticipantUpdate := p.onParticipantUpdate onClaimsChanged := p.onClaimsChanged p.lock.Unlock() - if !changed { - return - } - if onParticipantUpdate != nil { onParticipantUpdate(p) } @@ -312,16 +318,18 @@ func (p *ParticipantImpl) SetName(name string) { // SetMetadata attaches metadata to the participant func (p *ParticipantImpl) SetMetadata(metadata string) { p.lock.Lock() - changed := p.grants.Metadata != metadata + if p.grants.Metadata == metadata { + p.lock.Unlock() + return + } + p.grants.Metadata = metadata + p.dirty.Store(true) + onParticipantUpdate := p.onParticipantUpdate onClaimsChanged := p.onClaimsChanged p.lock.Unlock() - if !changed { - return - } - if onParticipantUpdate != nil { onParticipantUpdate(p) } @@ -350,6 +358,7 @@ func (p *ParticipantImpl) SetPermission(permission *livekit.ParticipantPermissio } video.UpdateFromPermission(permission) + p.dirty.Store(true) canPublish := video.GetCanPublish() canSubscribe := video.GetCanSubscribe() @@ -392,24 +401,37 @@ func (p *ParticipantImpl) SetPermission(permission *livekit.ParticipantPermissio return true } -func (p *ParticipantImpl) ToProto() *livekit.ParticipantInfo { +func (p *ParticipantImpl) ToProtoWithVersion() (*livekit.ParticipantInfo, utils.TimedVersion) { + v := p.version.Load() + piv := p.timedVersion.Load() + if p.dirty.Swap(false) { + v = p.version.Inc() + piv = p.params.VersionGenerator.Next() + p.timedVersion.Update(&piv) + } + p.lock.RLock() - info := &livekit.ParticipantInfo{ + pi := &livekit.ParticipantInfo{ Sid: string(p.params.SID), Identity: string(p.params.Identity), Name: p.grants.Name, State: p.State(), JoinedAt: p.ConnectedAt().Unix(), - Version: p.version.Inc(), + Version: v, Permission: p.grants.Video.ToPermission(), Metadata: p.grants.Metadata, Region: p.params.Region, IsPublisher: p.IsPublisher(), } p.lock.RUnlock() - info.Tracks = p.UpTrackManager.ToProto() + pi.Tracks = p.UpTrackManager.ToProto() - return info + return pi, piv +} + +func (p *ParticipantImpl) ToProto() *livekit.ParticipantInfo { + pi, _ := p.ToProtoWithVersion() + return pi } // callbacks for clients @@ -545,6 +567,10 @@ func (p *ParticipantImpl) handleMigrateMutedTrack() { } } p.mutedTrackNotFired = append(p.mutedTrackNotFired, addedTracks...) + + if len(addedTracks) != 0 { + p.dirty.Store(true) + } p.pendingTracksLock.Unlock() // launch callbacks in goroutine since they could block. @@ -754,6 +780,7 @@ func (p *ParticipantImpl) SetMigrateState(s types.MigrateState) { p.params.Logger.Debugw("SetMigrateState", "state", s) p.migrateState.Store(s) + p.dirty.Store(true) processPendingOffer := false if s == types.MigrateStateSync { @@ -1054,6 +1081,7 @@ func (p *ParticipantImpl) setupUpTrackManager() { }) p.UpTrackManager.OnPublishedTrackUpdated(func(track types.MediaTrack) { + p.dirty.Store(true) p.lock.RLock() onTrackUpdated := p.onTrackUpdated p.lock.RUnlock() @@ -1084,8 +1112,11 @@ func (p *ParticipantImpl) updateState(state livekit.ParticipantInfo_State) { if state == oldState { return } - p.state.Store(state) + p.params.Logger.Debugw("updating participant state", "state", state.String()) + p.state.Store(state) + p.dirty.Store(true) + p.lock.RLock() onStateChange := p.onStateChange p.lock.RUnlock() @@ -1103,6 +1134,8 @@ func (p *ParticipantImpl) updateState(state livekit.ParticipantInfo_State) { func (p *ParticipantImpl) setIsPublisher(isPublisher bool) { if p.isPublisher.Swap(isPublisher) != isPublisher { + p.dirty.Store(true) + // trigger update as well if participant is already fully connected if p.State() == livekit.ParticipantInfo_ACTIVE { p.lock.RLock() @@ -1164,6 +1197,8 @@ func (p *ParticipantImpl) onMediaTrack(track *webrtc.TrackRemote, rtpReceiver *w "mime", track.Codec().MimeType, ) + p.dirty.Store(true) + if !isNewTrack && !publishedTrack.HasPendingCodec() && p.IsReady() { p.lock.RLock() onTrackUpdated := p.onTrackUpdated @@ -1515,6 +1550,7 @@ func (p *ParticipantImpl) SetTrackMuted(trackID livekit.TrackID, muted bool, fro } func (p *ParticipantImpl) setTrackMuted(trackID livekit.TrackID, muted bool) { + p.dirty.Store(true) p.supervisor.SetPublicationMute(trackID, muted) track := p.UpTrackManager.SetPublishedTrackMuted(trackID, muted) @@ -1579,6 +1615,7 @@ func (p *ParticipantImpl) mediaTrackReceived(track *webrtc.TrackRemote, rtpRecei ti.MimeType = track.Codec().MimeType mt = p.addMediaTrack(signalCid, track.ID(), ti) newTrack = true + p.dirty.Store(true) } ssrc := uint32(track.SSRC()) @@ -1699,6 +1736,8 @@ func (p *ParticipantImpl) addMediaTrack(signalCid string, sdpCid string, ti *liv } p.pendingTracksLock.Unlock() + p.dirty.Store(true) + if !p.IsClosed() { // unpublished events aren't necessary when participant is closed p.params.Logger.Infow("unpublished track", "trackID", ti.Sid, "trackInfo", ti) diff --git a/pkg/rtc/participant_internal_test.go b/pkg/rtc/participant_internal_test.go index 3c585c3b0..74e0b38e0 100644 --- a/pkg/rtc/participant_internal_test.go +++ b/pkg/rtc/participant_internal_test.go @@ -683,6 +683,7 @@ func newParticipantForTestWithOpts(identity livekit.ParticipantIdentity, opts *p ClientInfo: ClientInfo{ClientInfo: opts.clientInfo}, Logger: LoggerWithParticipant(logger.GetLogger(), identity, sid, false), Telemetry: &telemetryfakes.FakeTelemetryService{}, + VersionGenerator: utils.NewDefaultTimedVersionGenerator(), }) p.isPublisher.Store(opts.publisher) p.updateState(livekit.ParticipantInfo_ACTIVE) diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index 911191027..e9d8b65d8 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -526,7 +526,7 @@ func (r *Room) SyncState(participant types.LocalParticipant, state *livekit.Sync } func (r *Room) UpdateSubscriptionPermission(participant types.LocalParticipant, subscriptionPermission *livekit.SubscriptionPermission) error { - if err := participant.UpdateSubscriptionPermission(subscriptionPermission, nil, r.GetParticipant, r.GetParticipantByID); err != nil { + if err := participant.UpdateSubscriptionPermission(subscriptionPermission, utils.TimedVersion{}, r.GetParticipant, r.GetParticipantByID); err != nil { return err } for _, track := range participant.GetPublishedTracks() { diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index d2dfe1f3a..83b22b833 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -10,6 +10,7 @@ import ( "github.com/livekit/protocol/auth" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/utils" "github.com/livekit/livekit-server/pkg/routing" "github.com/livekit/livekit-server/pkg/sfu" @@ -195,12 +196,12 @@ type Participant interface { Start() Close(sendLeave bool, reason ParticipantCloseReason) error - SubscriptionPermission() (*livekit.SubscriptionPermission, *livekit.TimedVersion) + SubscriptionPermission() (*livekit.SubscriptionPermission, utils.TimedVersion) // updates from remotes UpdateSubscriptionPermission( subscriptionPermission *livekit.SubscriptionPermission, - timedVersion *livekit.TimedVersion, + timedVersion utils.TimedVersion, resolverByIdentity func(participantIdentity livekit.ParticipantIdentity) LocalParticipant, resolverBySid func(participantID livekit.ParticipantID) LocalParticipant, ) error @@ -229,6 +230,8 @@ type AddTrackParams struct { type LocalParticipant interface { Participant + ToProtoWithVersion() (*livekit.ParticipantInfo, utils.TimedVersion) + // getters GetLogger() logger.Logger GetAdaptiveStream() bool diff --git a/pkg/rtc/types/typesfakes/fake_local_participant.go b/pkg/rtc/types/typesfakes/fake_local_participant.go index d9e73aa64..f4966a5c5 100644 --- a/pkg/rtc/types/typesfakes/fake_local_participant.go +++ b/pkg/rtc/types/typesfakes/fake_local_participant.go @@ -12,6 +12,7 @@ import ( "github.com/livekit/protocol/auth" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/utils" "github.com/pion/rtcp" webrtc "github.com/pion/webrtc/v3" ) @@ -339,6 +340,10 @@ type FakeLocalParticipant struct { identityReturnsOnCall map[int]struct { result1 livekit.ParticipantIdentity } + InvalidateVersionStub func() + invalidateVersionMutex sync.RWMutex + invalidateVersionArgsForCall []struct { + } IsClosedStub func() bool isClosedMutex sync.RWMutex isClosedArgsForCall []struct { @@ -705,17 +710,17 @@ type FakeLocalParticipant struct { subscriberAsPrimaryReturnsOnCall map[int]struct { result1 bool } - SubscriptionPermissionStub func() (*livekit.SubscriptionPermission, *livekit.TimedVersion) + SubscriptionPermissionStub func() (*livekit.SubscriptionPermission, utils.TimedVersion) subscriptionPermissionMutex sync.RWMutex subscriptionPermissionArgsForCall []struct { } subscriptionPermissionReturns struct { result1 *livekit.SubscriptionPermission - result2 *livekit.TimedVersion + result2 utils.TimedVersion } subscriptionPermissionReturnsOnCall map[int]struct { result1 *livekit.SubscriptionPermission - result2 *livekit.TimedVersion + result2 utils.TimedVersion } SubscriptionPermissionUpdateStub func(livekit.ParticipantID, livekit.TrackID, bool) subscriptionPermissionUpdateMutex sync.RWMutex @@ -734,6 +739,18 @@ type FakeLocalParticipant struct { toProtoReturnsOnCall map[int]struct { result1 *livekit.ParticipantInfo } + ToProtoWithVersionStub func() (*livekit.ParticipantInfo, utils.TimedVersion) + toProtoWithVersionMutex sync.RWMutex + toProtoWithVersionArgsForCall []struct { + } + toProtoWithVersionReturns struct { + result1 *livekit.ParticipantInfo + result2 utils.TimedVersion + } + toProtoWithVersionReturnsOnCall map[int]struct { + result1 *livekit.ParticipantInfo + result2 utils.TimedVersion + } UncacheDownTrackStub func(*webrtc.RTPTransceiver) uncacheDownTrackMutex sync.RWMutex uncacheDownTrackArgsForCall []struct { @@ -790,11 +807,11 @@ type FakeLocalParticipant struct { arg1 livekit.TrackID arg2 *livekit.UpdateTrackSettings } - UpdateSubscriptionPermissionStub func(*livekit.SubscriptionPermission, *livekit.TimedVersion, func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, func(participantID livekit.ParticipantID) types.LocalParticipant) error + UpdateSubscriptionPermissionStub func(*livekit.SubscriptionPermission, utils.TimedVersion, func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, func(participantID livekit.ParticipantID) types.LocalParticipant) error updateSubscriptionPermissionMutex sync.RWMutex updateSubscriptionPermissionArgsForCall []struct { arg1 *livekit.SubscriptionPermission - arg2 *livekit.TimedVersion + arg2 utils.TimedVersion arg3 func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant arg4 func(participantID livekit.ParticipantID) types.LocalParticipant } @@ -2520,6 +2537,30 @@ func (fake *FakeLocalParticipant) IdentityReturnsOnCall(i int, result1 livekit.P }{result1} } +func (fake *FakeLocalParticipant) InvalidateVersion() { + fake.invalidateVersionMutex.Lock() + fake.invalidateVersionArgsForCall = append(fake.invalidateVersionArgsForCall, struct { + }{}) + stub := fake.InvalidateVersionStub + fake.recordInvocation("InvalidateVersion", []interface{}{}) + fake.invalidateVersionMutex.Unlock() + if stub != nil { + fake.InvalidateVersionStub() + } +} + +func (fake *FakeLocalParticipant) InvalidateVersionCallCount() int { + fake.invalidateVersionMutex.RLock() + defer fake.invalidateVersionMutex.RUnlock() + return len(fake.invalidateVersionArgsForCall) +} + +func (fake *FakeLocalParticipant) InvalidateVersionCalls(stub func()) { + fake.invalidateVersionMutex.Lock() + defer fake.invalidateVersionMutex.Unlock() + fake.InvalidateVersionStub = stub +} + func (fake *FakeLocalParticipant) IsClosed() bool { fake.isClosedMutex.Lock() ret, specificReturn := fake.isClosedReturnsOnCall[len(fake.isClosedArgsForCall)] @@ -4613,7 +4654,7 @@ func (fake *FakeLocalParticipant) SubscriberAsPrimaryReturnsOnCall(i int, result }{result1} } -func (fake *FakeLocalParticipant) SubscriptionPermission() (*livekit.SubscriptionPermission, *livekit.TimedVersion) { +func (fake *FakeLocalParticipant) SubscriptionPermission() (*livekit.SubscriptionPermission, utils.TimedVersion) { fake.subscriptionPermissionMutex.Lock() ret, specificReturn := fake.subscriptionPermissionReturnsOnCall[len(fake.subscriptionPermissionArgsForCall)] fake.subscriptionPermissionArgsForCall = append(fake.subscriptionPermissionArgsForCall, struct { @@ -4637,35 +4678,35 @@ func (fake *FakeLocalParticipant) SubscriptionPermissionCallCount() int { return len(fake.subscriptionPermissionArgsForCall) } -func (fake *FakeLocalParticipant) SubscriptionPermissionCalls(stub func() (*livekit.SubscriptionPermission, *livekit.TimedVersion)) { +func (fake *FakeLocalParticipant) SubscriptionPermissionCalls(stub func() (*livekit.SubscriptionPermission, utils.TimedVersion)) { fake.subscriptionPermissionMutex.Lock() defer fake.subscriptionPermissionMutex.Unlock() fake.SubscriptionPermissionStub = stub } -func (fake *FakeLocalParticipant) SubscriptionPermissionReturns(result1 *livekit.SubscriptionPermission, result2 *livekit.TimedVersion) { +func (fake *FakeLocalParticipant) SubscriptionPermissionReturns(result1 *livekit.SubscriptionPermission, result2 utils.TimedVersion) { fake.subscriptionPermissionMutex.Lock() defer fake.subscriptionPermissionMutex.Unlock() fake.SubscriptionPermissionStub = nil fake.subscriptionPermissionReturns = struct { result1 *livekit.SubscriptionPermission - result2 *livekit.TimedVersion + result2 utils.TimedVersion }{result1, result2} } -func (fake *FakeLocalParticipant) SubscriptionPermissionReturnsOnCall(i int, result1 *livekit.SubscriptionPermission, result2 *livekit.TimedVersion) { +func (fake *FakeLocalParticipant) SubscriptionPermissionReturnsOnCall(i int, result1 *livekit.SubscriptionPermission, result2 utils.TimedVersion) { fake.subscriptionPermissionMutex.Lock() defer fake.subscriptionPermissionMutex.Unlock() fake.SubscriptionPermissionStub = nil if fake.subscriptionPermissionReturnsOnCall == nil { fake.subscriptionPermissionReturnsOnCall = make(map[int]struct { result1 *livekit.SubscriptionPermission - result2 *livekit.TimedVersion + result2 utils.TimedVersion }) } fake.subscriptionPermissionReturnsOnCall[i] = struct { result1 *livekit.SubscriptionPermission - result2 *livekit.TimedVersion + result2 utils.TimedVersion }{result1, result2} } @@ -4756,6 +4797,62 @@ func (fake *FakeLocalParticipant) ToProtoReturnsOnCall(i int, result1 *livekit.P }{result1} } +func (fake *FakeLocalParticipant) ToProtoWithVersion() (*livekit.ParticipantInfo, utils.TimedVersion) { + fake.toProtoWithVersionMutex.Lock() + ret, specificReturn := fake.toProtoWithVersionReturnsOnCall[len(fake.toProtoWithVersionArgsForCall)] + fake.toProtoWithVersionArgsForCall = append(fake.toProtoWithVersionArgsForCall, struct { + }{}) + stub := fake.ToProtoWithVersionStub + fakeReturns := fake.toProtoWithVersionReturns + fake.recordInvocation("ToProtoWithVersion", []interface{}{}) + fake.toProtoWithVersionMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeLocalParticipant) ToProtoWithVersionCallCount() int { + fake.toProtoWithVersionMutex.RLock() + defer fake.toProtoWithVersionMutex.RUnlock() + return len(fake.toProtoWithVersionArgsForCall) +} + +func (fake *FakeLocalParticipant) ToProtoWithVersionCalls(stub func() (*livekit.ParticipantInfo, utils.TimedVersion)) { + fake.toProtoWithVersionMutex.Lock() + defer fake.toProtoWithVersionMutex.Unlock() + fake.ToProtoWithVersionStub = stub +} + +func (fake *FakeLocalParticipant) ToProtoWithVersionReturns(result1 *livekit.ParticipantInfo, result2 utils.TimedVersion) { + fake.toProtoWithVersionMutex.Lock() + defer fake.toProtoWithVersionMutex.Unlock() + fake.ToProtoWithVersionStub = nil + fake.toProtoWithVersionReturns = struct { + result1 *livekit.ParticipantInfo + result2 utils.TimedVersion + }{result1, result2} +} + +func (fake *FakeLocalParticipant) ToProtoWithVersionReturnsOnCall(i int, result1 *livekit.ParticipantInfo, result2 utils.TimedVersion) { + fake.toProtoWithVersionMutex.Lock() + defer fake.toProtoWithVersionMutex.Unlock() + fake.ToProtoWithVersionStub = nil + if fake.toProtoWithVersionReturnsOnCall == nil { + fake.toProtoWithVersionReturnsOnCall = make(map[int]struct { + result1 *livekit.ParticipantInfo + result2 utils.TimedVersion + }) + } + fake.toProtoWithVersionReturnsOnCall[i] = struct { + result1 *livekit.ParticipantInfo + result2 utils.TimedVersion + }{result1, result2} +} + func (fake *FakeLocalParticipant) UncacheDownTrack(arg1 *webrtc.RTPTransceiver) { fake.uncacheDownTrackMutex.Lock() fake.uncacheDownTrackArgsForCall = append(fake.uncacheDownTrackArgsForCall, struct { @@ -5072,12 +5169,12 @@ func (fake *FakeLocalParticipant) UpdateSubscribedTrackSettingsArgsForCall(i int return argsForCall.arg1, argsForCall.arg2 } -func (fake *FakeLocalParticipant) UpdateSubscriptionPermission(arg1 *livekit.SubscriptionPermission, arg2 *livekit.TimedVersion, arg3 func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, arg4 func(participantID livekit.ParticipantID) types.LocalParticipant) error { +func (fake *FakeLocalParticipant) UpdateSubscriptionPermission(arg1 *livekit.SubscriptionPermission, arg2 utils.TimedVersion, arg3 func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, arg4 func(participantID livekit.ParticipantID) types.LocalParticipant) error { fake.updateSubscriptionPermissionMutex.Lock() ret, specificReturn := fake.updateSubscriptionPermissionReturnsOnCall[len(fake.updateSubscriptionPermissionArgsForCall)] fake.updateSubscriptionPermissionArgsForCall = append(fake.updateSubscriptionPermissionArgsForCall, struct { arg1 *livekit.SubscriptionPermission - arg2 *livekit.TimedVersion + arg2 utils.TimedVersion arg3 func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant arg4 func(participantID livekit.ParticipantID) types.LocalParticipant }{arg1, arg2, arg3, arg4}) @@ -5100,13 +5197,13 @@ func (fake *FakeLocalParticipant) UpdateSubscriptionPermissionCallCount() int { return len(fake.updateSubscriptionPermissionArgsForCall) } -func (fake *FakeLocalParticipant) UpdateSubscriptionPermissionCalls(stub func(*livekit.SubscriptionPermission, *livekit.TimedVersion, func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, func(participantID livekit.ParticipantID) types.LocalParticipant) error) { +func (fake *FakeLocalParticipant) UpdateSubscriptionPermissionCalls(stub func(*livekit.SubscriptionPermission, utils.TimedVersion, func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, func(participantID livekit.ParticipantID) types.LocalParticipant) error) { fake.updateSubscriptionPermissionMutex.Lock() defer fake.updateSubscriptionPermissionMutex.Unlock() fake.UpdateSubscriptionPermissionStub = stub } -func (fake *FakeLocalParticipant) UpdateSubscriptionPermissionArgsForCall(i int) (*livekit.SubscriptionPermission, *livekit.TimedVersion, func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, func(participantID livekit.ParticipantID) types.LocalParticipant) { +func (fake *FakeLocalParticipant) UpdateSubscriptionPermissionArgsForCall(i int) (*livekit.SubscriptionPermission, utils.TimedVersion, func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, func(participantID livekit.ParticipantID) types.LocalParticipant) { fake.updateSubscriptionPermissionMutex.RLock() defer fake.updateSubscriptionPermissionMutex.RUnlock() argsForCall := fake.updateSubscriptionPermissionArgsForCall[i] @@ -5360,6 +5457,8 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.iDMutex.RUnlock() fake.identityMutex.RLock() defer fake.identityMutex.RUnlock() + fake.invalidateVersionMutex.RLock() + defer fake.invalidateVersionMutex.RUnlock() fake.isClosedMutex.RLock() defer fake.isClosedMutex.RUnlock() fake.isDisconnectedMutex.RLock() @@ -5462,6 +5561,8 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.subscriptionPermissionUpdateMutex.RUnlock() fake.toProtoMutex.RLock() defer fake.toProtoMutex.RUnlock() + fake.toProtoWithVersionMutex.RLock() + defer fake.toProtoWithVersionMutex.RUnlock() fake.uncacheDownTrackMutex.RLock() defer fake.uncacheDownTrackMutex.RUnlock() fake.unsubscribeFromTrackMutex.RLock() diff --git a/pkg/rtc/types/typesfakes/fake_participant.go b/pkg/rtc/types/typesfakes/fake_participant.go index efacc7a5b..29c8cb032 100644 --- a/pkg/rtc/types/typesfakes/fake_participant.go +++ b/pkg/rtc/types/typesfakes/fake_participant.go @@ -6,6 +6,7 @@ import ( "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/utils" ) type FakeParticipant struct { @@ -135,17 +136,17 @@ type FakeParticipant struct { stateReturnsOnCall map[int]struct { result1 livekit.ParticipantInfo_State } - SubscriptionPermissionStub func() (*livekit.SubscriptionPermission, *livekit.TimedVersion) + SubscriptionPermissionStub func() (*livekit.SubscriptionPermission, utils.TimedVersion) subscriptionPermissionMutex sync.RWMutex subscriptionPermissionArgsForCall []struct { } subscriptionPermissionReturns struct { result1 *livekit.SubscriptionPermission - result2 *livekit.TimedVersion + result2 utils.TimedVersion } subscriptionPermissionReturnsOnCall map[int]struct { result1 *livekit.SubscriptionPermission - result2 *livekit.TimedVersion + result2 utils.TimedVersion } ToProtoStub func() *livekit.ParticipantInfo toProtoMutex sync.RWMutex @@ -157,11 +158,11 @@ type FakeParticipant struct { toProtoReturnsOnCall map[int]struct { result1 *livekit.ParticipantInfo } - UpdateSubscriptionPermissionStub func(*livekit.SubscriptionPermission, *livekit.TimedVersion, func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, func(participantID livekit.ParticipantID) types.LocalParticipant) error + UpdateSubscriptionPermissionStub func(*livekit.SubscriptionPermission, utils.TimedVersion, func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, func(participantID livekit.ParticipantID) types.LocalParticipant) error updateSubscriptionPermissionMutex sync.RWMutex updateSubscriptionPermissionArgsForCall []struct { arg1 *livekit.SubscriptionPermission - arg2 *livekit.TimedVersion + arg2 utils.TimedVersion arg3 func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant arg4 func(participantID livekit.ParticipantID) types.LocalParticipant } @@ -864,7 +865,7 @@ func (fake *FakeParticipant) StateReturnsOnCall(i int, result1 livekit.Participa }{result1} } -func (fake *FakeParticipant) SubscriptionPermission() (*livekit.SubscriptionPermission, *livekit.TimedVersion) { +func (fake *FakeParticipant) SubscriptionPermission() (*livekit.SubscriptionPermission, utils.TimedVersion) { fake.subscriptionPermissionMutex.Lock() ret, specificReturn := fake.subscriptionPermissionReturnsOnCall[len(fake.subscriptionPermissionArgsForCall)] fake.subscriptionPermissionArgsForCall = append(fake.subscriptionPermissionArgsForCall, struct { @@ -888,35 +889,35 @@ func (fake *FakeParticipant) SubscriptionPermissionCallCount() int { return len(fake.subscriptionPermissionArgsForCall) } -func (fake *FakeParticipant) SubscriptionPermissionCalls(stub func() (*livekit.SubscriptionPermission, *livekit.TimedVersion)) { +func (fake *FakeParticipant) SubscriptionPermissionCalls(stub func() (*livekit.SubscriptionPermission, utils.TimedVersion)) { fake.subscriptionPermissionMutex.Lock() defer fake.subscriptionPermissionMutex.Unlock() fake.SubscriptionPermissionStub = stub } -func (fake *FakeParticipant) SubscriptionPermissionReturns(result1 *livekit.SubscriptionPermission, result2 *livekit.TimedVersion) { +func (fake *FakeParticipant) SubscriptionPermissionReturns(result1 *livekit.SubscriptionPermission, result2 utils.TimedVersion) { fake.subscriptionPermissionMutex.Lock() defer fake.subscriptionPermissionMutex.Unlock() fake.SubscriptionPermissionStub = nil fake.subscriptionPermissionReturns = struct { result1 *livekit.SubscriptionPermission - result2 *livekit.TimedVersion + result2 utils.TimedVersion }{result1, result2} } -func (fake *FakeParticipant) SubscriptionPermissionReturnsOnCall(i int, result1 *livekit.SubscriptionPermission, result2 *livekit.TimedVersion) { +func (fake *FakeParticipant) SubscriptionPermissionReturnsOnCall(i int, result1 *livekit.SubscriptionPermission, result2 utils.TimedVersion) { fake.subscriptionPermissionMutex.Lock() defer fake.subscriptionPermissionMutex.Unlock() fake.SubscriptionPermissionStub = nil if fake.subscriptionPermissionReturnsOnCall == nil { fake.subscriptionPermissionReturnsOnCall = make(map[int]struct { result1 *livekit.SubscriptionPermission - result2 *livekit.TimedVersion + result2 utils.TimedVersion }) } fake.subscriptionPermissionReturnsOnCall[i] = struct { result1 *livekit.SubscriptionPermission - result2 *livekit.TimedVersion + result2 utils.TimedVersion }{result1, result2} } @@ -973,12 +974,12 @@ func (fake *FakeParticipant) ToProtoReturnsOnCall(i int, result1 *livekit.Partic }{result1} } -func (fake *FakeParticipant) UpdateSubscriptionPermission(arg1 *livekit.SubscriptionPermission, arg2 *livekit.TimedVersion, arg3 func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, arg4 func(participantID livekit.ParticipantID) types.LocalParticipant) error { +func (fake *FakeParticipant) UpdateSubscriptionPermission(arg1 *livekit.SubscriptionPermission, arg2 utils.TimedVersion, arg3 func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, arg4 func(participantID livekit.ParticipantID) types.LocalParticipant) error { fake.updateSubscriptionPermissionMutex.Lock() ret, specificReturn := fake.updateSubscriptionPermissionReturnsOnCall[len(fake.updateSubscriptionPermissionArgsForCall)] fake.updateSubscriptionPermissionArgsForCall = append(fake.updateSubscriptionPermissionArgsForCall, struct { arg1 *livekit.SubscriptionPermission - arg2 *livekit.TimedVersion + arg2 utils.TimedVersion arg3 func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant arg4 func(participantID livekit.ParticipantID) types.LocalParticipant }{arg1, arg2, arg3, arg4}) @@ -1001,13 +1002,13 @@ func (fake *FakeParticipant) UpdateSubscriptionPermissionCallCount() int { return len(fake.updateSubscriptionPermissionArgsForCall) } -func (fake *FakeParticipant) UpdateSubscriptionPermissionCalls(stub func(*livekit.SubscriptionPermission, *livekit.TimedVersion, func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, func(participantID livekit.ParticipantID) types.LocalParticipant) error) { +func (fake *FakeParticipant) UpdateSubscriptionPermissionCalls(stub func(*livekit.SubscriptionPermission, utils.TimedVersion, func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, func(participantID livekit.ParticipantID) types.LocalParticipant) error) { fake.updateSubscriptionPermissionMutex.Lock() defer fake.updateSubscriptionPermissionMutex.Unlock() fake.UpdateSubscriptionPermissionStub = stub } -func (fake *FakeParticipant) UpdateSubscriptionPermissionArgsForCall(i int) (*livekit.SubscriptionPermission, *livekit.TimedVersion, func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, func(participantID livekit.ParticipantID) types.LocalParticipant) { +func (fake *FakeParticipant) UpdateSubscriptionPermissionArgsForCall(i int) (*livekit.SubscriptionPermission, utils.TimedVersion, func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, func(participantID livekit.ParticipantID) types.LocalParticipant) { fake.updateSubscriptionPermissionMutex.RLock() defer fake.updateSubscriptionPermissionMutex.RUnlock() argsForCall := fake.updateSubscriptionPermissionArgsForCall[i] diff --git a/pkg/rtc/uptrackmanager.go b/pkg/rtc/uptrackmanager.go index 76c380a5b..10e75227c 100644 --- a/pkg/rtc/uptrackmanager.go +++ b/pkg/rtc/uptrackmanager.go @@ -30,7 +30,7 @@ type UpTrackManager struct { // publishedTracks that participant is publishing publishedTracks map[livekit.TrackID]types.MediaTrack subscriptionPermission *livekit.SubscriptionPermission - subscriptionPermissionVersion *utils.TimedVersion + subscriptionPermissionVersion utils.TimedVersion // subscriber permission for published tracks subscriberPermissions map[livekit.ParticipantIdentity]*livekit.TrackPermission // subscriberIdentity => *livekit.TrackPermission @@ -127,47 +127,36 @@ func (u *UpTrackManager) GetPublishedTracks() []types.MediaTrack { func (u *UpTrackManager) UpdateSubscriptionPermission( subscriptionPermission *livekit.SubscriptionPermission, - timedVersion *livekit.TimedVersion, + timedVersion utils.TimedVersion, resolverByIdentity func(participantIdentity livekit.ParticipantIdentity) types.LocalParticipant, resolverBySid func(participantID livekit.ParticipantID) types.LocalParticipant, ) error { u.lock.Lock() - if timedVersion != nil { + if !timedVersion.IsZero() { // it's possible for permission updates to come from another node. In that case // they would be the authority for this participant's permissions // we do not want to initialize subscriptionPermissionVersion too early since if another machine is the // owner for the data, we'd prefer to use their TimedVersion - if u.subscriptionPermissionVersion != nil { - tv := utils.NewTimedVersionFromProto(timedVersion) - // ignore older version - if !tv.After(u.subscriptionPermissionVersion) { - perms := "" - if u.subscriptionPermission != nil { - perms = u.subscriptionPermission.String() - } - u.params.Logger.Debugw( - "skipping older subscription permission version", - "existingValue", perms, - "existingVersion", u.subscriptionPermissionVersion.ToProto().String(), - "requestingValue", subscriptionPermission.String(), - "requestingVersion", timedVersion.String(), - ) - u.lock.Unlock() - return nil + // ignore older version + if !timedVersion.After(&u.subscriptionPermissionVersion) { + perms := "" + if u.subscriptionPermission != nil { + perms = u.subscriptionPermission.String() } - u.subscriptionPermissionVersion.Update(tv) - } else { - u.subscriptionPermissionVersion = utils.NewTimedVersionFromProto(timedVersion) + u.params.Logger.Debugw( + "skipping older subscription permission version", + "existingValue", perms, + "existingVersion", u.subscriptionPermissionVersion.ToProto().String(), + "requestingValue", subscriptionPermission.String(), + "requestingVersion", timedVersion.String(), + ) + u.lock.Unlock() + return nil } + u.subscriptionPermissionVersion.Update(&timedVersion) } else { // for requests coming from the current node, use local versions - tv := u.params.VersionGenerator.New() - // use current time as the new/updated version - if u.subscriptionPermissionVersion == nil { - u.subscriptionPermissionVersion = tv - } else { - u.subscriptionPermissionVersion.Update(tv) - } + u.subscriptionPermissionVersion.Update(u.params.VersionGenerator.New()) } // store as is for use when migrating @@ -205,15 +194,15 @@ func (u *UpTrackManager) UpdateSubscriptionPermission( return nil } -func (u *UpTrackManager) SubscriptionPermission() (*livekit.SubscriptionPermission, *livekit.TimedVersion) { +func (u *UpTrackManager) SubscriptionPermission() (*livekit.SubscriptionPermission, utils.TimedVersion) { u.lock.RLock() defer u.lock.RUnlock() - if u.subscriptionPermissionVersion == nil { - return nil, nil + if u.subscriptionPermissionVersion.IsZero() { + return nil, u.subscriptionPermissionVersion.Load() } - return u.subscriptionPermission, u.subscriptionPermissionVersion.ToProto() + return u.subscriptionPermission, u.subscriptionPermissionVersion.Load() } func (u *UpTrackManager) HasPermission(trackID livekit.TrackID, subIdentity livekit.ParticipantIdentity) bool { diff --git a/pkg/rtc/uptrackmanager_test.go b/pkg/rtc/uptrackmanager_test.go index 88e4c2ee7..e75d92881 100644 --- a/pkg/rtc/uptrackmanager_test.go +++ b/pkg/rtc/uptrackmanager_test.go @@ -21,6 +21,7 @@ var defaultUptrackManagerParams = UpTrackManagerParams{ func TestUpdateSubscriptionPermission(t *testing.T) { t.Run("updates subscription permission", func(t *testing.T) { um := NewUpTrackManager(defaultUptrackManagerParams) + vg := utils.NewDefaultTimedVersionGenerator() tra := &typesfakes.FakeMediaTrack{} tra.IDReturns("audio") @@ -34,14 +35,14 @@ func TestUpdateSubscriptionPermission(t *testing.T) { subscriptionPermission := &livekit.SubscriptionPermission{ AllParticipants: true, } - um.UpdateSubscriptionPermission(subscriptionPermission, nil, nil, nil) + um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil, nil) require.Nil(t, um.subscriberPermissions) // nobody is allowed to subscribe subscriptionPermission = &livekit.SubscriptionPermission{ TrackPermissions: []*livekit.TrackPermission{}, } - um.UpdateSubscriptionPermission(subscriptionPermission, nil, nil, nil) + um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil, nil) require.NotNil(t, um.subscriberPermissions) require.Equal(t, 0, len(um.subscriberPermissions)) @@ -77,7 +78,7 @@ func TestUpdateSubscriptionPermission(t *testing.T) { perms2, }, } - um.UpdateSubscriptionPermission(subscriptionPermission, nil, nil, sidResolver) + um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil, sidResolver) require.Equal(t, 2, len(um.subscriberPermissions)) require.EqualValues(t, perms1, um.subscriberPermissions["p1"]) require.EqualValues(t, perms2, um.subscriberPermissions["p2"]) @@ -102,7 +103,7 @@ func TestUpdateSubscriptionPermission(t *testing.T) { perms3, }, } - um.UpdateSubscriptionPermission(subscriptionPermission, nil, nil, nil) + um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil, nil) require.Equal(t, 3, len(um.subscriberPermissions)) require.EqualValues(t, perms1, um.subscriberPermissions["p1"]) require.EqualValues(t, perms2, um.subscriberPermissions["p2"]) @@ -111,6 +112,7 @@ func TestUpdateSubscriptionPermission(t *testing.T) { t.Run("updates subscription permission using both", func(t *testing.T) { um := NewUpTrackManager(defaultUptrackManagerParams) + vg := utils.NewDefaultTimedVersionGenerator() tra := &typesfakes.FakeMediaTrack{} tra.IDReturns("audio") @@ -154,7 +156,7 @@ func TestUpdateSubscriptionPermission(t *testing.T) { perms2, }, } - err := um.UpdateSubscriptionPermission(subscriptionPermission, nil, nil, sidResolver) + err := um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil, sidResolver) require.NoError(t, err) require.Equal(t, 2, len(um.subscriberPermissions)) require.EqualValues(t, perms1, um.subscriberPermissions["p1"]) @@ -173,17 +175,37 @@ func TestUpdateSubscriptionPermission(t *testing.T) { return nil } - err = um.UpdateSubscriptionPermission(subscriptionPermission, nil, nil, badSidResolver) + err = um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil, badSidResolver) require.NoError(t, err) require.Equal(t, 2, len(um.subscriberPermissions)) require.EqualValues(t, perms1, um.subscriberPermissions["p1"]) require.EqualValues(t, perms2, um.subscriberPermissions["p2"]) }) + + t.Run("update versions", func(t *testing.T) { + um := NewUpTrackManager(defaultUptrackManagerParams) + vg := utils.NewDefaultTimedVersionGenerator() + + v0, v1, v2 := vg.Next(), vg.Next(), vg.Next() + + um.UpdateSubscriptionPermission(&livekit.SubscriptionPermission{}, v1, nil, nil) + require.Equal(t, v1.Load(), um.subscriptionPermissionVersion.Load(), "first update should be applied") + + um.UpdateSubscriptionPermission(&livekit.SubscriptionPermission{}, v2, nil, nil) + require.Equal(t, v2.Load(), um.subscriptionPermissionVersion.Load(), "ordered updates should be applied") + + um.UpdateSubscriptionPermission(&livekit.SubscriptionPermission{}, v0, nil, nil) + require.Equal(t, v2.Load(), um.subscriptionPermissionVersion.Load(), "out of order updates should be ignored") + + um.UpdateSubscriptionPermission(&livekit.SubscriptionPermission{}, utils.TimedVersion{}, nil, nil) + require.True(t, um.subscriptionPermissionVersion.After(&v2), "zero version in updates should use next local version") + }) } func TestSubscriptionPermission(t *testing.T) { t.Run("checks subscription permission", func(t *testing.T) { um := NewUpTrackManager(defaultUptrackManagerParams) + vg := utils.NewDefaultTimedVersionGenerator() tra := &typesfakes.FakeMediaTrack{} tra.IDReturns("audio") @@ -197,7 +219,7 @@ func TestSubscriptionPermission(t *testing.T) { subscriptionPermission := &livekit.SubscriptionPermission{ AllParticipants: true, } - um.UpdateSubscriptionPermission(subscriptionPermission, nil, nil, nil) + um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil, nil) require.True(t, um.hasPermissionLocked("audio", "p1")) require.True(t, um.hasPermissionLocked("audio", "p2")) @@ -205,7 +227,7 @@ func TestSubscriptionPermission(t *testing.T) { subscriptionPermission = &livekit.SubscriptionPermission{ TrackPermissions: []*livekit.TrackPermission{}, } - um.UpdateSubscriptionPermission(subscriptionPermission, nil, nil, nil) + um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil, nil) require.False(t, um.hasPermissionLocked("audio", "p1")) require.False(t, um.hasPermissionLocked("audio", "p2")) @@ -222,7 +244,7 @@ func TestSubscriptionPermission(t *testing.T) { }, }, } - um.UpdateSubscriptionPermission(subscriptionPermission, nil, nil, nil) + um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil, nil) require.True(t, um.hasPermissionLocked("audio", "p1")) require.True(t, um.hasPermissionLocked("video", "p1")) require.True(t, um.hasPermissionLocked("audio", "p2")) @@ -257,7 +279,7 @@ func TestSubscriptionPermission(t *testing.T) { }, }, } - um.UpdateSubscriptionPermission(subscriptionPermission, nil, nil, nil) + um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil, nil) require.True(t, um.hasPermissionLocked("audio", "p1")) require.True(t, um.hasPermissionLocked("video", "p1")) require.True(t, um.hasPermissionLocked("screen", "p1")) From 3f64828a77e6f06aae96934616e1d9d4d54fa333 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Sat, 22 Apr 2023 21:08:59 -0700 Subject: [PATCH 107/324] Send Room updates when participant counts change (#1647) Reduces the number of unneeded generation with ProtoProxy --- go.mod | 2 +- go.sum | 4 +- pkg/rtc/room.go | 99 +++++++++++--------- pkg/rtc/room_test.go | 25 ++++- pkg/rtc/types/interfaces.go | 2 +- pkg/rtc/types/typesfakes/fake_participant.go | 65 +++++++++++++ pkg/service/roommanager.go | 2 +- 7 files changed, 150 insertions(+), 49 deletions(-) diff --git a/go.mod b/go.mod index ec383e2e8..e64a54ef9 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 - github.com/livekit/protocol v1.5.5-0.20230422131440-eb3ef0f4bf36 + github.com/livekit/protocol v1.5.5 github.com/livekit/psrpc v0.3.0 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.14.0 diff --git a/go.sum b/go.sum index fa24f3bd0..dd6b0e056 100644 --- a/go.sum +++ b/go.sum @@ -121,8 +121,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 h1:QlQFyMwCDgjyySsrgmrMcVbEBA6KZcyTzvK+z346tUA= github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26/go.mod h1:eDA41kiySZoG+wy4Etsjb3w0jjLx69i/vAmSjG4bteA= -github.com/livekit/protocol v1.5.5-0.20230422131440-eb3ef0f4bf36 h1:Qrl0N7dAeR2iLOk6u4WNelFhTh2OetyZzd/h8kbxLSI= -github.com/livekit/protocol v1.5.5-0.20230422131440-eb3ef0f4bf36/go.mod h1:iZ289+6H5xn/9kP2iqpRvVWxuc8GXBMqN0qI7LdN9HI= +github.com/livekit/protocol v1.5.5 h1:vuSU3TI/w58WnAWnyC59nMzY/JE+ZznU6W/iRgWw4JQ= +github.com/livekit/protocol v1.5.5/go.mod h1:iZ289+6H5xn/9kP2iqpRvVWxuc8GXBMqN0qI7LdN9HI= github.com/livekit/psrpc v0.3.0 h1:giBZsfM3CWA0oIYXofsMITbVQtyW7u/ES9sQmVspHPM= github.com/livekit/psrpc v0.3.0/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index e9d8b65d8..5a58ebe3c 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -37,6 +37,7 @@ const ( var ( // var to allow unit test override RoomDepartureGrace uint32 = 20 + roomUpdateInterval = 5 * time.Second // frequency to update room participant counts ) type broadcastOptions struct { @@ -47,9 +48,10 @@ type broadcastOptions struct { type Room struct { lock sync.RWMutex - protoRoom *livekit.Room - internal *livekit.RoomInternal - Logger logger.Logger + protoRoom *livekit.Room + internal *livekit.RoomInternal + protoProxy *utils.ProtoProxy[*livekit.Room] + Logger logger.Logger config WebRTCConfig audioConfig *config.AudioConfig @@ -76,7 +78,7 @@ type Room struct { closed chan struct{} onParticipantChanged func(p types.LocalParticipant) - onMetadataUpdate func(metadata string) + onRoomUpdated func() onClose func() } @@ -110,6 +112,7 @@ func NewRoom( batchedUpdates: make(map[livekit.ParticipantIdentity]*livekit.ParticipantInfo), closed: make(chan struct{}), } + r.protoProxy = utils.NewProtoProxy[*livekit.Room](roomUpdateInterval, r.updateProto) if r.protoRoom.EmptyTimeout == 0 { r.protoRoom.EmptyTimeout = DefaultEmptyTimeout } @@ -119,16 +122,13 @@ func NewRoom( go r.audioUpdateWorker() go r.connectionQualityWorker() - go r.subscriberBroadcastWorker() + go r.changeUpdateWorker() return r } func (r *Room) ToProto() *livekit.Room { - r.lock.RLock() - defer r.lock.RUnlock() - - return proto.Clone(r.protoRoom).(*livekit.Room) + return r.protoProxy.Get() } func (r *Room) Name() livekit.RoomName { @@ -244,14 +244,13 @@ func (r *Room) Join(participant types.LocalParticipant, requestSource routing.Me } if r.protoRoom.MaxParticipants > 0 && !participant.IsRecorder() { - participantCount := 0 + numParticipants := uint32(0) for _, p := range r.participants { if !p.IsRecorder() { - participantCount++ + numParticipants++ } } - - if participantCount >= int(r.protoRoom.MaxParticipants) { + if numParticipants >= r.protoRoom.MaxParticipants { return ErrMaxParticipantsExceeded } } @@ -259,9 +258,6 @@ func (r *Room) Join(participant types.LocalParticipant, requestSource routing.Me if r.FirstJoinedAt() == 0 { r.joinedAt.Store(time.Now().Unix()) } - if !participant.Hidden() { - r.protoRoom.NumParticipants++ - } // it's important to set this before connection, we don't want to miss out on any published tracks participant.OnTrackPublished(r.onTrackPublished) @@ -340,7 +336,9 @@ func (r *Room) Join(participant types.LocalParticipant, requestSource routing.Me if participant.IsRecorder() && !r.protoRoom.ActiveRecording { r.protoRoom.ActiveRecording = true - r.sendRoomUpdateLocked() + r.protoProxy.MarkDirty(true) + } else { + r.protoProxy.MarkDirty(false) } r.participants[participant.Identity()] = participant @@ -419,10 +417,7 @@ func (r *Room) ResumeParticipant(p types.LocalParticipant, requestSource routing return err } - r.lock.RLock() - p.SendRoomUpdate(r.protoRoom) - r.lock.RUnlock() - + p.SendRoomUpdate(r.ToProto()) p.ICERestart(nil) return nil } @@ -445,6 +440,7 @@ func (r *Room) RemoveParticipant(identity livekit.ParticipantIdentity, pID livek } } + immediateChange := false if (p != nil && p.IsRecorder()) || r.protoRoom.ActiveRecording { activeRecording := false for _, op := range r.participants { @@ -456,10 +452,12 @@ func (r *Room) RemoveParticipant(identity livekit.ParticipantIdentity, pID livek if r.protoRoom.ActiveRecording != activeRecording { r.protoRoom.ActiveRecording = activeRecording - r.sendRoomUpdateLocked() + immediateChange = true + } } r.lock.Unlock() + r.protoProxy.MarkDirty(immediateChange) if !ok { return @@ -634,6 +632,7 @@ func (r *Room) Close() { for _, p := range r.GetParticipants() { _ = p.Close(true, types.ParticipantCloseReasonRoomClose) } + r.protoProxy.Stop() if r.onClose != nil { r.onClose() } @@ -661,32 +660,26 @@ func (r *Room) SetMetadata(metadata string) { r.lock.Lock() r.protoRoom.Metadata = metadata r.lock.Unlock() - - r.lock.RLock() - r.sendRoomUpdateLocked() - r.lock.RUnlock() - - if r.onMetadataUpdate != nil { - r.onMetadataUpdate(metadata) - } + r.protoProxy.MarkDirty(true) } -func (r *Room) sendRoomUpdateLocked() { +func (r *Room) sendRoomUpdate() { + roomInfo := r.ToProto() // Send update to participants - for _, p := range r.participants { + for _, p := range r.GetParticipants() { if !p.IsReady() { continue } - err := p.SendRoomUpdate(r.protoRoom) + err := p.SendRoomUpdate(roomInfo) if err != nil { r.Logger.Warnw("failed to send room update", err, "participant", p.Identity()) } } } -func (r *Room) OnMetadataUpdate(f func(metadata string)) { - r.onMetadataUpdate = f +func (r *Room) OnRoomUpdated(f func()) { + r.onRoomUpdated = f } func (r *Room) SimulateScenario(participant types.LocalParticipant, simulateScenario *livekit.SimulateScenario) error { @@ -760,11 +753,9 @@ func (r *Room) createJoinResponseLocked(participant types.LocalParticipant, iceS } return &livekit.JoinResponse{ - Room: r.protoRoom, + Room: r.ToProto(), Participant: participant.ToProto(), OtherParticipants: otherParticipants, - ServerVersion: r.serverInfo.Version, - ServerRegion: r.serverInfo.Region, IceServers: iceServers, // indicates both server and client support subscriber as primary SubscriberPrimary: participant.SubscriberAsPrimary(), @@ -1003,15 +994,39 @@ func (r *Room) pushAndDequeueUpdates(pi *livekit.ParticipantInfo, isImmediate bo return updates } -func (r *Room) subscriberBroadcastWorker() { - ticker := time.NewTicker(subscriberUpdateInterval) - defer ticker.Stop() +func (r *Room) updateProto() *livekit.Room { + r.lock.RLock() + room := proto.Clone(r.protoRoom).(*livekit.Room) + r.lock.RUnlock() + + room.NumPublishers = 0 + room.NumParticipants = 0 + for _, p := range r.GetParticipants() { + if !p.IsRecorder() { + room.NumParticipants++ + } + if p.IsPublisher() { + room.NumPublishers++ + } + } + + return room +} + +func (r *Room) changeUpdateWorker() { + subTicker := time.NewTicker(subscriberUpdateInterval) + defer subTicker.Stop() for !r.IsClosed() { select { case <-r.closed: return - case <-ticker.C: + case <-r.protoProxy.Updated(): + if r.onRoomUpdated != nil { + r.onRoomUpdated() + } + r.sendRoomUpdate() + case <-subTicker.C: r.batchedUpdatesMu.Lock() updatesMap := r.batchedUpdates r.batchedUpdates = make(map[livekit.ParticipantIdentity]*livekit.ParticipantInfo) diff --git a/pkg/rtc/room_test.go b/pkg/rtc/room_test.go index 8d93367bf..f95fb5e2c 100644 --- a/pkg/rtc/room_test.go +++ b/pkg/rtc/room_test.go @@ -35,6 +35,7 @@ func init() { }) // allow immediate closure in testing RoomDepartureGrace = 1 + roomUpdateInterval = defaultDelay } var iceServersForRoom = []*livekit.ICEServer{{Urls: []string{"stun:stun.l.google.com:19302"}}} @@ -645,7 +646,7 @@ func TestHiddenParticipants(t *testing.T) { require.Len(t, res.OtherParticipants, 2) require.Len(t, rm.GetParticipants(), 4) require.NotEmpty(t, res.IceServers) - require.Equal(t, "testregion", res.ServerRegion) + require.Equal(t, "testregion", res.ServerInfo.Region) }) t.Run("hidden participant subscribes to tracks", func(t *testing.T) { @@ -665,15 +666,35 @@ func TestHiddenParticipants(t *testing.T) { } func TestRoomUpdate(t *testing.T) { + t.Run("updates are sent when participant joined", func(t *testing.T) { + rm := newRoomWithParticipants(t, testRoomOpts{num: 1}) + defer rm.Close() + + p1 := rm.GetParticipants()[0].(*typesfakes.FakeLocalParticipant) + require.Equal(t, 0, p1.SendRoomUpdateCallCount()) + + p2 := newMockParticipant("p2", types.CurrentProtocol, false, false) + require.NoError(t, rm.Join(p2, nil, nil, iceServersForRoom)) + + // p1 should have received an update + time.Sleep(2 * defaultDelay) + require.Equal(t, 1, p1.SendRoomUpdateCallCount()) + require.EqualValues(t, 2, p1.SendRoomUpdateArgsForCall(0).NumParticipants) + }) + t.Run("participants should receive metadata update", func(t *testing.T) { rm := newRoomWithParticipants(t, testRoomOpts{num: 2}) defer rm.Close() rm.SetMetadata("test metadata...") + // callbacks are updated from goroutine + time.Sleep(2 * defaultDelay) + for _, op := range rm.GetParticipants() { fp := op.(*typesfakes.FakeLocalParticipant) - require.Equal(t, 1, fp.SendRoomUpdateCallCount()) + // room updates are now sent for both participant joining and room metadata + require.GreaterOrEqual(t, fp.SendRoomUpdateCallCount(), 1) } }) } diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index 83b22b833..1163cc1d2 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -181,6 +181,7 @@ type Participant interface { SetName(name string) SetMetadata(metadata string) + IsPublisher() bool GetPublishedTrack(sid livekit.TrackID) MediaTrack GetPublishedTracks() []MediaTrack RemovePublishedTrack(track MediaTrack, willBeResumed bool, shouldClose bool) @@ -284,7 +285,6 @@ type LocalParticipant interface { // returns list of participant identities that the current participant is subscribed to GetSubscribedParticipants() []livekit.ParticipantID IsSubscribedTo(sid livekit.ParticipantID) bool - IsPublisher() bool GetAudioLevel() (smoothedLevel float64, active bool) GetConnectionQuality() *livekit.ConnectionQualityInfo diff --git a/pkg/rtc/types/typesfakes/fake_participant.go b/pkg/rtc/types/typesfakes/fake_participant.go index 29c8cb032..404af0a17 100644 --- a/pkg/rtc/types/typesfakes/fake_participant.go +++ b/pkg/rtc/types/typesfakes/fake_participant.go @@ -95,6 +95,16 @@ type FakeParticipant struct { identityReturnsOnCall map[int]struct { result1 livekit.ParticipantIdentity } + IsPublisherStub func() bool + isPublisherMutex sync.RWMutex + isPublisherArgsForCall []struct { + } + isPublisherReturns struct { + result1 bool + } + isPublisherReturnsOnCall map[int]struct { + result1 bool + } IsRecorderStub func() bool isRecorderMutex sync.RWMutex isRecorderArgsForCall []struct { @@ -637,6 +647,59 @@ func (fake *FakeParticipant) IdentityReturnsOnCall(i int, result1 livekit.Partic }{result1} } +func (fake *FakeParticipant) IsPublisher() bool { + fake.isPublisherMutex.Lock() + ret, specificReturn := fake.isPublisherReturnsOnCall[len(fake.isPublisherArgsForCall)] + fake.isPublisherArgsForCall = append(fake.isPublisherArgsForCall, struct { + }{}) + stub := fake.IsPublisherStub + fakeReturns := fake.isPublisherReturns + fake.recordInvocation("IsPublisher", []interface{}{}) + fake.isPublisherMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeParticipant) IsPublisherCallCount() int { + fake.isPublisherMutex.RLock() + defer fake.isPublisherMutex.RUnlock() + return len(fake.isPublisherArgsForCall) +} + +func (fake *FakeParticipant) IsPublisherCalls(stub func() bool) { + fake.isPublisherMutex.Lock() + defer fake.isPublisherMutex.Unlock() + fake.IsPublisherStub = stub +} + +func (fake *FakeParticipant) IsPublisherReturns(result1 bool) { + fake.isPublisherMutex.Lock() + defer fake.isPublisherMutex.Unlock() + fake.IsPublisherStub = nil + fake.isPublisherReturns = struct { + result1 bool + }{result1} +} + +func (fake *FakeParticipant) IsPublisherReturnsOnCall(i int, result1 bool) { + fake.isPublisherMutex.Lock() + defer fake.isPublisherMutex.Unlock() + fake.IsPublisherStub = nil + if fake.isPublisherReturnsOnCall == nil { + fake.isPublisherReturnsOnCall = make(map[int]struct { + result1 bool + }) + } + fake.isPublisherReturnsOnCall[i] = struct { + result1 bool + }{result1} +} + func (fake *FakeParticipant) IsRecorder() bool { fake.isRecorderMutex.Lock() ret, specificReturn := fake.isRecorderReturnsOnCall[len(fake.isRecorderArgsForCall)] @@ -1118,6 +1181,8 @@ func (fake *FakeParticipant) Invocations() map[string][][]interface{} { defer fake.iDMutex.RUnlock() fake.identityMutex.RLock() defer fake.identityMutex.RUnlock() + fake.isPublisherMutex.RLock() + defer fake.isPublisherMutex.RUnlock() fake.isRecorderMutex.RLock() defer fake.isRecorderMutex.RUnlock() fake.removePublishedTrackMutex.RLock() diff --git a/pkg/service/roommanager.go b/pkg/service/roommanager.go index 0244abb84..4a63ea4e1 100644 --- a/pkg/service/roommanager.go +++ b/pkg/service/roommanager.go @@ -451,7 +451,7 @@ func (r *RoomManager) getOrCreateRoom(ctx context.Context, roomName livekit.Room newRoom.Logger.Infow("room closed") }) - newRoom.OnMetadataUpdate(func(metadata string) { + newRoom.OnRoomUpdated(func() { if err := r.roomStore.StoreRoom(ctx, newRoom.ToProto(), newRoom.Internal()); err != nil { newRoom.Logger.Errorw("could not handle metadata update", err) } From cf0fcc33f3bfc392caabfd1794fe7b8d708a4c7b Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 22 Apr 2023 21:15:48 -0700 Subject: [PATCH 108/324] Update module github.com/rs/cors to v1.9.0 (#1644) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e64a54ef9..fd38e3264 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.15.0 github.com/redis/go-redis/v9 v9.0.3 - github.com/rs/cors v1.8.3 + github.com/rs/cors v1.9.0 github.com/stretchr/testify v1.8.2 github.com/thoas/go-funk v0.9.3 github.com/twitchtv/twirp v8.1.3+incompatible diff --git a/go.sum b/go.sum index dd6b0e056..e7d56672f 100644 --- a/go.sum +++ b/go.sum @@ -231,8 +231,8 @@ github.com/redis/go-redis/v9 v9.0.3 h1:+7mmR26M0IvyLxGZUHxu4GiBkJkVDid0Un+j4ScYu github.com/redis/go-redis/v9 v9.0.3/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= -github.com/rs/cors v1.8.3 h1:O+qNyWn7Z+F9M0ILBHgMVPuB1xTOucVd5gtaYyXBpRo= -github.com/rs/cors v1.8.3/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= +github.com/rs/cors v1.9.0 h1:l9HGsTsHJcvW14Nk7J9KFz8bzeAWXn3CG6bgt7LsrAE= +github.com/rs/cors v1.9.0/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= From 1cb189a3dddbc7f65b95d983fcc2632981ec3867 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 22 Apr 2023 21:16:11 -0700 Subject: [PATCH 109/324] Update module github.com/pion/transport/v2 to v2.2.0 (#1626) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index fd38e3264..ff4c85b7f 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/pion/rtp v1.7.13 github.com/pion/sdp/v3 v3.0.6 github.com/pion/stun v0.4.0 - github.com/pion/transport/v2 v2.0.2 + github.com/pion/transport/v2 v2.2.0 github.com/pion/turn/v2 v2.1.0 github.com/pion/webrtc/v3 v3.1.60 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index e7d56672f..4895d7034 100644 --- a/go.sum +++ b/go.sum @@ -207,8 +207,9 @@ github.com/pion/stun v0.4.0/go.mod h1:QPsh1/SbXASntw3zkkrIk3ZJVKz4saBY2G7S10P3wC github.com/pion/transport v0.14.1 h1:XSM6olwW+o8J4SCmOBb/BpwZypkHeyM0PGFCxNQBr40= github.com/pion/transport v0.14.1/go.mod h1:4tGmbk00NeYA3rUa9+n+dzCCoKkcy3YlYb99Jn2fNnI= github.com/pion/transport/v2 v2.0.0/go.mod h1:HS2MEBJTwD+1ZI2eSXSvHJx/HnzQqRy2/LXxt6eVMHc= -github.com/pion/transport/v2 v2.0.2 h1:St+8o+1PEzPT51O9bv+tH/KYYLMNR5Vwm5Z3Qkjsywg= github.com/pion/transport/v2 v2.0.2/go.mod h1:vrz6bUbFr/cjdwbnxq8OdDDzHf7JJfGsIRkxfpZoTA0= +github.com/pion/transport/v2 v2.2.0 h1:u5lFqFHkXLMXMzai8tixZDfVjb8eOjH35yCunhPeb1c= +github.com/pion/transport/v2 v2.2.0/go.mod h1:AdSw4YBZVDkZm8fpoz+fclXyQwANWmZAlDuQdctTThQ= github.com/pion/turn/v2 v2.1.0 h1:5wGHSgGhJhP/RpabkUb/T9PdsAjkGLS6toYz5HNzoSI= github.com/pion/turn/v2 v2.1.0/go.mod h1:yrT5XbXSGX1VFSF31A3c1kCNB5bBZgk/uu5LET162qs= github.com/pion/udp/v2 v2.0.1 h1:xP0z6WNux1zWEjhC7onRA3EwwSliXqu1ElUZAQhUP54= From 279b3604c36b01a5a69b617903678f9ffb1f2cdb Mon Sep 17 00:00:00 2001 From: David Zhao Date: Sun, 23 Apr 2023 23:10:00 -0700 Subject: [PATCH 110/324] Add back ServerRegion and ServerVersion (#1650) clients are still dependent on them --- pkg/rtc/room.go | 8 +++++--- pkg/rtc/room_test.go | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index 5a58ebe3c..7c789cba4 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -761,9 +761,11 @@ func (r *Room) createJoinResponseLocked(participant types.LocalParticipant, iceS SubscriberPrimary: participant.SubscriberAsPrimary(), ClientConfiguration: participant.GetClientConfiguration(), // sane defaults for ping interval & timeout - PingInterval: 10, - PingTimeout: 20, - ServerInfo: r.serverInfo, + PingInterval: 10, + PingTimeout: 20, + ServerInfo: r.serverInfo, + ServerVersion: r.serverInfo.Version, + ServerRegion: r.serverInfo.Region, } } diff --git a/pkg/rtc/room_test.go b/pkg/rtc/room_test.go index f95fb5e2c..80987785a 100644 --- a/pkg/rtc/room_test.go +++ b/pkg/rtc/room_test.go @@ -678,8 +678,8 @@ func TestRoomUpdate(t *testing.T) { // p1 should have received an update time.Sleep(2 * defaultDelay) - require.Equal(t, 1, p1.SendRoomUpdateCallCount()) - require.EqualValues(t, 2, p1.SendRoomUpdateArgsForCall(0).NumParticipants) + require.GreaterOrEqual(t, p1.SendRoomUpdateCallCount(), 1) + require.EqualValues(t, 2, p1.SendRoomUpdateArgsForCall(p1.SendRoomUpdateCallCount()-1).NumParticipants) }) t.Run("participants should receive metadata update", func(t *testing.T) { From 7dc8f8bcd3c599ea13520c889d205e0c633cc73d Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Sun, 23 Apr 2023 23:51:02 -0700 Subject: [PATCH 111/324] update psprc (#1651) --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index ff4c85b7f..83e9abbc2 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 github.com/livekit/protocol v1.5.5 - github.com/livekit/psrpc v0.3.0 + github.com/livekit/psrpc v0.3.1-0.20230424064451-65c6a2dd048b github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.14.0 github.com/maxbrunsfeld/counterfeiter/v6 v6.6.1 @@ -98,7 +98,7 @@ require ( golang.org/x/sys v0.7.0 // indirect golang.org/x/text v0.9.0 // indirect golang.org/x/tools v0.6.0 // indirect - google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd // indirect + google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect google.golang.org/grpc v1.54.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 4895d7034..dbb146402 100644 --- a/go.sum +++ b/go.sum @@ -123,8 +123,8 @@ github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 h1:QlQF github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26/go.mod h1:eDA41kiySZoG+wy4Etsjb3w0jjLx69i/vAmSjG4bteA= github.com/livekit/protocol v1.5.5 h1:vuSU3TI/w58WnAWnyC59nMzY/JE+ZznU6W/iRgWw4JQ= github.com/livekit/protocol v1.5.5/go.mod h1:iZ289+6H5xn/9kP2iqpRvVWxuc8GXBMqN0qI7LdN9HI= -github.com/livekit/psrpc v0.3.0 h1:giBZsfM3CWA0oIYXofsMITbVQtyW7u/ES9sQmVspHPM= -github.com/livekit/psrpc v0.3.0/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= +github.com/livekit/psrpc v0.3.1-0.20230424064451-65c6a2dd048b h1:WqRJoXeycdrhEuA9C0jJ7aN6i5wx8B3c+M0hIKnnFZE= +github.com/livekit/psrpc v0.3.1-0.20230424064451-65c6a2dd048b/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= github.com/mackerelio/go-osstat v0.2.4/go.mod h1:Zy+qzGdZs3A9cuIqmgbJvwbmLQH9dJvtio5ZjJTbdlQ= github.com/magefile/mage v1.14.0 h1:6QDX3g6z1YvJ4olPhT1wksUcSa/V0a1B+pJb73fBjyo= @@ -390,8 +390,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd h1:sLpv7bNL1AsX3fdnWh9WVh7ejIzXdOc1RRHGeAmeStU= -google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= From 09c0b25787df9a4f0f7366515c201b35a85f6228 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 24 Apr 2023 23:39:30 +0530 Subject: [PATCH 112/324] Ensure that RR is not received for a while before running scorer on nil (#1653) data. Without the check, it was getting tripped by publisher not publishing any data. Both conditions returned nil, but in one case, the receiver report should have been received, but no movement in number of packets. --- pkg/sfu/buffer/rtpstats.go | 2 +- pkg/sfu/connectionquality/connectionstats.go | 31 ++++++++++++++------ 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index f1176f5d2..3b9ec4a8d 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -869,7 +869,7 @@ func (r *RTPStats) DeltaInfo(snapshotId uint32) *RTPDeltaInfo { } if packetsExpected == 0 { if r.params.IsReceiverReportDriven { - // not received RTCP RR + // not received RTCP RR (OR) publisher is not producing any data return nil } diff --git a/pkg/sfu/connectionquality/connectionstats.go b/pkg/sfu/connectionquality/connectionstats.go index fb6caa91f..7eca5d515 100644 --- a/pkg/sfu/connectionquality/connectionstats.go +++ b/pkg/sfu/connectionquality/connectionstats.go @@ -15,9 +15,10 @@ import ( ) const ( - UpdateInterval = 5 * time.Second - processThreshold = 0.95 - noStatsTooLongMultiplier = 2 + UpdateInterval = 5 * time.Second + processThreshold = 0.95 + noStatsTooLongMultiplier = 2 + noReceiverReportTooLongThreshold = 10 * time.Second ) type ConnectionStatsParams struct { @@ -38,9 +39,10 @@ type ConnectionStats struct { onStatsUpdate func(cs *ConnectionStats, stat *livekit.AnalyticsStat) - lock sync.RWMutex - lastStatsAt time.Time - statsInProcess bool + lock sync.RWMutex + lastStatsAt time.Time + statsInProcess bool + lastReceiverReportAt time.Time scorer *qualityScorer @@ -103,6 +105,10 @@ func (cs *ConnectionStats) GetScoreAndQuality() (float32, livekit.ConnectionQual } func (cs *ConnectionStats) ReceiverReportReceived(at time.Time) { + cs.lock.Lock() + cs.lastReceiverReportAt = time.Now() + cs.lock.Unlock() + cs.getStat(at) } @@ -160,8 +166,8 @@ func (cs *ConnectionStats) updateLastStatsAt(at time.Time) { } func (cs *ConnectionStats) isTooLongSinceLastStats() bool { - cs.lock.Lock() - defer cs.lock.Unlock() + cs.lock.RLock() + defer cs.lock.RUnlock() interval := cs.params.UpdateInterval if interval == 0 { @@ -170,6 +176,13 @@ func (cs *ConnectionStats) isTooLongSinceLastStats() bool { return !cs.lastStatsAt.IsZero() && time.Since(cs.lastStatsAt) > interval*noStatsTooLongMultiplier } +func (cs *ConnectionStats) isTooLongSinceLastReceiverReport() bool { + cs.lock.RLock() + defer cs.lock.RUnlock() + + return !cs.lastReceiverReportAt.IsZero() && time.Since(cs.lastReceiverReportAt) > noReceiverReportTooLongThreshold +} + func (cs *ConnectionStats) clearInProcess() { cs.lock.Lock() defer cs.lock.Unlock() @@ -189,7 +202,7 @@ func (cs *ConnectionStats) getStat(at time.Time) { streams := cs.params.GetDeltaStats() if len(streams) == 0 { - if cs.isTooLongSinceLastStats() { + if cs.isTooLongSinceLastStats() && cs.isTooLongSinceLastReceiverReport() { cs.updateLastStatsAt(at) cs.updateScore(streams, at) } From b4ea4de5c07346e9a10c3f2c240a5ffe80220d24 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Mon, 24 Apr 2023 15:24:50 -0700 Subject: [PATCH 113/324] Skip room updates to participants unless they are active --- pkg/rtc/room.go | 6 ++++-- pkg/rtc/room_test.go | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index 7c789cba4..ba5d5db44 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -417,7 +417,7 @@ func (r *Room) ResumeParticipant(p types.LocalParticipant, requestSource routing return err } - p.SendRoomUpdate(r.ToProto()) + _ = p.SendRoomUpdate(r.ToProto()) p.ICERestart(nil) return nil } @@ -667,7 +667,9 @@ func (r *Room) sendRoomUpdate() { roomInfo := r.ToProto() // Send update to participants for _, p := range r.GetParticipants() { - if !p.IsReady() { + // new participants receive the update as part of JoinResponse + // skip inactive participants + if p.State() != livekit.ParticipantInfo_ACTIVE { continue } diff --git a/pkg/rtc/room_test.go b/pkg/rtc/room_test.go index 80987785a..35cc3caa9 100644 --- a/pkg/rtc/room_test.go +++ b/pkg/rtc/room_test.go @@ -678,7 +678,7 @@ func TestRoomUpdate(t *testing.T) { // p1 should have received an update time.Sleep(2 * defaultDelay) - require.GreaterOrEqual(t, p1.SendRoomUpdateCallCount(), 1) + require.Equal(t, 1, p1.SendRoomUpdateCallCount()) require.EqualValues(t, 2, p1.SendRoomUpdateArgsForCall(p1.SendRoomUpdateCallCount()-1).NumParticipants) }) From 8ed193b231aca1b145c1cba0d0bfb148c18e326a Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 25 Apr 2023 07:31:22 +0530 Subject: [PATCH 114/324] `StreamAllocator` - tracking more things (#1652) * WIP commit * Add a probe cluster mode * better variable naming * fix units * WIP commit * WIP commit * WIP commit * new file * WIP commit * Maintain history of a few things * correct signal * fix typo * WIP commmit * gofmt * rate not sum * adjust edges of rate monitor * fmt * remove debug --- go.mod | 4 +- go.sum | 8 +- pkg/sfu/downtrack.go | 60 +++++-- pkg/sfu/streamallocator/channelobserver.go | 6 +- pkg/sfu/streamallocator/nacktracker.go | 22 ++- pkg/sfu/streamallocator/prober.go | 127 +++++++++++---- pkg/sfu/streamallocator/ratemonitor.go | 158 ++++++++++++++++++ pkg/sfu/streamallocator/streamallocator.go | 119 +++++++++++++- pkg/sfu/streamallocator/track.go | 178 ++++++++++++++++++++- 9 files changed, 625 insertions(+), 57 deletions(-) create mode 100644 pkg/sfu/streamallocator/ratemonitor.go diff --git a/go.mod b/go.mod index 83e9abbc2..d8b16b22a 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 - github.com/livekit/protocol v1.5.5 + github.com/livekit/protocol v1.5.6-0.20230424073901-c54f5f7f4182 github.com/livekit/psrpc v0.3.1-0.20230424064451-65c6a2dd048b github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.14.0 @@ -35,7 +35,7 @@ require ( github.com/pion/stun v0.4.0 github.com/pion/transport/v2 v2.2.0 github.com/pion/turn/v2 v2.1.0 - github.com/pion/webrtc/v3 v3.1.60 + github.com/pion/webrtc/v3 v3.1.61 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.15.0 github.com/redis/go-redis/v9 v9.0.3 diff --git a/go.sum b/go.sum index dbb146402..d36f3ccad 100644 --- a/go.sum +++ b/go.sum @@ -121,8 +121,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 h1:QlQFyMwCDgjyySsrgmrMcVbEBA6KZcyTzvK+z346tUA= github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26/go.mod h1:eDA41kiySZoG+wy4Etsjb3w0jjLx69i/vAmSjG4bteA= -github.com/livekit/protocol v1.5.5 h1:vuSU3TI/w58WnAWnyC59nMzY/JE+ZznU6W/iRgWw4JQ= -github.com/livekit/protocol v1.5.5/go.mod h1:iZ289+6H5xn/9kP2iqpRvVWxuc8GXBMqN0qI7LdN9HI= +github.com/livekit/protocol v1.5.6-0.20230424073901-c54f5f7f4182 h1:rpaYN8Jy5F1ZhxH4Q61+6Cc42I/evI63SlSXUMZ+VdU= +github.com/livekit/protocol v1.5.6-0.20230424073901-c54f5f7f4182/go.mod h1:B7Ns8diIKB3y39oRHm7ZluU9ZGCxCWQT+uKcbY3MCG4= github.com/livekit/psrpc v0.3.1-0.20230424064451-65c6a2dd048b h1:WqRJoXeycdrhEuA9C0jJ7aN6i5wx8B3c+M0hIKnnFZE= github.com/livekit/psrpc v0.3.1-0.20230424064451-65c6a2dd048b/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= @@ -214,8 +214,8 @@ github.com/pion/turn/v2 v2.1.0 h1:5wGHSgGhJhP/RpabkUb/T9PdsAjkGLS6toYz5HNzoSI= github.com/pion/turn/v2 v2.1.0/go.mod h1:yrT5XbXSGX1VFSF31A3c1kCNB5bBZgk/uu5LET162qs= github.com/pion/udp/v2 v2.0.1 h1:xP0z6WNux1zWEjhC7onRA3EwwSliXqu1ElUZAQhUP54= github.com/pion/udp/v2 v2.0.1/go.mod h1:B7uvTMP00lzWdyMr/1PVZXtV3wpPIxBRd4Wl6AksXn8= -github.com/pion/webrtc/v3 v3.1.60 h1:FLF6HT3x3CMHtPz5JbdAARfIUpMZu2YeOSzkVxaeF+k= -github.com/pion/webrtc/v3 v3.1.60/go.mod h1:65gfOgxrmszb6ec7kEiZp32QwnmDNIrJK8hgo/0niWY= +github.com/pion/webrtc/v3 v3.1.61 h1:WG6p786t7jxXO/3miw6HmAQmO3p/n+QLRa2xLaovcr8= +github.com/pion/webrtc/v3 v3.1.61/go.mod h1:uk/4AJmgEUpSExaP7aexCyODwfbHap8hAnQzRV7zKcE= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 87498d987..538b524e1 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -119,6 +119,12 @@ func (d DownTrackState) String() string { // ------------------------------------------------------------------- +type NackInfo struct { + Timestamp uint32 + SequenceNumber uint16 + Attempts uint8 +} + type DownTrackStreamAllocatorListener interface { // RTCP received OnREMB(dt *DownTrack, remb *rtcp.ReceiverEstimatedMaximumBitrate) @@ -147,6 +153,12 @@ type DownTrackStreamAllocatorListener interface { // packet(s) sent OnPacketsSent(dt *DownTrack, size int) + + // NACKs received + OnNACK(dt *DownTrack, nackInfos []NackInfo) + + // RTCP Receiver Report received + OnRTCPReceiverReport(dt *DownTrack, rr rtcp.ReceptionReport) } type ReceiverReportListener func(dt *DownTrack, report *rtcp.ReceiverReport) @@ -198,8 +210,7 @@ type DownTrack struct { rtpStats *buffer.RTPStats - statsLock sync.RWMutex - totalRepeatedNACKs uint32 + totalRepeatedNACKs atomic.Uint32 keyFrameRequestGeneration atomic.Uint32 @@ -219,6 +230,8 @@ type DownTrack struct { streamAllocatorListener DownTrackStreamAllocatorListener streamAllocatorReportGeneration int streamAllocatorBytesCounter atomic.Uint32 + bytesSent atomic.Uint32 + bytesRetransmitted atomic.Uint32 // update stats onStatsUpdate func(dt *DownTrack, stat *livekit.AnalyticsStat) @@ -593,7 +606,9 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { return err } + // STREAM-ALLOCATOR-TODO: remove this stream allocator bytes counter once stream allocator changes fully to pull bytes counter d.streamAllocatorBytesCounter.Add(uint32(hdr.MarshalSize() + len(payload))) + d.bytesSent.Add(uint32(hdr.MarshalSize() + len(payload))) if tp.isSwitchingToMaxSpatial && d.onMaxSubscribedLayerChanged != nil && d.kind == webrtc.RTPCodecTypeVideo { d.onMaxSubscribedLayerChanged(d, layer) @@ -1166,6 +1181,7 @@ func (d *DownTrack) writeBlankFrameRTP(duration float32, generation uint32) chan } d.streamAllocatorBytesCounter.Add(uint32(pktSize)) + d.bytesSent.Add(uint32(pktSize)) // only the first frame will need frameEndNeeded to close out the // previous picture, rest are small key frames (for the video case) @@ -1314,6 +1330,10 @@ func (d *DownTrack) handleRTCP(bytes []byte) { if isRttChanged { rttToReport = rtt } + + if sal := d.getStreamAllocatorListener(); sal != nil { + sal.OnRTCPReceiverReport(d, r) + } } if len(rr.Reports) > 0 { d.listenerLock.RLock() @@ -1399,12 +1419,18 @@ func (d *DownTrack) retransmitPackets(nacks []uint16) { nackAcks := uint32(0) nackMisses := uint32(0) numRepeatedNACKs := uint32(0) + nackInfos := make([]NackInfo, 0, len(filtered)) for _, meta := range d.sequencer.getPacketsMeta(filtered) { if disallowedLayers[meta.layer] { continue } nackAcks++ + nackInfos = append(nackInfos, NackInfo{ + SequenceNumber: meta.targetSeqNo, + Timestamp: meta.timestamp, + Attempts: meta.nacked, + }) if pool != nil { PacketFactory.Put(pool) @@ -1465,16 +1491,30 @@ func (d *DownTrack) retransmitPackets(nacks []uint16) { d.logger.Errorw("writing rtx packet err", err) } else { d.streamAllocatorBytesCounter.Add(uint32(pkt.Header.MarshalSize() + len(payload))) + d.bytesRetransmitted.Add(uint32(pkt.Header.MarshalSize() + len(payload))) d.rtpStats.Update(&pkt.Header, len(payload), 0, time.Now().UnixNano()) } } - d.statsLock.Lock() - d.totalRepeatedNACKs += numRepeatedNACKs - d.statsLock.Unlock() + d.totalRepeatedNACKs.Add(numRepeatedNACKs) d.rtpStats.UpdateNackProcessed(nackAcks, nackMisses, numRepeatedNACKs) + // STREAM-ALLOCATOR-EXPERIMENTAL-TODO-START + // Need to check on the following + // - get all NACKs from sequencer even if SFU is not acknowledging, + // i. e. SFU does not acknowledge even same sequence number is NACKed too closely, + // but if sequencer return those also (even if not actually retransmitting), + // will that provide a signal? + // - get padding NACKs also? Maybe only look at them when their NACK count is 2? + // because padding runs in a separate path, it could get out of order with + // primary packets. So, it could be NACKed once. But, a repeat NACK means they + // were probably lost. But, as we do not retransmit padding packets, more than + // the second try does not provide any useful signal. + // STREAM-ALLOCATOR-EXPERIMENTAL-TODO-END + if sal := d.getStreamAllocatorListener(); sal != nil && len(nackInfos) != 0 { + sal.OnNACK(d, nackInfos) + } } type extensionData struct { @@ -1606,14 +1646,14 @@ func (d *DownTrack) getDeltaStats() map[uint32]*buffer.StreamStatsWithLayers { func (d *DownTrack) GetNackStats() (totalPackets uint32, totalRepeatedNACKs uint32) { totalPackets = d.rtpStats.GetTotalPacketsPrimary() - - d.statsLock.RLock() - totalRepeatedNACKs = d.totalRepeatedNACKs - d.statsLock.RUnlock() - + totalRepeatedNACKs = d.totalRepeatedNACKs.Load() return } +func (d *DownTrack) GetAndResetBytesSent() (uint32, uint32) { + return d.bytesSent.Swap(0), d.bytesRetransmitted.Swap(0) +} + func (d *DownTrack) onBindAndConnected() { if d.connected.Load() && d.bound.Load() && !d.bindAndConnectedOnce.Swap(true) { if d.kind == webrtc.RTPCodecTypeVideo { diff --git a/pkg/sfu/streamallocator/channelobserver.go b/pkg/sfu/streamallocator/channelobserver.go index 25dfe7cac..1960b6c2d 100644 --- a/pkg/sfu/streamallocator/channelobserver.go +++ b/pkg/sfu/streamallocator/channelobserver.go @@ -89,7 +89,7 @@ func NewChannelObserver(params ChannelObserverParams, logger logger.Logger) *Cha CollapseThreshold: params.EstimateCollapseThreshold, }), nackTracker: NewNackTracker(NackTrackerParams{ - Name: params.Name + "-estimate", + Name: params.Name + "-nack", Logger: logger, WindowMinDuration: params.NackWindowMinDuration, WindowMaxDuration: params.NackWindowMaxDuration, @@ -122,6 +122,10 @@ func (c *ChannelObserver) GetNackRatio() float64 { return c.nackTracker.GetRatio() } +func (c *ChannelObserver) GetNackHistory() []string { + return c.nackTracker.GetHistory() +} + func (c *ChannelObserver) GetTrend() (ChannelTrend, ChannelCongestionReason) { estimateDirection := c.estimateTrend.GetDirection() diff --git a/pkg/sfu/streamallocator/nacktracker.go b/pkg/sfu/streamallocator/nacktracker.go index cc91bfe98..74104d625 100644 --- a/pkg/sfu/streamallocator/nacktracker.go +++ b/pkg/sfu/streamallocator/nacktracker.go @@ -23,16 +23,22 @@ type NackTracker struct { windowStartTime time.Time packets uint32 repeatedNacks uint32 + + // STREAM-ALLOCATOR-EXPERIMENTAL-TODO: remove when cleaning up experimental stuff + history []string } func NewNackTracker(params NackTrackerParams) *NackTracker { return &NackTracker{ - params: params, + params: params, + history: make([]string, 0, 10), } } func (n *NackTracker) Add(packets uint32, repeatedNacks uint32) { if n.params.WindowMaxDuration != 0 && !n.windowStartTime.IsZero() && time.Since(n.windowStartTime) > n.params.WindowMaxDuration { + n.updateHistory() + n.windowStartTime = time.Time{} n.packets = 0 n.repeatedNacks = 0 @@ -81,7 +87,19 @@ func (n *NackTracker) ToString() string { elapsed := now.Sub(n.windowStartTime).Seconds() window = fmt.Sprintf("t: %+v|%+v|%.2fs", n.windowStartTime.Format(time.UnixDate), now.Format(time.UnixDate), elapsed) } - return fmt.Sprintf("n: %s, t: %s, p: %d, rn: %d, rn/p: %.2f", n.params.Name, window, n.packets, n.repeatedNacks, n.GetRatio()) + return fmt.Sprintf("n: %s, %s, p: %d, rn: %d, rn/p: %.2f", n.params.Name, window, n.packets, n.repeatedNacks, n.GetRatio()) +} + +func (n *NackTracker) GetHistory() []string { + return n.history +} + +func (n *NackTracker) updateHistory() { + if len(n.history) >= 10 { + n.history = n.history[1:] + } + + n.history = append(n.history, n.ToString()) } // ------------------------------------------------ diff --git a/pkg/sfu/streamallocator/prober.go b/pkg/sfu/streamallocator/prober.go index dd18401bd..a1b1a3205 100644 --- a/pkg/sfu/streamallocator/prober.go +++ b/pkg/sfu/streamallocator/prober.go @@ -195,13 +195,13 @@ func (p *Prober) Reset() { p.processActiveStateQueue() } -func (p *Prober) AddCluster(desiredRateBps int, expectedRateBps int, minDuration time.Duration, maxDuration time.Duration) ProbeClusterId { +func (p *Prober) AddCluster(mode ProbeClusterMode, desiredRateBps int, expectedRateBps int, minDuration time.Duration, maxDuration time.Duration) ProbeClusterId { if desiredRateBps <= 0 { return ProbeClusterIdInvalid } clusterId := ProbeClusterId(p.clusterId.Inc()) - cluster := NewCluster(clusterId, desiredRateBps, expectedRateBps, minDuration, maxDuration) + cluster := NewCluster(clusterId, mode, desiredRateBps, expectedRateBps, minDuration, maxDuration) p.logger.Debugw("cluster added", "cluster", cluster.String()) p.pushBackClusterAndMaybeStart(cluster) @@ -353,47 +353,117 @@ type ProbeClusterId uint32 const ( ProbeClusterIdInvalid ProbeClusterId = 0 + + bucketDuration = time.Second + bytesPerProbe = 1000 + minProbeRateBps = 10000 ) +// ----------------------------------- + +type ProbeClusterMode int + +const ( + ProbeClusterModeUniform ProbeClusterMode = iota + ProbeClusterModeLinearChirp +) + +func (p ProbeClusterMode) String() string { + switch p { + case ProbeClusterModeUniform: + return "UNIFORM" + case ProbeClusterModeLinearChirp: + return "LINEAR_CHIRP" + default: + return fmt.Sprintf("%d", int(p)) + } +} + +// --------------------------------------------------------------------------- + type ProbeClusterInfo struct { Id ProbeClusterId BytesSent int Duration time.Duration } +type clusterBucket struct { + desiredBytes int + desiredElapsedTime time.Duration + sleepDuration time.Duration +} + type Cluster struct { lock sync.RWMutex id ProbeClusterId + mode ProbeClusterMode desiredBytes int minDuration time.Duration maxDuration time.Duration - sleepDuration time.Duration + buckets []clusterBucket + bucketIdx int bytesSentProbe int bytesSentNonProbe int startTime time.Time } -func NewCluster(id ProbeClusterId, desiredRateBps int, expectedRateBps int, minDuration time.Duration, maxDuration time.Duration) *Cluster { - minDurationMs := minDuration.Milliseconds() - desiredBytes := int((int64(desiredRateBps)*minDurationMs/time.Second.Milliseconds() + 7) / 8) - expectedBytes := int((int64(expectedRateBps)*minDurationMs/time.Second.Milliseconds() + 7) / 8) - - // pace based on sending approximately 1000 bytes per probe - numProbes := (desiredBytes - expectedBytes + 999) / 1000 - sleepDurationMicroSeconds := int(float64(minDurationMs*1000)/float64(numProbes) + 0.5) +func NewCluster(id ProbeClusterId, mode ProbeClusterMode, desiredRateBps int, expectedRateBps int, minDuration time.Duration, maxDuration time.Duration) *Cluster { c := &Cluster{ - id: id, - desiredBytes: desiredBytes, - minDuration: minDuration, - maxDuration: maxDuration, - sleepDuration: time.Duration(sleepDurationMicroSeconds) * time.Microsecond, + id: id, + mode: mode, + minDuration: minDuration, + maxDuration: maxDuration, } + c.initBuckets(desiredRateBps, expectedRateBps, minDuration) + c.desiredBytes = c.buckets[len(c.buckets)-1].desiredBytes return c } +func (c *Cluster) initBuckets(desiredRateBps int, expectedRateBps int, minDuration time.Duration) { + // split into 1-second bucket + // NOTE: splitting even if mode is unitform + numBuckets := int((minDuration.Milliseconds() + bucketDuration.Milliseconds() - 1) / bucketDuration.Milliseconds()) + if numBuckets < 1 { + numBuckets = 1 + } + + expectedRateBytes := (expectedRateBps + 7) / 8 + baseProbeRateBps := (desiredRateBps - expectedRateBps + numBuckets - 1) / numBuckets + + runningDesiredBytes := 0 + runningDesiredElapsedTime := time.Duration(0) + + c.buckets = make([]clusterBucket, 0, numBuckets) + for bucketIdx := 0; bucketIdx < numBuckets; bucketIdx++ { + multiplier := numBuckets + if c.mode == ProbeClusterModeLinearChirp { + multiplier = bucketIdx + 1 + } + + bucketProbeRateBps := baseProbeRateBps * multiplier + if bucketProbeRateBps < minProbeRateBps { + bucketProbeRateBps = minProbeRateBps + } + bucketProbeRateBytes := (bucketProbeRateBps + 7) / 8 + + // pace based on bytes per probe + numProbes := (bucketProbeRateBytes + bytesPerProbe - 1) / bytesPerProbe + sleepDurationMicroSeconds := int(float64(1_000_000)/float64(numProbes) + 0.5) + + runningDesiredBytes += bucketProbeRateBytes + expectedRateBytes + runningDesiredElapsedTime += bucketDuration + + c.buckets = append(c.buckets, clusterBucket{ + desiredBytes: runningDesiredBytes, + desiredElapsedTime: runningDesiredElapsedTime, + sleepDuration: time.Duration(sleepDurationMicroSeconds) * time.Microsecond, + }) + } +} + func (c *Cluster) Start() { c.lock.Lock() defer c.lock.Unlock() @@ -407,7 +477,7 @@ func (c *Cluster) GetSleepDuration() time.Duration { c.lock.RLock() defer c.lock.RUnlock() - return c.sleepDuration + return c.buckets[c.bucketIdx].sleepDuration } func (c *Cluster) PacketsSent(size int) { @@ -456,7 +526,6 @@ func (c *Cluster) GetInfo() ProbeClusterInfo { func (c *Cluster) Process(pl ProberListener) { c.lock.RLock() - timeElapsed := time.Since(c.startTime) // Calculate number of probe bytes that should have been sent since start. @@ -464,14 +533,7 @@ func (c *Cluster) Process(pl ProberListener) { // However, it is possible that timeElapsed is more than minDuration due // to scheduling variance. When overshooting time budget, use a capped // short fall if there is a grace period given. - windowDone := float64(timeElapsed) / float64(c.minDuration) - if windowDone > 1.0 { - // cluster has been running for longer than minDuration - windowDone = 1.0 - } - - bytesShouldHaveBeenSent := int(windowDone * float64(c.desiredBytes)) - bytesShortFall := bytesShouldHaveBeenSent - c.bytesSentProbe - c.bytesSentNonProbe + bytesShortFall := c.buckets[c.bucketIdx].desiredBytes - c.bytesSentProbe - c.bytesSentNonProbe if bytesShortFall < 0 { bytesShortFall = 0 } @@ -482,6 +544,14 @@ func (c *Cluster) Process(pl ProberListener) { } // round up to packet size bytesShortFall = ((bytesShortFall + 274) / 275) * 275 + + // move to next bucket if necessary + if timeElapsed > c.buckets[c.bucketIdx].desiredElapsedTime { + c.bucketIdx++ + if c.bucketIdx >= len(c.buckets) { + c.bucketIdx = len(c.buckets) - 1 + } + } c.lock.RUnlock() if bytesShortFall > 0 && pl != nil { @@ -497,8 +567,9 @@ func (c *Cluster) String() string { activeTimeMs = time.Since(c.startTime).Milliseconds() } - return fmt.Sprintf("id: %d, bytes: desired %d / probe %d / non-probe %d / remaining: %d, time(ms): active %d / min %d / max %d", + return fmt.Sprintf("id: %d, mode: %s, bytes: desired %d / probe %d / non-probe %d / remaining: %d, time(ms): active %d / min %d / max %d", c.id, + c.mode, c.desiredBytes, c.bytesSentProbe, c.bytesSentNonProbe, @@ -507,3 +578,5 @@ func (c *Cluster) String() string { c.minDuration.Milliseconds(), c.maxDuration.Milliseconds()) } + +// ---------------------------------------------------------------------- diff --git a/pkg/sfu/streamallocator/ratemonitor.go b/pkg/sfu/streamallocator/ratemonitor.go new file mode 100644 index 000000000..06445ea44 --- /dev/null +++ b/pkg/sfu/streamallocator/ratemonitor.go @@ -0,0 +1,158 @@ +package streamallocator + +import ( + "fmt" + "time" + + "github.com/livekit/protocol/utils/timeseries" +) + +// ------------------------------------------------ + +const ( + rateMonitorWindow = 10 * time.Second + queueMonitorWindow = 2 * time.Second +) + +// ------------------------------------------------ + +type RateMonitor struct { + bitrateEstimate *timeseries.TimeSeries[int64] + managedBytesSent *timeseries.TimeSeries[uint32] + managedBytesRetransmitted *timeseries.TimeSeries[uint32] + unmanagedBytesSent *timeseries.TimeSeries[uint32] + unmanagedBytesRetransmitted *timeseries.TimeSeries[uint32] + + // STREAM-ALLOCATOR-EXPERIMENTAL-TODO: remove after experimental + history []string +} + +func NewRateMonitor() *RateMonitor { + return &RateMonitor{ + bitrateEstimate: timeseries.NewTimeSeries[int64](timeseries.TimeSeriesParams{ + UpdateOp: timeseries.TimeSeriesUpdateOpLatest, + Window: rateMonitorWindow, + }), + managedBytesSent: timeseries.NewTimeSeries[uint32](timeseries.TimeSeriesParams{ + UpdateOp: timeseries.TimeSeriesUpdateOpAdd, + Window: rateMonitorWindow, + }), + managedBytesRetransmitted: timeseries.NewTimeSeries[uint32](timeseries.TimeSeriesParams{ + UpdateOp: timeseries.TimeSeriesUpdateOpAdd, + Window: rateMonitorWindow, + }), + unmanagedBytesSent: timeseries.NewTimeSeries[uint32](timeseries.TimeSeriesParams{ + UpdateOp: timeseries.TimeSeriesUpdateOpAdd, + Window: rateMonitorWindow, + }), + unmanagedBytesRetransmitted: timeseries.NewTimeSeries[uint32](timeseries.TimeSeriesParams{ + UpdateOp: timeseries.TimeSeriesUpdateOpAdd, + Window: rateMonitorWindow, + }), + } +} + +func (r *RateMonitor) Update(estimate int64, managedBytesSent uint32, managedBytesRetransmitted uint32, unmanagedBytesSent uint32, unmanagedBytesRetransmitted uint32) { + now := time.Now() + r.bitrateEstimate.AddSampleAt(estimate, now) + r.managedBytesSent.AddSampleAt(managedBytesSent, now) + r.managedBytesRetransmitted.AddSampleAt(managedBytesRetransmitted, now) + r.unmanagedBytesSent.AddSampleAt(unmanagedBytesSent, now) + r.unmanagedBytesRetransmitted.AddSampleAt(unmanagedBytesRetransmitted, now) + + r.updateHistory() +} + +// STREAM-ALLOCATOR-TODO: +// This should be updated periodically to flush any pending. +// Reason is that the estimate could be higher than the actual rate by a significant amount. +// So, updating periodically to flush out samples that will not contribute to queueing would be good. +func (r *RateMonitor) GetQueuingGuess() float64 { + _, _, _, _, _, qd := r.getRates(queueMonitorWindow) + return qd +} + +func (r *RateMonitor) getRates(monitorDuration time.Duration) (float64, float64, float64, float64, float64, float64) { + threshold := time.Now().Add(-monitorDuration) + bitrateEstimateSamples := r.bitrateEstimate.GetSamplesAfter(threshold) + managedBytesSentSamples := r.managedBytesSent.GetSamplesAfter(threshold) + managedBytesRetransmittedSamples := r.managedBytesRetransmitted.GetSamplesAfter(threshold) + unmanagedBytesSentSamples := r.unmanagedBytesSent.GetSamplesAfter(threshold) + unmanagedBytesRetransmittedSamples := r.unmanagedBytesRetransmitted.GetSamplesAfter(threshold) + + if len(bitrateEstimateSamples) == 0 || (len(managedBytesSentSamples)+len(managedBytesRetransmittedSamples)+len(unmanagedBytesSentSamples)+len(unmanagedBytesRetransmittedSamples)) == 0 { + return 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 + } + + totalBitrateEstimate := getTimeWeightedSum(bitrateEstimateSamples) + totalManagedSent := getRate(managedBytesSentSamples) * 8 + totalManagedRetransmitted := getRate(managedBytesRetransmittedSamples) * 8 + totalUnmanagedSent := getRate(unmanagedBytesSentSamples) * 8 + totalUnmanagedRetransmitted := getRate(unmanagedBytesRetransmittedSamples) * 8 + totalBits := totalManagedSent + totalManagedRetransmitted + totalUnmanagedSent + totalUnmanagedRetransmitted + + queuingDelay := float64(0.0) + if totalBits > totalBitrateEstimate { + latestBitrateEstimate := bitrateEstimateSamples[len(bitrateEstimateSamples)-1].Value + excessBits := totalBits - totalBitrateEstimate + queuingDelay = excessBits / float64(latestBitrateEstimate) + } + return totalBitrateEstimate, totalManagedSent, totalManagedRetransmitted, totalUnmanagedSent, totalUnmanagedRetransmitted, queuingDelay +} + +func (r *RateMonitor) updateHistory() { + if len(r.history) >= 10 { + r.history = r.history[1:] + } + + e, m, mr, um, umr, qd := r.getRates(time.Second) + if e == 0.0 { + return + } + + r.history = append( + r.history, + fmt.Sprintf("t: %+v, e: %.2f, m: %.2f/%.2f, um: %.2f/%.2f, qd: %.2f", time.Now().UnixMilli(), e, m, mr, um, umr, qd), + ) +} + +func (r *RateMonitor) GetHistory() []string { + return r.history +} + +// ------------------------------------------------ + +func getTimeWeightedSum[T int64 | uint32](samples []timeseries.TimeSeriesSample[T]) float64 { + if len(samples) < 2 { + return 0.0 + } + + sum := 0.0 + for i := 1; i < len(samples); i++ { + diff := samples[i].At.Sub(samples[i-1].At).Seconds() + sum += diff * float64(samples[i-1].Value) + } + + diff := time.Now().Sub(samples[len(samples)-1].At).Seconds() + sum += diff * float64(samples[len(samples)-1].Value) + return sum +} + +func getRate[T int64 | uint32](samples []timeseries.TimeSeriesSample[T]) float64 { + if len(samples) < 2 { + return 0.0 + } + + sum := 0.0 + // start at 1 as the first sample duration is not available + for i := 1; i < len(samples); i++ { + sum += float64(samples[i].Value) + } + + duration := samples[len(samples)-1].At.Sub(samples[0].At) + if duration == 0 { + return 0.0 + } + + return sum / duration.Seconds() +} diff --git a/pkg/sfu/streamallocator/streamallocator.go b/pkg/sfu/streamallocator/streamallocator.go index cb73976d3..b680ae817 100644 --- a/pkg/sfu/streamallocator/streamallocator.go +++ b/pkg/sfu/streamallocator/streamallocator.go @@ -66,7 +66,7 @@ var ( Name: "non-probe", EstimateRequiredSamples: 8, EstimateDownwardTrendThreshold: -0.5, - EstimateCollapseThreshold: 500 * time.Millisecond, + EstimateCollapseThreshold: 250 * time.Millisecond, NackWindowMinDuration: 1 * time.Second, NackWindowMaxDuration: 2 * time.Second, NackRatioThreshold: 0.08, @@ -108,6 +108,8 @@ const ( streamAllocatorSignalResume streamAllocatorSignalSetAllowPause streamAllocatorSignalSetChannelCapacity + streamAllocatorSignalNACK + streamAllocatorSignalRTCPReceiverReport ) func (s streamAllocatorSignal) String() string { @@ -132,6 +134,10 @@ func (s streamAllocatorSignal) String() string { return "SET_ALLOW_PAUSE" case streamAllocatorSignalSetChannelCapacity: return "SET_CHANNEL_CAPACITY" + case streamAllocatorSignalNACK: + return "NACK" + case streamAllocatorSignalRTCPReceiverReport: + return "RTCP_RECEIVER_REPORT" default: return fmt.Sprintf("%d", int(s)) } @@ -180,6 +186,7 @@ type StreamAllocator struct { prober *Prober channelObserver *ChannelObserver + rateMonitor *RateMonitor videoTracksMu sync.RWMutex videoTracks map[livekit.TrackID]*Track @@ -201,8 +208,9 @@ func NewStreamAllocator(params StreamAllocatorParams) *StreamAllocator { prober: NewProber(ProberParams{ Logger: params.Logger, }), + rateMonitor: NewRateMonitor(), videoTracks: make(map[livekit.TrackID]*Track), - eventCh: make(chan Event, 200), + eventCh: make(chan Event, 1000), } s.resetState() @@ -463,11 +471,30 @@ func (s *StreamAllocator) OnResume(downTrack *sfu.DownTrack) { }) } -// called when a video DownTrack sends a packet +// called by a video DownTrack to report packet send func (s *StreamAllocator) OnPacketsSent(downTrack *sfu.DownTrack, size int) { s.prober.PacketsSent(size) } +// called by a video DownTrack when it processes NACKs +func (s *StreamAllocator) OnNACK(downTrack *sfu.DownTrack, nackInfos []sfu.NackInfo) { + s.postEvent(Event{ + Signal: streamAllocatorSignalNACK, + TrackID: livekit.TrackID(downTrack.ID()), + Data: nackInfos, + }) +} + +// called by a video DownTrack when it receives an RTCP Receiver Report +// STREAM-ALLOCATOR-TODO: this should probably be done for audio tracks also +func (s *StreamAllocator) OnRTCPReceiverReport(downTrack *sfu.DownTrack, rr rtcp.ReceptionReport) { + s.postEvent(Event{ + Signal: streamAllocatorSignalRTCPReceiverReport, + TrackID: livekit.TrackID(downTrack.ID()), + Data: rr, + }) +} + // called when prober wants to send packet(s) func (s *StreamAllocator) OnSendProbe(bytesToSend int) { s.postEvent(Event{ @@ -538,7 +565,7 @@ func (s *StreamAllocator) processEvents() { } func (s *StreamAllocator) ping() { - ticker := time.NewTicker(time.Second) + ticker := time.NewTicker(500 * time.Millisecond) defer ticker.Stop() for { @@ -575,6 +602,10 @@ func (s *StreamAllocator) handleEvent(event *Event) { s.handleSignalSetAllowPause(event) case streamAllocatorSignalSetChannelCapacity: s.handleSignalSetChannelCapacity(event) + case streamAllocatorSignalNACK: + s.handleSignalNACK(event) + case streamAllocatorSignalRTCPReceiverReport: + s.handleSignalRTCPReceiverReport(event) } } @@ -608,6 +639,7 @@ func (s *StreamAllocator) handleSignalAdjustState(event *Event) { func (s *StreamAllocator) handleSignalEstimate(event *Event) { receivedEstimate, _ := event.Data.(int64) s.lastReceivedEstimate = receivedEstimate + s.monitorRate(receivedEstimate) // while probing, maintain estimate separately to enable keeping current committed estimate if probe fails if s.isInProbe() { @@ -627,6 +659,8 @@ func (s *StreamAllocator) handleSignalPeriodicPing(event *Event) { if s.state == streamAllocatorStateDeficient { s.maybeProbe() } + + s.updateTracksHistory() } func (s *StreamAllocator) handleSignalSendProbe(event *Event) { @@ -702,6 +736,30 @@ func (s *StreamAllocator) handleSignalSetChannelCapacity(event *Event) { } } +func (s *StreamAllocator) handleSignalNACK(event *Event) { + nackInfos := event.Data.([]sfu.NackInfo) + + s.videoTracksMu.Lock() + track := s.videoTracks[event.TrackID] + s.videoTracksMu.Unlock() + + if track != nil { + track.UpdateNack(nackInfos) + } +} + +func (s *StreamAllocator) handleSignalRTCPReceiverReport(event *Event) { + rr := event.Data.(rtcp.ReceptionReport) + + s.videoTracksMu.Lock() + track := s.videoTracks[event.TrackID] + s.videoTracksMu.Unlock() + + if track != nil { + track.ProcessRTCPReceiverReport(rr) + } +} + func (s *StreamAllocator) setState(state streamAllocatorState) { if s.state == state { return @@ -801,6 +859,13 @@ func (s *StreamAllocator) handleNewEstimateInNonProbe() { "expectedUsage(bps)", expectedBandwidthUsage, "channel", s.channelObserver.ToString(), ) + s.params.Logger.Infow( + "stream allocator: channel congestion detected, updating channel capacity: experimental", + "rateHistory", s.rateMonitor.GetHistory(), + "expectedQueuing", s.rateMonitor.GetQueuingGuess(), + "nackHistory", s.channelObserver.GetNackHistory(), + "trackHistory", s.getTracksHistory(), + ) s.committedChannelCapacity = estimateToCommit // reset to get new set of samples for next trend @@ -919,7 +984,7 @@ func (s *StreamAllocator) finalizeProbe() { // // Reset estimator at the end of a probe irrespective of probe result to get fresh readings. - // With a failed probe, the latest estimate would be lower than committed estimate. + // With a failed probe, the latest estimate could be lower than committed estimate. // As bandwidth estimator (remote in REMB case, local in TWCC case) holds state, // subsequent estimates could start from the lower point. That should not trigger a // downward trend and get latched to committed estimate as that would trigger a re-allocation. @@ -1179,6 +1244,7 @@ func (s *StreamAllocator) initProbe(probeGoalDeltaBps int64) { s.channelObserver.SeedEstimate(s.lastReceivedEstimate) s.probeClusterId = s.prober.AddCluster( + ProbeClusterModeUniform, int(s.probeGoalBps), int(expectedBandwidthUsage), ProbeMinDuration, @@ -1342,4 +1408,47 @@ func (s *StreamAllocator) getMaxDistanceSortedDeficient() MaxDistanceSorter { return maxDistanceSorter } +// STREAM-ALLOCATOR-EXPERIMENTAL-TODO +// Monitor sent rate vs estimate to figure out queuing on congestion. +// Idea here is to pause all managed tracks on congestion detection immediately till queue drains. +// That will allow channel to clear up without more traffic added and a re-allocation can start afresh. +// Some bits to work out +// - how good is queuing estimate? +// - should we pause unmanaged tracks also? But, they will restart at highest layer and request a key frame. +// - what should be the channel capacity to use when resume re-allocation happens? +func (s *StreamAllocator) monitorRate(estimate int64) { + managedBytesSent := uint32(0) + managedBytesRetransmitted := uint32(0) + unmanagedBytesSent := uint32(0) + unmanagedBytesRetransmitted := uint32(0) + for _, track := range s.getTracks() { + b, r := track.GetAndResetBytesSent() + if track.IsManaged() { + managedBytesSent += b + managedBytesRetransmitted += r + } else { + unmanagedBytesSent += b + unmanagedBytesRetransmitted += r + } + } + + s.rateMonitor.Update(estimate, managedBytesSent, managedBytesRetransmitted, unmanagedBytesSent, unmanagedBytesRetransmitted) +} + +func (s *StreamAllocator) updateTracksHistory() { + for _, track := range s.getTracks() { + track.UpdateHistory() + } +} + +func (s *StreamAllocator) getTracksHistory() map[livekit.TrackID]string { + tracks := s.getTracks() + history := make(map[livekit.TrackID]string, len(tracks)) + for _, track := range tracks { + history[track.ID()] = track.GetHistory() + } + + return history +} + // ------------------------------------------------ diff --git a/pkg/sfu/streamallocator/track.go b/pkg/sfu/streamallocator/track.go index e309baec6..d3aedcef4 100644 --- a/pkg/sfu/streamallocator/track.go +++ b/pkg/sfu/streamallocator/track.go @@ -1,8 +1,14 @@ package streamallocator import ( + "fmt" + "sort" + "time" + + "github.com/livekit/mediatransportutil" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" + "github.com/pion/rtcp" "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/livekit-server/pkg/sfu/buffer" @@ -21,6 +27,19 @@ type Track struct { totalPackets uint32 totalRepeatedNacks uint32 + nackInfos map[uint16]sfu.NackInfo + // STREAM-ALLOCATOR-EXPERIMENTAL-TODO: remove after experimental + nackHistory []string + + receiverReportInitialized bool + totalLostAtLastRead uint32 + totalLost uint32 + highestSequenceNumberAtLastRead uint32 + highestSequenceNumber uint32 + maxRTT uint32 + // STREAM-ALLOCATOR-EXPERIMENTAL-TODO: remove after experimental + receiverReportHistory []string + isDirty bool isPaused bool @@ -34,12 +53,15 @@ func NewTrack( logger logger.Logger, ) *Track { t := &Track{ - downTrack: downTrack, - source: source, - isSimulcast: isSimulcast, - publisherID: publisherID, - logger: logger, - isPaused: true, + downTrack: downTrack, + source: source, + isSimulcast: isSimulcast, + publisherID: publisherID, + logger: logger, + nackInfos: make(map[uint16]sfu.NackInfo), + nackHistory: make([]string, 0, 10), + receiverReportHistory: make([]string, 0, 10), + isPaused: true, } t.SetPriority(0) t.SetMaxLayer(downTrack.MaxLayer()) @@ -176,6 +198,150 @@ func (t *Track) GetNackDelta() (uint32, uint32) { return packetDelta, nackDelta } +func (t *Track) UpdateNack(nackInfos []sfu.NackInfo) { + for _, ni := range nackInfos { + t.nackInfos[ni.SequenceNumber] = ni + } +} + +func (t *Track) GetAndResetNackStats() (lowest uint16, highest uint16, numNacked int, numNacks int, numRuns int) { + if len(t.nackInfos) == 0 { + return + } + + sns := make([]uint16, 0, len(t.nackInfos)) + for _, ni := range t.nackInfos { + if lowest == 0 || ni.SequenceNumber-lowest > (1<<15) { + lowest = ni.SequenceNumber + } + if highest == 0 || highest-ni.SequenceNumber > (1<<15) { + highest = ni.SequenceNumber + } + numNacks += int(ni.Attempts) + sns = append(sns, ni.SequenceNumber) + } + numNacked = len(t.nackInfos) + + // find number of runs, i. e. bursts of contiguous sequence numbers NACKed, does not include isolated NACKs + sort.Slice(sns, func(i, j int) bool { + return (sns[i] - sns[j]) > (1 << 15) + }) + + rsn := sns[0] + rsi := 0 + for i := 1; i < len(sns); i++ { + if sns[i] == rsn+1 { + continue + } + + if (i - rsi - 1) > 0 { + numRuns++ + } + + rsn = sns[i] + rsi = i + } + + t.nackInfos = make(map[uint16]sfu.NackInfo) + return +} + +func (t *Track) ProcessRTCPReceiverReport(rr rtcp.ReceptionReport) { + if !t.receiverReportInitialized { + t.receiverReportInitialized = true + t.totalLostAtLastRead = rr.TotalLost + t.highestSequenceNumberAtLastRead = rr.LastSequenceNumber + } + + t.totalLost = rr.TotalLost + t.highestSequenceNumber = rr.LastSequenceNumber + + if rtt, err := mediatransportutil.GetRttMsFromReceiverReportOnly(&rr); err != nil { + if rtt > t.maxRTT { + t.maxRTT = rtt + } + } + + t.updateReceiverReportHistory() +} + +func (t *Track) GetRTCPReceiverReportDelta() (uint32, uint32, uint32) { + deltaPackets := t.highestSequenceNumber - t.highestSequenceNumberAtLastRead + t.highestSequenceNumberAtLastRead = t.highestSequenceNumber + + deltaLost := t.totalLost - t.totalLostAtLastRead + t.totalLostAtLastRead = t.totalLost + + maxRTT := t.maxRTT + t.maxRTT = 0 + + return deltaLost, deltaPackets, maxRTT +} + +func (t *Track) GetAndResetBytesSent() (uint32, uint32) { + return t.downTrack.GetAndResetBytesSent() +} + +func (t *Track) UpdateHistory() { + t.updateNackHistory() +} + +func (t *Track) GetHistory() string { + return fmt.Sprintf("t: %+v, n: %+v, rr: %+v", time.Now(), t.nackHistory, t.receiverReportHistory) +} + +// STREAM-ALLOCATOR-EXPERIMENTAL-TODO: +// Idea is to check if this provides a good signal to detect congestion. +// This measures a few things +// 1. Spread: sequence number difference between highest and lowest NACK +// - shows how widespread the losses are +// 2. Number of runs of length more than 1: Counts number of burst losses. +// - could be a sign of congestion when losses are bursty +// 3. NACK density: how many sequence numbers in the spread were NACKed. +// - a high density could be a sign of congestion +// 4. NACK intensity: how many times those sequence numbers were NACKed. +// - high intensity could be a sign of congestion +// +// While these all could be good signals, some challenges in making use of these +// - aggregating across tracks +// - proper thresholing, i. e. something based on averages should not trip +// because of small numbers, e. g. a single NACK run of 2 sequence numbers +// is technically a burst, but is it a signal of congestion? +func (t *Track) updateNackHistory() { + if len(t.nackHistory) >= 10 { + t.nackHistory = t.nackHistory[1:] + } + + l, h, nnd, nns, nr := t.GetAndResetNackStats() + spread := h - l + 1 + density := float64(0.0) + if nnd != 0 { + density = float64(nnd) / float64(spread) + } else { + spread = 0 + } + intensity := float64(0.0) + if nnd != 0 { + intensity = float64(nns) / float64(nnd) + } + t.nackHistory = append( + t.nackHistory, + fmt.Sprintf("t: %+v, l: %d, h: %d, sp: %d, nnd: %d, dens: %.2f, nns: %d, int: %.2f, nr: %d", time.Now().UnixMilli(), l, h, spread, nnd, density, nns, intensity, nr), + ) +} + +func (t *Track) updateReceiverReportHistory() { + if len(t.receiverReportHistory) >= 10 { + t.receiverReportHistory = t.receiverReportHistory[1:] + } + + dl, dp, maxRTT := t.GetRTCPReceiverReportDelta() + t.receiverReportHistory = append( + t.receiverReportHistory, + fmt.Sprintf("t: %+v, l: %d, p: %d, rtt: %d", time.Now().Format(time.UnixDate), dl, dp, maxRTT), + ) +} + // ------------------------------------------------ type TrackSorter []*Track From 5751b8c0823a96d006afe78282a63ef56d7c5064 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Mon, 24 Apr 2023 20:02:28 -0700 Subject: [PATCH 115/324] rebase (#1654) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d8b16b22a..63b40cf5a 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 github.com/livekit/protocol v1.5.6-0.20230424073901-c54f5f7f4182 - github.com/livekit/psrpc v0.3.1-0.20230424064451-65c6a2dd048b + github.com/livekit/psrpc v0.3.1-0.20230425025640-5390915734c3 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.14.0 github.com/maxbrunsfeld/counterfeiter/v6 v6.6.1 diff --git a/go.sum b/go.sum index d36f3ccad..144995ed6 100644 --- a/go.sum +++ b/go.sum @@ -123,8 +123,8 @@ github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 h1:QlQF github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26/go.mod h1:eDA41kiySZoG+wy4Etsjb3w0jjLx69i/vAmSjG4bteA= github.com/livekit/protocol v1.5.6-0.20230424073901-c54f5f7f4182 h1:rpaYN8Jy5F1ZhxH4Q61+6Cc42I/evI63SlSXUMZ+VdU= github.com/livekit/protocol v1.5.6-0.20230424073901-c54f5f7f4182/go.mod h1:B7Ns8diIKB3y39oRHm7ZluU9ZGCxCWQT+uKcbY3MCG4= -github.com/livekit/psrpc v0.3.1-0.20230424064451-65c6a2dd048b h1:WqRJoXeycdrhEuA9C0jJ7aN6i5wx8B3c+M0hIKnnFZE= -github.com/livekit/psrpc v0.3.1-0.20230424064451-65c6a2dd048b/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= +github.com/livekit/psrpc v0.3.1-0.20230425025640-5390915734c3 h1:NXcxrluYLng7LTHcYNOj/MdR4SHWrKQAG2G+U930mTA= +github.com/livekit/psrpc v0.3.1-0.20230425025640-5390915734c3/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= github.com/mackerelio/go-osstat v0.2.4/go.mod h1:Zy+qzGdZs3A9cuIqmgbJvwbmLQH9dJvtio5ZjJTbdlQ= github.com/magefile/mage v1.14.0 h1:6QDX3g6z1YvJ4olPhT1wksUcSa/V0a1B+pJb73fBjyo= From 9db46bb86635145f059cace585ef83aef14ae7f8 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 26 Apr 2023 21:29:25 +0530 Subject: [PATCH 116/324] Avoid divide-by-zero and NaN (#1656) --- pkg/sfu/connectionquality/scorer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/sfu/connectionquality/scorer.go b/pkg/sfu/connectionquality/scorer.go index e13575a32..4de60b05f 100644 --- a/pkg/sfu/connectionquality/scorer.go +++ b/pkg/sfu/connectionquality/scorer.go @@ -349,7 +349,7 @@ func (q *qualityScorer) isLayerMuted() bool { } func (q *qualityScorer) getPacketLossWeight(stat *windowStat) float64 { - if stat == nil { + if stat == nil || stat.duration == 0 { return q.params.PacketLossWeight } From 11eedf45142e7a6db965d9150cb7a180cffdc14c Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Wed, 26 Apr 2023 17:11:33 -0700 Subject: [PATCH 117/324] update participant to support signal broadcast skipping (#1657) * update participant to support signal broadcast skipping * cleanup * lock * feedback * order * update requireBroadcast in SetPermissions --- pkg/rtc/participant.go | 20 +++- pkg/rtc/types/interfaces.go | 1 + .../typesfakes/fake_local_participant.go | 95 +++++++++++++------ pkg/rtc/types/typesfakes/fake_participant.go | 65 +++++++++++++ 4 files changed, 149 insertions(+), 32 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 532030aac..cb9f68e33 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -131,6 +131,7 @@ type ParticipantImpl struct { // keeps track of unpublished tracks in order to reuse trackID unpublishedTracks []*livekit.TrackInfo + requireBroadcast bool // queued participant updates before join response is sent // guarded by updateLock queuedUpdates []*livekit.ParticipantInfo @@ -324,6 +325,7 @@ func (p *ParticipantImpl) SetMetadata(metadata string) { } p.grants.Metadata = metadata + p.requireBroadcast = p.requireBroadcast || metadata != "" p.dirty.Store(true) onParticipantUpdate := p.onParticipantUpdate @@ -364,6 +366,9 @@ func (p *ParticipantImpl) SetPermission(permission *livekit.ParticipantPermissio canSubscribe := video.GetCanSubscribe() onParticipantUpdate := p.onParticipantUpdate onClaimsChanged := p.onClaimsChanged + + isPublisher := canPublish && p.TransportManager.IsPublisherEstablished() + p.requireBroadcast = p.requireBroadcast || isPublisher p.lock.Unlock() // publish permission has been revoked then remove offending tracks @@ -390,7 +395,7 @@ func (p *ParticipantImpl) SetPermission(permission *livekit.ParticipantPermissio } // update isPublisher attribute - p.isPublisher.Store(canPublish && p.TransportManager.IsPublisherEstablished()) + p.isPublisher.Store(isPublisher) if onParticipantUpdate != nil { onParticipantUpdate(p) @@ -401,6 +406,12 @@ func (p *ParticipantImpl) SetPermission(permission *livekit.ParticipantPermissio return true } +func (p *ParticipantImpl) CanSkipBroadcast() bool { + p.lock.RLock() + defer p.lock.RUnlock() + return !p.requireBroadcast +} + func (p *ParticipantImpl) ToProtoWithVersion() (*livekit.ParticipantInfo, utils.TimedVersion) { v := p.version.Load() piv := p.timedVersion.Load() @@ -1081,10 +1092,11 @@ func (p *ParticipantImpl) setupUpTrackManager() { }) p.UpTrackManager.OnPublishedTrackUpdated(func(track types.MediaTrack) { - p.dirty.Store(true) p.lock.RLock() onTrackUpdated := p.onTrackUpdated p.lock.RUnlock() + + p.dirty.Store(true) if onTrackUpdated != nil { onTrackUpdated(p, track) } @@ -1134,6 +1146,10 @@ func (p *ParticipantImpl) updateState(state livekit.ParticipantInfo_State) { func (p *ParticipantImpl) setIsPublisher(isPublisher bool) { if p.isPublisher.Swap(isPublisher) != isPublisher { + p.lock.Lock() + p.requireBroadcast = true + p.lock.Unlock() + p.dirty.Store(true) // trigger update as well if participant is already fully connected diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index 1163cc1d2..625c3ecb0 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -176,6 +176,7 @@ type Participant interface { Identity() livekit.ParticipantIdentity State() livekit.ParticipantInfo_State + CanSkipBroadcast() bool ToProto() *livekit.ParticipantInfo SetName(name string) diff --git a/pkg/rtc/types/typesfakes/fake_local_participant.go b/pkg/rtc/types/typesfakes/fake_local_participant.go index f4966a5c5..33416aa05 100644 --- a/pkg/rtc/types/typesfakes/fake_local_participant.go +++ b/pkg/rtc/types/typesfakes/fake_local_participant.go @@ -89,6 +89,16 @@ type FakeLocalParticipant struct { canPublishSourceReturnsOnCall map[int]struct { result1 bool } + CanSkipBroadcastStub func() bool + canSkipBroadcastMutex sync.RWMutex + canSkipBroadcastArgsForCall []struct { + } + canSkipBroadcastReturns struct { + result1 bool + } + canSkipBroadcastReturnsOnCall map[int]struct { + result1 bool + } CanSubscribeStub func() bool canSubscribeMutex sync.RWMutex canSubscribeArgsForCall []struct { @@ -340,10 +350,6 @@ type FakeLocalParticipant struct { identityReturnsOnCall map[int]struct { result1 livekit.ParticipantIdentity } - InvalidateVersionStub func() - invalidateVersionMutex sync.RWMutex - invalidateVersionArgsForCall []struct { - } IsClosedStub func() bool isClosedMutex sync.RWMutex isClosedArgsForCall []struct { @@ -1202,6 +1208,59 @@ func (fake *FakeLocalParticipant) CanPublishSourceReturnsOnCall(i int, result1 b }{result1} } +func (fake *FakeLocalParticipant) CanSkipBroadcast() bool { + fake.canSkipBroadcastMutex.Lock() + ret, specificReturn := fake.canSkipBroadcastReturnsOnCall[len(fake.canSkipBroadcastArgsForCall)] + fake.canSkipBroadcastArgsForCall = append(fake.canSkipBroadcastArgsForCall, struct { + }{}) + stub := fake.CanSkipBroadcastStub + fakeReturns := fake.canSkipBroadcastReturns + fake.recordInvocation("CanSkipBroadcast", []interface{}{}) + fake.canSkipBroadcastMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) CanSkipBroadcastCallCount() int { + fake.canSkipBroadcastMutex.RLock() + defer fake.canSkipBroadcastMutex.RUnlock() + return len(fake.canSkipBroadcastArgsForCall) +} + +func (fake *FakeLocalParticipant) CanSkipBroadcastCalls(stub func() bool) { + fake.canSkipBroadcastMutex.Lock() + defer fake.canSkipBroadcastMutex.Unlock() + fake.CanSkipBroadcastStub = stub +} + +func (fake *FakeLocalParticipant) CanSkipBroadcastReturns(result1 bool) { + fake.canSkipBroadcastMutex.Lock() + defer fake.canSkipBroadcastMutex.Unlock() + fake.CanSkipBroadcastStub = nil + fake.canSkipBroadcastReturns = struct { + result1 bool + }{result1} +} + +func (fake *FakeLocalParticipant) CanSkipBroadcastReturnsOnCall(i int, result1 bool) { + fake.canSkipBroadcastMutex.Lock() + defer fake.canSkipBroadcastMutex.Unlock() + fake.CanSkipBroadcastStub = nil + if fake.canSkipBroadcastReturnsOnCall == nil { + fake.canSkipBroadcastReturnsOnCall = make(map[int]struct { + result1 bool + }) + } + fake.canSkipBroadcastReturnsOnCall[i] = struct { + result1 bool + }{result1} +} + func (fake *FakeLocalParticipant) CanSubscribe() bool { fake.canSubscribeMutex.Lock() ret, specificReturn := fake.canSubscribeReturnsOnCall[len(fake.canSubscribeArgsForCall)] @@ -2537,30 +2596,6 @@ func (fake *FakeLocalParticipant) IdentityReturnsOnCall(i int, result1 livekit.P }{result1} } -func (fake *FakeLocalParticipant) InvalidateVersion() { - fake.invalidateVersionMutex.Lock() - fake.invalidateVersionArgsForCall = append(fake.invalidateVersionArgsForCall, struct { - }{}) - stub := fake.InvalidateVersionStub - fake.recordInvocation("InvalidateVersion", []interface{}{}) - fake.invalidateVersionMutex.Unlock() - if stub != nil { - fake.InvalidateVersionStub() - } -} - -func (fake *FakeLocalParticipant) InvalidateVersionCallCount() int { - fake.invalidateVersionMutex.RLock() - defer fake.invalidateVersionMutex.RUnlock() - return len(fake.invalidateVersionArgsForCall) -} - -func (fake *FakeLocalParticipant) InvalidateVersionCalls(stub func()) { - fake.invalidateVersionMutex.Lock() - defer fake.invalidateVersionMutex.Unlock() - fake.InvalidateVersionStub = stub -} - func (fake *FakeLocalParticipant) IsClosed() bool { fake.isClosedMutex.Lock() ret, specificReturn := fake.isClosedReturnsOnCall[len(fake.isClosedArgsForCall)] @@ -5405,6 +5440,8 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.canPublishDataMutex.RUnlock() fake.canPublishSourceMutex.RLock() defer fake.canPublishSourceMutex.RUnlock() + fake.canSkipBroadcastMutex.RLock() + defer fake.canSkipBroadcastMutex.RUnlock() fake.canSubscribeMutex.RLock() defer fake.canSubscribeMutex.RUnlock() fake.claimGrantsMutex.RLock() @@ -5457,8 +5494,6 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.iDMutex.RUnlock() fake.identityMutex.RLock() defer fake.identityMutex.RUnlock() - fake.invalidateVersionMutex.RLock() - defer fake.invalidateVersionMutex.RUnlock() fake.isClosedMutex.RLock() defer fake.isClosedMutex.RUnlock() fake.isDisconnectedMutex.RLock() diff --git a/pkg/rtc/types/typesfakes/fake_participant.go b/pkg/rtc/types/typesfakes/fake_participant.go index 404af0a17..c32b8a6ed 100644 --- a/pkg/rtc/types/typesfakes/fake_participant.go +++ b/pkg/rtc/types/typesfakes/fake_participant.go @@ -10,6 +10,16 @@ import ( ) type FakeParticipant struct { + CanSkipBroadcastStub func() bool + canSkipBroadcastMutex sync.RWMutex + canSkipBroadcastArgsForCall []struct { + } + canSkipBroadcastReturns struct { + result1 bool + } + canSkipBroadcastReturnsOnCall map[int]struct { + result1 bool + } CloseStub func(bool, types.ParticipantCloseReason) error closeMutex sync.RWMutex closeArgsForCall []struct { @@ -197,6 +207,59 @@ type FakeParticipant struct { invocationsMutex sync.RWMutex } +func (fake *FakeParticipant) CanSkipBroadcast() bool { + fake.canSkipBroadcastMutex.Lock() + ret, specificReturn := fake.canSkipBroadcastReturnsOnCall[len(fake.canSkipBroadcastArgsForCall)] + fake.canSkipBroadcastArgsForCall = append(fake.canSkipBroadcastArgsForCall, struct { + }{}) + stub := fake.CanSkipBroadcastStub + fakeReturns := fake.canSkipBroadcastReturns + fake.recordInvocation("CanSkipBroadcast", []interface{}{}) + fake.canSkipBroadcastMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeParticipant) CanSkipBroadcastCallCount() int { + fake.canSkipBroadcastMutex.RLock() + defer fake.canSkipBroadcastMutex.RUnlock() + return len(fake.canSkipBroadcastArgsForCall) +} + +func (fake *FakeParticipant) CanSkipBroadcastCalls(stub func() bool) { + fake.canSkipBroadcastMutex.Lock() + defer fake.canSkipBroadcastMutex.Unlock() + fake.CanSkipBroadcastStub = stub +} + +func (fake *FakeParticipant) CanSkipBroadcastReturns(result1 bool) { + fake.canSkipBroadcastMutex.Lock() + defer fake.canSkipBroadcastMutex.Unlock() + fake.CanSkipBroadcastStub = nil + fake.canSkipBroadcastReturns = struct { + result1 bool + }{result1} +} + +func (fake *FakeParticipant) CanSkipBroadcastReturnsOnCall(i int, result1 bool) { + fake.canSkipBroadcastMutex.Lock() + defer fake.canSkipBroadcastMutex.Unlock() + fake.CanSkipBroadcastStub = nil + if fake.canSkipBroadcastReturnsOnCall == nil { + fake.canSkipBroadcastReturnsOnCall = make(map[int]struct { + result1 bool + }) + } + fake.canSkipBroadcastReturnsOnCall[i] = struct { + result1 bool + }{result1} +} + func (fake *FakeParticipant) Close(arg1 bool, arg2 types.ParticipantCloseReason) error { fake.closeMutex.Lock() ret, specificReturn := fake.closeReturnsOnCall[len(fake.closeArgsForCall)] @@ -1165,6 +1228,8 @@ func (fake *FakeParticipant) UpdateVideoLayersReturnsOnCall(i int, result1 error func (fake *FakeParticipant) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() + fake.canSkipBroadcastMutex.RLock() + defer fake.canSkipBroadcastMutex.RUnlock() fake.closeMutex.RLock() defer fake.closeMutex.RUnlock() fake.debugInfoMutex.RLock() From f9f89cd7cf366f41979dacbbad36b5c69abf72c1 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Wed, 26 Apr 2023 22:59:39 -0700 Subject: [PATCH 118/324] close signal with reliable message (#1658) * close signal with reliable message * update protocol --- go.mod | 8 ++--- go.sum | 16 +++++----- pkg/routing/signal.go | 68 +++++++++++++++++----------------------- pkg/rtc/signalhandler.go | 2 +- pkg/service/signal.go | 15 ++------- 5 files changed, 45 insertions(+), 64 deletions(-) diff --git a/go.mod b/go.mod index 63b40cf5a..ccbf1201a 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 - github.com/livekit/protocol v1.5.6-0.20230424073901-c54f5f7f4182 + github.com/livekit/protocol v1.5.6-0.20230427055046-79477e28a150 github.com/livekit/psrpc v0.3.1-0.20230425025640-5390915734c3 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.14.0 @@ -27,7 +27,7 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/pion/dtls/v2 v2.2.6 github.com/pion/ice/v2 v2.3.2 - github.com/pion/interceptor v0.1.12 + github.com/pion/interceptor v0.1.13 github.com/pion/logging v0.2.2 github.com/pion/rtcp v1.2.10 github.com/pion/rtp v1.7.13 @@ -35,7 +35,7 @@ require ( github.com/pion/stun v0.4.0 github.com/pion/transport/v2 v2.2.0 github.com/pion/turn/v2 v2.1.0 - github.com/pion/webrtc/v3 v3.1.61 + github.com/pion/webrtc/v3 v3.1.62 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.15.0 github.com/redis/go-redis/v9 v9.0.3 @@ -92,7 +92,7 @@ require ( github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect go.uber.org/multierr v1.6.0 // indirect golang.org/x/crypto v0.8.0 // indirect - golang.org/x/exp v0.0.0-20230420155640-133eef4313cb // indirect + golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 // indirect golang.org/x/mod v0.8.0 // indirect golang.org/x/net v0.9.0 // indirect golang.org/x/sys v0.7.0 // indirect diff --git a/go.sum b/go.sum index 144995ed6..f3769ea35 100644 --- a/go.sum +++ b/go.sum @@ -121,8 +121,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 h1:QlQFyMwCDgjyySsrgmrMcVbEBA6KZcyTzvK+z346tUA= github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26/go.mod h1:eDA41kiySZoG+wy4Etsjb3w0jjLx69i/vAmSjG4bteA= -github.com/livekit/protocol v1.5.6-0.20230424073901-c54f5f7f4182 h1:rpaYN8Jy5F1ZhxH4Q61+6Cc42I/evI63SlSXUMZ+VdU= -github.com/livekit/protocol v1.5.6-0.20230424073901-c54f5f7f4182/go.mod h1:B7Ns8diIKB3y39oRHm7ZluU9ZGCxCWQT+uKcbY3MCG4= +github.com/livekit/protocol v1.5.6-0.20230427055046-79477e28a150 h1:jN0fW8H8Qgi5xsmmbk1s2qsXg1Y873zeWghE4mRV1Cc= +github.com/livekit/protocol v1.5.6-0.20230427055046-79477e28a150/go.mod h1:MBW05GWdhbl+o6u2gLLCQtDvr9EvcV4VWckpIYtoM2c= github.com/livekit/psrpc v0.3.1-0.20230425025640-5390915734c3 h1:NXcxrluYLng7LTHcYNOj/MdR4SHWrKQAG2G+U930mTA= github.com/livekit/psrpc v0.3.1-0.20230425025640-5390915734c3/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= @@ -183,8 +183,8 @@ github.com/pion/dtls/v2 v2.2.6 h1:yXMxKr0Skd+Ub6A8UqXTRLSywskx93ooMRHsQUtd+Z4= github.com/pion/dtls/v2 v2.2.6/go.mod h1:t8fWJCIquY5rlQZwA2yWxUS1+OCrAdXrhVKXB5oD/wY= github.com/pion/ice/v2 v2.3.2 h1:vh+fi4RkZ8H5fB4brZ/jm3j4BqFgMmNs+aB3X52Hu7M= github.com/pion/ice/v2 v2.3.2/go.mod h1:AMIpuJqcpe+UwloocNebmTSWhCZM1TUCo9v7nW50jX0= -github.com/pion/interceptor v0.1.12 h1:CslaNriCFUItiXS5o+hh5lpL0t0ytQkFnUcbbCs2Zq8= -github.com/pion/interceptor v0.1.12/go.mod h1:bDtgAD9dRkBZpWHGKaoKb42FhDHTG2rX8Ii9LRALLVA= +github.com/pion/interceptor v0.1.13 h1:tfJdEqPxnQrlstjd7SCL7B97WdjPkJtg5EpRMgJ61Ms= +github.com/pion/interceptor v0.1.13/go.mod h1:SY8kpmfVBvrbUzvj2bsXz7OJt5JvmVNZ+4Kjq7FcwrI= github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= github.com/pion/mdns v0.0.7 h1:P0UB4Sr6xDWEox0kTVxF0LmQihtCbSAdW0H2nEgkA3U= @@ -214,8 +214,8 @@ github.com/pion/turn/v2 v2.1.0 h1:5wGHSgGhJhP/RpabkUb/T9PdsAjkGLS6toYz5HNzoSI= github.com/pion/turn/v2 v2.1.0/go.mod h1:yrT5XbXSGX1VFSF31A3c1kCNB5bBZgk/uu5LET162qs= github.com/pion/udp/v2 v2.0.1 h1:xP0z6WNux1zWEjhC7onRA3EwwSliXqu1ElUZAQhUP54= github.com/pion/udp/v2 v2.0.1/go.mod h1:B7uvTMP00lzWdyMr/1PVZXtV3wpPIxBRd4Wl6AksXn8= -github.com/pion/webrtc/v3 v3.1.61 h1:WG6p786t7jxXO/3miw6HmAQmO3p/n+QLRa2xLaovcr8= -github.com/pion/webrtc/v3 v3.1.61/go.mod h1:uk/4AJmgEUpSExaP7aexCyODwfbHap8hAnQzRV7zKcE= +github.com/pion/webrtc/v3 v3.1.62 h1:B+QYCs+ajtRMJtC3nphzFWWjVoCorugOABu/JD0pJ3c= +github.com/pion/webrtc/v3 v3.1.62/go.mod h1:PaPsj1aigBfWK1jJRZPkWvdiPaAiJwAEMgDKXVO7NjI= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -282,8 +282,8 @@ golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ= golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= -golang.org/x/exp v0.0.0-20230420155640-133eef4313cb h1:rhjz/8Mbfa8xROFiH+MQphmAmgqRM0bOMnytznhWEXk= -golang.org/x/exp v0.0.0-20230420155640-133eef4313cb/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 h1:5llv2sWeaMSnA3w2kS57ouQQ4pudlXrR0dCgw51QK9o= +golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= diff --git a/pkg/routing/signal.go b/pkg/routing/signal.go index dfef77dec..9a83c6054 100644 --- a/pkg/routing/signal.go +++ b/pkg/routing/signal.go @@ -127,17 +127,11 @@ func (r *signalClient) StartParticipantSignal( type signalRequestMessageWriter struct{} -func (e signalRequestMessageWriter) WriteOne(seq uint64, msg proto.Message) *rpc.RelaySignalRequest { - return &rpc.RelaySignalRequest{ - Seq: seq, - Request: msg.(*livekit.SignalRequest), - } -} - -func (e signalRequestMessageWriter) WriteMany(seq uint64, msgs []proto.Message) *rpc.RelaySignalRequest { +func (e signalRequestMessageWriter) Write(seq uint64, close bool, msgs []proto.Message) *rpc.RelaySignalRequest { r := &rpc.RelaySignalRequest{ Seq: seq, Requests: make([]*livekit.SignalRequest, 0, len(msgs)), + Close: close, } for _, m := range msgs { r.Requests = append(r.Requests, m.(*livekit.SignalRequest)) @@ -148,10 +142,7 @@ func (e signalRequestMessageWriter) WriteMany(seq uint64, msgs []proto.Message) type signalResponseMessageReader struct{} func (e signalResponseMessageReader) Read(rm *rpc.RelaySignalResponse) ([]proto.Message, error) { - msgs := make([]proto.Message, 0, len(rm.Responses)+1) - if rm.Response != nil { - msgs = append(msgs, rm.Response) - } + msgs := make([]proto.Message, 0, len(rm.Responses)) for _, m := range rm.Responses { msgs = append(msgs, m) } @@ -161,11 +152,11 @@ func (e signalResponseMessageReader) Read(rm *rpc.RelaySignalResponse) ([]proto. type RelaySignalMessage interface { proto.Message GetSeq() uint64 + GetClose() bool } type SignalMessageWriter[SendType RelaySignalMessage] interface { - WriteOne(seq uint64, msg proto.Message) SendType - WriteMany(seq uint64, msgs []proto.Message) SendType + Write(seq uint64, close bool, msgs []proto.Message) SendType } type SignalMessageReader[RecvType RelaySignalMessage] interface { @@ -196,6 +187,10 @@ func CopySignalStreamToMessageChannel[SendType, RecvType RelaySignalMessage]( } prometheus.MessageCounter.WithLabelValues("signal", "success").Add(1) } + + if msg.GetClose() { + return psrpc.ErrStreamClosed + } } return stream.Err() } @@ -212,19 +207,18 @@ func (r *signalMessageReader[SendType, RecvType]) Read(msg RecvType) ([]proto.Me return nil, err } - if r.config.MinVersion >= 1 { - if r.seq < msg.GetSeq() { - return nil, ErrSignalMessageDropped - } - if r.seq > msg.GetSeq() { - n := int(r.seq - msg.GetSeq()) - if n > len(res) { - n = len(res) - } - res = res[n:] - } - r.seq += uint64(len(res)) + if r.seq < msg.GetSeq() { + return nil, ErrSignalMessageDropped } + if r.seq > msg.GetSeq() { + n := int(r.seq - msg.GetSeq()) + if n > len(res) { + n = len(res) + } + res = res[n:] + } + r.seq += uint64(len(res)) + return res, nil } @@ -256,7 +250,8 @@ func (s *signalMessageSink[SendType, RecvType]) Close() { s.mu.Lock() s.draining = true if !s.writing { - s.Stream.Close(nil) + s.writing = true + go s.write() } s.mu.Unlock() @@ -267,16 +262,6 @@ func (s *signalMessageSink[SendType, RecvType]) IsClosed() bool { return s.Stream.Err() != nil } -func (s *signalMessageSink[SendType, RecvType]) nextMessage() (msg SendType, n int) { - if len(s.queue) == 0 { - return - } - if s.Config.MinVersion >= 1 { - return s.Writer.WriteMany(s.seq, s.queue), len(s.queue) - } - return s.Writer.WriteOne(s.seq, s.queue[0]), 1 -} - func (s *signalMessageSink[SendType, RecvType]) write() { interval := s.Config.MinRetryInterval deadline := time.Now().Add(s.Config.RetryTimeout) @@ -284,10 +269,11 @@ func (s *signalMessageSink[SendType, RecvType]) write() { s.mu.Lock() for { - msg, n := s.nextMessage() - if n == 0 || s.IsClosed() { + close := s.draining + if (!close && len(s.queue) == 0) || s.IsClosed() { break } + msg, n := s.Writer.Write(s.seq, close, s.queue), len(s.queue) s.mu.Unlock() err = s.Stream.Send(msg, psrpc.WithTimeout(interval)) @@ -314,6 +300,10 @@ func (s *signalMessageSink[SendType, RecvType]) write() { s.seq += uint64(n) s.queue = s.queue[n:] + + if close { + break + } } } diff --git a/pkg/rtc/signalhandler.go b/pkg/rtc/signalhandler.go index e8749aa64..ee349201f 100644 --- a/pkg/rtc/signalhandler.go +++ b/pkg/rtc/signalhandler.go @@ -10,7 +10,7 @@ import ( func HandleParticipantSignal(room types.Room, participant types.LocalParticipant, req *livekit.SignalRequest, pLogger logger.Logger) error { participant.UpdateLastSeenSignal() - switch msg := req.Message.(type) { + switch msg := req.GetMessage().(type) { case *livekit.SignalRequest_Offer: participant.HandleOffer(FromProtoSessionDescription(msg.Offer)) case *livekit.SignalRequest_Answer: diff --git a/pkg/service/signal.go b/pkg/service/signal.go index 87b29d527..405ae79cb 100644 --- a/pkg/service/signal.go +++ b/pkg/service/signal.go @@ -162,17 +162,11 @@ func (r *signalService) RelaySignal(stream psrpc.ServerStream[*rpc.RelaySignalRe type signalResponseMessageWriter struct{} -func (e signalResponseMessageWriter) WriteOne(seq uint64, msg proto.Message) *rpc.RelaySignalResponse { - return &rpc.RelaySignalResponse{ - Seq: seq, - Response: msg.(*livekit.SignalResponse), - } -} - -func (e signalResponseMessageWriter) WriteMany(seq uint64, msgs []proto.Message) *rpc.RelaySignalResponse { +func (e signalResponseMessageWriter) Write(seq uint64, close bool, msgs []proto.Message) *rpc.RelaySignalResponse { r := &rpc.RelaySignalResponse{ Seq: seq, Responses: make([]*livekit.SignalResponse, 0, len(msgs)), + Close: close, } for _, m := range msgs { r.Responses = append(r.Responses, m.(*livekit.SignalResponse)) @@ -183,10 +177,7 @@ func (e signalResponseMessageWriter) WriteMany(seq uint64, msgs []proto.Message) type signalRequestMessageReader struct{} func (e signalRequestMessageReader) Read(rm *rpc.RelaySignalRequest) ([]proto.Message, error) { - msgs := make([]proto.Message, 0, len(rm.Requests)+1) - if rm.Request != nil { - msgs = append(msgs, rm.Request) - } + msgs := make([]proto.Message, 0, len(rm.Requests)) for _, m := range rm.Requests { msgs = append(msgs, m) } From c1c4e8aea028626ae4e3d9170f8f0a0247b18736 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 27 Apr 2023 14:39:05 +0530 Subject: [PATCH 119/324] Include packetsMissing field in string representation (#1659) * Include packetsMissing field in string representation * do not set stub directly --- pkg/rtc/helper_test.go | 12 ++++++------ pkg/rtc/participant_internal_test.go | 8 ++++---- pkg/rtc/subscriptionmanager_test.go | 12 ++++++------ pkg/sfu/connectionquality/scorer.go | 3 ++- 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/pkg/rtc/helper_test.go b/pkg/rtc/helper_test.go index dc97f13a0..c26e7aebc 100644 --- a/pkg/rtc/helper_test.go +++ b/pkg/rtc/helper_test.go @@ -31,7 +31,7 @@ func newMockParticipant(identity livekit.ParticipantIdentity, protocol types.Pro IsPublisher: publisher, }) - p.SetMetadataStub = func(m string) { + p.SetMetadataCalls(func(m string) { var f func(participant types.LocalParticipant) if p.OnParticipantUpdateCallCount() > 0 { f = p.OnParticipantUpdateArgsForCall(p.OnParticipantUpdateCallCount() - 1) @@ -39,7 +39,7 @@ func newMockParticipant(identity livekit.ParticipantIdentity, protocol types.Pro if f != nil { f(p) } - } + }) updateTrack := func() { var f func(participant types.LocalParticipant, track types.MediaTrack) if p.OnTrackUpdatedCallCount() > 0 { @@ -50,12 +50,12 @@ func newMockParticipant(identity livekit.ParticipantIdentity, protocol types.Pro } } - p.SetTrackMutedStub = func(sid livekit.TrackID, muted bool, fromServer bool) { + p.SetTrackMutedCalls(func(sid livekit.TrackID, muted bool, fromServer bool) { updateTrack() - } - p.AddTrackStub = func(req *livekit.AddTrackRequest) { + }) + p.AddTrackCalls(func(req *livekit.AddTrackRequest) { updateTrack() - } + }) return p } diff --git a/pkg/rtc/participant_internal_test.go b/pkg/rtc/participant_internal_test.go index 74e0b38e0..1b2dadefb 100644 --- a/pkg/rtc/participant_internal_test.go +++ b/pkg/rtc/participant_internal_test.go @@ -422,7 +422,7 @@ func TestDisableCodecs(t *testing.T) { participant.SetResponseSink(sink) var answer webrtc.SessionDescription var answerReceived atomic.Bool - sink.WriteMessageStub = func(msg proto.Message) error { + sink.WriteMessageCalls(func(msg proto.Message) error { if res, ok := msg.(*livekit.SignalResponse); ok { if res.GetAnswer() != nil { answer = FromProtoSessionDescription(res.GetAnswer()) @@ -430,7 +430,7 @@ func TestDisableCodecs(t *testing.T) { } } return nil - } + }) participant.HandleOffer(sdp) testutils.WithTimeout(t, func() string { @@ -579,7 +579,7 @@ func TestPreferAudioCodecForRed(t *testing.T) { participant.SetResponseSink(sink) var answer webrtc.SessionDescription var answerReceived atomic.Bool - sink.WriteMessageStub = func(msg proto.Message) error { + sink.WriteMessageCalls(func(msg proto.Message) error { if res, ok := msg.(*livekit.SignalResponse); ok { if res.GetAnswer() != nil { answer = FromProtoSessionDescription(res.GetAnswer()) @@ -588,7 +588,7 @@ func TestPreferAudioCodecForRed(t *testing.T) { } } return nil - } + }) participant.HandleOffer(sdp) require.Eventually(t, func() bool { return answerReceived.Load() }, 5*time.Second, 10*time.Millisecond) diff --git a/pkg/rtc/subscriptionmanager_test.go b/pkg/rtc/subscriptionmanager_test.go index 0c3de6955..2bd005f45 100644 --- a/pkg/rtc/subscriptionmanager_test.go +++ b/pkg/rtc/subscriptionmanager_test.go @@ -214,9 +214,9 @@ func TestUnsubscribe(t *testing.T) { st.OnClose(func(willBeResumed bool) { sm.handleSubscribedTrackClose(s, willBeResumed) }) - res.Track.(*typesfakes.FakeMediaTrack).RemoveSubscriberStub = func(pID livekit.ParticipantID, willBeResumed bool) { + res.Track.(*typesfakes.FakeMediaTrack).RemoveSubscriberCalls(func(pID livekit.ParticipantID, willBeResumed bool) { setTestSubscribedTrackClosed(t, st, willBeResumed) - } + }) sm.lock.Lock() sm.subscriptions["track"] = s @@ -280,12 +280,12 @@ func TestSubscribeStatusChanged(t *testing.T) { st2.OnClose(func(willBeResumed bool) { sm.handleSubscribedTrackClose(s2, willBeResumed) }) - st1.MediaTrack().(*typesfakes.FakeMediaTrack).RemoveSubscriberStub = func(pID livekit.ParticipantID, willBeResumed bool) { + st1.MediaTrack().(*typesfakes.FakeMediaTrack).RemoveSubscriberCalls(func(pID livekit.ParticipantID, willBeResumed bool) { setTestSubscribedTrackClosed(t, st1, willBeResumed) - } - st2.MediaTrack().(*typesfakes.FakeMediaTrack).RemoveSubscriberStub = func(pID livekit.ParticipantID, willBeResumed bool) { + }) + st2.MediaTrack().(*typesfakes.FakeMediaTrack).RemoveSubscriberCalls(func(pID livekit.ParticipantID, willBeResumed bool) { setTestSubscribedTrackClosed(t, st2, willBeResumed) - } + }) require.Equal(t, int32(1), numParticipantSubscribed.Load()) require.Equal(t, int32(0), numParticipantUnsubscribed.Load()) diff --git a/pkg/sfu/connectionquality/scorer.go b/pkg/sfu/connectionquality/scorer.go index 4de60b05f..d4c98c316 100644 --- a/pkg/sfu/connectionquality/scorer.go +++ b/pkg/sfu/connectionquality/scorer.go @@ -101,11 +101,12 @@ func (w *windowStat) calculateBitrateScore(expectedBitrate int64) float64 { } func (w *windowStat) String() string { - return fmt.Sprintf("start: %+v, dur: %+v, pe: %d, pl: %d, b: %d, rtt: %d, jitter: %0.2f", + return fmt.Sprintf("start: %+v, dur: %+v, pe: %d, pl: %d, pm: %d, b: %d, rtt: %d, jitter: %0.2f", w.startedAt, w.duration, w.packetsExpected, w.packetsLost, + w.packetsMissing, w.bytes, w.rttMax, w.jitterMax, From fc09cacfcec6acc2f457bfc75c48ba6f04be68c8 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Thu, 27 Apr 2023 08:30:40 -0700 Subject: [PATCH 120/324] increase level for signal stream closed log messages (#1660) * increase level for signal stream closed log messages * ensure stream closes on signal close receipt * cleanup --- pkg/routing/signal.go | 6 +++--- pkg/service/signal.go | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/routing/signal.go b/pkg/routing/signal.go index 9a83c6054..56aa25005 100644 --- a/pkg/routing/signal.go +++ b/pkg/routing/signal.go @@ -111,13 +111,13 @@ func (r *signalClient) StartParticipantSignal( r.active.Inc() defer r.active.Dec() - err = CopySignalStreamToMessageChannel[*rpc.RelaySignalRequest, *rpc.RelaySignalResponse]( + err := CopySignalStreamToMessageChannel[*rpc.RelaySignalRequest, *rpc.RelaySignalResponse]( stream, resChan, signalResponseMessageReader{}, r.config, ) - l.Debugw("participant signal stream closed", "error", err) + l.Infow("signal stream closed", "error", err) resChan.Close() }() @@ -189,7 +189,7 @@ func CopySignalStreamToMessageChannel[SendType, RecvType RelaySignalMessage]( } if msg.GetClose() { - return psrpc.ErrStreamClosed + return stream.Close(nil) } } return stream.Err() diff --git a/pkg/service/signal.go b/pkg/service/signal.go index 405ae79cb..a8ce00811 100644 --- a/pkg/service/signal.go +++ b/pkg/service/signal.go @@ -152,10 +152,11 @@ func (r *signalService) RelaySignal(stream psrpc.ServerStream[*rpc.RelaySignalRe err = r.sessionHandler(ctx, livekit.RoomName(ss.RoomName), *pi, livekit.ConnectionID(ss.ConnectionId), reqChan, sink) if err != nil { l.Errorw("could not handle new participant", err) + return } err = routing.CopySignalStreamToMessageChannel[*rpc.RelaySignalResponse, *rpc.RelaySignalRequest](stream, reqChan, signalRequestMessageReader{}, r.config) - l.Debugw("participant signal stream closed", "error", err) + l.Infow("signal stream closed", "error", err) return } From 3f3b02357cfe74bd6446ada79249df6721b376a3 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 27 Apr 2023 23:06:15 +0530 Subject: [PATCH 121/324] Check all transport connected for subscriber only properly. (#1661) --- pkg/rtc/participant.go | 6 +++++- pkg/rtc/transportmanager.go | 8 ++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index cb9f68e33..fe8bc3c57 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -735,7 +735,11 @@ func (p *ParticipantImpl) clearMigrationTimer() { } func (p *ParticipantImpl) MaybeStartMigration(force bool, onStart func()) bool { - if !force && !p.TransportManager.HaveAllTransportEverConnected() { + allTransportConnected := p.TransportManager.HasSubscriberEverConnected() + if p.IsPublisher() { + allTransportConnected = allTransportConnected && p.TransportManager.HasPublisherEverConnected() + } + if !force && !allTransportConnected { return false } diff --git a/pkg/rtc/transportmanager.go b/pkg/rtc/transportmanager.go index de7c0a32b..397482dab 100644 --- a/pkg/rtc/transportmanager.go +++ b/pkg/rtc/transportmanager.go @@ -190,10 +190,6 @@ func (t *TransportManager) Close() { t.subscriber.Close() } -func (t *TransportManager) HaveAllTransportEverConnected() bool { - return t.publisher.HasEverConnected() && t.subscriber.HasEverConnected() -} - func (t *TransportManager) SubscriberClose() { t.subscriber.Close() } @@ -217,6 +213,10 @@ func (t *TransportManager) OnPublisherTrack(f func(track *webrtc.TrackRemote, rt t.publisher.OnTrack(f) } +func (t *TransportManager) HasPublisherEverConnected() bool { + return t.publisher.HasEverConnected() +} + func (t *TransportManager) IsPublisherEstablished() bool { return t.publisher.IsEstablished() } From 50ab72a5f820cc6a025e6260ae26b918370ec94f Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 28 Apr 2023 14:50:06 +0530 Subject: [PATCH 122/324] DownTrack scoring when RR is not received. (#1664) --- pkg/sfu/buffer/rtpstats.go | 178 ++++++++++++------- pkg/sfu/connectionquality/connectionstats.go | 175 +++++++++--------- pkg/sfu/connectionquality/scorer.go | 2 + pkg/sfu/downtrack.go | 74 ++++---- 4 files changed, 249 insertions(+), 180 deletions(-) diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index 3b9ec4a8d..06417365e 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -66,6 +66,7 @@ type RTPDeltaInfo struct { type Snapshot struct { startTime time.Time extStartSN uint32 + extStartSNOverridden uint32 packetsDuplicate uint32 bytesDuplicate uint64 headerBytesDuplicate uint64 @@ -294,7 +295,11 @@ func (r *RTPStats) NewSnapshotId() uint32 { id := r.nextSnapshotId if r.initialized { - r.snapshots[id] = &Snapshot{startTime: time.Now(), extStartSN: r.extStartSN} + r.snapshots[id] = &Snapshot{ + startTime: time.Now(), + extStartSN: r.extStartSN, + extStartSNOverridden: r.extStartSN, + } } r.nextSnapshotId++ @@ -334,7 +339,11 @@ func (r *RTPStats) Update(rtph *rtp.Header, payloadSize int, paddingSize int, pa // initialize snapshots if any for i := uint32(FirstSnapshotId); i < r.nextSnapshotId; i++ { - r.snapshots[i] = &Snapshot{startTime: r.startTime, extStartSN: r.extStartSN} + r.snapshots[i] = &Snapshot{ + startTime: r.startTime, + extStartSN: r.extStartSN, + extStartSNOverridden: r.extStartSN, + } } } @@ -526,6 +535,13 @@ func (r *RTPStats) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt uint32 return } +func (r *RTPStats) LastReceiverReport() time.Time { + r.lock.RLock() + defer r.lock.RUnlock() + + return r.lastRRTime +} + func (r *RTPStats) UpdateNack(nackCount uint32) { r.lock.Lock() defer r.lock.Unlock() @@ -777,7 +793,7 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32, srDataExt *RTCPSenderReportD func (r *RTPStats) SnapshotRtcpReceptionReport(ssrc uint32, proxyFracLost uint8, snapshotId uint32) *rtcp.ReceptionReport { r.lock.Lock() - then, now := r.getAndResetSnapshot(snapshotId) + then, now := r.getAndResetSnapshot(snapshotId, false) r.lock.Unlock() if now == nil || then == nil { @@ -799,17 +815,8 @@ func (r *RTPStats) SnapshotRtcpReceptionReport(ssrc uint32, proxyFracLost uint8, return nil } - packetsLost := uint32(0) - if r.params.IsReceiverReportDriven { - // receiver report driven should not be set for streams that need to generate reception report, but including code here for consistency - packetsLost = now.packetsLostOverridden - then.packetsLostOverridden - if int32(packetsLost) < 0 { - packetsLost = 0 - } - } else { - intervalStats := r.getIntervalStats(uint16(then.extStartSN), uint16(now.extStartSN)) - packetsLost = intervalStats.packetsLost - } + intervalStats := r.getIntervalStats(uint16(then.extStartSN), uint16(now.extStartSN)) + packetsLost := intervalStats.packetsLost lossRate := float32(packetsLost) / float32(packetsExpected) fracLost := uint8(lossRate * 256.0) if proxyFracLost > fracLost { @@ -823,12 +830,6 @@ func (r *RTPStats) SnapshotRtcpReceptionReport(ssrc uint32, proxyFracLost uint8, dlsr |= (delayMS % 1e3) * 65536 / 1000 } - jitter := r.jitter - if r.params.IsReceiverReportDriven { - // receiver report driven should not be set for streams that need to generate reception report, but including code here for consistency - jitter = r.jitterOverridden - } - lastSR := uint32(0) if r.srDataExt != nil { lastSR = uint32(r.srDataExt.SenderReportData.NTPTimestamp >> 16) @@ -838,7 +839,7 @@ func (r *RTPStats) SnapshotRtcpReceptionReport(ssrc uint32, proxyFracLost uint8, FractionLost: fracLost, TotalLost: r.packetsLost, LastSequenceNumber: now.extStartSN, - Jitter: uint32(jitter), + Jitter: uint32(r.jitter), LastSenderReport: lastSR, Delay: dlsr, } @@ -846,7 +847,7 @@ func (r *RTPStats) SnapshotRtcpReceptionReport(ssrc uint32, proxyFracLost uint8, func (r *RTPStats) DeltaInfo(snapshotId uint32) *RTPDeltaInfo { r.lock.Lock() - then, now := r.getAndResetSnapshot(snapshotId) + then, now := r.getAndResetSnapshot(snapshotId, false) r.lock.Unlock() if now == nil || then == nil { @@ -868,53 +869,93 @@ func (r *RTPStats) DeltaInfo(snapshotId uint32) *RTPDeltaInfo { return nil } if packetsExpected == 0 { - if r.params.IsReceiverReportDriven { - // not received RTCP RR (OR) publisher is not producing any data - return nil - } - return &RTPDeltaInfo{ StartTime: startTime, Duration: endTime.Sub(startTime), } } - packetsLost := uint32(0) - packetsMissing := uint32(0) intervalStats := r.getIntervalStats(uint16(then.extStartSN), uint16(now.extStartSN)) - if r.params.IsReceiverReportDriven { - packetsMissing = intervalStats.packetsLost + return &RTPDeltaInfo{ + StartTime: startTime, + Duration: endTime.Sub(startTime), + Packets: packetsExpected - intervalStats.packetsPadding, + Bytes: intervalStats.bytes, + HeaderBytes: intervalStats.headerBytes, + PacketsDuplicate: now.packetsDuplicate - then.packetsDuplicate, + BytesDuplicate: now.bytesDuplicate - then.bytesDuplicate, + HeaderBytesDuplicate: now.headerBytesDuplicate - then.headerBytesDuplicate, + PacketsPadding: intervalStats.packetsPadding, + BytesPadding: intervalStats.bytesPadding, + HeaderBytesPadding: intervalStats.headerBytesPadding, + PacketsLost: intervalStats.packetsLost, + Frames: intervalStats.frames, + RttMax: then.maxRtt, + JitterMax: then.maxJitter / float64(r.params.ClockRate) * 1e6, + Nacks: now.nacks - then.nacks, + Plis: now.plis - then.plis, + Firs: now.firs - then.firs, + } +} - packetsLost = now.packetsLostOverridden - then.packetsLostOverridden - if int32(packetsLost) < 0 { - packetsLost = 0 - } - - if packetsLost > packetsExpected { - r.logger.Warnw( - "unexpected number of packets lost", - fmt.Errorf( - "start: %d, end: %d, expected: %d, lost: report: %d, interval: %d", - then.extStartSN, - now.extStartSN, - packetsExpected, - now.packetsLostOverridden-then.packetsLostOverridden, - intervalStats.packetsLost, - ), - ) - packetsLost = packetsExpected - } - } else { - packetsLost = intervalStats.packetsLost +func (r *RTPStats) DeltaInfoOverridden(snapshotId uint32) *RTPDeltaInfo { + if !r.params.IsReceiverReportDriven { + return nil } - maxJitter := then.maxJitter - if r.params.IsReceiverReportDriven { - // discount jitter from publisher side + internal processing - maxJitter = then.maxJitterOverridden - maxJitter - if maxJitter < 0.0 { - maxJitter = 0.0 - } + r.lock.Lock() + then, now := r.getAndResetSnapshot(snapshotId, true) + r.lock.Unlock() + + if now == nil || then == nil { + return nil + } + + r.lock.RLock() + defer r.lock.RUnlock() + + startTime := then.startTime + endTime := now.startTime + + packetsExpected := now.extStartSNOverridden - then.extStartSNOverridden + if packetsExpected > NumSequenceNumbers { + r.logger.Warnw( + "too many packets expected in delta", + fmt.Errorf("start: %d, end: %d, expected: %d", then.extStartSNOverridden, now.extStartSNOverridden, packetsExpected), + ) + return nil + } + if packetsExpected == 0 { + // not received RTCP RR (OR) publisher is not producing any data + return nil + } + + intervalStats := r.getIntervalStats(uint16(then.extStartSNOverridden), uint16(now.extStartSNOverridden)) + packetsMissing := intervalStats.packetsLost + packetsLost := now.packetsLostOverridden - then.packetsLostOverridden + if int32(packetsLost) < 0 { + packetsLost = 0 + } + + if packetsLost > packetsExpected { + r.logger.Warnw( + "unexpected number of packets lost", + fmt.Errorf( + "start: %d, end: %d, expected: %d, lost: report: %d, interval: %d", + then.extStartSNOverridden, + now.extStartSNOverridden, + packetsExpected, + now.packetsLostOverridden-then.packetsLostOverridden, + intervalStats.packetsLost, + ), + ) + packetsLost = packetsExpected + } + + // discount jitter from publisher side + internal processing + maxJitter := then.maxJitterOverridden - then.maxJitter + if maxJitter < 0.0 { + maxJitter = 0.0 } maxJitterTime := maxJitter / float64(r.params.ClockRate) * 1e6 @@ -1288,7 +1329,7 @@ func (r *RTPStats) updateGapHistogram(gap int) { } } -func (r *RTPStats) getAndResetSnapshot(snapshotId uint32) (*Snapshot, *Snapshot) { +func (r *RTPStats) getAndResetSnapshot(snapshotId uint32, override bool) (*Snapshot, *Snapshot) { if !r.initialized || (r.params.IsReceiverReportDriven && r.lastRRTime.IsZero()) { return nil, nil } @@ -1296,16 +1337,25 @@ func (r *RTPStats) getAndResetSnapshot(snapshotId uint32) (*Snapshot, *Snapshot) then := r.snapshots[snapshotId] if then == nil { then = &Snapshot{ - startTime: r.startTime, - extStartSN: r.extStartSN, + startTime: r.startTime, + extStartSN: r.extStartSN, + extStartSNOverridden: r.extStartSN, } r.snapshots[snapshotId] = then } + var startTime time.Time + if override && r.params.IsReceiverReportDriven { + startTime = r.lastRRTime + } else { + startTime = time.Now() + } + // snapshot now r.snapshots[snapshotId] = &Snapshot{ - startTime: time.Now(), - extStartSN: r.getExtHighestSNAdjusted() + 1, + startTime: startTime, + extStartSN: r.getExtHighestSN() + 1, + extStartSNOverridden: r.getExtHighestSNAdjusted() + 1, packetsDuplicate: r.packetsDuplicate, bytesDuplicate: r.bytesDuplicate, headerBytesDuplicate: r.headerBytesDuplicate, diff --git a/pkg/sfu/connectionquality/connectionstats.go b/pkg/sfu/connectionquality/connectionstats.go index 7eca5d515..c611c7fc0 100644 --- a/pkg/sfu/connectionquality/connectionstats.go +++ b/pkg/sfu/connectionquality/connectionstats.go @@ -18,17 +18,19 @@ const ( UpdateInterval = 5 * time.Second processThreshold = 0.95 noStatsTooLongMultiplier = 2 - noReceiverReportTooLongThreshold = 10 * time.Second + noReceiverReportTooLongThreshold = 30 * time.Second ) type ConnectionStatsParams struct { - UpdateInterval time.Duration - MimeType string - IsFECEnabled bool - IsDependentRTT bool - IsDependentJitter bool - GetDeltaStats func() map[uint32]*buffer.StreamStatsWithLayers - Logger logger.Logger + UpdateInterval time.Duration + MimeType string + IsFECEnabled bool + IsDependentRTT bool + IsDependentJitter bool + GetDeltaStats func() map[uint32]*buffer.StreamStatsWithLayers + GetDeltaStatsOverridden func() map[uint32]*buffer.StreamStatsWithLayers + GetLastReceiverReportTime func() time.Time + Logger logger.Logger } type ConnectionStats struct { @@ -39,10 +41,8 @@ type ConnectionStats struct { onStatsUpdate func(cs *ConnectionStats, stat *livekit.AnalyticsStat) - lock sync.RWMutex - lastStatsAt time.Time - statsInProcess bool - lastReceiverReportAt time.Time + lock sync.RWMutex + streamingStartedAt time.Time scorer *qualityScorer @@ -69,8 +69,6 @@ func (cs *ConnectionStats) Start(trackInfo *livekit.TrackInfo, at time.Time) { cs.isVideo.Store(trackInfo.Type == livekit.TrackType_VIDEO) - cs.updateLastStatsAt(time.Now()) // force an initial wait - cs.scorer.Start(at) go cs.updateStatsWorker() @@ -104,21 +102,7 @@ func (cs *ConnectionStats) GetScoreAndQuality() (float32, livekit.ConnectionQual return cs.scorer.GetMOSAndQuality() } -func (cs *ConnectionStats) ReceiverReportReceived(at time.Time) { - cs.lock.Lock() - cs.lastReceiverReportAt = time.Now() - cs.lock.Unlock() - - cs.getStat(at) -} - -func (cs *ConnectionStats) updateScore(streams map[uint32]*buffer.StreamStatsWithLayers, at time.Time) float32 { - deltaInfoList := make([]*buffer.RTPDeltaInfo, 0, len(streams)) - for _, s := range streams { - deltaInfoList = append(deltaInfoList, s.RTPStats) - } - agg := buffer.AggregateRTPDeltaInfo(deltaInfoList) - +func (cs *ConnectionStats) updateScoreWithAggregate(agg *buffer.RTPDeltaInfo, at time.Time) float32 { var stat windowStat if agg != nil { stat.startedAt = agg.StartTime @@ -136,58 +120,86 @@ func (cs *ConnectionStats) updateScore(streams map[uint32]*buffer.StreamStatsWit return mos } -func (cs *ConnectionStats) maybeMarkInProcess() bool { - cs.lock.Lock() - defer cs.lock.Unlock() - - if cs.statsInProcess { - // already running - return false +func (cs *ConnectionStats) updateScoreFromReceiverReport(at time.Time) float32 { + if cs.params.GetDeltaStatsOverridden == nil || cs.params.GetLastReceiverReportTime == nil { + return MinMOS } - interval := cs.params.UpdateInterval - if interval == 0 { - interval = UpdateInterval - } - - if cs.isStarted.Load() && time.Since(cs.lastStatsAt) > time.Duration(processThreshold*float64(interval)) { - cs.statsInProcess = true - return true - } - - return false -} - -func (cs *ConnectionStats) updateLastStatsAt(at time.Time) { - cs.lock.Lock() - defer cs.lock.Unlock() - - cs.lastStatsAt = at -} - -func (cs *ConnectionStats) isTooLongSinceLastStats() bool { cs.lock.RLock() - defer cs.lock.RUnlock() - - interval := cs.params.UpdateInterval - if interval == 0 { - interval = UpdateInterval + streamingStartedAt := cs.streamingStartedAt + cs.lock.RUnlock() + if streamingStartedAt.IsZero() { + // not streaming, just return current score + mos, _ := cs.scorer.GetMOSAndQuality() + return mos } - return !cs.lastStatsAt.IsZero() && time.Since(cs.lastStatsAt) > interval*noStatsTooLongMultiplier + + streams := cs.params.GetDeltaStatsOverridden() + if len(streams) == 0 { + // check for receiver report not received for a while + marker := cs.params.GetLastReceiverReportTime() + if marker.IsZero() || streamingStartedAt.After(marker) { + marker = streamingStartedAt + } + if time.Since(marker) > noReceiverReportTooLongThreshold { + // have not received receiver report for a long time when streaming, run with nil stat + return cs.updateScoreWithAggregate(nil, at) + } + + // wait for receiver report, return current score + mos, _ := cs.scorer.GetMOSAndQuality() + return mos + } + + // delta stat duration could be large due to not receiving receiver report for a long time (for example, due to mute), + // adjust to streaming start if necessary + agg := toAggregateDeltaInfo(streams) + if streamingStartedAt.After(cs.params.GetLastReceiverReportTime()) { + // last receiver report was before streaming started, wait for next one + mos, _ := cs.scorer.GetMOSAndQuality() + return mos + } + + if streamingStartedAt.After(agg.StartTime) { + agg.Duration = agg.StartTime.Add(agg.Duration).Sub(streamingStartedAt) + agg.StartTime = streamingStartedAt + } + return cs.updateScoreWithAggregate(agg, at) } -func (cs *ConnectionStats) isTooLongSinceLastReceiverReport() bool { - cs.lock.RLock() - defer cs.lock.RUnlock() +func (cs *ConnectionStats) updateScore(streams map[uint32]*buffer.StreamStatsWithLayers, at time.Time) float32 { + deltaInfoList := make([]*buffer.RTPDeltaInfo, 0, len(streams)) + for _, s := range streams { + deltaInfoList = append(deltaInfoList, s.RTPStats) + } + agg := buffer.AggregateRTPDeltaInfo(deltaInfoList) + if agg != nil && agg.Packets > 0 { + // not very accurate as streaming could have started part way in the window, but don't need accurate time + cs.maybeSetStreamingStart(agg.StartTime) + } else { + cs.clearStreamingStart() + } - return !cs.lastReceiverReportAt.IsZero() && time.Since(cs.lastReceiverReportAt) > noReceiverReportTooLongThreshold + if cs.params.GetDeltaStatsOverridden != nil { + // receiver report based quality scoring, use stats from receiver report for scoring + return cs.updateScoreFromReceiverReport(at) + } + + return cs.updateScoreWithAggregate(agg, at) } -func (cs *ConnectionStats) clearInProcess() { +func (cs *ConnectionStats) maybeSetStreamingStart(at time.Time) { cs.lock.Lock() - defer cs.lock.Unlock() + if cs.streamingStartedAt.IsZero() { + cs.streamingStartedAt = at + } + cs.lock.Unlock() +} - cs.statsInProcess = false +func (cs *ConnectionStats) clearStreamingStart() { + cs.lock.Lock() + cs.streamingStartedAt = time.Time{} + cs.lock.Unlock() } func (cs *ConnectionStats) getStat(at time.Time) { @@ -195,24 +207,11 @@ func (cs *ConnectionStats) getStat(at time.Time) { return } - if !cs.maybeMarkInProcess() { - // not yet time to process - return - } - streams := cs.params.GetDeltaStats() if len(streams) == 0 { - if cs.isTooLongSinceLastStats() && cs.isTooLongSinceLastReceiverReport() { - cs.updateLastStatsAt(at) - cs.updateScore(streams, at) - } - cs.clearInProcess() return } - // stats available, update last stats time - cs.updateLastStatsAt(at) - score := cs.updateScore(streams, at) if cs.onStatsUpdate != nil { @@ -240,8 +239,6 @@ func (cs *ConnectionStats) getStat(at time.Time) { Mime: cs.params.MimeType, }) } - - cs.clearInProcess() } func (cs *ConnectionStats) updateStatsWorker() { @@ -308,6 +305,14 @@ func getPacketLossWeight(mimeType string, isFecEnabled bool) float64 { return plw } +func toAggregateDeltaInfo(streams map[uint32]*buffer.StreamStatsWithLayers) *buffer.RTPDeltaInfo { + deltaInfoList := make([]*buffer.RTPDeltaInfo, 0, len(streams)) + for _, s := range streams { + deltaInfoList = append(deltaInfoList, s.RTPStats) + } + return buffer.AggregateRTPDeltaInfo(deltaInfoList) +} + func toAnalyticsStream(ssrc uint32, deltaStats *buffer.RTPDeltaInfo) *livekit.AnalyticsStream { return &livekit.AnalyticsStream{ Ssrc: ssrc, diff --git a/pkg/sfu/connectionquality/scorer.go b/pkg/sfu/connectionquality/scorer.go index d4c98c316..0c0a40e49 100644 --- a/pkg/sfu/connectionquality/scorer.go +++ b/pkg/sfu/connectionquality/scorer.go @@ -12,6 +12,7 @@ import ( const ( MaxMOS = float32(4.5) + MinMOS = float32(1.0) maxScore = float64(100.0) poorScore = float64(30.0) @@ -362,6 +363,7 @@ func (q *qualityScorer) getPacketLossWeight(stat *windowStat) float64 { pps := float64(stat.packetsExpected) / stat.duration.Seconds() if pps > q.maxPPS { q.maxPPS = pps + q.params.Logger.Debugw("updating maxPPS", "expected", stat.packetsExpected, "duration", stat.duration.Seconds(), "pps", pps) } if q.maxPPS == 0 { diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 538b524e1..6ad785500 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -107,9 +107,10 @@ var ( // ------------------------------------------------------------------- type DownTrackState struct { - RTPStats *buffer.RTPStats - DeltaStatsSnapshotId uint32 - ForwarderState ForwarderState + RTPStats *buffer.RTPStats + DeltaStatsSnapshotId uint32 + DeltaStatsOverriddenSnapshotId uint32 + ForwarderState ForwarderState } func (d DownTrackState) String() string { @@ -216,8 +217,9 @@ type DownTrack struct { blankFramesGeneration atomic.Uint32 - connectionStats *connectionquality.ConnectionStats - deltaStatsSnapshotId uint32 + connectionStats *connectionquality.ConnectionStats + deltaStatsSnapshotId uint32 + deltaStatsOverriddenSnapshotId uint32 // for throttling error logs writeIOErrors atomic.Uint32 @@ -281,25 +283,28 @@ func NewDownTrack( } }) - d.connectionStats = connectionquality.NewConnectionStats(connectionquality.ConnectionStatsParams{ - MimeType: codecs[0].MimeType, // LK-TODO have to notify on codec change - IsFECEnabled: strings.EqualFold(codecs[0].MimeType, webrtc.MimeTypeOpus) && strings.Contains(strings.ToLower(codecs[0].SDPFmtpLine), "fec"), - IsDependentJitter: true, - GetDeltaStats: d.getDeltaStats, - Logger: d.logger.WithValues("direction", "down"), - }) - d.connectionStats.OnStatsUpdate(func(_cs *connectionquality.ConnectionStats, stat *livekit.AnalyticsStat) { - if d.onStatsUpdate != nil { - d.onStatsUpdate(d, stat) - } - }) - d.rtpStats = buffer.NewRTPStats(buffer.RTPStatsParams{ ClockRate: d.codec.ClockRate, IsReceiverReportDriven: true, Logger: d.logger, }) d.deltaStatsSnapshotId = d.rtpStats.NewSnapshotId() + d.deltaStatsOverriddenSnapshotId = d.rtpStats.NewSnapshotId() + + d.connectionStats = connectionquality.NewConnectionStats(connectionquality.ConnectionStatsParams{ + MimeType: codecs[0].MimeType, // LK-TODO have to notify on codec change + IsFECEnabled: strings.EqualFold(codecs[0].MimeType, webrtc.MimeTypeOpus) && strings.Contains(strings.ToLower(codecs[0].SDPFmtpLine), "fec"), + IsDependentJitter: true, + GetDeltaStats: d.getDeltaStats, + GetDeltaStatsOverridden: d.getDeltaStatsOverridden, + GetLastReceiverReportTime: func() time.Time { return d.rtpStats.LastReceiverReport() }, + Logger: d.logger.WithValues("direction", "down"), + }) + d.connectionStats.OnStatsUpdate(func(_cs *connectionquality.ConnectionStats, stat *livekit.AnalyticsStat) { + if d.onStatsUpdate != nil { + d.onStatsUpdate(d, stat) + } + }) return d, nil } @@ -913,16 +918,19 @@ func (d *DownTrack) MaxLayer() buffer.VideoLayer { } func (d *DownTrack) GetState() DownTrackState { - return DownTrackState{ - RTPStats: d.rtpStats, - DeltaStatsSnapshotId: d.deltaStatsSnapshotId, - ForwarderState: d.forwarder.GetState(), + dts := DownTrackState{ + RTPStats: d.rtpStats, + DeltaStatsSnapshotId: d.deltaStatsSnapshotId, + DeltaStatsOverriddenSnapshotId: d.deltaStatsOverriddenSnapshotId, + ForwarderState: d.forwarder.GetState(), } + return dts } func (d *DownTrack) SeedState(state DownTrackState) { d.rtpStats.Seed(state.RTPStats) d.deltaStatsSnapshotId = state.DeltaStatsSnapshotId + d.deltaStatsOverriddenSnapshotId = state.DeltaStatsOverriddenSnapshotId d.forwarder.SeedState(state.ForwarderState) } @@ -1341,8 +1349,6 @@ func (d *DownTrack) handleRTCP(bytes []byte) { l(d, rr) } d.listenerLock.RUnlock() - - d.connectionStats.ReceiverReportReceived(time.Now()) } case *rtcp.TransportLayerNack: @@ -1626,24 +1632,30 @@ func (d *DownTrack) GetTrackStats() *livekit.RTPStats { return d.rtpStats.ToProto() } -func (d *DownTrack) getDeltaStats() map[uint32]*buffer.StreamStatsWithLayers { - streamStats := make(map[uint32]*buffer.StreamStatsWithLayers, 1) - - deltaStats := d.rtpStats.DeltaInfo(d.deltaStatsSnapshotId) - if deltaStats == nil { +func (d *DownTrack) deltaStats(ds *buffer.RTPDeltaInfo) map[uint32]*buffer.StreamStatsWithLayers { + if ds == nil { return nil } + streamStats := make(map[uint32]*buffer.StreamStatsWithLayers, 1) streamStats[d.ssrc] = &buffer.StreamStatsWithLayers{ - RTPStats: deltaStats, + RTPStats: ds, Layers: map[int32]*buffer.RTPDeltaInfo{ - 0: deltaStats, + 0: ds, }, } return streamStats } +func (d *DownTrack) getDeltaStats() map[uint32]*buffer.StreamStatsWithLayers { + return d.deltaStats(d.rtpStats.DeltaInfo(d.deltaStatsSnapshotId)) +} + +func (d *DownTrack) getDeltaStatsOverridden() map[uint32]*buffer.StreamStatsWithLayers { + return d.deltaStats(d.rtpStats.DeltaInfoOverridden(d.deltaStatsOverriddenSnapshotId)) +} + func (d *DownTrack) GetNackStats() (totalPackets uint32, totalRepeatedNACKs uint32) { totalPackets = d.rtpStats.GetTotalPacketsPrimary() totalRepeatedNACKs = d.totalRepeatedNACKs.Load() From 1148d389781b743a0d8a38989a97f51f810c100a Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 28 Apr 2023 17:02:31 +0530 Subject: [PATCH 123/324] hopefully more stable tests (#1665) * hopefully more stable tests * do eventual checks as some callbacks happen in go routines. Needs a bit more work to ensure that some conditions do not happen. But, with goroutines, the amount of wait is always tricky.`` --- pkg/rtc/room_test.go | 2 +- pkg/rtc/subscriptionmanager_test.go | 27 ++++++++++++++++++++------- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/pkg/rtc/room_test.go b/pkg/rtc/room_test.go index 35cc3caa9..66076b98a 100644 --- a/pkg/rtc/room_test.go +++ b/pkg/rtc/room_test.go @@ -678,7 +678,7 @@ func TestRoomUpdate(t *testing.T) { // p1 should have received an update time.Sleep(2 * defaultDelay) - require.Equal(t, 1, p1.SendRoomUpdateCallCount()) + require.LessOrEqual(t, 1, p1.SendRoomUpdateCallCount()) require.EqualValues(t, 2, p1.SendRoomUpdateArgsForCall(p1.SendRoomUpdateCallCount()-1).NumParticipants) }) diff --git a/pkg/rtc/subscriptionmanager_test.go b/pkg/rtc/subscriptionmanager_test.go index 2bd005f45..c5fe60f59 100644 --- a/pkg/rtc/subscriptionmanager_test.go +++ b/pkg/rtc/subscriptionmanager_test.go @@ -88,7 +88,6 @@ func TestSubscribe(t *testing.T) { // ensure bound setTestSubscribedTrackBound(t, s.getSubscribedTrack()) - require.Eventually(t, func() bool { return !s.needsBind() }, subSettleTimeout, subCheckInterval, "track was not bound") @@ -99,9 +98,13 @@ func TestSubscribe(t *testing.T) { time.Sleep(notFoundTimeout) require.False(t, failed.Load()) + resolver.SetPause(true) // ensure its resilience after being closed setTestSubscribedTrackClosed(t, s.getSubscribedTrack(), false) - require.True(t, s.needsSubscribe()) + require.Eventually(t, func() bool { + return s.needsSubscribe() + }, subSettleTimeout, subCheckInterval, "needs subscribe did not persist across track close") + resolver.SetPause(false) require.Eventually(t, func() bool { return s.isDesired() && !s.needsSubscribe() @@ -287,7 +290,9 @@ func TestSubscribeStatusChanged(t *testing.T) { setTestSubscribedTrackClosed(t, st2, willBeResumed) }) - require.Equal(t, int32(1), numParticipantSubscribed.Load()) + require.Eventually(t, func() bool { + return numParticipantSubscribed.Load() == 1 + }, subSettleTimeout, subCheckInterval, "should be subscribed to publisher") require.Equal(t, int32(0), numParticipantUnsubscribed.Load()) require.True(t, sm.IsSubscribedTo("pubID")) @@ -303,7 +308,9 @@ func TestSubscribeStatusChanged(t *testing.T) { require.Eventually(t, func() bool { return !s1.needsUnsubscribe() }, subSettleTimeout, subCheckInterval, "track1 should be unsubscribed") - require.Equal(t, int32(1), numParticipantUnsubscribed.Load()) + require.Eventually(t, func() bool { + return numParticipantUnsubscribed.Load() == 1 + }, subSettleTimeout, subCheckInterval, "should be subscribed to publisher") require.False(t, sm.IsSubscribedTo("pubID")) } @@ -387,7 +394,6 @@ func TestSubscriptionLimits(t *testing.T) { // ensure bound setTestSubscribedTrackBound(t, s.getSubscribedTrack()) - require.Eventually(t, func() bool { return !s.needsBind() }, subSettleTimeout, subCheckInterval, "track was not bound") @@ -423,7 +429,6 @@ func TestSubscriptionLimits(t *testing.T) { // ensure bound setTestSubscribedTrackBound(t, s2.getSubscribedTrack()) - require.Eventually(t, func() bool { return !s2.needsBind() }, subSettleTimeout, subCheckInterval, "track was not bound") @@ -474,6 +479,8 @@ type testResolver struct { hasTrack bool pubIdentity livekit.ParticipantIdentity pubID livekit.ParticipantID + + paused bool } func newTestResolver(hasPermission bool, hasTrack bool, pubIdentity livekit.ParticipantIdentity, pubID livekit.ParticipantID) *testResolver { @@ -485,6 +492,12 @@ func newTestResolver(hasPermission bool, hasTrack bool, pubIdentity livekit.Part } } +func (t *testResolver) SetPause(paused bool) { + t.lock.Lock() + defer t.lock.Unlock() + t.paused = paused +} + func (t *testResolver) Resolve(identity livekit.ParticipantIdentity, trackID livekit.TrackID) types.MediaResolverResult { t.lock.Lock() defer t.lock.Unlock() @@ -495,7 +508,7 @@ func (t *testResolver) Resolve(identity livekit.ParticipantIdentity, trackID liv PublisherID: t.pubID, PublisherIdentity: t.pubIdentity, } - if t.hasTrack { + if t.hasTrack && !t.paused { mt := &typesfakes.FakeMediaTrack{} st := &typesfakes.FakeSubscribedTrack{} st.IDReturns(trackID) From a08cd23b6d3a6b3cc5d640a625b5ccdafaf07edf Mon Sep 17 00:00:00 2001 From: Benjamin Pracht Date: Fri, 28 Apr 2023 10:51:41 -0700 Subject: [PATCH 124/324] Adopt pion logging initialization moving to protocol (#1667) --- cmd/server/main.go | 3 +- go.mod | 6 +- go.sum | 4 +- pkg/config/config.go | 11 ++++ pkg/logger/logadapter.go | 118 ------------------------------------ pkg/logger/logger.go | 66 -------------------- pkg/rtc/config.go | 4 +- pkg/rtc/room_test.go | 3 +- pkg/rtc/transport.go | 4 +- pkg/service/turn.go | 4 +- test/integration_helpers.go | 3 +- 11 files changed, 25 insertions(+), 201 deletions(-) delete mode 100644 pkg/logger/logadapter.go delete mode 100644 pkg/logger/logger.go diff --git a/cmd/server/main.go b/cmd/server/main.go index 31abfaae2..29b8a6588 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -12,7 +12,6 @@ import ( "github.com/urfave/cli/v2" - serverlogger "github.com/livekit/livekit-server/pkg/logger" "github.com/livekit/livekit-server/pkg/rtc" "github.com/livekit/livekit-server/pkg/telemetry/prometheus" "github.com/livekit/protocol/logger" @@ -188,7 +187,7 @@ func getConfig(c *cli.Context) (*config.Config, error) { if err != nil { return nil, err } - serverlogger.InitFromConfig(conf.Logging) + config.InitLoggerFromConfig(conf.Logging) if c.String("config") == "" && c.String("config-body") == "" && conf.Development { // use single port UDP when no config is provided diff --git a/go.mod b/go.mod index ccbf1201a..8b996b883 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 - github.com/livekit/protocol v1.5.6-0.20230427055046-79477e28a150 + github.com/livekit/protocol v1.5.6-0.20230428011359-db5afb1c7f9b github.com/livekit/psrpc v0.3.1-0.20230425025640-5390915734c3 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.14.0 @@ -28,7 +28,6 @@ require ( github.com/pion/dtls/v2 v2.2.6 github.com/pion/ice/v2 v2.3.2 github.com/pion/interceptor v0.1.13 - github.com/pion/logging v0.2.2 github.com/pion/rtcp v1.2.10 github.com/pion/rtp v1.7.13 github.com/pion/sdp/v3 v3.0.6 @@ -47,7 +46,6 @@ require ( github.com/urfave/cli/v2 v2.25.1 github.com/urfave/negroni/v3 v3.0.0 go.uber.org/atomic v1.10.0 - go.uber.org/zap v1.24.0 golang.org/x/sync v0.1.0 google.golang.org/protobuf v1.30.0 gopkg.in/yaml.v3 v3.0.1 @@ -79,6 +77,7 @@ require ( github.com/nats-io/nkeys v0.4.4 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/pion/datachannel v1.5.5 // indirect + github.com/pion/logging v0.2.2 // indirect github.com/pion/mdns v0.0.7 // indirect github.com/pion/randutil v0.1.0 // indirect github.com/pion/sctp v1.8.6 // indirect @@ -91,6 +90,7 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect go.uber.org/multierr v1.6.0 // indirect + go.uber.org/zap v1.24.0 // indirect golang.org/x/crypto v0.8.0 // indirect golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 // indirect golang.org/x/mod v0.8.0 // indirect diff --git a/go.sum b/go.sum index f3769ea35..052a35a44 100644 --- a/go.sum +++ b/go.sum @@ -121,8 +121,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 h1:QlQFyMwCDgjyySsrgmrMcVbEBA6KZcyTzvK+z346tUA= github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26/go.mod h1:eDA41kiySZoG+wy4Etsjb3w0jjLx69i/vAmSjG4bteA= -github.com/livekit/protocol v1.5.6-0.20230427055046-79477e28a150 h1:jN0fW8H8Qgi5xsmmbk1s2qsXg1Y873zeWghE4mRV1Cc= -github.com/livekit/protocol v1.5.6-0.20230427055046-79477e28a150/go.mod h1:MBW05GWdhbl+o6u2gLLCQtDvr9EvcV4VWckpIYtoM2c= +github.com/livekit/protocol v1.5.6-0.20230428011359-db5afb1c7f9b h1:UEFMJr1OTF0yAX9mRRaQQ/YrTA6H7hCkbuABWfT6wLk= +github.com/livekit/protocol v1.5.6-0.20230428011359-db5afb1c7f9b/go.mod h1:MBW05GWdhbl+o6u2gLLCQtDvr9EvcV4VWckpIYtoM2c= github.com/livekit/psrpc v0.3.1-0.20230425025640-5390915734c3 h1:NXcxrluYLng7LTHcYNOj/MdR4SHWrKQAG2G+U930mTA= github.com/livekit/psrpc v0.3.1-0.20230425025640-5390915734c3/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= diff --git a/pkg/config/config.go b/pkg/config/config.go index bff9d19e1..f0dc1e1c1 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -14,6 +14,7 @@ import ( "gopkg.in/yaml.v3" "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/logger/pionlogger" redisLiveKit "github.com/livekit/protocol/redis" ) @@ -744,3 +745,13 @@ func (conf *Config) unmarshalKeys(keys string) error { } return nil } + +// Note: only pass in logr.Logger with default depth +func SetLogger(l logger.Logger) { + logger.SetLogger(l, "livekit") +} + +func InitLoggerFromConfig(config LoggingConfig) { + pionlogger.SetLogLevel(config.PionLevel) + logger.InitFromConfig(config.Config, "livekit") +} diff --git a/pkg/logger/logadapter.go b/pkg/logger/logadapter.go deleted file mode 100644 index a55a68ed0..000000000 --- a/pkg/logger/logadapter.go +++ /dev/null @@ -1,118 +0,0 @@ -package serverlogger - -import ( - "fmt" - "strings" - - "go.uber.org/zap/zapcore" - - "github.com/livekit/protocol/logger" -) - -// implements webrtc.LeveledLogger -type logAdapter struct { - logger logger.Logger - level zapcore.Level - ignoredPrefixes []string -} - -func (l *logAdapter) Trace(msg string) { - // ignore trace -} - -func (l *logAdapter) Tracef(format string, args ...interface{}) { - // ignore trace -} - -func (l *logAdapter) Debug(msg string) { - if l.level > zapcore.DebugLevel { - return - } - if l.shouldIgnore(msg) { - return - } - l.logger.Debugw(msg) -} - -func (l *logAdapter) Debugf(format string, args ...interface{}) { - if l.level > zapcore.DebugLevel { - return - } - msg := fmt.Sprintf(format, args...) - if l.shouldIgnore(msg) { - return - } - l.logger.Debugw(msg) -} - -func (l *logAdapter) Info(msg string) { - if l.level > zapcore.InfoLevel { - return - } - if l.shouldIgnore(msg) { - return - } - l.logger.Infow(msg) -} - -func (l *logAdapter) Infof(format string, args ...interface{}) { - if l.level > zapcore.InfoLevel { - return - } - msg := fmt.Sprintf(format, args...) - if l.shouldIgnore(msg) { - return - } - l.logger.Infow(msg) -} - -func (l *logAdapter) Warn(msg string) { - if l.level > zapcore.WarnLevel { - return - } - if l.shouldIgnore(msg) { - return - } - l.logger.Warnw(msg, nil) -} - -func (l *logAdapter) Warnf(format string, args ...interface{}) { - if l.level > zapcore.WarnLevel { - return - } - msg := fmt.Sprintf(format, args...) - if l.shouldIgnore(msg) { - return - } - l.logger.Warnw(msg, nil) -} - -func (l *logAdapter) Error(msg string) { - if l.level > zapcore.ErrorLevel { - return - } - if l.shouldIgnore(msg) { - return - } - l.logger.Errorw(msg, nil) -} - -func (l *logAdapter) Errorf(format string, args ...interface{}) { - if l.level > zapcore.ErrorLevel { - return - } - msg := fmt.Sprintf(format, args...) - if l.shouldIgnore(msg) { - return - } - l.logger.Errorw(msg, nil) -} - -func (l *logAdapter) shouldIgnore(msg string) bool { - for _, prefix := range l.ignoredPrefixes { - if strings.HasPrefix(msg, prefix) { - return true - } - } - return false -} diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go deleted file mode 100644 index 7449ef555..000000000 --- a/pkg/logger/logger.go +++ /dev/null @@ -1,66 +0,0 @@ -package serverlogger - -import ( - "github.com/pion/logging" - "go.uber.org/zap/zapcore" - - "github.com/livekit/protocol/logger" - - "github.com/livekit/livekit-server/pkg/config" -) - -var ( - pionLevel zapcore.Level - pionIgnoredPrefixes = map[string][]string{ - "ice": { - "pingAllCandidates called with no candidate pairs", - "failed to send packet: io: read/write on closed pipe", - "Ignoring remote candidate with tcpType active", - "discard message from", - "Failed to discover mDNS candidate", - "Failed to read from candidate tcp", - "remote mDNS candidate added, but mDNS is disabled", - }, - "pc": { - "Failed to accept RTCP stream is already closed", - "Failed to accept RTP stream is already closed", - "Incoming unhandled RTCP ssrc", - }, - "tcp_mux": { - "Error reading first packet from", - "error closing connection", - }, - "turn": { - "error when handling datagram", - }, - } -) - -// implements webrtc.LoggerFactory -type LoggerFactory struct { - logger logger.Logger -} - -func NewLoggerFactory(logger logger.Logger) *LoggerFactory { - return &LoggerFactory{ - logger: logger, - } -} - -func (f *LoggerFactory) NewLogger(scope string) logging.LeveledLogger { - return &logAdapter{ - logger: f.logger.WithName(scope), - level: pionLevel, - ignoredPrefixes: pionIgnoredPrefixes[scope], - } -} - -// Note: only pass in logr.Logger with default depth -func SetLogger(l logger.Logger) { - logger.SetLogger(l, "livekit") -} - -func InitFromConfig(config config.LoggingConfig) { - pionLevel = logger.ParseZapLevel(config.PionLevel) - logger.InitFromConfig(config.Config, "livekit") -} diff --git a/pkg/rtc/config.go b/pkg/rtc/config.go index 027c05637..b87ed857f 100644 --- a/pkg/rtc/config.go +++ b/pkg/rtc/config.go @@ -15,10 +15,10 @@ import ( "github.com/pion/webrtc/v3" "github.com/livekit/livekit-server/pkg/config" - logging "github.com/livekit/livekit-server/pkg/logger" "github.com/livekit/livekit-server/pkg/sfu/buffer" dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/logger/pionlogger" ) const ( @@ -73,7 +73,7 @@ func NewWebRTCConfig(conf *config.Config, externalIP string) (*WebRTCConfig, err SDPSemantics: webrtc.SDPSemanticsUnifiedPlan, } s := webrtc.SettingEngine{ - LoggerFactory: logging.NewLoggerFactory(logger.GetLogger()), + LoggerFactory: pionlogger.NewLoggerFactory(logger.GetLogger()), } var ifFilter func(string) bool diff --git a/pkg/rtc/room_test.go b/pkg/rtc/room_test.go index 66076b98a..9179c14fc 100644 --- a/pkg/rtc/room_test.go +++ b/pkg/rtc/room_test.go @@ -14,7 +14,6 @@ import ( "github.com/livekit/protocol/webhook" "github.com/livekit/livekit-server/pkg/config" - serverlogger "github.com/livekit/livekit-server/pkg/logger" "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/rtc/types/typesfakes" "github.com/livekit/livekit-server/pkg/sfu/audio" @@ -30,7 +29,7 @@ const ( ) func init() { - serverlogger.InitFromConfig(config.LoggingConfig{ + config.InitLoggerFromConfig(config.LoggingConfig{ Config: logger.Config{Level: "debug"}, }) // allow immediate closure in testing diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index 00a7da629..0b438f4f3 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -22,10 +22,10 @@ import ( "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/logger/pionlogger" lksdp "github.com/livekit/protocol/sdp" "github.com/livekit/livekit-server/pkg/config" - serverlogger "github.com/livekit/livekit-server/pkg/logger" "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/sfu/streamallocator" "github.com/livekit/livekit-server/pkg/telemetry" @@ -298,7 +298,7 @@ func newPeerConnection(params TransportParams, onBandwidthEstimator func(estimat } } - lf := serverlogger.NewLoggerFactory(params.Logger) + lf := pionlogger.NewLoggerFactory(params.Logger) if lf != nil { se.LoggerFactory = lf } diff --git a/pkg/service/turn.go b/pkg/service/turn.go index 3cdd39072..325eca32b 100644 --- a/pkg/service/turn.go +++ b/pkg/service/turn.go @@ -11,9 +11,9 @@ import ( "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/logger/pionlogger" "github.com/livekit/livekit-server/pkg/config" - logging "github.com/livekit/livekit-server/pkg/logger" "github.com/livekit/livekit-server/pkg/telemetry" "github.com/livekit/livekit-server/pkg/telemetry/prometheus" ) @@ -39,7 +39,7 @@ func NewTurnServer(conf *config.Config, authHandler turn.AuthHandler, standalone serverConfig := turn.ServerConfig{ Realm: LivekitRealm, AuthHandler: authHandler, - LoggerFactory: logging.NewLoggerFactory(logger.GetLogger()), + LoggerFactory: pionlogger.NewLoggerFactory(logger.GetLogger()), } var relayAddrGen turn.RelayAddressGenerator = &turn.RelayAddressGeneratorPortRange{ RelayAddress: net.ParseIP(conf.RTC.NodeIP), diff --git a/test/integration_helpers.go b/test/integration_helpers.go index ab2f10a23..a9e633f83 100644 --- a/test/integration_helpers.go +++ b/test/integration_helpers.go @@ -12,7 +12,6 @@ import ( "github.com/twitchtv/twirp" "github.com/livekit/livekit-server/pkg/config" - serverlogger "github.com/livekit/livekit-server/pkg/logger" "github.com/livekit/livekit-server/pkg/routing" "github.com/livekit/livekit-server/pkg/service" "github.com/livekit/livekit-server/pkg/telemetry/prometheus" @@ -42,7 +41,7 @@ const ( var roomClient livekit.RoomService func init() { - serverlogger.InitFromConfig(config.LoggingConfig{ + config.InitLoggerFromConfig(config.LoggingConfig{ Config: logger.Config{Level: "debug"}, }) From 35b8319b08cc749ad6500ea71dbc7df8cf74450f Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sat, 29 Apr 2023 09:18:07 +0530 Subject: [PATCH 125/324] Remove disallowed subscriptions on close. (#1668) With subscription manager, there is no need to tell a publisher about a subscriber going away. Before subscription manager, the up track manager of a participant (i. e. the publisher side) was holding a list of pending subscriptions for its published tracks and that had to be cleaned up if one of the subscriber goes away. That is not the case any more. Also set publisherID early so that subscription permission update has the right publisherID. In fact, saw an empty ID in the logs and saw that we still have the disallowed subscription handling which is not necessary any more. --- pkg/rtc/participant.go | 24 +++---------------- pkg/rtc/room.go | 14 ----------- pkg/rtc/subscriptionmanager.go | 3 ++- pkg/rtc/types/interfaces.go | 2 +- .../typesfakes/fake_local_participant.go | 12 +++++----- pkg/service/roommanager.go | 4 +--- 6 files changed, 13 insertions(+), 46 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index fe8bc3c57..793defe93 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -126,8 +126,6 @@ type ParticipantImpl struct { *UpTrackManager *SubscriptionManager - // tracks and participants that this participant isn't allowed to subscribe to - disallowedSubscriptions map[livekit.TrackID]livekit.ParticipantID // trackID -> publisherID // keeps track of unpublished tracks in order to reuse trackID unpublishedTracks []*livekit.TrackInfo @@ -163,7 +161,7 @@ type ParticipantImpl struct { migrateState atomic.Value // types.MigrateState - onClose func(types.LocalParticipant, map[livekit.TrackID]livekit.ParticipantID) + onClose func(types.LocalParticipant) onClaimsChanged func(participant types.LocalParticipant) onICEConfigChanged func(participant types.LocalParticipant, iceConfig *livekit.ICEConfig) @@ -187,7 +185,6 @@ func NewParticipant(params ParticipantParams) (*ParticipantImpl, error) { rtcpCh: make(chan []rtcp.Packet, 100), pendingTracks: make(map[string]*pendingTrackInfo), pendingPublishingTracks: make(map[livekit.TrackID]*pendingTrackInfo), - disallowedSubscriptions: make(map[livekit.TrackID]livekit.ParticipantID), connectedAt: time.Now(), rttUpdatedAt: time.Now(), cachedDownTracks: make(map[livekit.TrackID]*downTrackState), @@ -496,7 +493,7 @@ func (p *ParticipantImpl) OnDataPacket(callback func(types.LocalParticipant, *li p.lock.Unlock() } -func (p *ParticipantImpl) OnClose(callback func(types.LocalParticipant, map[livekit.TrackID]livekit.ParticipantID)) { +func (p *ParticipantImpl) OnClose(callback func(types.LocalParticipant)) { p.lock.Lock() p.onClose = callback p.lock.Unlock() @@ -684,13 +681,6 @@ func (p *ParticipantImpl) Close(sendLeave bool, reason types.ParticipantCloseRea p.UpTrackManager.Close(!sendLeave) - p.lock.Lock() - disallowedSubscriptions := make(map[livekit.TrackID]livekit.ParticipantID) - for trackID, publisherID := range p.disallowedSubscriptions { - disallowedSubscriptions[trackID] = publisherID - } - p.lock.Unlock() - p.updateState(livekit.ParticipantInfo_DISCONNECTED) // ensure this is synchronized @@ -699,7 +689,7 @@ func (p *ParticipantImpl) Close(sendLeave bool, reason types.ParticipantCloseRea onClose := p.onClose p.lock.RUnlock() if onClose != nil { - onClose(p, disallowedSubscriptions) + onClose(p) } // Close peer connections without blocking participant Close. If peer connections are gathering candidates @@ -976,14 +966,6 @@ func (p *ParticipantImpl) onTrackUnsubscribed(subTrack types.SubscribedTrack) { } func (p *ParticipantImpl) SubscriptionPermissionUpdate(publisherID livekit.ParticipantID, trackID livekit.TrackID, allowed bool) { - p.lock.Lock() - if allowed { - delete(p.disallowedSubscriptions, trackID) - } else { - p.disallowedSubscriptions[trackID] = publisherID - } - p.lock.Unlock() - p.params.Logger.Debugw("sending subscription permission update", "publisherID", publisherID, "trackID", trackID, "allowed", allowed) err := p.writeMessage(&livekit.SignalResponse{ Message: &livekit.SignalResponse_SubscriptionPermissionUpdate{ diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index ba5d5db44..72cbb206f 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -533,20 +533,6 @@ func (r *Room) UpdateSubscriptionPermission(participant types.LocalParticipant, return nil } -func (r *Room) RemoveDisallowedSubscriptions(sub types.LocalParticipant, disallowedSubscriptions map[livekit.TrackID]livekit.ParticipantID) { - for trackID, publisherID := range disallowedSubscriptions { - pub := r.GetParticipantByID(publisherID) - if pub == nil { - continue - } - - track := pub.GetPublishedTrack(trackID) - if track != nil { - track.RemoveSubscriber(sub.ID(), false) - } - } -} - func (r *Room) UpdateVideoLayers(participant types.Participant, updateVideoLayers *livekit.UpdateVideoLayers) error { return participant.UpdateVideoLayers(updateVideoLayers) } diff --git a/pkg/rtc/subscriptionmanager.go b/pkg/rtc/subscriptionmanager.go index 640e85d19..0e43f400f 100644 --- a/pkg/rtc/subscriptionmanager.go +++ b/pkg/rtc/subscriptionmanager.go @@ -449,6 +449,8 @@ func (m *SubscriptionManager) subscribe(s *trackSubscription) error { return ErrSubscriptionLimitExceeded } + s.setPublisher(res.PublisherIdentity, res.PublisherID) + // since hasPermission defaults to true, we will want to send a message to the client the first time // that we discover permissions were denied permChanged := s.setHasPermission(res.HasPermission) @@ -459,7 +461,6 @@ func (m *SubscriptionManager) subscribe(s *trackSubscription) error { return ErrNoTrackPermission } - s.setPublisher(res.PublisherIdentity, res.PublisherID) subTrack, err := track.AddSubscriber(m.params.Participant) if err != nil && err != errAlreadySubscribed { // ignore already subscribed error diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index 625c3ecb0..3c6299f9e 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -315,7 +315,7 @@ type LocalParticipant interface { OnParticipantUpdate(callback func(LocalParticipant)) OnDataPacket(callback func(LocalParticipant, *livekit.DataPacket)) OnSubscribeStatusChanged(fn func(publisherID livekit.ParticipantID, subscribed bool)) - OnClose(callback func(LocalParticipant, map[livekit.TrackID]livekit.ParticipantID)) + OnClose(callback func(LocalParticipant)) OnClaimsChanged(callback func(LocalParticipant)) OnReceiverReport(dt *sfu.DownTrack, report *rtcp.ReceiverReport) diff --git a/pkg/rtc/types/typesfakes/fake_local_participant.go b/pkg/rtc/types/typesfakes/fake_local_participant.go index 33416aa05..e4ddfea5f 100644 --- a/pkg/rtc/types/typesfakes/fake_local_participant.go +++ b/pkg/rtc/types/typesfakes/fake_local_participant.go @@ -458,10 +458,10 @@ type FakeLocalParticipant struct { onClaimsChangedArgsForCall []struct { arg1 func(types.LocalParticipant) } - OnCloseStub func(func(types.LocalParticipant, map[livekit.TrackID]livekit.ParticipantID)) + OnCloseStub func(func(types.LocalParticipant)) onCloseMutex sync.RWMutex onCloseArgsForCall []struct { - arg1 func(types.LocalParticipant, map[livekit.TrackID]livekit.ParticipantID) + arg1 func(types.LocalParticipant) } OnDataPacketStub func(func(types.LocalParticipant, *livekit.DataPacket)) onDataPacketMutex sync.RWMutex @@ -3186,10 +3186,10 @@ func (fake *FakeLocalParticipant) OnClaimsChangedArgsForCall(i int) func(types.L return argsForCall.arg1 } -func (fake *FakeLocalParticipant) OnClose(arg1 func(types.LocalParticipant, map[livekit.TrackID]livekit.ParticipantID)) { +func (fake *FakeLocalParticipant) OnClose(arg1 func(types.LocalParticipant)) { fake.onCloseMutex.Lock() fake.onCloseArgsForCall = append(fake.onCloseArgsForCall, struct { - arg1 func(types.LocalParticipant, map[livekit.TrackID]livekit.ParticipantID) + arg1 func(types.LocalParticipant) }{arg1}) stub := fake.OnCloseStub fake.recordInvocation("OnClose", []interface{}{arg1}) @@ -3205,13 +3205,13 @@ func (fake *FakeLocalParticipant) OnCloseCallCount() int { return len(fake.onCloseArgsForCall) } -func (fake *FakeLocalParticipant) OnCloseCalls(stub func(func(types.LocalParticipant, map[livekit.TrackID]livekit.ParticipantID))) { +func (fake *FakeLocalParticipant) OnCloseCalls(stub func(func(types.LocalParticipant))) { fake.onCloseMutex.Lock() defer fake.onCloseMutex.Unlock() fake.OnCloseStub = stub } -func (fake *FakeLocalParticipant) OnCloseArgsForCall(i int) func(types.LocalParticipant, map[livekit.TrackID]livekit.ParticipantID) { +func (fake *FakeLocalParticipant) OnCloseArgsForCall(i int) func(types.LocalParticipant) { fake.onCloseMutex.RLock() defer fake.onCloseMutex.RUnlock() argsForCall := fake.onCloseArgsForCall[i] diff --git a/pkg/service/roommanager.go b/pkg/service/roommanager.go index 4a63ea4e1..3966d4f97 100644 --- a/pkg/service/roommanager.go +++ b/pkg/service/roommanager.go @@ -376,7 +376,7 @@ func (r *RoomManager) StartSession( clientMeta := &livekit.AnalyticsClientMeta{Region: r.currentNode.Region, Node: r.currentNode.Id} r.telemetry.ParticipantJoined(ctx, protoRoom, participant.ToProto(), pi.Client, clientMeta, true) - participant.OnClose(func(p types.LocalParticipant, disallowedSubscriptions map[livekit.TrackID]livekit.ParticipantID) { + participant.OnClose(func(p types.LocalParticipant) { if err := r.roomStore.DeleteParticipant(ctx, roomName, p.Identity()); err != nil { pLogger.Errorw("could not delete participant", err) } @@ -385,8 +385,6 @@ func (r *RoomManager) StartSession( proto := room.ToProto() persistRoomForParticipantCount(proto) r.telemetry.ParticipantLeft(ctx, proto, p.ToProto(), true) - - room.RemoveDisallowedSubscriptions(p, disallowedSubscriptions) }) participant.OnClaimsChanged(func(participant types.LocalParticipant) { pLogger.Debugw("refreshing client token after claims change") From dfa3d77945078bba32f9c66bffa8d4111dea38ed Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sun, 30 Apr 2023 15:42:39 +0530 Subject: [PATCH 126/324] Misc changes (#1669) --- go.mod | 6 ++-- go.sum | 13 +++++---- pkg/sfu/buffer/rtpstats.go | 56 ++++++++++++++++++++++++++++++++++++-- 3 files changed, 63 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 8b996b883..df3c56d6b 100644 --- a/go.mod +++ b/go.mod @@ -27,14 +27,14 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/pion/dtls/v2 v2.2.6 github.com/pion/ice/v2 v2.3.2 - github.com/pion/interceptor v0.1.13 + github.com/pion/interceptor v0.1.16 github.com/pion/rtcp v1.2.10 github.com/pion/rtp v1.7.13 github.com/pion/sdp/v3 v3.0.6 github.com/pion/stun v0.4.0 github.com/pion/transport/v2 v2.2.0 github.com/pion/turn/v2 v2.1.0 - github.com/pion/webrtc/v3 v3.1.62 + github.com/pion/webrtc/v3 v3.2.1 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.15.0 github.com/redis/go-redis/v9 v9.0.3 @@ -80,7 +80,7 @@ require ( github.com/pion/logging v0.2.2 // indirect github.com/pion/mdns v0.0.7 // indirect github.com/pion/randutil v0.1.0 // indirect - github.com/pion/sctp v1.8.6 // indirect + github.com/pion/sctp v1.8.7 // indirect github.com/pion/srtp/v2 v2.0.12 // indirect github.com/pion/udp/v2 v2.0.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/go.sum b/go.sum index 052a35a44..967527630 100644 --- a/go.sum +++ b/go.sum @@ -183,8 +183,8 @@ github.com/pion/dtls/v2 v2.2.6 h1:yXMxKr0Skd+Ub6A8UqXTRLSywskx93ooMRHsQUtd+Z4= github.com/pion/dtls/v2 v2.2.6/go.mod h1:t8fWJCIquY5rlQZwA2yWxUS1+OCrAdXrhVKXB5oD/wY= github.com/pion/ice/v2 v2.3.2 h1:vh+fi4RkZ8H5fB4brZ/jm3j4BqFgMmNs+aB3X52Hu7M= github.com/pion/ice/v2 v2.3.2/go.mod h1:AMIpuJqcpe+UwloocNebmTSWhCZM1TUCo9v7nW50jX0= -github.com/pion/interceptor v0.1.13 h1:tfJdEqPxnQrlstjd7SCL7B97WdjPkJtg5EpRMgJ61Ms= -github.com/pion/interceptor v0.1.13/go.mod h1:SY8kpmfVBvrbUzvj2bsXz7OJt5JvmVNZ+4Kjq7FcwrI= +github.com/pion/interceptor v0.1.16 h1:0GDZrfNO+BmVNWymS31fMlVtPO2IJVBzy2Qq5XCYMIg= +github.com/pion/interceptor v0.1.16/go.mod h1:SY8kpmfVBvrbUzvj2bsXz7OJt5JvmVNZ+4Kjq7FcwrI= github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= github.com/pion/mdns v0.0.7 h1:P0UB4Sr6xDWEox0kTVxF0LmQihtCbSAdW0H2nEgkA3U= @@ -196,8 +196,8 @@ github.com/pion/rtcp v1.2.10/go.mod h1:ztfEwXZNLGyF1oQDttz/ZKIBaeeg/oWbRYqzBM9TL github.com/pion/rtp v1.7.13 h1:qcHwlmtiI50t1XivvoawdCGTP4Uiypzfrsap+bijcoA= github.com/pion/rtp v1.7.13/go.mod h1:bDb5n+BFZxXx0Ea7E5qe+klMuqiBrP+w8XSjiWtCUko= github.com/pion/sctp v1.8.5/go.mod h1:SUFFfDpViyKejTAdwD1d/HQsCu+V/40cCs2nZIvC3s0= -github.com/pion/sctp v1.8.6 h1:CUex11Vkt9YS++VhLf8b55O3VqKrWL6W3SDwX4jAqsI= -github.com/pion/sctp v1.8.6/go.mod h1:SUFFfDpViyKejTAdwD1d/HQsCu+V/40cCs2nZIvC3s0= +github.com/pion/sctp v1.8.7 h1:JnABvFakZueGAn4KU/4PSKg+GWbF6QWbKTWZOSGJjXw= +github.com/pion/sctp v1.8.7/go.mod h1:g1Ul+ARqZq5JEmoFy87Q/4CePtKnTJ1QCL9dBBdN6AU= github.com/pion/sdp/v3 v3.0.6 h1:WuDLhtuFUUVpTfus9ILC4HRyHsW6TdugjEX/QY9OiUw= github.com/pion/sdp/v3 v3.0.6/go.mod h1:iiFWFpQO8Fy3S5ldclBkpXqmWy02ns78NOKoLLL0YQw= github.com/pion/srtp/v2 v2.0.12 h1:WrmiVCubGMOAObBU1vwWjG0H3VSyQHawKeer2PVA5rY= @@ -208,14 +208,15 @@ github.com/pion/transport v0.14.1 h1:XSM6olwW+o8J4SCmOBb/BpwZypkHeyM0PGFCxNQBr40 github.com/pion/transport v0.14.1/go.mod h1:4tGmbk00NeYA3rUa9+n+dzCCoKkcy3YlYb99Jn2fNnI= github.com/pion/transport/v2 v2.0.0/go.mod h1:HS2MEBJTwD+1ZI2eSXSvHJx/HnzQqRy2/LXxt6eVMHc= github.com/pion/transport/v2 v2.0.2/go.mod h1:vrz6bUbFr/cjdwbnxq8OdDDzHf7JJfGsIRkxfpZoTA0= +github.com/pion/transport/v2 v2.1.0/go.mod h1:AdSw4YBZVDkZm8fpoz+fclXyQwANWmZAlDuQdctTThQ= github.com/pion/transport/v2 v2.2.0 h1:u5lFqFHkXLMXMzai8tixZDfVjb8eOjH35yCunhPeb1c= github.com/pion/transport/v2 v2.2.0/go.mod h1:AdSw4YBZVDkZm8fpoz+fclXyQwANWmZAlDuQdctTThQ= github.com/pion/turn/v2 v2.1.0 h1:5wGHSgGhJhP/RpabkUb/T9PdsAjkGLS6toYz5HNzoSI= github.com/pion/turn/v2 v2.1.0/go.mod h1:yrT5XbXSGX1VFSF31A3c1kCNB5bBZgk/uu5LET162qs= github.com/pion/udp/v2 v2.0.1 h1:xP0z6WNux1zWEjhC7onRA3EwwSliXqu1ElUZAQhUP54= github.com/pion/udp/v2 v2.0.1/go.mod h1:B7uvTMP00lzWdyMr/1PVZXtV3wpPIxBRd4Wl6AksXn8= -github.com/pion/webrtc/v3 v3.1.62 h1:B+QYCs+ajtRMJtC3nphzFWWjVoCorugOABu/JD0pJ3c= -github.com/pion/webrtc/v3 v3.1.62/go.mod h1:PaPsj1aigBfWK1jJRZPkWvdiPaAiJwAEMgDKXVO7NjI= +github.com/pion/webrtc/v3 v3.2.1 h1:eehbYzkM6xWoH3LXoIBnZTb4TOrjwmVzI78JO1+5kgQ= +github.com/pion/webrtc/v3 v3.2.1/go.mod h1:sQVqop5YhZezvKyyz6Nywvf15LhlXUWiXWdN5DV4zHs= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index 06417365e..887008cdb 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -124,6 +124,7 @@ type RTPStats struct { lastRR rtcp.ReceptionReport highestTS uint32 + tsCycles uint32 highestTime int64 lastTransit uint32 @@ -174,7 +175,9 @@ type RTPStats struct { rtt uint32 maxRtt uint32 - srDataExt *RTCPSenderReportDataExt + srDataExt *RTCPSenderReportDataExt + firstSenderReportNTP mediatransportutil.NtpTime + firstSenderReportRTP uint32 nextSnapshotId uint32 snapshots map[uint32]*Snapshot @@ -212,6 +215,7 @@ func (r *RTPStats) Seed(from *RTPStats) { r.lastRR = from.lastRR r.highestTS = from.highestTS + r.tsCycles = from.tsCycles r.highestTime = from.highestTime r.lastTransit = from.lastTransit @@ -270,6 +274,8 @@ func (r *RTPStats) Seed(from *RTPStats) { } else { r.srDataExt = nil } + r.firstSenderReportNTP = from.firstSenderReportNTP + r.firstSenderReportRTP = from.firstSenderReportRTP r.nextSnapshotId = from.nextSnapshotId for id, ss := range from.snapshots { @@ -334,6 +340,7 @@ func (r *RTPStats) Update(rtph *rtp.Header, payloadSize int, paddingSize int, pa r.extStartSN = uint32(rtph.SequenceNumber) r.cycles = 0 + r.tsCycles = 0 first = true @@ -400,6 +407,9 @@ func (r *RTPStats) Update(rtph *rtp.Header, payloadSize int, paddingSize int, pa r.cycles++ } r.highestSN = rtph.SequenceNumber + if rtph.Timestamp < r.highestTS && !first { + r.tsCycles++ + } r.highestTS = rtph.Timestamp r.highestTime = packetTime } @@ -706,6 +716,11 @@ func (r *RTPStats) SetRtcpSenderReportData(srData *RTCPSenderReportData) { // prevent against extreme case of anachronous sender reports if r.srDataExt != nil && r.srDataExt.SenderReportData.NTPTimestamp > srData.NTPTimestamp { + r.logger.Debugw( + "anachronous RTCP sender report", + "current", srData.NTPTimestamp.Time(), + "last", r.srDataExt.SenderReportData.NTPTimestamp.Time(), + ) return } @@ -750,8 +765,8 @@ func (r *RTPStats) GetRtcpSenderReportDataExt() *RTCPSenderReportDataExt { } func (r *RTPStats) GetRtcpSenderReport(ssrc uint32, srDataExt *RTCPSenderReportDataExt) *rtcp.SenderReport { - r.lock.RLock() - defer r.lock.RUnlock() + r.lock.Lock() + defer r.lock.Unlock() if !r.initialized { return nil @@ -782,6 +797,37 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32, srDataExt *RTCPSenderReportD nowRTP = srDataExt.SenderReportData.RTPTimestamp + uint32(now.Sub(smoothedLocalTimeOfLatestSenderReportNTP).Milliseconds()*int64(r.params.ClockRate)/1000) } + // TODO-REMOVE-AFTER-DEBUG + if r.firstSenderReportNTP == 0 { + r.firstSenderReportNTP = nowNTP + r.firstSenderReportRTP = nowRTP + } else { + highestTime := time.Unix(0, r.highestTime) + ntpTime := nowNTP.Time() + ntpDiff := ntpTime.Sub(highestTime) + rtpDiff := int32(nowRTP - r.highestTS) + rtpOffset := int32(nowRTP - r.highestTS - uint32(ntpDiff.Milliseconds()*int64(r.params.ClockRate)/1000)) + + timeSinceFirst := nowNTP.Time().Sub(r.firstSenderReportNTP.Time()) + rtpDiffSinceFirst := getExtTS(nowRTP, r.tsCycles) - getExtTS(r.firstSenderReportRTP, 0) + drift := int64(uint64(timeSinceFirst.Milliseconds()*int64(r.params.ClockRate)/1000) - rtpDiffSinceFirst) + driftTime := float64(drift) / float64(r.params.ClockRate) / 1000 + r.logger.Debugw( + "sender report", + "highestTS", r.highestTS, + "reportTS", nowRTP, + "rtpDiff", rtpDiff, + "highestTime", highestTime, + "reportTime", ntpTime, + "timeDiff", ntpDiff, + "rtpOffset", rtpOffset, + "timeSinceFirst", timeSinceFirst, + "rtpDiffSinceFirst", rtpDiffSinceFirst, + "drift", drift, + "driftTime(ms)", driftTime, + ) + } + return &rtcp.SenderReport{ SSRC: ssrc, NTPTime: uint64(nowNTP), @@ -1375,6 +1421,10 @@ func (r *RTPStats) getAndResetSnapshot(snapshotId uint32, override bool) (*Snaps // ---------------------------------- +func getExtTS(ts uint32, cycles uint32) uint64 { + return (uint64(cycles) << 32) | uint64(ts) +} + func AggregateRTPStats(statsList []*livekit.RTPStats) *livekit.RTPStats { if len(statsList) == 0 { return nil From faebb79ebeb06ec7cb318898a073af62cd7cbb13 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 30 Apr 2023 22:16:03 -0700 Subject: [PATCH 127/324] Update go deps (#1648) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index df3c56d6b..11dff2106 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.18 require ( github.com/bep/debounce v1.2.1 - github.com/d5/tengo/v2 v2.14.0 + github.com/d5/tengo/v2 v2.16.0 github.com/dustin/go-humanize v1.0.1 github.com/elliotchance/orderedmap/v2 v2.2.0 github.com/florianl/go-tc v0.4.2 @@ -43,7 +43,7 @@ require ( github.com/thoas/go-funk v0.9.3 github.com/twitchtv/twirp v8.1.3+incompatible github.com/ua-parser/uap-go v0.0.0-20211112212520-00c877edfe0f - github.com/urfave/cli/v2 v2.25.1 + github.com/urfave/cli/v2 v2.25.2 github.com/urfave/negroni/v3 v3.0.0 go.uber.org/atomic v1.10.0 golang.org/x/sync v0.1.0 diff --git a/go.sum b/go.sum index 967527630..5d67de0e3 100644 --- a/go.sum +++ b/go.sum @@ -14,8 +14,8 @@ github.com/cilium/ebpf v0.8.1/go.mod h1:f5zLIM0FSNuAkSyLAN7X+Hy6yznlF1mNiWUMfxMt github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/d5/tengo/v2 v2.14.0 h1:ZTUyb1tGvxXGah1xDdCjaIq6ZdG4Z8PP1ZlBSN5OChg= -github.com/d5/tengo/v2 v2.14.0/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8= +github.com/d5/tengo/v2 v2.16.0 h1:HEpo2Rk8fIiXmTGtkRMJZ4RTQdpgiL8n/tNrRUDj75c= +github.com/d5/tengo/v2 v2.16.0/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -258,8 +258,8 @@ github.com/twitchtv/twirp v8.1.3+incompatible h1:+F4TdErPgSUbMZMwp13Q/KgDVuI7HJX github.com/twitchtv/twirp v8.1.3+incompatible/go.mod h1:RRJoFSAmTEh2weEqWtpPE3vFK5YBhA6bqp2l1kfCC5A= github.com/ua-parser/uap-go v0.0.0-20211112212520-00c877edfe0f h1:A+MmlgpvrHLeUP8dkBVn4Pnf5Bp5Yk2OALm7SEJLLE8= github.com/ua-parser/uap-go v0.0.0-20211112212520-00c877edfe0f/go.mod h1:OBcG9bn7sHtXgarhUEb3OfCnNsgtGnkVf41ilSZ3K3E= -github.com/urfave/cli/v2 v2.25.1 h1:zw8dSP7ghX0Gmm8vugrs6q9Ku0wzweqPyshy+syu9Gw= -github.com/urfave/cli/v2 v2.25.1/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= +github.com/urfave/cli/v2 v2.25.2 h1:rgeK7wmjwH+d3DqXDDSV20GZAvNzmzu/VEsg1om3Qwg= +github.com/urfave/cli/v2 v2.25.2/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= github.com/urfave/negroni/v3 v3.0.0 h1:Vo8CeZfu1lFR9gW8GnAb6dOGCJyijfil9j/jKKc/JhU= github.com/urfave/negroni/v3 v3.0.0/go.mod h1:jWvnX03kcSjDBl/ShB0iHvx5uOs7mAzZXW+JvJ5XYAs= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= From 08fd0685113720fcc12c8a74c5fa892a8733e13a Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 1 May 2023 11:39:37 +0530 Subject: [PATCH 128/324] logging feed side last report for clues (#1670) --- pkg/sfu/buffer/rtpstats.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index 887008cdb..0803b1505 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -407,10 +407,12 @@ func (r *RTPStats) Update(rtph *rtp.Header, payloadSize int, paddingSize int, pa r.cycles++ } r.highestSN = rtph.SequenceNumber + if rtph.Timestamp < r.highestTS && !first { r.tsCycles++ } r.highestTS = rtph.Timestamp + r.highestTime = packetTime } @@ -785,8 +787,10 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32, srDataExt *RTCPSenderReportD nowNTP := mediatransportutil.ToNtpTime(now) nowRTP := r.highestTS + isUsingSmoothed := true smoothedLocalTimeOfLatestSenderReportNTP := srDataExt.SenderReportData.NTPTimestamp.Time().Add(srDataExt.SmoothedOWD) if smoothedLocalTimeOfLatestSenderReportNTP.After(now) { + isUsingSmoothed = false r.logger.Debugw("smoothed time of NTP is ahead", "now", now, "smoothed", smoothedLocalTimeOfLatestSenderReportNTP, @@ -806,12 +810,12 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32, srDataExt *RTCPSenderReportD ntpTime := nowNTP.Time() ntpDiff := ntpTime.Sub(highestTime) rtpDiff := int32(nowRTP - r.highestTS) - rtpOffset := int32(nowRTP - r.highestTS - uint32(ntpDiff.Milliseconds()*int64(r.params.ClockRate)/1000)) + rtpOffset := int32(nowRTP - r.highestTS - uint32(ntpDiff.Nanoseconds()*int64(r.params.ClockRate)/1e9)) timeSinceFirst := nowNTP.Time().Sub(r.firstSenderReportNTP.Time()) rtpDiffSinceFirst := getExtTS(nowRTP, r.tsCycles) - getExtTS(r.firstSenderReportRTP, 0) - drift := int64(uint64(timeSinceFirst.Milliseconds()*int64(r.params.ClockRate)/1000) - rtpDiffSinceFirst) - driftTime := float64(drift) / float64(r.params.ClockRate) / 1000 + drift := int64(uint64(timeSinceFirst.Nanoseconds()*int64(r.params.ClockRate)/1e9) - rtpDiffSinceFirst) + driftTime := (float64(drift) * 1000) / float64(r.params.ClockRate) r.logger.Debugw( "sender report", "highestTS", r.highestTS, @@ -825,6 +829,11 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32, srDataExt *RTCPSenderReportD "rtpDiffSinceFirst", rtpDiffSinceFirst, "drift", drift, "driftTime(ms)", driftTime, + "smoothed", isUsingSmoothed, + "feedRTP", srDataExt.SenderReportData.RTPTimestamp, + "feedNTP", srDataExt.SenderReportData.NTPTimestamp.Time(), + "feedArrival", srDataExt.SenderReportData.ArrivalTime, + "smoothedOWD", srDataExt.SmoothedOWD, ) } From 7dbd086c3e81caf0be4a42a3b3b9d2964efe7208 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 1 May 2023 11:55:27 +0530 Subject: [PATCH 129/324] Calculating feed drift and log them for more clues. (#1671) --- pkg/sfu/buffer/rtpstats.go | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index 0803b1505..ac1e421ed 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -175,9 +175,11 @@ type RTPStats struct { rtt uint32 maxRtt uint32 - srDataExt *RTCPSenderReportDataExt - firstSenderReportNTP mediatransportutil.NtpTime - firstSenderReportRTP uint32 + srDataExt *RTCPSenderReportDataExt + firstSenderReportNTP mediatransportutil.NtpTime + firstSenderReportRTP uint32 + firstFeedSenderReportNTP mediatransportutil.NtpTime + firstFeedSenderReportRTP uint32 nextSnapshotId uint32 snapshots map[uint32]*Snapshot @@ -276,6 +278,8 @@ func (r *RTPStats) Seed(from *RTPStats) { } r.firstSenderReportNTP = from.firstSenderReportNTP r.firstSenderReportRTP = from.firstSenderReportRTP + r.firstFeedSenderReportNTP = from.firstFeedSenderReportNTP + r.firstFeedSenderReportRTP = from.firstFeedSenderReportRTP r.nextSnapshotId = from.nextSnapshotId for id, ss := range from.snapshots { @@ -805,6 +809,9 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32, srDataExt *RTCPSenderReportD if r.firstSenderReportNTP == 0 { r.firstSenderReportNTP = nowNTP r.firstSenderReportRTP = nowRTP + + r.firstFeedSenderReportNTP = srDataExt.SenderReportData.NTPTimestamp + r.firstFeedSenderReportRTP = srDataExt.SenderReportData.RTPTimestamp } else { highestTime := time.Unix(0, r.highestTime) ntpTime := nowNTP.Time() @@ -816,6 +823,13 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32, srDataExt *RTCPSenderReportD rtpDiffSinceFirst := getExtTS(nowRTP, r.tsCycles) - getExtTS(r.firstSenderReportRTP, 0) drift := int64(uint64(timeSinceFirst.Nanoseconds()*int64(r.params.ClockRate)/1e9) - rtpDiffSinceFirst) driftTime := (float64(drift) * 1000) / float64(r.params.ClockRate) + + feedTimeSinceFirst := srDataExt.SenderReportData.NTPTimestamp.Time().Sub(r.firstFeedSenderReportNTP.Time()) + // using tsCycles for extending feed time stamp too + feedRtpDiffSinceFirst := getExtTS(srDataExt.SenderReportData.RTPTimestamp, r.tsCycles) - getExtTS(r.firstFeedSenderReportRTP, 0) + feedDrift := int64(uint64(feedTimeSinceFirst.Nanoseconds()*int64(r.params.ClockRate)/1e9) - feedRtpDiffSinceFirst) + feedDriftTime := (float64(feedDrift) * 1000) / float64(r.params.ClockRate) + r.logger.Debugw( "sender report", "highestTS", r.highestTS, @@ -834,6 +848,10 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32, srDataExt *RTCPSenderReportD "feedNTP", srDataExt.SenderReportData.NTPTimestamp.Time(), "feedArrival", srDataExt.SenderReportData.ArrivalTime, "smoothedOWD", srDataExt.SmoothedOWD, + "feedTimeSinceFirst", feedTimeSinceFirst, + "feedRtpDiffSinceFirst", feedRtpDiffSinceFirst, + "feedDrift", feedDrift, + "feedDriftTime(ms)", feedDriftTime, ) } From 3070e976c3f2ed5669695eddb69b3119b6f6f0d6 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 2 May 2023 00:22:33 +0530 Subject: [PATCH 130/324] Log received sender report of audio for debugging (#1673) * Log received sender report of audio for debugging * log OWD also * add some more bits --- pkg/sfu/buffer/rtpstats.go | 43 ++++++++++++++++++++++++++++---------- pkg/sfu/forwarder.go | 1 + 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index ac1e421ed..0e17414e1 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -750,15 +750,27 @@ func (r *RTPStats) SetRtcpSenderReportData(srData *RTCPSenderReportData) { if r.srDataExt != nil { smoothedOwd = r.srDataExt.SmoothedOWD } + smoothedOwd = (owd + smoothedOwd) / 2 + // TODO-REMOVE-AFTER-DEBUG + if r.params.ClockRate != 90000 { // log only for audio as it is less frequent + r.logger.Debugw( + "received sender report", + "ntp", srData.NTPTimestamp.Time(), + "rtp", srData.RTPTimestamp, + "arrival", srData.ArrivalTime, + "owd", owd, + "smoothedOwd", smoothedOwd, + ) + } r.srDataExt = &RTCPSenderReportDataExt{ SenderReportData: *srData, - SmoothedOWD: (owd + smoothedOwd) / 2, + SmoothedOWD: smoothedOwd, } } func (r *RTPStats) GetRtcpSenderReportDataExt() *RTCPSenderReportDataExt { - r.lock.Lock() - defer r.lock.Unlock() + r.lock.RLock() + defer r.lock.RUnlock() if r.srDataExt == nil { return nil @@ -815,9 +827,14 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32, srDataExt *RTCPSenderReportD } else { highestTime := time.Unix(0, r.highestTime) ntpTime := nowNTP.Time() - ntpDiff := ntpTime.Sub(highestTime) - rtpDiff := int32(nowRTP - r.highestTS) - rtpOffset := int32(nowRTP - r.highestTS - uint32(ntpDiff.Nanoseconds()*int64(r.params.ClockRate)/1e9)) + + ntpDiffLocal := ntpTime.Sub(highestTime) + rtpDiffLocal := int32(nowRTP - r.highestTS) + rtpOffsetLocal := int32(nowRTP - r.highestTS - uint32(ntpDiffLocal.Nanoseconds()*int64(r.params.ClockRate)/1e9)) + + ntpDiffSmoothed := ntpTime.Sub(smoothedLocalTimeOfLatestSenderReportNTP) + rtpDiffSmoothed := int32(nowRTP - srDataExt.SenderReportData.RTPTimestamp) + rtpOffsetSmoothed := int32(nowRTP - srDataExt.SenderReportData.RTPTimestamp - uint32(ntpDiffSmoothed.Nanoseconds()*int64(r.params.ClockRate)/1e9)) timeSinceFirst := nowNTP.Time().Sub(r.firstSenderReportNTP.Time()) rtpDiffSinceFirst := getExtTS(nowRTP, r.tsCycles) - getExtTS(r.firstSenderReportRTP, 0) @@ -831,14 +848,18 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32, srDataExt *RTCPSenderReportD feedDriftTime := (float64(feedDrift) * 1000) / float64(r.params.ClockRate) r.logger.Debugw( - "sender report", + "sending sender report", "highestTS", r.highestTS, - "reportTS", nowRTP, - "rtpDiff", rtpDiff, "highestTime", highestTime, + "smoothedTime", smoothedLocalTimeOfLatestSenderReportNTP, + "reportTS", nowRTP, "reportTime", ntpTime, - "timeDiff", ntpDiff, - "rtpOffset", rtpOffset, + "rtpDiffLocal", rtpDiffLocal, + "ntpDiffLocal", ntpDiffLocal, + "rtpOffsetLocal", rtpOffsetLocal, + "rtpDiffSmoothed", rtpDiffSmoothed, + "ntpDiffSmoothed", ntpDiffSmoothed, + "rtpOffsetSmoothed", rtpOffsetSmoothed, "timeSinceFirst", timeSinceFirst, "rtpDiffSinceFirst", rtpDiffSinceFirst, "drift", drift, diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 069132850..4c1ba5028 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -1459,6 +1459,7 @@ func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer i // log jumps greater than 0.5 seconds f.logger.Debugw("reference timestamp too far ahead", "lastTS", last.LastTS, "refTS", refTS, "td", td) } + f.logger.Debugw("reference timestamp on switch", "lastTS", last.LastTS, "refTS", refTS, "td", int32(td), "switchingAt", time.Now()) } else { f.logger.Debugw("reference timestamp get error, using default", "error", err) } From 87e2b2366e5f6a2c01c4ed6196ecddc8a39bfd2a Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Tue, 2 May 2023 08:31:12 -0700 Subject: [PATCH 131/324] reduce log level of signal close errors (#1675) * reduce log level of signal close errors * update psrpc * cleanup * cleanup --- go.mod | 4 ++-- go.sum | 8 ++++---- pkg/rtc/participant_signal.go | 8 +++++++- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 11dff2106..b6292142b 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 github.com/livekit/protocol v1.5.6-0.20230428011359-db5afb1c7f9b - github.com/livekit/psrpc v0.3.1-0.20230425025640-5390915734c3 + github.com/livekit/psrpc v0.3.1-0.20230502152150-df9dd21fba11 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.14.0 github.com/maxbrunsfeld/counterfeiter/v6 v6.6.1 @@ -37,7 +37,7 @@ require ( github.com/pion/webrtc/v3 v3.2.1 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.15.0 - github.com/redis/go-redis/v9 v9.0.3 + github.com/redis/go-redis/v9 v9.0.4 github.com/rs/cors v1.9.0 github.com/stretchr/testify v1.8.2 github.com/thoas/go-funk v0.9.3 diff --git a/go.sum b/go.sum index 5d67de0e3..3b4cb4271 100644 --- a/go.sum +++ b/go.sum @@ -123,8 +123,8 @@ github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 h1:QlQF github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26/go.mod h1:eDA41kiySZoG+wy4Etsjb3w0jjLx69i/vAmSjG4bteA= github.com/livekit/protocol v1.5.6-0.20230428011359-db5afb1c7f9b h1:UEFMJr1OTF0yAX9mRRaQQ/YrTA6H7hCkbuABWfT6wLk= github.com/livekit/protocol v1.5.6-0.20230428011359-db5afb1c7f9b/go.mod h1:MBW05GWdhbl+o6u2gLLCQtDvr9EvcV4VWckpIYtoM2c= -github.com/livekit/psrpc v0.3.1-0.20230425025640-5390915734c3 h1:NXcxrluYLng7LTHcYNOj/MdR4SHWrKQAG2G+U930mTA= -github.com/livekit/psrpc v0.3.1-0.20230425025640-5390915734c3/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= +github.com/livekit/psrpc v0.3.1-0.20230502152150-df9dd21fba11 h1:VS23iVQu/TNiLEM5XjbBSY28+B6nSewjKWPDbieg0Ho= +github.com/livekit/psrpc v0.3.1-0.20230502152150-df9dd21fba11/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= github.com/mackerelio/go-osstat v0.2.4/go.mod h1:Zy+qzGdZs3A9cuIqmgbJvwbmLQH9dJvtio5ZjJTbdlQ= github.com/magefile/mage v1.14.0 h1:6QDX3g6z1YvJ4olPhT1wksUcSa/V0a1B+pJb73fBjyo= @@ -229,8 +229,8 @@ github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= -github.com/redis/go-redis/v9 v9.0.3 h1:+7mmR26M0IvyLxGZUHxu4GiBkJkVDid0Un+j4ScYu4k= -github.com/redis/go-redis/v9 v9.0.3/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= +github.com/redis/go-redis/v9 v9.0.4 h1:FC82T+CHJ/Q/PdyLW++GeCO+Ol59Y4T7R4jbgjvktgc= +github.com/redis/go-redis/v9 v9.0.4/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rs/cors v1.9.0 h1:l9HGsTsHJcvW14Nk7J9KFz8bzeAWXn3CG6bgt7LsrAE= diff --git a/pkg/rtc/participant_signal.go b/pkg/rtc/participant_signal.go index 106c8370a..e09ff85ab 100644 --- a/pkg/rtc/participant_signal.go +++ b/pkg/rtc/participant_signal.go @@ -1,12 +1,14 @@ package rtc import ( + "errors" "fmt" "time" "github.com/pion/webrtc/v3" "github.com/livekit/protocol/livekit" + "github.com/livekit/psrpc" "github.com/livekit/livekit-server/pkg/routing" ) @@ -265,7 +267,11 @@ func (p *ParticipantImpl) writeMessage(msg *livekit.SignalResponse) error { } err := sink.WriteMessage(msg) - if err != nil { + if errors.Is(err, psrpc.Canceled) { + p.params.Logger.Debugw("could not send message to participant", + "error", err, "messageType", fmt.Sprintf("%T", msg.Message)) + return nil + } else if err != nil { p.params.Logger.Warnw("could not send message to participant", err, "messageType", fmt.Sprintf("%T", msg.Message)) return err From 00217c7af17bf1050b07c6bc64faac24231c1418 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 2 May 2023 22:43:37 +0530 Subject: [PATCH 132/324] Logging delta of receiver report (#1676) --- pkg/sfu/buffer/rtpstats.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index 0e17414e1..c7b8575f9 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -753,11 +753,24 @@ func (r *RTPStats) SetRtcpSenderReportData(srData *RTCPSenderReportData) { smoothedOwd = (owd + smoothedOwd) / 2 // TODO-REMOVE-AFTER-DEBUG if r.params.ClockRate != 90000 { // log only for audio as it is less frequent + ntpTime := srData.NTPTimestamp.Time() + + var ntpDiffSinceLast, arrivalDiffSinceLast time.Duration + var rtpDiffSinceLast uint32 + if r.srDataExt != nil { + ntpDiffSinceLast = ntpTime.Sub(r.srDataExt.SenderReportData.NTPTimestamp.Time()) + rtpDiffSinceLast = srData.RTPTimestamp - r.srDataExt.SenderReportData.RTPTimestamp + arrivalDiffSinceLast = srData.ArrivalTime.Sub(r.srDataExt.SenderReportData.ArrivalTime) + } r.logger.Debugw( "received sender report", - "ntp", srData.NTPTimestamp.Time(), + "ntp", ntpTime, "rtp", srData.RTPTimestamp, "arrival", srData.ArrivalTime, + "ntpDiff", ntpDiffSinceLast, + "rtpDiff", rtpDiffSinceLast, + "arrivalDiff", arrivalDiffSinceLast, + "expectedTimeDiff", float64(rtpDiffSinceLast)/float64(r.params.ClockRate), "owd", owd, "smoothedOwd", smoothedOwd, ) @@ -1381,6 +1394,7 @@ func (r *RTPStats) getIntervalStats(startInclusive uint16, endExclusive uint16) "start", startInclusive, "end", endExclusive, "count", packetsNotFound, + "highestSN", r.highestSN, ) } return From 11749eace9cc52e89067a8135dfb3267a9f0c4ba Mon Sep 17 00:00:00 2001 From: Benjamin Pracht Date: Tue, 2 May 2023 13:26:47 -0700 Subject: [PATCH 133/324] Add support for creating WHIP ingress (#1674) --- config-sample.yaml | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- pkg/config/config.go | 1 + pkg/service/ingress.go | 12 +++++++++++- 5 files changed, 17 insertions(+), 6 deletions(-) diff --git a/config-sample.yaml b/config-sample.yaml index 7375b2398..122fac283 100644 --- a/config-sample.yaml +++ b/config-sample.yaml @@ -224,9 +224,9 @@ keys: # ingress server # ingress: # # Prefix used to generate RTMP URLs for RTMP ingress. -# # The stream_key will be appended to this base and returned as part of the -# # ingress info # rtmp_base_url: "rtmp://my.domain.com/live" +# # Prefix used to generate WHIP URLs for WHIP ingress. +# whip_base_url: "http://my.domain.com/whip" # egress server # egress: diff --git a/go.mod b/go.mod index b6292142b..25a0643bc 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 - github.com/livekit/protocol v1.5.6-0.20230428011359-db5afb1c7f9b + github.com/livekit/protocol v1.5.6-0.20230501213748-b2974692ebfd github.com/livekit/psrpc v0.3.1-0.20230502152150-df9dd21fba11 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.14.0 diff --git a/go.sum b/go.sum index 3b4cb4271..01f701878 100644 --- a/go.sum +++ b/go.sum @@ -121,8 +121,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 h1:QlQFyMwCDgjyySsrgmrMcVbEBA6KZcyTzvK+z346tUA= github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26/go.mod h1:eDA41kiySZoG+wy4Etsjb3w0jjLx69i/vAmSjG4bteA= -github.com/livekit/protocol v1.5.6-0.20230428011359-db5afb1c7f9b h1:UEFMJr1OTF0yAX9mRRaQQ/YrTA6H7hCkbuABWfT6wLk= -github.com/livekit/protocol v1.5.6-0.20230428011359-db5afb1c7f9b/go.mod h1:MBW05GWdhbl+o6u2gLLCQtDvr9EvcV4VWckpIYtoM2c= +github.com/livekit/protocol v1.5.6-0.20230501213748-b2974692ebfd h1:5gCLKGemm4JCEZvKIPpKVPOpguFt3/S2ZzYQd+4tInA= +github.com/livekit/protocol v1.5.6-0.20230501213748-b2974692ebfd/go.mod h1:MBW05GWdhbl+o6u2gLLCQtDvr9EvcV4VWckpIYtoM2c= github.com/livekit/psrpc v0.3.1-0.20230502152150-df9dd21fba11 h1:VS23iVQu/TNiLEM5XjbBSY28+B6nSewjKWPDbieg0Ho= github.com/livekit/psrpc v0.3.1-0.20230502152150-df9dd21fba11/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= diff --git a/pkg/config/config.go b/pkg/config/config.go index f0dc1e1c1..315fb3aae 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -259,6 +259,7 @@ type EgressConfig struct { type IngressConfig struct { RTMPBaseURL string `yaml:"rtmp_base_url"` + WHIPBaseURL string `yaml:"whip_base_url"` } // not exposed to YAML diff --git a/pkg/service/ingress.go b/pkg/service/ingress.go index 52ff81e69..9456224a0 100644 --- a/pkg/service/ingress.go +++ b/pkg/service/ingress.go @@ -56,7 +56,17 @@ func (s *IngressService) CreateIngress(ctx context.Context, req *livekit.CreateI AppendLogFields(ctx, fields...) }() - ig, err := s.CreateIngressWithUrlPrefix(ctx, s.conf.RTMPBaseURL, req) + var urlPrefix string + switch req.InputType { + case livekit.IngressInput_RTMP_INPUT: + urlPrefix = s.conf.RTMPBaseURL + case livekit.IngressInput_WHIP_INPUT: + urlPrefix = s.conf.WHIPBaseURL + default: + return nil, ingress.ErrInvalidIngressType + } + + ig, err := s.CreateIngressWithUrlPrefix(ctx, urlPrefix, req) if err != nil { return nil, err } From 0e9e7e994dc3ec12bca872acacf9a6c9c6762e4d Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 3 May 2023 00:08:12 -0700 Subject: [PATCH 134/324] Update go deps (#1672) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 25a0643bc..3ec29847c 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,7 @@ require ( github.com/thoas/go-funk v0.9.3 github.com/twitchtv/twirp v8.1.3+incompatible github.com/ua-parser/uap-go v0.0.0-20211112212520-00c877edfe0f - github.com/urfave/cli/v2 v2.25.2 + github.com/urfave/cli/v2 v2.25.3 github.com/urfave/negroni/v3 v3.0.0 go.uber.org/atomic v1.10.0 golang.org/x/sync v0.1.0 diff --git a/go.sum b/go.sum index 01f701878..0604fd34a 100644 --- a/go.sum +++ b/go.sum @@ -258,8 +258,8 @@ github.com/twitchtv/twirp v8.1.3+incompatible h1:+F4TdErPgSUbMZMwp13Q/KgDVuI7HJX github.com/twitchtv/twirp v8.1.3+incompatible/go.mod h1:RRJoFSAmTEh2weEqWtpPE3vFK5YBhA6bqp2l1kfCC5A= github.com/ua-parser/uap-go v0.0.0-20211112212520-00c877edfe0f h1:A+MmlgpvrHLeUP8dkBVn4Pnf5Bp5Yk2OALm7SEJLLE8= github.com/ua-parser/uap-go v0.0.0-20211112212520-00c877edfe0f/go.mod h1:OBcG9bn7sHtXgarhUEb3OfCnNsgtGnkVf41ilSZ3K3E= -github.com/urfave/cli/v2 v2.25.2 h1:rgeK7wmjwH+d3DqXDDSV20GZAvNzmzu/VEsg1om3Qwg= -github.com/urfave/cli/v2 v2.25.2/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= +github.com/urfave/cli/v2 v2.25.3 h1:VJkt6wvEBOoSjPFQvOkv6iWIrsJyCrKGtCtxXWwmGeY= +github.com/urfave/cli/v2 v2.25.3/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= github.com/urfave/negroni/v3 v3.0.0 h1:Vo8CeZfu1lFR9gW8GnAb6dOGCJyijfil9j/jKKc/JhU= github.com/urfave/negroni/v3 v3.0.0/go.mod h1:jWvnX03kcSjDBl/ShB0iHvx5uOs7mAzZXW+JvJ5XYAs= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= From 4bd20c77b7dea74c282fef59ebf251077d866ef6 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 3 May 2023 09:33:20 -0700 Subject: [PATCH 135/324] Update module github.com/prometheus/client_golang to v1.15.1 (#1677) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 3ec29847c..8c5ed846b 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/pion/turn/v2 v2.1.0 github.com/pion/webrtc/v3 v3.2.1 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.15.0 + github.com/prometheus/client_golang v1.15.1 github.com/redis/go-redis/v9 v9.0.4 github.com/rs/cors v1.9.0 github.com/stretchr/testify v1.8.2 diff --git a/go.sum b/go.sum index 0604fd34a..f2afcbf39 100644 --- a/go.sum +++ b/go.sum @@ -221,8 +221,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.15.0 h1:5fCgGYogn0hFdhyhLbw7hEsWxufKtY9klyvdNfFlFhM= -github.com/prometheus/client_golang v1.15.0/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= +github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= +github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= From 5fcd682fb0c30e7a01dcf7d2da2a93997aee5ed1 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Wed, 3 May 2023 13:50:45 -0700 Subject: [PATCH 136/324] Refactor participant metadata updates to avoid duplication (#1679) * Refactor participant metadata updates to avoid duplication * generated fakes --- pkg/rtc/room.go | 9 ++++++ pkg/rtc/signalhandler.go | 7 +---- pkg/rtc/subscriptionmanager.go | 6 ++-- pkg/rtc/types/interfaces.go | 1 + pkg/rtc/types/typesfakes/fake_room.go | 43 +++++++++++++++++++++++++++ pkg/service/roommanager.go | 7 +---- 6 files changed, 58 insertions(+), 15 deletions(-) diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index 72cbb206f..6d339d83f 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -649,6 +649,15 @@ func (r *Room) SetMetadata(metadata string) { r.protoProxy.MarkDirty(true) } +func (r *Room) UpdateParticipantMetadata(participant types.LocalParticipant, name string, metadata string) { + if metadata != "" { + participant.SetMetadata(metadata) + } + if name != "" { + participant.SetName(name) + } +} + func (r *Room) sendRoomUpdate() { roomInfo := r.ToProto() // Send update to participants diff --git a/pkg/rtc/signalhandler.go b/pkg/rtc/signalhandler.go index ee349201f..c29baf10f 100644 --- a/pkg/rtc/signalhandler.go +++ b/pkg/rtc/signalhandler.go @@ -76,12 +76,7 @@ func HandleParticipantSignal(room types.Room, participant types.LocalParticipant case *livekit.SignalRequest_UpdateMetadata: if participant.ClaimGrants().Video.GetCanUpdateOwnMetadata() { - if msg.UpdateMetadata.Metadata != "" { - participant.SetMetadata(msg.UpdateMetadata.Metadata) - } - if msg.UpdateMetadata.Name != "" { - participant.SetName(msg.UpdateMetadata.Name) - } + room.UpdateParticipantMetadata(participant, msg.UpdateMetadata.Name, msg.UpdateMetadata.Metadata) } } return nil diff --git a/pkg/rtc/subscriptionmanager.go b/pkg/rtc/subscriptionmanager.go index 0e43f400f..329c067ab 100644 --- a/pkg/rtc/subscriptionmanager.go +++ b/pkg/rtc/subscriptionmanager.go @@ -391,7 +391,7 @@ func (m *SubscriptionManager) reconcileWorker() { } } -func (m *SubscriptionManager) hasCapcityForSubscription(kind livekit.TrackType) bool { +func (m *SubscriptionManager) hasCapacityForSubscription(kind livekit.TrackType) bool { switch kind { case livekit.TrackType_VIDEO: if m.params.SubscriptionLimitVideo > 0 && m.subscribedVideoCount.Load() >= m.params.SubscriptionLimitVideo { @@ -413,7 +413,7 @@ func (m *SubscriptionManager) subscribe(s *trackSubscription) error { return ErrNoSubscribePermission } - if kind, ok := s.getKind(); ok && !m.hasCapcityForSubscription(kind) { + if kind, ok := s.getKind(); ok && !m.hasCapacityForSubscription(kind) { return ErrSubscriptionLimitExceeded } @@ -445,7 +445,7 @@ func (m *SubscriptionManager) subscribe(s *trackSubscription) error { return ErrTrackNotFound } s.trySetKind(track.Kind()) - if !m.hasCapcityForSubscription(track.Kind()) { + if !m.hasCapacityForSubscription(track.Kind()) { return ErrSubscriptionLimitExceeded } diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index 3c6299f9e..84989179a 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -357,6 +357,7 @@ type Room interface { UpdateVideoLayers(participant Participant, updateVideoLayers *livekit.UpdateVideoLayers) error ResolveMediaTrackForSubscriber(subIdentity livekit.ParticipantIdentity, trackID livekit.TrackID) MediaResolverResult GetLocalParticipants() []LocalParticipant + UpdateParticipantMetadata(participant LocalParticipant, name string, metadata string) } // MediaTrack represents a media track diff --git a/pkg/rtc/types/typesfakes/fake_room.go b/pkg/rtc/types/typesfakes/fake_room.go index 84449cf56..67d68e8ca 100644 --- a/pkg/rtc/types/typesfakes/fake_room.go +++ b/pkg/rtc/types/typesfakes/fake_room.go @@ -82,6 +82,13 @@ type FakeRoom struct { syncStateReturnsOnCall map[int]struct { result1 error } + UpdateParticipantMetadataStub func(types.LocalParticipant, string, string) + updateParticipantMetadataMutex sync.RWMutex + updateParticipantMetadataArgsForCall []struct { + arg1 types.LocalParticipant + arg2 string + arg3 string + } UpdateSubscriptionPermissionStub func(types.LocalParticipant, *livekit.SubscriptionPermission) error updateSubscriptionPermissionMutex sync.RWMutex updateSubscriptionPermissionArgsForCall []struct { @@ -497,6 +504,40 @@ func (fake *FakeRoom) SyncStateReturnsOnCall(i int, result1 error) { }{result1} } +func (fake *FakeRoom) UpdateParticipantMetadata(arg1 types.LocalParticipant, arg2 string, arg3 string) { + fake.updateParticipantMetadataMutex.Lock() + fake.updateParticipantMetadataArgsForCall = append(fake.updateParticipantMetadataArgsForCall, struct { + arg1 types.LocalParticipant + arg2 string + arg3 string + }{arg1, arg2, arg3}) + stub := fake.UpdateParticipantMetadataStub + fake.recordInvocation("UpdateParticipantMetadata", []interface{}{arg1, arg2, arg3}) + fake.updateParticipantMetadataMutex.Unlock() + if stub != nil { + fake.UpdateParticipantMetadataStub(arg1, arg2, arg3) + } +} + +func (fake *FakeRoom) UpdateParticipantMetadataCallCount() int { + fake.updateParticipantMetadataMutex.RLock() + defer fake.updateParticipantMetadataMutex.RUnlock() + return len(fake.updateParticipantMetadataArgsForCall) +} + +func (fake *FakeRoom) UpdateParticipantMetadataCalls(stub func(types.LocalParticipant, string, string)) { + fake.updateParticipantMetadataMutex.Lock() + defer fake.updateParticipantMetadataMutex.Unlock() + fake.UpdateParticipantMetadataStub = stub +} + +func (fake *FakeRoom) UpdateParticipantMetadataArgsForCall(i int) (types.LocalParticipant, string, string) { + fake.updateParticipantMetadataMutex.RLock() + defer fake.updateParticipantMetadataMutex.RUnlock() + argsForCall := fake.updateParticipantMetadataArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + func (fake *FakeRoom) UpdateSubscriptionPermission(arg1 types.LocalParticipant, arg2 *livekit.SubscriptionPermission) error { fake.updateSubscriptionPermissionMutex.Lock() ret, specificReturn := fake.updateSubscriptionPermissionReturnsOnCall[len(fake.updateSubscriptionPermissionArgsForCall)] @@ -683,6 +724,8 @@ func (fake *FakeRoom) Invocations() map[string][][]interface{} { defer fake.simulateScenarioMutex.RUnlock() fake.syncStateMutex.RLock() defer fake.syncStateMutex.RUnlock() + fake.updateParticipantMetadataMutex.RLock() + defer fake.updateParticipantMetadataMutex.RUnlock() fake.updateSubscriptionPermissionMutex.RLock() defer fake.updateSubscriptionPermissionMutex.RUnlock() fake.updateSubscriptionsMutex.RLock() diff --git a/pkg/service/roommanager.go b/pkg/service/roommanager.go index 3966d4f97..5b24c44bc 100644 --- a/pkg/service/roommanager.go +++ b/pkg/service/roommanager.go @@ -590,12 +590,7 @@ func (r *RoomManager) handleRTCMessage(ctx context.Context, roomName livekit.Roo } pLogger.Debugw("updating participant", "metadata", rm.UpdateParticipant.Metadata, "permission", rm.UpdateParticipant.Permission) - if rm.UpdateParticipant.Name != "" { - participant.SetName(rm.UpdateParticipant.Name) - } - if rm.UpdateParticipant.Metadata != "" { - participant.SetMetadata(rm.UpdateParticipant.Metadata) - } + room.UpdateParticipantMetadata(participant, rm.UpdateParticipant.Name, rm.UpdateParticipant.Metadata) if rm.UpdateParticipant.Permission != nil { participant.SetPermission(rm.UpdateParticipant.Permission) } From 298ebaee7874867d786b6093fa032a3a1488ac9f Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Thu, 4 May 2023 15:14:20 +0800 Subject: [PATCH 137/324] Suppress error log of setPrefferedCodec for simulcast codec track (#1682) --- pkg/rtc/mediatrack.go | 1 + pkg/rtc/participant_sdp.go | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/rtc/mediatrack.go b/pkg/rtc/mediatrack.go index f492930e3..4095013fa 100644 --- a/pkg/rtc/mediatrack.go +++ b/pkg/rtc/mediatrack.go @@ -190,6 +190,7 @@ func (t *MediaTrack) SetPendingCodecSid(codecs []*livekit.SimulcastCodec) { } } t.params.TrackInfo = ti + t.MediaTrackReceiver.UpdateTrackInfo(ti) } // AddReceiver adds a new RTP receiver to the track, returns true when receiver represents a new codec diff --git a/pkg/rtc/participant_sdp.go b/pkg/rtc/participant_sdp.go index 524a487fb..427c846a5 100644 --- a/pkg/rtc/participant_sdp.go +++ b/pkg/rtc/participant_sdp.go @@ -112,8 +112,15 @@ func (p *ParticipantImpl) setCodecPreferencesVideoForPublisher(offer webrtc.Sess continue } + var info *livekit.TrackInfo p.pendingTracksLock.RLock() - _, info := p.getPendingTrack(streamID, livekit.TrackType_VIDEO) + mt := p.getPublishedTrackBySdpCid(streamID) + if mt != nil { + info = mt.ToProto() + } else { + _, info = p.getPendingTrack(streamID, livekit.TrackType_VIDEO) + } + if info == nil { p.pendingTracksLock.RUnlock() continue From 15078eb9f4814d5837fea317b8dea15924bb1427 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 4 May 2023 13:00:57 +0530 Subject: [PATCH 138/324] Keep track of expected RTP time stamp and control drift. (#1681) * Keep track of expected RTP time stamp and control drift. - Use monotonic clock in RTCP Sender Report and packet times - Keep the time stamp close to expected time stamp on layer/SSRC switches * clean up * fix test compile * more test compile failures --- pkg/sfu/buffer/buffer.go | 28 +++--- pkg/sfu/buffer/rtpstats.go | 149 +++++++++++++++++++++----------- pkg/sfu/buffer/rtpstats_test.go | 14 +-- pkg/sfu/downtrack.go | 25 ++++-- pkg/sfu/forwarder.go | 90 +++++++++++++++---- pkg/sfu/forwarder_test.go | 2 +- pkg/sfu/streamtrackermanager.go | 5 +- pkg/sfu/testutils/data.go | 4 +- 8 files changed, 218 insertions(+), 99 deletions(-) diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go index 8b9a7094d..0faea1fe8 100644 --- a/pkg/sfu/buffer/buffer.go +++ b/pkg/sfu/buffer/buffer.go @@ -27,17 +27,17 @@ import ( ) const ( - ReportDelta = 1e9 + ReportDelta = time.Second ) type pendingPacket struct { - arrivalTime int64 + arrivalTime time.Time packet []byte } type ExtPacket struct { VideoLayer - Arrival int64 + Arrival time.Time Packet *rtp.Packet Payload interface{} KeyFrame bool @@ -58,7 +58,7 @@ type Buffer struct { closeOnce sync.Once mediaSSRC uint32 clockRate uint32 - lastReport int64 + lastReport time.Time twccExt uint8 audioLevelExt uint8 bound bool @@ -163,7 +163,7 @@ func (b *Buffer) Bind(params webrtc.RTPParameters, codec webrtc.RTPCodecCapabili b.deltaStatsSnapshotId = b.rtpStats.NewSnapshotId() b.clockRate = codec.ClockRate - b.lastReport = time.Now().UnixNano() + b.lastReport = time.Now() b.mime = strings.ToLower(codec.MimeType) for _, ext := range params.HeaderExtensions { @@ -260,12 +260,12 @@ func (b *Buffer) Write(pkt []byte) (n int, err error) { copy(packet, pkt) b.pPackets = append(b.pPackets, pendingPacket{ packet: packet, - arrivalTime: time.Now().UnixNano(), + arrivalTime: time.Now(), }) return } - b.calc(pkt, time.Now().UnixNano()) + b.calc(pkt, time.Now()) return } @@ -392,7 +392,7 @@ func (b *Buffer) SetRTT(rtt uint32) { } } -func (b *Buffer) calc(pkt []byte, arrivalTime int64) { +func (b *Buffer) calc(pkt []byte, arrivalTime time.Time) { pktBuf, err := b.bucket.AddPacket(pkt) if err != nil { // @@ -486,7 +486,7 @@ func (b *Buffer) doFpsCalc(ep *ExtPacket) { } } -func (b *Buffer) updateStreamState(p *rtp.Packet, arrivalTime int64) { +func (b *Buffer) updateStreamState(p *rtp.Packet, arrivalTime time.Time) { flowState := b.rtpStats.Update(&p.Header, len(p.Payload), int(p.PaddingSize), arrivalTime) if b.nacker != nil { @@ -500,12 +500,12 @@ func (b *Buffer) updateStreamState(p *rtp.Packet, arrivalTime int64) { } } -func (b *Buffer) processHeaderExtensions(p *rtp.Packet, arrivalTime int64) { +func (b *Buffer) processHeaderExtensions(p *rtp.Packet, arrivalTime time.Time) { // submit to TWCC even if it is a padding only packet. Clients use padding only packets as probes // for bandwidth estimation if b.twcc != nil && b.twccExt != 0 { if ext := p.GetExtension(b.twccExt); ext != nil { - b.twcc.Push(binary.BigEndian.Uint16(ext[0:2]), arrivalTime, p.Marker) + b.twcc.Push(binary.BigEndian.Uint16(ext[0:2]), arrivalTime.UnixNano(), p.Marker) } } @@ -530,7 +530,7 @@ func (b *Buffer) processHeaderExtensions(p *rtp.Packet, arrivalTime int64) { } } -func (b *Buffer) getExtPacket(rtpPacket *rtp.Packet, arrivalTime int64) *ExtPacket { +func (b *Buffer) getExtPacket(rtpPacket *rtp.Packet, arrivalTime time.Time) *ExtPacket { ep := &ExtPacket{ Packet: rtpPacket, Arrival: arrivalTime, @@ -615,8 +615,8 @@ func (b *Buffer) doNACKs() { } } -func (b *Buffer) doReports(arrivalTime int64) { - timeDiff := arrivalTime - b.lastReport +func (b *Buffer) doReports(arrivalTime time.Time) { + timeDiff := arrivalTime.Sub(b.lastReport) if timeDiff < ReportDelta { return } diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index c7b8575f9..b2cca9f5d 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -1,6 +1,7 @@ package buffer import ( + "errors" "fmt" "math" "sync" @@ -123,11 +124,15 @@ type RTPStats struct { lastRRTime time.Time lastRR rtcp.ReceptionReport - highestTS uint32 - tsCycles uint32 - highestTime int64 + extStartTS uint64 + highestTS uint32 + tsCycles uint32 - lastTransit uint32 + firstTime time.Time + highestTime time.Time + + lastTransit uint32 + lastJitterRTP uint32 bytes uint64 headerBytes uint64 @@ -180,6 +185,8 @@ type RTPStats struct { firstSenderReportRTP uint32 firstFeedSenderReportNTP mediatransportutil.NtpTime firstFeedSenderReportRTP uint32 + lastSRTime time.Time + lastSRNTP mediatransportutil.NtpTime nextSnapshotId uint32 snapshots map[uint32]*Snapshot @@ -206,7 +213,7 @@ func (r *RTPStats) Seed(from *RTPStats) { r.resyncOnNextPacket = from.resyncOnNextPacket r.startTime = from.startTime - // do not clone endTime as a non-zero endTime indiacates an ended object + // do not clone endTime as a non-zero endTime indicates an ended object r.extStartSN = from.extStartSN r.highestSN = from.highestSN @@ -216,11 +223,15 @@ func (r *RTPStats) Seed(from *RTPStats) { r.lastRRTime = from.lastRRTime r.lastRR = from.lastRR + r.extStartTS = from.extStartTS r.highestTS = from.highestTS r.tsCycles = from.tsCycles + + r.firstTime = from.firstTime r.highestTime = from.highestTime r.lastTransit = from.lastTransit + r.lastJitterRTP = from.lastJitterRTP r.bytes = from.bytes r.headerBytes = from.headerBytes @@ -280,6 +291,8 @@ func (r *RTPStats) Seed(from *RTPStats) { r.firstSenderReportRTP = from.firstSenderReportRTP r.firstFeedSenderReportNTP = from.firstFeedSenderReportNTP r.firstFeedSenderReportRTP = from.firstFeedSenderReportRTP + r.lastSRTime = from.lastSRTime + r.lastSRNTP = from.lastSRNTP r.nextSnapshotId = from.nextSnapshotId for id, ss := range from.snapshots { @@ -324,7 +337,7 @@ func (r *RTPStats) IsActive() bool { return r.initialized && r.endTime.IsZero() } -func (r *RTPStats) Update(rtph *rtp.Header, payloadSize int, paddingSize int, packetTime int64) (flowState RTPFlowState) { +func (r *RTPStats) Update(rtph *rtp.Header, payloadSize int, paddingSize int, packetTime time.Time) (flowState RTPFlowState) { r.lock.Lock() defer r.lock.Unlock() @@ -338,14 +351,17 @@ func (r *RTPStats) Update(rtph *rtp.Header, payloadSize int, paddingSize int, pa r.startTime = time.Now() - r.highestSN = rtph.SequenceNumber - 1 - r.highestTS = rtph.Timestamp - r.highestTime = packetTime - r.extStartSN = uint32(rtph.SequenceNumber) + r.highestSN = rtph.SequenceNumber - 1 r.cycles = 0 + + r.extStartTS = uint64(rtph.Timestamp) + r.highestTS = rtph.Timestamp r.tsCycles = 0 + r.firstTime = packetTime + r.highestTime = packetTime + first = true // initialize snapshots if any @@ -378,7 +394,7 @@ func (r *RTPStats) Update(rtph *rtp.Header, payloadSize int, paddingSize int, pa } // adjust start to account for out-of-order packets before a cycle completes - if !r.maybeAdjustStartSN(rtph, packetTime, pktSize, hdrSize, payloadSize) { + if !r.maybeAdjustStartSN(rtph, pktSize, hdrSize, payloadSize) { if !r.isSnInfoLost(rtph.SequenceNumber) { r.bytesDuplicate += pktSize r.headerBytesDuplicate += hdrSize @@ -447,7 +463,7 @@ func (r *RTPStats) ResyncOnNextPacket() { r.resyncOnNextPacket = true } -func (r *RTPStats) maybeAdjustStartSN(rtph *rtp.Header, packetTime int64, pktSize uint64, hdrSize uint64, payloadSize int) bool { +func (r *RTPStats) maybeAdjustStartSN(rtph *rtp.Header, pktSize uint64, hdrSize uint64, payloadSize int) bool { if (r.getExtHighestSN() - r.extStartSN + 1) >= (NumSequenceNumbers / 2) { return false } @@ -501,7 +517,7 @@ func (r *RTPStats) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt uint32 return } - rtt, err := mediatransportutil.GetRttMsFromReceiverReportOnly(&rr) + rtt, err := mediatransportutil.GetRttMs(&rr, r.lastSRNTP, r.lastSRTime) if err == nil { isRttChanged = rtt != r.rtt } else { @@ -795,6 +811,32 @@ func (r *RTPStats) GetRtcpSenderReportDataExt() *RTCPSenderReportDataExt { } } +func (r *RTPStats) GetExpectedRTPTimestamp(at time.Time) (uint32, error) { + r.lock.RLock() + defer r.lock.RUnlock() + + if !r.initialized { + return 0, errors.New("uninitilaized") + } + + timeDiff := at.Sub(r.firstTime) + rtpDiff := timeDiff.Nanoseconds() * int64(r.params.ClockRate) / 1e9 + expectedExtRTP := r.extStartTS + uint64(rtpDiff) + r.logger.Debugw( + "expected RTP timestamp", + "firstTime", r.firstTime.String(), + "checkAt", at.String(), + "timeDiff", timeDiff, + "firstRTP", r.extStartTS, + "rtpDiff", rtpDiff, + "expectedExtRTP", expectedExtRTP, + "expectedRTP", uint32(expectedExtRTP), + "highestTS", r.highestTS, + "highestTime", r.highestTime.String(), + ) + return uint32(expectedExtRTP), nil +} + func (r *RTPStats) GetRtcpSenderReport(ssrc uint32, srDataExt *RTCPSenderReportDataExt) *rtcp.SenderReport { r.lock.Lock() defer r.lock.Unlock() @@ -808,28 +850,27 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32, srDataExt *RTCPSenderReportD return nil } - // NTP timestamp in sender report from publisher side could have a different base, - // i. e. although it should be wall clock at time of send, have observed instances of older timer. - // It is not possible to accurately calculate current time in the NTP time base of the publisher side. - // So, using a smoothed version of one way delay for use in sender reports. - now := time.Now() + // construct current time based on monotonic clock + timeSinceFirst := time.Since(r.firstTime) + now := r.firstTime.Add(timeSinceFirst) nowNTP := mediatransportutil.ToNtpTime(now) - nowRTP := r.highestTS - isUsingSmoothed := true - smoothedLocalTimeOfLatestSenderReportNTP := srDataExt.SenderReportData.NTPTimestamp.Time().Add(srDataExt.SmoothedOWD) - if smoothedLocalTimeOfLatestSenderReportNTP.After(now) { - isUsingSmoothed = false - r.logger.Debugw("smoothed time of NTP is ahead", - "now", now, - "smoothed", smoothedLocalTimeOfLatestSenderReportNTP, - "diff", smoothedLocalTimeOfLatestSenderReportNTP.Sub(now), + expectedExtRTP := r.extStartTS + uint64(timeSinceFirst.Nanoseconds()*int64(r.params.ClockRate)/1e9) + if getExtTS(r.highestTS, r.tsCycles) > expectedExtRTP || now.Before(r.highestTime) { + r.logger.Debugw( + "anachronous sender report", + "firstTime", r.firstTime.String(), + "currentTime", now.String(), + "timSinceFirst", timeSinceFirst, + "extStartTS", r.extStartTS, + "highestExtRTP", getExtTS(r.highestTS, r.tsCycles), + "expectedExtRTP", expectedExtRTP, ) - nowRTP += uint32(now.Sub(time.Unix(0, r.highestTime)).Milliseconds() * int64(r.params.ClockRate) / 1000) - } else { - nowRTP = srDataExt.SenderReportData.RTPTimestamp + uint32(now.Sub(smoothedLocalTimeOfLatestSenderReportNTP).Milliseconds()*int64(r.params.ClockRate)/1000) } + timeSinceHighest := time.Since(r.highestTime) + nowRTP := r.highestTS + uint32(timeSinceHighest.Nanoseconds()*int64(r.params.ClockRate)/1e9) + // TODO-REMOVE-AFTER-DEBUG if r.firstSenderReportNTP == 0 { r.firstSenderReportNTP = nowNTP @@ -838,57 +879,50 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32, srDataExt *RTCPSenderReportD r.firstFeedSenderReportNTP = srDataExt.SenderReportData.NTPTimestamp r.firstFeedSenderReportRTP = srDataExt.SenderReportData.RTPTimestamp } else { - highestTime := time.Unix(0, r.highestTime) ntpTime := nowNTP.Time() - ntpDiffLocal := ntpTime.Sub(highestTime) + ntpDiffLocal := ntpTime.Sub(r.highestTime) rtpDiffLocal := int32(nowRTP - r.highestTS) rtpOffsetLocal := int32(nowRTP - r.highestTS - uint32(ntpDiffLocal.Nanoseconds()*int64(r.params.ClockRate)/1e9)) - ntpDiffSmoothed := ntpTime.Sub(smoothedLocalTimeOfLatestSenderReportNTP) - rtpDiffSmoothed := int32(nowRTP - srDataExt.SenderReportData.RTPTimestamp) - rtpOffsetSmoothed := int32(nowRTP - srDataExt.SenderReportData.RTPTimestamp - uint32(ntpDiffSmoothed.Nanoseconds()*int64(r.params.ClockRate)/1e9)) - timeSinceFirst := nowNTP.Time().Sub(r.firstSenderReportNTP.Time()) rtpDiffSinceFirst := getExtTS(nowRTP, r.tsCycles) - getExtTS(r.firstSenderReportRTP, 0) drift := int64(uint64(timeSinceFirst.Nanoseconds()*int64(r.params.ClockRate)/1e9) - rtpDiffSinceFirst) - driftTime := (float64(drift) * 1000) / float64(r.params.ClockRate) + driftMs := (float64(drift) * 1000) / float64(r.params.ClockRate) feedTimeSinceFirst := srDataExt.SenderReportData.NTPTimestamp.Time().Sub(r.firstFeedSenderReportNTP.Time()) // using tsCycles for extending feed time stamp too feedRtpDiffSinceFirst := getExtTS(srDataExt.SenderReportData.RTPTimestamp, r.tsCycles) - getExtTS(r.firstFeedSenderReportRTP, 0) feedDrift := int64(uint64(feedTimeSinceFirst.Nanoseconds()*int64(r.params.ClockRate)/1e9) - feedRtpDiffSinceFirst) - feedDriftTime := (float64(feedDrift) * 1000) / float64(r.params.ClockRate) + feedDriftMs := (float64(feedDrift) * 1000) / float64(r.params.ClockRate) r.logger.Debugw( "sending sender report", "highestTS", r.highestTS, - "highestTime", highestTime, - "smoothedTime", smoothedLocalTimeOfLatestSenderReportNTP, + "highestTime", r.highestTime.String(), "reportTS", nowRTP, - "reportTime", ntpTime, + "reportTime", ntpTime.String(), "rtpDiffLocal", rtpDiffLocal, "ntpDiffLocal", ntpDiffLocal, "rtpOffsetLocal", rtpOffsetLocal, - "rtpDiffSmoothed", rtpDiffSmoothed, - "ntpDiffSmoothed", ntpDiffSmoothed, - "rtpOffsetSmoothed", rtpOffsetSmoothed, "timeSinceFirst", timeSinceFirst, "rtpDiffSinceFirst", rtpDiffSinceFirst, "drift", drift, - "driftTime(ms)", driftTime, - "smoothed", isUsingSmoothed, + "driftMs", driftMs, "feedRTP", srDataExt.SenderReportData.RTPTimestamp, - "feedNTP", srDataExt.SenderReportData.NTPTimestamp.Time(), - "feedArrival", srDataExt.SenderReportData.ArrivalTime, + "feedNTP", srDataExt.SenderReportData.NTPTimestamp.Time().String(), + "feedArrival", srDataExt.SenderReportData.ArrivalTime.String(), "smoothedOWD", srDataExt.SmoothedOWD, "feedTimeSinceFirst", feedTimeSinceFirst, "feedRtpDiffSinceFirst", feedRtpDiffSinceFirst, "feedDrift", feedDrift, - "feedDriftTime(ms)", feedDriftTime, + "feedDriftMs", feedDriftMs, ) } + r.lastSRTime = now + r.lastSRNTP = nowNTP + return &rtcp.SenderReport{ SSRC: ssrc, NTPTime: uint64(nowNTP), @@ -1400,8 +1434,20 @@ func (r *RTPStats) getIntervalStats(startInclusive uint16, endExclusive uint16) return } -func (r *RTPStats) updateJitter(rtph *rtp.Header, packetTime int64) { - packetTimeRTP := uint32(packetTime / 1e6 * int64(r.params.ClockRate/1e3)) +func (r *RTPStats) updateJitter(rtph *rtp.Header, packetTime time.Time) { + // Do not update jitter on multiple packets of same frame. + // All packets of a frame have the same time stamp. + // NOTE: This does not protect against using more than one packet of the same frame + // if packets arrive out-of-order. For example, + // p1f1 -> p1f2 -> p2f1 + // In this case, p2f1 (packet 2, frame 1) will still be used in jitter calculation + // although it is the second packet of a frame because of out-of-order receival. + if r.lastJitterRTP == rtph.Timestamp { + return + } + + timeSinceFirst := packetTime.Sub(r.firstTime) + packetTimeRTP := uint32(timeSinceFirst.Nanoseconds() * int64(r.params.ClockRate) / 1e9) transit := packetTimeRTP - rtph.Timestamp if r.lastTransit != 0 { @@ -1422,6 +1468,7 @@ func (r *RTPStats) updateJitter(rtph *rtp.Header, packetTime int64) { } r.lastTransit = transit + r.lastJitterRTP = rtph.Timestamp } func (r *RTPStats) updateGapHistogram(gap int) { diff --git a/pkg/sfu/buffer/rtpstats_test.go b/pkg/sfu/buffer/rtpstats_test.go index b1be40650..70c852e7b 100644 --- a/pkg/sfu/buffer/rtpstats_test.go +++ b/pkg/sfu/buffer/rtpstats_test.go @@ -43,7 +43,7 @@ func TestRTPStats(t *testing.T) { timestamp += uint32(now.Sub(lastFrameTime).Seconds() * float64(clockRate)) for i := 0; i < packetsPerFrame; i++ { packet := getPacket(sequenceNumber, timestamp, packetSize) - r.Update(&packet.Header, len(packet.Payload), 0, time.Now().UnixNano()) + r.Update(&packet.Header, len(packet.Payload), 0, time.Now()) if (sequenceNumber % 100) == 0 { jump := uint16(rand.Float64() * 120.0) sequenceNumber += jump @@ -70,7 +70,7 @@ func TestRTPStats_Update(t *testing.T) { sequenceNumber := uint16(rand.Float64() * float64(1<<16)) timestamp := uint32(rand.Float64() * float64(1<<32)) packet := getPacket(sequenceNumber, timestamp, 1000) - flowState := r.Update(&packet.Header, len(packet.Payload), 0, time.Now().UnixNano()) + flowState := r.Update(&packet.Header, len(packet.Payload), 0, time.Now()) require.False(t, flowState.HasLoss) require.True(t, r.initialized) require.Equal(t, sequenceNumber, r.highestSN) @@ -80,14 +80,14 @@ func TestRTPStats_Update(t *testing.T) { sequenceNumber++ timestamp += 3000 packet = getPacket(sequenceNumber, timestamp, 1000) - flowState = r.Update(&packet.Header, len(packet.Payload), 0, time.Now().UnixNano()) + flowState = r.Update(&packet.Header, len(packet.Payload), 0, time.Now()) require.False(t, flowState.HasLoss) require.Equal(t, sequenceNumber, r.highestSN) require.Equal(t, timestamp, r.highestTS) // out-of-order packet = getPacket(sequenceNumber-10, timestamp-30000, 1000) - flowState = r.Update(&packet.Header, len(packet.Payload), 0, time.Now().UnixNano()) + flowState = r.Update(&packet.Header, len(packet.Payload), 0, time.Now()) require.False(t, flowState.HasLoss) require.Equal(t, sequenceNumber, r.highestSN) require.Equal(t, timestamp, r.highestTS) @@ -96,7 +96,7 @@ func TestRTPStats_Update(t *testing.T) { // duplicate packet = getPacket(sequenceNumber-10, timestamp-30000, 1000) - flowState = r.Update(&packet.Header, len(packet.Payload), 0, time.Now().UnixNano()) + flowState = r.Update(&packet.Header, len(packet.Payload), 0, time.Now()) require.False(t, flowState.HasLoss) require.Equal(t, sequenceNumber, r.highestSN) require.Equal(t, timestamp, r.highestTS) @@ -107,7 +107,7 @@ func TestRTPStats_Update(t *testing.T) { sequenceNumber += 10 timestamp += 30000 packet = getPacket(sequenceNumber, timestamp, 1000) - flowState = r.Update(&packet.Header, len(packet.Payload), 0, time.Now().UnixNano()) + flowState = r.Update(&packet.Header, len(packet.Payload), 0, time.Now()) require.True(t, flowState.HasLoss) require.Equal(t, sequenceNumber-9, flowState.LossStartInclusive) require.Equal(t, sequenceNumber, flowState.LossEndExclusive) @@ -115,7 +115,7 @@ func TestRTPStats_Update(t *testing.T) { // out-of-order should decrement number of lost packets packet = getPacket(sequenceNumber-15, timestamp-45000, 1000) - flowState = r.Update(&packet.Header, len(packet.Payload), 0, time.Now().UnixNano()) + flowState = r.Update(&packet.Header, len(packet.Payload), 0, time.Now()) require.False(t, flowState.HasLoss) require.Equal(t, sequenceNumber, r.highestSN) require.Equal(t, timestamp, r.highestTS) diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 6ad785500..0e6638b0e 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -276,7 +276,12 @@ func NewDownTrack( kind: kind, codec: codecs[0].RTPCodecCapability, } - d.forwarder = NewForwarder(d.kind, d.logger, d.receiver.GetReferenceLayerRTPTimestamp) + d.forwarder = NewForwarder( + d.kind, + d.logger, + d.receiver.GetReferenceLayerRTPTimestamp, + d.getExpectedRTPTimestamp, + ) d.forwarder.OnParkedLayerExpired(func() { if sal := d.getStreamAllocatorListener(); sal != nil { sal.OnSubscriptionChanged(d) @@ -638,7 +643,7 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { } } - d.rtpStats.Update(hdr, len(payload), 0, time.Now().UnixNano()) + d.rtpStats.Update(hdr, len(payload), 0, extPkt.Arrival) return nil } @@ -717,7 +722,7 @@ func (d *DownTrack) WritePaddingRTP(bytesToSend int, paddingOnMute bool) int { } if !paddingOnMute { - d.rtpStats.Update(&hdr, 0, len(payload), time.Now().UnixNano()) + d.rtpStats.Update(&hdr, 0, len(payload), time.Now()) } // @@ -1214,7 +1219,7 @@ func (d *DownTrack) writeOpusBlankFrame(hdr *rtp.Header, frameEndNeeded bool) (i _, err := d.writeStream.WriteRTP(hdr, payload) if err == nil { - d.rtpStats.Update(hdr, len(payload), 0, time.Now().UnixNano()) + d.rtpStats.Update(hdr, len(payload), 0, time.Now()) } return hdr.MarshalSize() + len(payload), err } @@ -1233,7 +1238,7 @@ func (d *DownTrack) writeOpusRedBlankFrame(hdr *rtp.Header, frameEndNeeded bool) _, err := d.writeStream.WriteRTP(hdr, payload) if err == nil { - d.rtpStats.Update(hdr, len(payload), 0, time.Now().UnixNano()) + d.rtpStats.Update(hdr, len(payload), 0, time.Now()) } return hdr.MarshalSize() + len(payload), err } @@ -1254,7 +1259,7 @@ func (d *DownTrack) writeVP8BlankFrame(hdr *rtp.Header, frameEndNeeded bool) (in _, err = d.writeStream.WriteRTP(hdr, payload) if err == nil { - d.rtpStats.Update(hdr, len(payload), 0, time.Now().UnixNano()) + d.rtpStats.Update(hdr, len(payload), 0, time.Now()) } return hdr.MarshalSize() + len(payload), err } @@ -1276,7 +1281,7 @@ func (d *DownTrack) writeH264BlankFrame(hdr *rtp.Header, frameEndNeeded bool) (i payload := buf[:offset] _, err := d.writeStream.WriteRTP(hdr, payload) if err == nil { - d.rtpStats.Update(hdr, len(payload), 0, time.Now().UnixNano()) + d.rtpStats.Update(hdr, len(payload), 0, time.Now()) } return hdr.MarshalSize() + offset, err } @@ -1499,7 +1504,7 @@ func (d *DownTrack) retransmitPackets(nacks []uint16) { d.streamAllocatorBytesCounter.Add(uint32(pkt.Header.MarshalSize() + len(payload))) d.bytesRetransmitted.Add(uint32(pkt.Header.MarshalSize() + len(payload))) - d.rtpStats.Update(&pkt.Header, len(payload), 0, time.Now().UnixNano()) + d.rtpStats.Update(&pkt.Header, len(payload), 0, time.Now()) } } @@ -1624,6 +1629,10 @@ func (d *DownTrack) DebugInfo() map[string]interface{} { } } +func (d *DownTrack) getExpectedRTPTimestamp(at time.Time) (uint32, error) { + return d.rtpStats.GetExpectedRTPTimestamp(at) +} + func (d *DownTrack) GetConnectionScoreAndQuality() (float32, livekit.ConnectionQuality) { return d.connectionStats.GetScoreAndQuality() } diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 4c1ba5028..e6da06859 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -158,6 +158,7 @@ type Forwarder struct { kind webrtc.RTPCodecType logger logger.Logger getReferenceLayerRTPTimestamp func(ts uint32, layer int32, referenceLayer int32) (uint32, error) + getExpectedRTPTimestamp func(at time.Time) (uint32, error) muted bool pubMuted bool @@ -185,11 +186,13 @@ func NewForwarder( kind webrtc.RTPCodecType, logger logger.Logger, getReferenceLayerRTPTimestamp func(ts uint32, layer int32, referenceLayer int32) (uint32, error), + getExpectedRTPTimestamp func(at time.Time) (uint32, error), ) *Forwarder { f := &Forwarder{ kind: kind, logger: logger, getReferenceLayerRTPTimestamp: getReferenceLayerRTPTimestamp, + getExpectedRTPTimestamp: getExpectedRTPTimestamp, referenceLayerSpatial: buffer.InvalidLayerSpatial, lastAllocation: VideoAllocationDefault, rtpMunger: NewRTPMunger(logger), @@ -1446,26 +1449,46 @@ func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer i // Compute how much time passed between the old RTP extPkt // and the current packet, and fix timestamp on source change - td := uint32(1) + // + // There are three time stamps to consider here + // 1. lastTS -> time stamp of last sent packet + // 2. refTS -> time stamp of this packet (after munging) calculated using feed's RTCP sender report + // 3. expectedTS -> time stamp of this packet (after munging) calculated using this stream's RTCP sender report + // Ideally, refTS and expectedTS should be very close and lastTS should be before both of those. + // But, cases like muting/unmuting, clock vagaries make them not satisfy those conditions always. + // + // There are 6 orderings to consider (considering only inequalities). Resolve them using following rules + // 1. Timestamp has to move forward + // 2. Keep next time stamp close to expected + lastTS := f.rtpMunger.GetLast().LastTS + refTS := lastTS + expectedTS := lastTS + switchingAt := time.Now() if f.getReferenceLayerRTPTimestamp != nil { - refTS, err := f.getReferenceLayerRTPTimestamp(extPkt.Packet.Timestamp, layer, f.referenceLayerSpatial) + ts, err := f.getReferenceLayerRTPTimestamp(extPkt.Packet.Timestamp, layer, f.referenceLayerSpatial) if err == nil { - last := f.rtpMunger.GetLast() - td = refTS - last.LastTS - if td == 0 || td > (1<<31) { - f.logger.Debugw("reference timestamp out-of-order, using default", "lastTS", last.LastTS, "refTS", refTS, "td", int32(td)) - td = 1 - } else if td > uint32(0.5*float32(f.codec.ClockRate)) { - // log jumps greater than 0.5 seconds - f.logger.Debugw("reference timestamp too far ahead", "lastTS", last.LastTS, "refTS", refTS, "td", td) - } - f.logger.Debugw("reference timestamp on switch", "lastTS", last.LastTS, "refTS", refTS, "td", int32(td), "switchingAt", time.Now()) - } else { - f.logger.Debugw("reference timestamp get error, using default", "error", err) + refTS = ts } } + if f.getExpectedRTPTimestamp != nil { + ts, err := f.getExpectedRTPTimestamp(switchingAt) + if err == nil { + expectedTS = ts + } + } + nextTS, explain := getNextTimestamp(lastTS, refTS, expectedTS) + f.logger.Debugw( + "next timestamp on switch", + "switchingAt", switchingAt.String(), + "lastTS", lastTS, + "refTS", refTS, + "expectedTS", expectedTS, + "nextTS", nextTS, + "jump", nextTS-lastTS, + "explanation", explain, + ) - f.rtpMunger.UpdateSnTsOffsets(extPkt, 1, td) + f.rtpMunger.UpdateSnTsOffsets(extPkt, 1, nextTS-lastTS) f.codecMunger.UpdateOffsets(extPkt) } @@ -1734,3 +1757,40 @@ done: return float64(distance) / float64(maxSeenLayer.Temporal+1) } + +func getNextTimestamp(lastTS uint32, refTS uint32, expectedTS uint32) (uint32, string) { + isInOrder := func(val1, val2 uint32) bool { + diff := val1 - val2 + return diff != 0 && diff < (1<<31) + } + + rl := isInOrder(refTS, lastTS) + el := isInOrder(expectedTS, lastTS) + er := isInOrder(expectedTS, refTS) + + nextTS := lastTS + 1 + explain := "l = r = e" + + switch { + case rl && el && er: // lastTS < refTS < expectedTS + nextTS = uint32(float64(refTS) + 0.95*float64(expectedTS-refTS)) + explain = fmt.Sprintf("l < r < e, %d, %d", refTS-lastTS, expectedTS-refTS) + case rl && el && !er: // lastTS < expectedTS < refTS + nextTS = uint32(float64(expectedTS) + 0.5*float64(refTS-expectedTS)) + explain = fmt.Sprintf("l < e < r, %d, %d", expectedTS-lastTS, refTS-expectedTS) + case !rl && el && er: // refTS < lastTS < expectedTS + nextTS = uint32(float64(lastTS) + 0.5*float64(expectedTS-lastTS)) + explain = fmt.Sprintf("r < l < e, %d, %d", lastTS-refTS, expectedTS-lastTS) + case !rl && !el && er: // refTS < expectedTS < lastTS + nextTS = lastTS + 1 + explain = fmt.Sprintf("r < e < l, %d, %d", expectedTS-refTS, lastTS-expectedTS) + case rl && !el && !er: // expectedTS < lastTS < refTS + nextTS = uint32(float64(lastTS) + 0.5*float64(refTS-lastTS)) + explain = fmt.Sprintf("e < l < r, %d, %d", lastTS-expectedTS, refTS-lastTS) + case !rl && !el && !er: // expectedTS < refTS < lastTS + nextTS = lastTS + 1 + explain = fmt.Sprintf("e < r < l, %d, %d", refTS-expectedTS, lastTS-refTS) + } + + return nextTS, explain +} diff --git a/pkg/sfu/forwarder_test.go b/pkg/sfu/forwarder_test.go index ac3166f70..c99374ded 100644 --- a/pkg/sfu/forwarder_test.go +++ b/pkg/sfu/forwarder_test.go @@ -18,7 +18,7 @@ func disable(f *Forwarder) { } func newForwarder(codec webrtc.RTPCodecCapability, kind webrtc.RTPCodecType) *Forwarder { - f := NewForwarder(kind, logger.GetLogger(), nil) + f := NewForwarder(kind, logger.GetLogger(), nil, nil) f.DetermineCodec(codec, nil) return f } diff --git a/pkg/sfu/streamtrackermanager.go b/pkg/sfu/streamtrackermanager.go index e4598e675..8e324e6a3 100644 --- a/pkg/sfu/streamtrackermanager.go +++ b/pkg/sfu/streamtrackermanager.go @@ -529,8 +529,9 @@ func (s *StreamTrackerManager) GetReferenceLayerRTPTimestamp(ts uint32, layer in // NOTE: It is possible that reference layer has stopped (due to dynacast/adaptive streaming OR publisher // constraints). It should be okay even if the layer has stopped for a long time when using modulo arithmetic for // RTP time stamp (uint32 arithmetic). - ntpDiff := float64(int64(srRef.SenderReportData.NTPTimestamp-srLayer.SenderReportData.NTPTimestamp)) / float64(1<<32) - normalizedTS := srLayer.SenderReportData.RTPTimestamp + uint32(ntpDiff*float64(s.clockRate)) + ntpDiff := srRef.SenderReportData.NTPTimestamp.Time().Sub(srLayer.SenderReportData.NTPTimestamp.Time()) + rtpDiff := ntpDiff.Nanoseconds() * int64(s.clockRate) / 1e9 + normalizedTS := srLayer.SenderReportData.RTPTimestamp + uint32(rtpDiff) // now that both RTP timestamps correspond to roughly the same NTP time, // the diff between them is the offset in RTP timestamp units between layer and referenceLayer. diff --git a/pkg/sfu/testutils/data.go b/pkg/sfu/testutils/data.go index 8b243b40f..2ab28767a 100644 --- a/pkg/sfu/testutils/data.go +++ b/pkg/sfu/testutils/data.go @@ -1,6 +1,8 @@ package testutils import ( + "time" + "github.com/pion/rtp" "github.com/pion/webrtc/v3" @@ -18,7 +20,7 @@ type TestExtPacketParams struct { SSRC uint32 PayloadSize int PaddingSize byte - ArrivalTime int64 + ArrivalTime time.Time VideoLayer buffer.VideoLayer } From 28a8a808f2645754160da890704ef5beeecb1ee5 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 5 May 2023 08:59:08 +0530 Subject: [PATCH 139/324] Do not add empty video layers in stats. (#1685) --- pkg/sfu/connectionquality/connectionstats.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pkg/sfu/connectionquality/connectionstats.go b/pkg/sfu/connectionquality/connectionstats.go index c611c7fc0..0da29996a 100644 --- a/pkg/sfu/connectionquality/connectionstats.go +++ b/pkg/sfu/connectionquality/connectionstats.go @@ -226,7 +226,10 @@ func (cs *ConnectionStats) getStat(at time.Time) { // if (len(streams) > 1 || len(stream.Layers) > 1) && cs.isVideo.Load() { for layer, layerStats := range stream.Layers { - as.VideoLayers = append(as.VideoLayers, toAnalyticsVideoLayer(layer, layerStats)) + avl := toAnalyticsVideoLayer(layer, layerStats) + if avl != nil { + as.VideoLayers = append(as.VideoLayers, avl) + } } } @@ -333,10 +336,15 @@ func toAnalyticsStream(ssrc uint32, deltaStats *buffer.RTPDeltaInfo) *livekit.An } func toAnalyticsVideoLayer(layer int32, layerStats *buffer.RTPDeltaInfo) *livekit.AnalyticsVideoLayer { - return &livekit.AnalyticsVideoLayer{ + avl := &livekit.AnalyticsVideoLayer{ Layer: layer, Packets: layerStats.Packets + layerStats.PacketsDuplicate + layerStats.PacketsPadding, Bytes: layerStats.Bytes + layerStats.BytesDuplicate + layerStats.BytesPadding, Frames: layerStats.Frames, } + if avl.Packets == 0 || avl.Bytes == 0 || avl.Frames == 0 { + return nil + } + + return avl } From e00ff50cd640efa322bc096d74e029442c459de3 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 4 May 2023 22:33:22 -0700 Subject: [PATCH 140/324] Update go deps (#1680) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 8c5ed846b..49e083c66 100644 --- a/go.mod +++ b/go.mod @@ -45,8 +45,8 @@ require ( github.com/ua-parser/uap-go v0.0.0-20211112212520-00c877edfe0f github.com/urfave/cli/v2 v2.25.3 github.com/urfave/negroni/v3 v3.0.0 - go.uber.org/atomic v1.10.0 - golang.org/x/sync v0.1.0 + go.uber.org/atomic v1.11.0 + golang.org/x/sync v0.2.0 google.golang.org/protobuf v1.30.0 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go.sum b/go.sum index f2afcbf39..8ff9983c4 100644 --- a/go.sum +++ b/go.sum @@ -267,8 +267,8 @@ github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsr github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= -go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= @@ -322,8 +322,9 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= +golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= From 25d6fd751f7e58ec1137bfc45a619340dc7c52f5 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 5 May 2023 13:14:12 +0530 Subject: [PATCH 141/324] Cleaning up smoothed OWD calculation for sender report. (#1684) * Keep track of expected RTP time stamp and control drift. - Use monotonic clock in RTCP Sender Report and packet times - Keep the time stamp close to expected time stamp on layer/SSRC switches * clean up * fix test compile * more test compile failures * anticipatory clean up * further clean up * add received sender report logging --- pkg/rtc/wrappedreceiver.go | 8 -- pkg/sfu/buffer/buffer.go | 4 +- pkg/sfu/buffer/rtpstats.go | 190 +++++++++++--------------------- pkg/sfu/downtrack.go | 2 +- pkg/sfu/forwarder.go | 7 -- pkg/sfu/receiver.go | 7 +- pkg/sfu/streamtrackermanager.go | 29 ++--- 7 files changed, 80 insertions(+), 167 deletions(-) diff --git a/pkg/rtc/wrappedreceiver.go b/pkg/rtc/wrappedreceiver.go index b962ba809..068f8b69d 100644 --- a/pkg/rtc/wrappedreceiver.go +++ b/pkg/rtc/wrappedreceiver.go @@ -12,7 +12,6 @@ import ( "github.com/livekit/protocol/logger" "github.com/livekit/livekit-server/pkg/sfu" - "github.com/livekit/livekit-server/pkg/sfu/buffer" ) // wrapper around WebRTC receiver, overriding its ID @@ -289,13 +288,6 @@ func (d *DummyReceiver) GetRedReceiver() sfu.TrackReceiver { return d } -func (d *DummyReceiver) GetRTCPSenderReportDataExt(layer int32) *buffer.RTCPSenderReportDataExt { - if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { - return r.GetRTCPSenderReportDataExt(layer) - } - return nil -} - func (d *DummyReceiver) GetReferenceLayerRTPTimestamp(ts uint32, layer int32, referenceLayer int32) (uint32, error) { if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { return r.GetReferenceLayerRTPTimestamp(ts, layer, referenceLayer) diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go index 0faea1fe8..63949a52c 100644 --- a/pkg/sfu/buffer/buffer.go +++ b/pkg/sfu/buffer/buffer.go @@ -668,12 +668,12 @@ func (b *Buffer) SetSenderReportData(rtpTime uint32, ntpTime uint64) { } } -func (b *Buffer) GetSenderReportDataExt() *RTCPSenderReportDataExt { +func (b *Buffer) GetSenderReportData() *RTCPSenderReportData { b.RLock() defer b.RUnlock() if b.rtpStats != nil { - return b.rtpStats.GetRtcpSenderReportDataExt() + return b.rtpStats.GetRtcpSenderReportData() } return nil diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index b2cca9f5d..339959e6d 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -3,7 +3,6 @@ package buffer import ( "errors" "fmt" - "math" "sync" "time" @@ -93,11 +92,6 @@ type RTCPSenderReportData struct { ArrivalTime time.Time } -type RTCPSenderReportDataExt struct { - SenderReportData RTCPSenderReportData - SmoothedOWD time.Duration -} - type RTPStatsParams struct { ClockRate uint32 IsReceiverReportDriven bool @@ -180,13 +174,9 @@ type RTPStats struct { rtt uint32 maxRtt uint32 - srDataExt *RTCPSenderReportDataExt - firstSenderReportNTP mediatransportutil.NtpTime - firstSenderReportRTP uint32 - firstFeedSenderReportNTP mediatransportutil.NtpTime - firstFeedSenderReportRTP uint32 - lastSRTime time.Time - lastSRNTP mediatransportutil.NtpTime + srData *RTCPSenderReportData + lastSRTime time.Time + lastSRNTP mediatransportutil.NtpTime nextSnapshotId uint32 snapshots map[uint32]*Snapshot @@ -279,18 +269,12 @@ func (r *RTPStats) Seed(from *RTPStats) { r.rtt = from.rtt r.maxRtt = from.maxRtt - if from.srDataExt != nil { - r.srDataExt = &RTCPSenderReportDataExt{ - SenderReportData: from.srDataExt.SenderReportData, - SmoothedOWD: from.srDataExt.SmoothedOWD, - } + if from.srData != nil { + srData := *from.srData + r.srData = &srData } else { - r.srDataExt = nil + r.srData = nil } - r.firstSenderReportNTP = from.firstSenderReportNTP - r.firstSenderReportRTP = from.firstSenderReportRTP - r.firstFeedSenderReportNTP = from.firstFeedSenderReportNTP - r.firstFeedSenderReportRTP = from.firstFeedSenderReportRTP r.lastSRTime = from.lastSRTime r.lastSRNTP = from.lastSRNTP @@ -732,52 +716,37 @@ func (r *RTPStats) SetRtcpSenderReportData(srData *RTCPSenderReportData) { defer r.lock.Unlock() if srData == nil { - r.srDataExt = nil + r.srData = nil return } // prevent against extreme case of anachronous sender reports - if r.srDataExt != nil && r.srDataExt.SenderReportData.NTPTimestamp > srData.NTPTimestamp { + if r.srData != nil && r.srData.NTPTimestamp > srData.NTPTimestamp { r.logger.Debugw( "anachronous RTCP sender report", "current", srData.NTPTimestamp.Time(), - "last", r.srDataExt.SenderReportData.NTPTimestamp.Time(), + "last", r.srData.NTPTimestamp.Time(), ) return } - // Low pass filter one-way-delay (owd) to normalize time stamp to local time base when sending RTCP Sender Report. - // Forwarding RTCP Sender Report would be ideal. But, there are a couple of issues with that - // 1. Senders could have different clocks. - // 2. Adjusting to current time as required by RTCP spec. - // By normalizing to local clock, these issues can be addressed. However, normalization is not straightforward - // as it is not possible to know the propagation delay and processing delay at both ends (send side processing - // after time stamping the RTCP packet and receive side processing after reading packet off the wire). - // Smoothed version of OWD is used to alleviate irregularities somewhat. - owd := srData.ArrivalTime.Sub(srData.NTPTimestamp.Time()) - if r.srDataExt != nil { - prevOwd := r.srDataExt.SenderReportData.ArrivalTime.Sub(r.srDataExt.SenderReportData.NTPTimestamp.Time()) - if time.Duration(math.Abs(float64(owd)-float64(prevOwd))) > TooLargeOWDDelta { - r.logger.Debugw("large delta in one-way-delay", "owd", owd, "prevOwd", prevOwd) - } - } - - smoothedOwd := owd - if r.srDataExt != nil { - smoothedOwd = r.srDataExt.SmoothedOWD - } - smoothedOwd = (owd + smoothedOwd) / 2 - // TODO-REMOVE-AFTER-DEBUG + // TODO-REMOVE-AFTER-DEBUG-START if r.params.ClockRate != 90000 { // log only for audio as it is less frequent ntpTime := srData.NTPTimestamp.Time() var ntpDiffSinceLast, arrivalDiffSinceLast time.Duration var rtpDiffSinceLast uint32 - if r.srDataExt != nil { - ntpDiffSinceLast = ntpTime.Sub(r.srDataExt.SenderReportData.NTPTimestamp.Time()) - rtpDiffSinceLast = srData.RTPTimestamp - r.srDataExt.SenderReportData.RTPTimestamp - arrivalDiffSinceLast = srData.ArrivalTime.Sub(r.srDataExt.SenderReportData.ArrivalTime) + if r.srData != nil { + ntpDiffSinceLast = ntpTime.Sub(r.srData.NTPTimestamp.Time()) + rtpDiffSinceLast = srData.RTPTimestamp - r.srData.RTPTimestamp + arrivalDiffSinceLast = srData.ArrivalTime.Sub(r.srData.ArrivalTime) } + + timeSinceFirst := srData.NTPTimestamp.Time().Sub(r.firstTime) + rtpDiffSinceFirst := getExtTS(srData.RTPTimestamp, r.tsCycles) - r.extStartTS + drift := int64(uint64(timeSinceFirst.Nanoseconds()*int64(r.params.ClockRate)/1e9) - rtpDiffSinceFirst) + driftMs := (float64(drift) * 1000) / float64(r.params.ClockRate) + r.logger.Debugw( "received sender report", "ntp", ntpTime, @@ -787,28 +756,28 @@ func (r *RTPStats) SetRtcpSenderReportData(srData *RTCPSenderReportData) { "rtpDiff", rtpDiffSinceLast, "arrivalDiff", arrivalDiffSinceLast, "expectedTimeDiff", float64(rtpDiffSinceLast)/float64(r.params.ClockRate), - "owd", owd, - "smoothedOwd", smoothedOwd, + "timeSinceFirst", timeSinceFirst, + "rtpDiffSinceFirst", rtpDiffSinceFirst, + "drift", drift, + "driftMs", driftMs, ) } - r.srDataExt = &RTCPSenderReportDataExt{ - SenderReportData: *srData, - SmoothedOWD: smoothedOwd, - } + // TODO-REMOVE-AFTER-DEBUG-END + + srDataCopy := *srData + r.srData = &srDataCopy } -func (r *RTPStats) GetRtcpSenderReportDataExt() *RTCPSenderReportDataExt { +func (r *RTPStats) GetRtcpSenderReportData() *RTCPSenderReportData { r.lock.RLock() defer r.lock.RUnlock() - if r.srDataExt == nil { + if r.srData == nil { return nil } - return &RTCPSenderReportDataExt{ - SenderReportData: r.srDataExt.SenderReportData, - SmoothedOWD: r.srDataExt.SmoothedOWD, - } + srDataCopy := *r.srData + return &srDataCopy } func (r *RTPStats) GetExpectedRTPTimestamp(at time.Time) (uint32, error) { @@ -820,15 +789,15 @@ func (r *RTPStats) GetExpectedRTPTimestamp(at time.Time) (uint32, error) { } timeDiff := at.Sub(r.firstTime) - rtpDiff := timeDiff.Nanoseconds() * int64(r.params.ClockRate) / 1e9 - expectedExtRTP := r.extStartTS + uint64(rtpDiff) + expectedRTPDiff := timeDiff.Nanoseconds() * int64(r.params.ClockRate) / 1e9 + expectedExtRTP := r.extStartTS + uint64(expectedRTPDiff) r.logger.Debugw( "expected RTP timestamp", "firstTime", r.firstTime.String(), "checkAt", at.String(), "timeDiff", timeDiff, "firstRTP", r.extStartTS, - "rtpDiff", rtpDiff, + "expectedRTPDiff", expectedRTPDiff, "expectedExtRTP", expectedExtRTP, "expectedRTP", uint32(expectedExtRTP), "highestTS", r.highestTS, @@ -837,7 +806,7 @@ func (r *RTPStats) GetExpectedRTPTimestamp(at time.Time) (uint32, error) { return uint32(expectedExtRTP), nil } -func (r *RTPStats) GetRtcpSenderReport(ssrc uint32, srDataExt *RTCPSenderReportDataExt) *rtcp.SenderReport { +func (r *RTPStats) GetRtcpSenderReport(ssrc uint32) *rtcp.SenderReport { r.lock.Lock() defer r.lock.Unlock() @@ -845,11 +814,6 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32, srDataExt *RTCPSenderReportD return nil } - if srDataExt == nil || srDataExt.SenderReportData.NTPTimestamp == 0 || srDataExt.SenderReportData.ArrivalTime.IsZero() { - // no sender report from publisher - return nil - } - // construct current time based on monotonic clock timeSinceFirst := time.Since(r.firstTime) now := r.firstTime.Add(timeSinceFirst) @@ -861,7 +825,8 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32, srDataExt *RTCPSenderReportD "anachronous sender report", "firstTime", r.firstTime.String(), "currentTime", now.String(), - "timSinceFirst", timeSinceFirst, + "highestTime", r.highestTime.String(), + "timeSinceFirst", timeSinceFirst, "extStartTS", r.extStartTS, "highestExtRTP", getExtTS(r.highestTS, r.tsCycles), "expectedExtRTP", expectedExtRTP, @@ -871,54 +836,33 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32, srDataExt *RTCPSenderReportD timeSinceHighest := time.Since(r.highestTime) nowRTP := r.highestTS + uint32(timeSinceHighest.Nanoseconds()*int64(r.params.ClockRate)/1e9) - // TODO-REMOVE-AFTER-DEBUG - if r.firstSenderReportNTP == 0 { - r.firstSenderReportNTP = nowNTP - r.firstSenderReportRTP = nowRTP + // TODO-REMOVE-AFTER-DEBUG-START + ntpTime := nowNTP.Time() - r.firstFeedSenderReportNTP = srDataExt.SenderReportData.NTPTimestamp - r.firstFeedSenderReportRTP = srDataExt.SenderReportData.RTPTimestamp - } else { - ntpTime := nowNTP.Time() + ntpDiffLocal := ntpTime.Sub(r.highestTime) + rtpDiffLocal := int32(nowRTP - r.highestTS) + rtpOffsetLocal := int32(nowRTP - r.highestTS - uint32(ntpDiffLocal.Nanoseconds()*int64(r.params.ClockRate)/1e9)) - ntpDiffLocal := ntpTime.Sub(r.highestTime) - rtpDiffLocal := int32(nowRTP - r.highestTS) - rtpOffsetLocal := int32(nowRTP - r.highestTS - uint32(ntpDiffLocal.Nanoseconds()*int64(r.params.ClockRate)/1e9)) + timeSinceFirst = nowNTP.Time().Sub(r.firstTime) + rtpDiffSinceFirst := getExtTS(nowRTP, r.tsCycles) - r.extStartTS + drift := int64(uint64(timeSinceFirst.Nanoseconds()*int64(r.params.ClockRate)/1e9) - rtpDiffSinceFirst) + driftMs := (float64(drift) * 1000) / float64(r.params.ClockRate) - timeSinceFirst := nowNTP.Time().Sub(r.firstSenderReportNTP.Time()) - rtpDiffSinceFirst := getExtTS(nowRTP, r.tsCycles) - getExtTS(r.firstSenderReportRTP, 0) - drift := int64(uint64(timeSinceFirst.Nanoseconds()*int64(r.params.ClockRate)/1e9) - rtpDiffSinceFirst) - driftMs := (float64(drift) * 1000) / float64(r.params.ClockRate) - - feedTimeSinceFirst := srDataExt.SenderReportData.NTPTimestamp.Time().Sub(r.firstFeedSenderReportNTP.Time()) - // using tsCycles for extending feed time stamp too - feedRtpDiffSinceFirst := getExtTS(srDataExt.SenderReportData.RTPTimestamp, r.tsCycles) - getExtTS(r.firstFeedSenderReportRTP, 0) - feedDrift := int64(uint64(feedTimeSinceFirst.Nanoseconds()*int64(r.params.ClockRate)/1e9) - feedRtpDiffSinceFirst) - feedDriftMs := (float64(feedDrift) * 1000) / float64(r.params.ClockRate) - - r.logger.Debugw( - "sending sender report", - "highestTS", r.highestTS, - "highestTime", r.highestTime.String(), - "reportTS", nowRTP, - "reportTime", ntpTime.String(), - "rtpDiffLocal", rtpDiffLocal, - "ntpDiffLocal", ntpDiffLocal, - "rtpOffsetLocal", rtpOffsetLocal, - "timeSinceFirst", timeSinceFirst, - "rtpDiffSinceFirst", rtpDiffSinceFirst, - "drift", drift, - "driftMs", driftMs, - "feedRTP", srDataExt.SenderReportData.RTPTimestamp, - "feedNTP", srDataExt.SenderReportData.NTPTimestamp.Time().String(), - "feedArrival", srDataExt.SenderReportData.ArrivalTime.String(), - "smoothedOWD", srDataExt.SmoothedOWD, - "feedTimeSinceFirst", feedTimeSinceFirst, - "feedRtpDiffSinceFirst", feedRtpDiffSinceFirst, - "feedDrift", feedDrift, - "feedDriftMs", feedDriftMs, - ) - } + r.logger.Debugw( + "sending sender report", + "highestTS", r.highestTS, + "highestTime", r.highestTime.String(), + "reportTS", nowRTP, + "reportTime", ntpTime.String(), + "rtpDiffLocal", rtpDiffLocal, + "ntpDiffLocal", ntpDiffLocal, + "rtpOffsetLocal", rtpOffsetLocal, + "timeSinceFirst", timeSinceFirst, + "rtpDiffSinceFirst", rtpDiffSinceFirst, + "drift", drift, + "driftMs", driftMs, + ) + // TODO-REMOVE-AFTER-DEBUG-END r.lastSRTime = now r.lastSRNTP = nowNTP @@ -965,15 +909,15 @@ func (r *RTPStats) SnapshotRtcpReceptionReport(ssrc uint32, proxyFracLost uint8, } var dlsr uint32 - if r.srDataExt != nil && !r.srDataExt.SenderReportData.ArrivalTime.IsZero() { - delayMS := uint32(time.Since(r.srDataExt.SenderReportData.ArrivalTime).Milliseconds()) + if r.srData != nil && !r.srData.ArrivalTime.IsZero() { + delayMS := uint32(time.Since(r.srData.ArrivalTime).Milliseconds()) dlsr = (delayMS / 1e3) << 16 dlsr |= (delayMS % 1e3) * 65536 / 1000 } lastSR := uint32(0) - if r.srDataExt != nil { - lastSR = uint32(r.srDataExt.SenderReportData.NTPTimestamp >> 16) + if r.srData != nil { + lastSR = uint32(r.srData.NTPTimestamp >> 16) } return &rtcp.ReceptionReport{ SSRC: ssrc, diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 0e6638b0e..b69f4dccb 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -1113,7 +1113,7 @@ func (d *DownTrack) CreateSenderReport() *rtcp.SenderReport { return nil } - return d.rtpStats.GetRtcpSenderReport(d.ssrc, d.receiver.GetRTCPSenderReportDataExt(d.forwarder.GetReferenceLayerSpatial())) + return d.rtpStats.GetRtcpSenderReport(d.ssrc) } func (d *DownTrack) writeBlankFrameRTP(duration float32, generation uint32) chan struct{} { diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index e6da06859..e8e8167b0 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -469,13 +469,6 @@ func (f *Forwarder) TargetLayer() buffer.VideoLayer { return f.vls.GetTarget() } -func (f *Forwarder) GetReferenceLayerSpatial() int32 { - f.lock.RLock() - defer f.lock.RUnlock() - - return f.referenceLayerSpatial -} - func (f *Forwarder) isDeficientLocked() bool { return f.lastAllocation.IsDeficient } diff --git a/pkg/sfu/receiver.go b/pkg/sfu/receiver.go index 2e38540d3..5fd083bed 100644 --- a/pkg/sfu/receiver.go +++ b/pkg/sfu/receiver.go @@ -65,7 +65,6 @@ type TrackReceiver interface { GetTemporalLayerFpsForSpatial(layer int32) []float32 - GetRTCPSenderReportDataExt(layer int32) *buffer.RTCPSenderReportDataExt GetReferenceLayerRTPTimestamp(ts uint32, layer int32, referenceLayer int32) (uint32, error) } @@ -311,7 +310,7 @@ func (w *WebRTCReceiver) AddUpTrack(track *webrtc.TrackRemote, buff *buffer.Buff }) buff.OnRtcpFeedback(w.sendRTCP) buff.OnRtcpSenderReport(func(srData *buffer.RTCPSenderReportData) { - w.streamTrackerManager.SetRTCPSenderReportDataExt(layer, buff.GetSenderReportDataExt()) + w.streamTrackerManager.SetRTCPSenderReportData(layer, buff.GetSenderReportData()) w.downTrackSpreader.Broadcast(func(dt TrackSender) { _ = dt.HandleRTCPSenderReportData(w.codec.PayloadType, layer, srData) @@ -746,10 +745,6 @@ func (w *WebRTCReceiver) GetTemporalLayerFpsForSpatial(layer int32) []float32 { return b.GetTemporalLayerFpsForSpatial(layer) } -func (w *WebRTCReceiver) GetRTCPSenderReportDataExt(layer int32) *buffer.RTCPSenderReportDataExt { - return w.streamTrackerManager.GetRTCPSenderReportDataExt(layer) -} - func (w *WebRTCReceiver) GetReferenceLayerRTPTimestamp(ts uint32, layer int32, referenceLayer int32) (uint32, error) { return w.streamTrackerManager.GetReferenceLayerRTPTimestamp(ts, layer, referenceLayer) } diff --git a/pkg/sfu/streamtrackermanager.go b/pkg/sfu/streamtrackermanager.go index 8e324e6a3..256b5c4ce 100644 --- a/pkg/sfu/streamtrackermanager.go +++ b/pkg/sfu/streamtrackermanager.go @@ -43,7 +43,7 @@ type StreamTrackerManager struct { paused bool senderReportMu sync.RWMutex - senderReports [buffer.DefaultMaxLayerSpatial + 1]*buffer.RTCPSenderReportDataExt + senderReports [buffer.DefaultMaxLayerSpatial + 1]*buffer.RTCPSenderReportData closed core.Fuse @@ -475,7 +475,7 @@ func (s *StreamTrackerManager) maxExpectedLayerFromTrackInfo() { } } -func (s *StreamTrackerManager) SetRTCPSenderReportDataExt(layer int32, senderReport *buffer.RTCPSenderReportDataExt) { +func (s *StreamTrackerManager) SetRTCPSenderReportData(layer int32, senderReport *buffer.RTCPSenderReportData) { s.senderReportMu.Lock() defer s.senderReportMu.Unlock() @@ -486,17 +486,6 @@ func (s *StreamTrackerManager) SetRTCPSenderReportDataExt(layer int32, senderRep s.senderReports[layer] = senderReport } -func (s *StreamTrackerManager) GetRTCPSenderReportDataExt(layer int32) *buffer.RTCPSenderReportDataExt { - s.senderReportMu.RLock() - defer s.senderReportMu.RUnlock() - - if layer < 0 || int(layer) >= len(s.senderReports) { - return nil - } - - return s.senderReports[layer] -} - func (s *StreamTrackerManager) GetReferenceLayerRTPTimestamp(ts uint32, layer int32, referenceLayer int32) (uint32, error) { s.senderReportMu.RLock() defer s.senderReportMu.RUnlock() @@ -509,19 +498,19 @@ func (s *StreamTrackerManager) GetReferenceLayerRTPTimestamp(ts uint32, layer in return ts, nil } - var srLayer *buffer.RTCPSenderReportDataExt + var srLayer *buffer.RTCPSenderReportData if int(layer) < len(s.senderReports) { srLayer = s.senderReports[layer] } - if srLayer == nil || srLayer.SenderReportData.NTPTimestamp == 0 { + if srLayer == nil || srLayer.NTPTimestamp == 0 { return 0, fmt.Errorf("layer rtcp sender report not available: %d", layer) } - var srRef *buffer.RTCPSenderReportDataExt + var srRef *buffer.RTCPSenderReportData if int(referenceLayer) < len(s.senderReports) { srRef = s.senderReports[referenceLayer] } - if srRef == nil || srRef.SenderReportData.NTPTimestamp == 0 { + if srRef == nil || srRef.NTPTimestamp == 0 { return 0, fmt.Errorf("reference layer rtcp sender report not available: %d", referenceLayer) } @@ -529,15 +518,15 @@ func (s *StreamTrackerManager) GetReferenceLayerRTPTimestamp(ts uint32, layer in // NOTE: It is possible that reference layer has stopped (due to dynacast/adaptive streaming OR publisher // constraints). It should be okay even if the layer has stopped for a long time when using modulo arithmetic for // RTP time stamp (uint32 arithmetic). - ntpDiff := srRef.SenderReportData.NTPTimestamp.Time().Sub(srLayer.SenderReportData.NTPTimestamp.Time()) + ntpDiff := srRef.NTPTimestamp.Time().Sub(srLayer.NTPTimestamp.Time()) rtpDiff := ntpDiff.Nanoseconds() * int64(s.clockRate) / 1e9 - normalizedTS := srLayer.SenderReportData.RTPTimestamp + uint32(rtpDiff) + normalizedTS := srLayer.RTPTimestamp + uint32(rtpDiff) // now that both RTP timestamps correspond to roughly the same NTP time, // the diff between them is the offset in RTP timestamp units between layer and referenceLayer. // Add the offset to layer's ts to map it to corresponding RTP timestamp in // the reference layer. - return ts + (srRef.SenderReportData.RTPTimestamp - normalizedTS), nil + return ts + (srRef.RTPTimestamp - normalizedTS), nil } func (s *StreamTrackerManager) GetMaxTemporalLayerSeen() int32 { From 0586009e0df08110937777ae240f97bce7fd3679 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Fri, 5 May 2023 22:38:17 -0700 Subject: [PATCH 142/324] Do not send hidden participants after resume (#1689) --- README.md | 5 +++-- pkg/rtc/room.go | 15 ++++++++++++++- pkg/rtc/utils.go | 10 ---------- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 16fd9dd8b..d6690a9e0 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,8 @@ # LiveKit: Real-time video, audio and data for developers -[LiveKit](https://livekit.io) is an open source project that provides scalable, multi-user conferencing based on WebRTC. It's designed to provide everything you need to build real-time video audio data capabilities in your applications. +[LiveKit](https://livekit.io) is an open source project that provides scalable, multi-user conferencing based on WebRTC. +It's designed to provide everything you need to build real-time video audio data capabilities in your applications. LiveKit's server is written in Go, using the awesome [Pion WebRTC](https://github.com/pion/webrtc) implementation. @@ -41,7 +42,7 @@ https://docs.livekit.io ## Live Demos -- [LiveKit Meet](https://meet.livekit.io) ([source](https://github.com/livekit/meet)) +- [LiveKit Meet](https://meet.livekit.io) ([source](https://github.com/livekit-examples/meet)) - [Spatial Audio](https://spatial-audio-demo.livekit.io/) ([source](https://github.com/livekit-examples/spatial-audio)) - Livestreaming from OBS Studio ([source](https://github.com/livekit-examples/livestream)) diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index 6d339d83f..0441a56b4 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -412,7 +412,8 @@ func (r *Room) ResumeParticipant(p types.LocalParticipant, requestSource routing return err } - updates := ToProtoParticipants(r.GetParticipants()) + // include the local participant's info as well, since metadata could have been changed + updates := r.getOtherParticipantInfo("") if err := p.SendParticipantUpdate(updates); err != nil { return err } @@ -730,6 +731,18 @@ func (r *Room) SimulateScenario(participant types.LocalParticipant, simulateScen return nil } +func (r *Room) getOtherParticipantInfo(identity livekit.ParticipantIdentity) []*livekit.ParticipantInfo { + participants := r.GetParticipants() + pi := make([]*livekit.ParticipantInfo, 0, len(participants)) + for _, p := range participants { + if !p.Hidden() && p.Identity() != identity { + pi = append(pi, p.ToProto()) + } + } + + return pi +} + // checks if participant should be autosubscribed to new tracks, assumes lock is already acquired func (r *Room) autoSubscribe(participant types.LocalParticipant) bool { opts := r.participantOpts[participant.Identity()] diff --git a/pkg/rtc/utils.go b/pkg/rtc/utils.go index 95db5606d..0b49d02af 100644 --- a/pkg/rtc/utils.go +++ b/pkg/rtc/utils.go @@ -10,8 +10,6 @@ import ( "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" - - "github.com/livekit/livekit-server/pkg/rtc/types" ) const ( @@ -45,14 +43,6 @@ func UnpackDataTrackLabel(packed string) (participantID livekit.ParticipantID, t return } -func ToProtoParticipants(participants []types.LocalParticipant) []*livekit.ParticipantInfo { - infos := make([]*livekit.ParticipantInfo, 0, len(participants)) - for _, op := range participants { - infos = append(infos, op.ToProto()) - } - return infos -} - func ToProtoSessionDescription(sd webrtc.SessionDescription) *livekit.SessionDescription { return &livekit.SessionDescription{ Type: sd.Type.String(), From 0250d9855ad9d0e0434a2b53c6c0be04679494af Mon Sep 17 00:00:00 2001 From: Danil Andreev Date: Sat, 6 May 2023 08:40:12 +0300 Subject: [PATCH 143/324] Correctly ignore .idea directory (#1686) --- .gitignore | 2 +- .idea/.gitignore | 8 -------- .idea/inspectionProfiles/Project_Default.xml | 7 ------- .idea/livekit-server.iml | 19 ------------------- .idea/modules.xml | 8 -------- .idea/protoeditor.xml | 20 -------------------- .idea/vcs.xml | 6 ------ 7 files changed, 1 insertion(+), 69 deletions(-) delete mode 100644 .idea/.gitignore delete mode 100644 .idea/inspectionProfiles/Project_Default.xml delete mode 100644 .idea/livekit-server.iml delete mode 100644 .idea/modules.xml delete mode 100644 .idea/protoeditor.xml delete mode 100644 .idea/vcs.xml diff --git a/.gitignore b/.gitignore index b44d8bdac..dec7e8aad 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,6 @@ proto/ .DS_Store # IDE -.idea +.idea/ dist/ diff --git a/.idea/.gitignore b/.idea/.gitignore deleted file mode 100644 index 73f69e095..000000000 --- a/.idea/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -# Default ignored files -/shelf/ -/workspace.xml -# Datasource local storage ignored files -/dataSources/ -/dataSources.local.xml -# Editor-based HTTP Client requests -/httpRequests/ diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml deleted file mode 100644 index 239034913..000000000 --- a/.idea/inspectionProfiles/Project_Default.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/livekit-server.iml b/.idea/livekit-server.iml deleted file mode 100644 index adec66b98..000000000 --- a/.idea/livekit-server.iml +++ /dev/null @@ -1,19 +0,0 @@ - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index 9c32c0523..000000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/protoeditor.xml b/.idea/protoeditor.xml deleted file mode 100644 index 6f5d7aadf..000000000 --- a/.idea/protoeditor.xml +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 94a25f7f4..000000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file From ae5d2c3261aa20035b543dc79ad10a47bda698fb Mon Sep 17 00:00:00 2001 From: David Zhao Date: Fri, 5 May 2023 23:11:22 -0700 Subject: [PATCH 144/324] Integrate fix for webhook delivery delays (#1690) --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 49e083c66..4ccb896b3 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 - github.com/livekit/protocol v1.5.6-0.20230501213748-b2974692ebfd + github.com/livekit/protocol v1.5.6 github.com/livekit/psrpc v0.3.1-0.20230502152150-df9dd21fba11 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.14.0 @@ -99,6 +99,6 @@ require ( golang.org/x/text v0.9.0 // indirect golang.org/x/tools v0.6.0 // indirect google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect - google.golang.org/grpc v1.54.0 // indirect + google.golang.org/grpc v1.55.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 8ff9983c4..51023823d 100644 --- a/go.sum +++ b/go.sum @@ -121,8 +121,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 h1:QlQFyMwCDgjyySsrgmrMcVbEBA6KZcyTzvK+z346tUA= github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26/go.mod h1:eDA41kiySZoG+wy4Etsjb3w0jjLx69i/vAmSjG4bteA= -github.com/livekit/protocol v1.5.6-0.20230501213748-b2974692ebfd h1:5gCLKGemm4JCEZvKIPpKVPOpguFt3/S2ZzYQd+4tInA= -github.com/livekit/protocol v1.5.6-0.20230501213748-b2974692ebfd/go.mod h1:MBW05GWdhbl+o6u2gLLCQtDvr9EvcV4VWckpIYtoM2c= +github.com/livekit/protocol v1.5.6 h1:2kwduElaTcYc4JKxs3aGp+jbMDC9g1z8L+ywlLBwMvo= +github.com/livekit/protocol v1.5.6/go.mod h1:CtvrXHdVzapR+avHHBr3RnCwwM4wjCJv9LQ4UCA14TU= github.com/livekit/psrpc v0.3.1-0.20230502152150-df9dd21fba11 h1:VS23iVQu/TNiLEM5XjbBSY28+B6nSewjKWPDbieg0Ho= github.com/livekit/psrpc v0.3.1-0.20230502152150-df9dd21fba11/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= @@ -394,8 +394,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= -google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= -google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= +google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= +google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= From 3fb93135f5f2e41c8ff6924b5c4b2cf2d7f83899 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sat, 6 May 2023 11:52:57 +0530 Subject: [PATCH 145/324] Experimental flag to try time stamp adjustment to control drift. (#1687) * Experimental flag to try time stamp adjustment to control drift. There is a config to enable this. Using a PID controller to try and keep the sample rate at expected value. Need to be seen if this works well. Adjustment are limited to 25 ms max at a time to ensure there are no large jumps. And it is applied when doing RTCP sender report which happens once in 5 seconds currently for both audio and video tracks. A nice introduction to PID controllers - https://alphaville.github.io/qub/pid-101/#/ Implementation borrowed from - https://github.com/pms67/PID A few things TODO 1. PID controller tuning is a process. Have picked values from test from that implementation above. May not be the best. Need to try. 2. Can potentially run this more often. Rather than running it only when running RTCP sender report (which is once in 5 seconds now), can potentially run it every second and limit the amount of change to something like 10 ms max. * remove unused variable * debug log a bit more --- pkg/config/config.go | 3 + pkg/rtc/mediatracksubscriptions.go | 1 + pkg/rtc/participant.go | 5 + pkg/rtc/types/interfaces.go | 2 + .../typesfakes/fake_local_participant.go | 65 ++++++ pkg/service/roommanager.go | 6 + pkg/sfu/buffer/rtpstats.go | 185 ++++++++++++++---- pkg/sfu/downtrack.go | 30 +-- pkg/sfu/forwarder.go | 7 + pkg/sfu/rtpmunger.go | 27 +++ pkg/sfu/streamtrackermanager.go | 16 ++ 11 files changed, 298 insertions(+), 49 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 315fb3aae..cdab66a70 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -107,6 +107,9 @@ type RTCConfig struct { // force a reconnect on a subscription error ReconnectOnSubscriptionError *bool `yaml:"reconnect_on_subscription_error,omitempty"` + + // allow time stamp adjust to keep drift low, this is experimental + AllowTimestampAdjustment *bool `yaml:"allow_timestamp_adjustment,omitempty"` } type TURNServer struct { diff --git a/pkg/rtc/mediatracksubscriptions.go b/pkg/rtc/mediatracksubscriptions.go index af82fe224..bbf439e43 100644 --- a/pkg/rtc/mediatracksubscriptions.go +++ b/pkg/rtc/mediatracksubscriptions.go @@ -104,6 +104,7 @@ func (t *MediaTrackSubscriptions) AddSubscriber(sub types.LocalParticipant, wr * sub.GetBufferFactory(), subscriberID, t.params.ReceiverConfig.PacketBufferSize, + sub.GetAllowTimestampAdjustment(), LoggerWithTrack(sub.GetLogger(), trackID, t.params.IsRelayed), ) if err != nil { diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 793defe93..91008754c 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -92,6 +92,7 @@ type ParticipantParams struct { SubscriberAllowPause bool SubscriptionLimitAudio int32 SubscriptionLimitVideo int32 + AllowTimestampAdjustment bool } type ParticipantImpl struct { @@ -228,6 +229,10 @@ func (p *ParticipantImpl) GetAdaptiveStream() bool { return p.params.AdaptiveStream } +func (p *ParticipantImpl) GetAllowTimestampAdjustment() bool { + return p.params.AllowTimestampAdjustment +} + func (p *ParticipantImpl) ID() livekit.ParticipantID { return p.params.SID } diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index 84989179a..1a0a9532c 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -341,6 +341,8 @@ type LocalParticipant interface { // down stream bandwidth management SetSubscriberAllowPause(allowPause bool) SetSubscriberChannelCapacity(channelCapacity int64) + + GetAllowTimestampAdjustment() bool } // Room is a container of participants, and can provide room-level actions diff --git a/pkg/rtc/types/typesfakes/fake_local_participant.go b/pkg/rtc/types/typesfakes/fake_local_participant.go index e4ddfea5f..3fbdcbc7d 100644 --- a/pkg/rtc/types/typesfakes/fake_local_participant.go +++ b/pkg/rtc/types/typesfakes/fake_local_participant.go @@ -165,6 +165,16 @@ type FakeLocalParticipant struct { getAdaptiveStreamReturnsOnCall map[int]struct { result1 bool } + GetAllowTimestampAdjustmentStub func() bool + getAllowTimestampAdjustmentMutex sync.RWMutex + getAllowTimestampAdjustmentArgsForCall []struct { + } + getAllowTimestampAdjustmentReturns struct { + result1 bool + } + getAllowTimestampAdjustmentReturnsOnCall map[int]struct { + result1 bool + } GetAudioLevelStub func() (float64, bool) getAudioLevelMutex sync.RWMutex getAudioLevelArgsForCall []struct { @@ -1612,6 +1622,59 @@ func (fake *FakeLocalParticipant) GetAdaptiveStreamReturnsOnCall(i int, result1 }{result1} } +func (fake *FakeLocalParticipant) GetAllowTimestampAdjustment() bool { + fake.getAllowTimestampAdjustmentMutex.Lock() + ret, specificReturn := fake.getAllowTimestampAdjustmentReturnsOnCall[len(fake.getAllowTimestampAdjustmentArgsForCall)] + fake.getAllowTimestampAdjustmentArgsForCall = append(fake.getAllowTimestampAdjustmentArgsForCall, struct { + }{}) + stub := fake.GetAllowTimestampAdjustmentStub + fakeReturns := fake.getAllowTimestampAdjustmentReturns + fake.recordInvocation("GetAllowTimestampAdjustment", []interface{}{}) + fake.getAllowTimestampAdjustmentMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) GetAllowTimestampAdjustmentCallCount() int { + fake.getAllowTimestampAdjustmentMutex.RLock() + defer fake.getAllowTimestampAdjustmentMutex.RUnlock() + return len(fake.getAllowTimestampAdjustmentArgsForCall) +} + +func (fake *FakeLocalParticipant) GetAllowTimestampAdjustmentCalls(stub func() bool) { + fake.getAllowTimestampAdjustmentMutex.Lock() + defer fake.getAllowTimestampAdjustmentMutex.Unlock() + fake.GetAllowTimestampAdjustmentStub = stub +} + +func (fake *FakeLocalParticipant) GetAllowTimestampAdjustmentReturns(result1 bool) { + fake.getAllowTimestampAdjustmentMutex.Lock() + defer fake.getAllowTimestampAdjustmentMutex.Unlock() + fake.GetAllowTimestampAdjustmentStub = nil + fake.getAllowTimestampAdjustmentReturns = struct { + result1 bool + }{result1} +} + +func (fake *FakeLocalParticipant) GetAllowTimestampAdjustmentReturnsOnCall(i int, result1 bool) { + fake.getAllowTimestampAdjustmentMutex.Lock() + defer fake.getAllowTimestampAdjustmentMutex.Unlock() + fake.GetAllowTimestampAdjustmentStub = nil + if fake.getAllowTimestampAdjustmentReturnsOnCall == nil { + fake.getAllowTimestampAdjustmentReturnsOnCall = make(map[int]struct { + result1 bool + }) + } + fake.getAllowTimestampAdjustmentReturnsOnCall[i] = struct { + result1 bool + }{result1} +} + func (fake *FakeLocalParticipant) GetAudioLevel() (float64, bool) { fake.getAudioLevelMutex.Lock() ret, specificReturn := fake.getAudioLevelReturnsOnCall[len(fake.getAudioLevelArgsForCall)] @@ -5456,6 +5519,8 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.debugInfoMutex.RUnlock() fake.getAdaptiveStreamMutex.RLock() defer fake.getAdaptiveStreamMutex.RUnlock() + fake.getAllowTimestampAdjustmentMutex.RLock() + defer fake.getAllowTimestampAdjustmentMutex.RUnlock() fake.getAudioLevelMutex.RLock() defer fake.getAudioLevelMutex.RUnlock() fake.getBufferFactoryMutex.RLock() diff --git a/pkg/service/roommanager.go b/pkg/service/roommanager.go index 5b24c44bc..c9986e92a 100644 --- a/pkg/service/roommanager.go +++ b/pkg/service/roommanager.go @@ -309,6 +309,11 @@ func (r *RoomManager) StartSession( if pi.SubscriberAllowPause != nil { subscriberAllowPause = *pi.SubscriberAllowPause } + // default do not allow timestamp adjustment + allowTimestampAdjustment := false + if r.config.RTC.AllowTimestampAdjustment != nil { + allowTimestampAdjustment = *r.config.RTC.AllowTimestampAdjustment + } participant, err = rtc.NewParticipant(rtc.ParticipantParams{ Identity: pi.Identity, Name: pi.Name, @@ -343,6 +348,7 @@ func (r *RoomManager) StartSession( SubscriberAllowPause: subscriberAllowPause, SubscriptionLimitAudio: r.config.Limit.SubscriptionLimitAudio, SubscriptionLimitVideo: r.config.Limit.SubscriptionLimitVideo, + AllowTimestampAdjustment: allowTimestampAdjustment, }) if err != nil { return err diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index 339959e6d..9b274f4e3 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -174,21 +174,28 @@ type RTPStats struct { rtt uint32 maxRtt uint32 - srData *RTCPSenderReportData - lastSRTime time.Time - lastSRNTP mediatransportutil.NtpTime + srData *RTCPSenderReportData + lastSRTime time.Time + lastSRNTP mediatransportutil.NtpTime + pidController *PIDController nextSnapshotId uint32 snapshots map[uint32]*Snapshot } func NewRTPStats(params RTPStatsParams) *RTPStats { - return &RTPStats{ + r := &RTPStats{ params: params, logger: params.Logger, nextSnapshotId: FirstSnapshotId, snapshots: make(map[uint32]*Snapshot), + pidController: NewPIDController(), } + + r.pidController.SetGains(2.0, 0.5, 0.25) + r.pidController.SetDerivativeLPF(0.02) + r.pidController.SetOutputLimits(-0.025*float64(params.ClockRate), 0.025*float64(params.ClockRate)) + return r } func (r *RTPStats) Seed(from *RTPStats) { @@ -731,37 +738,35 @@ func (r *RTPStats) SetRtcpSenderReportData(srData *RTCPSenderReportData) { } // TODO-REMOVE-AFTER-DEBUG-START - if r.params.ClockRate != 90000 { // log only for audio as it is less frequent - ntpTime := srData.NTPTimestamp.Time() + ntpTime := srData.NTPTimestamp.Time() - var ntpDiffSinceLast, arrivalDiffSinceLast time.Duration - var rtpDiffSinceLast uint32 - if r.srData != nil { - ntpDiffSinceLast = ntpTime.Sub(r.srData.NTPTimestamp.Time()) - rtpDiffSinceLast = srData.RTPTimestamp - r.srData.RTPTimestamp - arrivalDiffSinceLast = srData.ArrivalTime.Sub(r.srData.ArrivalTime) - } - - timeSinceFirst := srData.NTPTimestamp.Time().Sub(r.firstTime) - rtpDiffSinceFirst := getExtTS(srData.RTPTimestamp, r.tsCycles) - r.extStartTS - drift := int64(uint64(timeSinceFirst.Nanoseconds()*int64(r.params.ClockRate)/1e9) - rtpDiffSinceFirst) - driftMs := (float64(drift) * 1000) / float64(r.params.ClockRate) - - r.logger.Debugw( - "received sender report", - "ntp", ntpTime, - "rtp", srData.RTPTimestamp, - "arrival", srData.ArrivalTime, - "ntpDiff", ntpDiffSinceLast, - "rtpDiff", rtpDiffSinceLast, - "arrivalDiff", arrivalDiffSinceLast, - "expectedTimeDiff", float64(rtpDiffSinceLast)/float64(r.params.ClockRate), - "timeSinceFirst", timeSinceFirst, - "rtpDiffSinceFirst", rtpDiffSinceFirst, - "drift", drift, - "driftMs", driftMs, - ) + var ntpDiffSinceLast, arrivalDiffSinceLast time.Duration + var rtpDiffSinceLast uint32 + if r.srData != nil { + ntpDiffSinceLast = ntpTime.Sub(r.srData.NTPTimestamp.Time()) + rtpDiffSinceLast = srData.RTPTimestamp - r.srData.RTPTimestamp + arrivalDiffSinceLast = srData.ArrivalTime.Sub(r.srData.ArrivalTime) } + + timeSinceFirst := time.Since(r.firstTime) // ideally should use NTP time from SR, but that is a different time base, now is a resonable approximation + rtpDiffSinceFirst := getExtTS(srData.RTPTimestamp, r.tsCycles) - r.extStartTS + drift := int64(uint64(timeSinceFirst.Nanoseconds()*int64(r.params.ClockRate)/1e9) - rtpDiffSinceFirst) + driftMs := (float64(drift) * 1000) / float64(r.params.ClockRate) + + r.logger.Debugw( + "received sender report", + "ntp", ntpTime, + "rtp", srData.RTPTimestamp, + "arrival", srData.ArrivalTime, + "ntpDiff", ntpDiffSinceLast, + "rtpDiff", rtpDiffSinceLast, + "arrivalDiff", arrivalDiffSinceLast, + "expectedTimeDiff", float64(rtpDiffSinceLast)/float64(r.params.ClockRate), + "timeSinceFirst", timeSinceFirst, + "rtpDiffSinceFirst", rtpDiffSinceFirst, + "drift", drift, + "driftMs", driftMs, + ) // TODO-REMOVE-AFTER-DEBUG-END srDataCopy := *srData @@ -806,12 +811,12 @@ func (r *RTPStats) GetExpectedRTPTimestamp(at time.Time) (uint32, error) { return uint32(expectedExtRTP), nil } -func (r *RTPStats) GetRtcpSenderReport(ssrc uint32) *rtcp.SenderReport { +func (r *RTPStats) GetRtcpSenderReport(ssrc uint32) (*rtcp.SenderReport, float64) { r.lock.Lock() defer r.lock.Unlock() if !r.initialized { - return nil + return nil, 0.0 } // construct current time based on monotonic clock @@ -836,6 +841,23 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32) *rtcp.SenderReport { timeSinceHighest := time.Since(r.highestTime) nowRTP := r.highestTS + uint32(timeSinceHighest.Nanoseconds()*int64(r.params.ClockRate)/1e9) + // TODO-REMOVE-AFTER-DEBUG-START + timeSinceFirst = nowNTP.Time().Sub(r.firstTime) + rtpDiffSinceFirst := getExtTS(nowRTP, r.tsCycles) - r.extStartTS + measurement := float64(rtpDiffSinceFirst) / timeSinceFirst.Seconds() + pidOutput := r.pidController.Update( + float64(r.params.ClockRate), + measurement, + now, + ) + r.logger.Debugw( + "pid controller output", + "measurement", measurement, + "errorTerm", float64(r.params.ClockRate)-measurement, + "pidOutput", pidOutput, + ) + // TODO-REMOVE-AFTER-DEBUG-STOP + // TODO-REMOVE-AFTER-DEBUG-START ntpTime := nowNTP.Time() @@ -843,8 +865,6 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32) *rtcp.SenderReport { rtpDiffLocal := int32(nowRTP - r.highestTS) rtpOffsetLocal := int32(nowRTP - r.highestTS - uint32(ntpDiffLocal.Nanoseconds()*int64(r.params.ClockRate)/1e9)) - timeSinceFirst = nowNTP.Time().Sub(r.firstTime) - rtpDiffSinceFirst := getExtTS(nowRTP, r.tsCycles) - r.extStartTS drift := int64(uint64(timeSinceFirst.Nanoseconds()*int64(r.params.ClockRate)/1e9) - rtpDiffSinceFirst) driftMs := (float64(drift) * 1000) / float64(r.params.ClockRate) @@ -853,6 +873,7 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32) *rtcp.SenderReport { "highestTS", r.highestTS, "highestTime", r.highestTime.String(), "reportTS", nowRTP, + "expectedTS", uint32(expectedExtRTP), "reportTime", ntpTime.String(), "rtpDiffLocal", rtpDiffLocal, "ntpDiffLocal", ntpDiffLocal, @@ -861,6 +882,7 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32) *rtcp.SenderReport { "rtpDiffSinceFirst", rtpDiffSinceFirst, "drift", drift, "driftMs", driftMs, + "rate", measurement, ) // TODO-REMOVE-AFTER-DEBUG-END @@ -873,7 +895,7 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32) *rtcp.SenderReport { RTPTime: nowRTP, PacketCount: r.getTotalPacketsPrimary() + r.packetsDuplicate + r.packetsPadding, OctetCount: uint32(r.bytes + r.bytesDuplicate + r.bytesPadding), - } + }, pidOutput } func (r *RTPStats) SnapshotRtcpReceptionReport(ssrc uint32, proxyFracLost uint8, snapshotId uint32) *rtcp.ReceptionReport { @@ -1747,3 +1769,90 @@ func AggregateRTPDeltaInfo(deltaInfoList []*RTPDeltaInfo) *RTPDeltaInfo { Firs: firs, } } + +// ------------------------------------------------------------------- + +type PIDController struct { + kp, ki, kd float64 + + tau float64 // low pass filter of D, time constant + + outMin, outMax float64 + isOutLimitsSet bool + + iMin, iMax float64 + isILimitsSet bool + + iVal, dVal float64 + + prevError, prevMeasurement float64 + prevMeasurementTime time.Time +} + +func NewPIDController() *PIDController { + return &PIDController{} +} + +func (p *PIDController) SetGains(kp, ki, kd float64) { + p.kp = kp + p.ki = ki + p.kd = kd +} + +func (p *PIDController) SetDerivativeLPF(tau float64) { + p.tau = tau +} + +func (p *PIDController) SetOutputLimits(min, max float64) { + p.outMin = min + p.outMax = max + p.isOutLimitsSet = true +} + +func (p *PIDController) SetIntegralLimits(min, max float64) { + p.iMin = min + p.iMax = max + p.isILimitsSet = true +} + +func (p *PIDController) Update(setpoint, measurement float64, at time.Time) float64 { + diff := setpoint - measurement + if p.prevMeasurementTime.IsZero() { + p.prevError = diff + p.prevMeasurement = measurement + p.prevMeasurementTime = at + return 0 + } + + proportional := p.kp * diff + + duration := at.Sub(p.prevMeasurementTime).Seconds() + p.iVal = p.iVal + (0.5 * p.ki * duration * (diff + p.prevError)) + if p.isILimitsSet { + if p.iVal > p.iMax { + p.iVal = p.iMax + } + if p.iVal < p.iMin { + p.iVal = p.iMin + } + } + + p.dVal = (-2.0 * p.kd * (measurement - p.prevMeasurement)) + (((2.0*p.tau - duration) * p.dVal) / (2.0*p.tau + duration)) + + output := proportional + p.iVal + p.dVal + if p.isOutLimitsSet { + if output > p.outMax { + output = p.outMax + } + if output < p.outMin { + output = p.outMin + } + } + + p.prevError = diff + p.prevMeasurement = measurement + p.prevMeasurementTime = at + return output +} + +// ------------------------------------------------------------------- diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index b69f4dccb..e9bca290f 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -185,6 +185,8 @@ type DownTrack struct { sequencer *sequencer bufferFactory *buffer.Factory + allowTimestampAdjustment bool + forwarder *Forwarder upstreamCodecs []webrtc.RTPCodecParameters @@ -252,6 +254,7 @@ func NewDownTrack( bf *buffer.Factory, subID livekit.ParticipantID, mt int, + allowTimestampAdjustment bool, logger logger.Logger, ) (*DownTrack, error) { var kind webrtc.RTPCodecType @@ -265,16 +268,17 @@ func NewDownTrack( } d := &DownTrack{ - logger: logger, - id: r.TrackID(), - subscriberID: subID, - maxTrack: mt, - streamID: r.StreamID(), - bufferFactory: bf, - receiver: r, - upstreamCodecs: codecs, - kind: kind, - codec: codecs[0].RTPCodecCapability, + logger: logger, + id: r.TrackID(), + subscriberID: subID, + maxTrack: mt, + streamID: r.StreamID(), + bufferFactory: bf, + allowTimestampAdjustment: allowTimestampAdjustment, + receiver: r, + upstreamCodecs: codecs, + kind: kind, + codec: codecs[0].RTPCodecCapability, } d.forwarder = NewForwarder( d.kind, @@ -1113,7 +1117,11 @@ func (d *DownTrack) CreateSenderReport() *rtcp.SenderReport { return nil } - return d.rtpStats.GetRtcpSenderReport(d.ssrc) + sr, tsAdjust := d.rtpStats.GetRtcpSenderReport(d.ssrc) + if d.allowTimestampAdjustment { + d.forwarder.AdjustTimestamp(tsAdjust) + } + return sr } func (d *DownTrack) writeBlankFrameRTP(duration float32, generation uint32) chan struct{} { diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index e8e8167b0..b347be756 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -1630,6 +1630,13 @@ func (f *Forwarder) GetRTPMungerParams() RTPMungerParams { return f.rtpMunger.GetParams() } +func (f *Forwarder) AdjustTimestamp(tsAdjust float64) { + f.lock.Lock() + defer f.lock.Unlock() + + f.rtpMunger.UpdateTsOffset(uint32(tsAdjust)) +} + // ----------------------------------------------------------------------------- func getOptimalBandwidthNeeded(muted bool, pubMuted bool, maxPublishedLayer int32, brs Bitrates, maxLayer buffer.VideoLayer) int64 { diff --git a/pkg/sfu/rtpmunger.go b/pkg/sfu/rtpmunger.go index 35ea2b25f..8fa1d95d4 100644 --- a/pkg/sfu/rtpmunger.go +++ b/pkg/sfu/rtpmunger.go @@ -58,6 +58,7 @@ type RTPMungerParams struct { highestIncomingSN uint16 lastSN uint16 snOffset uint16 + highestIncomingTS uint32 lastTS uint32 tsOffset uint32 lastMarker bool @@ -88,6 +89,7 @@ func (r *RTPMunger) GetParams() RTPMungerParams { highestIncomingSN: r.highestIncomingSN, lastSN: r.lastSN, snOffset: r.snOffset, + highestIncomingTS: r.highestIncomingTS, lastTS: r.lastTS, tsOffset: r.tsOffset, lastMarker: r.lastMarker, @@ -110,6 +112,7 @@ func (r *RTPMunger) SeedLast(state RTPMungerState) { func (r *RTPMunger) SetLastSnTs(extPkt *buffer.ExtPacket) { r.highestIncomingSN = extPkt.Packet.SequenceNumber - 1 + r.highestIncomingTS = extPkt.Packet.Timestamp if !r.started { r.lastSN = extPkt.Packet.SequenceNumber r.lastTS = extPkt.Packet.Timestamp @@ -122,6 +125,7 @@ func (r *RTPMunger) SetLastSnTs(extPkt *buffer.ExtPacket) { func (r *RTPMunger) UpdateSnTsOffsets(extPkt *buffer.ExtPacket, snAdjust uint16, tsAdjust uint32) { r.highestIncomingSN = extPkt.Packet.SequenceNumber - 1 + r.highestIncomingTS = extPkt.Packet.Timestamp r.snOffset = extPkt.Packet.SequenceNumber - r.lastSN - snAdjust r.tsOffset = extPkt.Packet.Timestamp - r.lastTS - tsAdjust @@ -138,6 +142,10 @@ func (r *RTPMunger) PacketDropped(extPkt *buffer.ExtPacket) { r.lastSN = extPkt.Packet.SequenceNumber - r.snOffset } +func (r *RTPMunger) UpdateTsOffset(tsAdjust uint32) { + r.tsOffset -= tsAdjust +} + func (r *RTPMunger) UpdateAndGetSnTs(extPkt *buffer.ExtPacket) (*TranslationParamsRTP, error) { // if out-of-order, look up sequence number offset cache diff := extPkt.Packet.SequenceNumber - r.highestIncomingSN @@ -200,8 +208,27 @@ func (r *RTPMunger) UpdateAndGetSnTs(extPkt *buffer.ExtPacket) (*TranslationPara mungedSN := extPkt.Packet.SequenceNumber - r.snOffset mungedTS := extPkt.Packet.Timestamp - r.tsOffset + // with timestamp adjustment, it is possible that the adjustment causes munged timestamp to move backwards, + // detect that and adjust so that it does not move back + if extPkt.Packet.Timestamp != r.highestIncomingTS && (((mungedTS - r.lastTS) == 0) || (mungedTS-r.lastTS) > (1<<31)) { + adjustedMungedTS := r.lastTS + 1 + adjustedTSOffset := extPkt.Packet.Timestamp - adjustedMungedTS + r.logger.Infow( + "adjust out-of-order timestamp offset", + "mungedTS", mungedTS, + "lastTS", r.lastTS, + "incomingTS", extPkt.Packet.Timestamp, + "offset", r.tsOffset, + "adjustedMungedTS", adjustedMungedTS, + "adjustedTSOffset", adjustedTSOffset, + ) + mungedTS = adjustedMungedTS + r.tsOffset = adjustedTSOffset + } + r.highestIncomingSN = extPkt.Packet.SequenceNumber r.lastSN = mungedSN + r.highestIncomingTS = extPkt.Packet.Timestamp r.lastTS = mungedTS r.lastMarker = extPkt.Packet.Marker diff --git a/pkg/sfu/streamtrackermanager.go b/pkg/sfu/streamtrackermanager.go index 256b5c4ce..ee5f039e7 100644 --- a/pkg/sfu/streamtrackermanager.go +++ b/pkg/sfu/streamtrackermanager.go @@ -494,9 +494,11 @@ func (s *StreamTrackerManager) GetReferenceLayerRTPTimestamp(ts uint32, layer in return 0, fmt.Errorf("invalid layer, target: %d, reference: %d", layer, referenceLayer) } + /* TODO-RESTORE-AFTER-DEBUG - this is just fast path, below calculations should yield same if layer == referenceLayer { return ts, nil } + */ var srLayer *buffer.RTCPSenderReportData if int(layer) < len(s.senderReports) { @@ -521,6 +523,20 @@ func (s *StreamTrackerManager) GetReferenceLayerRTPTimestamp(ts uint32, layer in ntpDiff := srRef.NTPTimestamp.Time().Sub(srLayer.NTPTimestamp.Time()) rtpDiff := ntpDiff.Nanoseconds() * int64(s.clockRate) / 1e9 normalizedTS := srLayer.RTPTimestamp + uint32(rtpDiff) + s.logger.Debugw( + "getting reference timestaml", + "layer", layer, + "referenceLayer", referenceLayer, + "incomingTS", ts, + "layerNTP", srLayer.NTPTimestamp.Time().String(), + "refNTP", srRef.NTPTimestamp.Time().String(), + "ntpDiff", ntpDiff.String(), + "layerRTP", srLayer.RTPTimestamp, + "refRTP", srRef.RTPTimestamp, + "rtpDiff", rtpDiff, + "normalizedTS", normalizedTS, + "mappedTS", ts+(srRef.RTPTimestamp-normalizedTS), + ) // now that both RTP timestamps correspond to roughly the same NTP time, // the diff between them is the offset in RTP timestamp units between layer and referenceLayer. From 14a2d06bcd594bb7d2db5f9aeed99c4415f8e527 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sun, 7 May 2023 10:09:30 +0530 Subject: [PATCH 146/324] RTCP sender reports every three seconds. (#1692) * RTCP sender reports every three seconds. Ideally, we should be sending this based on data rate. But, increasing frequency a little as a lost sender report means the client may not have sender report for 10 seconds and that could affect sync. We do receiver reports once a second. Thought of setting this to that level too, but not making a big change from existing rate. Also, simplifying the RTCP send loop. Don't need to hold and do the processing after collecting all reports. * consistent use of GetSubscribedTracks --- pkg/rtc/participant.go | 55 ++++++++++++++---------------------------- 1 file changed, 18 insertions(+), 37 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 91008754c..ebe987428 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -34,7 +34,7 @@ import ( ) const ( - sdBatchSize = 20 + sdBatchSize = 30 rttUpdateInterval = 5 * time.Second disconnectCleanupDuration = 15 * time.Second @@ -391,7 +391,7 @@ func (p *ParticipantImpl) SetPermission(permission *livekit.ParticipantPermissio p.SubscriptionManager.queueReconcile("") } else { // revoke all subscriptions - for _, st := range p.GetSubscribedTracks() { + for _, st := range p.SubscriptionManager.GetSubscribedTracks() { st.MediaTrack().RemoveSubscriber(p.ID(), false) } } @@ -1330,57 +1330,38 @@ func (p *ParticipantImpl) subscriberRTCPWorker() { return } - var srs []rtcp.Packet - var sd []rtcp.SourceDescriptionChunk subscribedTracks := p.SubscriptionManager.GetSubscribedTracks() - p.lock.RLock() + + // send in batches of sdBatchSize + batchSize := 0 + var pkts []rtcp.Packet + var sd []rtcp.SourceDescriptionChunk for _, subTrack := range subscribedTracks { sr := subTrack.DownTrack().CreateSenderReport() chunks := subTrack.DownTrack().CreateSourceDescriptionChunks() if sr == nil || chunks == nil { continue } - srs = append(srs, sr) + + pkts = append(pkts, sr) sd = append(sd, chunks...) - } - p.lock.RUnlock() - - // now send in batches of sdBatchSize - var batch []rtcp.SourceDescriptionChunk - var pkts []rtcp.Packet - batchSize := 0 - for len(sd) > 0 || len(srs) > 0 { - numSRs := len(srs) - if numSRs > 0 { - if numSRs > sdBatchSize { - numSRs = sdBatchSize - } - pkts = append(pkts, srs[:numSRs]...) - srs = srs[numSRs:] - } - - size := len(sd) - spaceRemain := sdBatchSize - batchSize - if spaceRemain > 0 && size > 0 { - if size > spaceRemain { - size = spaceRemain - } - batch = sd[:size] - sd = sd[size:] - pkts = append(pkts, &rtcp.SourceDescription{Chunks: batch}) + batchSize = batchSize + 1 + len(chunks) + if batchSize >= sdBatchSize { + pkts = append(pkts, &rtcp.SourceDescription{Chunks: sd}) if err := p.TransportManager.WriteSubscriberRTCP(pkts); err != nil { if err == io.EOF || err == io.ErrClosedPipe { return } p.params.Logger.Errorw("could not send down track reports", err) } - } - pkts = pkts[:0] - batchSize = 0 + pkts = pkts[:0] + sd = sd[:0] + batchSize = 0 + } } - time.Sleep(5 * time.Second) + time.Sleep(3 * time.Second) } } @@ -1987,7 +1968,7 @@ func (p *ParticipantImpl) postRtcp(pkts []rtcp.Packet) { } func (p *ParticipantImpl) setDowntracksConnected() { - for _, t := range p.GetSubscribedTracks() { + for _, t := range p.SubscriptionManager.GetSubscribedTracks() { if dt := t.DownTrack(); dt != nil { dt.SetConnected() } From ddcb8342ef7f40313f385f3e554eb31df3555b55 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sun, 7 May 2023 18:36:26 +0530 Subject: [PATCH 147/324] Fix Dervivative equation wrong brackets (#1693) --- pkg/sfu/buffer/rtpstats.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index 9b274f4e3..6ef8a60da 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -194,7 +194,9 @@ func NewRTPStats(params RTPStatsParams) *RTPStats { r.pidController.SetGains(2.0, 0.5, 0.25) r.pidController.SetDerivativeLPF(0.02) - r.pidController.SetOutputLimits(-0.025*float64(params.ClockRate), 0.025*float64(params.ClockRate)) + outMin, outMax := -0.025*float64(params.ClockRate), 0.025*float64(params.ClockRate) + r.pidController.SetOutputLimits(outMin, outMax) + r.pidController.SetIntegralLimits(outMin/2.0, outMax/2.0) return r } @@ -1837,7 +1839,7 @@ func (p *PIDController) Update(setpoint, measurement float64, at time.Time) floa } } - p.dVal = (-2.0 * p.kd * (measurement - p.prevMeasurement)) + (((2.0*p.tau - duration) * p.dVal) / (2.0*p.tau + duration)) + p.dVal = (-2.0*p.kd*(measurement-p.prevMeasurement) + (2.0*p.tau-duration)*p.dVal) / (2.0*p.tau + duration) output := proportional + p.iVal + p.dVal if p.isOutLimitsSet { From 153f02091ced62575d3306b4e5db90d811d49e78 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 8 May 2023 19:51:23 +0530 Subject: [PATCH 148/324] Use measurement in window instead of since start. (#1695) This captues chnages within a measurement window. --- pkg/sfu/buffer/rtpstats.go | 103 ++++++++++++++++++++------------ pkg/sfu/forwarder.go | 2 +- pkg/sfu/streamtrackermanager.go | 2 +- 3 files changed, 68 insertions(+), 39 deletions(-) diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index 6ef8a60da..b10dbff76 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -177,6 +177,7 @@ type RTPStats struct { srData *RTCPSenderReportData lastSRTime time.Time lastSRNTP mediatransportutil.NtpTime + lastSRRTP uint32 pidController *PIDController nextSnapshotId uint32 @@ -189,12 +190,12 @@ func NewRTPStats(params RTPStatsParams) *RTPStats { logger: params.Logger, nextSnapshotId: FirstSnapshotId, snapshots: make(map[uint32]*Snapshot), - pidController: NewPIDController(), + pidController: NewPIDController(params.Logger), } r.pidController.SetGains(2.0, 0.5, 0.25) r.pidController.SetDerivativeLPF(0.02) - outMin, outMax := -0.025*float64(params.ClockRate), 0.025*float64(params.ClockRate) + outMin, outMax := -0.025*float64(r.params.ClockRate), 0.025*float64(r.params.ClockRate) r.pidController.SetOutputLimits(outMin, outMax) r.pidController.SetIntegralLimits(outMin/2.0, outMax/2.0) return r @@ -286,6 +287,7 @@ func (r *RTPStats) Seed(from *RTPStats) { } r.lastSRTime = from.lastSRTime r.lastSRNTP = from.lastSRNTP + r.lastSRRTP = from.lastSRRTP r.nextSnapshotId = from.nextSnapshotId for id, ss := range from.snapshots { @@ -296,6 +298,7 @@ func (r *RTPStats) Seed(from *RTPStats) { func (r *RTPStats) SetLogger(logger logger.Logger) { r.logger = logger + r.pidController.SetLogger(logger) } func (r *RTPStats) Stop() { @@ -732,7 +735,7 @@ func (r *RTPStats) SetRtcpSenderReportData(srData *RTCPSenderReportData) { // prevent against extreme case of anachronous sender reports if r.srData != nil && r.srData.NTPTimestamp > srData.NTPTimestamp { r.logger.Debugw( - "anachronous RTCP sender report", + "received anachronous sender report", "current", srData.NTPTimestamp.Time(), "last", r.srData.NTPTimestamp.Time(), ) @@ -752,7 +755,7 @@ func (r *RTPStats) SetRtcpSenderReportData(srData *RTCPSenderReportData) { timeSinceFirst := time.Since(r.firstTime) // ideally should use NTP time from SR, but that is a different time base, now is a resonable approximation rtpDiffSinceFirst := getExtTS(srData.RTPTimestamp, r.tsCycles) - r.extStartTS - drift := int64(uint64(timeSinceFirst.Nanoseconds()*int64(r.params.ClockRate)/1e9) - rtpDiffSinceFirst) + drift := int64(rtpDiffSinceFirst - uint64(timeSinceFirst.Nanoseconds()*int64(r.params.ClockRate)/1e9)) driftMs := (float64(drift) * 1000) / float64(r.params.ClockRate) r.logger.Debugw( @@ -768,6 +771,7 @@ func (r *RTPStats) SetRtcpSenderReportData(srData *RTCPSenderReportData) { "rtpDiffSinceFirst", rtpDiffSinceFirst, "drift", drift, "driftMs", driftMs, + "rate", float64(rtpDiffSinceFirst)/timeSinceFirst.Seconds(), ) // TODO-REMOVE-AFTER-DEBUG-END @@ -829,7 +833,7 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32) (*rtcp.SenderReport, float64 expectedExtRTP := r.extStartTS + uint64(timeSinceFirst.Nanoseconds()*int64(r.params.ClockRate)/1e9) if getExtTS(r.highestTS, r.tsCycles) > expectedExtRTP || now.Before(r.highestTime) { r.logger.Debugw( - "anachronous sender report", + "sending anachronous sender report", "firstTime", r.firstTime.String(), "currentTime", now.String(), "highestTime", r.highestTime.String(), @@ -844,20 +848,17 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32) (*rtcp.SenderReport, float64 nowRTP := r.highestTS + uint32(timeSinceHighest.Nanoseconds()*int64(r.params.ClockRate)/1e9) // TODO-REMOVE-AFTER-DEBUG-START - timeSinceFirst = nowNTP.Time().Sub(r.firstTime) - rtpDiffSinceFirst := getExtTS(nowRTP, r.tsCycles) - r.extStartTS - measurement := float64(rtpDiffSinceFirst) / timeSinceFirst.Seconds() - pidOutput := r.pidController.Update( - float64(r.params.ClockRate), - measurement, - now, - ) - r.logger.Debugw( - "pid controller output", - "measurement", measurement, - "errorTerm", float64(r.params.ClockRate)-measurement, - "pidOutput", pidOutput, - ) + pidOutput := float64(0.0) + if !r.lastSRTime.IsZero() { + timeSinceLast := now.Sub(r.lastSRTime) + rtpDiffSinceLast := nowRTP - r.lastSRRTP + rate := float64(rtpDiffSinceLast) / timeSinceLast.Seconds() + pidOutput = r.pidController.Update( + float64(r.params.ClockRate), + rate, + now, + ) + } // TODO-REMOVE-AFTER-DEBUG-STOP // TODO-REMOVE-AFTER-DEBUG-START @@ -867,9 +868,9 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32) (*rtcp.SenderReport, float64 rtpDiffLocal := int32(nowRTP - r.highestTS) rtpOffsetLocal := int32(nowRTP - r.highestTS - uint32(ntpDiffLocal.Nanoseconds()*int64(r.params.ClockRate)/1e9)) - drift := int64(uint64(timeSinceFirst.Nanoseconds()*int64(r.params.ClockRate)/1e9) - rtpDiffSinceFirst) + rtpDiffSinceFirst := getExtTS(nowRTP, r.tsCycles) - r.extStartTS + drift := int64(rtpDiffSinceFirst - uint64(timeSinceFirst.Nanoseconds()*int64(r.params.ClockRate)/1e9)) driftMs := (float64(drift) * 1000) / float64(r.params.ClockRate) - r.logger.Debugw( "sending sender report", "highestTS", r.highestTS, @@ -884,12 +885,13 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32) (*rtcp.SenderReport, float64 "rtpDiffSinceFirst", rtpDiffSinceFirst, "drift", drift, "driftMs", driftMs, - "rate", measurement, + "rate", float64(rtpDiffSinceFirst)/timeSinceFirst.Seconds(), ) // TODO-REMOVE-AFTER-DEBUG-END r.lastSRTime = now r.lastSRNTP = nowNTP + r.lastSRRTP = nowRTP return &rtcp.SenderReport{ SSRC: ssrc, @@ -1775,6 +1777,8 @@ func AggregateRTPDeltaInfo(deltaInfoList []*RTPDeltaInfo) *RTPDeltaInfo { // ------------------------------------------------------------------- type PIDController struct { + logger logger.Logger + kp, ki, kd float64 tau float64 // low pass filter of D, time constant @@ -1791,8 +1795,14 @@ type PIDController struct { prevMeasurementTime time.Time } -func NewPIDController() *PIDController { - return &PIDController{} +func NewPIDController(logger logger.Logger) *PIDController { + return &PIDController{ + logger: logger, + } +} + +func (p *PIDController) SetLogger(logger logger.Logger) { + p.logger = logger } func (p *PIDController) SetGains(kp, ki, kd float64) { @@ -1818,43 +1828,62 @@ func (p *PIDController) SetIntegralLimits(min, max float64) { } func (p *PIDController) Update(setpoint, measurement float64, at time.Time) float64 { - diff := setpoint - measurement + errorTerm := setpoint - measurement if p.prevMeasurementTime.IsZero() { - p.prevError = diff + p.prevError = errorTerm p.prevMeasurement = measurement p.prevMeasurementTime = at return 0 } - proportional := p.kp * diff - duration := at.Sub(p.prevMeasurementTime).Seconds() - p.iVal = p.iVal + (0.5 * p.ki * duration * (diff + p.prevError)) + if duration == 0 { + return 0 + } + + proportional := p.kp * errorTerm + + iVal := p.iVal + (0.5 * p.ki * duration * (errorTerm + p.prevError)) + boundIVal := iVal if p.isILimitsSet { - if p.iVal > p.iMax { - p.iVal = p.iMax + if iVal > p.iMax { + boundIVal = p.iMax } - if p.iVal < p.iMin { - p.iVal = p.iMin + if iVal < p.iMin { + boundIVal = p.iMin } } + p.iVal = boundIVal p.dVal = (-2.0*p.kd*(measurement-p.prevMeasurement) + (2.0*p.tau-duration)*p.dVal) / (2.0*p.tau + duration) output := proportional + p.iVal + p.dVal + boundOutput := output if p.isOutLimitsSet { if output > p.outMax { - output = p.outMax + boundOutput = p.outMax } if output < p.outMin { - output = p.outMin + boundOutput = p.outMin } } - p.prevError = diff + p.prevError = errorTerm p.prevMeasurement = measurement p.prevMeasurementTime = at - return output + p.logger.Debugw( + "pid controller", + "setpoint", setpoint, + "measurement", measurement, + "errorTerm", errorTerm, + "proportional", proportional, + "integral", iVal, + "integralLimited", boundIVal, + "derivative", p.dVal, + "output", output, + "outputLimited", boundOutput, + ) + return boundOutput } // ------------------------------------------------------------------- diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index b347be756..c7159d810 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -1634,7 +1634,7 @@ func (f *Forwarder) AdjustTimestamp(tsAdjust float64) { f.lock.Lock() defer f.lock.Unlock() - f.rtpMunger.UpdateTsOffset(uint32(tsAdjust)) + f.rtpMunger.UpdateTsOffset(uint32(tsAdjust + 0.5)) } // ----------------------------------------------------------------------------- diff --git a/pkg/sfu/streamtrackermanager.go b/pkg/sfu/streamtrackermanager.go index ee5f039e7..6b79914f9 100644 --- a/pkg/sfu/streamtrackermanager.go +++ b/pkg/sfu/streamtrackermanager.go @@ -524,7 +524,7 @@ func (s *StreamTrackerManager) GetReferenceLayerRTPTimestamp(ts uint32, layer in rtpDiff := ntpDiff.Nanoseconds() * int64(s.clockRate) / 1e9 normalizedTS := srLayer.RTPTimestamp + uint32(rtpDiff) s.logger.Debugw( - "getting reference timestaml", + "getting reference timestamp", "layer", layer, "referenceLayer", referenceLayer, "incomingTS", ts, From 0e582ec82aa0968b1ca3df2799156bf298cb7fe9 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 9 May 2023 00:13:01 +0530 Subject: [PATCH 149/324] fix the negative sign scope (#1696) --- pkg/sfu/buffer/rtpstats.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index b10dbff76..33de3c19d 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -1855,7 +1855,7 @@ func (p *PIDController) Update(setpoint, measurement float64, at time.Time) floa } p.iVal = boundIVal - p.dVal = (-2.0*p.kd*(measurement-p.prevMeasurement) + (2.0*p.tau-duration)*p.dVal) / (2.0*p.tau + duration) + p.dVal = -(2.0*p.kd*(measurement-p.prevMeasurement) + (2.0*p.tau-duration)*p.dVal) / (2.0*p.tau + duration) output := proportional + p.iVal + p.dVal boundOutput := output From cf2a0785798dbfe7477ec9beea68344355470a33 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 9 May 2023 12:39:11 +0530 Subject: [PATCH 150/324] Apply time stamp adjustment only at the start of a frame. (#1698) It was possible that the adjustment applied in the middle of a frame resulting in the same frame having multiple time stamps. That would have caused video to pause/jump. Apply the offset only at the start of the frame so that all packets of a frame get the same offset. --- pkg/sfu/rtpmunger.go | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/pkg/sfu/rtpmunger.go b/pkg/sfu/rtpmunger.go index 8fa1d95d4..a7971deb2 100644 --- a/pkg/sfu/rtpmunger.go +++ b/pkg/sfu/rtpmunger.go @@ -54,14 +54,15 @@ func (r RTPMungerState) String() string { // ---------------------------------------------------------------------- type RTPMungerParams struct { - started bool - highestIncomingSN uint16 - lastSN uint16 - snOffset uint16 - highestIncomingTS uint32 - lastTS uint32 - tsOffset uint32 - lastMarker bool + started bool + highestIncomingSN uint16 + lastSN uint16 + snOffset uint16 + highestIncomingTS uint32 + lastTS uint32 + tsOffset uint32 + tsOffsetAdjustment uint32 + lastMarker bool snOffsets [SnOffsetCacheSize]uint16 snOffsetsWritePtr int @@ -143,7 +144,7 @@ func (r *RTPMunger) PacketDropped(extPkt *buffer.ExtPacket) { } func (r *RTPMunger) UpdateTsOffset(tsAdjust uint32) { - r.tsOffset -= tsAdjust + r.tsOffsetAdjustment = tsAdjust } func (r *RTPMunger) UpdateAndGetSnTs(extPkt *buffer.ExtPacket) (*TranslationParamsRTP, error) { @@ -199,6 +200,12 @@ func (r *RTPMunger) UpdateAndGetSnTs(extPkt *buffer.ExtPacket) (*TranslationPara } } + // apply timestamp offset adjustment at the start of a frame only + if extPkt.Packet.Timestamp != r.highestIncomingTS && r.tsOffsetAdjustment != 0 { + r.tsOffset -= r.tsOffsetAdjustment + r.tsOffsetAdjustment = 0 + } + // in-order incoming packet, may or may not be contiguous. // In the case of loss (i.e. incoming sequence number is not contiguous), // forward even if it is a padding only packet. With temporal scalability, From f543e3f8d0ac842c7b2363de2d85c410248cf96b Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 9 May 2023 18:46:30 +0530 Subject: [PATCH 151/324] Send left over RTCP packets. (#1699) --- pkg/rtc/participant.go | 16 +++++++++++++++- pkg/sfu/buffer/rtpstats.go | 2 +- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index ebe987428..9e37fe99c 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -1347,7 +1347,9 @@ func (p *ParticipantImpl) subscriberRTCPWorker() { sd = append(sd, chunks...) batchSize = batchSize + 1 + len(chunks) if batchSize >= sdBatchSize { - pkts = append(pkts, &rtcp.SourceDescription{Chunks: sd}) + if len(sd) != 0 { + pkts = append(pkts, &rtcp.SourceDescription{Chunks: sd}) + } if err := p.TransportManager.WriteSubscriberRTCP(pkts); err != nil { if err == io.EOF || err == io.ErrClosedPipe { return @@ -1361,6 +1363,18 @@ func (p *ParticipantImpl) subscriberRTCPWorker() { } } + if len(pkts) != 0 || len(sd) != 0 { + if len(sd) != 0 { + pkts = append(pkts, &rtcp.SourceDescription{Chunks: sd}) + } + if err := p.TransportManager.WriteSubscriberRTCP(pkts); err != nil { + if err == io.EOF || err == io.ErrClosedPipe { + return + } + p.params.Logger.Errorw("could not send down track reports", err) + } + } + time.Sleep(3 * time.Second) } } diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index 33de3c19d..7e2871f7b 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -1107,7 +1107,7 @@ func (r *RTPStats) ToString() string { str := fmt.Sprintf("t: %+v|%+v|%.2fs", p.StartTime.AsTime().Format(time.UnixDate), p.EndTime.AsTime().Format(time.UnixDate), p.Duration) - str += fmt.Sprintf(" sn: %d|%d", r.extStartSN, r.getExtHighestSN()) + str += fmt.Sprintf(", sn: %d|%d", r.extStartSN, r.getExtHighestSN()) str += fmt.Sprintf(", ep: %d|%.2f/s", expectedPackets, expectedPacketRate) str += fmt.Sprintf(", p: %d|%.2f/s", p.Packets, p.PacketRate) From 0a3c22993e9dc7fd7a5dab8154ac14ab9c3d6523 Mon Sep 17 00:00:00 2001 From: David Colburn Date: Tue, 9 May 2023 16:54:32 -0700 Subject: [PATCH 152/324] Remove deprecated egress client (#1701) * remove deprecated egress client * don't copy mutex --- pkg/config/config.go | 5 -- pkg/service/egress.go | 104 ++++++++++---------------------------- pkg/service/ioinfo.go | 71 +++++--------------------- pkg/service/redisstore.go | 5 +- pkg/service/wire.go | 12 +---- pkg/service/wire_gen.go | 18 ++----- 6 files changed, 48 insertions(+), 167 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index cdab66a70..b379140f6 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -55,7 +55,6 @@ type Config struct { Video VideoConfig `yaml:"video,omitempty"` Room RoomConfig `yaml:"room,omitempty"` TURN TURNConfig `yaml:"turn,omitempty"` - Egress EgressConfig `yaml:"egress,omitempty"` Ingress IngressConfig `yaml:"ingress,omitempty"` WebHook WebHookConfig `yaml:"webhook,omitempty"` NodeSelector NodeSelectorConfig `yaml:"node_selector,omitempty"` @@ -256,10 +255,6 @@ type LimitConfig struct { SubscriptionLimitAudio int32 `yaml:"subscription_limit_audio,omitempty"` } -type EgressConfig struct { - UsePsRPC bool `yaml:"use_psrpc"` -} - type IngressConfig struct { RTMPBaseURL string `yaml:"rtmp_base_url"` WHIPBaseURL string `yaml:"whip_base_url"` diff --git a/pkg/service/egress.go b/pkg/service/egress.go index 80008f82b..99e232b96 100644 --- a/pkg/service/egress.go +++ b/pkg/service/egress.go @@ -10,7 +10,6 @@ import ( "github.com/livekit/livekit-server/pkg/rtc" "github.com/livekit/livekit-server/pkg/telemetry" - "github.com/livekit/protocol/egress" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/rpc" @@ -18,42 +17,37 @@ import ( ) type EgressService struct { - psrpcClient rpc.EgressClient - clientDeprecated egress.RPCClient - store ServiceStore - es EgressStore - roomService livekit.RoomService - telemetry telemetry.TelemetryService - launcher rtc.EgressLauncher + client rpc.EgressClient + store ServiceStore + es EgressStore + roomService livekit.RoomService + telemetry telemetry.TelemetryService + launcher rtc.EgressLauncher } type egressLauncher struct { - psrpcClient rpc.EgressClient - clientDeprecated egress.RPCClient - es EgressStore - telemetry telemetry.TelemetryService + client rpc.EgressClient + es EgressStore + telemetry telemetry.TelemetryService } func NewEgressLauncher( - psrpcClient rpc.EgressClient, - clientDeprecated egress.RPCClient, + client rpc.EgressClient, es EgressStore, ts telemetry.TelemetryService) rtc.EgressLauncher { - if psrpcClient == nil && clientDeprecated == nil { + if client == nil { return nil } return &egressLauncher{ - psrpcClient: psrpcClient, - clientDeprecated: clientDeprecated, - es: es, - telemetry: ts, + client: client, + es: es, + telemetry: ts, } } func NewEgressService( - psrpcClient rpc.EgressClient, - clientDeprecated egress.RPCClient, + client rpc.EgressClient, store ServiceStore, es EgressStore, rs livekit.RoomService, @@ -61,13 +55,12 @@ func NewEgressService( launcher rtc.EgressLauncher, ) *EgressService { return &EgressService{ - psrpcClient: psrpcClient, - clientDeprecated: clientDeprecated, - store: store, - es: es, - roomService: rs, - telemetry: ts, - launcher: launcher, + client: client, + store: store, + es: es, + roomService: rs, + telemetry: ts, + launcher: launcher, } } @@ -175,21 +168,12 @@ func (s *egressLauncher) StartEgress(ctx context.Context, req *rpc.StartEgressRe return s.StartEgressWithClusterId(ctx, "", req) } func (s *egressLauncher) StartEgressWithClusterId(ctx context.Context, clusterId string, req *rpc.StartEgressRequest) (*livekit.EgressInfo, error) { - var info *livekit.EgressInfo - var err error - // Ensure we have an Egress ID if req.EgressId == "" { req.EgressId = utils.NewGuid(utils.EgressPrefix) } - if s.psrpcClient != nil { - info, err = s.psrpcClient.StartEgress(ctx, clusterId, req) - } else { - logger.Warnw("Using deprecated egress client. Upgrade egress to v1.5.6+ and use egress:use_psrpc:true in your livekit config", nil) - // SendRequest will transform rpc.StartEgressRequest into deprecated livekit.StartEgressRequest - info, err = s.clientDeprecated.SendRequest(ctx, req) - } + info, err := s.client.StartEgress(ctx, clusterId, req) if err != nil { return nil, err } @@ -213,7 +197,7 @@ func (s *EgressService) UpdateLayout(ctx context.Context, req *livekit.UpdateLay if err := EnsureRecordPermission(ctx); err != nil { return nil, twirpAuthError(err) } - if s.psrpcClient == nil && s.clientDeprecated == nil { + if s.client == nil { return nil, ErrEgressNotConnected } @@ -249,27 +233,11 @@ func (s *EgressService) UpdateStream(ctx context.Context, req *livekit.UpdateStr return nil, twirpAuthError(err) } - if s.psrpcClient == nil && s.clientDeprecated == nil { + if s.client == nil { return nil, ErrEgressNotConnected } - race := rpc.NewRace[livekit.EgressInfo](ctx) - if s.clientDeprecated != nil { - race.Go(func(ctx context.Context) (*livekit.EgressInfo, error) { - return s.clientDeprecated.SendRequest(ctx, &livekit.EgressRequest{ - EgressId: req.EgressId, - Request: &livekit.EgressRequest_UpdateStream{ - UpdateStream: req, - }, - }) - }) - } - if s.psrpcClient != nil { - race.Go(func(ctx context.Context) (*livekit.EgressInfo, error) { - return s.psrpcClient.UpdateStream(ctx, req.EgressId, req) - }) - } - _, info, err := race.Wait() + info, err := s.client.UpdateStream(ctx, req.EgressId, req) if err != nil { return nil, err } @@ -290,7 +258,7 @@ func (s *EgressService) ListEgress(ctx context.Context, req *livekit.ListEgressR if err := EnsureRecordPermission(ctx); err != nil { return nil, twirpAuthError(err) } - if s.psrpcClient == nil && s.clientDeprecated == nil { + if s.client == nil { return nil, ErrEgressNotConnected } @@ -321,7 +289,7 @@ func (s *EgressService) StopEgress(ctx context.Context, req *livekit.StopEgressR return nil, twirpAuthError(err) } - if s.psrpcClient == nil && s.clientDeprecated == nil { + if s.client == nil { return nil, ErrEgressNotConnected } @@ -335,23 +303,7 @@ func (s *EgressService) StopEgress(ctx context.Context, req *livekit.StopEgressR } } - race := rpc.NewRace[livekit.EgressInfo](ctx) - if s.clientDeprecated != nil { - race.Go(func(ctx context.Context) (*livekit.EgressInfo, error) { - return s.clientDeprecated.SendRequest(ctx, &livekit.EgressRequest{ - EgressId: req.EgressId, - Request: &livekit.EgressRequest_Stop{ - Stop: req, - }, - }) - }) - } - if s.psrpcClient != nil { - race.Go(func(ctx context.Context) (*livekit.EgressInfo, error) { - return s.psrpcClient.StopEgress(ctx, req.EgressId, req) - }) - } - _, info, err = race.Wait() + info, err = s.client.StopEgress(ctx, req.EgressId, req) if err != nil { return nil, err } diff --git a/pkg/service/ioinfo.go b/pkg/service/ioinfo.go index e8fc77f9c..529eec8f9 100644 --- a/pkg/service/ioinfo.go +++ b/pkg/service/ioinfo.go @@ -4,11 +4,9 @@ import ( "context" "errors" - "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/emptypb" "github.com/livekit/livekit-server/pkg/telemetry" - "github.com/livekit/protocol/egress" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/rpc" @@ -16,12 +14,11 @@ import ( ) type IOInfoService struct { - psrpcServer rpc.IOInfoServer - es EgressStore - is IngressStore - telemetry telemetry.TelemetryService - ecDeprecated egress.RPCClient - shutdown chan struct{} + ioServer rpc.IOInfoServer + es EgressStore + is IngressStore + telemetry telemetry.TelemetryService + shutdown chan struct{} } func NewIOInfoService( @@ -30,22 +27,20 @@ func NewIOInfoService( es EgressStore, is IngressStore, ts telemetry.TelemetryService, - ec egress.RPCClient, ) (*IOInfoService, error) { s := &IOInfoService{ - es: es, - is: is, - telemetry: ts, - ecDeprecated: ec, - shutdown: make(chan struct{}), + es: es, + is: is, + telemetry: ts, + shutdown: make(chan struct{}), } if bus != nil { - psrpcServer, err := rpc.NewIOInfoServer(string(nodeID), s, bus) + ioServer, err := rpc.NewIOInfoServer(string(nodeID), s, bus) if err != nil { return nil, err } - s.psrpcServer = psrpcServer + s.ioServer = ioServer } return s, nil @@ -59,8 +54,6 @@ func (s *IOInfoService) Start() error { logger.Errorw("failed to start redis egress worker", err) return err } - - go s.egressWorkerDeprecated() } return nil @@ -126,45 +119,7 @@ func (s *IOInfoService) UpdateIngressState(ctx context.Context, req *rpc.UpdateI func (s *IOInfoService) Stop() { close(s.shutdown) - if s.psrpcServer != nil { - s.psrpcServer.Shutdown() + if s.ioServer != nil { + s.ioServer.Shutdown() } } - -// Deprecated -func (s *IOInfoService) egressWorkerDeprecated() error { - if s.ecDeprecated == nil { - return nil - } - - go func() { - sub, err := s.ecDeprecated.GetUpdateChannel(context.Background()) - if err != nil { - logger.Errorw("failed to subscribe to results channel", err) - } - - resChan := sub.Channel() - for { - select { - case msg := <-resChan: - b := sub.Payload(msg) - info := &livekit.EgressInfo{} - if err = proto.Unmarshal(b, info); err != nil { - logger.Errorw("failed to read results", err) - continue - } - _, err = s.UpdateEgressInfo(context.Background(), info) - if err != nil { - logger.Errorw("failed to update egress info", err) - } - - case <-s.shutdown: - _ = sub.Close() - s.es.(*RedisStore).Stop() - return - } - } - }() - - return nil -} diff --git a/pkg/service/redisstore.go b/pkg/service/redisstore.go index e7a615ba4..a11c318ad 100644 --- a/pkg/service/redisstore.go +++ b/pkg/service/redisstore.go @@ -505,11 +505,10 @@ func (s *RedisStore) storeIngress(_ context.Context, info *livekit.IngressInfo) } // ignore state - infoCopy := livekit.IngressInfo{} - infoCopy = *info + infoCopy := proto.Clone(info).(*livekit.IngressInfo) infoCopy.State = nil - data, err := proto.Marshal(&infoCopy) + data, err := proto.Marshal(infoCopy) if err != nil { return err } diff --git a/pkg/service/wire.go b/pkg/service/wire.go index 5da0e2c79..126542780 100644 --- a/pkg/service/wire.go +++ b/pkg/service/wire.go @@ -18,7 +18,6 @@ import ( "github.com/livekit/livekit-server/pkg/routing" "github.com/livekit/livekit-server/pkg/telemetry" "github.com/livekit/protocol/auth" - "github.com/livekit/protocol/egress" "github.com/livekit/protocol/livekit" redisLiveKit "github.com/livekit/protocol/redis" "github.com/livekit/protocol/rpc" @@ -45,8 +44,7 @@ func InitializeServer(conf *config.Config, currentNode routing.LocalNode) (*Live telemetry.NewTelemetryService, getMessageBus, NewIOInfoService, - getEgressClient, - egress.NewRedisRPCClient, + rpc.NewEgressClient, getEgressStore, NewEgressLauncher, NewEgressService, @@ -148,14 +146,6 @@ func getMessageBus(rc redis.UniversalClient) psrpc.MessageBus { return psrpc.NewRedisMessageBus(rc) } -func getEgressClient(conf *config.Config, nodeID livekit.NodeID, bus psrpc.MessageBus) (rpc.EgressClient, error) { - if conf.Egress.UsePsRPC { - return rpc.NewEgressClient(nodeID, bus) - } - - return nil, nil -} - func getEgressStore(s ObjectStore) EgressStore { switch store := s.(type) { case *RedisStore: diff --git a/pkg/service/wire_gen.go b/pkg/service/wire_gen.go index bce220b63..1ce19893a 100644 --- a/pkg/service/wire_gen.go +++ b/pkg/service/wire_gen.go @@ -13,7 +13,6 @@ import ( "github.com/livekit/livekit-server/pkg/routing" "github.com/livekit/livekit-server/pkg/telemetry" "github.com/livekit/protocol/auth" - "github.com/livekit/protocol/egress" "github.com/livekit/protocol/livekit" redis2 "github.com/livekit/protocol/redis" "github.com/livekit/protocol/rpc" @@ -53,11 +52,10 @@ func InitializeServer(conf *config.Config, currentNode routing.LocalNode) (*Live if err != nil { return nil, err } - egressClient, err := getEgressClient(conf, nodeID, messageBus) + egressClient, err := rpc.NewEgressClient(nodeID, messageBus) if err != nil { return nil, err } - rpcClient := egress.NewRedisRPCClient(nodeID, universalClient) egressStore := getEgressStore(objectStore) keyProvider, err := createKeyProvider(conf) if err != nil { @@ -69,12 +67,12 @@ func InitializeServer(conf *config.Config, currentNode routing.LocalNode) (*Live } analyticsService := telemetry.NewAnalyticsService(conf, currentNode) telemetryService := telemetry.NewTelemetryService(queuedNotifier, analyticsService) - rtcEgressLauncher := NewEgressLauncher(egressClient, rpcClient, egressStore, telemetryService) + rtcEgressLauncher := NewEgressLauncher(egressClient, egressStore, telemetryService) roomService, err := NewRoomService(roomConfig, apiConfig, router, roomAllocator, objectStore, rtcEgressLauncher) if err != nil { return nil, err } - egressService := NewEgressService(egressClient, rpcClient, objectStore, egressStore, roomService, telemetryService, rtcEgressLauncher) + egressService := NewEgressService(egressClient, objectStore, egressStore, roomService, telemetryService, rtcEgressLauncher) ingressConfig := getIngressConfig(conf) ingressClient, err := rpc.NewIngressClient(nodeID, messageBus) if err != nil { @@ -82,7 +80,7 @@ func InitializeServer(conf *config.Config, currentNode routing.LocalNode) (*Live } ingressStore := getIngressStore(objectStore) ingressService := NewIngressService(ingressConfig, nodeID, messageBus, ingressClient, ingressStore, roomService, telemetryService) - ioInfoService, err := NewIOInfoService(nodeID, messageBus, egressStore, ingressStore, telemetryService, rpcClient) + ioInfoService, err := NewIOInfoService(nodeID, messageBus, egressStore, ingressStore, telemetryService) if err != nil { return nil, err } @@ -193,14 +191,6 @@ func getMessageBus(rc redis.UniversalClient) psrpc.MessageBus { return psrpc.NewRedisMessageBus(rc) } -func getEgressClient(conf *config.Config, nodeID livekit.NodeID, bus psrpc.MessageBus) (rpc.EgressClient, error) { - if conf.Egress.UsePsRPC { - return rpc.NewEgressClient(nodeID, bus) - } - - return nil, nil -} - func getEgressStore(s ObjectStore) EgressStore { switch store := s.(type) { case *RedisStore: From 2ccee369a667194329ae7321b8200d4679ef20cc Mon Sep 17 00:00:00 2001 From: David Colburn Date: Tue, 9 May 2023 20:52:22 -0700 Subject: [PATCH 153/324] update notifier (#1702) --- go.mod | 4 ++-- go.sum | 8 ++++---- pkg/telemetry/telemetryservice.go | 1 - 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 4ccb896b3..fff0ea1dc 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/dustin/go-humanize v1.0.1 github.com/elliotchance/orderedmap/v2 v2.2.0 github.com/florianl/go-tc v0.4.2 - github.com/frostbyte73/core v0.0.5 + github.com/frostbyte73/core v0.0.9 github.com/gammazero/deque v0.2.1 github.com/gammazero/workerpool v1.1.3 github.com/google/wire v0.5.0 @@ -18,7 +18,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 - github.com/livekit/protocol v1.5.6 + github.com/livekit/protocol v1.5.7-0.20230510002113-cadccd54108e github.com/livekit/psrpc v0.3.1-0.20230502152150-df9dd21fba11 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.14.0 diff --git a/go.sum b/go.sum index 51023823d..e52434e66 100644 --- a/go.sum +++ b/go.sum @@ -33,8 +33,8 @@ github.com/florianl/go-tc v0.4.2 h1:jan5zcOWCLhA9SRBHZhQ0SSAq7cmDUagiRPngAi5AOQ= github.com/florianl/go-tc v0.4.2/go.mod h1:2W1jSMFryiYlpQigr4ZpSSpE9XNze+bW7cTsCXWbMwo= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= -github.com/frostbyte73/core v0.0.5 h1:+oHjXDyQyQzEx04mtmmafYP07n7EToKpUGafWbNVQ9I= -github.com/frostbyte73/core v0.0.5/go.mod h1:mqHHSVFS5DE6kSdhU1/s9Mm0YCnLB8Ou2DD/eX1Zbr4= +github.com/frostbyte73/core v0.0.9 h1:AmE9GjgGpPsWk9ZkmY3HsYUs2hf2tZt+/W6r49URBQI= +github.com/frostbyte73/core v0.0.9/go.mod h1:XsOGqrqe/VEV7+8vJ+3a8qnCIXNbKsoEiu/czs7nrcU= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/gammazero/deque v0.2.1 h1:qSdsbG6pgp6nL7A0+K/B7s12mcCY/5l5SIUpMOl+dC0= @@ -121,8 +121,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 h1:QlQFyMwCDgjyySsrgmrMcVbEBA6KZcyTzvK+z346tUA= github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26/go.mod h1:eDA41kiySZoG+wy4Etsjb3w0jjLx69i/vAmSjG4bteA= -github.com/livekit/protocol v1.5.6 h1:2kwduElaTcYc4JKxs3aGp+jbMDC9g1z8L+ywlLBwMvo= -github.com/livekit/protocol v1.5.6/go.mod h1:CtvrXHdVzapR+avHHBr3RnCwwM4wjCJv9LQ4UCA14TU= +github.com/livekit/protocol v1.5.7-0.20230510002113-cadccd54108e h1:0F3RjOkUS71P2ODHI5ZW09Cbrr6A0Pg8vCFFy6gcqkk= +github.com/livekit/protocol v1.5.7-0.20230510002113-cadccd54108e/go.mod h1:vjGsR1YxXnN5BLS0yr/YjGnJOPrS0ymddCF3JwxSHGM= github.com/livekit/psrpc v0.3.1-0.20230502152150-df9dd21fba11 h1:VS23iVQu/TNiLEM5XjbBSY28+B6nSewjKWPDbieg0Ho= github.com/livekit/psrpc v0.3.1-0.20230502152150-df9dd21fba11/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= diff --git a/pkg/telemetry/telemetryservice.go b/pkg/telemetry/telemetryservice.go index 9e6ef65ae..fdc6ff052 100644 --- a/pkg/telemetry/telemetryservice.go +++ b/pkg/telemetry/telemetryservice.go @@ -62,7 +62,6 @@ type TelemetryService interface { } const ( - maxWebhookWorkers = 50 workerCleanupWait = 3 * time.Minute jobQueueBufferSize = 10000 ) From 2d380868dc65ef18429983d14a7978fa775a2fd6 Mon Sep 17 00:00:00 2001 From: Russ d'Sa Date: Tue, 9 May 2023 20:58:43 -0700 Subject: [PATCH 154/324] Update README.md (#1691) * Update README.md * Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index d6690a9e0..6f0cf205c 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,7 @@ https://docs.livekit.io - [LiveKit Meet](https://meet.livekit.io) ([source](https://github.com/livekit-examples/meet)) - [Spatial Audio](https://spatial-audio-demo.livekit.io/) ([source](https://github.com/livekit-examples/spatial-audio)) - Livestreaming from OBS Studio ([source](https://github.com/livekit-examples/livestream)) +- [AI voice assistant using ChatGPT](https://livekit.io/kitt) ([source](https://github.com/livekit-examples/kitt)) ## SDKs & Tools From 678cd0624195a374a289028484f1bcdd61199b64 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 10 May 2023 10:26:36 +0530 Subject: [PATCH 155/324] Infow -> Debugw (#1703) --- pkg/sfu/rtpmunger.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/sfu/rtpmunger.go b/pkg/sfu/rtpmunger.go index a7971deb2..7880dfee7 100644 --- a/pkg/sfu/rtpmunger.go +++ b/pkg/sfu/rtpmunger.go @@ -220,7 +220,7 @@ func (r *RTPMunger) UpdateAndGetSnTs(extPkt *buffer.ExtPacket) (*TranslationPara if extPkt.Packet.Timestamp != r.highestIncomingTS && (((mungedTS - r.lastTS) == 0) || (mungedTS-r.lastTS) > (1<<31)) { adjustedMungedTS := r.lastTS + 1 adjustedTSOffset := extPkt.Packet.Timestamp - adjustedMungedTS - r.logger.Infow( + r.logger.Debugw( "adjust out-of-order timestamp offset", "mungedTS", mungedTS, "lastTS", r.lastTS, From 4419cd56b88459beec0c6a3a4c0341ab6c5dc133 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 10 May 2023 11:01:51 +0530 Subject: [PATCH 156/324] Switch to rate since first time. (#1704) With short term measurements, the adjustment itself was causing some oscillations and drift tend to settle at some small value and oscillated around it due to push/pull affecting small window measurement. --- pkg/sfu/buffer/rtpstats.go | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index 7e2871f7b..7ba38e694 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -177,7 +177,6 @@ type RTPStats struct { srData *RTCPSenderReportData lastSRTime time.Time lastSRNTP mediatransportutil.NtpTime - lastSRRTP uint32 pidController *PIDController nextSnapshotId uint32 @@ -287,7 +286,6 @@ func (r *RTPStats) Seed(from *RTPStats) { } r.lastSRTime = from.lastSRTime r.lastSRNTP = from.lastSRNTP - r.lastSRRTP = from.lastSRRTP r.nextSnapshotId = from.nextSnapshotId for id, ss := range from.snapshots { @@ -848,17 +846,13 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32) (*rtcp.SenderReport, float64 nowRTP := r.highestTS + uint32(timeSinceHighest.Nanoseconds()*int64(r.params.ClockRate)/1e9) // TODO-REMOVE-AFTER-DEBUG-START - pidOutput := float64(0.0) - if !r.lastSRTime.IsZero() { - timeSinceLast := now.Sub(r.lastSRTime) - rtpDiffSinceLast := nowRTP - r.lastSRRTP - rate := float64(rtpDiffSinceLast) / timeSinceLast.Seconds() - pidOutput = r.pidController.Update( - float64(r.params.ClockRate), - rate, - now, - ) - } + rtpDiffSinceFirst := getExtTS(nowRTP, r.tsCycles) - r.extStartTS + rate := float64(rtpDiffSinceFirst) / timeSinceFirst.Seconds() + pidOutput := r.pidController.Update( + float64(r.params.ClockRate), + rate, + now, + ) // TODO-REMOVE-AFTER-DEBUG-STOP // TODO-REMOVE-AFTER-DEBUG-START @@ -868,7 +862,6 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32) (*rtcp.SenderReport, float64 rtpDiffLocal := int32(nowRTP - r.highestTS) rtpOffsetLocal := int32(nowRTP - r.highestTS - uint32(ntpDiffLocal.Nanoseconds()*int64(r.params.ClockRate)/1e9)) - rtpDiffSinceFirst := getExtTS(nowRTP, r.tsCycles) - r.extStartTS drift := int64(rtpDiffSinceFirst - uint64(timeSinceFirst.Nanoseconds()*int64(r.params.ClockRate)/1e9)) driftMs := (float64(drift) * 1000) / float64(r.params.ClockRate) r.logger.Debugw( @@ -885,13 +878,12 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32) (*rtcp.SenderReport, float64 "rtpDiffSinceFirst", rtpDiffSinceFirst, "drift", drift, "driftMs", driftMs, - "rate", float64(rtpDiffSinceFirst)/timeSinceFirst.Seconds(), + "rate", rate, ) // TODO-REMOVE-AFTER-DEBUG-END r.lastSRTime = now r.lastSRNTP = nowNTP - r.lastSRRTP = nowRTP return &rtcp.SenderReport{ SSRC: ssrc, From c254e5713fa69ce0fa8ea7afc0f0f3764ad1b05f Mon Sep 17 00:00:00 2001 From: David Zhao Date: Tue, 9 May 2023 23:41:34 -0700 Subject: [PATCH 157/324] v1.4.2 release notes (#1663) --- CHANGELOG | 38 ++++++++++++++++++++++++++++++++++++++ config-sample.yaml | 6 ------ version/version.go | 2 +- 3 files changed, 39 insertions(+), 7 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index d2c66ec7b..43f763a1f 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,6 +2,44 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.4.2] - 2023-04-27 +### Added +- VP9 codec with SVC support (#1586) +- Support for source-specific permissions and client-initiated metadata updates (#1590) +- Batch support for signal relay (#1593 #1596) +- Support for simulating subscriber bandwidth (#1609) +- Support for subscription limits (#1629) +- Send Room updates when participant counts change (#1647) + +### Fixed +- Fixed process return code to 0 (#1589) +- Fixed VP9 stutter when not using dependency descriptors (#1595) +- Fixed stutter when using dependency descriptors (#1600) +- Fixed Redis cluster support when using Egress or Ingress (#1606) +- Fixed simulcast parsing error for slower clients (camera and screenshare) (#1621) +- Don't close RTCP reader if Downtrack will be resumed (#1632) +- Restore VP8 munger state properly. (#1634) +- Fixed incorrect node routing when using signal relay (#1645) +- Do not send hidden participants to others after resume (#1689) +- Fix for potential webhook delivery delays (#1690) + +### Changed +- Refactored video layer selector (#1588 #1591 #1592) +- Improved transport fallback when client is resuming (#1597) +- Improved webhook reliability with delivery retries (#1607 #1615) +- Congestion controller improvements (#1614 #1616 #1617 #1623 #1628 #1631 #1652) +- Reduced memory usage by releasing ParticipantInfo after JoinResponse is transmitted (#1619) +- Ensure safe access in sequencer (#1625) +- Run quality scorer when there are no streams. (#1633) +- Participant version is only incremented after updates (#1646) +- Connection quality attribution improvements (#1653 #1664) +- Remove disallowed subscriptions on close. (#1668) +- A/V sync improvements (#1681 #1684 #1687 #1693 #1695 #1696 #1698 #1704) +- RTCP sender reports every three seconds. (#1692) + +### Removed +- Remove deprecated (non-psrpc) egress client (#1701) + ## [1.4.1] - 2023-04-05 ### Added - Added prometheus metrics for internal signaling API #1571 diff --git a/config-sample.yaml b/config-sample.yaml index 122fac283..e1cd63731 100644 --- a/config-sample.yaml +++ b/config-sample.yaml @@ -228,12 +228,6 @@ keys: # # Prefix used to generate WHIP URLs for WHIP ingress. # whip_base_url: "http://my.domain.com/whip" -# egress server -# egress: -# # Whether to use the PSRPC enabled RPC implementation. This requires livekit egress version >=1.5.4 -# # The legacy, non PSRPC RPC implementation will be removed eventually -# use_psrpc: false - # Region of the current node. Required if using regionaware node selector # region: us-west-2 diff --git a/version/version.go b/version/version.go index bf695e80d..3835a32bc 100644 --- a/version/version.go +++ b/version/version.go @@ -1,3 +1,3 @@ package version -const Version = "1.4.1" +const Version = "1.4.2" From b61fad339feeb2c7eb319370d7fed9fcd8f00bea Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 10 May 2023 18:31:49 +0530 Subject: [PATCH 158/324] Handle time stamp increment across mute. (#1705) * Handle time stamp increment across mute. Two cases handled 1. Starting on mute could inject blank frame/padding packets. These time stamps are randomly generated. So, when the publisher unmutes, the time stamp was jumping ahead by only 1. Make it so that they jump ahead by elapsed time since starting the blank frames/ padding packets. 2. When generating blank frames at the end of a down track, if the track was muted at that time, the blank frame time stamps could have been off (i. e. would have been pointing to time after the last forwarded frame). Here also use current time to adjust time stamp. Maybe, this could help in some cases where we are seeing unflushed video buffer? * remove unnecessary check * address feedback and also maintain first synthesized time stamp --- pkg/sfu/forwarder.go | 91 +++++++++++++++++++++++++++++++++------ pkg/sfu/forwarder_test.go | 11 ++++- pkg/sfu/rtpmunger.go | 48 +++++++++------------ pkg/sfu/rtpmunger_test.go | 64 ++++++--------------------- 4 files changed, 121 insertions(+), 93 deletions(-) diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index c7159d810..312689133 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -3,10 +3,12 @@ package sfu import ( "fmt" "math" + "math/rand" "strings" "sync" "time" + "github.com/pion/rtp" "github.com/pion/webrtc/v3" "github.com/livekit/protocol/logger" @@ -136,9 +138,12 @@ type TranslationParams struct { // ------------------------------------------------------------------- type ForwarderState struct { - Started bool - RTP RTPMungerState - Codec interface{} + Started bool + PreStartTime time.Time + FirstTS uint32 + RefTSOffset uint32 + RTP RTPMungerState + Codec interface{} } func (f ForwarderState) String() string { @@ -147,7 +152,14 @@ func (f ForwarderState) String() string { case codecmunger.VP8State: codecString = codecState.String() } - return fmt.Sprintf("ForwarderState{started: %v, rtp: %s, codec: %s}", f.Started, f.RTP.String(), codecString) + return fmt.Sprintf("ForwarderState{started: %v, preStartTime: %s, firstTS: %d, refTSOffset: %d, rtp: %s, codec: %s}", + f.Started, + f.PreStartTime.String(), + f.FirstTS, + f.RefTSOffset, + f.RTP.String(), + codecString, + ) } // ------------------------------------------------------------------- @@ -164,8 +176,11 @@ type Forwarder struct { pubMuted bool started bool + preStartTime time.Time + firstTS uint32 lastSSRC uint32 referenceLayerSpatial int32 + refTSOffset uint32 parkedLayerTimer *time.Timer @@ -313,13 +328,14 @@ func (f *Forwarder) GetState() ForwarderState { return ForwarderState{} } - state := ForwarderState{ - Started: f.started, - RTP: f.rtpMunger.GetLast(), - Codec: f.codecMunger.GetState(), + return ForwarderState{ + Started: f.started, + PreStartTime: f.preStartTime, + FirstTS: f.firstTS, + RefTSOffset: f.refTSOffset, + RTP: f.rtpMunger.GetLast(), + Codec: f.codecMunger.GetState(), } - - return state } func (f *Forwarder) SeedState(state ForwarderState) { @@ -334,6 +350,9 @@ func (f *Forwarder) SeedState(state ForwarderState) { f.codecMunger.SeedState(state.Codec) f.started = true + f.preStartTime = state.PreStartTime + f.firstTS = state.FirstTS + f.refTSOffset = state.RefTSOffset } func (f *Forwarder) Mute(muted bool) (bool, buffer.VideoLayer) { @@ -1467,8 +1486,17 @@ func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer i ts, err := f.getExpectedRTPTimestamp(switchingAt) if err == nil { expectedTS = ts + } else { + rtpDiff := uint32(0) + if !f.preStartTime.IsZero() && f.refTSOffset == 0 { + timeSinceFirst := time.Since(f.preStartTime) + rtpDiff = uint32(timeSinceFirst.Nanoseconds() * int64(f.codec.ClockRate) / 1e9) + f.refTSOffset = f.firstTS + rtpDiff - refTS + } + expectedTS += rtpDiff } } + refTS += f.refTSOffset nextTS, explain := getNextTimestamp(lastTS, refTS, expectedTS) f.logger.Debugw( "next timestamp on switch", @@ -1588,12 +1616,35 @@ func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer in return tp, nil } +func (f *Forwarder) maybeStart() { + if f.started { + return + } + + f.started = true + f.preStartTime = time.Now() + + extPkt := &buffer.ExtPacket{ + Packet: &rtp.Packet{ + Header: rtp.Header{ + SequenceNumber: uint16(rand.Intn(1<<14)) + uint16(1<<15), // a random number in third quartile of sequence number space + Timestamp: uint32(rand.Intn(1<<30)) + uint32(1<<31), // a random number in third quartile of time stamp space + }, + }, + } + f.rtpMunger.SetLastSnTs(extPkt) + + f.firstTS = extPkt.Packet.Timestamp +} + func (f *Forwarder) GetSnTsForPadding(num int) ([]SnTs, error) { f.lock.Lock() defer f.lock.Unlock() - // padding is used for probing. Padding packets should be - // at only the frame boundaries to ensure decoder sequencer does + f.maybeStart() + + // padding is used for probing. Padding packets should only + // be at frame boundaries to ensure decoder sequencer does // not get out-of-sync. But, when a stream is paused, // force a frame marker as a restart of the stream will // start with a key frame which will reset the decoder. @@ -1601,18 +1652,30 @@ func (f *Forwarder) GetSnTsForPadding(num int) ([]SnTs, error) { if !f.vls.GetTarget().IsValid() { forceMarker = true } - return f.rtpMunger.UpdateAndGetPaddingSnTs(num, 0, 0, forceMarker) + return f.rtpMunger.UpdateAndGetPaddingSnTs(num, 0, 0, forceMarker, 0) } func (f *Forwarder) GetSnTsForBlankFrames(frameRate uint32, numPackets int) ([]SnTs, bool, error) { f.lock.Lock() defer f.lock.Unlock() + f.maybeStart() + frameEndNeeded := !f.rtpMunger.IsOnFrameBoundary() if frameEndNeeded { numPackets++ } - snts, err := f.rtpMunger.UpdateAndGetPaddingSnTs(numPackets, f.codec.ClockRate, frameRate, frameEndNeeded) + + lastTS := f.rtpMunger.GetLast().LastTS + expectedTS := lastTS + if f.getExpectedRTPTimestamp != nil { + ts, err := f.getExpectedRTPTimestamp(time.Now()) + if err == nil { + expectedTS = ts + } + } + nextTS, _ := getNextTimestamp(lastTS, expectedTS, expectedTS) + snts, err := f.rtpMunger.UpdateAndGetPaddingSnTs(numPackets, f.codec.ClockRate, frameRate, frameEndNeeded, nextTS) return snts, frameEndNeeded, err } diff --git a/pkg/sfu/forwarder_test.go b/pkg/sfu/forwarder_test.go index c99374ded..df03a058a 100644 --- a/pkg/sfu/forwarder_test.go +++ b/pkg/sfu/forwarder_test.go @@ -1844,9 +1844,15 @@ func TestForwardGetSnTsForBlankFrames(t *testing.T) { frameRate := uint32(30) var sntsExpected = make([]SnTs, numPadding) for i := 0; i < numPadding; i++ { + // first blank frame should have same timestamp as last frame as end frame is synthesized + ts := params.Timestamp + if i != 0 { + // +1 here due to expected time stamp bumpint by at least one so that time stamp is always moving ahead + ts = params.Timestamp + 1 + ((uint32(i)*clockRate)+frameRate-1)/frameRate + } sntsExpected[i] = SnTs{ sequenceNumber: params.SequenceNumber + uint16(i) + 1, - timestamp: params.Timestamp + (uint32(i)*clockRate)/frameRate, + timestamp: ts, } } require.Equal(t, sntsExpected, snts) @@ -1858,7 +1864,8 @@ func TestForwardGetSnTsForBlankFrames(t *testing.T) { for i := 0; i < numPadding; i++ { sntsExpected[i] = SnTs{ sequenceNumber: params.SequenceNumber + uint16(len(snts)) + uint16(i) + 1, - timestamp: snts[len(snts)-1].timestamp + (uint32(i+1)*clockRate)/frameRate, + // +1 here due to expected time stamp bumpint by at least one so that time stamp is always moving ahead + timestamp: snts[len(snts)-1].timestamp + 1 + ((uint32(i+1)*clockRate)+frameRate-1)/frameRate, } } snts, frameEndNeeded, err = f.GetSnTsForBlankFrames(30, numBlankFrames) diff --git a/pkg/sfu/rtpmunger.go b/pkg/sfu/rtpmunger.go index 7880dfee7..a80ad6000 100644 --- a/pkg/sfu/rtpmunger.go +++ b/pkg/sfu/rtpmunger.go @@ -2,7 +2,6 @@ package sfu import ( "fmt" - "math/rand" "github.com/livekit/protocol/logger" @@ -42,19 +41,17 @@ type SnTs struct { // ---------------------------------------------------------------------- type RTPMungerState struct { - Started bool - LastSN uint16 - LastTS uint32 + LastSN uint16 + LastTS uint32 } func (r RTPMungerState) String() string { - return fmt.Sprintf("RTPMungerState{started: %v, lastSN: %d, lastTS: %d)", r.Started, r.LastSN, r.LastTS) + return fmt.Sprintf("RTPMungerState{lastSN: %d, lastTS: %d)", r.LastSN, r.LastTS) } // ---------------------------------------------------------------------- type RTPMungerParams struct { - started bool highestIncomingSN uint16 lastSN uint16 snOffset uint16 @@ -86,7 +83,6 @@ func NewRTPMunger(logger logger.Logger) *RTPMunger { func (r *RTPMunger) GetParams() RTPMungerParams { return RTPMungerParams{ - started: r.started, highestIncomingSN: r.highestIncomingSN, lastSN: r.lastSN, snOffset: r.snOffset, @@ -99,14 +95,12 @@ func (r *RTPMunger) GetParams() RTPMungerParams { func (r *RTPMunger) GetLast() RTPMungerState { return RTPMungerState{ - Started: r.started, - LastSN: r.lastSN, - LastTS: r.lastTS, + LastSN: r.lastSN, + LastTS: r.lastTS, } } func (r *RTPMunger) SeedLast(state RTPMungerState) { - r.started = state.Started r.lastSN = state.LastSN r.lastTS = state.LastTS } @@ -114,14 +108,8 @@ func (r *RTPMunger) SeedLast(state RTPMungerState) { func (r *RTPMunger) SetLastSnTs(extPkt *buffer.ExtPacket) { r.highestIncomingSN = extPkt.Packet.SequenceNumber - 1 r.highestIncomingTS = extPkt.Packet.Timestamp - if !r.started { - r.lastSN = extPkt.Packet.SequenceNumber - r.lastTS = extPkt.Packet.Timestamp - } else { - r.snOffset = extPkt.Packet.SequenceNumber - r.lastSN - 1 - r.tsOffset = extPkt.Packet.Timestamp - r.lastTS - 1 - } - r.started = true + r.lastSN = extPkt.Packet.SequenceNumber + r.lastTS = extPkt.Packet.Timestamp } func (r *RTPMunger) UpdateSnTsOffsets(extPkt *buffer.ExtPacket, snAdjust uint16, tsAdjust uint32) { @@ -270,7 +258,8 @@ func (r *RTPMunger) FilterRTX(nacks []uint16) []uint16 { return filtered } -func (r *RTPMunger) UpdateAndGetPaddingSnTs(num int, clockRate uint32, frameRate uint32, forceMarker bool) ([]SnTs, error) { +func (r *RTPMunger) UpdateAndGetPaddingSnTs(num int, clockRate uint32, frameRate uint32, forceMarker bool, rtpTimestamp uint32) ([]SnTs, error) { + useLastTSForFirst := false tsOffset := 0 if !r.lastMarker { if !forceMarker { @@ -278,20 +267,25 @@ func (r *RTPMunger) UpdateAndGetPaddingSnTs(num int, clockRate uint32, frameRate } // if forcing frame end, use timestamp of latest received frame for the first one + useLastTSForFirst = true tsOffset = 1 } - if !r.started { - r.lastSN = uint16(rand.Intn(1<<14)) + uint16(1<<15) // a random number in third quartile of sequence number space - r.lastTS = uint32(rand.Intn(1<<30)) + uint32(1<<31) // a random number in third quartile of time stamp space - r.started = true - } - + lastTS := r.lastTS vals := make([]SnTs, num) for i := 0; i < num; i++ { vals[i].sequenceNumber = r.lastSN + uint16(i) + 1 if frameRate != 0 { - vals[i].timestamp = r.lastTS + uint32(i+1-tsOffset)*(clockRate/frameRate) + if useLastTSForFirst && i == 0 { + vals[i].timestamp = r.lastTS + } else { + ts := rtpTimestamp + ((uint32(i+1-tsOffset)*clockRate)+frameRate-1)/frameRate + if (ts-lastTS) == 0 || (ts-lastTS) > (1<<31) { + ts = lastTS + 1 + lastTS = ts + } + vals[i].timestamp = ts + } } else { vals[i].timestamp = r.lastTS } diff --git a/pkg/sfu/rtpmunger_test.go b/pkg/sfu/rtpmunger_test.go index e1e772023..68ea60716 100644 --- a/pkg/sfu/rtpmunger_test.go +++ b/pkg/sfu/rtpmunger_test.go @@ -28,49 +28,11 @@ func TestSetLastSnTs(t *testing.T) { r.SetLastSnTs(extPkt) require.Equal(t, uint16(23332), r.highestIncomingSN) + require.Equal(t, uint32(0xabcdef), r.highestIncomingTS) require.Equal(t, uint16(23333), r.lastSN) require.Equal(t, uint32(0xabcdef), r.lastTS) require.Equal(t, uint16(0), r.snOffset) require.Equal(t, uint32(0), r.tsOffset) - require.True(t, r.started) - - // force re-start - r.started = false - - params = &testutils.TestExtPacketParams{ - SequenceNumber: 43, - Timestamp: 0xabcdef, - SSRC: 0x12345678, - } - extPkt, err = testutils.GetTestExtPacket(params) - require.NoError(t, err) - require.NotNil(t, extPkt) - - r.SetLastSnTs(extPkt) - require.Equal(t, uint16(42), r.highestIncomingSN) - require.Equal(t, uint16(43), r.lastSN) - require.Equal(t, uint32(0xabcdef), r.lastTS) - require.Equal(t, uint16(0), r.snOffset) - require.Equal(t, uint32(0), r.tsOffset) - require.True(t, r.started) - - // set on a started munger - params = &testutils.TestExtPacketParams{ - SequenceNumber: 23457, - Timestamp: 0xabcdef, - SSRC: 0x12345678, - } - extPkt, err = testutils.GetTestExtPacket(params) - require.NoError(t, err) - require.NotNil(t, extPkt) - - r.SetLastSnTs(extPkt) - require.Equal(t, uint16(23456), r.highestIncomingSN) - require.Equal(t, uint16(43), r.lastSN) - require.Equal(t, uint32(0xabcdef), r.lastTS) - require.Equal(t, uint16(23413), r.snOffset) - require.Equal(t, uint32(0xffffffff), r.tsOffset) - require.True(t, r.started) } func TestUpdateSnTsOffsets(t *testing.T) { @@ -92,6 +54,7 @@ func TestUpdateSnTsOffsets(t *testing.T) { extPkt, _ = testutils.GetTestExtPacket(params) r.UpdateSnTsOffsets(extPkt, 1, 1) require.Equal(t, uint16(33332), r.highestIncomingSN) + require.Equal(t, uint32(0xabcdef), r.highestIncomingTS) require.Equal(t, uint16(23333), r.lastSN) require.Equal(t, uint32(0xabcdef), r.lastTS) require.Equal(t, uint16(9999), r.snOffset) @@ -109,9 +72,10 @@ func TestPacketDropped(t *testing.T) { } extPkt, _ := testutils.GetTestExtPacket(params) r.SetLastSnTs(extPkt) - require.Equal(t, r.highestIncomingSN, uint16(23332)) - require.Equal(t, r.lastSN, uint16(23333)) - require.Equal(t, r.lastTS, uint32(0xabcdef)) + require.Equal(t, uint16(23332), r.highestIncomingSN) + require.Equal(t, uint32(0xabcdef), r.highestIncomingTS) + require.Equal(t, uint16(23333), r.lastSN) + require.Equal(t, uint32(0xabcdef), r.lastTS) require.Equal(t, uint16(0), r.snOffset) require.Equal(t, uint32(0), r.tsOffset) @@ -126,8 +90,8 @@ func TestPacketDropped(t *testing.T) { } extPkt, _ = testutils.GetTestExtPacket(params) r.PacketDropped(extPkt) - require.Equal(t, r.highestIncomingSN, uint16(23333)) - require.Equal(t, r.lastSN, uint16(23333)) + require.Equal(t, uint16(23333), r.highestIncomingSN) + require.Equal(t, uint16(23333), r.lastSN) require.Equal(t, uint16(0), r.snOffset) // drop a head packet and check offset increases @@ -160,7 +124,7 @@ func TestPacketDropped(t *testing.T) { require.Equal(t, uint16(1), r.snOffsets[snOffsetWritePtr]) snOffsetWritePtr = (snOffsetWritePtr + 1) & SnOffsetCacheMask require.Equal(t, snOffsetWritePtr, r.snOffsetsWritePtr) - require.Equal(t, r.lastSN, uint16(44444)) + require.Equal(t, uint16(44444), r.lastSN) require.Equal(t, uint16(1), r.snOffset) } @@ -464,7 +428,7 @@ func TestUpdateAndGetPaddingSnTs(t *testing.T) { r.SetLastSnTs(extPkt) // getting padding without forcing marker should fail - _, err := r.UpdateAndGetPaddingSnTs(10, 10, 5, false) + _, err := r.UpdateAndGetPaddingSnTs(10, 10, 5, false, 0) require.Error(t, err) require.ErrorIs(t, err, ErrPaddingNotOnFrameBoundary) @@ -477,10 +441,10 @@ func TestUpdateAndGetPaddingSnTs(t *testing.T) { for i := 0; i < numPadding; i++ { sntsExpected[i] = SnTs{ sequenceNumber: params.SequenceNumber + uint16(i) + 1, - timestamp: params.Timestamp + (uint32(i)*clockRate)/frameRate, + timestamp: params.Timestamp + ((uint32(i)*clockRate)+frameRate-1)/frameRate, } } - snts, err := r.UpdateAndGetPaddingSnTs(numPadding, clockRate, frameRate, true) + snts, err := r.UpdateAndGetPaddingSnTs(numPadding, clockRate, frameRate, true, params.Timestamp) require.NoError(t, err) require.Equal(t, sntsExpected, snts) @@ -488,10 +452,10 @@ func TestUpdateAndGetPaddingSnTs(t *testing.T) { for i := 0; i < numPadding; i++ { sntsExpected[i] = SnTs{ sequenceNumber: params.SequenceNumber + uint16(len(snts)) + uint16(i) + 1, - timestamp: snts[len(snts)-1].timestamp + (uint32(i+1)*clockRate)/frameRate, + timestamp: snts[len(snts)-1].timestamp + ((uint32(i+1)*clockRate)+frameRate-1)/frameRate, } } - snts, err = r.UpdateAndGetPaddingSnTs(numPadding, clockRate, frameRate, false) + snts, err = r.UpdateAndGetPaddingSnTs(numPadding, clockRate, frameRate, false, snts[len(snts)-1].timestamp) require.NoError(t, err) require.Equal(t, sntsExpected, snts) } From 4244542840449ca93ae8178eeea0e4f3b2af51bf Mon Sep 17 00:00:00 2001 From: Benjamin Pracht Date: Wed, 10 May 2023 20:00:34 -0700 Subject: [PATCH 159/324] Adopt WebRTCConfig from mediatransportutil (#1707) This also adds support for inline fields in ToCLIFlagNames --- go.mod | 4 +- go.sum | 4 +- pkg/config/config.go | 101 +++----- pkg/config/ip.go | 200 -------------- pkg/rtc/config.go | 373 ++------------------------- pkg/rtc/participant_internal_test.go | 2 +- pkg/rtc/rtc_unix.go | 40 --- pkg/rtc/rtc_windows.go | 7 - pkg/service/roommanager.go | 5 +- test/client/client.go | 5 +- 10 files changed, 57 insertions(+), 684 deletions(-) delete mode 100644 pkg/config/ip.go delete mode 100644 pkg/rtc/rtc_unix.go delete mode 100644 pkg/rtc/rtc_windows.go diff --git a/go.mod b/go.mod index fff0ea1dc..8418df750 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.2 github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 - github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 + github.com/livekit/mediatransportutil v0.0.0-20230511025422-058ebf6b48c9 github.com/livekit/protocol v1.5.7-0.20230510002113-cadccd54108e github.com/livekit/psrpc v0.3.1-0.20230502152150-df9dd21fba11 github.com/mackerelio/go-osstat v0.2.4 @@ -31,7 +31,6 @@ require ( github.com/pion/rtcp v1.2.10 github.com/pion/rtp v1.7.13 github.com/pion/sdp/v3 v3.0.6 - github.com/pion/stun v0.4.0 github.com/pion/transport/v2 v2.2.0 github.com/pion/turn/v2 v2.1.0 github.com/pion/webrtc/v3 v3.2.1 @@ -82,6 +81,7 @@ require ( github.com/pion/randutil v0.1.0 // indirect github.com/pion/sctp v1.8.7 // indirect github.com/pion/srtp/v2 v2.0.12 // indirect + github.com/pion/stun v0.4.0 // indirect github.com/pion/udp/v2 v2.0.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect diff --git a/go.sum b/go.sum index e52434e66..c2fb05eda 100644 --- a/go.sum +++ b/go.sum @@ -119,8 +119,8 @@ github.com/lithammer/shortuuid/v4 v4.0.0 h1:QRbbVkfgNippHOS8PXDkti4NaWeyYfcBTHtw github.com/lithammer/shortuuid/v4 v4.0.0/go.mod h1:Zs8puNcrvf2rV9rTH51ZLLcj7ZXqQI3lv67aw4KiB1Y= github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkDaKb5iXdynYrzB84ErPPO4LbRASk58= github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= -github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26 h1:QlQFyMwCDgjyySsrgmrMcVbEBA6KZcyTzvK+z346tUA= -github.com/livekit/mediatransportutil v0.0.0-20230326055817-ed569ca13d26/go.mod h1:eDA41kiySZoG+wy4Etsjb3w0jjLx69i/vAmSjG4bteA= +github.com/livekit/mediatransportutil v0.0.0-20230511025422-058ebf6b48c9 h1:aqivx5Tal2Fa6z1ZQrrBk/vYShosQx3ecl1aMwcQRV0= +github.com/livekit/mediatransportutil v0.0.0-20230511025422-058ebf6b48c9/go.mod h1:MRc0zSOSzXuFt0X218SgabzlaKevkvCckPgBEoHYc34= github.com/livekit/protocol v1.5.7-0.20230510002113-cadccd54108e h1:0F3RjOkUS71P2ODHI5ZW09Cbrr6A0Pg8vCFFy6gcqkk= github.com/livekit/protocol v1.5.7-0.20230510002113-cadccd54108e/go.mod h1:vjGsR1YxXnN5BLS0yr/YjGnJOPrS0ymddCF3JwxSHGM= github.com/livekit/psrpc v0.3.1-0.20230502152150-df9dd21fba11 h1:VS23iVQu/TNiLEM5XjbBSY28+B6nSewjKWPDbieg0Ho= diff --git a/pkg/config/config.go b/pkg/config/config.go index b379140f6..a9f61aad4 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -13,16 +13,12 @@ import ( "github.com/urfave/cli/v2" "gopkg.in/yaml.v3" + "github.com/livekit/mediatransportutil/pkg/rtcconfig" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/logger/pionlogger" redisLiveKit "github.com/livekit/protocol/redis" ) -var DefaultStunServers = []string{ - "stun.l.google.com:19302", - "stun1.l.google.com:19302", -} - type CongestionControlProbeMode string type StreamTrackerType string @@ -71,21 +67,9 @@ type Config struct { } type RTCConfig struct { - UDPPort uint32 `yaml:"udp_port,omitempty"` - TCPPort uint32 `yaml:"tcp_port,omitempty"` - ICEPortRangeStart uint32 `yaml:"port_range_start,omitempty"` - ICEPortRangeEnd uint32 `yaml:"port_range_end,omitempty"` - NodeIP string `yaml:"node_ip,omitempty"` - NodeIPAutoGenerated bool `yaml:"-"` - STUNServers []string `yaml:"stun_servers,omitempty"` - TURNServers []TURNServer `yaml:"turn_servers,omitempty"` - UseExternalIP bool `yaml:"use_external_ip"` - UseICELite bool `yaml:"use_ice_lite,omitempty"` - Interfaces InterfacesConfig `yaml:"interfaces,omitempty"` - IPs IPsConfig `yaml:"ips,omitempty"` - EnableLoopbackCandidate bool `yaml:"enable_loopback_candidate"` - UseMDNS bool `yaml:"use_mdns,omitempty"` - StrictACKs bool `yaml:"strict_acks,omitempty"` + rtcconfig.RTCConfig `yaml:",inline"` + + StrictACKs bool `yaml:"strict_acks,omitempty"` // Number of packets to buffer for NACK PacketBufferSize int `yaml:"packet_buffer_size,omitempty"` @@ -98,9 +82,6 @@ type RTCConfig struct { // allow TCP and TURN/TLS fallback AllowTCPFallback *bool `yaml:"allow_tcp_fallback,omitempty"` - // for testing, disable UDP - ForceTCP bool `yaml:"force_tcp,omitempty"` - // force a reconnect on a publication error ReconnectOnPublicationError *bool `yaml:"reconnect_on_publication_error,omitempty"` @@ -111,14 +92,6 @@ type RTCConfig struct { AllowTimestampAdjustment *bool `yaml:"allow_timestamp_adjustment,omitempty"` } -type TURNServer struct { - Host string `yaml:"host"` - Port int `yaml:"port"` - Protocol string `yaml:"protocol"` - Username string `yaml:"username,omitempty"` - Credential string `yaml:"credential,omitempty"` -} - type PLIThrottleConfig struct { LowQuality time.Duration `yaml:"low_quality,omitempty"` MidQuality time.Duration `yaml:"mid_quality,omitempty"` @@ -133,16 +106,6 @@ type CongestionControlConfig struct { MinChannelCapacity int64 `yaml:"min_channel_capacity,omitempty"` } -type InterfacesConfig struct { - Includes []string `yaml:"includes,omitempty"` - Excludes []string `yaml:"excludes,omitempty"` -} - -type IPsConfig struct { - Includes []string `yaml:"includes,omitempty"` - Excludes []string `yaml:"excludes,omitempty"` -} - type AudioConfig struct { // minimum level to be considered active, 0-127, where 0 is loudest ActiveLevel uint8 `yaml:"active_level,omitempty"` @@ -281,14 +244,16 @@ func NewConfig(confString string, strictMode bool, c *cli.Context, baseFlags []c conf := &Config{ Port: 7880, RTC: RTCConfig{ - UseExternalIP: false, - TCPPort: 7881, - UDPPort: 0, - ICEPortRangeStart: 0, - ICEPortRangeEnd: 0, - STUNServers: []string{}, - PacketBufferSize: 500, - StrictACKs: true, + RTCConfig: rtcconfig.RTCConfig{ + UseExternalIP: false, + TCPPort: 7881, + UDPPort: 0, + ICEPortRangeStart: 0, + ICEPortRangeEnd: 0, + STUNServers: []string{}, + }, + PacketBufferSize: 500, + StrictACKs: true, PLIThrottle: PLIThrottleConfig{ LowQuality: 500 * time.Millisecond, MidQuality: time.Second, @@ -426,6 +391,10 @@ func NewConfig(confString string, strictMode bool, c *cli.Context, baseFlags []c } } + if err := conf.RTC.Validate(conf.Development); err != nil { + return nil, fmt.Errorf("could not validate RTC config: %v", err) + } + if c != nil { if err := conf.updateFromCLI(c, baseFlags); err != nil { return nil, err @@ -439,17 +408,6 @@ func NewConfig(confString string, strictMode bool, c *cli.Context, baseFlags []c } conf.KeyFile = file - // set defaults for ports if none are set - if conf.RTC.UDPPort == 0 && conf.RTC.ICEPortRangeStart == 0 { - // to make it easier to run in dev mode/docker, default to single port - if conf.Development { - conf.RTC.UDPPort = 7882 - } else { - conf.RTC.ICEPortRangeStart = 50000 - conf.RTC.ICEPortRangeEnd = 60000 - } - } - // set defaults for Turn relay if none are set if conf.TURN.RelayPortRangeStart == 0 || conf.TURN.RelayPortRangeEnd == 0 { // to make it easier to run in dev mode/docker, default to two ports @@ -462,14 +420,6 @@ func NewConfig(confString string, strictMode bool, c *cli.Context, baseFlags []c } } - if conf.RTC.NodeIP == "" { - conf.RTC.NodeIP, err = conf.determineIP() - if err != nil { - return nil, err - } - conf.RTC.NodeIPAutoGenerated = true - } - if conf.LogLevel != "" { conf.Logging.Level = conf.LogLevel } @@ -517,13 +467,22 @@ func (conf *Config) ToCLIFlagNames(existingFlags []cli.Flag) map[string]reflect. for i := 0; i < currNode.TypeNode.NumField(); i++ { // inspect yaml tag from struct field to get path field := currNode.TypeNode.Type().Field(i) - yamlTag := strings.SplitN(field.Tag.Get("yaml"), ",", 2)[0] - if yamlTag == "" || yamlTag == "-" { + yamlTagArray := strings.SplitN(field.Tag.Get("yaml"), ",", 2) + yamlTag := yamlTagArray[0] + isInline := false + if len(yamlTagArray) > 1 && yamlTagArray[1] == "inline" { + isInline = true + } + if (yamlTag == "" && (!isInline || currNode.TagPrefix == "")) || yamlTag == "-" { continue } yamlPath := yamlTag if currNode.TagPrefix != "" { - yamlPath = fmt.Sprintf("%s.%s", currNode.TagPrefix, yamlTag) + if isInline { + yamlPath = currNode.TagPrefix + } else { + yamlPath = fmt.Sprintf("%s.%s", currNode.TagPrefix, yamlTag) + } } if existingFlagNames[yamlPath] { continue diff --git a/pkg/config/ip.go b/pkg/config/ip.go deleted file mode 100644 index e854c8e2d..000000000 --- a/pkg/config/ip.go +++ /dev/null @@ -1,200 +0,0 @@ -package config - -import ( - "context" - "fmt" - "net" - "time" - - "github.com/pion/stun" - "github.com/pkg/errors" - - "github.com/livekit/protocol/logger" -) - -func (conf *Config) determineIP() (string, error) { - if conf.RTC.UseExternalIP { - stunServers := conf.RTC.STUNServers - if len(stunServers) == 0 { - stunServers = DefaultStunServers - } - var err error - for i := 0; i < 3; i++ { - var ip string - ip, err = GetExternalIP(context.Background(), stunServers, nil) - if err == nil { - return ip, nil - } else { - time.Sleep(500 * time.Millisecond) - } - } - return "", errors.Errorf("could not resolve external IP: %v", err) - } - - // use local ip instead - addresses, err := GetLocalIPAddresses(false) - if len(addresses) > 0 { - return addresses[0], err - } - return "", err -} - -func GetLocalIPAddresses(includeLoopback bool) ([]string, error) { - ifaces, err := net.Interfaces() - if err != nil { - return nil, err - } - loopBacks := make([]string, 0) - addresses := make([]string, 0) - for _, iface := range ifaces { - addrs, err := iface.Addrs() - if err != nil { - continue - } - for _, addr := range addrs { - var ip net.IP - switch typedAddr := addr.(type) { - case *net.IPNet: - ip = typedAddr.IP.To4() - case *net.IPAddr: - ip = typedAddr.IP.To4() - default: - continue - } - if ip == nil { - continue - } - if ip.IsLoopback() { - loopBacks = append(loopBacks, ip.String()) - } else { - addresses = append(addresses, ip.String()) - } - } - } - - if includeLoopback { - addresses = append(addresses, loopBacks...) - } - - if len(addresses) > 0 { - return addresses, nil - } - if len(loopBacks) > 0 { - return loopBacks, nil - } - return nil, fmt.Errorf("could not find local IP address") -} - -// GetExternalIP return external IP for localAddr from stun server. If localAddr is nil, a local address is chosen automatically, -// else the address will be used to validate the external IP is accessible from the outside. -func GetExternalIP(ctx context.Context, stunServers []string, localAddr net.Addr) (string, error) { - if len(stunServers) == 0 { - return "", errors.New("STUN servers are required but not defined") - } - dialer := &net.Dialer{ - LocalAddr: localAddr, - } - conn, err := dialer.Dial("udp4", stunServers[0]) - if err != nil { - return "", err - } - c, err := stun.NewClient(conn) - if err != nil { - return "", err - } - defer c.Close() - - message, err := stun.Build(stun.TransactionID, stun.BindingRequest) - if err != nil { - return "", err - } - - var stunErr error - // sufficiently large buffer to not block it - ipChan := make(chan string, 20) - err = c.Start(message, func(res stun.Event) { - if res.Error != nil { - stunErr = res.Error - return - } - - var xorAddr stun.XORMappedAddress - if err := xorAddr.GetFrom(res.Message); err != nil { - stunErr = err - return - } - ip := xorAddr.IP.To4() - if ip != nil { - ipChan <- ip.String() - } - }) - if err != nil { - return "", err - } - - ctx1, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - select { - case nodeIP := <-ipChan: - if localAddr == nil { - return nodeIP, nil - } - _ = c.Close() - return nodeIP, validateExternalIP(ctx1, nodeIP, localAddr.(*net.UDPAddr)) - case <-ctx1.Done(): - msg := "could not determine public IP" - if stunErr != nil { - return "", errors.Wrap(stunErr, msg) - } else { - return "", fmt.Errorf(msg) - } - } -} - -// validateExternalIP validates that the external IP is accessible from the outside by listen the local address, -// it will send a magic string to the external IP and check the string is received by the local address. -func validateExternalIP(ctx context.Context, nodeIP string, addr *net.UDPAddr) error { - srv, err := net.ListenUDP("udp", addr) - if err != nil { - return err - } - defer srv.Close() - - magicString := "9#B8D2Nvg2xg5P$ZRwJ+f)*^Nne6*W3WamGY" - - validCh := make(chan struct{}) - go func() { - buf := make([]byte, 1024) - for { - n, err := srv.Read(buf) - if err != nil { - logger.Debugw("error reading from UDP socket", "err", err) - return - } - if string(buf[:n]) == magicString { - close(validCh) - return - } - } - }() - - cli, err := net.DialUDP("udp", nil, &net.UDPAddr{IP: net.ParseIP(nodeIP), Port: srv.LocalAddr().(*net.UDPAddr).Port}) - if err != nil { - return err - } - defer cli.Close() - - if _, err = cli.Write([]byte(magicString)); err != nil { - return err - } - - ctx1, cancel := context.WithTimeout(ctx, 1*time.Second) - defer cancel() - select { - case <-validCh: - return nil - case <-ctx1.Done(): - break - } - return fmt.Errorf("could not validate external IP") -} diff --git a/pkg/rtc/config.go b/pkg/rtc/config.go index b87ed857f..f3dbc2402 100644 --- a/pkg/rtc/config.go +++ b/pkg/rtc/config.go @@ -1,43 +1,26 @@ package rtc import ( - "context" - "errors" - "fmt" - "math/rand" - "net" - "strings" - "sync" - "time" - - "github.com/pion/ice/v2" "github.com/pion/sdp/v3" "github.com/pion/webrtc/v3" "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/sfu/buffer" dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" - "github.com/livekit/protocol/logger" - "github.com/livekit/protocol/logger/pionlogger" + "github.com/livekit/mediatransportutil/pkg/rtcconfig" ) const ( - minUDPBufferSize = 5_000_000 - defaultUDPBufferSize = 16_777_216 - frameMarking = "urn:ietf:params:rtp-hdrext:framemarking" + frameMarking = "urn:ietf:params:rtp-hdrext:framemarking" ) type WebRTCConfig struct { - Configuration webrtc.Configuration - SettingEngine webrtc.SettingEngine - Receiver ReceiverConfig - BufferFactory *buffer.Factory - UDPMux ice.UDPMux - TCPMuxListener *net.TCPListener - Publisher DirectionConfig - Subscriber DirectionConfig - NAT1To1IPs []string - UseMDNS bool + rtcconfig.WebRTCConfig + + BufferFactory *buffer.Factory + Receiver ReceiverConfig + Publisher DirectionConfig + Subscriber DirectionConfig } type ReceiverConfig struct { @@ -60,138 +43,18 @@ type DirectionConfig struct { StrictACKs bool } -const ( - // number of packets to buffer up - readBufferSize = 50 - - writeBufferSizeInBytes = 4 * 1024 * 1024 -) - -func NewWebRTCConfig(conf *config.Config, externalIP string) (*WebRTCConfig, error) { +func NewWebRTCConfig(conf *config.Config) (*WebRTCConfig, error) { rtcConf := conf.RTC - c := webrtc.Configuration{ - SDPSemantics: webrtc.SDPSemanticsUnifiedPlan, - } - s := webrtc.SettingEngine{ - LoggerFactory: pionlogger.NewLoggerFactory(logger.GetLogger()), - } - var ifFilter func(string) bool - if len(rtcConf.Interfaces.Includes) != 0 || len(rtcConf.Interfaces.Excludes) != 0 { - ifFilter = InterfaceFilterFromConf(rtcConf.Interfaces) - s.SetInterfaceFilter(ifFilter) - } - - var ipFilter func(net.IP) bool - if len(rtcConf.IPs.Includes) != 0 || len(rtcConf.IPs.Excludes) != 0 { - filter, err := IPFilterFromConf(rtcConf.IPs) - if err != nil { - return nil, err - } - ipFilter = filter - s.SetIPFilter(filter) - } - - if !rtcConf.UseMDNS { - s.SetICEMulticastDNSMode(ice.MulticastDNSModeDisabled) - } - - var nat1to1IPs []string - // force it to the node IPs that the user has set - if externalIP != "" && (conf.RTC.UseExternalIP || (conf.RTC.NodeIP != "" && !conf.RTC.NodeIPAutoGenerated)) { - if conf.RTC.UseExternalIP { - ips, err := getNAT1to1IPsForConf(conf, ipFilter) - if err != nil { - return nil, err - } - if len(ips) == 0 { - logger.Infow("no external IPs found, using node IP for NAT1To1Ips", "ip", externalIP) - s.SetNAT1To1IPs([]string{externalIP}, webrtc.ICECandidateTypeHost) - } else { - logger.Infow("using external IPs", "ips", ips) - s.SetNAT1To1IPs(ips, webrtc.ICECandidateTypeHost) - } - nat1to1IPs = ips - } else { - s.SetNAT1To1IPs([]string{externalIP}, webrtc.ICECandidateTypeHost) - } + webRTCConfig, err := rtcconfig.NewWebRTCConfig(&rtcConf.RTCConfig, conf.Development) + if err != nil { + return nil, err } if rtcConf.PacketBufferSize == 0 { rtcConf.PacketBufferSize = 500 } - var udpMux ice.UDPMux - var err error - networkTypes := make([]webrtc.NetworkType, 0, 4) - - if !rtcConf.ForceTCP { - networkTypes = append(networkTypes, - webrtc.NetworkTypeUDP4, webrtc.NetworkTypeUDP6, - ) - if rtcConf.ICEPortRangeStart != 0 && rtcConf.ICEPortRangeEnd != 0 { - if err := s.SetEphemeralUDPPortRange(uint16(rtcConf.ICEPortRangeStart), uint16(rtcConf.ICEPortRangeEnd)); err != nil { - return nil, err - } - } else if rtcConf.UDPPort != 0 { - opts := []ice.UDPMuxFromPortOption{ - ice.UDPMuxFromPortWithReadBufferSize(defaultUDPBufferSize), - ice.UDPMuxFromPortWithWriteBufferSize(defaultUDPBufferSize), - ice.UDPMuxFromPortWithLogger(s.LoggerFactory.NewLogger("udp_mux")), - } - if rtcConf.EnableLoopbackCandidate { - opts = append(opts, ice.UDPMuxFromPortWithLoopback()) - } - if ipFilter != nil { - opts = append(opts, ice.UDPMuxFromPortWithIPFilter(ipFilter)) - } - if ifFilter != nil { - opts = append(opts, ice.UDPMuxFromPortWithInterfaceFilter(ifFilter)) - } - udpMux, err := ice.NewMultiUDPMuxFromPort(int(rtcConf.UDPPort), opts...) - if err != nil { - return nil, err - } - - s.SetICEUDPMux(udpMux) - if !conf.Development { - checkUDPReadBuffer() - } - } - } - - // use TCP mux when it's set - var tcpListener *net.TCPListener - if rtcConf.TCPPort != 0 { - networkTypes = append(networkTypes, - webrtc.NetworkTypeTCP4, webrtc.NetworkTypeTCP6, - ) - tcpListener, err = net.ListenTCP("tcp", &net.TCPAddr{ - Port: int(rtcConf.TCPPort), - }) - if err != nil { - return nil, err - } - - tcpMux := ice.NewTCPMuxDefault(ice.TCPMuxParams{ - Logger: s.LoggerFactory.NewLogger("tcp_mux"), - Listener: tcpListener, - ReadBufferSize: readBufferSize, - WriteBufferSize: writeBufferSizeInBytes, - }) - - s.SetICETCPMux(tcpMux) - } - - if len(networkTypes) == 0 { - return nil, errors.New("TCP is forced but not configured") - } - s.SetNetworkTypes(networkTypes) - - if rtcConf.EnableLoopbackCandidate { - s.SetIncludeLoopbackCandidate(true) - } - // publisher configuration publisherConfig := DirectionConfig{ StrictACKs: true, // publisher is dialed, and will always reply with ACK @@ -244,32 +107,13 @@ func NewWebRTCConfig(conf *config.Config, externalIP string) (*WebRTCConfig, err subscriberConfig.RTCPFeedback.Video = append(subscriberConfig.RTCPFeedback.Video, webrtc.RTCPFeedback{Type: webrtc.TypeRTCPFBGoogREMB}) } - if rtcConf.UseICELite { - s.SetLite(true) - } else if rtcConf.NodeIP == "" && !rtcConf.UseExternalIP { - // use STUN servers for server to support NAT - // when deployed in production, we expect UseExternalIP to be used, and ports accessible - // this is not compatible with ICE Lite - // Do not automatically add STUN servers if nodeIP is set - if len(rtcConf.STUNServers) > 0 { - c.ICEServers = []webrtc.ICEServer{iceServerForStunServers(rtcConf.STUNServers)} - } else { - c.ICEServers = []webrtc.ICEServer{iceServerForStunServers(config.DefaultStunServers)} - } - } - return &WebRTCConfig{ - Configuration: c, - SettingEngine: s, + WebRTCConfig: *webRTCConfig, Receiver: ReceiverConfig{ PacketBufferSize: rtcConf.PacketBufferSize, }, - UDPMux: udpMux, - TCPMuxListener: tcpListener, - Publisher: publisherConfig, - Subscriber: subscriberConfig, - NAT1To1IPs: nat1to1IPs, - UseMDNS: rtcConf.UseMDNS, + Publisher: publisherConfig, + Subscriber: subscriberConfig, }, nil } @@ -277,190 +121,3 @@ func (c *WebRTCConfig) SetBufferFactory(factory *buffer.Factory) { c.BufferFactory = factory c.SettingEngine.BufferFactory = factory.GetOrNew } - -func iceServerForStunServers(servers []string) webrtc.ICEServer { - iceServer := webrtc.ICEServer{} - for _, stunServer := range servers { - iceServer.URLs = append(iceServer.URLs, fmt.Sprintf("stun:%s", stunServer)) - } - return iceServer -} - -func getNAT1to1IPsForConf(conf *config.Config, ipFilter func(net.IP) bool) ([]string, error) { - stunServers := conf.RTC.STUNServers - if len(stunServers) == 0 { - stunServers = config.DefaultStunServers - } - localIPs, err := config.GetLocalIPAddresses(conf.RTC.EnableLoopbackCandidate) - if err != nil { - return nil, err - } - type ipmapping struct { - externalIP string - localIP string - } - addrCh := make(chan ipmapping, len(localIPs)) - - var udpPorts []int - if conf.RTC.ICEPortRangeStart != 0 && conf.RTC.ICEPortRangeEnd != 0 { - portRangeStart, portRangeEnd := uint16(conf.RTC.ICEPortRangeStart), uint16(conf.RTC.ICEPortRangeEnd) - for i := 0; i < 5; i++ { - udpPorts = append(udpPorts, rand.Intn(int(portRangeEnd-portRangeStart))+int(portRangeStart)) - } - } else if conf.RTC.UDPPort != 0 { - udpPorts = append(udpPorts, int(conf.RTC.UDPPort)) - } else { - udpPorts = append(udpPorts, 0) - } - - var wg sync.WaitGroup - ctx, cancel := context.WithCancel(context.Background()) - for _, ip := range localIPs { - if ipFilter != nil && !ipFilter(net.ParseIP(ip)) { - continue - } - - wg.Add(1) - go func(localIP string) { - defer wg.Done() - for _, port := range udpPorts { - addr, err := config.GetExternalIP(ctx, stunServers, &net.UDPAddr{IP: net.ParseIP(localIP), Port: port}) - if err != nil { - if strings.Contains(err.Error(), "address already in use") { - logger.Debugw("failed to get external ip, address already in use", "local", localIP, "port", port) - continue - } - logger.Infow("failed to get external ip", "local", localIP, "err", err) - return - } - addrCh <- ipmapping{externalIP: addr, localIP: localIP} - return - } - logger.Infow("failed to get external ip after all ports tried", "local", localIP, "ports", udpPorts) - }(ip) - } - - var firstResolved bool - natMapping := make(map[string]string) - timeout := time.NewTimer(5 * time.Second) - defer timeout.Stop() - -done: - for { - select { - case mapping := <-addrCh: - if !firstResolved { - firstResolved = true - timeout.Reset(1 * time.Second) - } - if local, ok := natMapping[mapping.externalIP]; ok { - logger.Infow("external ip already solved, ignore duplicate", - "external", mapping.externalIP, - "local", local, - "ignore", mapping.localIP) - } else { - natMapping[mapping.externalIP] = mapping.localIP - } - - case <-timeout.C: - break done - } - } - cancel() - wg.Wait() - - if len(natMapping) == 0 { - // no external ip resolved - return nil, nil - } - - // mapping unresolved local ip to itself - for _, local := range localIPs { - var found bool - for _, localIPMapping := range natMapping { - if local == localIPMapping { - found = true - break - } - } - if !found { - natMapping[local] = local - } - } - - nat1to1IPs := make([]string, 0, len(natMapping)) - for external, local := range natMapping { - nat1to1IPs = append(nat1to1IPs, fmt.Sprintf("%s/%s", external, local)) - } - return nat1to1IPs, nil -} - -func InterfaceFilterFromConf(ifs config.InterfacesConfig) func(string) bool { - includes := ifs.Includes - excludes := ifs.Excludes - return func(s string) bool { - // filter by include interfaces - if len(includes) > 0 { - for _, iface := range includes { - if iface == s { - return true - } - } - return false - } - - // filter by exclude interfaces - if len(excludes) > 0 { - for _, iface := range excludes { - if iface == s { - return false - } - } - } - return true - } -} - -func IPFilterFromConf(ips config.IPsConfig) (func(ip net.IP) bool, error) { - var ipnets [2][]*net.IPNet - var err error - for i, ips := range [][]string{ips.Includes, ips.Excludes} { - ipnets[i], err = func(fromIPs []string) ([]*net.IPNet, error) { - var toNets []*net.IPNet - for _, ip := range fromIPs { - _, ipnet, err := net.ParseCIDR(ip) - if err != nil { - return nil, err - } - toNets = append(toNets, ipnet) - } - return toNets, nil - }(ips) - - if err != nil { - return nil, err - } - } - - includes, excludes := ipnets[0], ipnets[1] - - return func(ip net.IP) bool { - if len(includes) > 0 { - for _, ipn := range includes { - if ipn.Contains(ip) { - return true - } - } - return false - } - - if len(excludes) > 0 { - for _, ipn := range excludes { - if ipn.Contains(ip) { - return false - } - } - } - return true - }, nil -} diff --git a/pkg/rtc/participant_internal_test.go b/pkg/rtc/participant_internal_test.go index 1b2dadefb..adbe5862c 100644 --- a/pkg/rtc/participant_internal_test.go +++ b/pkg/rtc/participant_internal_test.go @@ -649,7 +649,7 @@ func newParticipantForTestWithOpts(identity livekit.ParticipantIdentity, opts *p // disable mux, it doesn't play too well with unit test conf.RTC.UDPPort = 0 conf.RTC.TCPPort = 0 - rtcConf, err := NewWebRTCConfig(conf, "") + rtcConf, err := NewWebRTCConfig(conf) if err != nil { panic(err) } diff --git a/pkg/rtc/rtc_unix.go b/pkg/rtc/rtc_unix.go deleted file mode 100644 index 3b357a21f..000000000 --- a/pkg/rtc/rtc_unix.go +++ /dev/null @@ -1,40 +0,0 @@ -//go:build !windows -// +build !windows - -package rtc - -import ( - "net" - "syscall" - - "github.com/livekit/protocol/logger" -) - -func checkUDPReadBuffer() { - val, err := getUDPReadBuffer() - if err == nil { - if val < minUDPBufferSize { - logger.Warnw("UDP receive buffer is too small for a production set-up", nil, - "current", val, - "suggested", minUDPBufferSize) - } else { - logger.Debugw("UDP receive buffer size", "current", val) - } - } -} - -func getUDPReadBuffer() (int, error) { - conn, err := net.ListenUDP("udp4", nil) - if err != nil { - return 0, err - } - defer func() { _ = conn.Close() }() - _ = conn.SetReadBuffer(defaultUDPBufferSize) - fd, err := conn.File() - if err != nil { - return 0, nil - } - defer func() { _ = fd.Close() }() - - return syscall.GetsockoptInt(int(fd.Fd()), syscall.SOL_SOCKET, syscall.SO_RCVBUF) -} diff --git a/pkg/rtc/rtc_windows.go b/pkg/rtc/rtc_windows.go deleted file mode 100644 index beeab22cf..000000000 --- a/pkg/rtc/rtc_windows.go +++ /dev/null @@ -1,7 +0,0 @@ -//go:build windows -// +build windows - -package rtc - -func checkUDPReadBuffer() { -} diff --git a/pkg/service/roommanager.go b/pkg/service/roommanager.go index c9986e92a..061446b00 100644 --- a/pkg/service/roommanager.go +++ b/pkg/service/roommanager.go @@ -11,6 +11,7 @@ import ( "github.com/livekit/livekit-server/pkg/telemetry/prometheus" "github.com/livekit/livekit-server/version" + "github.com/livekit/mediatransportutil/pkg/rtcconfig" "github.com/livekit/protocol/auth" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" @@ -67,7 +68,7 @@ func NewLocalRoomManager( egressLauncher rtc.EgressLauncher, versionGenerator utils.TimedVersionGenerator, ) (*RoomManager, error) { - rtcConf, err := rtc.NewWebRTCConfig(conf, currentNode.Ip) + rtcConf, err := rtc.NewWebRTCConfig(conf) if err != nil { return nil, err } @@ -687,7 +688,7 @@ func (r *RoomManager) iceServersForRoom(ri *livekit.Room, tlsOnly bool) []*livek } if !hasSTUN { - iceServers = append(iceServers, iceServerForStunServers(config.DefaultStunServers)) + iceServers = append(iceServers, iceServerForStunServers(rtcconfig.DefaultStunServers)) } return iceServers } diff --git a/test/client/client.go b/test/client/client.go index 76a347e87..384cdff79 100644 --- a/test/client/client.go +++ b/test/client/client.go @@ -19,6 +19,7 @@ import ( "go.uber.org/atomic" "google.golang.org/protobuf/proto" + "github.com/livekit/mediatransportutil/pkg/rtcconfig" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" @@ -120,7 +121,9 @@ func NewRTCClient(conn *websocket.Conn) (*RTCClient, error) { c.ctx, c.cancel = context.WithCancel(context.Background()) conf := rtc.WebRTCConfig{ - Configuration: rtcConf, + WebRTCConfig: rtcconfig.WebRTCConfig{ + Configuration: rtcConf, + }, } conf.SettingEngine.SetLite(false) conf.SettingEngine.SetAnsweringDTLSRole(webrtc.DTLSRoleClient) From a085afc6ee53cc006848e22681dfdab23e15a718 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 12 May 2023 09:44:03 +0530 Subject: [PATCH 160/324] Send quality stats to prometheus. (#1708) --- pkg/rtc/participant.go | 47 ++++++++++++++++++++++++++++- pkg/telemetry/prometheus/node.go | 1 + pkg/telemetry/prometheus/quality.go | 47 +++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 pkg/telemetry/prometheus/quality.go diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 9e37fe99c..96db0b3d0 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -26,6 +26,7 @@ import ( "github.com/livekit/livekit-server/pkg/sfu/connectionquality" "github.com/livekit/livekit-server/pkg/sfu/streamallocator" "github.com/livekit/livekit-server/pkg/telemetry" + "github.com/livekit/livekit-server/pkg/telemetry/prometheus" "github.com/livekit/mediatransportutil/pkg/twcc" "github.com/livekit/protocol/auth" "github.com/livekit/protocol/livekit" @@ -169,6 +170,8 @@ type ParticipantImpl struct { cachedDownTracks map[livekit.TrackID]*downTrackState supervisor *supervisor.ParticipantSupervisor + + tracksQuality map[livekit.TrackID]livekit.ConnectionQuality } func NewParticipant(params ParticipantParams) (*ParticipantImpl, error) { @@ -193,7 +196,8 @@ func NewParticipant(params ParticipantParams) (*ParticipantImpl, error) { telemetry.BytesTrackIDForParticipantID(telemetry.BytesTrackTypeData, params.SID), params.SID, params.Telemetry), - supervisor: supervisor.NewParticipantSupervisor(supervisor.ParticipantSupervisorParams{Logger: params.Logger}), + supervisor: supervisor.NewParticipantSupervisor(supervisor.ParticipantSupervisorParams{Logger: params.Logger}), + tracksQuality: make(map[livekit.TrackID]livekit.ConnectionQuality), } p.version.Store(params.InitialVersion) p.timedVersion.Update(params.VersionGenerator.New()) @@ -857,6 +861,10 @@ func (p *ParticipantImpl) GetConnectionQuality() *livekit.ConnectionQualityInfo numTracks := 0 minQuality := livekit.ConnectionQuality_EXCELLENT minScore := float32(0.0) + numUpDrops := 0 + numDownDrops := 0 + + availableTracks := make(map[livekit.TrackID]bool) for _, pt := range p.GetPublishedTracks() { numTracks++ @@ -869,6 +877,19 @@ func (p *ParticipantImpl) GetConnectionQuality() *livekit.ConnectionQualityInfo } else if quality == minQuality && score < minScore { minScore = score } + + p.lock.Lock() + trackID := pt.ID() + if prevQuality, ok := p.tracksQuality[trackID]; ok { + // WARNING NOTE: comparing protobuf enums directly + if prevQuality > quality { + numUpDrops++ + } + } + p.tracksQuality[trackID] = quality + p.lock.Unlock() + + availableTracks[trackID] = true } subscribedTracks := p.SubscriptionManager.GetSubscribedTracks() @@ -883,6 +904,19 @@ func (p *ParticipantImpl) GetConnectionQuality() *livekit.ConnectionQualityInfo } else if quality == minQuality && score < minScore { minScore = score } + + p.lock.Lock() + trackID := subTrack.ID() + if prevQuality, ok := p.tracksQuality[trackID]; ok { + // WARNING NOTE: comparing protobuf enums directly + if prevQuality > quality { + numDownDrops++ + } + } + p.tracksQuality[trackID] = quality + p.lock.Unlock() + + availableTracks[trackID] = true } if numTracks == 0 { @@ -890,6 +924,17 @@ func (p *ParticipantImpl) GetConnectionQuality() *livekit.ConnectionQualityInfo minScore = connectionquality.MaxMOS } + prometheus.RecordQuality(minQuality, minScore, numUpDrops, numDownDrops) + + // remove unavailable tracks from track quality cache + p.lock.Lock() + for trackID := range p.tracksQuality { + if !availableTracks[trackID] { + delete(p.tracksQuality, trackID) + } + } + p.lock.Unlock() + return &livekit.ConnectionQualityInfo{ ParticipantSid: string(p.ID()), Quality: minQuality, diff --git a/pkg/telemetry/prometheus/node.go b/pkg/telemetry/prometheus/node.go index 54e7f9d33..48b438e65 100644 --- a/pkg/telemetry/prometheus/node.go +++ b/pkg/telemetry/prometheus/node.go @@ -96,6 +96,7 @@ func Init(nodeID string, nodeType livekit.NodeType, env string) { initPacketStats(nodeID, nodeType, env) initRoomStats(nodeID, nodeType, env) initPSRPCStats(nodeID, nodeType, env) + initQualityStats(nodeID, nodeType, env) } func GetUpdatedNodeStats(prev *livekit.NodeStats, prevAverage *livekit.NodeStats) (*livekit.NodeStats, bool, error) { diff --git a/pkg/telemetry/prometheus/quality.go b/pkg/telemetry/prometheus/quality.go new file mode 100644 index 000000000..708c654f0 --- /dev/null +++ b/pkg/telemetry/prometheus/quality.go @@ -0,0 +1,47 @@ +package prometheus + +import ( + "github.com/prometheus/client_golang/prometheus" + + "github.com/livekit/protocol/livekit" +) + +var ( + qualityRating prometheus.Histogram + qualityScore prometheus.Histogram + qualityDrop *prometheus.CounterVec +) + +func initQualityStats(nodeID string, nodeType livekit.NodeType, env string) { + qualityRating = prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: livekitNamespace, + Subsystem: "quality", + Name: "rating", + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + Buckets: []float64{0, 1, 2}, + }) + qualityScore = prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: livekitNamespace, + Subsystem: "quality", + Name: "score", + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + Buckets: []float64{1.0, 2.0, 2.5, 3.0, 3.25, 3.5, 3.75, 4.0, 4.25, 4.5}, + }) + qualityDrop = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: livekitNamespace, + Subsystem: "quality", + Name: "drop", + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String(), "env": env}, + }, []string{"direction"}) + + prometheus.MustRegister(qualityRating) + prometheus.MustRegister(qualityScore) + prometheus.MustRegister(qualityDrop) +} + +func RecordQuality(rating livekit.ConnectionQuality, score float32, numUpDrops int, numDownDrops int) { + qualityRating.Observe(float64(rating)) + qualityScore.Observe(float64(score)) + qualityDrop.WithLabelValues("up").Add(float64(numUpDrops)) + qualityDrop.WithLabelValues("down").Add(float64(numDownDrops)) +} From 61102533ae530d7afe0f56d777891187cb413791 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sat, 13 May 2023 18:41:09 +0530 Subject: [PATCH 161/324] Monitor and log RTP time stmap drifts (#1710) The PID controller seems to be working well. But, it is unclear where it can be applied as some of the data shows significant jumps (either caused by BT devices or possibly noise cancellation/cpu constraint) and although PID controller is slowly pulling things to expected sample rate, it could be a bit slow. Unfortunately, cannot munge too much in a middle box. However leaving the controller in there as it is doing its job for cases where things slip slowly. Changing things to log significant jumps (more than 200 ms away from expected) at Infow level. Also, recording drift and sample rate in RTP stats proto and string representation. --- go.mod | 2 +- go.sum | 4 +- pkg/sfu/buffer/rtpstats.go | 172 +++++++++++++++++++++---------------- 3 files changed, 103 insertions(+), 75 deletions(-) diff --git a/go.mod b/go.mod index 8418df750..92a6a321a 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230511025422-058ebf6b48c9 - github.com/livekit/protocol v1.5.7-0.20230510002113-cadccd54108e + github.com/livekit/protocol v1.5.7-0.20230513090813-c5dc103838fd github.com/livekit/psrpc v0.3.1-0.20230502152150-df9dd21fba11 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.14.0 diff --git a/go.sum b/go.sum index c2fb05eda..1dd29b473 100644 --- a/go.sum +++ b/go.sum @@ -121,8 +121,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20230511025422-058ebf6b48c9 h1:aqivx5Tal2Fa6z1ZQrrBk/vYShosQx3ecl1aMwcQRV0= github.com/livekit/mediatransportutil v0.0.0-20230511025422-058ebf6b48c9/go.mod h1:MRc0zSOSzXuFt0X218SgabzlaKevkvCckPgBEoHYc34= -github.com/livekit/protocol v1.5.7-0.20230510002113-cadccd54108e h1:0F3RjOkUS71P2ODHI5ZW09Cbrr6A0Pg8vCFFy6gcqkk= -github.com/livekit/protocol v1.5.7-0.20230510002113-cadccd54108e/go.mod h1:vjGsR1YxXnN5BLS0yr/YjGnJOPrS0ymddCF3JwxSHGM= +github.com/livekit/protocol v1.5.7-0.20230513090813-c5dc103838fd h1:wK+Vp0Oa0oggHYKBymHRJFeDWFzrcMjmyuOw4TLzT7c= +github.com/livekit/protocol v1.5.7-0.20230513090813-c5dc103838fd/go.mod h1:vjGsR1YxXnN5BLS0yr/YjGnJOPrS0ymddCF3JwxSHGM= github.com/livekit/psrpc v0.3.1-0.20230502152150-df9dd21fba11 h1:VS23iVQu/TNiLEM5XjbBSY28+B6nSewjKWPDbieg0Ho= github.com/livekit/psrpc v0.3.1-0.20230502152150-df9dd21fba11/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index 7ba38e694..10e147628 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -3,6 +3,7 @@ package buffer import ( "errors" "fmt" + "math" "sync" "time" @@ -177,6 +178,7 @@ type RTPStats struct { srData *RTCPSenderReportData lastSRTime time.Time lastSRNTP mediatransportutil.NtpTime + lastSRRTP uint32 pidController *PIDController nextSnapshotId uint32 @@ -286,6 +288,7 @@ func (r *RTPStats) Seed(from *RTPStats) { } r.lastSRTime = from.lastSRTime r.lastSRNTP = from.lastSRNTP + r.lastSRRTP = from.lastSRRTP r.nextSnapshotId = from.nextSnapshotId for id, ss := range from.snapshots { @@ -732,7 +735,7 @@ func (r *RTPStats) SetRtcpSenderReportData(srData *RTCPSenderReportData) { // prevent against extreme case of anachronous sender reports if r.srData != nil && r.srData.NTPTimestamp > srData.NTPTimestamp { - r.logger.Debugw( + r.logger.Infow( "received anachronous sender report", "current", srData.NTPTimestamp.Time(), "last", r.srData.NTPTimestamp.Time(), @@ -740,39 +743,39 @@ func (r *RTPStats) SetRtcpSenderReportData(srData *RTCPSenderReportData) { return } - // TODO-REMOVE-AFTER-DEBUG-START - ntpTime := srData.NTPTimestamp.Time() - - var ntpDiffSinceLast, arrivalDiffSinceLast time.Duration - var rtpDiffSinceLast uint32 + // monitor and log RTP timestamp anomalies + isWarped := false if r.srData != nil { - ntpDiffSinceLast = ntpTime.Sub(r.srData.NTPTimestamp.Time()) - rtpDiffSinceLast = srData.RTPTimestamp - r.srData.RTPTimestamp - arrivalDiffSinceLast = srData.ArrivalTime.Sub(r.srData.ArrivalTime) + ntpDiffSinceLast := srData.NTPTimestamp.Time().Sub(r.srData.NTPTimestamp.Time()) + rtpDiffSinceLast := srData.RTPTimestamp - r.srData.RTPTimestamp + arrivalDiffSinceLast := srData.ArrivalTime.Sub(r.srData.ArrivalTime) + + expectedTimeDiffSinceLast := float64(rtpDiffSinceLast) / float64(r.params.ClockRate) + if math.Abs(expectedTimeDiffSinceLast-ntpDiffSinceLast.Seconds()) > 0.2 { + // more than 200 ms away from expected delta + isWarped = true + } + + if isWarped { + timeSinceFirst, rtpDiffSinceFirst, drift, driftMs, sampleRate := r.getDrift() + r.logger.Infow( + "received sender report, time warp", + "ntp", srData.NTPTimestamp.Time().String(), + "rtp", srData.RTPTimestamp, + "arrival", srData.ArrivalTime.String(), + "ntpDiffSincelast", ntpDiffSinceLast.Seconds(), + "rtpDiffSincelast", rtpDiffSinceLast, + "arrivalDiffSincelast", arrivalDiffSinceLast.Seconds(), + "expectedTimeDiffSincelast", expectedTimeDiffSinceLast, + "timeSinceFirst", timeSinceFirst.Seconds(), + "rtpDiffSinceFirst", rtpDiffSinceFirst, + "drift", drift, + "driftMs", driftMs, + "sampleRate", sampleRate, + ) + } } - timeSinceFirst := time.Since(r.firstTime) // ideally should use NTP time from SR, but that is a different time base, now is a resonable approximation - rtpDiffSinceFirst := getExtTS(srData.RTPTimestamp, r.tsCycles) - r.extStartTS - drift := int64(rtpDiffSinceFirst - uint64(timeSinceFirst.Nanoseconds()*int64(r.params.ClockRate)/1e9)) - driftMs := (float64(drift) * 1000) / float64(r.params.ClockRate) - - r.logger.Debugw( - "received sender report", - "ntp", ntpTime, - "rtp", srData.RTPTimestamp, - "arrival", srData.ArrivalTime, - "ntpDiff", ntpDiffSinceLast, - "rtpDiff", rtpDiffSinceLast, - "arrivalDiff", arrivalDiffSinceLast, - "expectedTimeDiff", float64(rtpDiffSinceLast)/float64(r.params.ClockRate), - "timeSinceFirst", timeSinceFirst, - "rtpDiffSinceFirst", rtpDiffSinceFirst, - "drift", drift, - "driftMs", driftMs, - "rate", float64(rtpDiffSinceFirst)/timeSinceFirst.Seconds(), - ) - // TODO-REMOVE-AFTER-DEBUG-END - srDataCopy := *srData r.srData = &srDataCopy } @@ -828,24 +831,9 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32) (*rtcp.SenderReport, float64 now := r.firstTime.Add(timeSinceFirst) nowNTP := mediatransportutil.ToNtpTime(now) - expectedExtRTP := r.extStartTS + uint64(timeSinceFirst.Nanoseconds()*int64(r.params.ClockRate)/1e9) - if getExtTS(r.highestTS, r.tsCycles) > expectedExtRTP || now.Before(r.highestTime) { - r.logger.Debugw( - "sending anachronous sender report", - "firstTime", r.firstTime.String(), - "currentTime", now.String(), - "highestTime", r.highestTime.String(), - "timeSinceFirst", timeSinceFirst, - "extStartTS", r.extStartTS, - "highestExtRTP", getExtTS(r.highestTS, r.tsCycles), - "expectedExtRTP", expectedExtRTP, - ) - } - - timeSinceHighest := time.Since(r.highestTime) + timeSinceHighest := now.Sub(r.highestTime) nowRTP := r.highestTS + uint32(timeSinceHighest.Nanoseconds()*int64(r.params.ClockRate)/1e9) - // TODO-REMOVE-AFTER-DEBUG-START rtpDiffSinceFirst := getExtTS(nowRTP, r.tsCycles) - r.extStartTS rate := float64(rtpDiffSinceFirst) / timeSinceFirst.Seconds() pidOutput := r.pidController.Update( @@ -853,37 +841,51 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32) (*rtcp.SenderReport, float64 rate, now, ) - // TODO-REMOVE-AFTER-DEBUG-STOP - // TODO-REMOVE-AFTER-DEBUG-START - ntpTime := nowNTP.Time() + // monitor and log RTP timestamp anomalies + isWarped := false + if r.lastSRNTP != 0 { + ntpDiffSinceLast := nowNTP.Time().Sub(r.lastSRNTP.Time()) + rtpDiffSinceLast := nowRTP - r.lastSRRTP + departureDiffSinceLast := now.Sub(r.lastSRTime) - ntpDiffLocal := ntpTime.Sub(r.highestTime) - rtpDiffLocal := int32(nowRTP - r.highestTS) - rtpOffsetLocal := int32(nowRTP - r.highestTS - uint32(ntpDiffLocal.Nanoseconds()*int64(r.params.ClockRate)/1e9)) + expectedTimeDiffSinceLast := float64(rtpDiffSinceLast) / float64(r.params.ClockRate) + if math.Abs(expectedTimeDiffSinceLast-ntpDiffSinceLast.Seconds()) > 0.2 { + // more than 200 ms away from expected delta + isWarped = true + } - drift := int64(rtpDiffSinceFirst - uint64(timeSinceFirst.Nanoseconds()*int64(r.params.ClockRate)/1e9)) - driftMs := (float64(drift) * 1000) / float64(r.params.ClockRate) - r.logger.Debugw( - "sending sender report", - "highestTS", r.highestTS, - "highestTime", r.highestTime.String(), - "reportTS", nowRTP, - "expectedTS", uint32(expectedExtRTP), - "reportTime", ntpTime.String(), - "rtpDiffLocal", rtpDiffLocal, - "ntpDiffLocal", ntpDiffLocal, - "rtpOffsetLocal", rtpOffsetLocal, - "timeSinceFirst", timeSinceFirst, - "rtpDiffSinceFirst", rtpDiffSinceFirst, - "drift", drift, - "driftMs", driftMs, - "rate", rate, - ) - // TODO-REMOVE-AFTER-DEBUG-END + if isWarped { + expectedExtRTP := r.extStartTS + uint64(timeSinceFirst.Nanoseconds()*int64(r.params.ClockRate)/1e9) + ntpDiffLocal := nowNTP.Time().Sub(r.highestTime) + rtpDiffLocal := int32(nowRTP - r.highestTS) + timeSinceFirst, rtpDiffSinceFirst, drift, driftMs, sampleRate := r.getDrift() + r.logger.Infow( + "sending sender report, time warp", + "ntp", nowNTP.Time().String(), + "rtp", nowRTP, + "expectedRTP", uint32(expectedExtRTP), + "departure", now.String(), + "ntpDiffSincelast", ntpDiffSinceLast.Seconds(), + "rtpDiffSincelast", rtpDiffSinceLast, + "departureDiffSincelast", departureDiffSinceLast.Seconds(), + "expectedTimeDiffSincelast", expectedTimeDiffSinceLast, + "timeSinceFirst", timeSinceFirst.Seconds(), + "rtpDiffSinceFirst", rtpDiffSinceFirst, + "drift", drift, + "driftMs", driftMs, + "sampleRate", sampleRate, + "highestTS", r.highestTS, + "highestTime", r.highestTime.String(), + "rtpDiffLocal", rtpDiffLocal, + "ntpDiffLocal", ntpDiffLocal, + ) + } + } r.lastSRTime = now r.lastSRNTP = nowNTP + r.lastSRRTP = nowRTP return &rtcp.SenderReport{ SSRC: ssrc, @@ -1152,6 +1154,12 @@ func (r *RTPStats) ToString() string { str += ", rtt(ms):" str += fmt.Sprintf("%d|%d", p.RttCurrent, p.RttMax) + str += ", drift(ms):" + str += fmt.Sprintf("%.2f", p.DriftMs) + + str += ", sr(Hz):" + str += fmt.Sprintf("%.2f", p.SampleRate) + return str } @@ -1199,6 +1207,8 @@ func (r *RTPStats) ToProto() *livekit.RTPStats { jitterTime := jitter / float64(r.params.ClockRate) * 1e6 maxJitterTime := maxJitter / float64(r.params.ClockRate) * 1e6 + _, _, _, driftMs, sampleRate := r.getDrift() + p := &livekit.RTPStats{ StartTime: timestamppb.New(r.startTime), EndTime: timestamppb.New(endTime), @@ -1240,6 +1250,8 @@ func (r *RTPStats) ToProto() *livekit.RTPStats { LastFir: timestamppb.New(r.lastFir), RttCurrent: r.rtt, RttMax: r.maxRtt, + DriftMs: driftMs, + SampleRate: sampleRate, } gapsPresent := false @@ -1433,6 +1445,15 @@ func (r *RTPStats) updateJitter(rtph *rtp.Header, packetTime time.Time) { r.lastJitterRTP = rtph.Timestamp } +func (r *RTPStats) getDrift() (timeSinceFirst time.Duration, rtpDiffSinceFirst uint64, drift int64, driftMs float64, sampleRate float64) { + timeSinceFirst = r.highestTime.Sub(r.firstTime) + rtpDiffSinceFirst = getExtTS(r.highestTS, r.tsCycles) - r.extStartTS + drift = int64(rtpDiffSinceFirst - uint64(timeSinceFirst.Nanoseconds()*int64(r.params.ClockRate)/1e9)) + driftMs = (float64(drift) * 1000) / float64(r.params.ClockRate) + sampleRate = float64(rtpDiffSinceFirst) / timeSinceFirst.Seconds() + return +} + func (r *RTPStats) updateGapHistogram(gap int) { if gap < 2 { return @@ -1533,6 +1554,8 @@ func AggregateRTPStats(statsList []*livekit.RTPStats) *livekit.RTPStats { lastFir := time.Time{} rtt := uint32(0) maxRtt := uint32(0) + driftMs := float64(0.0) + sampleRate := float64(0.0) for _, stats := range statsList { if startTime.IsZero() || startTime.After(stats.StartTime.AsTime()) { @@ -1599,6 +1622,9 @@ func AggregateRTPStats(statsList []*livekit.RTPStats) *livekit.RTPStats { if stats.RttMax > maxRtt { maxRtt = stats.RttMax } + + driftMs += stats.DriftMs + sampleRate += stats.SampleRate } if endTime.IsZero() { @@ -1661,6 +1687,8 @@ func AggregateRTPStats(statsList []*livekit.RTPStats) *livekit.RTPStats { LastFir: timestamppb.New(lastFir), RttCurrent: rtt / uint32(len(statsList)), RttMax: maxRtt, + DriftMs: driftMs / float64(len(statsList)), + SampleRate: sampleRate / float64(len(statsList)), } } From c79e0ce06f1d2f0c6bddacd8782701dfaf19641d Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 16 May 2023 14:08:17 +0530 Subject: [PATCH 162/324] Make signal close async. (#1711) * Make signal close async. Left notes about async close in code. Also reducing retry config timeout - Timeout to 7.5 seconds (making it 1/4th of current config) - max retry to 4 seconds - so, it can do 4 tries now in 7.5 seconds (with retries ending at 0.5 seconds, 1.5 seconds, 3.5 seconds, 7.5 seconds). The change of max to 4 seconds is not really needed, but it lined up with 7.5. So, made the change. * update comments a bit --- pkg/config/config.go | 4 ++-- pkg/routing/signal.go | 17 ++++++++++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index a9f61aad4..876e74773 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -375,9 +375,9 @@ func NewConfig(confString string, strictMode bool, c *cli.Context, baseFlags []c }, SignalRelay: SignalRelayConfig{ Enabled: false, - RetryTimeout: 30 * time.Second, + RetryTimeout: 7500 * time.Millisecond, MinRetryInterval: 500 * time.Millisecond, - MaxRetryInterval: 5 * time.Second, + MaxRetryInterval: 4 * time.Second, StreamBufferSize: 1000, }, Keys: map[string]string{}, diff --git a/pkg/routing/signal.go b/pkg/routing/signal.go index 56aa25005..af8392e48 100644 --- a/pkg/routing/signal.go +++ b/pkg/routing/signal.go @@ -255,7 +255,22 @@ func (s *signalMessageSink[SendType, RecvType]) Close() { } s.mu.Unlock() - <-s.Stream.Context().Done() + // NOTE: not waiting for stream context to be done. + // Waiting for stream context to be done is confirmation + // that the close message has been processed by the other side. + // In ideal conditions, waiting for it is a clean end. + // + // But, in cases where the remote side goes away abruptly, waiting + // for stream context to be done could block connection progress + // till the timeout hits. + // + // The abrupt case happens when one side of the signal + // relay is shut down due to scale down or a crash. + // + // Uncomment the following line to wait for close acknowledgement, + // but the system should be able to wait long enough (till timeout) + // without adverse impact if waiting for close acknowledgement. + //<-s.Stream.Context().Done() } func (s *signalMessageSink[SendType, RecvType]) IsClosed() bool { From 9395f0b1fb5645d41100919ab3a47eba68b7b428 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 16 May 2023 21:48:10 +0530 Subject: [PATCH 163/324] More time stamp dance. (#1712) Two things - Somehow the publisher RTCP sender report time stamp goes back some times. Log it differently. Also, use signed type for logging so that negative is easy to see. - On down track, because of silence frame injection on mute, the RTCP sender report time stamp might be ahead of timestamp we will use on unmute. If so, ensure that next timestamp is also not before what was sent in RTCP sender report. --- pkg/sfu/buffer/rtpstats.go | 39 +++++++++++++++++++++----------------- pkg/sfu/downtrack.go | 2 +- pkg/sfu/forwarder.go | 23 +++++++++++++++------- 3 files changed, 39 insertions(+), 25 deletions(-) diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index 10e147628..0667f8464 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -744,29 +744,34 @@ func (r *RTPStats) SetRtcpSenderReportData(srData *RTCPSenderReportData) { } // monitor and log RTP timestamp anomalies - isWarped := false if r.srData != nil { ntpDiffSinceLast := srData.NTPTimestamp.Time().Sub(r.srData.NTPTimestamp.Time()) rtpDiffSinceLast := srData.RTPTimestamp - r.srData.RTPTimestamp arrivalDiffSinceLast := srData.ArrivalTime.Sub(r.srData.ArrivalTime) expectedTimeDiffSinceLast := float64(rtpDiffSinceLast) / float64(r.params.ClockRate) - if math.Abs(expectedTimeDiffSinceLast-ntpDiffSinceLast.Seconds()) > 0.2 { - // more than 200 ms away from expected delta - isWarped = true + + var reason string + if (srData.RTPTimestamp - r.srData.RTPTimestamp) > (1 << 31) { + reason = "received sender report, out-of-order" + } else { + if math.Abs(expectedTimeDiffSinceLast-ntpDiffSinceLast.Seconds()) > 0.2 { + // more than 200 ms away from expected delta + reason = "received sender report, time warp" + } } - if isWarped { + if reason != "" { timeSinceFirst, rtpDiffSinceFirst, drift, driftMs, sampleRate := r.getDrift() r.logger.Infow( - "received sender report, time warp", + reason, "ntp", srData.NTPTimestamp.Time().String(), "rtp", srData.RTPTimestamp, "arrival", srData.ArrivalTime.String(), - "ntpDiffSincelast", ntpDiffSinceLast.Seconds(), - "rtpDiffSincelast", rtpDiffSinceLast, - "arrivalDiffSincelast", arrivalDiffSinceLast.Seconds(), - "expectedTimeDiffSincelast", expectedTimeDiffSinceLast, + "ntpDiffSinceLast", ntpDiffSinceLast.Seconds(), + "rtpDiffSinceLast", int32(rtpDiffSinceLast), + "arrivalDiffSinceLast", arrivalDiffSinceLast.Seconds(), + "expectedTimeDiffSinceLast", expectedTimeDiffSinceLast, "timeSinceFirst", timeSinceFirst.Seconds(), "rtpDiffSinceFirst", rtpDiffSinceFirst, "drift", drift, @@ -792,12 +797,12 @@ func (r *RTPStats) GetRtcpSenderReportData() *RTCPSenderReportData { return &srDataCopy } -func (r *RTPStats) GetExpectedRTPTimestamp(at time.Time) (uint32, error) { +func (r *RTPStats) GetExpectedRTPTimestamp(at time.Time) (uint32, uint32, error) { r.lock.RLock() defer r.lock.RUnlock() if !r.initialized { - return 0, errors.New("uninitilaized") + return 0, 0, errors.New("uninitilaized") } timeDiff := at.Sub(r.firstTime) @@ -815,7 +820,7 @@ func (r *RTPStats) GetExpectedRTPTimestamp(at time.Time) (uint32, error) { "highestTS", r.highestTS, "highestTime", r.highestTime.String(), ) - return uint32(expectedExtRTP), nil + return uint32(expectedExtRTP), r.lastSRRTP, nil } func (r *RTPStats) GetRtcpSenderReport(ssrc uint32) (*rtcp.SenderReport, float64) { @@ -866,10 +871,10 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32) (*rtcp.SenderReport, float64 "rtp", nowRTP, "expectedRTP", uint32(expectedExtRTP), "departure", now.String(), - "ntpDiffSincelast", ntpDiffSinceLast.Seconds(), - "rtpDiffSincelast", rtpDiffSinceLast, - "departureDiffSincelast", departureDiffSinceLast.Seconds(), - "expectedTimeDiffSincelast", expectedTimeDiffSinceLast, + "ntpDiffSinceLast", ntpDiffSinceLast.Seconds(), + "rtpDiffSinceLast", rtpDiffSinceLast, + "departureDiffSinceLast", departureDiffSinceLast.Seconds(), + "expectedTimeDiffSinceLast", expectedTimeDiffSinceLast, "timeSinceFirst", timeSinceFirst.Seconds(), "rtpDiffSinceFirst", rtpDiffSinceFirst, "drift", drift, diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index e9bca290f..6769dec9e 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -1637,7 +1637,7 @@ func (d *DownTrack) DebugInfo() map[string]interface{} { } } -func (d *DownTrack) getExpectedRTPTimestamp(at time.Time) (uint32, error) { +func (d *DownTrack) getExpectedRTPTimestamp(at time.Time) (uint32, uint32, error) { return d.rtpStats.GetExpectedRTPTimestamp(at) } diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 312689133..ebede6e0e 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -170,7 +170,7 @@ type Forwarder struct { kind webrtc.RTPCodecType logger logger.Logger getReferenceLayerRTPTimestamp func(ts uint32, layer int32, referenceLayer int32) (uint32, error) - getExpectedRTPTimestamp func(at time.Time) (uint32, error) + getExpectedRTPTimestamp func(at time.Time) (uint32, uint32, error) muted bool pubMuted bool @@ -201,7 +201,7 @@ func NewForwarder( kind webrtc.RTPCodecType, logger logger.Logger, getReferenceLayerRTPTimestamp func(ts uint32, layer int32, referenceLayer int32) (uint32, error), - getExpectedRTPTimestamp func(at time.Time) (uint32, error), + getExpectedRTPTimestamp func(at time.Time) (uint32, uint32, error), ) *Forwarder { f := &Forwarder{ kind: kind, @@ -1475,6 +1475,7 @@ func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer i lastTS := f.rtpMunger.GetLast().LastTS refTS := lastTS expectedTS := lastTS + minTS := lastTS switchingAt := time.Now() if f.getReferenceLayerRTPTimestamp != nil { ts, err := f.getReferenceLayerRTPTimestamp(extPkt.Packet.Timestamp, layer, f.referenceLayerSpatial) @@ -1483,9 +1484,10 @@ func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer i } } if f.getExpectedRTPTimestamp != nil { - ts, err := f.getExpectedRTPTimestamp(switchingAt) + ts, min, err := f.getExpectedRTPTimestamp(switchingAt) if err == nil { expectedTS = ts + minTS = min } else { rtpDiff := uint32(0) if !f.preStartTime.IsZero() && f.refTSOffset == 0 { @@ -1497,13 +1499,14 @@ func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer i } } refTS += f.refTSOffset - nextTS, explain := getNextTimestamp(lastTS, refTS, expectedTS) + nextTS, explain := getNextTimestamp(lastTS, refTS, expectedTS, minTS) f.logger.Debugw( "next timestamp on switch", "switchingAt", switchingAt.String(), "lastTS", lastTS, "refTS", refTS, "expectedTS", expectedTS, + "minTS", minTS, "nextTS", nextTS, "jump", nextTS-lastTS, "explanation", explain, @@ -1668,13 +1671,15 @@ func (f *Forwarder) GetSnTsForBlankFrames(frameRate uint32, numPackets int) ([]S lastTS := f.rtpMunger.GetLast().LastTS expectedTS := lastTS + minTS := lastTS if f.getExpectedRTPTimestamp != nil { - ts, err := f.getExpectedRTPTimestamp(time.Now()) + ts, min, err := f.getExpectedRTPTimestamp(time.Now()) if err == nil { expectedTS = ts + minTS = min } } - nextTS, _ := getNextTimestamp(lastTS, expectedTS, expectedTS) + nextTS, _ := getNextTimestamp(lastTS, expectedTS, expectedTS, minTS) snts, err := f.rtpMunger.UpdateAndGetPaddingSnTs(numPackets, f.codec.ClockRate, frameRate, frameEndNeeded, nextTS) return snts, frameEndNeeded, err } @@ -1821,7 +1826,7 @@ done: return float64(distance) / float64(maxSeenLayer.Temporal+1) } -func getNextTimestamp(lastTS uint32, refTS uint32, expectedTS uint32) (uint32, string) { +func getNextTimestamp(lastTS uint32, refTS uint32, expectedTS uint32, minTS uint32) (uint32, string) { isInOrder := func(val1, val2 uint32) bool { diff := val1 - val2 return diff != 0 && diff < (1<<31) @@ -1855,5 +1860,9 @@ func getNextTimestamp(lastTS uint32, refTS uint32, expectedTS uint32) (uint32, s explain = fmt.Sprintf("e < r < l, %d, %d", refTS-expectedTS, lastTS-refTS) } + if !isInOrder(nextTS, minTS) { + nextTS = minTS + 1 + } + return nextTS, explain } From 1c88a0336674725c660b62d2b2caba2d094a3994 Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Wed, 17 May 2023 18:48:54 +0800 Subject: [PATCH 164/324] Don't add nack if it is already present in track codec (#1714) --- pkg/rtc/transport.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index 0b438f4f3..a2e88f9f2 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -1958,7 +1958,16 @@ func configureAudioTransceiver(tr *webrtc.RTPTransceiver, stereo bool, nack bool c.SDPFmtpLine += ";sprop-stereo=1" } if nack { - c.RTCPFeedback = append(c.RTCPFeedback, webrtc.RTCPFeedback{Type: webrtc.TypeRTCPFBNACK}) + var nackFound bool + for _, fb := range c.RTCPFeedback { + if fb.Type == webrtc.TypeRTCPFBNACK { + nackFound = true + break + } + } + if !nackFound { + c.RTCPFeedback = append(c.RTCPFeedback, webrtc.RTCPFeedback{Type: webrtc.TypeRTCPFBNACK}) + } } } configCodecs = append(configCodecs, c) From f401c44a46db3202e9bb91fb2357b1964a4de22e Mon Sep 17 00:00:00 2001 From: Benjamin Pracht Date: Wed, 17 May 2023 15:24:17 -0700 Subject: [PATCH 165/324] Move TURNServers back to livekit-server (#1715) --- go.mod | 2 +- go.sum | 4 ++-- pkg/config/config.go | 10 ++++++++++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 92a6a321a..970675c27 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.2 github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 - github.com/livekit/mediatransportutil v0.0.0-20230511025422-058ebf6b48c9 + github.com/livekit/mediatransportutil v0.0.0-20230517210015-117bec6a19a8 github.com/livekit/protocol v1.5.7-0.20230513090813-c5dc103838fd github.com/livekit/psrpc v0.3.1-0.20230502152150-df9dd21fba11 github.com/mackerelio/go-osstat v0.2.4 diff --git a/go.sum b/go.sum index 1dd29b473..10e4d5b01 100644 --- a/go.sum +++ b/go.sum @@ -119,8 +119,8 @@ github.com/lithammer/shortuuid/v4 v4.0.0 h1:QRbbVkfgNippHOS8PXDkti4NaWeyYfcBTHtw github.com/lithammer/shortuuid/v4 v4.0.0/go.mod h1:Zs8puNcrvf2rV9rTH51ZLLcj7ZXqQI3lv67aw4KiB1Y= github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkDaKb5iXdynYrzB84ErPPO4LbRASk58= github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= -github.com/livekit/mediatransportutil v0.0.0-20230511025422-058ebf6b48c9 h1:aqivx5Tal2Fa6z1ZQrrBk/vYShosQx3ecl1aMwcQRV0= -github.com/livekit/mediatransportutil v0.0.0-20230511025422-058ebf6b48c9/go.mod h1:MRc0zSOSzXuFt0X218SgabzlaKevkvCckPgBEoHYc34= +github.com/livekit/mediatransportutil v0.0.0-20230517210015-117bec6a19a8 h1:YgBDljjYPJc57sSwaoyUgiviThQDyS7SyWsXJSRsZH8= +github.com/livekit/mediatransportutil v0.0.0-20230517210015-117bec6a19a8/go.mod h1:MRc0zSOSzXuFt0X218SgabzlaKevkvCckPgBEoHYc34= github.com/livekit/protocol v1.5.7-0.20230513090813-c5dc103838fd h1:wK+Vp0Oa0oggHYKBymHRJFeDWFzrcMjmyuOw4TLzT7c= github.com/livekit/protocol v1.5.7-0.20230513090813-c5dc103838fd/go.mod h1:vjGsR1YxXnN5BLS0yr/YjGnJOPrS0ymddCF3JwxSHGM= github.com/livekit/psrpc v0.3.1-0.20230502152150-df9dd21fba11 h1:VS23iVQu/TNiLEM5XjbBSY28+B6nSewjKWPDbieg0Ho= diff --git a/pkg/config/config.go b/pkg/config/config.go index 876e74773..86138a0d9 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -69,6 +69,8 @@ type Config struct { type RTCConfig struct { rtcconfig.RTCConfig `yaml:",inline"` + TURNServers []TURNServer `yaml:"turn_servers,omitempty"` + StrictACKs bool `yaml:"strict_acks,omitempty"` // Number of packets to buffer for NACK @@ -92,6 +94,14 @@ type RTCConfig struct { AllowTimestampAdjustment *bool `yaml:"allow_timestamp_adjustment,omitempty"` } +type TURNServer struct { + Host string `yaml:"host"` + Port int `yaml:"port"` + Protocol string `yaml:"protocol"` + Username string `yaml:"username,omitempty"` + Credential string `yaml:"credential,omitempty"` +} + type PLIThrottleConfig struct { LowQuality time.Duration `yaml:"low_quality,omitempty"` MidQuality time.Duration `yaml:"mid_quality,omitempty"` From c3d6ecca6e16d442a1b7187d23ed8427218f094d Mon Sep 17 00:00:00 2001 From: David Colburn Date: Wed, 17 May 2023 16:46:22 -0700 Subject: [PATCH 166/324] check egress status on UpdateStream failure (#1716) --- pkg/service/egress.go | 38 +++++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/pkg/service/egress.go b/pkg/service/egress.go index 99e232b96..a05444077 100644 --- a/pkg/service/egress.go +++ b/pkg/service/egress.go @@ -239,7 +239,20 @@ func (s *EgressService) UpdateStream(ctx context.Context, req *livekit.UpdateStr info, err := s.client.UpdateStream(ctx, req.EgressId, req) if err != nil { - return nil, err + var loadErr error + info, loadErr = s.es.LoadEgress(ctx, req.EgressId) + if loadErr != nil { + return nil, loadErr + } + + switch info.Status { + case livekit.EgressStatus_EGRESS_STARTING, + livekit.EgressStatus_EGRESS_ACTIVE: + return nil, err + default: + return nil, twirp.NewError(twirp.FailedPrecondition, + fmt.Sprintf("egress with status %s cannot be updated", info.Status.String())) + } } go func() { @@ -293,19 +306,22 @@ func (s *EgressService) StopEgress(ctx context.Context, req *livekit.StopEgressR return nil, ErrEgressNotConnected } - info, err := s.es.LoadEgress(ctx, req.EgressId) + info, err := s.client.StopEgress(ctx, req.EgressId, req) if err != nil { - return nil, err - } else { - if info.Status != livekit.EgressStatus_EGRESS_STARTING && - info.Status != livekit.EgressStatus_EGRESS_ACTIVE { - return nil, twirp.NewError(twirp.FailedPrecondition, fmt.Sprintf("egress with status %s cannot be stopped", info.Status.String())) + var loadErr error + info, loadErr = s.es.LoadEgress(ctx, req.EgressId) + if loadErr != nil { + return nil, loadErr } - } - info, err = s.client.StopEgress(ctx, req.EgressId, req) - if err != nil { - return nil, err + switch info.Status { + case livekit.EgressStatus_EGRESS_STARTING, + livekit.EgressStatus_EGRESS_ACTIVE: + return nil, err + default: + return nil, twirp.NewError(twirp.FailedPrecondition, + fmt.Sprintf("egress with status %s cannot be stopped", info.Status.String())) + } } go func() { From 1d3faefc5eb91caab0524e10ea1584b48a31a7a2 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 18 May 2023 20:16:43 +0530 Subject: [PATCH 167/324] More scoring tweaks (#1719) 1. Completely removing RTT and jitter from score calculation. Need to do more work there. a. Jitter is slow moving (RFC 3550 formula is designed that way). But, we still get high values at times. Ideally, that should penalise the score, but due to jitter buffer, effect may not be too bad. b. Need to smooth RTT. It is based on receiver report and if one sample causes a high number, score could be penalised (this was being used in down track direction only). One option is to smooth it like the jitter formula above and try using it. But, for now, disabling that also. 2. When receiving lesser number of packets (for example DTX), reduce the weight of packet loss with a quadratic relationship to packet loss ratio. Previously using a square root and it was potentially weighting it too high. For example, if only 5 packets were received due to DTX instead of 50, we were still giving 30% weight (sqrt(0.1)). Now, it gets 1% weight. So, if one of those 5 packets were lost (20% packet loss ratio), it still does not get much weight as the number of packets is low., 3. Slightly slower decrease in score (in EWMA) 4. When using RED, increase packet loss weight thresholds to be able to take more loss before penalizing score. --- pkg/sfu/buffer/rtpstats.go | 26 +++-- pkg/sfu/connectionquality/connectionstats.go | 18 +-- .../connectionquality/connectionstats_test.go | 108 +++++++++--------- pkg/sfu/connectionquality/scorer.go | 23 ++-- pkg/sfu/downtrack.go | 1 - pkg/sfu/receiver.go | 9 +- 6 files changed, 93 insertions(+), 92 deletions(-) diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index 0667f8464..873c8d59d 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -1896,18 +1896,20 @@ func (p *PIDController) Update(setpoint, measurement float64, at time.Time) floa p.prevError = errorTerm p.prevMeasurement = measurement p.prevMeasurementTime = at - p.logger.Debugw( - "pid controller", - "setpoint", setpoint, - "measurement", measurement, - "errorTerm", errorTerm, - "proportional", proportional, - "integral", iVal, - "integralLimited", boundIVal, - "derivative", p.dVal, - "output", output, - "outputLimited", boundOutput, - ) + /* + p.logger.Debugw( + "pid controller", + "setpoint", setpoint, + "measurement", measurement, + "errorTerm", errorTerm, + "proportional", proportional, + "integral", iVal, + "integralLimited", boundIVal, + "derivative", p.dVal, + "output", output, + "outputLimited", boundOutput, + ) + */ return boundOutput } diff --git a/pkg/sfu/connectionquality/connectionstats.go b/pkg/sfu/connectionquality/connectionstats.go index 0da29996a..9cb5c716c 100644 --- a/pkg/sfu/connectionquality/connectionstats.go +++ b/pkg/sfu/connectionquality/connectionstats.go @@ -25,8 +25,8 @@ type ConnectionStatsParams struct { UpdateInterval time.Duration MimeType string IsFECEnabled bool - IsDependentRTT bool - IsDependentJitter bool + IncludeRTT bool + IncludeJitter bool GetDeltaStats func() map[uint32]*buffer.StreamStatsWithLayers GetDeltaStatsOverridden func() map[uint32]*buffer.StreamStatsWithLayers GetLastReceiverReportTime func() time.Time @@ -53,10 +53,10 @@ func NewConnectionStats(params ConnectionStatsParams) *ConnectionStats { return &ConnectionStats{ params: params, scorer: newQualityScorer(qualityScorerParams{ - PacketLossWeight: getPacketLossWeight(params.MimeType, params.IsFECEnabled), // LK-TODO: have to notify codec change? - IsDependentRTT: params.IsDependentRTT, - IsDependentJitter: params.IsDependentJitter, - Logger: params.Logger, + PacketLossWeight: getPacketLossWeight(params.MimeType, params.IsFECEnabled), // LK-TODO: have to notify codec change? + IncludeRTT: params.IncludeRTT, + IncludeJitter: params.IncludeJitter, + Logger: params.Logger, }), done: core.NewFuse(), } @@ -293,10 +293,10 @@ func getPacketLossWeight(mimeType string, isFecEnabled bool) float64 { } case strings.EqualFold(mimeType, "audio/red"): - // 6.66%: fall to GOOD, 20.0%: fall to POOR - plw = 3.0 + // 10%: fall to GOOD, 30.0%: fall to POOR + plw = 2.0 if isFecEnabled { - // 10%: fall to GOOD, 30.0%: fall to POOR + // 15%: fall to GOOD, 45.0%: fall to POOR plw /= 1.5 } diff --git a/pkg/sfu/connectionquality/connectionstats_test.go b/pkg/sfu/connectionquality/connectionstats_test.go index 2ed5f3789..57793ef85 100644 --- a/pkg/sfu/connectionquality/connectionstats_test.go +++ b/pkg/sfu/connectionquality/connectionstats_test.go @@ -12,19 +12,19 @@ import ( "github.com/livekit/protocol/logger" ) -func newConnectionStats(mimeType string, isFECEnabled bool, isDependentRTT bool, isDependentJitter bool) *ConnectionStats { +func newConnectionStats(mimeType string, isFECEnabled bool, includeRTT bool, includeJitter bool) *ConnectionStats { return NewConnectionStats(ConnectionStatsParams{ - MimeType: mimeType, - IsFECEnabled: isFECEnabled, - IsDependentRTT: isDependentRTT, - IsDependentJitter: isDependentJitter, - Logger: logger.GetLogger(), + MimeType: mimeType, + IsFECEnabled: isFECEnabled, + IncludeRTT: includeRTT, + IncludeJitter: includeJitter, + Logger: logger.GetLogger(), }) } func TestConnectionQuality(t *testing.T) { t.Run("quality scorer state machine", func(t *testing.T) { - cs := newConnectionStats("audio/opus", false, false, false) + cs := newConnectionStats("audio/opus", false, true, true) duration := 5 * time.Second now := time.Now() @@ -184,7 +184,7 @@ func TestConnectionQuality(t *testing.T) { StartTime: now, Duration: duration, Packets: 250, - PacketsLost: 25, + PacketsLost: 30, }, }, } @@ -239,7 +239,7 @@ func TestConnectionQuality(t *testing.T) { cs.UpdateMute(false, now.Add(2*time.Second)) // with lesser number of packet (simulating DTX). - // even higher loss (like 10%) should only knock down quality to GOOD, typically would be POOR at that loss rate + // even higher loss (like 10%) should not knock down quality due to quadratic weighting of packet loss ratio streams = map[uint32]*buffer.StreamStatsWithLayers{ 1: { RTPStats: &buffer.RTPDeltaInfo{ @@ -252,8 +252,8 @@ func TestConnectionQuality(t *testing.T) { } cs.updateScore(streams, now.Add(duration)) mos, quality = cs.GetScoreAndQuality() - require.Greater(t, float32(4.1), mos) - require.Equal(t, livekit.ConnectionQuality_GOOD, quality) + require.Greater(t, float32(4.6), mos) + require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality) // mute/unmute to bring quality back up now = now.Add(duration) @@ -269,7 +269,7 @@ func TestConnectionQuality(t *testing.T) { Duration: duration, Packets: 250, PacketsLost: 5, - RttMax: 300, + RttMax: 400, JitterMax: 30000, }, }, @@ -294,7 +294,7 @@ func TestConnectionQuality(t *testing.T) { StartTime: now, Duration: duration, Packets: 250, - Bytes: 8_000_000 / 8 / 4, + Bytes: 8_000_000 / 8 / 5, }, }, } @@ -320,7 +320,7 @@ func TestConnectionQuality(t *testing.T) { StartTime: now, Duration: duration, Packets: 250, - Bytes: 8_000_000 / 8 / 4, + Bytes: 8_000_000 / 8 / 5, }, }, } @@ -345,7 +345,7 @@ func TestConnectionQuality(t *testing.T) { StartTime: now, Duration: duration, Packets: 250, - Bytes: 8_000_000 / 8 / 4, + Bytes: 8_000_000 / 8 / 5, }, }, } @@ -356,7 +356,7 @@ func TestConnectionQuality(t *testing.T) { }) t.Run("quality scorer dependent rtt", func(t *testing.T) { - cs := newConnectionStats("audio/opus", false, true, false) + cs := newConnectionStats("audio/opus", false, false, true) duration := 5 * time.Second now := time.Now() @@ -384,7 +384,7 @@ func TestConnectionQuality(t *testing.T) { }) t.Run("quality scorer dependent jitter", func(t *testing.T) { - cs := newConnectionStats("audio/opus", false, false, true) + cs := newConnectionStats("audio/opus", false, true, false) duration := 5 * time.Second now := time.Now() @@ -462,47 +462,23 @@ func TestConnectionQuality(t *testing.T) { expectedQuality: livekit.ConnectionQuality_EXCELLENT, }, { - packetLossPercentage: 4.1, + packetLossPercentage: 4.4, expectedMOS: 4.1, expectedQuality: livekit.ConnectionQuality_GOOD, }, { - packetLossPercentage: 13.2, + packetLossPercentage: 15.0, expectedMOS: 2.1, expectedQuality: livekit.ConnectionQuality_POOR, }, }, }, - // "audio/red" - no fec - 0 <= loss < 6.66%: EXCELLENT, 6.66% <= loss < 20%: GOOD, >= 20%: POOR + // "audio/red" - no fec - 0 <= loss < 10%: EXCELLENT, 10% <= loss < 30%: GOOD, >= 30%: POOR { name: "audio/red - no fec", mimeType: "audio/red", isFECEnabled: false, packetsExpected: 200, - expectedQualities: []expectedQuality{ - { - packetLossPercentage: 6.0, - expectedMOS: 4.6, - expectedQuality: livekit.ConnectionQuality_EXCELLENT, - }, - { - packetLossPercentage: 10.0, - expectedMOS: 4.1, - expectedQuality: livekit.ConnectionQuality_GOOD, - }, - { - packetLossPercentage: 23.0, - expectedMOS: 2.1, - expectedQuality: livekit.ConnectionQuality_POOR, - }, - }, - }, - // "audio/red" - fec - 0 <= loss < 10%: EXCELLENT, 10% <= loss < 30%: GOOD, >= 30%: POOR - { - name: "audio/red - fec", - mimeType: "audio/red", - isFECEnabled: true, - packetsExpected: 200, expectedQualities: []expectedQuality{ { packetLossPercentage: 8.0, @@ -510,12 +486,36 @@ func TestConnectionQuality(t *testing.T) { expectedQuality: livekit.ConnectionQuality_EXCELLENT, }, { - packetLossPercentage: 18.0, + packetLossPercentage: 12.0, expectedMOS: 4.1, expectedQuality: livekit.ConnectionQuality_GOOD, }, { - packetLossPercentage: 36.0, + packetLossPercentage: 39.0, + expectedMOS: 2.1, + expectedQuality: livekit.ConnectionQuality_POOR, + }, + }, + }, + // "audio/red" - fec - 0 <= loss < 15%: EXCELLENT, 15% <= loss < 45%: GOOD, >= 45%: POOR + { + name: "audio/red - fec", + mimeType: "audio/red", + isFECEnabled: true, + packetsExpected: 200, + expectedQualities: []expectedQuality{ + { + packetLossPercentage: 12.0, + expectedMOS: 4.6, + expectedQuality: livekit.ConnectionQuality_EXCELLENT, + }, + { + packetLossPercentage: 20.0, + expectedMOS: 4.1, + expectedQuality: livekit.ConnectionQuality_GOOD, + }, + { + packetLossPercentage: 60.0, expectedMOS: 2.1, expectedQuality: livekit.ConnectionQuality_POOR, }, @@ -534,12 +534,12 @@ func TestConnectionQuality(t *testing.T) { expectedQuality: livekit.ConnectionQuality_EXCELLENT, }, { - packetLossPercentage: 2.5, + packetLossPercentage: 3.5, expectedMOS: 4.1, expectedQuality: livekit.ConnectionQuality_GOOD, }, { - packetLossPercentage: 7.0, + packetLossPercentage: 8.0, expectedMOS: 2.1, expectedQuality: livekit.ConnectionQuality_POOR, }, @@ -549,7 +549,7 @@ func TestConnectionQuality(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - cs := newConnectionStats(tc.mimeType, tc.isFECEnabled, false, false) + cs := newConnectionStats(tc.mimeType, tc.isFECEnabled, true, true) duration := 5 * time.Second now := time.Now() @@ -619,7 +619,7 @@ func TestConnectionQuality(t *testing.T) { offset: 3 * time.Second, }, }, - bytes: uint64(math.Ceil(7_000_000.0 / 8.0 / 3.5)), + bytes: uint64(math.Ceil(7_000_000.0 / 8.0 / 4.2)), expectedMOS: 4.1, expectedQuality: livekit.ConnectionQuality_GOOD, }, @@ -634,7 +634,7 @@ func TestConnectionQuality(t *testing.T) { offset: 3 * time.Second, }, }, - bytes: uint64(math.Ceil(8_000_000.0 / 8.0 / 43.0)), + bytes: uint64(math.Ceil(8_000_000.0 / 8.0 / 75.0)), expectedMOS: 2.1, expectedQuality: livekit.ConnectionQuality_POOR, }, @@ -642,7 +642,7 @@ func TestConnectionQuality(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - cs := newConnectionStats("video/vp8", false, false, false) + cs := newConnectionStats("video/vp8", false, true, true) duration := 5 * time.Second now := time.Now() @@ -718,7 +718,7 @@ func TestConnectionQuality(t *testing.T) { distance: 2.0, }, { - distance: 2.2, + distance: 2.6, offset: 1 * time.Second, }, }, @@ -729,7 +729,7 @@ func TestConnectionQuality(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - cs := newConnectionStats("video/vp8", false, false, false) + cs := newConnectionStats("video/vp8", false, true, true) duration := 5 * time.Second now := time.Now() diff --git a/pkg/sfu/connectionquality/scorer.go b/pkg/sfu/connectionquality/scorer.go index 0c0a40e49..56e008a10 100644 --- a/pkg/sfu/connectionquality/scorer.go +++ b/pkg/sfu/connectionquality/scorer.go @@ -18,8 +18,8 @@ const ( poorScore = float64(30.0) minScore = float64(20.0) - increaseFactor = float64(0.4) // slow increase - decreaseFactor = float64(0.8) // fast decrease + increaseFactor = float64(0.4) // slower increase, i. e. when score is recovering move up slower -> conservative + decreaseFactor = float64(0.7) // faster decrease, i. e. when score is dropping move down faster -> aggressive to be responsive to quality drops distanceWeight = float64(35.0) // each spatial layer missed drops a quality level @@ -39,7 +39,7 @@ type windowStat struct { jitterMax float64 } -func (w *windowStat) calculatePacketScore(plw float64, isDependentRTT bool, isDependentJitter bool) float64 { +func (w *windowStat) calculatePacketScore(plw float64, includeRTT bool, includeJitter bool) float64 { // this is based on simplified E-model based on packet loss, rtt, jitter as // outlined at https://www.pingman.com/kb/article/how-is-mos-calculated-in-pingplotter-pro-50.html. effectiveDelay := 0.0 @@ -48,10 +48,10 @@ func (w *windowStat) calculatePacketScore(plw float64, isDependentRTT bool, isDe // 1. in the up stream, RTT cannot be measured without RTCP-XR, it is using down stream RTT. // 2. in the down stream, up stream jitter affects it. although jitter can be adjusted to account for up stream // jitter, this lever can be used to discount jitter in scoring. - if !isDependentRTT { + if includeRTT { effectiveDelay += float64(w.rttMax) / 2.0 } - if !isDependentJitter { + if includeJitter { effectiveDelay += (w.jitterMax * 2.0) / 1000.0 } delayEffect := effectiveDelay / 40.0 @@ -127,10 +127,10 @@ type layerTransition struct { } type qualityScorerParams struct { - PacketLossWeight float64 - IsDependentRTT bool - IsDependentJitter bool - Logger logger.Logger + PacketLossWeight float64 + IncludeRTT bool + IncludeJitter bool + Logger logger.Logger } type qualityScorer struct { @@ -261,7 +261,7 @@ func (q *qualityScorer) Update(stat *windowStat, at time.Time) { reason = "dry" score = poorScore } else { - packetScore := stat.calculatePacketScore(plw, q.params.IsDependentRTT, q.params.IsDependentJitter) + packetScore := stat.calculatePacketScore(plw, q.params.IncludeRTT, q.params.IncludeJitter) bitrateScore := stat.calculateBitrateScore(expectedBitrate) layerScore := math.Max(math.Min(maxScore, maxScore-(expectedDistance*distanceWeight)), 0.0) @@ -370,7 +370,8 @@ func (q *qualityScorer) getPacketLossWeight(stat *windowStat) float64 { return q.params.PacketLossWeight } - return math.Sqrt(pps/q.maxPPS) * q.params.PacketLossWeight + packetRatio := pps / q.maxPPS + return packetRatio * packetRatio * q.params.PacketLossWeight } func (q *qualityScorer) getExpectedBitsAndUpdateTransitions(at time.Time) int64 { diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 6769dec9e..c072645c6 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -303,7 +303,6 @@ func NewDownTrack( d.connectionStats = connectionquality.NewConnectionStats(connectionquality.ConnectionStatsParams{ MimeType: codecs[0].MimeType, // LK-TODO have to notify on codec change IsFECEnabled: strings.EqualFold(codecs[0].MimeType, webrtc.MimeTypeOpus) && strings.Contains(strings.ToLower(codecs[0].SDPFmtpLine), "fec"), - IsDependentJitter: true, GetDeltaStats: d.getDeltaStats, GetDeltaStatsOverridden: d.getDeltaStatsOverridden, GetLastReceiverReportTime: func() time.Time { return d.rtpStats.LastReceiverReport() }, diff --git a/pkg/sfu/receiver.go b/pkg/sfu/receiver.go index 5fd083bed..496e90ad0 100644 --- a/pkg/sfu/receiver.go +++ b/pkg/sfu/receiver.go @@ -203,11 +203,10 @@ func NewWebRTCReceiver( }) w.connectionStats = connectionquality.NewConnectionStats(connectionquality.ConnectionStatsParams{ - MimeType: w.codec.MimeType, - IsFECEnabled: strings.EqualFold(w.codec.MimeType, webrtc.MimeTypeOpus) && strings.Contains(strings.ToLower(w.codec.SDPFmtpLine), "fec"), - IsDependentRTT: true, - GetDeltaStats: w.getDeltaStats, - Logger: w.logger.WithValues("direction", "up"), + MimeType: w.codec.MimeType, + IsFECEnabled: strings.EqualFold(w.codec.MimeType, webrtc.MimeTypeOpus) && strings.Contains(strings.ToLower(w.codec.SDPFmtpLine), "fec"), + GetDeltaStats: w.getDeltaStats, + Logger: w.logger.WithValues("direction", "up"), }) w.connectionStats.OnStatsUpdate(func(_cs *connectionquality.ConnectionStats, stat *livekit.AnalyticsStat) { if w.onStatsUpdate != nil { From 2e93d386fed9cf146810bab59441ef1089cbe151 Mon Sep 17 00:00:00 2001 From: shishirng Date: Thu, 18 May 2023 13:50:54 -0400 Subject: [PATCH 168/324] send min/median connection score along with avg (#1720) * send min/median connection score along with avg * guard against divide by zero for avg score calculation * update median calculation Signed-off-by: shishir gowda --- go.mod | 18 +++++++++--------- go.sum | 36 +++++++++++++++++++----------------- pkg/telemetry/statsworker.go | 27 ++++++++++++++++++++++----- pkg/utils/math.go | 22 ++++++++++++++++++++++ 4 files changed, 72 insertions(+), 31 deletions(-) create mode 100644 pkg/utils/math.go diff --git a/go.mod b/go.mod index 970675c27..f916462b6 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230517210015-117bec6a19a8 - github.com/livekit/protocol v1.5.7-0.20230513090813-c5dc103838fd + github.com/livekit/protocol v1.5.7-0.20230518171313-8999a6b785c9 github.com/livekit/psrpc v0.3.1-0.20230502152150-df9dd21fba11 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.14.0 @@ -26,14 +26,14 @@ require ( github.com/mitchellh/go-homedir v1.1.0 github.com/olekukonko/tablewriter v0.0.5 github.com/pion/dtls/v2 v2.2.6 - github.com/pion/ice/v2 v2.3.2 + github.com/pion/ice/v2 v2.3.4 github.com/pion/interceptor v0.1.16 github.com/pion/rtcp v1.2.10 github.com/pion/rtp v1.7.13 github.com/pion/sdp/v3 v3.0.6 github.com/pion/transport/v2 v2.2.0 github.com/pion/turn/v2 v2.1.0 - github.com/pion/webrtc/v3 v3.2.1 + github.com/pion/webrtc/v3 v3.2.3 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.15.1 github.com/redis/go-redis/v9 v9.0.4 @@ -80,8 +80,8 @@ require ( github.com/pion/mdns v0.0.7 // indirect github.com/pion/randutil v0.1.0 // indirect github.com/pion/sctp v1.8.7 // indirect - github.com/pion/srtp/v2 v2.0.12 // indirect - github.com/pion/stun v0.4.0 // indirect + github.com/pion/srtp/v2 v2.0.14 // indirect + github.com/pion/stun v0.5.2 // indirect github.com/pion/udp/v2 v2.0.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect @@ -91,11 +91,11 @@ require ( github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.24.0 // indirect - golang.org/x/crypto v0.8.0 // indirect - golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 // indirect + golang.org/x/crypto v0.9.0 // indirect + golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect golang.org/x/mod v0.8.0 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sys v0.7.0 // indirect + golang.org/x/net v0.10.0 // indirect + golang.org/x/sys v0.8.0 // indirect golang.org/x/text v0.9.0 // indirect golang.org/x/tools v0.6.0 // indirect google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect diff --git a/go.sum b/go.sum index 10e4d5b01..cf0f2536e 100644 --- a/go.sum +++ b/go.sum @@ -121,8 +121,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20230517210015-117bec6a19a8 h1:YgBDljjYPJc57sSwaoyUgiviThQDyS7SyWsXJSRsZH8= github.com/livekit/mediatransportutil v0.0.0-20230517210015-117bec6a19a8/go.mod h1:MRc0zSOSzXuFt0X218SgabzlaKevkvCckPgBEoHYc34= -github.com/livekit/protocol v1.5.7-0.20230513090813-c5dc103838fd h1:wK+Vp0Oa0oggHYKBymHRJFeDWFzrcMjmyuOw4TLzT7c= -github.com/livekit/protocol v1.5.7-0.20230513090813-c5dc103838fd/go.mod h1:vjGsR1YxXnN5BLS0yr/YjGnJOPrS0ymddCF3JwxSHGM= +github.com/livekit/protocol v1.5.7-0.20230518171313-8999a6b785c9 h1:i61dBfZbe4MSF+5EVv9/kwVPq9pj1bWciBolSajl374= +github.com/livekit/protocol v1.5.7-0.20230518171313-8999a6b785c9/go.mod h1:nJvfqOFq0yenjwaJR0K5PCGf/6tbDts9QZ8bts+RBvk= github.com/livekit/psrpc v0.3.1-0.20230502152150-df9dd21fba11 h1:VS23iVQu/TNiLEM5XjbBSY28+B6nSewjKWPDbieg0Ho= github.com/livekit/psrpc v0.3.1-0.20230502152150-df9dd21fba11/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= @@ -181,8 +181,8 @@ github.com/pion/datachannel v1.5.5 h1:10ef4kwdjije+M9d7Xm9im2Y3O6A6ccQb0zcqZcJew github.com/pion/datachannel v1.5.5/go.mod h1:iMz+lECmfdCMqFRhXhcA/219B0SQlbpoR2V118yimL0= github.com/pion/dtls/v2 v2.2.6 h1:yXMxKr0Skd+Ub6A8UqXTRLSywskx93ooMRHsQUtd+Z4= github.com/pion/dtls/v2 v2.2.6/go.mod h1:t8fWJCIquY5rlQZwA2yWxUS1+OCrAdXrhVKXB5oD/wY= -github.com/pion/ice/v2 v2.3.2 h1:vh+fi4RkZ8H5fB4brZ/jm3j4BqFgMmNs+aB3X52Hu7M= -github.com/pion/ice/v2 v2.3.2/go.mod h1:AMIpuJqcpe+UwloocNebmTSWhCZM1TUCo9v7nW50jX0= +github.com/pion/ice/v2 v2.3.4 h1:tjYjTLpWyZzUjpDnzk6T1y3oQyhyY2DiM2t095iDhyQ= +github.com/pion/ice/v2 v2.3.4/go.mod h1:jVbxqPWQDK5+/V/YqpinUcP0YtDGYqd24n2lusVdX80= github.com/pion/interceptor v0.1.16 h1:0GDZrfNO+BmVNWymS31fMlVtPO2IJVBzy2Qq5XCYMIg= github.com/pion/interceptor v0.1.16/go.mod h1:SY8kpmfVBvrbUzvj2bsXz7OJt5JvmVNZ+4Kjq7FcwrI= github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= @@ -200,10 +200,11 @@ github.com/pion/sctp v1.8.7 h1:JnABvFakZueGAn4KU/4PSKg+GWbF6QWbKTWZOSGJjXw= github.com/pion/sctp v1.8.7/go.mod h1:g1Ul+ARqZq5JEmoFy87Q/4CePtKnTJ1QCL9dBBdN6AU= github.com/pion/sdp/v3 v3.0.6 h1:WuDLhtuFUUVpTfus9ILC4HRyHsW6TdugjEX/QY9OiUw= github.com/pion/sdp/v3 v3.0.6/go.mod h1:iiFWFpQO8Fy3S5ldclBkpXqmWy02ns78NOKoLLL0YQw= -github.com/pion/srtp/v2 v2.0.12 h1:WrmiVCubGMOAObBU1vwWjG0H3VSyQHawKeer2PVA5rY= -github.com/pion/srtp/v2 v2.0.12/go.mod h1:C3Ep44hlOo2qEYaq4ddsmK5dL63eLehXFbHaZ9F5V9Y= -github.com/pion/stun v0.4.0 h1:vgRrbBE2htWHy7l3Zsxckk7rkjnjOsSM7PHZnBwo8rk= +github.com/pion/srtp/v2 v2.0.14 h1:Glt0MqEvINrDxL+aanmK4DiFjvs+uN2iYc6XD/iKpoY= +github.com/pion/srtp/v2 v2.0.14/go.mod h1:b/pQOlDrbB0HEH5EUAQXzSYxikFbNcNuKmF8tM0hCtw= github.com/pion/stun v0.4.0/go.mod h1:QPsh1/SbXASntw3zkkrIk3ZJVKz4saBY2G7S10P3wCw= +github.com/pion/stun v0.5.2 h1:J/8glQnDV91dfk2+ZnGN0o9bUJgABhTNljwfQWByoXE= +github.com/pion/stun v0.5.2/go.mod h1:TNo1HjyjaFVpMZsvowqPeV8TfwRytympQC0//neaksA= github.com/pion/transport v0.14.1 h1:XSM6olwW+o8J4SCmOBb/BpwZypkHeyM0PGFCxNQBr40= github.com/pion/transport v0.14.1/go.mod h1:4tGmbk00NeYA3rUa9+n+dzCCoKkcy3YlYb99Jn2fNnI= github.com/pion/transport/v2 v2.0.0/go.mod h1:HS2MEBJTwD+1ZI2eSXSvHJx/HnzQqRy2/LXxt6eVMHc= @@ -215,8 +216,8 @@ github.com/pion/turn/v2 v2.1.0 h1:5wGHSgGhJhP/RpabkUb/T9PdsAjkGLS6toYz5HNzoSI= github.com/pion/turn/v2 v2.1.0/go.mod h1:yrT5XbXSGX1VFSF31A3c1kCNB5bBZgk/uu5LET162qs= github.com/pion/udp/v2 v2.0.1 h1:xP0z6WNux1zWEjhC7onRA3EwwSliXqu1ElUZAQhUP54= github.com/pion/udp/v2 v2.0.1/go.mod h1:B7uvTMP00lzWdyMr/1PVZXtV3wpPIxBRd4Wl6AksXn8= -github.com/pion/webrtc/v3 v3.2.1 h1:eehbYzkM6xWoH3LXoIBnZTb4TOrjwmVzI78JO1+5kgQ= -github.com/pion/webrtc/v3 v3.2.1/go.mod h1:sQVqop5YhZezvKyyz6Nywvf15LhlXUWiXWdN5DV4zHs= +github.com/pion/webrtc/v3 v3.2.3 h1:xHxxc4Tl7VWiZJtlITQadsAZq0vq9jw6ehRtH+3kRzs= +github.com/pion/webrtc/v3 v3.2.3/go.mod h1:UOxBQxi5gPxeJA5gDWk6pjrHgXUq8TCNWnEshhAVfko= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -280,11 +281,10 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= -golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ= -golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= -golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 h1:5llv2sWeaMSnA3w2kS57ouQQ4pudlXrR0dCgw51QK9o= -golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= +golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc h1:mCRnTeVUjcrhlRmO0VK8a6k6Rrf6TF9htwo2pJVSjIU= +golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= @@ -314,8 +314,8 @@ golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -362,14 +362,16 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/pkg/telemetry/statsworker.go b/pkg/telemetry/statsworker.go index 9447703b6..37dbc3aeb 100644 --- a/pkg/telemetry/statsworker.go +++ b/pkg/telemetry/statsworker.go @@ -8,6 +8,7 @@ import ( "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" + "github.com/livekit/livekit-server/pkg/utils" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" ) @@ -143,7 +144,9 @@ func coalesce(stats []*livekit.AnalyticsStat) *livekit.AnalyticsStat { } // find aggregates across streams - score := float32(0.0) + scoreSum := float32(0.0) // used for average + minScore := float32(0.0) // min score in batched stats + var scores []float32 // used for median maxRtt := uint32(0) maxJitter := uint32(0) coalescedVideoLayers := make(map[int32]*livekit.AnalyticsVideoLayer) @@ -154,7 +157,15 @@ func coalesce(stats []*livekit.AnalyticsStat) *livekit.AnalyticsStat { continue } - score += stat.Score + // only consider non-zero scores + if stat.Score > 0 { + if stat.Score < minScore { + minScore = stat.Score + } + scoreSum += stat.Score + scores = append(scores, stat.Score) + } + for _, analyticsStream := range stat.Streams { if analyticsStream.Rtt > maxRtt { maxRtt = analyticsStream.Rtt @@ -201,10 +212,16 @@ func coalesce(stats []*livekit.AnalyticsStat) *livekit.AnalyticsStat { } } - return &livekit.AnalyticsStat{ - Score: score / float32(len(stats)), - Streams: []*livekit.AnalyticsStream{coalescedStream}, + stat := &livekit.AnalyticsStat{ + MinScore: minScore, + MedianScore: utils.MedianFloat32(scores), + Streams: []*livekit.AnalyticsStream{coalescedStream}, } + numScores := len(scores) + if numScores > 0 { + stat.Score = scoreSum / float32(numScores) + } + return stat } func isValid(stat *livekit.AnalyticsStat) bool { diff --git a/pkg/utils/math.go b/pkg/utils/math.go new file mode 100644 index 000000000..0ac6319af --- /dev/null +++ b/pkg/utils/math.go @@ -0,0 +1,22 @@ +package utils + +import "sort" + +// MedianFloat32 gets median value for an array of float32 +func MedianFloat32(input []float32) float32 { + num := len(input) + if num == 0 { + return 0 + } else if num == 1 { + return input[0] + } + sort.Slice(input, func(i, j int) bool { + return input[i] < input[j] + }) + if num%2 != 0 { + return input[num/2] + } + left := input[num/2-1] + right := input[num/2] + return (left + right) / 2 +} From e03b7ef8deee0498f557c10c66bfcf7a896fce20 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Thu, 18 May 2023 12:39:02 -0700 Subject: [PATCH 169/324] start signal relay sessions with the correct node (#1721) * start signal relay sessions with the correct node * enable signal relay in multiregion integration test --- pkg/config/config.go | 1 - pkg/routing/localrouter.go | 2 +- pkg/service/signal_test.go | 1 - test/integration_helpers.go | 1 + 4 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 86138a0d9..eded926ca 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -210,7 +210,6 @@ type SignalRelayConfig struct { MinRetryInterval time.Duration `yaml:"min_retry_interval,omitempty"` MaxRetryInterval time.Duration `yaml:"max_retry_interval,omitempty"` StreamBufferSize int `yaml:"stream_buffer_size,omitempty"` - MinVersion int `yaml:"min_version,omitempty"` } // RegionConfig lists available regions and their latitude/longitude, so the selector would prefer diff --git a/pkg/routing/localrouter.go b/pkg/routing/localrouter.go index 2c5c39a98..da43566db 100644 --- a/pkg/routing/localrouter.go +++ b/pkg/routing/localrouter.go @@ -88,7 +88,7 @@ func (r *LocalRouter) StartParticipantSignal(ctx context.Context, roomName livek } func (r *LocalRouter) StartParticipantSignalWithNodeID(ctx context.Context, roomName livekit.RoomName, pi ParticipantInit, nodeID livekit.NodeID) (connectionID livekit.ConnectionID, reqSink MessageSink, resSource MessageSource, err error) { - connectionID, reqSink, resSource, err = r.signalClient.StartParticipantSignal(ctx, roomName, pi, livekit.NodeID(r.currentNode.Id)) + connectionID, reqSink, resSource, err = r.signalClient.StartParticipantSignal(ctx, roomName, pi, nodeID) if err != nil { logger.Errorw("could not handle new participant", err, "room", roomName, diff --git a/pkg/service/signal_test.go b/pkg/service/signal_test.go index a953fd9dc..946675864 100644 --- a/pkg/service/signal_test.go +++ b/pkg/service/signal_test.go @@ -28,7 +28,6 @@ func TestSignal(t *testing.T) { MinRetryInterval: 500 * time.Millisecond, MaxRetryInterval: 5 * time.Second, StreamBufferSize: 1000, - MinVersion: 1, } reqMessageIn := &livekit.SignalRequest{ diff --git a/test/integration_helpers.go b/test/integration_helpers.go index a9e633f83..b0354d866 100644 --- a/test/integration_helpers.go +++ b/test/integration_helpers.go @@ -170,6 +170,7 @@ func createMultiNodeServer(nodeID string, port uint32) *service.LivekitServer { conf.RTC.TCPPort = port + 2 conf.Redis.Address = "localhost:6379" conf.Keys = map[string]string{testApiKey: testApiSecret} + conf.SignalRelay.Enabled = true currentNode, err := routing.NewLocalNode(conf) if err != nil { From 5f3ea75a1e14c4cb632119593320857cc1ae2731 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Thu, 18 May 2023 13:53:20 -0700 Subject: [PATCH 170/324] conditionally block on signal relay close (#1722) --- pkg/routing/signal.go | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/pkg/routing/signal.go b/pkg/routing/signal.go index af8392e48..fab96ef7c 100644 --- a/pkg/routing/signal.go +++ b/pkg/routing/signal.go @@ -104,6 +104,7 @@ func (r *signalClient) StartParticipantSignal( Config: r.config, Writer: signalRequestMessageWriter{}, CloseOnFailure: true, + BlockOnClose: true, }) resChan := NewDefaultMessageChannel() @@ -228,6 +229,7 @@ type SignalSinkParams[SendType, RecvType RelaySignalMessage] struct { Config config.SignalRelayConfig Writer SignalMessageWriter[SendType] CloseOnFailure bool + BlockOnClose bool } func NewSignalMessageSink[SendType, RecvType RelaySignalMessage](params SignalSinkParams[SendType, RecvType]) MessageSink { @@ -255,22 +257,19 @@ func (s *signalMessageSink[SendType, RecvType]) Close() { } s.mu.Unlock() - // NOTE: not waiting for stream context to be done. - // Waiting for stream context to be done is confirmation - // that the close message has been processed by the other side. - // In ideal conditions, waiting for it is a clean end. + // conditionally block while closing to wait for outgoing messages to drain // - // But, in cases where the remote side goes away abruptly, waiting - // for stream context to be done could block connection progress - // till the timeout hits. + // on media the signal sink shares a goroutine with other signal connection + // attempts from the same participant so blocking delays establishing new + // sessions during reconnect. // - // The abrupt case happens when one side of the signal - // relay is shut down due to scale down or a crash. - // - // Uncomment the following line to wait for close acknowledgement, - // but the system should be able to wait long enough (till timeout) - // without adverse impact if waiting for close acknowledgement. - //<-s.Stream.Context().Done() + // on controller closing without waiting for the outstanding messages to + // drain causes leave messages to be dropped from the write queue. when + // this happens other participants in the room aren't notified about the + // departure until the participant times out. + if s.BlockOnClose { + <-s.Stream.Context().Done() + } } func (s *signalMessageSink[SendType, RecvType]) IsClosed() bool { From 93d6651d60c21a10b2edb8b884aaea70c9afb1fe Mon Sep 17 00:00:00 2001 From: David Zhao Date: Thu, 18 May 2023 14:10:40 -0700 Subject: [PATCH 171/324] Improve error message when WaitUntil fails. (#1723) --- pkg/testutils/timeout.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/testutils/timeout.go b/pkg/testutils/timeout.go index ab6414711..af70a48f9 100644 --- a/pkg/testutils/timeout.go +++ b/pkg/testutils/timeout.go @@ -4,8 +4,6 @@ import ( "context" "testing" "time" - - "github.com/stretchr/testify/require" ) var ( @@ -19,7 +17,9 @@ func WithTimeout(t *testing.T, f func() string) { for { select { case <-ctx.Done(): - require.Empty(t, lastErr) + if lastErr != "" { + t.Fatalf("did not reach expected state after %v: %s", ConnectTimeout, lastErr) + } case <-time.After(10 * time.Millisecond): lastErr = f() if lastErr == "" { From 0bb89575eb114488ec68f182ad73c47c5f4698df Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 19 May 2023 12:43:19 +0530 Subject: [PATCH 172/324] Fix min TS before first sender report (#1724) --- pkg/sfu/buffer/rtpstats.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index 873c8d59d..abc5dda08 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -808,6 +808,11 @@ func (r *RTPStats) GetExpectedRTPTimestamp(at time.Time) (uint32, uint32, error) timeDiff := at.Sub(r.firstTime) expectedRTPDiff := timeDiff.Nanoseconds() * int64(r.params.ClockRate) / 1e9 expectedExtRTP := r.extStartTS + uint64(expectedRTPDiff) + + minTS := r.lastSRRTP + if r.lastSRNTP == 0 { + minTS = uint32(expectedExtRTP) + } r.logger.Debugw( "expected RTP timestamp", "firstTime", r.firstTime.String(), @@ -817,10 +822,11 @@ func (r *RTPStats) GetExpectedRTPTimestamp(at time.Time) (uint32, uint32, error) "expectedRTPDiff", expectedRTPDiff, "expectedExtRTP", expectedExtRTP, "expectedRTP", uint32(expectedExtRTP), + "minTS", minTS, "highestTS", r.highestTS, "highestTime", r.highestTime.String(), ) - return uint32(expectedExtRTP), r.lastSRRTP, nil + return uint32(expectedExtRTP), minTS, nil } func (r *RTPStats) GetRtcpSenderReport(ssrc uint32) (*rtcp.SenderReport, float64) { From 3de51181ec3a4809c4f2733d0dd731d4d4558301 Mon Sep 17 00:00:00 2001 From: shishirng Date: Fri, 19 May 2023 11:00:32 -0400 Subject: [PATCH 173/324] Fix setting minscore - initialized to 0 (#1725) Signed-off-by: shishir gowda --- pkg/telemetry/statsworker.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/telemetry/statsworker.go b/pkg/telemetry/statsworker.go index 37dbc3aeb..f963097ee 100644 --- a/pkg/telemetry/statsworker.go +++ b/pkg/telemetry/statsworker.go @@ -159,7 +159,9 @@ func coalesce(stats []*livekit.AnalyticsStat) *livekit.AnalyticsStat { // only consider non-zero scores if stat.Score > 0 { - if stat.Score < minScore { + if minScore == 0 { + minScore = stat.Score + } else if stat.Score < minScore { minScore = stat.Score } scoreSum += stat.Score From 5260907ffec9e0e6f8a30da0a787a2969d845bc5 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Fri, 19 May 2023 23:00:06 -0700 Subject: [PATCH 174/324] Disable active TCP (#1726) Active TCP was added in pion/ice v2.3.4. This is causing a couple of issues for us. Active TCP does not make sense for an SFU. Clients are expected to be behind NAT and we should not be dialing them. Instead, LiveKit exposes a TCP port so clients could dial in Active TCP is causing all iOS clients to become disconnected immediately. This is impacting all version of libwebrtc-based iOS clients (tested from M104 to M111) --- pkg/rtc/transport.go | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index a2e88f9f2..0502c752b 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -1478,11 +1478,16 @@ func (t *PCTransport) handleLocalICECandidate(e *event) error { c := e.data.(*webrtc.ICECandidate) filtered := false - if t.preferTCP.Load() && c != nil && c.Protocol != webrtc.ICEProtocolTCP { - cstr := c.String() - t.params.Logger.Debugw("filtering out local candidate", "candidate", cstr) - t.filteredLocalCandidates = append(t.filteredLocalCandidates, cstr) - filtered = true + if c != nil { + if t.preferTCP.Load() && c.Protocol != webrtc.ICEProtocolTCP { + cstr := c.String() + t.params.Logger.Debugw("filtering out local candidate", "candidate", cstr) + t.filteredLocalCandidates = append(t.filteredLocalCandidates, cstr) + filtered = true + } else if c.Protocol == webrtc.ICEProtocolTCP && c.TCPType == ice.TCPTypeActive.String() { + // SFU should not support TCP active candidates. clients should connect to us + filtered = true + } } if filtered { @@ -1512,6 +1517,11 @@ func (t *PCTransport) handleRemoteICECandidate(e *event) error { t.params.Logger.Debugw("filtering out remote candidate", "candidate", c.Candidate) t.filteredRemoteCandidates = append(t.filteredRemoteCandidates, c.Candidate) filtered = true + } else if candidate, err := ice.UnmarshalCandidate(c.Candidate); err == nil { + if candidate != nil && candidate.TCPType() == ice.TCPTypePassive { + // SFU should ignore client's passive TCP, so Pion doesn't attempt to connect to it + filtered = true + } } if filtered { From d9e682a0d253faf793e283db952931d0e42aebb8 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 22 May 2023 18:46:56 +0530 Subject: [PATCH 175/324] Fix unwrap (#1729) * Fix unwrap An out-or-order packet wrapping back after a wrap around had already happened was not using proper cycle ounter to calculate unerapped value. * update mediatransportutil --- go.mod | 2 +- go.sum | 4 +-- pkg/sfu/utils/wraparound.go | 15 ++++++++-- pkg/sfu/utils/wraparound_test.go | 50 +++++++++++++++++++++++++------- 4 files changed, 56 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index f916462b6..c8784cc15 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.2 github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 - github.com/livekit/mediatransportutil v0.0.0-20230517210015-117bec6a19a8 + github.com/livekit/mediatransportutil v0.0.0-20230522130635-cc5a3793d7b5 github.com/livekit/protocol v1.5.7-0.20230518171313-8999a6b785c9 github.com/livekit/psrpc v0.3.1-0.20230502152150-df9dd21fba11 github.com/mackerelio/go-osstat v0.2.4 diff --git a/go.sum b/go.sum index cf0f2536e..4ddfa5abb 100644 --- a/go.sum +++ b/go.sum @@ -119,8 +119,8 @@ github.com/lithammer/shortuuid/v4 v4.0.0 h1:QRbbVkfgNippHOS8PXDkti4NaWeyYfcBTHtw github.com/lithammer/shortuuid/v4 v4.0.0/go.mod h1:Zs8puNcrvf2rV9rTH51ZLLcj7ZXqQI3lv67aw4KiB1Y= github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkDaKb5iXdynYrzB84ErPPO4LbRASk58= github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= -github.com/livekit/mediatransportutil v0.0.0-20230517210015-117bec6a19a8 h1:YgBDljjYPJc57sSwaoyUgiviThQDyS7SyWsXJSRsZH8= -github.com/livekit/mediatransportutil v0.0.0-20230517210015-117bec6a19a8/go.mod h1:MRc0zSOSzXuFt0X218SgabzlaKevkvCckPgBEoHYc34= +github.com/livekit/mediatransportutil v0.0.0-20230522130635-cc5a3793d7b5 h1:Pifg96a/cWZ9tyEk/41LzPmkww7GqBhzJ8E+jgxmQvQ= +github.com/livekit/mediatransportutil v0.0.0-20230522130635-cc5a3793d7b5/go.mod h1:MRc0zSOSzXuFt0X218SgabzlaKevkvCckPgBEoHYc34= github.com/livekit/protocol v1.5.7-0.20230518171313-8999a6b785c9 h1:i61dBfZbe4MSF+5EVv9/kwVPq9pj1bWciBolSajl374= github.com/livekit/protocol v1.5.7-0.20230518171313-8999a6b785c9/go.mod h1:nJvfqOFq0yenjwaJR0K5PCGf/6tbDts9QZ8bts+RBvk= github.com/livekit/psrpc v0.3.1-0.20230502152150-df9dd21fba11 h1:VS23iVQu/TNiLEM5XjbBSY28+B6nSewjKWPDbieg0Ho= diff --git a/pkg/sfu/utils/wraparound.go b/pkg/sfu/utils/wraparound.go index e6c8bbf2c..ea9a09887 100644 --- a/pkg/sfu/utils/wraparound.go +++ b/pkg/sfu/utils/wraparound.go @@ -93,17 +93,24 @@ func (w *WrapAround[T, ET]) GetExtendedHighest() ET { } func (w *WrapAround[T, ET]) maybeAdjustStart(val T) (isRestart bool, preExtendedStart ET, extendedVal ET) { + isWrapBack := func() bool { + return ET(w.highest) < (w.fullRange>>1) && ET(val) >= (w.fullRange>>1) + } + // re-adjust start if necessary. The conditions are // 1. Not seen more than half the range yet // 1. wrap around compared to start and not completed a half cycle, sequences like (10, 65530) in uint16 space // 2. no wrap around, but out-of-order compared to start and not completed a half cycle , sequences like (10, 9), (65530, 65528) in uint16 space + cycles := w.cycles totalNum := w.GetExtendedHighest() - w.GetExtendedStart() + 1 if totalNum > (w.fullRange >> 1) { - extendedVal = ET(w.cycles)*w.fullRange + ET(val) + if isWrapBack() { + cycles-- + } + extendedVal = ET(cycles)*w.fullRange + ET(val) return } - cycles := w.cycles if val-w.start > T(w.fullRange>>1) { // out-of-order with existing start => a new start isRestart = true @@ -115,6 +122,10 @@ func (w *WrapAround[T, ET]) maybeAdjustStart(val T) (isRestart bool, preExtended cycles = 0 } w.start = val + } else { + if isWrapBack() { + cycles-- + } } extendedVal = ET(cycles)*w.fullRange + ET(val) return diff --git a/pkg/sfu/utils/wraparound_test.go b/pkg/sfu/utils/wraparound_test.go index 828242f87..e9b6bd7a2 100644 --- a/pkg/sfu/utils/wraparound_test.go +++ b/pkg/sfu/utils/wraparound_test.go @@ -77,6 +77,21 @@ func TestWrapAroundUint16(t *testing.T) { highest: 10, extendedHighest: (1 << 16) + 10, }, + // out of order with highest, wrap back, but no restart + { + name: "out of order - no restart", + input: (1 << 16) - 3, + updated: wrapAroundUpdateResult[uint32]{ + IsRestart: false, + PreExtendedStart: 0, + PreExtendedHighest: (1 << 16) + 10, + ExtendedVal: (1 << 16) - 3, + }, + start: (1 << 16) - 12, + extendedStart: (1 << 16) - 12, + highest: 10, + extendedHighest: (1 << 16) + 10, + }, // duplicate should return same as highest { name: "duplicate", @@ -95,32 +110,47 @@ func TestWrapAroundUint16(t *testing.T) { // a significant jump in order should not reset start { name: "big in-order jump", - input: 1 << 15, + input: (1 << 15) - 10, updated: wrapAroundUpdateResult[uint32]{ IsRestart: false, PreExtendedStart: 0, PreExtendedHighest: (1 << 16) + 10, - ExtendedVal: (1 << 16) + (1 << 15), + ExtendedVal: (1 << 16) + (1 << 15) - 10, }, start: (1 << 16) - 12, extendedStart: (1 << 16) - 12, - highest: 1 << 15, - extendedHighest: (1 << 16) + (1 << 15), + highest: (1 << 15) - 10, + extendedHighest: (1 << 16) + (1 << 15) - 10, }, // now out-of-order should not reset start as half the range has been seen { name: "out-of-order after half range", - input: (1 << 15) - 1, + input: (1 << 15) - 11, updated: wrapAroundUpdateResult[uint32]{ IsRestart: false, PreExtendedStart: 0, - PreExtendedHighest: (1 << 16) + (1 << 15), - ExtendedVal: (1 << 16) + (1 << 15) - 1, + PreExtendedHighest: (1 << 16) + (1 << 15) - 10, + ExtendedVal: (1 << 16) + (1 << 15) - 11, }, start: (1 << 16) - 12, extendedStart: (1 << 16) - 12, - highest: 1 << 15, - extendedHighest: (1 << 16) + (1 << 15), + highest: (1 << 15) - 10, + extendedHighest: (1 << 16) + (1 << 15) - 10, + }, + // wrap back out-of-order + { + name: "wrap back out-of-order after half range", + input: (1 << 16) - 1, + updated: wrapAroundUpdateResult[uint32]{ + IsRestart: false, + PreExtendedStart: 0, + PreExtendedHighest: (1 << 16) + (1 << 15) - 10, + ExtendedVal: (1 << 16) - 1, + }, + start: (1 << 16) - 12, + extendedStart: (1 << 16) - 12, + highest: (1 << 15) - 10, + extendedHighest: (1 << 16) + (1 << 15) - 10, }, // in-order, should update highest { @@ -129,7 +159,7 @@ func TestWrapAroundUint16(t *testing.T) { updated: wrapAroundUpdateResult[uint32]{ IsRestart: false, PreExtendedStart: 0, - PreExtendedHighest: (1 << 16) + (1 << 15), + PreExtendedHighest: (1 << 16) + (1 << 15) - 10, ExtendedVal: (1 << 16) + (1 << 15) + 3, }, start: (1 << 16) - 12, From 04bcd601f000dfd2877587891cd1a55194d76c34 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 22 May 2023 13:29:52 -0700 Subject: [PATCH 176/324] Update livekit deps (#1599) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 10 +++++----- go.sum | 19 ++++++++++--------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index c8784cc15..278d53de3 100644 --- a/go.mod +++ b/go.mod @@ -17,9 +17,9 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.2 github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 - github.com/livekit/mediatransportutil v0.0.0-20230522130635-cc5a3793d7b5 - github.com/livekit/protocol v1.5.7-0.20230518171313-8999a6b785c9 - github.com/livekit/psrpc v0.3.1-0.20230502152150-df9dd21fba11 + github.com/livekit/mediatransportutil v0.0.0-20230518201646-90e22cdc7407 + github.com/livekit/protocol v1.5.7-0.20230522074955-efad03ffe4d0 + github.com/livekit/psrpc v0.3.1-0.20230518234341-6f6847e10b09 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.14.0 github.com/maxbrunsfeld/counterfeiter/v6 v6.6.1 @@ -33,12 +33,12 @@ require ( github.com/pion/sdp/v3 v3.0.6 github.com/pion/transport/v2 v2.2.0 github.com/pion/turn/v2 v2.1.0 - github.com/pion/webrtc/v3 v3.2.3 + github.com/pion/webrtc/v3 v3.2.4 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.15.1 github.com/redis/go-redis/v9 v9.0.4 github.com/rs/cors v1.9.0 - github.com/stretchr/testify v1.8.2 + github.com/stretchr/testify v1.8.3 github.com/thoas/go-funk v0.9.3 github.com/twitchtv/twirp v8.1.3+incompatible github.com/ua-parser/uap-go v0.0.0-20211112212520-00c877edfe0f diff --git a/go.sum b/go.sum index 4ddfa5abb..ba854ec69 100644 --- a/go.sum +++ b/go.sum @@ -119,12 +119,12 @@ github.com/lithammer/shortuuid/v4 v4.0.0 h1:QRbbVkfgNippHOS8PXDkti4NaWeyYfcBTHtw github.com/lithammer/shortuuid/v4 v4.0.0/go.mod h1:Zs8puNcrvf2rV9rTH51ZLLcj7ZXqQI3lv67aw4KiB1Y= github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkDaKb5iXdynYrzB84ErPPO4LbRASk58= github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= -github.com/livekit/mediatransportutil v0.0.0-20230522130635-cc5a3793d7b5 h1:Pifg96a/cWZ9tyEk/41LzPmkww7GqBhzJ8E+jgxmQvQ= -github.com/livekit/mediatransportutil v0.0.0-20230522130635-cc5a3793d7b5/go.mod h1:MRc0zSOSzXuFt0X218SgabzlaKevkvCckPgBEoHYc34= -github.com/livekit/protocol v1.5.7-0.20230518171313-8999a6b785c9 h1:i61dBfZbe4MSF+5EVv9/kwVPq9pj1bWciBolSajl374= -github.com/livekit/protocol v1.5.7-0.20230518171313-8999a6b785c9/go.mod h1:nJvfqOFq0yenjwaJR0K5PCGf/6tbDts9QZ8bts+RBvk= -github.com/livekit/psrpc v0.3.1-0.20230502152150-df9dd21fba11 h1:VS23iVQu/TNiLEM5XjbBSY28+B6nSewjKWPDbieg0Ho= -github.com/livekit/psrpc v0.3.1-0.20230502152150-df9dd21fba11/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= +github.com/livekit/mediatransportutil v0.0.0-20230518201646-90e22cdc7407 h1:DfalkHIgHJqjPbcvwIfh47seq4m8HD6g2H9dxUZ3JVI= +github.com/livekit/mediatransportutil v0.0.0-20230518201646-90e22cdc7407/go.mod h1:MRc0zSOSzXuFt0X218SgabzlaKevkvCckPgBEoHYc34= +github.com/livekit/protocol v1.5.7-0.20230522074955-efad03ffe4d0 h1:s2essRiWF//fVVBhjmxgjLHetpKYQ2QJfzi5w1a8rOA= +github.com/livekit/protocol v1.5.7-0.20230522074955-efad03ffe4d0/go.mod h1:ZaOnsvP+JS4s7vI1UO+JVdBagvvLp/lBXDAl2hkDS0I= +github.com/livekit/psrpc v0.3.1-0.20230518234341-6f6847e10b09 h1:mb6jRcg57U0HQ4tKRsueHHKcvTqBinL6+0Aa84vTtWk= +github.com/livekit/psrpc v0.3.1-0.20230518234341-6f6847e10b09/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= github.com/mackerelio/go-osstat v0.2.4/go.mod h1:Zy+qzGdZs3A9cuIqmgbJvwbmLQH9dJvtio5ZjJTbdlQ= github.com/magefile/mage v1.14.0 h1:6QDX3g6z1YvJ4olPhT1wksUcSa/V0a1B+pJb73fBjyo= @@ -216,8 +216,8 @@ github.com/pion/turn/v2 v2.1.0 h1:5wGHSgGhJhP/RpabkUb/T9PdsAjkGLS6toYz5HNzoSI= github.com/pion/turn/v2 v2.1.0/go.mod h1:yrT5XbXSGX1VFSF31A3c1kCNB5bBZgk/uu5LET162qs= github.com/pion/udp/v2 v2.0.1 h1:xP0z6WNux1zWEjhC7onRA3EwwSliXqu1ElUZAQhUP54= github.com/pion/udp/v2 v2.0.1/go.mod h1:B7uvTMP00lzWdyMr/1PVZXtV3wpPIxBRd4Wl6AksXn8= -github.com/pion/webrtc/v3 v3.2.3 h1:xHxxc4Tl7VWiZJtlITQadsAZq0vq9jw6ehRtH+3kRzs= -github.com/pion/webrtc/v3 v3.2.3/go.mod h1:UOxBQxi5gPxeJA5gDWk6pjrHgXUq8TCNWnEshhAVfko= +github.com/pion/webrtc/v3 v3.2.4 h1:gWSx4dqQb77051qBT9ipDrOyP6/sGYcAQP3UPjM8pU8= +github.com/pion/webrtc/v3 v3.2.4/go.mod h1:jtG9DOHcnIp7JMavANA9kTyz12sVnVRZx/rF0Awfd7I= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -251,8 +251,9 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/thoas/go-funk v0.9.3 h1:7+nAEx3kn5ZJcnDm2Bh23N2yOtweO14bi//dvRtgLpw= github.com/thoas/go-funk v0.9.3/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= github.com/twitchtv/twirp v8.1.3+incompatible h1:+F4TdErPgSUbMZMwp13Q/KgDVuI7HJXP61mNV3/7iuU= From ceac340ddcda146a5d65d8b7712a7190ed3dabae Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 22 May 2023 13:41:27 -0700 Subject: [PATCH 177/324] Update github.com/livekit/mediatransportutil digest to cc5a379 (#1731) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 278d53de3..c5cc63be0 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.2 github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 - github.com/livekit/mediatransportutil v0.0.0-20230518201646-90e22cdc7407 + github.com/livekit/mediatransportutil v0.0.0-20230522130635-cc5a3793d7b5 github.com/livekit/protocol v1.5.7-0.20230522074955-efad03ffe4d0 github.com/livekit/psrpc v0.3.1-0.20230518234341-6f6847e10b09 github.com/mackerelio/go-osstat v0.2.4 diff --git a/go.sum b/go.sum index ba854ec69..5cdb0404b 100644 --- a/go.sum +++ b/go.sum @@ -119,8 +119,8 @@ github.com/lithammer/shortuuid/v4 v4.0.0 h1:QRbbVkfgNippHOS8PXDkti4NaWeyYfcBTHtw github.com/lithammer/shortuuid/v4 v4.0.0/go.mod h1:Zs8puNcrvf2rV9rTH51ZLLcj7ZXqQI3lv67aw4KiB1Y= github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkDaKb5iXdynYrzB84ErPPO4LbRASk58= github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= -github.com/livekit/mediatransportutil v0.0.0-20230518201646-90e22cdc7407 h1:DfalkHIgHJqjPbcvwIfh47seq4m8HD6g2H9dxUZ3JVI= -github.com/livekit/mediatransportutil v0.0.0-20230518201646-90e22cdc7407/go.mod h1:MRc0zSOSzXuFt0X218SgabzlaKevkvCckPgBEoHYc34= +github.com/livekit/mediatransportutil v0.0.0-20230522130635-cc5a3793d7b5 h1:Pifg96a/cWZ9tyEk/41LzPmkww7GqBhzJ8E+jgxmQvQ= +github.com/livekit/mediatransportutil v0.0.0-20230522130635-cc5a3793d7b5/go.mod h1:MRc0zSOSzXuFt0X218SgabzlaKevkvCckPgBEoHYc34= github.com/livekit/protocol v1.5.7-0.20230522074955-efad03ffe4d0 h1:s2essRiWF//fVVBhjmxgjLHetpKYQ2QJfzi5w1a8rOA= github.com/livekit/protocol v1.5.7-0.20230522074955-efad03ffe4d0/go.mod h1:ZaOnsvP+JS4s7vI1UO+JVdBagvvLp/lBXDAl2hkDS0I= github.com/livekit/psrpc v0.3.1-0.20230518234341-6f6847e10b09 h1:mb6jRcg57U0HQ4tKRsueHHKcvTqBinL6+0Aa84vTtWk= From cba37389daab09aa0af24f8bb3405167cdd3851c Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 23 May 2023 10:02:36 +0530 Subject: [PATCH 178/324] mediatransportutil to get wrap back fix (#1732) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index c5cc63be0..c0e4ae49f 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.2 github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 - github.com/livekit/mediatransportutil v0.0.0-20230522130635-cc5a3793d7b5 + github.com/livekit/mediatransportutil v0.0.0-20230523035537-27577c4e1646 github.com/livekit/protocol v1.5.7-0.20230522074955-efad03ffe4d0 github.com/livekit/psrpc v0.3.1-0.20230518234341-6f6847e10b09 github.com/mackerelio/go-osstat v0.2.4 diff --git a/go.sum b/go.sum index 5cdb0404b..23fd47715 100644 --- a/go.sum +++ b/go.sum @@ -119,8 +119,8 @@ github.com/lithammer/shortuuid/v4 v4.0.0 h1:QRbbVkfgNippHOS8PXDkti4NaWeyYfcBTHtw github.com/lithammer/shortuuid/v4 v4.0.0/go.mod h1:Zs8puNcrvf2rV9rTH51ZLLcj7ZXqQI3lv67aw4KiB1Y= github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkDaKb5iXdynYrzB84ErPPO4LbRASk58= github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= -github.com/livekit/mediatransportutil v0.0.0-20230522130635-cc5a3793d7b5 h1:Pifg96a/cWZ9tyEk/41LzPmkww7GqBhzJ8E+jgxmQvQ= -github.com/livekit/mediatransportutil v0.0.0-20230522130635-cc5a3793d7b5/go.mod h1:MRc0zSOSzXuFt0X218SgabzlaKevkvCckPgBEoHYc34= +github.com/livekit/mediatransportutil v0.0.0-20230523035537-27577c4e1646 h1:acGSGkWJdut7TUWozCDheHu4dwWFDqqRzv+SBbIY9Xo= +github.com/livekit/mediatransportutil v0.0.0-20230523035537-27577c4e1646/go.mod h1:MRc0zSOSzXuFt0X218SgabzlaKevkvCckPgBEoHYc34= github.com/livekit/protocol v1.5.7-0.20230522074955-efad03ffe4d0 h1:s2essRiWF//fVVBhjmxgjLHetpKYQ2QJfzi5w1a8rOA= github.com/livekit/protocol v1.5.7-0.20230522074955-efad03ffe4d0/go.mod h1:ZaOnsvP+JS4s7vI1UO+JVdBagvvLp/lBXDAl2hkDS0I= github.com/livekit/psrpc v0.3.1-0.20230518234341-6f6847e10b09 h1:mb6jRcg57U0HQ4tKRsueHHKcvTqBinL6+0Aa84vTtWk= From 12c6f1e12cd495367b2d3dcf1fe64b8bf20f77cd Mon Sep 17 00:00:00 2001 From: David Zhao Date: Mon, 22 May 2023 21:38:56 -0700 Subject: [PATCH 179/324] Added Xiaomi 2201117TI to devices that does not support H.264 (#1728) --- go.mod | 2 +- go.sum | 4 +- pkg/clientconfiguration/conf.go | 15 +++++++- pkg/rtc/transportmanager.go | 34 ++++++++++++----- test/client/client.go | 27 +++++++++++++- test/singlenode_test.go | 66 +++++++++++++++++++++++++++++++++ 6 files changed, 132 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index c0e4ae49f..51cc6a595 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230523035537-27577c4e1646 - github.com/livekit/protocol v1.5.7-0.20230522074955-efad03ffe4d0 + github.com/livekit/protocol v1.5.7 github.com/livekit/psrpc v0.3.1-0.20230518234341-6f6847e10b09 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.14.0 diff --git a/go.sum b/go.sum index 23fd47715..38470cbe1 100644 --- a/go.sum +++ b/go.sum @@ -121,8 +121,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20230523035537-27577c4e1646 h1:acGSGkWJdut7TUWozCDheHu4dwWFDqqRzv+SBbIY9Xo= github.com/livekit/mediatransportutil v0.0.0-20230523035537-27577c4e1646/go.mod h1:MRc0zSOSzXuFt0X218SgabzlaKevkvCckPgBEoHYc34= -github.com/livekit/protocol v1.5.7-0.20230522074955-efad03ffe4d0 h1:s2essRiWF//fVVBhjmxgjLHetpKYQ2QJfzi5w1a8rOA= -github.com/livekit/protocol v1.5.7-0.20230522074955-efad03ffe4d0/go.mod h1:ZaOnsvP+JS4s7vI1UO+JVdBagvvLp/lBXDAl2hkDS0I= +github.com/livekit/protocol v1.5.7 h1:jZeFQEmLuIhFblXDGPRCBbfjVJHb+YU7AsO+SMoXF70= +github.com/livekit/protocol v1.5.7/go.mod h1:ZaOnsvP+JS4s7vI1UO+JVdBagvvLp/lBXDAl2hkDS0I= github.com/livekit/psrpc v0.3.1-0.20230518234341-6f6847e10b09 h1:mb6jRcg57U0HQ4tKRsueHHKcvTqBinL6+0Aa84vTtWk= github.com/livekit/psrpc v0.3.1-0.20230518234341-6f6847e10b09/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= diff --git a/pkg/clientconfiguration/conf.go b/pkg/clientconfiguration/conf.go index e7f84896e..916cd153f 100644 --- a/pkg/clientconfiguration/conf.go +++ b/pkg/clientconfiguration/conf.go @@ -1,6 +1,10 @@ package clientconfiguration -// configurations for livekit-client, add more configuration to StaticConfigurations as need +import ( + "github.com/livekit/protocol/livekit" +) + +// StaticConfigurations list specific device-side limitations that should be disabled at a global level var StaticConfigurations = []ConfigurationItem{ // { // Match: &ScriptMatch{Expr: `c.protocol <= 5 || c.browser == "firefox"`}, @@ -14,4 +18,13 @@ var StaticConfigurations = []ConfigurationItem{ // }}}, // Merge: false, // }, + { + Match: &ScriptMatch{Expr: `c.device_model == "Xiaomi 2201117TI" && c.os == "android"`}, + Configuration: &livekit.ClientConfiguration{ + DisabledCodecs: &livekit.DisabledCodecs{ + Publish: []*livekit.Codec{{Mime: "video/h264"}}, + }, + }, + Merge: false, + }, } diff --git a/pkg/rtc/transportmanager.go b/pkg/rtc/transportmanager.go index 397482dab..e34cb3e71 100644 --- a/pkg/rtc/transportmanager.go +++ b/pkg/rtc/transportmanager.go @@ -95,18 +95,32 @@ func NewTransportManager(params TransportManagerParams) (*TransportManager, erro } t.mediaLossProxy.OnMediaLossUpdate(t.onMediaLossUpdate) - enabledCodecs := make([]*livekit.Codec, 0, len(params.EnabledCodecs)) - for _, c := range params.EnabledCodecs { - var disabled bool - for _, disableCodec := range params.ClientConf.GetDisabledCodecs().GetCodecs() { + subscribeCodecs := make([]*livekit.Codec, 0, len(params.EnabledCodecs)) + publishCodecs := make([]*livekit.Codec, 0, len(params.EnabledCodecs)) + shouldDisable := func(c *livekit.Codec, disabledCodecs []*livekit.Codec) bool { + for _, disableCodec := range disabledCodecs { // disable codec's fmtp is empty means disable this codec entirely if strings.EqualFold(c.Mime, disableCodec.Mime) && (disableCodec.FmtpLine == "" || disableCodec.FmtpLine == c.FmtpLine) { - disabled = true - break + return true } } - if !disabled { - enabledCodecs = append(enabledCodecs, c) + return false + } + for _, c := range params.EnabledCodecs { + var publishDisabled bool + var subscribeDisabled bool + if shouldDisable(c, params.ClientConf.GetDisabledCodecs().GetCodecs()) { + publishDisabled = true + subscribeDisabled = true + } + if shouldDisable(c, params.ClientConf.GetDisabledCodecs().GetPublish()) { + publishDisabled = true + } + if !publishDisabled { + publishCodecs = append(publishCodecs, c) + } + if !subscribeDisabled { + subscribeCodecs = append(subscribeCodecs, c) } } @@ -118,7 +132,7 @@ func NewTransportManager(params TransportManagerParams) (*TransportManager, erro DirectionConfig: params.Config.Publisher, CongestionControlConfig: params.CongestionControlConfig, Telemetry: params.Telemetry, - EnabledCodecs: enabledCodecs, + EnabledCodecs: publishCodecs, Logger: LoggerWithPCTarget(params.Logger, livekit.SignalTarget_PUBLISHER), SimTracks: params.SimTracks, ClientInfo: params.ClientInfo, @@ -150,7 +164,7 @@ func NewTransportManager(params TransportManagerParams) (*TransportManager, erro DirectionConfig: params.Config.Subscriber, CongestionControlConfig: params.CongestionControlConfig, Telemetry: params.Telemetry, - EnabledCodecs: enabledCodecs, + EnabledCodecs: subscribeCodecs, Logger: LoggerWithPCTarget(params.Logger, livekit.SignalTarget_SUBSCRIBER), ClientInfo: params.ClientInfo, IsOfferer: true, diff --git a/test/client/client.go b/test/client/client.go index 384cdff79..17b085b3f 100644 --- a/test/client/client.go +++ b/test/client/client.go @@ -48,6 +48,7 @@ type RTCClient struct { publisherFullyEstablished atomic.Bool subscriberFullyEstablished atomic.Bool pongReceivedAt atomic.Int64 + lastAnswer atomic.Pointer[webrtc.SessionDescription] // tracks waiting to be acked, cid => trackInfo pendingPublishedTracks map[string]*livekit.TrackInfo @@ -81,6 +82,7 @@ var ( type Options struct { AutoSubscribe bool Publish string + ClientInfo *livekit.ClientInfo } func NewWebSocketConn(host, token string, opts *Options) (*websocket.Conn, error) { @@ -93,8 +95,18 @@ func NewWebSocketConn(host, token string, opts *Options) (*websocket.Conn, error connectUrl := u.String() if opts != nil { - connectUrl = fmt.Sprintf("%s&auto_subscribe=%t&publish=%s", - connectUrl, opts.AutoSubscribe, opts.Publish) + connectUrl = fmt.Sprintf("%s&auto_subscribe=%t", connectUrl, opts.AutoSubscribe) + if opts.Publish != "" { + connectUrl += encodeQueryParam("publish", opts.Publish) + } + if opts.ClientInfo != nil { + if opts.ClientInfo.DeviceModel != "" { + connectUrl += encodeQueryParam("device_model", opts.ClientInfo.DeviceModel) + } + if opts.ClientInfo.Os != "" { + connectUrl += encodeQueryParam("os", opts.ClientInfo.Os) + } + } } conn, _, err := websocket.DefaultDialer.Dial(connectUrl, requestHeader) return conn, err @@ -610,6 +622,11 @@ func (c *RTCClient) GetPublishedTrackIDs() []string { return trackIDs } +// LastAnswer return SDP of the last answer for the publisher connection +func (c *RTCClient) LastAnswer() *webrtc.SessionDescription { + return c.lastAnswer.Load() +} + func (c *RTCClient) ensurePublisherConnected() error { if c.publisher.HasEverConnected() { return nil @@ -654,6 +671,8 @@ func (c *RTCClient) handleOffer(desc webrtc.SessionDescription) { // the client handles answer on the publisher PC func (c *RTCClient) handleAnswer(desc webrtc.SessionDescription) { logger.Infow("handling server answer", "participant", c.localParticipant.Identity) + + c.lastAnswer.Store(&desc) // remote answered the offer, establish connection c.publisher.HandleRemoteDescription(desc) } @@ -744,3 +763,7 @@ func (c *RTCClient) SendNacks(count int) { _ = c.subscriber.WriteRTCP(packets) } + +func encodeQueryParam(key, value string) string { + return fmt.Sprintf("&%s=%s", url.QueryEscape(key), url.QueryEscape(value)) +} diff --git a/test/singlenode_test.go b/test/singlenode_test.go index 5af6602e0..a658a4bce 100644 --- a/test/singlenode_test.go +++ b/test/singlenode_test.go @@ -8,6 +8,7 @@ import ( "testing" "time" + "github.com/pion/sdp/v3" "github.com/pion/webrtc/v3" "github.com/stretchr/testify/require" "github.com/thoas/go-funk" @@ -22,6 +23,11 @@ import ( testclient "github.com/livekit/livekit-server/test/client" ) +const ( + waitTick = 10 * time.Millisecond + waitTimeout = 5 * time.Second +) + func TestClientCouldConnect(t *testing.T) { if testing.Short() { t.SkipNow() @@ -477,3 +483,63 @@ func TestSingleNodeUpdateSubscriptionPermissions(t *testing.T) { } }) } + +// TestDeviceCodecOverride checks that codecs that are incompatible with a device is not +// negotiated by the server +func TestDeviceCodecOverride(t *testing.T) { + if testing.Short() { + t.SkipNow() + return + } + + _, finish := setupSingleNodeTest("TestDeviceCodecOverride") + defer finish() + + // simulate device that isn't compatible with H.264 + c1 := createRTCClient("c1", defaultServerPort, &testclient.Options{ + ClientInfo: &livekit.ClientInfo{ + Os: "android", + DeviceModel: "Xiaomi 2201117TI", + }, + }) + defer c1.Stop() + waitUntilConnected(t, c1) + + // it doesn't really matter what the codec set here is, uses default Pion MediaEngine codecs + tw, err := c1.AddStaticTrack("video/h264", "video", "webcam") + require.NoError(t, err) + defer stopWriters(tw) + + // wait for server to receive track + require.Eventually(t, func() bool { + return c1.LastAnswer() != nil + }, waitTimeout, waitTick, "did not receive answer") + + sd := webrtc.SessionDescription{ + Type: webrtc.SDPTypeAnswer, + SDP: c1.LastAnswer().SDP, + } + answer, err := sd.Unmarshal() + require.NoError(t, err) + + // video and data channel + require.Len(t, answer.MediaDescriptions, 2) + var desc *sdp.MediaDescription + for _, md := range answer.MediaDescriptions { + if md.MediaName.Media == "video" { + desc = md + break + } + } + require.NotNil(t, desc) + hasSeenVP8 := false + for _, a := range desc.Attributes { + if a.Key == "rtpmap" { + require.NotContains(t, a.Value, "H264", "should not contain H264 codec") + if strings.Contains(a.Value, "VP8") { + hasSeenVP8 = true + } + } + } + require.True(t, hasSeenVP8, "should have seen VP8 codec in SDP") +} From bbbe815260f524730ae3766b82b88b2a7fc98b4b Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 23 May 2023 12:55:24 +0530 Subject: [PATCH 180/324] Init min to max MOS (#1734) * Init min to max MOS Could have been contributing to low p50 score in prom stats. * don't need to reset on no tracks as default is that --- pkg/rtc/participant.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 96db0b3d0..876f3c1d7 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -860,7 +860,7 @@ func (p *ParticipantImpl) GetAudioLevel() (level float64, active bool) { func (p *ParticipantImpl) GetConnectionQuality() *livekit.ConnectionQualityInfo { numTracks := 0 minQuality := livekit.ConnectionQuality_EXCELLENT - minScore := float32(0.0) + minScore := connectionquality.MaxMOS numUpDrops := 0 numDownDrops := 0 @@ -919,11 +919,6 @@ func (p *ParticipantImpl) GetConnectionQuality() *livekit.ConnectionQualityInfo availableTracks[trackID] = true } - if numTracks == 0 { - minQuality = livekit.ConnectionQuality_EXCELLENT - minScore = connectionquality.MaxMOS - } - prometheus.RecordQuality(minQuality, minScore, numUpDrops, numDownDrops) // remove unavailable tracks from track quality cache From 61d393e7096a92938540349041889d8755e5f1d0 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Tue, 23 May 2023 21:27:03 -0700 Subject: [PATCH 181/324] Disable active TCP by rolling back to ICE v2.3.3 (#1735) * Revert "Disable active TCP (#1726)" This reverts commit 5260907ffec9e0e6f8a30da0a787a2969d845bc5. * Disable active TCP by rolling back to ICE v2.3.3 --- go.mod | 10 ++++++---- go.sum | 17 +++++++++++------ pkg/rtc/transport.go | 20 +++++--------------- 3 files changed, 22 insertions(+), 25 deletions(-) diff --git a/go.mod b/go.mod index 51cc6a595..f631f2b85 100644 --- a/go.mod +++ b/go.mod @@ -25,13 +25,13 @@ require ( github.com/maxbrunsfeld/counterfeiter/v6 v6.6.1 github.com/mitchellh/go-homedir v1.1.0 github.com/olekukonko/tablewriter v0.0.5 - github.com/pion/dtls/v2 v2.2.6 + github.com/pion/dtls/v2 v2.2.7 github.com/pion/ice/v2 v2.3.4 github.com/pion/interceptor v0.1.16 github.com/pion/rtcp v1.2.10 github.com/pion/rtp v1.7.13 github.com/pion/sdp/v3 v3.0.6 - github.com/pion/transport/v2 v2.2.0 + github.com/pion/transport/v2 v2.2.1 github.com/pion/turn/v2 v2.1.0 github.com/pion/webrtc/v3 v3.2.4 github.com/pkg/errors v0.9.1 @@ -50,6 +50,9 @@ require ( gopkg.in/yaml.v3 v3.0.1 ) +// fix to version before active TCP was introduced, until there's a way to disable it +replace github.com/pion/ice/v2 => github.com/pion/ice/v2 v2.3.3 + require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect @@ -81,8 +84,7 @@ require ( github.com/pion/randutil v0.1.0 // indirect github.com/pion/sctp v1.8.7 // indirect github.com/pion/srtp/v2 v2.0.14 // indirect - github.com/pion/stun v0.5.2 // indirect - github.com/pion/udp/v2 v2.0.1 // indirect + github.com/pion/stun v0.6.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect diff --git a/go.sum b/go.sum index 38470cbe1..8c63d4a19 100644 --- a/go.sum +++ b/go.sum @@ -179,10 +179,11 @@ github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAl github.com/onsi/gomega v1.26.0 h1:03cDLK28U6hWvCAns6NeydX3zIm4SF3ci69ulidS32Q= github.com/pion/datachannel v1.5.5 h1:10ef4kwdjije+M9d7Xm9im2Y3O6A6ccQb0zcqZcJew8= github.com/pion/datachannel v1.5.5/go.mod h1:iMz+lECmfdCMqFRhXhcA/219B0SQlbpoR2V118yimL0= -github.com/pion/dtls/v2 v2.2.6 h1:yXMxKr0Skd+Ub6A8UqXTRLSywskx93ooMRHsQUtd+Z4= github.com/pion/dtls/v2 v2.2.6/go.mod h1:t8fWJCIquY5rlQZwA2yWxUS1+OCrAdXrhVKXB5oD/wY= -github.com/pion/ice/v2 v2.3.4 h1:tjYjTLpWyZzUjpDnzk6T1y3oQyhyY2DiM2t095iDhyQ= -github.com/pion/ice/v2 v2.3.4/go.mod h1:jVbxqPWQDK5+/V/YqpinUcP0YtDGYqd24n2lusVdX80= +github.com/pion/dtls/v2 v2.2.7 h1:cSUBsETxepsCSFSxC3mc/aDo14qQLMSL+O6IjG28yV8= +github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= +github.com/pion/ice/v2 v2.3.3 h1:uGrUwn0DanTmXgFiKDTot7iQzo0J9NjTjHPG+Kt+kNE= +github.com/pion/ice/v2 v2.3.3/go.mod h1:jVbxqPWQDK5+/V/YqpinUcP0YtDGYqd24n2lusVdX80= github.com/pion/interceptor v0.1.16 h1:0GDZrfNO+BmVNWymS31fMlVtPO2IJVBzy2Qq5XCYMIg= github.com/pion/interceptor v0.1.16/go.mod h1:SY8kpmfVBvrbUzvj2bsXz7OJt5JvmVNZ+4Kjq7FcwrI= github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= @@ -203,18 +204,19 @@ github.com/pion/sdp/v3 v3.0.6/go.mod h1:iiFWFpQO8Fy3S5ldclBkpXqmWy02ns78NOKoLLL0 github.com/pion/srtp/v2 v2.0.14 h1:Glt0MqEvINrDxL+aanmK4DiFjvs+uN2iYc6XD/iKpoY= github.com/pion/srtp/v2 v2.0.14/go.mod h1:b/pQOlDrbB0HEH5EUAQXzSYxikFbNcNuKmF8tM0hCtw= github.com/pion/stun v0.4.0/go.mod h1:QPsh1/SbXASntw3zkkrIk3ZJVKz4saBY2G7S10P3wCw= -github.com/pion/stun v0.5.2 h1:J/8glQnDV91dfk2+ZnGN0o9bUJgABhTNljwfQWByoXE= github.com/pion/stun v0.5.2/go.mod h1:TNo1HjyjaFVpMZsvowqPeV8TfwRytympQC0//neaksA= +github.com/pion/stun v0.6.0 h1:JHT/2iyGDPrFWE8NNC15wnddBN8KifsEDw8swQmrEmU= +github.com/pion/stun v0.6.0/go.mod h1:HPqcfoeqQn9cuaet7AOmB5e5xkObu9DwBdurwLKO9oA= github.com/pion/transport v0.14.1 h1:XSM6olwW+o8J4SCmOBb/BpwZypkHeyM0PGFCxNQBr40= github.com/pion/transport v0.14.1/go.mod h1:4tGmbk00NeYA3rUa9+n+dzCCoKkcy3YlYb99Jn2fNnI= github.com/pion/transport/v2 v2.0.0/go.mod h1:HS2MEBJTwD+1ZI2eSXSvHJx/HnzQqRy2/LXxt6eVMHc= github.com/pion/transport/v2 v2.0.2/go.mod h1:vrz6bUbFr/cjdwbnxq8OdDDzHf7JJfGsIRkxfpZoTA0= github.com/pion/transport/v2 v2.1.0/go.mod h1:AdSw4YBZVDkZm8fpoz+fclXyQwANWmZAlDuQdctTThQ= -github.com/pion/transport/v2 v2.2.0 h1:u5lFqFHkXLMXMzai8tixZDfVjb8eOjH35yCunhPeb1c= github.com/pion/transport/v2 v2.2.0/go.mod h1:AdSw4YBZVDkZm8fpoz+fclXyQwANWmZAlDuQdctTThQ= +github.com/pion/transport/v2 v2.2.1 h1:7qYnCBlpgSJNYMbLCKuSY9KbQdBFoETvPNETv0y4N7c= +github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g= github.com/pion/turn/v2 v2.1.0 h1:5wGHSgGhJhP/RpabkUb/T9PdsAjkGLS6toYz5HNzoSI= github.com/pion/turn/v2 v2.1.0/go.mod h1:yrT5XbXSGX1VFSF31A3c1kCNB5bBZgk/uu5LET162qs= -github.com/pion/udp/v2 v2.0.1 h1:xP0z6WNux1zWEjhC7onRA3EwwSliXqu1ElUZAQhUP54= github.com/pion/udp/v2 v2.0.1/go.mod h1:B7uvTMP00lzWdyMr/1PVZXtV3wpPIxBRd4Wl6AksXn8= github.com/pion/webrtc/v3 v3.2.4 h1:gWSx4dqQb77051qBT9ipDrOyP6/sGYcAQP3UPjM8pU8= github.com/pion/webrtc/v3 v3.2.4/go.mod h1:jtG9DOHcnIp7JMavANA9kTyz12sVnVRZx/rF0Awfd7I= @@ -282,6 +284,7 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= +golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc h1:mCRnTeVUjcrhlRmO0VK8a6k6Rrf6TF9htwo2pJVSjIU= @@ -315,6 +318,7 @@ golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -372,6 +376,7 @@ golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index 0502c752b..a2e88f9f2 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -1478,16 +1478,11 @@ func (t *PCTransport) handleLocalICECandidate(e *event) error { c := e.data.(*webrtc.ICECandidate) filtered := false - if c != nil { - if t.preferTCP.Load() && c.Protocol != webrtc.ICEProtocolTCP { - cstr := c.String() - t.params.Logger.Debugw("filtering out local candidate", "candidate", cstr) - t.filteredLocalCandidates = append(t.filteredLocalCandidates, cstr) - filtered = true - } else if c.Protocol == webrtc.ICEProtocolTCP && c.TCPType == ice.TCPTypeActive.String() { - // SFU should not support TCP active candidates. clients should connect to us - filtered = true - } + if t.preferTCP.Load() && c != nil && c.Protocol != webrtc.ICEProtocolTCP { + cstr := c.String() + t.params.Logger.Debugw("filtering out local candidate", "candidate", cstr) + t.filteredLocalCandidates = append(t.filteredLocalCandidates, cstr) + filtered = true } if filtered { @@ -1517,11 +1512,6 @@ func (t *PCTransport) handleRemoteICECandidate(e *event) error { t.params.Logger.Debugw("filtering out remote candidate", "candidate", c.Candidate) t.filteredRemoteCandidates = append(t.filteredRemoteCandidates, c.Candidate) filtered = true - } else if candidate, err := ice.UnmarshalCandidate(c.Candidate); err == nil { - if candidate != nil && candidate.TCPType() == ice.TCPTypePassive { - // SFU should ignore client's passive TCP, so Pion doesn't attempt to connect to it - filtered = true - } } if filtered { From 07252b7ce35d2b3174c0ba562950a81ae376396c Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 24 May 2023 12:32:12 +0530 Subject: [PATCH 182/324] Filter not last SR error (#1737) --- pkg/sfu/buffer/rtpstats.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index abc5dda08..7720a10b3 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -518,7 +518,7 @@ func (r *RTPStats) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt uint32 if err == nil { isRttChanged = rtt != r.rtt } else { - if err != mediatransportutil.ErrRttNoLastSenderReport { + if !errors.Is(err, mediatransportutil.ErrRttNotLastSenderReport) { r.logger.Warnw("error getting rtt", err) } } From 11c5737e04afd7faf59fd682bada2288000fe437 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 24 May 2023 12:41:47 +0530 Subject: [PATCH 183/324] Filter another expected error. (#1738) Actually, was not filtering the not last sender report error before. Previous PR did that. This PR restores the old no last sender report filter. Both are filterable errors. --- pkg/sfu/buffer/rtpstats.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index 7720a10b3..0aab6e16a 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -518,7 +518,7 @@ func (r *RTPStats) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt uint32 if err == nil { isRttChanged = rtt != r.rtt } else { - if !errors.Is(err, mediatransportutil.ErrRttNotLastSenderReport) { + if !errors.Is(err, mediatransportutil.ErrRttNotLastSenderReport) && !errors.Is(err, mediatransportutil.ErrRttNoLastSenderReport) { r.logger.Warnw("error getting rtt", err) } } From 0354626bfc1b6aaacbf60f8aecdf719dce1507f2 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 25 May 2023 21:55:54 +0530 Subject: [PATCH 184/324] Adjust sender report time stamp for slow publishers. (#1740) It is possible that publisher paces the media. So, RTCP sender report from publisher could be ahead of what is being fowarded by a good amount (have seen up to 2 seconds ahead). Using the forwarded time stamp for RTCP sender report in the down stream leads to jumps back and forth in the down track RTCP sender report. So, look at the publisher's RTCP sender report to check for it being ahead and use the publisher rate as a guide. --- pkg/rtc/wrappedreceiver.go | 8 + pkg/sfu/buffer/buffer.go | 6 +- pkg/sfu/buffer/rtpstats.go | 313 ++++++++++++++++++++------------ pkg/sfu/downtrack.go | 5 +- pkg/sfu/forwarder.go | 21 ++- pkg/sfu/receiver.go | 8 +- pkg/sfu/streamtrackermanager.go | 27 ++- 7 files changed, 253 insertions(+), 135 deletions(-) diff --git a/pkg/rtc/wrappedreceiver.go b/pkg/rtc/wrappedreceiver.go index 068f8b69d..84d4d9910 100644 --- a/pkg/rtc/wrappedreceiver.go +++ b/pkg/rtc/wrappedreceiver.go @@ -12,6 +12,7 @@ import ( "github.com/livekit/protocol/logger" "github.com/livekit/livekit-server/pkg/sfu" + "github.com/livekit/livekit-server/pkg/sfu/buffer" ) // wrapper around WebRTC receiver, overriding its ID @@ -288,6 +289,13 @@ func (d *DummyReceiver) GetRedReceiver() sfu.TrackReceiver { return d } +func (d *DummyReceiver) GetRTCPSenderReportData(layer int32) (*buffer.RTCPSenderReportData, *buffer.RTCPSenderReportData) { + if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { + return r.GetRTCPSenderReportData(layer) + } + return nil, nil +} + func (d *DummyReceiver) GetReferenceLayerRTPTimestamp(ts uint32, layer int32, referenceLayer int32) (uint32, error) { if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { return r.GetReferenceLayerRTPTimestamp(ts, layer, referenceLayer) diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go index 63949a52c..575f2d433 100644 --- a/pkg/sfu/buffer/buffer.go +++ b/pkg/sfu/buffer/buffer.go @@ -654,7 +654,7 @@ func (b *Buffer) SetSenderReportData(rtpTime uint32, ntpTime uint64) { srData := &RTCPSenderReportData{ RTPTimestamp: rtpTime, NTPTimestamp: mediatransportutil.NtpTime(ntpTime), - ArrivalTime: time.Now(), + At: time.Now(), } b.RLock() @@ -668,7 +668,7 @@ func (b *Buffer) SetSenderReportData(rtpTime uint32, ntpTime uint64) { } } -func (b *Buffer) GetSenderReportData() *RTCPSenderReportData { +func (b *Buffer) GetSenderReportData() (*RTCPSenderReportData, *RTCPSenderReportData) { b.RLock() defer b.RUnlock() @@ -676,7 +676,7 @@ func (b *Buffer) GetSenderReportData() *RTCPSenderReportData { return b.rtpStats.GetRtcpSenderReportData() } - return nil + return nil, nil } func (b *Buffer) SetLastFractionLostReport(lost uint8) { diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index 0aab6e16a..d650f33a0 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -25,6 +25,28 @@ const ( TooLargeOWDDelta = 400 * time.Millisecond ) +// ------------------------------------------------------- + +type driftResult struct { + timeSinceFirst time.Duration + rtpDiffSinceFirst uint64 + driftSamples int64 + driftMs float64 + sampleRate float64 +} + +func (d driftResult) String() string { + return fmt.Sprintf("time: %+v, rtp: %d, driftSamples: %d, driftMs: %.02f, sampleRate: %.02f", + d.timeSinceFirst, + d.rtpDiffSinceFirst, + d.driftSamples, + d.driftMs, + d.sampleRate, + ) +} + +// ------------------------------------------------------- + type RTPFlowState struct { HasLoss bool LossStartInclusive uint16 @@ -88,9 +110,10 @@ type SnInfo struct { } type RTCPSenderReportData struct { - RTPTimestamp uint32 - NTPTimestamp mediatransportutil.NtpTime - ArrivalTime time.Time + RTPTimestamp uint32 + RTPTimestampExt uint64 + NTPTimestamp mediatransportutil.NtpTime + At time.Time } type RTPStatsParams struct { @@ -175,10 +198,9 @@ type RTPStats struct { rtt uint32 maxRtt uint32 - srData *RTCPSenderReportData - lastSRTime time.Time - lastSRNTP mediatransportutil.NtpTime - lastSRRTP uint32 + srFirst *RTCPSenderReportData + srNewest *RTCPSenderReportData + pidController *PIDController nextSnapshotId uint32 @@ -280,15 +302,18 @@ func (r *RTPStats) Seed(from *RTPStats) { r.rtt = from.rtt r.maxRtt = from.maxRtt - if from.srData != nil { - srData := *from.srData - r.srData = &srData + if from.srFirst != nil { + srFirst := *from.srFirst + r.srFirst = &srFirst } else { - r.srData = nil + r.srFirst = nil + } + if from.srNewest != nil { + srNewest := *from.srNewest + r.srNewest = &srNewest + } else { + r.srNewest = nil } - r.lastSRTime = from.lastSRTime - r.lastSRNTP = from.lastSRNTP - r.lastSRRTP = from.lastSRRTP r.nextSnapshotId = from.nextSnapshotId for id, ss := range from.snapshots { @@ -425,12 +450,16 @@ func (r *RTPStats) Update(rtph *rtp.Header, payloadSize int, paddingSize int, pa } r.highestSN = rtph.SequenceNumber - if rtph.Timestamp < r.highestTS && !first { - r.tsCycles++ - } - r.highestTS = rtph.Timestamp + if rtph.Timestamp != r.highestTS { + if rtph.Timestamp < r.highestTS && !first { + r.tsCycles++ + } + r.highestTS = rtph.Timestamp - r.highestTime = packetTime + // update only on first packet as same timestamp could be in multiple packets. + // NOTE: this may not be the first packet with this time stamp if there is packet loss. + r.highestTime = packetTime + } } if !isDuplicate { @@ -514,12 +543,15 @@ func (r *RTPStats) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt uint32 return } - rtt, err := mediatransportutil.GetRttMs(&rr, r.lastSRNTP, r.lastSRTime) - if err == nil { - isRttChanged = rtt != r.rtt - } else { - if !errors.Is(err, mediatransportutil.ErrRttNotLastSenderReport) && !errors.Is(err, mediatransportutil.ErrRttNoLastSenderReport) { - r.logger.Warnw("error getting rtt", err) + var err error + if r.srNewest != nil { + rtt, err = mediatransportutil.GetRttMs(&rr, r.srNewest.NTPTimestamp, r.srNewest.At) + if err == nil { + isRttChanged = rtt != r.rtt + } else { + if !errors.Is(err, mediatransportutil.ErrRttNotLastSenderReport) && !errors.Is(err, mediatransportutil.ErrRttNoLastSenderReport) { + r.logger.Warnw("error getting rtt", err) + } } } @@ -725,79 +757,97 @@ func (r *RTPStats) GetRtt() uint32 { } func (r *RTPStats) SetRtcpSenderReportData(srData *RTCPSenderReportData) { - r.lock.Lock() - defer r.lock.Unlock() - if srData == nil { - r.srData = nil return } + r.lock.Lock() + defer r.lock.Unlock() + // prevent against extreme case of anachronous sender reports - if r.srData != nil && r.srData.NTPTimestamp > srData.NTPTimestamp { + if r.srNewest != nil && r.srNewest.NTPTimestamp > srData.NTPTimestamp { r.logger.Infow( "received anachronous sender report", "current", srData.NTPTimestamp.Time(), - "last", r.srData.NTPTimestamp.Time(), + "last", r.srNewest.NTPTimestamp.Time(), ) return } // monitor and log RTP timestamp anomalies - if r.srData != nil { - ntpDiffSinceLast := srData.NTPTimestamp.Time().Sub(r.srData.NTPTimestamp.Time()) - rtpDiffSinceLast := srData.RTPTimestamp - r.srData.RTPTimestamp - arrivalDiffSinceLast := srData.ArrivalTime.Sub(r.srData.ArrivalTime) + var ntpDiffSinceLast time.Duration + var rtpDiffSinceLast uint32 + var arrivalDiffSinceLast time.Duration + var expectedTimeDiffSinceLast float64 + var reason string + if r.srNewest != nil { + ntpDiffSinceLast = srData.NTPTimestamp.Time().Sub(r.srNewest.NTPTimestamp.Time()) + rtpDiffSinceLast = srData.RTPTimestamp - r.srNewest.RTPTimestamp + arrivalDiffSinceLast = srData.At.Sub(r.srNewest.At) - expectedTimeDiffSinceLast := float64(rtpDiffSinceLast) / float64(r.params.ClockRate) + expectedTimeDiffSinceLast = float64(rtpDiffSinceLast) / float64(r.params.ClockRate) - var reason string - if (srData.RTPTimestamp - r.srData.RTPTimestamp) > (1 << 31) { - reason = "received sender report, out-of-order" + if (srData.RTPTimestamp - r.srNewest.RTPTimestamp) > (1 << 31) { + reason = "received sender report, out-of-order" // should not happen, just a sanity check } else { if math.Abs(expectedTimeDiffSinceLast-ntpDiffSinceLast.Seconds()) > 0.2 { // more than 200 ms away from expected delta reason = "received sender report, time warp" } } + } - if reason != "" { - timeSinceFirst, rtpDiffSinceFirst, drift, driftMs, sampleRate := r.getDrift() - r.logger.Infow( - reason, - "ntp", srData.NTPTimestamp.Time().String(), - "rtp", srData.RTPTimestamp, - "arrival", srData.ArrivalTime.String(), - "ntpDiffSinceLast", ntpDiffSinceLast.Seconds(), - "rtpDiffSinceLast", int32(rtpDiffSinceLast), - "arrivalDiffSinceLast", arrivalDiffSinceLast.Seconds(), - "expectedTimeDiffSinceLast", expectedTimeDiffSinceLast, - "timeSinceFirst", timeSinceFirst.Seconds(), - "rtpDiffSinceFirst", rtpDiffSinceFirst, - "drift", drift, - "driftMs", driftMs, - "sampleRate", sampleRate, - ) + cycles := uint64(0) + if r.srNewest != nil { + cycles = r.srNewest.RTPTimestampExt & 0xFF_FF_FF_FF_00_00_00_00 + if (srData.RTPTimestamp-r.srNewest.RTPTimestamp) < (1<<31) && srData.RTPTimestamp < r.srNewest.RTPTimestamp { + cycles += (1 << 32) } } srDataCopy := *srData - r.srData = &srDataCopy + srDataCopy.RTPTimestampExt = uint64(srDataCopy.RTPTimestamp) + cycles + r.srNewest = &srDataCopy + if r.srFirst == nil { + r.srFirst = &srDataCopy + } + + if reason != "" { + packetDriftResult, reportDriftResult := r.getDrift() + r.logger.Infow( + reason, + "ntp", srData.NTPTimestamp.Time().String(), + "rtp", srData.RTPTimestamp, + "arrival", srData.At.String(), + "ntpDiffSinceLast", ntpDiffSinceLast.Seconds(), + "rtpDiffSinceLast", int32(rtpDiffSinceLast), + "arrivalDiffSinceLast", arrivalDiffSinceLast.Seconds(), + "expectedTimeDiffSinceLast", expectedTimeDiffSinceLast, + "packetDrift", packetDriftResult.String(), + "reportDrift", reportDriftResult.String(), + "highestTS", r.highestTS, + "highestTime", r.highestTime.String(), + ) + } } -func (r *RTPStats) GetRtcpSenderReportData() *RTCPSenderReportData { +func (r *RTPStats) GetRtcpSenderReportData() (srFirst *RTCPSenderReportData, srNewest *RTCPSenderReportData) { r.lock.RLock() defer r.lock.RUnlock() - if r.srData == nil { - return nil + if r.srFirst != nil { + srFirstCopy := *r.srFirst + srFirst = &srFirstCopy } - srDataCopy := *r.srData - return &srDataCopy + if r.srNewest != nil { + srNewestCopy := *r.srNewest + srNewest = &srNewestCopy + } + return } -func (r *RTPStats) GetExpectedRTPTimestamp(at time.Time) (uint32, uint32, error) { +func (r *RTPStats) GetExpectedRTPTimestamp(at time.Time) (uint32, uint64, error) { r.lock.RLock() defer r.lock.RUnlock() @@ -809,9 +859,9 @@ func (r *RTPStats) GetExpectedRTPTimestamp(at time.Time) (uint32, uint32, error) expectedRTPDiff := timeDiff.Nanoseconds() * int64(r.params.ClockRate) / 1e9 expectedExtRTP := r.extStartTS + uint64(expectedRTPDiff) - minTS := r.lastSRRTP - if r.lastSRNTP == 0 { - minTS = uint32(expectedExtRTP) + minTS := ^uint64(0) + if r.srNewest != nil { + minTS = r.srNewest.RTPTimestampExt } r.logger.Debugw( "expected RTP timestamp", @@ -829,7 +879,7 @@ func (r *RTPStats) GetExpectedRTPTimestamp(at time.Time) (uint32, uint32, error) return uint32(expectedExtRTP), minTS, nil } -func (r *RTPStats) GetRtcpSenderReport(ssrc uint32) (*rtcp.SenderReport, float64) { +func (r *RTPStats) GetRtcpSenderReport(ssrc uint32, srFirst *RTCPSenderReportData, srNewest *RTCPSenderReportData) (*rtcp.SenderReport, float64) { r.lock.Lock() defer r.lock.Unlock() @@ -845,7 +895,27 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32) (*rtcp.SenderReport, float64 timeSinceHighest := now.Sub(r.highestTime) nowRTP := r.highestTS + uint32(timeSinceHighest.Nanoseconds()*int64(r.params.ClockRate)/1e9) - rtpDiffSinceFirst := getExtTS(nowRTP, r.tsCycles) - r.extStartTS + // It is possible that publisher is pacing at a slower rate. + // That would make `highestTS` to be lagging the RTP time stamp in the RTCP Sender Report from publisher. + // Check for that and use the later time stamp if applicable. + tsCycles := r.tsCycles + if nowRTP < r.highestTS { + tsCycles++ + } + nowRTPExt := getExtTS(nowRTP, tsCycles) + if srFirst != nil && srNewest != nil && srFirst.RTPTimestamp != srNewest.RTPTimestamp { + // use incoming rate as a guide + tsf := srNewest.NTPTimestamp.Time().Sub(srFirst.NTPTimestamp.Time()) + rdsf := srNewest.RTPTimestampExt - srFirst.RTPTimestampExt + sr := float64(rdsf) / tsf.Seconds() + nowRTPExtUsingRate := r.extStartTS + uint64(sr*timeSinceFirst.Seconds()) + if nowRTPExtUsingRate > nowRTPExt { + nowRTPExt = nowRTPExtUsingRate + nowRTP = uint32(nowRTPExtUsingRate) + } + } + + rtpDiffSinceFirst := nowRTPExt - r.extStartTS rate := float64(rtpDiffSinceFirst) / timeSinceFirst.Seconds() pidOutput := r.pidController.Update( float64(r.params.ClockRate), @@ -854,49 +924,50 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32) (*rtcp.SenderReport, float64 ) // monitor and log RTP timestamp anomalies - isWarped := false - if r.lastSRNTP != 0 { - ntpDiffSinceLast := nowNTP.Time().Sub(r.lastSRNTP.Time()) - rtpDiffSinceLast := nowRTP - r.lastSRRTP - departureDiffSinceLast := now.Sub(r.lastSRTime) + var ntpDiffSinceLast time.Duration + var rtpDiffSinceLast uint32 + var departureDiffSinceLast time.Duration + var expectedTimeDiffSinceLast float64 + var isWarped bool + if r.srNewest != nil { + ntpDiffSinceLast = nowNTP.Time().Sub(r.srNewest.NTPTimestamp.Time()) + rtpDiffSinceLast = nowRTP - r.srNewest.RTPTimestamp + departureDiffSinceLast = now.Sub(r.srNewest.At) - expectedTimeDiffSinceLast := float64(rtpDiffSinceLast) / float64(r.params.ClockRate) + expectedTimeDiffSinceLast = float64(rtpDiffSinceLast) / float64(r.params.ClockRate) if math.Abs(expectedTimeDiffSinceLast-ntpDiffSinceLast.Seconds()) > 0.2 { // more than 200 ms away from expected delta isWarped = true } - - if isWarped { - expectedExtRTP := r.extStartTS + uint64(timeSinceFirst.Nanoseconds()*int64(r.params.ClockRate)/1e9) - ntpDiffLocal := nowNTP.Time().Sub(r.highestTime) - rtpDiffLocal := int32(nowRTP - r.highestTS) - timeSinceFirst, rtpDiffSinceFirst, drift, driftMs, sampleRate := r.getDrift() - r.logger.Infow( - "sending sender report, time warp", - "ntp", nowNTP.Time().String(), - "rtp", nowRTP, - "expectedRTP", uint32(expectedExtRTP), - "departure", now.String(), - "ntpDiffSinceLast", ntpDiffSinceLast.Seconds(), - "rtpDiffSinceLast", rtpDiffSinceLast, - "departureDiffSinceLast", departureDiffSinceLast.Seconds(), - "expectedTimeDiffSinceLast", expectedTimeDiffSinceLast, - "timeSinceFirst", timeSinceFirst.Seconds(), - "rtpDiffSinceFirst", rtpDiffSinceFirst, - "drift", drift, - "driftMs", driftMs, - "sampleRate", sampleRate, - "highestTS", r.highestTS, - "highestTime", r.highestTime.String(), - "rtpDiffLocal", rtpDiffLocal, - "ntpDiffLocal", ntpDiffLocal, - ) - } } - r.lastSRTime = now - r.lastSRNTP = nowNTP - r.lastSRRTP = nowRTP + r.srNewest = &RTCPSenderReportData{ + NTPTimestamp: nowNTP, + RTPTimestamp: nowRTP, + RTPTimestampExt: nowRTPExt, + At: now, + } + if r.srFirst == nil { + r.srFirst = r.srNewest + } + + if isWarped { + packetDriftResult, reportDriftResult := r.getDrift() + r.logger.Infow( + "sending sender report, time warp", + "ntp", nowNTP.Time().String(), + "rtp", nowRTP, + "departure", now.String(), + "ntpDiffSinceLast", ntpDiffSinceLast.Seconds(), + "rtpDiffSinceLast", int32(rtpDiffSinceLast), + "departureDiffSinceLast", departureDiffSinceLast.Seconds(), + "expectedTimeDiffSinceLast", expectedTimeDiffSinceLast, + "packetDrift", packetDriftResult.String(), + "reportDrift", reportDriftResult.String(), + "highestTS", r.highestTS, + "highestTime", r.highestTime.String(), + ) + } return &rtcp.SenderReport{ SSRC: ssrc, @@ -940,15 +1011,15 @@ func (r *RTPStats) SnapshotRtcpReceptionReport(ssrc uint32, proxyFracLost uint8, } var dlsr uint32 - if r.srData != nil && !r.srData.ArrivalTime.IsZero() { - delayMS := uint32(time.Since(r.srData.ArrivalTime).Milliseconds()) + if r.srNewest != nil && !r.srNewest.At.IsZero() { + delayMS := uint32(time.Since(r.srNewest.At).Milliseconds()) dlsr = (delayMS / 1e3) << 16 dlsr |= (delayMS % 1e3) * 65536 / 1000 } lastSR := uint32(0) - if r.srData != nil { - lastSR = uint32(r.srData.NTPTimestamp >> 16) + if r.srNewest != nil { + lastSR = uint32(r.srNewest.NTPTimestamp >> 16) } return &rtcp.ReceptionReport{ SSRC: ssrc, @@ -1218,7 +1289,7 @@ func (r *RTPStats) ToProto() *livekit.RTPStats { jitterTime := jitter / float64(r.params.ClockRate) * 1e6 maxJitterTime := maxJitter / float64(r.params.ClockRate) * 1e6 - _, _, _, driftMs, sampleRate := r.getDrift() + packetDrift, _ := r.getDrift() p := &livekit.RTPStats{ StartTime: timestamppb.New(r.startTime), @@ -1261,8 +1332,8 @@ func (r *RTPStats) ToProto() *livekit.RTPStats { LastFir: timestamppb.New(r.lastFir), RttCurrent: r.rtt, RttMax: r.maxRtt, - DriftMs: driftMs, - SampleRate: sampleRate, + DriftMs: packetDrift.driftMs, + SampleRate: packetDrift.sampleRate, } gapsPresent := false @@ -1456,12 +1527,20 @@ func (r *RTPStats) updateJitter(rtph *rtp.Header, packetTime time.Time) { r.lastJitterRTP = rtph.Timestamp } -func (r *RTPStats) getDrift() (timeSinceFirst time.Duration, rtpDiffSinceFirst uint64, drift int64, driftMs float64, sampleRate float64) { - timeSinceFirst = r.highestTime.Sub(r.firstTime) - rtpDiffSinceFirst = getExtTS(r.highestTS, r.tsCycles) - r.extStartTS - drift = int64(rtpDiffSinceFirst - uint64(timeSinceFirst.Nanoseconds()*int64(r.params.ClockRate)/1e9)) - driftMs = (float64(drift) * 1000) / float64(r.params.ClockRate) - sampleRate = float64(rtpDiffSinceFirst) / timeSinceFirst.Seconds() +func (r *RTPStats) getDrift() (packetDrift driftResult, reportDrift driftResult) { + packetDrift.timeSinceFirst = r.highestTime.Sub(r.firstTime) + packetDrift.rtpDiffSinceFirst = getExtTS(r.highestTS, r.tsCycles) - r.extStartTS + packetDrift.driftSamples = int64(packetDrift.rtpDiffSinceFirst - uint64(packetDrift.timeSinceFirst.Nanoseconds()*int64(r.params.ClockRate)/1e9)) + packetDrift.driftMs = (float64(packetDrift.driftSamples) * 1000) / float64(r.params.ClockRate) + packetDrift.sampleRate = float64(packetDrift.rtpDiffSinceFirst) / packetDrift.timeSinceFirst.Seconds() + + if r.srFirst != nil && r.srNewest != nil && r.srFirst.RTPTimestamp != r.srNewest.RTPTimestamp { + reportDrift.timeSinceFirst = r.srNewest.NTPTimestamp.Time().Sub(r.srFirst.NTPTimestamp.Time()) + reportDrift.rtpDiffSinceFirst = r.srNewest.RTPTimestampExt - r.srFirst.RTPTimestampExt + reportDrift.driftSamples = int64(reportDrift.rtpDiffSinceFirst - uint64(reportDrift.timeSinceFirst.Nanoseconds()*int64(r.params.ClockRate)/1e9)) + reportDrift.driftMs = (float64(reportDrift.driftSamples) * 1000) / float64(r.params.ClockRate) + reportDrift.sampleRate = float64(reportDrift.rtpDiffSinceFirst) / reportDrift.timeSinceFirst.Seconds() + } return } diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index c072645c6..edd0f9149 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -1116,7 +1116,8 @@ func (d *DownTrack) CreateSenderReport() *rtcp.SenderReport { return nil } - sr, tsAdjust := d.rtpStats.GetRtcpSenderReport(d.ssrc) + srFirst, srNewest := d.receiver.GetRTCPSenderReportData(d.forwarder.GetReferenceLayerSpatial()) + sr, tsAdjust := d.rtpStats.GetRtcpSenderReport(d.ssrc, srFirst, srNewest) if d.allowTimestampAdjustment { d.forwarder.AdjustTimestamp(tsAdjust) } @@ -1636,7 +1637,7 @@ func (d *DownTrack) DebugInfo() map[string]interface{} { } } -func (d *DownTrack) getExpectedRTPTimestamp(at time.Time) (uint32, uint32, error) { +func (d *DownTrack) getExpectedRTPTimestamp(at time.Time) (uint32, uint64, error) { return d.rtpStats.GetExpectedRTPTimestamp(at) } diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index ebede6e0e..b4f373c39 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -170,7 +170,7 @@ type Forwarder struct { kind webrtc.RTPCodecType logger logger.Logger getReferenceLayerRTPTimestamp func(ts uint32, layer int32, referenceLayer int32) (uint32, error) - getExpectedRTPTimestamp func(at time.Time) (uint32, uint32, error) + getExpectedRTPTimestamp func(at time.Time) (uint32, uint64, error) muted bool pubMuted bool @@ -201,7 +201,7 @@ func NewForwarder( kind webrtc.RTPCodecType, logger logger.Logger, getReferenceLayerRTPTimestamp func(ts uint32, layer int32, referenceLayer int32) (uint32, error), - getExpectedRTPTimestamp func(at time.Time) (uint32, uint32, error), + getExpectedRTPTimestamp func(at time.Time) (uint32, uint64, error), ) *Forwarder { f := &Forwarder{ kind: kind, @@ -488,6 +488,13 @@ func (f *Forwarder) TargetLayer() buffer.VideoLayer { return f.vls.GetTarget() } +func (f *Forwarder) GetReferenceLayerSpatial() int32 { + f.lock.RLock() + defer f.lock.RUnlock() + + return f.referenceLayerSpatial +} + func (f *Forwarder) isDeficientLocked() bool { return f.lastAllocation.IsDeficient } @@ -1475,7 +1482,7 @@ func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer i lastTS := f.rtpMunger.GetLast().LastTS refTS := lastTS expectedTS := lastTS - minTS := lastTS + minTS := uint64(lastTS) switchingAt := time.Now() if f.getReferenceLayerRTPTimestamp != nil { ts, err := f.getReferenceLayerRTPTimestamp(extPkt.Packet.Timestamp, layer, f.referenceLayerSpatial) @@ -1671,7 +1678,7 @@ func (f *Forwarder) GetSnTsForBlankFrames(frameRate uint32, numPackets int) ([]S lastTS := f.rtpMunger.GetLast().LastTS expectedTS := lastTS - minTS := lastTS + minTS := uint64(lastTS) if f.getExpectedRTPTimestamp != nil { ts, min, err := f.getExpectedRTPTimestamp(time.Now()) if err == nil { @@ -1826,7 +1833,7 @@ done: return float64(distance) / float64(maxSeenLayer.Temporal+1) } -func getNextTimestamp(lastTS uint32, refTS uint32, expectedTS uint32, minTS uint32) (uint32, string) { +func getNextTimestamp(lastTS uint32, refTS uint32, expectedTS uint32, minTS uint64) (uint32, string) { isInOrder := func(val1, val2 uint32) bool { diff := val1 - val2 return diff != 0 && diff < (1<<31) @@ -1860,8 +1867,8 @@ func getNextTimestamp(lastTS uint32, refTS uint32, expectedTS uint32, minTS uint explain = fmt.Sprintf("e < r < l, %d, %d", refTS-expectedTS, lastTS-refTS) } - if !isInOrder(nextTS, minTS) { - nextTS = minTS + 1 + if minTS != ^uint64(0) && !isInOrder(nextTS, uint32(minTS)) { + nextTS = uint32(minTS) + 1 } return nextTS, explain diff --git a/pkg/sfu/receiver.go b/pkg/sfu/receiver.go index 496e90ad0..eda4f7412 100644 --- a/pkg/sfu/receiver.go +++ b/pkg/sfu/receiver.go @@ -65,6 +65,7 @@ type TrackReceiver interface { GetTemporalLayerFpsForSpatial(layer int32) []float32 + GetRTCPSenderReportData(layer int32) (*buffer.RTCPSenderReportData, *buffer.RTCPSenderReportData) GetReferenceLayerRTPTimestamp(ts uint32, layer int32, referenceLayer int32) (uint32, error) } @@ -309,7 +310,8 @@ func (w *WebRTCReceiver) AddUpTrack(track *webrtc.TrackRemote, buff *buffer.Buff }) buff.OnRtcpFeedback(w.sendRTCP) buff.OnRtcpSenderReport(func(srData *buffer.RTCPSenderReportData) { - w.streamTrackerManager.SetRTCPSenderReportData(layer, buff.GetSenderReportData()) + srFirst, srNewest := buff.GetSenderReportData() + w.streamTrackerManager.SetRTCPSenderReportData(layer, srFirst, srNewest) w.downTrackSpreader.Broadcast(func(dt TrackSender) { _ = dt.HandleRTCPSenderReportData(w.codec.PayloadType, layer, srData) @@ -744,6 +746,10 @@ func (w *WebRTCReceiver) GetTemporalLayerFpsForSpatial(layer int32) []float32 { return b.GetTemporalLayerFpsForSpatial(layer) } +func (w *WebRTCReceiver) GetRTCPSenderReportData(layer int32) (*buffer.RTCPSenderReportData, *buffer.RTCPSenderReportData) { + return w.streamTrackerManager.GetRTCPSenderReportData(layer) +} + func (w *WebRTCReceiver) GetReferenceLayerRTPTimestamp(ts uint32, layer int32, referenceLayer int32) (uint32, error) { return w.streamTrackerManager.GetReferenceLayerRTPTimestamp(ts, layer, referenceLayer) } diff --git a/pkg/sfu/streamtrackermanager.go b/pkg/sfu/streamtrackermanager.go index 6b79914f9..cdb42f4ab 100644 --- a/pkg/sfu/streamtrackermanager.go +++ b/pkg/sfu/streamtrackermanager.go @@ -24,6 +24,11 @@ type StreamTrackerManagerListener interface { OnBitrateReport(availableLayers []int32, bitrates Bitrates) } +type endsSenderReport struct { + first *buffer.RTCPSenderReportData + newest *buffer.RTCPSenderReportData +} + type StreamTrackerManager struct { logger logger.Logger trackInfo *livekit.TrackInfo @@ -43,7 +48,7 @@ type StreamTrackerManager struct { paused bool senderReportMu sync.RWMutex - senderReports [buffer.DefaultMaxLayerSpatial + 1]*buffer.RTCPSenderReportData + senderReports [buffer.DefaultMaxLayerSpatial + 1]endsSenderReport closed core.Fuse @@ -475,7 +480,7 @@ func (s *StreamTrackerManager) maxExpectedLayerFromTrackInfo() { } } -func (s *StreamTrackerManager) SetRTCPSenderReportData(layer int32, senderReport *buffer.RTCPSenderReportData) { +func (s *StreamTrackerManager) SetRTCPSenderReportData(layer int32, srFirst *buffer.RTCPSenderReportData, srNewest *buffer.RTCPSenderReportData) { s.senderReportMu.Lock() defer s.senderReportMu.Unlock() @@ -483,7 +488,19 @@ func (s *StreamTrackerManager) SetRTCPSenderReportData(layer int32, senderReport return } - s.senderReports[layer] = senderReport + s.senderReports[layer].first = srFirst + s.senderReports[layer].newest = srNewest +} + +func (s *StreamTrackerManager) GetRTCPSenderReportData(layer int32) (*buffer.RTCPSenderReportData, *buffer.RTCPSenderReportData) { + s.senderReportMu.RLock() + defer s.senderReportMu.RUnlock() + + if layer < 0 || int(layer) >= len(s.senderReports) { + return nil, nil + } + + return s.senderReports[layer].first, s.senderReports[layer].newest } func (s *StreamTrackerManager) GetReferenceLayerRTPTimestamp(ts uint32, layer int32, referenceLayer int32) (uint32, error) { @@ -502,7 +519,7 @@ func (s *StreamTrackerManager) GetReferenceLayerRTPTimestamp(ts uint32, layer in var srLayer *buffer.RTCPSenderReportData if int(layer) < len(s.senderReports) { - srLayer = s.senderReports[layer] + srLayer = s.senderReports[layer].newest } if srLayer == nil || srLayer.NTPTimestamp == 0 { return 0, fmt.Errorf("layer rtcp sender report not available: %d", layer) @@ -510,7 +527,7 @@ func (s *StreamTrackerManager) GetReferenceLayerRTPTimestamp(ts uint32, layer in var srRef *buffer.RTCPSenderReportData if int(referenceLayer) < len(s.senderReports) { - srRef = s.senderReports[referenceLayer] + srRef = s.senderReports[referenceLayer].newest } if srRef == nil || srRef.NTPTimestamp == 0 { return 0, fmt.Errorf("reference layer rtcp sender report not available: %d", referenceLayer) From fc8375f1503aa918855a89dc4550c34995b755f3 Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Fri, 26 May 2023 14:34:35 +0800 Subject: [PATCH 185/324] Fix dynacast for svc codec (#1742) --- pkg/sfu/downtrack.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index edd0f9149..053faa5dc 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -624,7 +624,7 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { d.bytesSent.Add(uint32(hdr.MarshalSize() + len(payload))) if tp.isSwitchingToMaxSpatial && d.onMaxSubscribedLayerChanged != nil && d.kind == webrtc.RTPCodecTypeVideo { - d.onMaxSubscribedLayerChanged(d, layer) + d.onMaxSubscribedLayerChanged(d, d.forwarder.MaxLayer().Spatial) } if extPkt.KeyFrame { From 1c920812d335b87117755267bb38c2ec54187269 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 26 May 2023 12:49:31 +0530 Subject: [PATCH 186/324] Return max spatial layer from selectors. (#1743) * Return max spatial layer from selectors. With differing requirements of SVC and allowing overshoot in Simulcast, selectors are best placed to indicate what is the max spatial layer when they indicate a switch to max spatial layer. * fix test * prevent race --- pkg/rtc/room_test.go | 6 ++++++ pkg/sfu/downtrack.go | 2 +- pkg/sfu/forwarder.go | 2 ++ pkg/sfu/forwarder_test.go | 1 + pkg/sfu/videolayerselector/dependencydescriptor.go | 1 + pkg/sfu/videolayerselector/simulcast.go | 2 ++ pkg/sfu/videolayerselector/videolayerselector.go | 1 + pkg/sfu/videolayerselector/vp9.go | 1 + 8 files changed, 15 insertions(+), 1 deletion(-) diff --git a/pkg/rtc/room_test.go b/pkg/rtc/room_test.go index 9179c14fc..9afba5250 100644 --- a/pkg/rtc/room_test.go +++ b/pkg/rtc/room_test.go @@ -140,7 +140,9 @@ func TestRoomJoin(t *testing.T) { t.Run("cannot exceed max participants", func(t *testing.T) { rm := newRoomWithParticipants(t, testRoomOpts{num: 1}) + rm.lock.Lock() rm.protoRoom.MaxParticipants = 1 + rm.lock.Unlock() p := newMockParticipant("second", types.ProtocolVersion(0), false, false) err := rm.Join(p, nil, nil, iceServersForRoom) @@ -347,8 +349,10 @@ func TestRoomClosure(t *testing.T) { isClosed = true }) p := rm.GetParticipants()[0] + rm.lock.Lock() // allows immediate close after rm.protoRoom.EmptyTimeout = 0 + rm.lock.Unlock() rm.RemoveParticipant(p.Identity(), p.ID(), types.ParticipantCloseReasonClientRequestLeave) time.Sleep(time.Duration(RoomDepartureGrace)*time.Second + defaultDelay) @@ -377,7 +381,9 @@ func TestRoomClosure(t *testing.T) { rm.OnClose(func() { isClosed = true }) + rm.lock.Lock() rm.protoRoom.EmptyTimeout = 1 + rm.lock.Unlock() time.Sleep(1010 * time.Millisecond) rm.CloseIfEmpty() diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 053faa5dc..e8704da08 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -624,7 +624,7 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { d.bytesSent.Add(uint32(hdr.MarshalSize() + len(payload))) if tp.isSwitchingToMaxSpatial && d.onMaxSubscribedLayerChanged != nil && d.kind == webrtc.RTPCodecTypeVideo { - d.onMaxSubscribedLayerChanged(d, d.forwarder.MaxLayer().Spatial) + d.onMaxSubscribedLayerChanged(d, tp.maxSpatialLayer) } if extPkt.KeyFrame { diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index b4f373c39..0690d4084 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -129,6 +129,7 @@ type TranslationParams struct { isResuming bool isSwitchingToRequestSpatial bool isSwitchingToMaxSpatial bool + maxSpatialLayer int32 rtp *TranslationParamsRTP codecBytes []byte ddBytes []byte @@ -1570,6 +1571,7 @@ func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer in tp.isResuming = result.IsResuming tp.isSwitchingToRequestSpatial = result.IsSwitchingToRequestSpatial tp.isSwitchingToMaxSpatial = result.IsSwitchingToMaxSpatial + tp.maxSpatialLayer = result.MaxSpatialLayer tp.ddBytes = result.DependencyDescriptorExtension tp.marker = result.RTPMarker diff --git a/pkg/sfu/forwarder_test.go b/pkg/sfu/forwarder_test.go index df03a058a..06c294f9d 100644 --- a/pkg/sfu/forwarder_test.go +++ b/pkg/sfu/forwarder_test.go @@ -1717,6 +1717,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { require.NoError(t, err) expectedTP = TranslationParams{ isSwitchingToMaxSpatial: true, + maxSpatialLayer: 1, rtp: &TranslationParamsRTP{ snOrdering: SequenceNumberOrderingContiguous, sequenceNumber: 23339, diff --git a/pkg/sfu/videolayerselector/dependencydescriptor.go b/pkg/sfu/videolayerselector/dependencydescriptor.go index 1fa1fc52a..1c04ef442 100644 --- a/pkg/sfu/videolayerselector/dependencydescriptor.go +++ b/pkg/sfu/videolayerselector/dependencydescriptor.go @@ -230,6 +230,7 @@ func (d *DependencyDescriptor) Select(extPkt *buffer.ExtPacket, _layer int32) (r } if d.currentLayer.Spatial == d.maxLayer.Spatial { result.IsSwitchingToMaxSpatial = true + result.MaxSpatialLayer = d.currentLayer.Spatial d.logger.Infow( "reached max layer", "current", d.currentLayer, diff --git a/pkg/sfu/videolayerselector/simulcast.go b/pkg/sfu/videolayerselector/simulcast.go index fc15c4db0..fe6f33659 100644 --- a/pkg/sfu/videolayerselector/simulcast.go +++ b/pkg/sfu/videolayerselector/simulcast.go @@ -98,6 +98,7 @@ func (s *Simulcast) Select(extPkt *buffer.ExtPacket, layer int32) (result VideoL if s.currentLayer.Spatial >= s.maxLayer.Spatial { result.IsSwitchingToMaxSpatial = true + result.MaxSpatialLayer = s.currentLayer.Spatial s.logger.Infow( "reached max layer", "current", s.currentLayer, @@ -136,6 +137,7 @@ func (s *Simulcast) Select(extPkt *buffer.ExtPacket, layer int32) (result VideoL if s.currentLayer.Spatial >= s.maxLayer.Spatial { result.IsSwitchingToMaxSpatial = true + result.MaxSpatialLayer = s.currentLayer.Spatial } if s.currentLayer.Spatial >= s.maxLayer.Spatial || s.currentLayer.Spatial == s.maxSeenLayer.Spatial { diff --git a/pkg/sfu/videolayerselector/videolayerselector.go b/pkg/sfu/videolayerselector/videolayerselector.go index 14229e182..fa1e71d83 100644 --- a/pkg/sfu/videolayerselector/videolayerselector.go +++ b/pkg/sfu/videolayerselector/videolayerselector.go @@ -11,6 +11,7 @@ type VideoLayerSelectorResult struct { IsResuming bool IsSwitchingToRequestSpatial bool IsSwitchingToMaxSpatial bool + MaxSpatialLayer int32 RTPMarker bool DependencyDescriptorExtension []byte } diff --git a/pkg/sfu/videolayerselector/vp9.go b/pkg/sfu/videolayerselector/vp9.go index 9f7533352..4a77f3675 100644 --- a/pkg/sfu/videolayerselector/vp9.go +++ b/pkg/sfu/videolayerselector/vp9.go @@ -85,6 +85,7 @@ func (v *VP9) Select(extPkt *buffer.ExtPacket, _layer int32) (result VideoLayerS if v.currentLayer.Spatial != v.maxLayer.Spatial && updatedLayer.Spatial == v.maxLayer.Spatial { result.IsSwitchingToMaxSpatial = true + result.MaxSpatialLayer = updatedLayer.Spatial v.logger.Infow( "reached max layer", "current", v.currentLayer, From 9dd2ebc9603193fbcaea216ae3df555eaee0a83f Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sat, 27 May 2023 12:19:30 +0530 Subject: [PATCH 187/324] Change too many packets log to error to get back trace. (#1744) --- pkg/sfu/buffer/rtpstats.go | 2 +- pkg/sfu/downtrack.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index d650f33a0..fda74a257 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -1049,7 +1049,7 @@ func (r *RTPStats) DeltaInfo(snapshotId uint32) *RTPDeltaInfo { packetsExpected := now.extStartSN - then.extStartSN if packetsExpected > NumSequenceNumbers { - r.logger.Warnw( + r.logger.Errorw( "too many packets expected in delta", fmt.Errorf("start: %d, end: %d, expected: %d", then.extStartSN, now.extStartSN, packetsExpected), ) diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index e8704da08..8fc30d542 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -114,8 +114,8 @@ type DownTrackState struct { } func (d DownTrackState) String() string { - return fmt.Sprintf("DownTrackState{rtpStats: %s, delta: %d, forwarder: %s}", - d.RTPStats.ToString(), d.DeltaStatsSnapshotId, d.ForwarderState.String()) + return fmt.Sprintf("DownTrackState{rtpStats: %s, delta: %d, deltaOverridden: %d, forwarder: %s}", + d.RTPStats.ToString(), d.DeltaStatsSnapshotId, d.DeltaStatsOverriddenSnapshotId, d.ForwarderState.String()) } // ------------------------------------------------------------------- From aefbdde3b880a5d15eb253a4921b69d9d68d0d64 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 27 May 2023 21:33:24 -0700 Subject: [PATCH 188/324] Update pion deps (#1706) * Update pion deps Generated by renovateBot * remove active TCP override --------- Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: David Zhao --- go.mod | 11 ++++------- go.sum | 19 ++++++++----------- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index f631f2b85..82ba7b49f 100644 --- a/go.mod +++ b/go.mod @@ -26,14 +26,14 @@ require ( github.com/mitchellh/go-homedir v1.1.0 github.com/olekukonko/tablewriter v0.0.5 github.com/pion/dtls/v2 v2.2.7 - github.com/pion/ice/v2 v2.3.4 - github.com/pion/interceptor v0.1.16 + github.com/pion/ice/v2 v2.3.6 + github.com/pion/interceptor v0.1.17 github.com/pion/rtcp v1.2.10 github.com/pion/rtp v1.7.13 github.com/pion/sdp/v3 v3.0.6 github.com/pion/transport/v2 v2.2.1 github.com/pion/turn/v2 v2.1.0 - github.com/pion/webrtc/v3 v3.2.4 + github.com/pion/webrtc/v3 v3.2.8 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.15.1 github.com/redis/go-redis/v9 v9.0.4 @@ -50,9 +50,6 @@ require ( gopkg.in/yaml.v3 v3.0.1 ) -// fix to version before active TCP was introduced, until there's a way to disable it -replace github.com/pion/ice/v2 => github.com/pion/ice/v2 v2.3.3 - require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect @@ -83,7 +80,7 @@ require ( github.com/pion/mdns v0.0.7 // indirect github.com/pion/randutil v0.1.0 // indirect github.com/pion/sctp v1.8.7 // indirect - github.com/pion/srtp/v2 v2.0.14 // indirect + github.com/pion/srtp/v2 v2.0.15 // indirect github.com/pion/stun v0.6.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect diff --git a/go.sum b/go.sum index 8c63d4a19..c105d047b 100644 --- a/go.sum +++ b/go.sum @@ -179,13 +179,12 @@ github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAl github.com/onsi/gomega v1.26.0 h1:03cDLK28U6hWvCAns6NeydX3zIm4SF3ci69ulidS32Q= github.com/pion/datachannel v1.5.5 h1:10ef4kwdjije+M9d7Xm9im2Y3O6A6ccQb0zcqZcJew8= github.com/pion/datachannel v1.5.5/go.mod h1:iMz+lECmfdCMqFRhXhcA/219B0SQlbpoR2V118yimL0= -github.com/pion/dtls/v2 v2.2.6/go.mod h1:t8fWJCIquY5rlQZwA2yWxUS1+OCrAdXrhVKXB5oD/wY= github.com/pion/dtls/v2 v2.2.7 h1:cSUBsETxepsCSFSxC3mc/aDo14qQLMSL+O6IjG28yV8= github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= -github.com/pion/ice/v2 v2.3.3 h1:uGrUwn0DanTmXgFiKDTot7iQzo0J9NjTjHPG+Kt+kNE= -github.com/pion/ice/v2 v2.3.3/go.mod h1:jVbxqPWQDK5+/V/YqpinUcP0YtDGYqd24n2lusVdX80= -github.com/pion/interceptor v0.1.16 h1:0GDZrfNO+BmVNWymS31fMlVtPO2IJVBzy2Qq5XCYMIg= -github.com/pion/interceptor v0.1.16/go.mod h1:SY8kpmfVBvrbUzvj2bsXz7OJt5JvmVNZ+4Kjq7FcwrI= +github.com/pion/ice/v2 v2.3.6 h1:Jgqw36cAud47iD+N6rNX225uHvrgWtAlHfVyOQc3Heg= +github.com/pion/ice/v2 v2.3.6/go.mod h1:9/TzKDRwBVAPsC+YOrKH/e3xDrubeTRACU9/sHQarsU= +github.com/pion/interceptor v0.1.17 h1:prJtgwFh/gB8zMqGZoOgJPHivOwVAp61i2aG61Du/1w= +github.com/pion/interceptor v0.1.17/go.mod h1:SY8kpmfVBvrbUzvj2bsXz7OJt5JvmVNZ+4Kjq7FcwrI= github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= github.com/pion/mdns v0.0.7 h1:P0UB4Sr6xDWEox0kTVxF0LmQihtCbSAdW0H2nEgkA3U= @@ -201,10 +200,9 @@ github.com/pion/sctp v1.8.7 h1:JnABvFakZueGAn4KU/4PSKg+GWbF6QWbKTWZOSGJjXw= github.com/pion/sctp v1.8.7/go.mod h1:g1Ul+ARqZq5JEmoFy87Q/4CePtKnTJ1QCL9dBBdN6AU= github.com/pion/sdp/v3 v3.0.6 h1:WuDLhtuFUUVpTfus9ILC4HRyHsW6TdugjEX/QY9OiUw= github.com/pion/sdp/v3 v3.0.6/go.mod h1:iiFWFpQO8Fy3S5ldclBkpXqmWy02ns78NOKoLLL0YQw= -github.com/pion/srtp/v2 v2.0.14 h1:Glt0MqEvINrDxL+aanmK4DiFjvs+uN2iYc6XD/iKpoY= -github.com/pion/srtp/v2 v2.0.14/go.mod h1:b/pQOlDrbB0HEH5EUAQXzSYxikFbNcNuKmF8tM0hCtw= +github.com/pion/srtp/v2 v2.0.15 h1:+tqRtXGsGwHC0G0IUIAzRmdkHvriF79IHVfZGfHrQoA= +github.com/pion/srtp/v2 v2.0.15/go.mod h1:b/pQOlDrbB0HEH5EUAQXzSYxikFbNcNuKmF8tM0hCtw= github.com/pion/stun v0.4.0/go.mod h1:QPsh1/SbXASntw3zkkrIk3ZJVKz4saBY2G7S10P3wCw= -github.com/pion/stun v0.5.2/go.mod h1:TNo1HjyjaFVpMZsvowqPeV8TfwRytympQC0//neaksA= github.com/pion/stun v0.6.0 h1:JHT/2iyGDPrFWE8NNC15wnddBN8KifsEDw8swQmrEmU= github.com/pion/stun v0.6.0/go.mod h1:HPqcfoeqQn9cuaet7AOmB5e5xkObu9DwBdurwLKO9oA= github.com/pion/transport v0.14.1 h1:XSM6olwW+o8J4SCmOBb/BpwZypkHeyM0PGFCxNQBr40= @@ -218,8 +216,8 @@ github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1A github.com/pion/turn/v2 v2.1.0 h1:5wGHSgGhJhP/RpabkUb/T9PdsAjkGLS6toYz5HNzoSI= github.com/pion/turn/v2 v2.1.0/go.mod h1:yrT5XbXSGX1VFSF31A3c1kCNB5bBZgk/uu5LET162qs= github.com/pion/udp/v2 v2.0.1/go.mod h1:B7uvTMP00lzWdyMr/1PVZXtV3wpPIxBRd4Wl6AksXn8= -github.com/pion/webrtc/v3 v3.2.4 h1:gWSx4dqQb77051qBT9ipDrOyP6/sGYcAQP3UPjM8pU8= -github.com/pion/webrtc/v3 v3.2.4/go.mod h1:jtG9DOHcnIp7JMavANA9kTyz12sVnVRZx/rF0Awfd7I= +github.com/pion/webrtc/v3 v3.2.8 h1:RmDEz7wjK3k0sAuCSMptfxp095pBYSkSSm5ySiJYIHI= +github.com/pion/webrtc/v3 v3.2.8/go.mod h1:6/7wF1P86AQAw4iTmKIgdzaevaQ8qh9SfrFyypqmN6w= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -283,7 +281,6 @@ golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= From e99aabd908fe31ff7274836da2b892e17141e5b6 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 27 May 2023 21:34:15 -0700 Subject: [PATCH 189/324] Update go deps (#1697) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 82ba7b49f..fa5d628b1 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/livekit/protocol v1.5.7 github.com/livekit/psrpc v0.3.1-0.20230518234341-6f6847e10b09 github.com/mackerelio/go-osstat v0.2.4 - github.com/magefile/mage v1.14.0 + github.com/magefile/mage v1.15.0 github.com/maxbrunsfeld/counterfeiter/v6 v6.6.1 github.com/mitchellh/go-homedir v1.1.0 github.com/olekukonko/tablewriter v0.0.5 @@ -42,7 +42,7 @@ require ( github.com/thoas/go-funk v0.9.3 github.com/twitchtv/twirp v8.1.3+incompatible github.com/ua-parser/uap-go v0.0.0-20211112212520-00c877edfe0f - github.com/urfave/cli/v2 v2.25.3 + github.com/urfave/cli/v2 v2.25.4 github.com/urfave/negroni/v3 v3.0.0 go.uber.org/atomic v1.11.0 golang.org/x/sync v0.2.0 diff --git a/go.sum b/go.sum index c105d047b..a60a6ea9c 100644 --- a/go.sum +++ b/go.sum @@ -127,8 +127,8 @@ github.com/livekit/psrpc v0.3.1-0.20230518234341-6f6847e10b09 h1:mb6jRcg57U0HQ4t github.com/livekit/psrpc v0.3.1-0.20230518234341-6f6847e10b09/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= github.com/mackerelio/go-osstat v0.2.4/go.mod h1:Zy+qzGdZs3A9cuIqmgbJvwbmLQH9dJvtio5ZjJTbdlQ= -github.com/magefile/mage v1.14.0 h1:6QDX3g6z1YvJ4olPhT1wksUcSa/V0a1B+pJb73fBjyo= -github.com/magefile/mage v1.14.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= +github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= +github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= @@ -260,8 +260,8 @@ github.com/twitchtv/twirp v8.1.3+incompatible h1:+F4TdErPgSUbMZMwp13Q/KgDVuI7HJX github.com/twitchtv/twirp v8.1.3+incompatible/go.mod h1:RRJoFSAmTEh2weEqWtpPE3vFK5YBhA6bqp2l1kfCC5A= github.com/ua-parser/uap-go v0.0.0-20211112212520-00c877edfe0f h1:A+MmlgpvrHLeUP8dkBVn4Pnf5Bp5Yk2OALm7SEJLLE8= github.com/ua-parser/uap-go v0.0.0-20211112212520-00c877edfe0f/go.mod h1:OBcG9bn7sHtXgarhUEb3OfCnNsgtGnkVf41ilSZ3K3E= -github.com/urfave/cli/v2 v2.25.3 h1:VJkt6wvEBOoSjPFQvOkv6iWIrsJyCrKGtCtxXWwmGeY= -github.com/urfave/cli/v2 v2.25.3/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= +github.com/urfave/cli/v2 v2.25.4 h1:HyYwPrTO3im9rYhUff/ZNs78eolxt0nJ4LN+9yJKSH4= +github.com/urfave/cli/v2 v2.25.4/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= github.com/urfave/negroni/v3 v3.0.0 h1:Vo8CeZfu1lFR9gW8GnAb6dOGCJyijfil9j/jKKc/JhU= github.com/urfave/negroni/v3 v3.0.0/go.mod h1:jWvnX03kcSjDBl/ShB0iHvx5uOs7mAzZXW+JvJ5XYAs= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= From ea57e4f2c1edf5f6dae1df48d63ea5ee4d991351 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sun, 28 May 2023 10:05:35 +0530 Subject: [PATCH 190/324] Ignore receiver reports that have a sequence number before first packet. (#1745) --- pkg/sfu/buffer/rtpstats.go | 10 ++++++---- pkg/sfu/downtrack.go | 8 ++++---- pkg/sfu/forwarder.go | 3 +-- pkg/sfu/forwarder_test.go | 4 ++-- pkg/sfu/streamallocator/track.go | 2 +- 5 files changed, 14 insertions(+), 13 deletions(-) diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index fda74a257..87a403bfa 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -539,7 +539,9 @@ func (r *RTPStats) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt uint32 r.lock.Lock() defer r.lock.Unlock() - if !r.endTime.IsZero() || !r.params.IsReceiverReportDriven { + if !r.endTime.IsZero() || !r.params.IsReceiverReportDriven || rr.LastSequenceNumber < r.extStartSN { + // it is possible that the `LastSequenceNumber` in the receiver report is before the starting + // sequence number when dummy packets are used to trigger Pion's OnTrack path. return } @@ -1107,7 +1109,7 @@ func (r *RTPStats) DeltaInfoOverridden(snapshotId uint32) *RTPDeltaInfo { packetsExpected := now.extStartSNOverridden - then.extStartSNOverridden if packetsExpected > NumSequenceNumbers { r.logger.Warnw( - "too many packets expected in delta", + "too many packets expected in delta (overridden)", fmt.Errorf("start: %d, end: %d, expected: %d", then.extStartSNOverridden, now.extStartSNOverridden, packetsExpected), ) return nil @@ -1558,7 +1560,7 @@ func (r *RTPStats) updateGapHistogram(gap int) { } func (r *RTPStats) getAndResetSnapshot(snapshotId uint32, override bool) (*Snapshot, *Snapshot) { - if !r.initialized || (r.params.IsReceiverReportDriven && r.lastRRTime.IsZero()) { + if !r.initialized || (override && r.lastRRTime.IsZero()) { return nil, nil } @@ -1573,7 +1575,7 @@ func (r *RTPStats) getAndResetSnapshot(snapshotId uint32, override bool) (*Snaps } var startTime time.Time - if override && r.params.IsReceiverReportDriven { + if override { startTime = r.lastRRTime } else { startTime = time.Now() diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 8fc30d542..0e8e12dbe 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -652,7 +652,7 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { // WritePaddingRTP tries to write as many padding only RTP packets as necessary // to satisfy given size to the DownTrack -func (d *DownTrack) WritePaddingRTP(bytesToSend int, paddingOnMute bool) int { +func (d *DownTrack) WritePaddingRTP(bytesToSend int, paddingOnMute bool, forceMarker bool) int { if !d.rtpStats.IsActive() && !paddingOnMute { return 0 } @@ -684,7 +684,7 @@ func (d *DownTrack) WritePaddingRTP(bytesToSend int, paddingOnMute bool) int { return 0 } - snts, err := d.forwarder.GetSnTsForPadding(num) + snts, err := d.forwarder.GetSnTsForPadding(num, forceMarker) if err != nil { return 0 } @@ -1699,10 +1699,10 @@ func (d *DownTrack) onBindAndConnected() { } func (d *DownTrack) sendPaddingOnMute() { - d.logger.Debugw("sending padding on mute") // let uptrack have chance to send packet before we send padding time.Sleep(waitBeforeSendPaddingOnMute) + d.logger.Debugw("sending padding on mute") if d.kind == webrtc.RTPCodecTypeVideo { d.sendPaddingOnMuteForVideo() } else if d.mime == "audio/opus" { @@ -1717,7 +1717,7 @@ func (d *DownTrack) sendPaddingOnMuteForVideo() { if d.rtpStats.IsActive() || d.IsClosed() { return } - d.WritePaddingRTP(20, true) + d.WritePaddingRTP(20, true, true) time.Sleep(paddingOnMuteInterval) } } diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 0690d4084..b2c4f4f27 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -1649,7 +1649,7 @@ func (f *Forwarder) maybeStart() { f.firstTS = extPkt.Packet.Timestamp } -func (f *Forwarder) GetSnTsForPadding(num int) ([]SnTs, error) { +func (f *Forwarder) GetSnTsForPadding(num int, forceMarker bool) ([]SnTs, error) { f.lock.Lock() defer f.lock.Unlock() @@ -1660,7 +1660,6 @@ func (f *Forwarder) GetSnTsForPadding(num int) ([]SnTs, error) { // not get out-of-sync. But, when a stream is paused, // force a frame marker as a restart of the stream will // start with a key frame which will reset the decoder. - forceMarker := false if !f.vls.GetTarget().IsValid() { forceMarker = true } diff --git a/pkg/sfu/forwarder_test.go b/pkg/sfu/forwarder_test.go index 06c294f9d..df82fa86d 100644 --- a/pkg/sfu/forwarder_test.go +++ b/pkg/sfu/forwarder_test.go @@ -1770,7 +1770,7 @@ func TestForwardGetSnTsForPadding(t *testing.T) { disable(f) // should get back frame end needed as the last packet did not have RTP marker set - snts, err := f.GetSnTsForPadding(5) + snts, err := f.GetSnTsForPadding(5, false) require.NoError(t, err) numPadding := 5 @@ -1786,7 +1786,7 @@ func TestForwardGetSnTsForPadding(t *testing.T) { require.Equal(t, sntsExpected, snts) // now that there is a marker, timestamp should jump on first padding when asked again - snts, err = f.GetSnTsForPadding(numPadding) + snts, err = f.GetSnTsForPadding(numPadding, false) require.NoError(t, err) for i := 0; i < numPadding; i++ { diff --git a/pkg/sfu/streamallocator/track.go b/pkg/sfu/streamallocator/track.go index d3aedcef4..4c1a125e4 100644 --- a/pkg/sfu/streamallocator/track.go +++ b/pkg/sfu/streamallocator/track.go @@ -135,7 +135,7 @@ func (t *Track) SetMaxLayer(layer buffer.VideoLayer) bool { } func (t *Track) WritePaddingRTP(bytesToSend int) int { - return t.downTrack.WritePaddingRTP(bytesToSend, false) + return t.downTrack.WritePaddingRTP(bytesToSend, false, false) } func (t *Track) AllocateOptimal(allowOvershoot bool) sfu.VideoAllocation { From 2edd257705b9ad8b2d298f31f49c34e9b4314aaa Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Sun, 28 May 2023 12:54:23 -0700 Subject: [PATCH 191/324] update psrpc (#1749) --- go.mod | 7 ++++--- go.sum | 15 ++++++++------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index fa5d628b1..06d693ed8 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230523035537-27577c4e1646 github.com/livekit/protocol v1.5.7 - github.com/livekit/psrpc v0.3.1-0.20230518234341-6f6847e10b09 + github.com/livekit/psrpc v0.3.1-0.20230528083849-53d664c6d912 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 github.com/maxbrunsfeld/counterfeiter/v6 v6.6.1 @@ -67,12 +67,13 @@ require ( github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.7.2 // indirect github.com/josharian/native v1.1.0 // indirect + github.com/klauspost/compress v1.16.5 // indirect github.com/lithammer/shortuuid/v4 v4.0.0 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mdlayher/netlink v1.7.1 // indirect github.com/mdlayher/socket v0.4.0 // indirect - github.com/nats-io/nats.go v1.25.0 // indirect + github.com/nats-io/nats.go v1.26.0 // indirect github.com/nats-io/nkeys v0.4.4 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/pion/datachannel v1.5.5 // indirect @@ -97,7 +98,7 @@ require ( golang.org/x/sys v0.8.0 // indirect golang.org/x/text v0.9.0 // indirect golang.org/x/tools v0.6.0 // indirect - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e // indirect google.golang.org/grpc v1.55.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index a60a6ea9c..bef82bb4b 100644 --- a/go.sum +++ b/go.sum @@ -106,7 +106,8 @@ github.com/jsimonetti/rtnetlink v0.0.0-20211022192332-93da33804786 h1:N527AHMa79 github.com/jsimonetti/rtnetlink v0.0.0-20211022192332-93da33804786/go.mod h1:v4hqbTdfQngbVSZJVWUhGE/lbTFf9jb+ygmNUDQMuOs= github.com/jxskiss/base62 v1.1.0 h1:A5zbF8v8WXx2xixnAKD2w+abC+sIzYJX+nxmhA6HWFw= github.com/jxskiss/base62 v1.1.0/go.mod h1:HhWAlUXvxKThfOlZbcuFzsqwtF5TcqS9ru3y5GfjWAc= -github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw= +github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= +github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= @@ -123,8 +124,8 @@ github.com/livekit/mediatransportutil v0.0.0-20230523035537-27577c4e1646 h1:acGS github.com/livekit/mediatransportutil v0.0.0-20230523035537-27577c4e1646/go.mod h1:MRc0zSOSzXuFt0X218SgabzlaKevkvCckPgBEoHYc34= github.com/livekit/protocol v1.5.7 h1:jZeFQEmLuIhFblXDGPRCBbfjVJHb+YU7AsO+SMoXF70= github.com/livekit/protocol v1.5.7/go.mod h1:ZaOnsvP+JS4s7vI1UO+JVdBagvvLp/lBXDAl2hkDS0I= -github.com/livekit/psrpc v0.3.1-0.20230518234341-6f6847e10b09 h1:mb6jRcg57U0HQ4tKRsueHHKcvTqBinL6+0Aa84vTtWk= -github.com/livekit/psrpc v0.3.1-0.20230518234341-6f6847e10b09/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= +github.com/livekit/psrpc v0.3.1-0.20230528083849-53d664c6d912 h1:q6i7ptxK6yZ220eyCWx5l1ZsUMQNyuYZ5feG91sM8EI= +github.com/livekit/psrpc v0.3.1-0.20230528083849-53d664c6d912/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= github.com/mackerelio/go-osstat v0.2.4/go.mod h1:Zy+qzGdZs3A9cuIqmgbJvwbmLQH9dJvtio5ZjJTbdlQ= github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= @@ -159,8 +160,8 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/nats-io/jwt/v2 v2.3.0 h1:z2mA1a7tIf5ShggOFlR1oBPgd6hGqcDYsISxZByUzdI= github.com/nats-io/nats-server/v2 v2.9.8 h1:jgxZsv+A3Reb3MgwxaINcNq/za8xZInKhDg9Q0cGN1o= -github.com/nats-io/nats.go v1.25.0 h1:t5/wCPGciR7X3Mu8QOi4jiJaXaWM8qtkLu4lzGZvYHE= -github.com/nats-io/nats.go v1.25.0/go.mod h1:D2WALIhz7V8M0pH8Scx8JZXlg6Oqz5VG+nQkK8nJdvg= +github.com/nats-io/nats.go v1.26.0 h1:fWJTYPnZ8DzxIaqIHOAMfColuznchnd5Ab5dbJpgPIE= +github.com/nats-io/nats.go v1.26.0/go.mod h1:XpbWUlOElGwTYbMR7imivs7jJj9GtK7ypv321Wp6pjc= github.com/nats-io/nkeys v0.4.4 h1:xvBJ8d69TznjcQl9t6//Q5xXuVhyYiSos6RPtvQNTwA= github.com/nats-io/nkeys v0.4.4/go.mod h1:XUkxdLPTufzlihbamfzQ7mw/VGx6ObUs+0bN5sNvt64= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= @@ -397,8 +398,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e h1:NumxXLPfHSndr3wBBdeKiVHjGVFzi9RX2HwwQke94iY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= From fdfd830394b64de6671f230f4c25810beb669142 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 29 May 2023 14:41:44 +0530 Subject: [PATCH 192/324] Split probe controller from StreamAllocator. (#1751) * Split probe controller from StreamAllocator. With TWCC, there is a need to check for probe status in a separate goroutine. So, probe specific stuff need locking. Split out the probe controller to make that cleaner. * remove defer --- pkg/sfu/streamallocator/probe_controller.go | 263 ++++++++++++++++++++ pkg/sfu/streamallocator/streamallocator.go | 175 ++----------- 2 files changed, 290 insertions(+), 148 deletions(-) create mode 100644 pkg/sfu/streamallocator/probe_controller.go diff --git a/pkg/sfu/streamallocator/probe_controller.go b/pkg/sfu/streamallocator/probe_controller.go new file mode 100644 index 000000000..f32c38b14 --- /dev/null +++ b/pkg/sfu/streamallocator/probe_controller.go @@ -0,0 +1,263 @@ +package streamallocator + +import ( + "sync" + "time" + + "github.com/livekit/protocol/logger" +) + +const ( + ProbeWaitBase = 5 * time.Second + ProbeBackoffFactor = 1.5 + ProbeWaitMax = 30 * time.Second + ProbeSettleWait = 250 + ProbeTrendWait = 2 * time.Second + + ProbePct = 120 + ProbeMinBps = 200 * 1000 // 200 kbps + ProbeMinDuration = 20 * time.Second + ProbeMaxDuration = 21 * time.Second +) + +// --------------------------------------------------------------------------- + +type ProbeControllerParams struct { + Prober *Prober + Logger logger.Logger +} + +type ProbeController struct { + params ProbeControllerParams + + lock sync.RWMutex + probeInterval time.Duration + lastProbeStartTime time.Time + probeGoalBps int64 + probeClusterId ProbeClusterId + abortedProbeClusterId ProbeClusterId + probeTrendObserved bool + probeEndTime time.Time + + onProbeSuccess func() +} + +func NewProbeController(params ProbeControllerParams) *ProbeController { + p := &ProbeController{ + params: params, + } + + p.Reset() + return p +} + +func (p *ProbeController) OnProbeSuccess(f func()) { + p.lock.Lock() + defer p.lock.Unlock() + + p.onProbeSuccess = f +} + +func (p *ProbeController) Reset() { + p.lock.Lock() + defer p.lock.Unlock() + + p.lastProbeStartTime = time.Now() + + p.resetProbeIntervalLocked() + + p.clearProbeLocked() +} + +func (p *ProbeController) ProbeClusterDone(info ProbeClusterInfo, lowestEstimate int64) { + p.lock.Lock() + if p.probeClusterId != info.Id { + p.lock.Unlock() + return + } + + if p.abortedProbeClusterId == ProbeClusterIdInvalid { + // successful probe, finalize + isSuccessful := p.finalizeProbeLocked() + + var onProbeSuccess func() + if isSuccessful { + onProbeSuccess = p.onProbeSuccess + } + p.lock.Unlock() + + if onProbeSuccess != nil { + onProbeSuccess() + } + return + } + + // ensure probe queue is flushed + // STREAM-ALLOCATOR-TODO: ProbeSettleWait should actually be a certain number of RTTs. + expectedDuration := float64(info.BytesSent*8*1000) / float64(lowestEstimate) + queueTime := expectedDuration - float64(info.Duration.Milliseconds()) + if queueTime < 0.0 { + queueTime = 0.0 + } + queueWait := time.Duration(queueTime+float64(ProbeSettleWait)) * time.Millisecond + p.probeEndTime = p.lastProbeStartTime.Add(queueWait) + p.lock.Unlock() +} + +func (p *ProbeController) CheckProbe(trend ChannelTrend, highestEstimate int64) { + p.lock.Lock() + defer p.lock.Unlock() + + if p.probeClusterId == ProbeClusterIdInvalid { + return + } + + if trend != ChannelTrendNeutral { + p.probeTrendObserved = true + } + + switch { + case !p.probeTrendObserved && time.Since(p.lastProbeStartTime) > ProbeTrendWait: + // + // More of a safety net. + // In rare cases, the estimate gets stuck. Prevent from probe running amok + // STREAM-ALLOCATOR-TODO: Need more testing here to ensure that probe does not cause a lot of damage + // + p.params.Logger.Infow("stream allocator: probe: aborting, no trend", "cluster", p.probeClusterId) + p.abortProbeLocked() + + case trend == ChannelTrendCongesting: + // stop immediately if the probe is congesting channel more + p.params.Logger.Infow("stream allocator: probe: aborting, channel is congesting", "cluster", p.probeClusterId) + p.abortProbeLocked() + + case highestEstimate > p.probeGoalBps: + // reached goal, stop probing + p.params.Logger.Infow( + "stream allocator: probe: stopping, goal reached", + "cluster", p.probeClusterId, + "goal", p.probeGoalBps, + "highest", highestEstimate, + ) + p.StopProbe() + } +} + +func (p *ProbeController) MaybeFinalizeProbe() { + p.lock.Lock() + isSuccessful := false + if p.isInProbeLocked() && !p.probeEndTime.IsZero() && time.Now().After(p.probeEndTime) { + isSuccessful = p.finalizeProbeLocked() + } + + var onProbeSuccess func() + if isSuccessful { + onProbeSuccess = p.onProbeSuccess + } + p.lock.Unlock() + + if onProbeSuccess != nil { + onProbeSuccess() + } +} + +func (p *ProbeController) DoesProbeNeedFinalize() bool { + p.lock.RLock() + defer p.lock.RUnlock() + + return p.abortedProbeClusterId != ProbeClusterIdInvalid +} + +func (p *ProbeController) finalizeProbeLocked() bool { + aborted := p.probeClusterId == p.abortedProbeClusterId + + p.clearProbeLocked() + + if aborted { + // failed probe, backoff + p.backoffProbeIntervalLocked() + return false + } + + // reset probe interval on a successful probe + p.resetProbeIntervalLocked() + return true +} + +func (p *ProbeController) InitProbe(probeGoalDeltaBps int64, expectedBandwidthUsage int64) (ProbeClusterId, int64) { + p.lock.Lock() + defer p.lock.Unlock() + + p.lastProbeStartTime = time.Now() + + // overshoot a bit to account for noise (in measurement/estimate etc) + p.probeGoalBps = expectedBandwidthUsage + ((probeGoalDeltaBps * ProbePct) / 100) + + p.abortedProbeClusterId = ProbeClusterIdInvalid + + p.probeTrendObserved = false + + p.probeEndTime = time.Time{} + + p.probeClusterId = p.params.Prober.AddCluster( + ProbeClusterModeUniform, + int(p.probeGoalBps), + int(expectedBandwidthUsage), + ProbeMinDuration, + ProbeMaxDuration, + ) + + return p.probeClusterId, p.probeGoalBps +} + +func (p *ProbeController) clearProbeLocked() { + p.probeClusterId = ProbeClusterIdInvalid + p.abortedProbeClusterId = ProbeClusterIdInvalid +} + +func (p *ProbeController) backoffProbeIntervalLocked() { + p.probeInterval = time.Duration(p.probeInterval.Seconds()*ProbeBackoffFactor) * time.Second + if p.probeInterval > ProbeWaitMax { + p.probeInterval = ProbeWaitMax + } +} + +func (p *ProbeController) resetProbeIntervalLocked() { + p.probeInterval = ProbeWaitBase +} + +func (p *ProbeController) StopProbe() { + p.params.Prober.Reset() +} + +func (p *ProbeController) AbortProbe() { + p.lock.Lock() + defer p.lock.Unlock() + + p.abortProbeLocked() +} + +func (p *ProbeController) abortProbeLocked() { + p.abortedProbeClusterId = p.probeClusterId + p.StopProbe() +} + +func (p *ProbeController) IsInProbe() bool { + p.lock.RLock() + defer p.lock.RUnlock() + + return p.isInProbeLocked() +} + +func (p *ProbeController) isInProbeLocked() bool { + return p.probeClusterId != ProbeClusterIdInvalid +} + +func (p *ProbeController) CanProbe() bool { + p.lock.RLock() + defer p.lock.RUnlock() + + return time.Since(p.lastProbeStartTime) >= p.probeInterval && p.probeClusterId == ProbeClusterIdInvalid +} + +// ------------------------------------------------ diff --git a/pkg/sfu/streamallocator/streamallocator.go b/pkg/sfu/streamallocator/streamallocator.go index b680ae817..7cc35f84a 100644 --- a/pkg/sfu/streamallocator/streamallocator.go +++ b/pkg/sfu/streamallocator/streamallocator.go @@ -25,17 +25,6 @@ const ( NackRatioAttenuator = 0.4 // how much to attenuate NACK ratio while calculating loss adjusted estimate - ProbeWaitBase = 5 * time.Second - ProbeBackoffFactor = 1.5 - ProbeWaitMax = 30 * time.Second - ProbeSettleWait = 250 - ProbeTrendWait = 2 * time.Second - - ProbePct = 120 - ProbeMinBps = 200 * 1000 // 200 kbps - ProbeMinDuration = 20 * time.Second - ProbeMaxDuration = 21 * time.Second - PriorityMin = uint8(1) PriorityMax = uint8(255) PriorityDefaultScreenshare = PriorityMax @@ -175,13 +164,7 @@ type StreamAllocator struct { committedChannelCapacity int64 overriddenChannelCapacity int64 - probeInterval time.Duration - lastProbeStartTime time.Time - probeGoalBps int64 - probeClusterId ProbeClusterId - abortedProbeClusterId ProbeClusterId - probeTrendObserved bool - probeEndTime time.Time + probeController *ProbeController prober *Prober @@ -213,6 +196,12 @@ func NewStreamAllocator(params StreamAllocatorParams) *StreamAllocator { eventCh: make(chan Event, 1000), } + s.probeController = NewProbeController(ProbeControllerParams{ + Prober: s.prober, + Logger: params.Logger, + }) + s.probeController.OnProbeSuccess(s.onProbeSuccess) + s.resetState() s.prober.SetProberListener(s) @@ -319,7 +308,7 @@ func (s *StreamAllocator) SetChannelCapacity(channelCapacity int64) { func (s *StreamAllocator) resetState() { s.channelObserver = s.newChannelObserverNonProbe() - s.resetProbe() + s.probeController.Reset() s.state = streamAllocatorStateStable } @@ -561,7 +550,7 @@ func (s *StreamAllocator) processEvents() { s.handleEvent(&event) } - s.stopProbe() + s.probeController.StopProbe() } func (s *StreamAllocator) ping() { @@ -642,7 +631,7 @@ func (s *StreamAllocator) handleSignalEstimate(event *Event) { s.monitorRate(receivedEstimate) // while probing, maintain estimate separately to enable keeping current committed estimate if probe fails - if s.isInProbe() { + if s.probeController.IsInProbe() { s.handleNewEstimateInProbe() } else { s.handleNewEstimateInNonProbe() @@ -651,9 +640,7 @@ func (s *StreamAllocator) handleSignalEstimate(event *Event) { func (s *StreamAllocator) handleSignalPeriodicPing(event *Event) { // finalize probe if necessary - if s.isInProbe() && !s.probeEndTime.IsZero() && time.Now().After(s.probeEndTime) { - s.finalizeProbe() - } + s.probeController.MaybeFinalizeProbe() // probe if necessary and timing is right if s.state == streamAllocatorStateDeficient { @@ -686,26 +673,7 @@ func (s *StreamAllocator) handleSignalSendProbe(event *Event) { func (s *StreamAllocator) handleSignalProbeClusterDone(event *Event) { info, _ := event.Data.(ProbeClusterInfo) - if s.probeClusterId != info.Id { - return - } - - if s.abortedProbeClusterId == ProbeClusterIdInvalid { - // successful probe, finalize - s.finalizeProbe() - return - } - - // ensure probe queue is flushed - // STREAM-ALLOCATOR-TODO: ProbeSettleWait should actually be a certain number of RTTs. - lowestEstimate := int64(math.Min(float64(s.committedChannelCapacity), float64(s.channelObserver.GetLowestEstimate()))) - expectedDuration := float64(info.BytesSent*8*1000) / float64(lowestEstimate) - queueTime := expectedDuration - float64(info.Duration.Milliseconds()) - if queueTime < 0.0 { - queueTime = 0.0 - } - queueWait := time.Duration(queueTime+float64(ProbeSettleWait)) * time.Millisecond - s.probeEndTime = s.lastProbeStartTime.Add(queueWait) + s.probeController.ProbeClusterDone(info, int64(math.Min(float64(s.committedChannelCapacity), float64(s.channelObserver.GetLowestEstimate())))) } func (s *StreamAllocator) handleSignalResume(event *Event) { @@ -732,7 +700,7 @@ func (s *StreamAllocator) handleSignalSetChannelCapacity(event *Event) { s.params.Logger.Infow("allocating on override channel capacity", "override", s.overriddenChannelCapacity) s.allocateAllTracks() } else { - s.params.Logger.Infow("clearing override channel capacity") + s.params.Logger.Infow("clearing override channel capacity") } } @@ -769,7 +737,7 @@ func (s *StreamAllocator) setState(state streamAllocatorState) { s.state = state // reset probe to enforce a delay after state change before probing - s.lastProbeStartTime = time.Now() + s.probeController.Reset() } func (s *StreamAllocator) adjustState() { @@ -787,7 +755,7 @@ func (s *StreamAllocator) handleNewEstimateInProbe() { // always update NACKs, even if aborted packetDelta, repeatedNackDelta := s.getNackDelta() - if s.abortedProbeClusterId != ProbeClusterIdInvalid { + if s.probeController.DoesProbeNeedFinalize() { // waiting for aborted probe to finalize return } @@ -796,35 +764,7 @@ func (s *StreamAllocator) handleNewEstimateInProbe() { s.channelObserver.AddNack(packetDelta, repeatedNackDelta) trend, _ := s.channelObserver.GetTrend() - if trend != ChannelTrendNeutral { - s.probeTrendObserved = true - } - - switch { - case !s.probeTrendObserved && time.Since(s.lastProbeStartTime) > ProbeTrendWait: - // - // More of a safety net. - // In rare cases, the estimate gets stuck. Prevent from probe running amok - // STREAM-ALLOCATOR-TODO: Need more testing here to ensure that probe does not cause a lot of damage - // - s.params.Logger.Infow("stream allocator: probe: aborting, no trend", "cluster", s.probeClusterId) - s.abortProbe() - - case trend == ChannelTrendCongesting: - // stop immediately if the probe is congesting channel more - s.params.Logger.Infow("stream allocator: probe: aborting, channel is congesting", "cluster", s.probeClusterId) - s.abortProbe() - - case s.channelObserver.GetHighestEstimate() > s.probeGoalBps: - // reached goal, stop probing - s.params.Logger.Infow( - "stream allocator: probe: stopping, goal reached", - "cluster", s.probeClusterId, - "goal", s.probeGoalBps, - "highest", s.channelObserver.GetHighestEstimate(), - ) - s.stopProbe() - } + s.probeController.CheckProbe(trend, s.channelObserver.GetHighestEstimate()) } func (s *StreamAllocator) handleNewEstimateInNonProbe() { @@ -872,14 +812,14 @@ func (s *StreamAllocator) handleNewEstimateInNonProbe() { s.channelObserver = s.newChannelObserverNonProbe() // reset probe to ensure it does not start too soon after a downward trend - s.resetProbe() + s.probeController.Reset() s.allocateAllTracks() } func (s *StreamAllocator) allocateTrack(track *Track) { // abort any probe that may be running when a track specific change needs allocation - s.abortProbe() + s.probeController.AbortProbe() // if not deficient, free pass allocate track if !s.params.Config.Enabled || s.state == streamAllocatorStateStable || !track.IsManaged() { @@ -976,12 +916,9 @@ func (s *StreamAllocator) allocateTrack(track *Track) { s.adjustState() } -func (s *StreamAllocator) finalizeProbe() { - aborted := s.probeClusterId == s.abortedProbeClusterId +func (s *StreamAllocator) onProbeSuccess() { highestEstimateInProbe := s.channelObserver.GetHighestEstimate() - s.clearProbe() - // // Reset estimator at the end of a probe irrespective of probe result to get fresh readings. // With a failed probe, the latest estimate could be lower than committed estimate. @@ -995,15 +932,6 @@ func (s *StreamAllocator) finalizeProbe() { // s.channelObserver = s.newChannelObserverNonProbe() - if aborted { - // failed probe, backoff - s.backoffProbeInterval() - return - } - - // reset probe interval on a successful probe - s.resetProbeInterval() - // probe estimate is same or higher, commit it and try to allocate deficient tracks s.params.Logger.Infow( "successful probe, updating channel capacity", @@ -1211,8 +1139,6 @@ func (s *StreamAllocator) newChannelObserverNonProbe() *ChannelObserver { } func (s *StreamAllocator) initProbe(probeGoalDeltaBps int64) { - s.lastProbeStartTime = time.Now() - expectedBandwidthUsage := s.getExpectedBandwidthUsage() if float64(expectedBandwidthUsage) > 1.5*float64(s.committedChannelCapacity) { // STREAM-ALLOCATOR-TODO-START @@ -1227,14 +1153,8 @@ func (s *StreamAllocator) initProbe(probeGoalDeltaBps int64) { fmt.Errorf("expected too high, expected: %d, committed: %d", expectedBandwidthUsage, s.committedChannelCapacity), ) } - // overshoot a bit to account for noise (in measurement/estimate etc) - s.probeGoalBps = expectedBandwidthUsage + ((probeGoalDeltaBps * ProbePct) / 100) - s.abortedProbeClusterId = ProbeClusterIdInvalid - - s.probeTrendObserved = false - - s.probeEndTime = time.Time{} + probeClusterId, probeGoalBps := s.probeController.InitProbe(probeGoalDeltaBps, expectedBandwidthUsage) channelState := "" if s.channelObserver != nil { @@ -1243,67 +1163,26 @@ func (s *StreamAllocator) initProbe(probeGoalDeltaBps int64) { s.channelObserver = s.newChannelObserverProbe() s.channelObserver.SeedEstimate(s.lastReceivedEstimate) - s.probeClusterId = s.prober.AddCluster( - ProbeClusterModeUniform, - int(s.probeGoalBps), - int(expectedBandwidthUsage), - ProbeMinDuration, - ProbeMaxDuration, - ) s.params.Logger.Infow( "stream allocator: starting probe", - "probeClusterId", s.probeClusterId, + "probeClusterId", probeClusterId, "current usage", expectedBandwidthUsage, "committed", s.committedChannelCapacity, "lastReceived", s.lastReceivedEstimate, "channel", channelState, "probeGoalDeltaBps", probeGoalDeltaBps, - "goalBps", s.probeGoalBps, + "goalBps", probeGoalBps, ) } -func (s *StreamAllocator) resetProbe() { - s.lastProbeStartTime = time.Now() - - s.resetProbeInterval() - - s.clearProbe() -} - -func (s *StreamAllocator) clearProbe() { - s.probeClusterId = ProbeClusterIdInvalid - s.abortedProbeClusterId = ProbeClusterIdInvalid -} - -func (s *StreamAllocator) backoffProbeInterval() { - s.probeInterval = time.Duration(s.probeInterval.Seconds()*ProbeBackoffFactor) * time.Second - if s.probeInterval > ProbeWaitMax { - s.probeInterval = ProbeWaitMax - } -} - -func (s *StreamAllocator) resetProbeInterval() { - s.probeInterval = ProbeWaitBase -} - -func (s *StreamAllocator) stopProbe() { - s.prober.Reset() -} - -func (s *StreamAllocator) abortProbe() { - s.abortedProbeClusterId = s.probeClusterId - s.stopProbe() -} - -func (s *StreamAllocator) isInProbe() bool { - return s.probeClusterId != ProbeClusterIdInvalid -} - func (s *StreamAllocator) maybeProbe() { - if time.Since(s.lastProbeStartTime) < s.probeInterval || s.probeClusterId != ProbeClusterIdInvalid || s.overriddenChannelCapacity > 0 { + if s.overriddenChannelCapacity > 0 { // do not probe if channel capacity is overridden return } + if !s.probeController.CanProbe() { + return + } switch s.params.Config.ProbeMode { case config.CongestionControlProbeModeMedia: @@ -1328,7 +1207,7 @@ func (s *StreamAllocator) maybeProbeWithMedia() { } s.maybeSendUpdate(update) - s.lastProbeStartTime = time.Now() + s.probeController.Reset() break } } From 956735ae05abd2820ea1c5801171d3edacc03234 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Mon, 29 May 2023 10:53:08 -0700 Subject: [PATCH 193/324] Fix node stats updates on Windows (#1748) Because we aren't able to get CPU count/load info on Windows, they are stubbed out to return placeholders. This restores compatibility to run on Windows. --- pkg/service/server.go | 4 ++ pkg/telemetry/prometheus/node.go | 3 +- pkg/telemetry/prometheus/node_linux.go | 27 ---------- pkg/telemetry/prometheus/node_nonlinux.go | 32 ------------ pkg/telemetry/prometheus/node_nonwindows.go | 56 +++++++++++++++++++++ pkg/telemetry/prometheus/node_windows.go | 27 ++++++++++ 6 files changed, 88 insertions(+), 61 deletions(-) create mode 100644 pkg/telemetry/prometheus/node_nonwindows.go create mode 100644 pkg/telemetry/prometheus/node_windows.go diff --git a/pkg/service/server.go b/pkg/service/server.go index bbd9f6775..69e3d404c 100644 --- a/pkg/service/server.go +++ b/pkg/service/server.go @@ -8,6 +8,7 @@ import ( "net" "net/http" _ "net/http/pprof" + "runtime" "runtime/pprof" "time" @@ -217,6 +218,9 @@ func (s *LivekitServer) Start() error { values = append(values, "region", s.config.Region) } logger.Infow("starting LiveKit server", values...) + if runtime.GOOS == "windows" { + logger.Infow("Windows detected, capacity management is unavailable") + } for _, promLn := range promListeners { go s.promServer.Serve(promLn) diff --git a/pkg/telemetry/prometheus/node.go b/pkg/telemetry/prometheus/node.go index 48b438e65..5dc04a3c8 100644 --- a/pkg/telemetry/prometheus/node.go +++ b/pkg/telemetry/prometheus/node.go @@ -3,7 +3,6 @@ package prometheus import ( "time" - "github.com/mackerelio/go-osstat/loadavg" "github.com/mackerelio/go-osstat/memory" "github.com/prometheus/client_golang/prometheus" "go.uber.org/atomic" @@ -100,7 +99,7 @@ func Init(nodeID string, nodeType livekit.NodeType, env string) { } func GetUpdatedNodeStats(prev *livekit.NodeStats, prevAverage *livekit.NodeStats) (*livekit.NodeStats, bool, error) { - loadAvg, err := loadavg.Get() + loadAvg, err := getLoadAvg() if err != nil { return nil, false, err } diff --git a/pkg/telemetry/prometheus/node_linux.go b/pkg/telemetry/prometheus/node_linux.go index b2e06c3d9..9854f056e 100644 --- a/pkg/telemetry/prometheus/node_linux.go +++ b/pkg/telemetry/prometheus/node_linux.go @@ -5,37 +5,10 @@ package prometheus import ( "fmt" - "sync" "github.com/florianl/go-tc" - "github.com/mackerelio/go-osstat/cpu" ) -var ( - cpuStatsLock sync.RWMutex - lastCPUTotal, lastCPUIdle uint64 -) - -func getCPUStats() (cpuLoad float32, numCPUs uint32, err error) { - cpuInfo, err := cpu.Get() - if err != nil { - return - } - - cpuStatsLock.Lock() - if lastCPUTotal > 0 && lastCPUTotal < cpuInfo.Total { - cpuLoad = 1 - float32(cpuInfo.Idle-lastCPUIdle)/float32(cpuInfo.Total-lastCPUTotal) - } - - lastCPUTotal = cpuInfo.Total - lastCPUIdle = cpuInfo.Idle // + cpu.Iowait - cpuStatsLock.Unlock() - - numCPUs = uint32(cpuInfo.CPUCount) - - return -} - func getTCStats() (packets, drops uint32, err error) { rtnl, err := tc.Open(&tc.Config{}) if err != nil { diff --git a/pkg/telemetry/prometheus/node_nonlinux.go b/pkg/telemetry/prometheus/node_nonlinux.go index 581456413..5cd36c9c1 100644 --- a/pkg/telemetry/prometheus/node_nonlinux.go +++ b/pkg/telemetry/prometheus/node_nonlinux.go @@ -2,38 +2,6 @@ package prometheus -import ( - "runtime" - "sync" - - "github.com/mackerelio/go-osstat/cpu" -) - -var ( - cpuStatsLock sync.RWMutex - lastCPUTotal, lastCPUIdle uint64 -) - -func getCPUStats() (cpuLoad float32, numCPUs uint32, err error) { - cpuInfo, err := cpu.Get() - if err != nil { - return - } - - cpuStatsLock.Lock() - if lastCPUTotal > 0 && lastCPUTotal < cpuInfo.Total { - cpuLoad = 1 - float32(cpuInfo.Idle-lastCPUIdle)/float32(cpuInfo.Total-lastCPUTotal) - } - - lastCPUTotal = cpuInfo.Total - lastCPUIdle = cpuInfo.Idle - cpuStatsLock.Unlock() - - numCPUs = uint32(runtime.NumCPU()) - - return -} - func getTCStats() (packets, drops uint32, err error) { // linux only return diff --git a/pkg/telemetry/prometheus/node_nonwindows.go b/pkg/telemetry/prometheus/node_nonwindows.go new file mode 100644 index 000000000..b765ce271 --- /dev/null +++ b/pkg/telemetry/prometheus/node_nonwindows.go @@ -0,0 +1,56 @@ +//go:build !windows + +/* + * Copyright 2023 LiveKit, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package prometheus + +import ( + "runtime" + "sync" + + "github.com/mackerelio/go-osstat/cpu" + "github.com/mackerelio/go-osstat/loadavg" +) + +var ( + cpuStatsLock sync.RWMutex + lastCPUTotal, lastCPUIdle uint64 +) + +func getLoadAvg() (*loadavg.Stats, error) { + return loadavg.Get() +} + +func getCPUStats() (cpuLoad float32, numCPUs uint32, err error) { + cpuInfo, err := cpu.Get() + if err != nil { + return + } + + cpuStatsLock.Lock() + if lastCPUTotal > 0 && lastCPUTotal < cpuInfo.Total { + cpuLoad = 1 - float32(cpuInfo.Idle-lastCPUIdle)/float32(cpuInfo.Total-lastCPUTotal) + } + + lastCPUTotal = cpuInfo.Total + lastCPUIdle = cpuInfo.Idle + cpuStatsLock.Unlock() + + numCPUs = uint32(runtime.NumCPU()) + + return +} diff --git a/pkg/telemetry/prometheus/node_windows.go b/pkg/telemetry/prometheus/node_windows.go new file mode 100644 index 000000000..d52e588ce --- /dev/null +++ b/pkg/telemetry/prometheus/node_windows.go @@ -0,0 +1,27 @@ +//go:build windows + +/* + * Copyright 2023 LiveKit, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package prometheus + +func getLoadAvg() (*loadavg.Stats, error) { + return &loadavg.Stats{}, nil +} + +func getCPUStats() (cpuLoad float32, numCPUs uint32, err error) { + return 1, 1, nil +} From ca3e9ab524cf99b48095dd528f9bafa805ef43da Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 29 May 2023 22:50:24 -0700 Subject: [PATCH 194/324] Update go deps (#1750) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 06d693ed8..52792aa9f 100644 --- a/go.mod +++ b/go.mod @@ -36,13 +36,13 @@ require ( github.com/pion/webrtc/v3 v3.2.8 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.15.1 - github.com/redis/go-redis/v9 v9.0.4 + github.com/redis/go-redis/v9 v9.0.5 github.com/rs/cors v1.9.0 github.com/stretchr/testify v1.8.3 github.com/thoas/go-funk v0.9.3 github.com/twitchtv/twirp v8.1.3+incompatible github.com/ua-parser/uap-go v0.0.0-20211112212520-00c877edfe0f - github.com/urfave/cli/v2 v2.25.4 + github.com/urfave/cli/v2 v2.25.5 github.com/urfave/negroni/v3 v3.0.0 go.uber.org/atomic v1.11.0 golang.org/x/sync v0.2.0 diff --git a/go.sum b/go.sum index bef82bb4b..ef1c229cb 100644 --- a/go.sum +++ b/go.sum @@ -231,8 +231,8 @@ github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= -github.com/redis/go-redis/v9 v9.0.4 h1:FC82T+CHJ/Q/PdyLW++GeCO+Ol59Y4T7R4jbgjvktgc= -github.com/redis/go-redis/v9 v9.0.4/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= +github.com/redis/go-redis/v9 v9.0.5 h1:CuQcn5HIEeK7BgElubPP8CGtE0KakrnbBSTLjathl5o= +github.com/redis/go-redis/v9 v9.0.5/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rs/cors v1.9.0 h1:l9HGsTsHJcvW14Nk7J9KFz8bzeAWXn3CG6bgt7LsrAE= @@ -261,8 +261,8 @@ github.com/twitchtv/twirp v8.1.3+incompatible h1:+F4TdErPgSUbMZMwp13Q/KgDVuI7HJX github.com/twitchtv/twirp v8.1.3+incompatible/go.mod h1:RRJoFSAmTEh2weEqWtpPE3vFK5YBhA6bqp2l1kfCC5A= github.com/ua-parser/uap-go v0.0.0-20211112212520-00c877edfe0f h1:A+MmlgpvrHLeUP8dkBVn4Pnf5Bp5Yk2OALm7SEJLLE8= github.com/ua-parser/uap-go v0.0.0-20211112212520-00c877edfe0f/go.mod h1:OBcG9bn7sHtXgarhUEb3OfCnNsgtGnkVf41ilSZ3K3E= -github.com/urfave/cli/v2 v2.25.4 h1:HyYwPrTO3im9rYhUff/ZNs78eolxt0nJ4LN+9yJKSH4= -github.com/urfave/cli/v2 v2.25.4/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= +github.com/urfave/cli/v2 v2.25.5 h1:d0NIAyhh5shGscroL7ek/Ya9QYQE0KNabJgiUinIQkc= +github.com/urfave/cli/v2 v2.25.5/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= github.com/urfave/negroni/v3 v3.0.0 h1:Vo8CeZfu1lFR9gW8GnAb6dOGCJyijfil9j/jKKc/JhU= github.com/urfave/negroni/v3 v3.0.0/go.mod h1:jWvnX03kcSjDBl/ShB0iHvx5uOs7mAzZXW+JvJ5XYAs= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= From d598e06d9f171747acd96641b87e91919231bf05 Mon Sep 17 00:00:00 2001 From: Benjamin Pracht Date: Tue, 30 May 2023 13:41:12 -0700 Subject: [PATCH 195/324] Add support for bypass_transcoding field in ingress (#1741) --- go.mod | 4 ++-- go.sum | 8 ++++---- pkg/service/ingress.go | 4 ++++ 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 52792aa9f..1e64e453c 100644 --- a/go.mod +++ b/go.mod @@ -18,8 +18,8 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230523035537-27577c4e1646 - github.com/livekit/protocol v1.5.7 - github.com/livekit/psrpc v0.3.1-0.20230528083849-53d664c6d912 + github.com/livekit/protocol v1.5.8-0.20230525225706-52cca4239cf2 + github.com/livekit/psrpc v0.3.1-0.20230518234341-6f6847e10b09 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 github.com/maxbrunsfeld/counterfeiter/v6 v6.6.1 diff --git a/go.sum b/go.sum index ef1c229cb..5e6a6c282 100644 --- a/go.sum +++ b/go.sum @@ -122,10 +122,10 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20230523035537-27577c4e1646 h1:acGSGkWJdut7TUWozCDheHu4dwWFDqqRzv+SBbIY9Xo= github.com/livekit/mediatransportutil v0.0.0-20230523035537-27577c4e1646/go.mod h1:MRc0zSOSzXuFt0X218SgabzlaKevkvCckPgBEoHYc34= -github.com/livekit/protocol v1.5.7 h1:jZeFQEmLuIhFblXDGPRCBbfjVJHb+YU7AsO+SMoXF70= -github.com/livekit/protocol v1.5.7/go.mod h1:ZaOnsvP+JS4s7vI1UO+JVdBagvvLp/lBXDAl2hkDS0I= -github.com/livekit/psrpc v0.3.1-0.20230528083849-53d664c6d912 h1:q6i7ptxK6yZ220eyCWx5l1ZsUMQNyuYZ5feG91sM8EI= -github.com/livekit/psrpc v0.3.1-0.20230528083849-53d664c6d912/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= +github.com/livekit/protocol v1.5.8-0.20230525225706-52cca4239cf2 h1:cE/OQiQyre2NFQlHwwZXGV97mTTFOHess37qkWt6Ctg= +github.com/livekit/protocol v1.5.8-0.20230525225706-52cca4239cf2/go.mod h1:ZaOnsvP+JS4s7vI1UO+JVdBagvvLp/lBXDAl2hkDS0I= +github.com/livekit/psrpc v0.3.1-0.20230518234341-6f6847e10b09 h1:mb6jRcg57U0HQ4tKRsueHHKcvTqBinL6+0Aa84vTtWk= +github.com/livekit/psrpc v0.3.1-0.20230518234341-6f6847e10b09/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= github.com/mackerelio/go-osstat v0.2.4/go.mod h1:Zy+qzGdZs3A9cuIqmgbJvwbmLQH9dJvtio5ZjJTbdlQ= github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= diff --git a/pkg/service/ingress.go b/pkg/service/ingress.go index 9456224a0..cb5604e2c 100644 --- a/pkg/service/ingress.go +++ b/pkg/service/ingress.go @@ -94,6 +94,7 @@ func (s *IngressService) CreateIngressWithUrlPrefix(ctx context.Context, urlPref InputType: req.InputType, Audio: req.Audio, Video: req.Video, + BypassTranscoding: req.BypassTranscoding, RoomName: req.RoomName, ParticipantIdentity: req.ParticipantIdentity, ParticipantName: req.ParticipantName, @@ -126,6 +127,9 @@ func updateInfoUsingRequest(req *livekit.UpdateIngressRequest, info *livekit.Ing if req.ParticipantName != "" { info.ParticipantName = req.ParticipantName } + if req.BypassTranscoding != nil { + info.BypassTranscoding = *req.BypassTranscoding + } if req.Audio != nil { info.Audio = req.Audio } From 13d599d2d998570294b2fccc8e978524d43cf4b9 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 31 May 2023 06:35:25 +0530 Subject: [PATCH 196/324] Comment out noisy log. (#1757) --- pkg/sfu/buffer/rtpstats.go | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index 87a403bfa..50d020dfd 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -865,19 +865,21 @@ func (r *RTPStats) GetExpectedRTPTimestamp(at time.Time) (uint32, uint64, error) if r.srNewest != nil { minTS = r.srNewest.RTPTimestampExt } - r.logger.Debugw( - "expected RTP timestamp", - "firstTime", r.firstTime.String(), - "checkAt", at.String(), - "timeDiff", timeDiff, - "firstRTP", r.extStartTS, - "expectedRTPDiff", expectedRTPDiff, - "expectedExtRTP", expectedExtRTP, - "expectedRTP", uint32(expectedExtRTP), - "minTS", minTS, - "highestTS", r.highestTS, - "highestTime", r.highestTime.String(), - ) + /* + r.logger.Debugw( + "expected RTP timestamp", + "firstTime", r.firstTime.String(), + "checkAt", at.String(), + "timeDiff", timeDiff, + "firstRTP", r.extStartTS, + "expectedRTPDiff", expectedRTPDiff, + "expectedExtRTP", expectedExtRTP, + "expectedRTP", uint32(expectedExtRTP), + "minTS", minTS, + "highestTS", r.highestTS, + "highestTime", r.highestTime.String(), + ) + */ return uint32(expectedExtRTP), minTS, nil } From c1842cb54f64a686ffe9445de0c813d6ccf46442 Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Wed, 31 May 2023 11:41:22 +0800 Subject: [PATCH 197/324] Avoid reconnect loop for unsupported downtrack (#1754) * Avoid reconnect loop for unsupported downtrack If the client subscribes to a track which codec is unsupported by the client, sfu will trigger negotiation failed and issue a full reconnect after received client answer. If the client try to subscribe that track then it will got full reconnect again. That will cause a infinite reconnect loop until the client don't subscribe that track. This PR will unsubscribe the error track for the client and send a SubscriptionResponse that contain the reason to indicates the track's codec is not supported to avoid the reconnect loop. --- go.mod | 4 +- go.sum | 8 +- pkg/rtc/mediatracksubscriptions.go | 8 +- pkg/rtc/participant.go | 26 ++++- pkg/rtc/subscribedtrack.go | 16 +-- pkg/rtc/subscriptionmanager.go | 16 ++- pkg/rtc/subscriptionmanager_test.go | 12 +- pkg/rtc/transport.go | 8 +- pkg/rtc/types/interfaces.go | 2 +- .../types/typesfakes/fake_subscribed_track.go | 12 +- pkg/sfu/downtrack.go | 14 ++- test/client/client.go | 35 +++++- test/integration_helpers.go | 4 +- test/singlenode_test.go | 108 ++++++++++++++++++ 14 files changed, 226 insertions(+), 47 deletions(-) diff --git a/go.mod b/go.mod index 1e64e453c..799001739 100644 --- a/go.mod +++ b/go.mod @@ -18,8 +18,8 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230523035537-27577c4e1646 - github.com/livekit/protocol v1.5.8-0.20230525225706-52cca4239cf2 - github.com/livekit/psrpc v0.3.1-0.20230518234341-6f6847e10b09 + github.com/livekit/protocol v1.5.8-0.20230531030840-2a4d1a607ba3 + github.com/livekit/psrpc v0.3.1-0.20230528083849-53d664c6d912 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 github.com/maxbrunsfeld/counterfeiter/v6 v6.6.1 diff --git a/go.sum b/go.sum index 5e6a6c282..c064abc4e 100644 --- a/go.sum +++ b/go.sum @@ -122,10 +122,10 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20230523035537-27577c4e1646 h1:acGSGkWJdut7TUWozCDheHu4dwWFDqqRzv+SBbIY9Xo= github.com/livekit/mediatransportutil v0.0.0-20230523035537-27577c4e1646/go.mod h1:MRc0zSOSzXuFt0X218SgabzlaKevkvCckPgBEoHYc34= -github.com/livekit/protocol v1.5.8-0.20230525225706-52cca4239cf2 h1:cE/OQiQyre2NFQlHwwZXGV97mTTFOHess37qkWt6Ctg= -github.com/livekit/protocol v1.5.8-0.20230525225706-52cca4239cf2/go.mod h1:ZaOnsvP+JS4s7vI1UO+JVdBagvvLp/lBXDAl2hkDS0I= -github.com/livekit/psrpc v0.3.1-0.20230518234341-6f6847e10b09 h1:mb6jRcg57U0HQ4tKRsueHHKcvTqBinL6+0Aa84vTtWk= -github.com/livekit/psrpc v0.3.1-0.20230518234341-6f6847e10b09/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= +github.com/livekit/protocol v1.5.8-0.20230531030840-2a4d1a607ba3 h1:UtySLPpOMgT9D/t0MOOxHShAyQQ2WsGKrUJ0XZCJlcI= +github.com/livekit/protocol v1.5.8-0.20230531030840-2a4d1a607ba3/go.mod h1:ZaOnsvP+JS4s7vI1UO+JVdBagvvLp/lBXDAl2hkDS0I= +github.com/livekit/psrpc v0.3.1-0.20230528083849-53d664c6d912 h1:q6i7ptxK6yZ220eyCWx5l1ZsUMQNyuYZ5feG91sM8EI= +github.com/livekit/psrpc v0.3.1-0.20230528083849-53d664c6d912/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= github.com/mackerelio/go-osstat v0.2.4/go.mod h1:Zy+qzGdZs3A9cuIqmgbJvwbmLQH9dJvtio5ZjJTbdlQ= github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= diff --git a/pkg/rtc/mediatracksubscriptions.go b/pkg/rtc/mediatracksubscriptions.go index bbf439e43..94b6f2b88 100644 --- a/pkg/rtc/mediatracksubscriptions.go +++ b/pkg/rtc/mediatracksubscriptions.go @@ -128,7 +128,11 @@ func (t *MediaTrackSubscriptions) AddSubscriber(sub types.LocalParticipant, wr * // Bind callback can happen from replaceTrack, so set it up early var reusingTransceiver atomic.Bool var dtState sfu.DownTrackState - downTrack.OnBinding(func() { + downTrack.OnBinding(func(err error) { + if err != nil { + go subTrack.Bound(err) + return + } wr.DetermineReceiver(downTrack.Codec()) if reusingTransceiver.Load() { downTrack.SeedState(dtState) @@ -142,7 +146,7 @@ func (t *MediaTrackSubscriptions) AddSubscriber(sub types.LocalParticipant, wr * ) } - go subTrack.Bound() + go subTrack.Bound(nil) subTrack.SetPublisherMuted(t.params.MediaTrack.IsMuted()) }) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 876f3c1d7..a098cba07 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -997,7 +997,10 @@ func (p *ParticipantImpl) onTrackSubscribed(subTrack types.SubscribedTrack) { subTrack.DownTrack().SetActivePaddingOnMuteUpTrack() } - subTrack.AddOnBind(func() { + subTrack.AddOnBind(func(err error) { + if err != nil { + return + } if p.TransportManager.HasSubscriberEverConnected() { subTrack.DownTrack().SetConnected() } @@ -2083,8 +2086,25 @@ func (p *ParticipantImpl) onPublicationError(trackID livekit.TrackID) { } } -func (p *ParticipantImpl) onSubscriptionError(trackID livekit.TrackID) { - if p.params.ReconnectOnSubscriptionError { +func (p *ParticipantImpl) onSubscriptionError(trackID livekit.TrackID, fatal bool, err error) { + signalErr := livekit.SubscriptionError_SE_UNKOWN + switch { + case errors.Is(err, webrtc.ErrUnsupportedCodec): + signalErr = livekit.SubscriptionError_SE_CODEC_UNSUPPORTED + case errors.Is(err, ErrTrackNotFound): + signalErr = livekit.SubscriptionError_SE_TRACK_NOTFOUND + } + + _ = p.writeMessage(&livekit.SignalResponse{ + Message: &livekit.SignalResponse_SubscriptionResponse{ + SubscriptionResponse: &livekit.SubscriptionResponse{ + TrackSid: string(trackID), + Err: signalErr, + }, + }, + }) + + if p.params.ReconnectOnSubscriptionError && fatal { p.params.Logger.Infow("issuing full reconnect on subscription error", "trackID", trackID) p.IssueFullReconnect(types.ParticipantCloseReasonPublicationError) } diff --git a/pkg/rtc/subscribedtrack.go b/pkg/rtc/subscribedtrack.go index 3a08c731f..7bb8ab294 100644 --- a/pkg/rtc/subscribedtrack.go +++ b/pkg/rtc/subscribedtrack.go @@ -40,7 +40,7 @@ type SubscribedTrack struct { needsNegotiation atomic.Bool bindLock sync.Mutex - onBindCallbacks []func() + onBindCallbacks []func(error) onClose atomic.Value // func(bool) bound atomic.Bool @@ -61,7 +61,7 @@ func NewSubscribedTrack(params SubscribedTrackParams) *SubscribedTrack { return s } -func (t *SubscribedTrack) AddOnBind(f func()) { +func (t *SubscribedTrack) AddOnBind(f func(error)) { t.bindLock.Lock() bound := t.bound.Load() if !bound { @@ -71,19 +71,21 @@ func (t *SubscribedTrack) AddOnBind(f func()) { if bound { // fire immediately, do not need to persist since bind is a one time event - go f() + go f(nil) } } // for DownTrack callback to notify us that it's bound -func (t *SubscribedTrack) Bound() { +func (t *SubscribedTrack) Bound(err error) { t.bindLock.Lock() - t.bound.Store(true) + if err == nil { + t.bound.Store(true) + } callbacks := t.onBindCallbacks t.onBindCallbacks = nil t.bindLock.Unlock() - if t.MediaTrack().Kind() == livekit.TrackType_VIDEO { + if err == nil && t.MediaTrack().Kind() == livekit.TrackType_VIDEO { // When AdaptiveStream is enabled, default the subscriber to LOW quality stream // we would want LOW instead of OFF for a couple of reasons // 1. when a subscriber unsubscribes from a track, we would forget their previously defined settings @@ -107,7 +109,7 @@ func (t *SubscribedTrack) Bound() { } for _, cb := range callbacks { - go cb() + go cb(err) } } diff --git a/pkg/rtc/subscriptionmanager.go b/pkg/rtc/subscriptionmanager.go index 329c067ab..5bc0b54d7 100644 --- a/pkg/rtc/subscriptionmanager.go +++ b/pkg/rtc/subscriptionmanager.go @@ -52,7 +52,7 @@ type SubscriptionManagerParams struct { TrackResolver types.MediaTrackResolver OnTrackSubscribed func(subTrack types.SubscribedTrack) OnTrackUnsubscribed func(subTrack types.SubscribedTrack) - OnSubscriptionError func(trackID livekit.TrackID) + OnSubscriptionError func(trackID livekit.TrackID, fatal bool, err error) Telemetry telemetry.TelemetryService SubscriptionLimitVideo, SubscriptionLimitAudio int32 @@ -309,6 +309,7 @@ func (m *SubscriptionManager) reconcileSubscription(s *trackSubscription) { s.logger.Infow("unsubscribing from track after notFoundTimeout", "error", err) s.setDesired(false) m.queueReconcile(s.trackID) + m.params.OnSubscriptionError(s.trackID, false, err) } default: // all other errors @@ -317,7 +318,7 @@ func (m *SubscriptionManager) reconcileSubscription(s *trackSubscription) { "attempt", numAttempts, ) s.maybeRecordError(m.params.Telemetry, m.params.Participant.ID(), err, false) - m.params.OnSubscriptionError(s.trackID) + m.params.OnSubscriptionError(s.trackID, true, err) } else { s.logger.Debugw("failed to subscribe, retrying", "error", err, @@ -353,7 +354,7 @@ func (m *SubscriptionManager) reconcileSubscription(s *trackSubscription) { if s.durationSinceStart() > subscriptionTimeout { s.logger.Errorw("track not bound after timeout", nil) s.maybeRecordError(m.params.Telemetry, m.params.Participant.ID(), ErrTrackNotBound, false) - m.params.OnSubscriptionError(s.trackID) + m.params.OnSubscriptionError(s.trackID, true, ErrTrackNotBound) } } } @@ -470,7 +471,14 @@ func (m *SubscriptionManager) subscribe(s *trackSubscription) error { subTrack.OnClose(func(willBeResumed bool) { m.handleSubscribedTrackClose(s, willBeResumed) }) - subTrack.AddOnBind(func() { + subTrack.AddOnBind(func(err error) { + if err != nil { + s.logger.Infow("failed to bind track", "err", err) + s.maybeRecordError(m.params.Telemetry, m.params.Participant.ID(), err, true) + m.UnsubscribeFromTrack(s.trackID) + m.params.OnSubscriptionError(s.trackID, false, err) + return + } s.setBound() s.maybeRecordSuccess(m.params.Telemetry, m.params.Participant.ID()) }) diff --git a/pkg/rtc/subscriptionmanager_test.go b/pkg/rtc/subscriptionmanager_test.go index c5fe60f59..944655a0c 100644 --- a/pkg/rtc/subscriptionmanager_test.go +++ b/pkg/rtc/subscriptionmanager_test.go @@ -54,7 +54,7 @@ func TestSubscribe(t *testing.T) { sm.params.OnTrackSubscribed = func(subTrack types.SubscribedTrack) { subCount.Add(1) } - sm.params.OnSubscriptionError = func(trackID livekit.TrackID) { + sm.params.OnSubscriptionError = func(trackID livekit.TrackID, fatal bool, err error) { failed.Store(true) } numParticipantSubscribed := atomic.Int32{} @@ -123,7 +123,7 @@ func TestSubscribe(t *testing.T) { resolver := newTestResolver(false, true, "pub", "pubID") sm.params.TrackResolver = resolver.Resolve failed := atomic.Bool{} - sm.params.OnSubscriptionError = func(trackID livekit.TrackID) { + sm.params.OnSubscriptionError = func(trackID livekit.TrackID, fatal bool, err error) { failed.Store(true) } @@ -164,7 +164,7 @@ func TestSubscribe(t *testing.T) { resolver := newTestResolver(true, true, "pub", "pubID") sm.params.TrackResolver = resolver.Resolve failed := atomic.Bool{} - sm.params.OnSubscriptionError = func(trackID livekit.TrackID) { + sm.params.OnSubscriptionError = func(trackID livekit.TrackID, fatal bool, err error) { failed.Store(true) } @@ -360,7 +360,7 @@ func TestSubscriptionLimits(t *testing.T) { sm.params.OnTrackSubscribed = func(subTrack types.SubscribedTrack) { subCount.Add(1) } - sm.params.OnSubscriptionError = func(trackID livekit.TrackID) { + sm.params.OnSubscriptionError = func(trackID livekit.TrackID, fatal bool, err error) { failed.Store(true) } numParticipantSubscribed := atomic.Int32{} @@ -463,7 +463,7 @@ func newTestSubscriptionManagerWithParams(t *testing.T, params testSubscriptionP Logger: logger.GetLogger(), OnTrackSubscribed: func(subTrack types.SubscribedTrack) {}, OnTrackUnsubscribed: func(subTrack types.SubscribedTrack) {}, - OnSubscriptionError: func(trackID livekit.TrackID) {}, + OnSubscriptionError: func(trackID livekit.TrackID, fatal bool, err error) {}, TrackResolver: func(identity livekit.ParticipantIdentity, trackID livekit.TrackID) types.MediaResolverResult { return types.MediaResolverResult{} }, @@ -526,7 +526,7 @@ func setTestSubscribedTrackBound(t *testing.T, st types.SubscribedTrack) { require.True(t, ok) for i := 0; i < fst.AddOnBindCallCount(); i++ { - fst.AddOnBindArgsForCall(i)() + fst.AddOnBindArgsForCall(i)(nil) } } diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index a2e88f9f2..c1913bf7a 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -1865,7 +1865,13 @@ func (t *PCTransport) handleRemoteOfferReceived(sd *webrtc.SessionDescription) e func (t *PCTransport) handleRemoteAnswerReceived(sd *webrtc.SessionDescription) error { if err := t.setRemoteDescription(*sd); err != nil { - return err + // Pion will call RTPSender.Send method for each new added Downtrack, and return error if the DownTrack.Bind + // returns error. In case of Downtrack.Bind returns ErrUnsupportedCodec, the signal state will be stable as negotiation is aleady compelted + // before startRTPSenders, and the peerconnection state can be recovered by next negotiation which will be triggered + // by the SubscriptionManager unsubscribe the failure DownTrack. So don't treat this error as negotiation failure. + if !errors.Is(err, webrtc.ErrUnsupportedCodec) { + return err + } } t.clearSignalStateCheckTimer() diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index 1a0a9532c..c7c963372 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -427,7 +427,7 @@ type LocalMediaTrack interface { //counterfeiter:generate . SubscribedTrack type SubscribedTrack interface { - AddOnBind(f func()) + AddOnBind(f func(error)) IsBound() bool Close(willBeResumed bool) OnClose(f func(willBeResumed bool)) diff --git a/pkg/rtc/types/typesfakes/fake_subscribed_track.go b/pkg/rtc/types/typesfakes/fake_subscribed_track.go index b14ceb2c4..833b8e4b3 100644 --- a/pkg/rtc/types/typesfakes/fake_subscribed_track.go +++ b/pkg/rtc/types/typesfakes/fake_subscribed_track.go @@ -11,10 +11,10 @@ import ( ) type FakeSubscribedTrack struct { - AddOnBindStub func(func()) + AddOnBindStub func(func(error)) addOnBindMutex sync.RWMutex addOnBindArgsForCall []struct { - arg1 func() + arg1 func(error) } CloseStub func(bool) closeMutex sync.RWMutex @@ -174,10 +174,10 @@ type FakeSubscribedTrack struct { invocationsMutex sync.RWMutex } -func (fake *FakeSubscribedTrack) AddOnBind(arg1 func()) { +func (fake *FakeSubscribedTrack) AddOnBind(arg1 func(error)) { fake.addOnBindMutex.Lock() fake.addOnBindArgsForCall = append(fake.addOnBindArgsForCall, struct { - arg1 func() + arg1 func(error) }{arg1}) stub := fake.AddOnBindStub fake.recordInvocation("AddOnBind", []interface{}{arg1}) @@ -193,13 +193,13 @@ func (fake *FakeSubscribedTrack) AddOnBindCallCount() int { return len(fake.addOnBindArgsForCall) } -func (fake *FakeSubscribedTrack) AddOnBindCalls(stub func(func())) { +func (fake *FakeSubscribedTrack) AddOnBindCalls(stub func(func(error))) { fake.addOnBindMutex.Lock() defer fake.addOnBindMutex.Unlock() fake.AddOnBindStub = stub } -func (fake *FakeSubscribedTrack) AddOnBindArgsForCall(i int) func() { +func (fake *FakeSubscribedTrack) AddOnBindArgsForCall(i int) func(error) { fake.addOnBindMutex.RLock() defer fake.addOnBindMutex.RUnlock() argsForCall := fake.addOnBindArgsForCall[i] diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 0e8e12dbe..0d26ad127 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -199,7 +199,7 @@ type DownTrack struct { writeStream webrtc.TrackLocalWriter rtcpReader *buffer.RTCPReader onCloseHandler func(willBeResumed bool) - onBinding func() + onBinding func(error) listenerLock sync.RWMutex receiverReportListeners []ReceiverReportListener @@ -336,8 +336,14 @@ func (d *DownTrack) Bind(t webrtc.TrackLocalContext) (webrtc.RTPCodecParameters, } if codec.MimeType == "" { + err := webrtc.ErrUnsupportedCodec + onBinding := d.onBinding d.bindLock.Unlock() - return webrtc.RTPCodecParameters{}, webrtc.ErrUnsupportedCodec + d.logger.Infow("bind error for unsupported codec", "codecs", d.upstreamCodecs, "remoteParameters", t.CodecParameters()) + if onBinding != nil { + onBinding(err) + } + return webrtc.RTPCodecParameters{}, err } // if a downtrack is closed before bind, it already unsubscribed from client, don't do subsequent operation and return here. @@ -367,7 +373,7 @@ func (d *DownTrack) Bind(t webrtc.TrackLocalContext) (webrtc.RTPCodecParameters, d.codec = codec.RTPCodecCapability if d.onBinding != nil { - d.onBinding() + d.onBinding(nil) } d.bound.Store(true) d.bindLock.Unlock() @@ -990,7 +996,7 @@ func (d *DownTrack) OnCloseHandler(fn func(willBeResumed bool)) { d.onCloseHandler = fn } -func (d *DownTrack) OnBinding(fn func()) { +func (d *DownTrack) OnBinding(fn func(error)) { d.onBinding = fn } diff --git a/test/client/client.go b/test/client/client.go index 17b085b3f..fa9b485a1 100644 --- a/test/client/client.go +++ b/test/client/client.go @@ -8,6 +8,7 @@ import ( "net/http" "net/url" "path/filepath" + "strings" "sync" "time" @@ -61,6 +62,8 @@ type RTCClient struct { // map of livekit.ParticipantID and last packet lastPackets map[livekit.ParticipantID]*rtp.Packet bytesReceived map[livekit.ParticipantID]uint64 + + subscriptionResponse atomic.Pointer[livekit.SubscriptionResponse] } var ( @@ -80,9 +83,10 @@ var ( ) type Options struct { - AutoSubscribe bool - Publish string - ClientInfo *livekit.ClientInfo + AutoSubscribe bool + Publish string + ClientInfo *livekit.ClientInfo + DisabledCodecs []webrtc.RTPCodecCapability } func NewWebSocketConn(host, token string, opts *Options) (*websocket.Conn, error) { @@ -116,7 +120,7 @@ func SetAuthorizationToken(header http.Header, token string) { header.Set("Authorization", "Bearer "+token) } -func NewRTCClient(conn *websocket.Conn) (*RTCClient, error) { +func NewRTCClient(conn *websocket.Conn, opts *Options) (*RTCClient, error) { var err error c := &RTCClient{ @@ -139,7 +143,8 @@ func NewRTCClient(conn *websocket.Conn) (*RTCClient, error) { } conf.SettingEngine.SetLite(false) conf.SettingEngine.SetAnsweringDTLSRole(webrtc.DTLSRoleClient) - codecs := []*livekit.Codec{ + var codecs []*livekit.Codec + for _, codec := range []*livekit.Codec{ { Mime: "audio/opus", }, @@ -149,7 +154,21 @@ func NewRTCClient(conn *websocket.Conn) (*RTCClient, error) { { Mime: "video/h264", }, + } { + var disabled bool + if opts != nil { + for _, dc := range opts.DisabledCodecs { + if strings.EqualFold(dc.MimeType, codec.Mime) && (dc.SDPFmtpLine == "" || dc.SDPFmtpLine == codec.FmtpLine) { + disabled = true + break + } + } + } + if !disabled { + codecs = append(codecs, codec) + } } + // // The signal targets are from point of view of server. // From client side, they are flipped, @@ -348,6 +367,8 @@ func (c *RTCClient) Run() error { c.lock.Unlock() case *livekit.SignalResponse_Pong: c.pongReceivedAt.Store(msg.Pong) + case *livekit.SignalResponse_SubscriptionResponse: + c.subscriptionResponse.Store(msg.SubscriptionResponse) } } } @@ -452,6 +473,10 @@ func (c *RTCClient) PongReceivedAt() int64 { return c.pongReceivedAt.Load() } +func (c *RTCClient) GetSubscriptionResponseAndClear() *livekit.SubscriptionResponse { + return c.subscriptionResponse.Swap(nil) +} + func (c *RTCClient) SendPing() error { return c.SendRequest(&livekit.SignalRequest{ Message: &livekit.SignalRequest_Ping{ diff --git a/test/integration_helpers.go b/test/integration_helpers.go index b0354d866..ec0ec7fb2 100644 --- a/test/integration_helpers.go +++ b/test/integration_helpers.go @@ -196,7 +196,7 @@ func createRTCClient(name string, port int, opts *testclient.Options) *testclien panic(err) } - c, err := testclient.NewRTCClient(ws) + c, err := testclient.NewRTCClient(ws, opts) if err != nil { panic(err) } @@ -213,7 +213,7 @@ func createRTCClientWithToken(token string, port int, opts *testclient.Options) panic(err) } - c, err := testclient.NewRTCClient(ws) + c, err := testclient.NewRTCClient(ws, opts) if err != nil { panic(err) } diff --git a/test/singlenode_test.go b/test/singlenode_test.go index a658a4bce..8a00e6a8d 100644 --- a/test/singlenode_test.go +++ b/test/singlenode_test.go @@ -543,3 +543,111 @@ func TestDeviceCodecOverride(t *testing.T) { } require.True(t, hasSeenVP8, "should have seen VP8 codec in SDP") } + +func TestSubscribeToCodecUnsupported(t *testing.T) { + if testing.Short() { + t.SkipNow() + return + } + + _, finish := setupSingleNodeTest("TestSubscribeToCodecUnsupported") + defer finish() + + c1 := createRTCClient("c1", defaultServerPort, nil) + // create a client that doesn't support H264 + c2 := createRTCClient("c2", defaultServerPort, &testclient.Options{ + AutoSubscribe: true, + DisabledCodecs: []webrtc.RTPCodecCapability{ + {MimeType: "video/H264"}, + }, + }) + waitUntilConnected(t, c1, c2) + + // publish a vp8 video track and ensure c2 receives it ok + t1, err := c1.AddStaticTrack("audio/opus", "audio", "webcam") + require.NoError(t, err) + defer t1.Stop() + t2, err := c1.AddStaticTrack("video/vp8", "video", "webcam") + require.NoError(t, err) + defer t2.Stop() + + testutils.WithTimeout(t, func() string { + if len(c2.SubscribedTracks()) == 0 { + return "c2 was not subscribed to anything" + } + // should have received two tracks + if len(c2.SubscribedTracks()[c1.ID()]) != 2 { + return "c2 was not subscribed to tracks from c1" + } + + tracks := c2.SubscribedTracks()[c1.ID()] + for _, t := range tracks { + if strings.EqualFold(t.Codec().MimeType, "video/vp8") { + return "" + + } + } + return "did not receive track with vp8" + }) + require.Nil(t, c2.GetSubscriptionResponseAndClear()) + + // publish a h264 track and ensure c2 got subscription error + t3, err := c1.AddStaticTrackWithCodec(webrtc.RTPCodecCapability{ + MimeType: "video/h264", + ClockRate: 90000, + SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f", + }, "videoscreen", "screen") + defer t3.Stop() + require.NoError(t, err) + + var h264TrackID string + require.Eventually(t, func() bool { + remoteC1 := c2.GetRemoteParticipant(c1.ID()) + require.NotNil(t, remoteC1) + for _, track := range remoteC1.Tracks { + if strings.EqualFold(track.MimeType, "video/h264") { + h264TrackID = track.Sid + return true + } + } + return false + }, time.Second, 10*time.Millisecond, "did not receive track info with h264") + + require.Eventually(t, func() bool { + sr := c2.GetSubscriptionResponseAndClear() + if sr == nil { + return false + } + require.Equal(t, h264TrackID, sr.TrackSid) + require.Equal(t, livekit.SubscriptionError_SE_CODEC_UNSUPPORTED, sr.Err) + return true + }, time.Second, 10*time.Millisecond, "did not receive subscription response") + + // publish another vp8 track again, ensure the transport recovered by sfu and c2 can receive it + t4, err := c1.AddStaticTrack("video/vp8", "video2", "webcam2") + require.NoError(t, err) + defer t4.Stop() + + testutils.WithTimeout(t, func() string { + if len(c2.SubscribedTracks()) == 0 { + return "c2 was not subscribed to anything" + } + // should have received two tracks + if len(c2.SubscribedTracks()[c1.ID()]) != 3 { + return "c2 was not subscribed to tracks from c1" + } + + var vp8Count int + tracks := c2.SubscribedTracks()[c1.ID()] + for _, t := range tracks { + if strings.EqualFold(t.Codec().MimeType, "video/vp8") { + vp8Count++ + } + } + if vp8Count == 2 { + return "" + } + return "did not 2 receive track with vp8" + }) + require.Nil(t, c2.GetSubscriptionResponseAndClear()) +} From 5a8305f09b9e6109788f59144799e13eadfb2d4b Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 31 May 2023 22:18:26 -0700 Subject: [PATCH 198/324] Update module github.com/stretchr/testify to v1.8.4 (#1756) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 799001739..61ca19085 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,7 @@ require ( github.com/prometheus/client_golang v1.15.1 github.com/redis/go-redis/v9 v9.0.5 github.com/rs/cors v1.9.0 - github.com/stretchr/testify v1.8.3 + github.com/stretchr/testify v1.8.4 github.com/thoas/go-funk v0.9.3 github.com/twitchtv/twirp v8.1.3+incompatible github.com/ua-parser/uap-go v0.0.0-20211112212520-00c877edfe0f diff --git a/go.sum b/go.sum index c064abc4e..84996f05f 100644 --- a/go.sum +++ b/go.sum @@ -253,8 +253,9 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/thoas/go-funk v0.9.3 h1:7+nAEx3kn5ZJcnDm2Bh23N2yOtweO14bi//dvRtgLpw= github.com/thoas/go-funk v0.9.3/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= github.com/twitchtv/twirp v8.1.3+incompatible h1:+F4TdErPgSUbMZMwp13Q/KgDVuI7HJXP61mNV3/7iuU= From 9a698736d1c6b35bc866ab7d410573da6f4f19bc Mon Sep 17 00:00:00 2001 From: David Colburn Date: Thu, 1 Jun 2023 16:56:12 -0700 Subject: [PATCH 199/324] include await_start_signal (#1759) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 61ca19085..5481cd35e 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230523035537-27577c4e1646 - github.com/livekit/protocol v1.5.8-0.20230531030840-2a4d1a607ba3 + github.com/livekit/protocol v1.5.8-0.20230601212100-a186ecb11a98 github.com/livekit/psrpc v0.3.1-0.20230528083849-53d664c6d912 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 diff --git a/go.sum b/go.sum index 84996f05f..3a8f83488 100644 --- a/go.sum +++ b/go.sum @@ -122,8 +122,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20230523035537-27577c4e1646 h1:acGSGkWJdut7TUWozCDheHu4dwWFDqqRzv+SBbIY9Xo= github.com/livekit/mediatransportutil v0.0.0-20230523035537-27577c4e1646/go.mod h1:MRc0zSOSzXuFt0X218SgabzlaKevkvCckPgBEoHYc34= -github.com/livekit/protocol v1.5.8-0.20230531030840-2a4d1a607ba3 h1:UtySLPpOMgT9D/t0MOOxHShAyQQ2WsGKrUJ0XZCJlcI= -github.com/livekit/protocol v1.5.8-0.20230531030840-2a4d1a607ba3/go.mod h1:ZaOnsvP+JS4s7vI1UO+JVdBagvvLp/lBXDAl2hkDS0I= +github.com/livekit/protocol v1.5.8-0.20230601212100-a186ecb11a98 h1:OkQu5+sAgOzLqcpOH9WV5UyWsEcNwfvyi1KmpmXCKb0= +github.com/livekit/protocol v1.5.8-0.20230601212100-a186ecb11a98/go.mod h1:ZaOnsvP+JS4s7vI1UO+JVdBagvvLp/lBXDAl2hkDS0I= github.com/livekit/psrpc v0.3.1-0.20230528083849-53d664c6d912 h1:q6i7ptxK6yZ220eyCWx5l1ZsUMQNyuYZ5feG91sM8EI= github.com/livekit/psrpc v0.3.1-0.20230528083849-53d664c6d912/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= From b5c8fe5294842e49983924af50f1164a5ea359c6 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Fri, 2 Jun 2023 00:13:18 -0700 Subject: [PATCH 200/324] Perform unsubscribe in parallel to avoid blocking (#1760) * Perform unsubscribe in parallel to avoid blocking When unsubscribing from tracks, we flush a blank frame in order to prepare the transceivers for re-use. This process is blocking for ~200ms. If the unsubscribes are performed serially, it would prevent other subscribe operation from continuing. This PR parallelizes that operation, and ensures subsequent subscribe operations could reuse the existing transceivers. * also perform in parallel when uptrack close * fix a few log fields --- pkg/rtc/participant.go | 2 +- pkg/rtc/subscriptionmanager.go | 36 +++++++++++++++++++++++------ pkg/rtc/subscriptionmanager_test.go | 3 +++ pkg/service/roomservice.go | 4 ++-- pkg/sfu/downtrack.go | 2 +- pkg/sfu/receiver.go | 18 ++++++++++++--- pkg/sfu/redprimaryreceiver.go | 4 +--- pkg/sfu/redreceiver.go | 4 +--- 8 files changed, 53 insertions(+), 20 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index a098cba07..ecdfd2082 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -1686,7 +1686,7 @@ func (p *ParticipantImpl) addMigrateMutedTrack(cid string, ti *livekit.TrackInfo p.params.Logger.Debugw("add migrate muted track", "cid", cid, "track", ti.String()) rtpReceiver := p.TransportManager.GetPublisherRTPReceiver(ti.Mid) if rtpReceiver == nil { - p.params.Logger.Errorw("could not find receiver for migrated track", nil, "track", ti.Sid) + p.params.Logger.Errorw("could not find receiver for migrated track", nil, "trackID", ti.Sid) return nil } diff --git a/pkg/rtc/subscriptionmanager.go b/pkg/rtc/subscriptionmanager.go index 5bc0b54d7..18a3581ad 100644 --- a/pkg/rtc/subscriptionmanager.go +++ b/pkg/rtc/subscriptionmanager.go @@ -40,6 +40,7 @@ var ( // amount of time to try otherwise before flagging subscription as failed subscriptionTimeout = iceFailedTimeout trackRemoveGracePeriod = time.Second + maxUnsubscribeWait = time.Second ) const ( @@ -60,9 +61,10 @@ type SubscriptionManagerParams struct { // SubscriptionManager manages a participant's subscriptions type SubscriptionManager struct { - params SubscriptionManagerParams - lock sync.RWMutex - subscriptions map[livekit.TrackID]*trackSubscription + params SubscriptionManagerParams + lock sync.RWMutex + subscriptions map[livekit.TrackID]*trackSubscription + pendingUnsubscribes atomic.Int32 subscribedVideoCount, subscribedAudioCount atomic.Int32 @@ -109,8 +111,15 @@ func (m *SubscriptionManager) Close(willBeResumed bool) { } } - for _, dt := range downTracksToClose { - dt.CloseWithFlush(!willBeResumed) + if willBeResumed { + for _, dt := range downTracksToClose { + dt.CloseWithFlush(false) + } + } else { + // flush blocks, so execute in parallel + for _, dt := range downTracksToClose { + go dt.CloseWithFlush(true) + } } } @@ -274,6 +283,15 @@ func (m *SubscriptionManager) reconcileSubscription(s *trackSubscription) { return } if s.needsSubscribe() { + if m.pendingUnsubscribes.Load() != 0 && s.durationSinceStart() < maxUnsubscribeWait { + // enqueue this in a bit, after pending unsubscribes are complete + go func() { + time.Sleep(time.Duration(sfu.RTPBlankFramesCloseSeconds * float32(time.Second))) + m.queueReconcile(s.trackID) + }() + return + } + numAttempts := s.getNumAttempts() if numAttempts == 0 { m.params.Telemetry.TrackSubscribeRequested( @@ -498,7 +516,7 @@ func (m *SubscriptionManager) subscribe(s *trackSubscription) error { go m.params.OnTrackSubscribed(subTrack) } - m.params.Logger.Debugw("subscribed to track", "track", s.trackID, "subscribedAudioCount", m.subscribedAudioCount.Load(), "subscribedVideoCount", m.subscribedVideoCount.Load()) + m.params.Logger.Debugw("subscribed to track", "trackID", s.trackID, "subscribedAudioCount", m.subscribedAudioCount.Load(), "subscribedVideoCount", m.subscribedVideoCount.Load()) // add mark the participant as someone we've subscribed to firstSubscribe := false @@ -531,7 +549,11 @@ func (m *SubscriptionManager) unsubscribe(s *trackSubscription) error { track := subTrack.MediaTrack() pID := m.params.Participant.ID() - track.RemoveSubscriber(pID, false) + m.pendingUnsubscribes.Inc() + go func() { + defer m.pendingUnsubscribes.Dec() + track.RemoveSubscriber(pID, false) + }() return nil } diff --git a/pkg/rtc/subscriptionmanager_test.go b/pkg/rtc/subscriptionmanager_test.go index 944655a0c..9ad6f20ea 100644 --- a/pkg/rtc/subscriptionmanager_test.go +++ b/pkg/rtc/subscriptionmanager_test.go @@ -236,6 +236,9 @@ func TestUnsubscribe(t *testing.T) { if s.needsUnsubscribe() { return false } + if sm.pendingUnsubscribes.Load() != 0 { + return false + } sm.lock.RLock() subLen := len(sm.subscriptions) sm.lock.RUnlock() diff --git a/pkg/service/roomservice.go b/pkg/service/roomservice.go index d4a4fe5a0..749e367de 100644 --- a/pkg/service/roomservice.go +++ b/pkg/service/roomservice.go @@ -212,7 +212,7 @@ func (s *RoomService) RemoveParticipant(ctx context.Context, req *livekit.RoomPa } func (s *RoomService) MutePublishedTrack(ctx context.Context, req *livekit.MuteRoomTrackRequest) (*livekit.MuteRoomTrackResponse, error) { - AppendLogFields(ctx, "room", req.Room, "participant", req.Identity, "track", req.TrackSid, "muted", req.Muted) + AppendLogFields(ctx, "room", req.Room, "participant", req.Identity, "trackID", req.TrackSid, "muted", req.Muted) if err := EnsureAdminPermission(ctx, livekit.RoomName(req.Room)); err != nil { return nil, twirpAuthError(err) } @@ -298,7 +298,7 @@ func (s *RoomService) UpdateSubscriptions(ctx context.Context, req *livekit.Upda for _, pt := range req.ParticipantTracks { trackSIDs = append(trackSIDs, pt.TrackSids...) } - AppendLogFields(ctx, "room", req.Room, "participant", req.Identity, "track", trackSIDs) + AppendLogFields(ctx, "room", req.Room, "participant", req.Identity, "trackID", trackSIDs) err := s.writeParticipantMessage(ctx, livekit.RoomName(req.Room), livekit.ParticipantIdentity(req.Identity), &livekit.RTCNodeMessage{ Message: &livekit.RTCNodeMessage_UpdateSubscriptions{ UpdateSubscriptions: req, diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 0d26ad127..b90d8ebb5 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -830,7 +830,7 @@ func (d *DownTrack) Close() { d.CloseWithFlush(true) } -// Close track, flush used to indicate whether send blank frame to flush +// CloseWithFlush - flush used to indicate whether send blank frame to flush // decoder of client. // 1. When transceiver is reused by other participant's video track, // set flush=true to avoid previous video shows before new stream is displayed. diff --git a/pkg/sfu/receiver.go b/pkg/sfu/receiver.go index eda4f7412..07c072501 100644 --- a/pkg/sfu/receiver.go +++ b/pkg/sfu/receiver.go @@ -662,9 +662,7 @@ func (w *WebRTCReceiver) closeTracks() { w.connectionStats.Close() w.streamTrackerManager.Close() - for _, dt := range w.downTrackSpreader.ResetAndGetDownTracks() { - dt.Close() - } + closeTrackSenders(w.downTrackSpreader.ResetAndGetDownTracks()) if w.onCloseHandler != nil { w.onCloseHandler() @@ -753,3 +751,17 @@ func (w *WebRTCReceiver) GetRTCPSenderReportData(layer int32) (*buffer.RTCPSende func (w *WebRTCReceiver) GetReferenceLayerRTPTimestamp(ts uint32, layer int32, referenceLayer int32) (uint32, error) { return w.streamTrackerManager.GetReferenceLayerRTPTimestamp(ts, layer, referenceLayer) } + +// closes all track senders in parallel, returns when all are closed +func closeTrackSenders(senders []TrackSender) { + wg := sync.WaitGroup{} + for _, dt := range senders { + dt := dt + wg.Add(1) + go func() { + defer wg.Done() + dt.Close() + }() + } + wg.Wait() +} diff --git a/pkg/sfu/redprimaryreceiver.go b/pkg/sfu/redprimaryreceiver.go index 9789bb32e..5f6fd8183 100644 --- a/pkg/sfu/redprimaryreceiver.go +++ b/pkg/sfu/redprimaryreceiver.go @@ -98,9 +98,7 @@ func (r *RedPrimaryReceiver) CanClose() bool { func (r *RedPrimaryReceiver) Close() { r.closed.Store(true) - for _, dt := range r.downTrackSpreader.ResetAndGetDownTracks() { - dt.Close() - } + closeTrackSenders(r.downTrackSpreader.ResetAndGetDownTracks()) } func (r *RedPrimaryReceiver) ReadRTP(buf []byte, layer uint8, sn uint16) (int, error) { diff --git a/pkg/sfu/redreceiver.go b/pkg/sfu/redreceiver.go index 0d2876cf9..7a9adc700 100644 --- a/pkg/sfu/redreceiver.go +++ b/pkg/sfu/redreceiver.go @@ -95,9 +95,7 @@ func (r *RedReceiver) IsClosed() bool { func (r *RedReceiver) Close() { r.closed.Store(true) - for _, dt := range r.downTrackSpreader.ResetAndGetDownTracks() { - dt.Close() - } + closeTrackSenders(r.downTrackSpreader.ResetAndGetDownTracks()) } func (r *RedReceiver) ReadRTP(buf []byte, layer uint8, sn uint16) (int, error) { From c2ae34151cb1983eb1d34c6fed291928e70023db Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 2 Jun 2023 16:31:19 +0530 Subject: [PATCH 201/324] Enable some debug logs to debug freeze (#1761) * Enable some debug logs to debug freeze * log receiver sender report also --- pkg/sfu/buffer/rtpstats.go | 86 +++++++++++++++++++++++++------------- 1 file changed, 57 insertions(+), 29 deletions(-) diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index 50d020dfd..62fd5a960 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -830,6 +830,22 @@ func (r *RTPStats) SetRtcpSenderReportData(srData *RTCPSenderReportData) { "highestTS", r.highestTS, "highestTime", r.highestTime.String(), ) + } else { + packetDriftResult, reportDriftResult := r.getDrift() + r.logger.Debugw( + "received sender report", + "ntp", srData.NTPTimestamp.Time().String(), + "rtp", srData.RTPTimestamp, + "arrival", srData.At.String(), + "ntpDiffSinceLast", ntpDiffSinceLast.Seconds(), + "rtpDiffSinceLast", int32(rtpDiffSinceLast), + "arrivalDiffSinceLast", arrivalDiffSinceLast.Seconds(), + "expectedTimeDiffSinceLast", expectedTimeDiffSinceLast, + "packetDrift", packetDriftResult.String(), + "reportDrift", reportDriftResult.String(), + "highestTS", r.highestTS, + "highestTime", r.highestTime.String(), + ) } } @@ -865,21 +881,19 @@ func (r *RTPStats) GetExpectedRTPTimestamp(at time.Time) (uint32, uint64, error) if r.srNewest != nil { minTS = r.srNewest.RTPTimestampExt } - /* - r.logger.Debugw( - "expected RTP timestamp", - "firstTime", r.firstTime.String(), - "checkAt", at.String(), - "timeDiff", timeDiff, - "firstRTP", r.extStartTS, - "expectedRTPDiff", expectedRTPDiff, - "expectedExtRTP", expectedExtRTP, - "expectedRTP", uint32(expectedExtRTP), - "minTS", minTS, - "highestTS", r.highestTS, - "highestTime", r.highestTime.String(), - ) - */ + r.logger.Debugw( + "expected RTP timestamp", + "firstTime", r.firstTime.String(), + "checkAt", at.String(), + "timeDiff", timeDiff, + "firstRTP", r.extStartTS, + "expectedRTPDiff", expectedRTPDiff, + "expectedExtRTP", expectedExtRTP, + "expectedRTP", uint32(expectedExtRTP), + "minTS", minTS, + "highestTS", r.highestTS, + "highestTime", r.highestTime.String(), + ) return uint32(expectedExtRTP), minTS, nil } @@ -971,6 +985,22 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32, srFirst *RTCPSenderReportDat "highestTS", r.highestTS, "highestTime", r.highestTime.String(), ) + } else { + packetDriftResult, reportDriftResult := r.getDrift() + r.logger.Debugw( + "sending sender report", + "ntp", nowNTP.Time().String(), + "rtp", nowRTP, + "departure", now.String(), + "ntpDiffSinceLast", ntpDiffSinceLast.Seconds(), + "rtpDiffSinceLast", int32(rtpDiffSinceLast), + "departureDiffSinceLast", departureDiffSinceLast.Seconds(), + "expectedTimeDiffSinceLast", expectedTimeDiffSinceLast, + "packetDrift", packetDriftResult.String(), + "reportDrift", reportDriftResult.String(), + "highestTS", r.highestTS, + "highestTime", r.highestTime.String(), + ) } return &rtcp.SenderReport{ @@ -1985,20 +2015,18 @@ func (p *PIDController) Update(setpoint, measurement float64, at time.Time) floa p.prevError = errorTerm p.prevMeasurement = measurement p.prevMeasurementTime = at - /* - p.logger.Debugw( - "pid controller", - "setpoint", setpoint, - "measurement", measurement, - "errorTerm", errorTerm, - "proportional", proportional, - "integral", iVal, - "integralLimited", boundIVal, - "derivative", p.dVal, - "output", output, - "outputLimited", boundOutput, - ) - */ + p.logger.Debugw( + "pid controller", + "setpoint", setpoint, + "measurement", measurement, + "errorTerm", errorTerm, + "proportional", proportional, + "integral", iVal, + "integralLimited", boundIVal, + "derivative", p.dVal, + "output", output, + "outputLimited", boundOutput, + ) return boundOutput } From e7879a46fc08066e0a6ca1110092ebddc2d1dbfb Mon Sep 17 00:00:00 2001 From: Benjamin Pracht Date: Fri, 2 Jun 2023 17:38:19 -0700 Subject: [PATCH 202/324] Add ingress telemetry support (#1763) --- pkg/service/ingress.go | 4 + pkg/service/ioinfo.go | 27 +++ pkg/telemetry/events.go | 43 +++++ .../telemetryfakes/fake_telemetry_service.go | 164 ++++++++++++++++++ pkg/telemetry/telemetryservice.go | 4 + 5 files changed, 242 insertions(+) diff --git a/pkg/service/ingress.go b/pkg/service/ingress.go index cb5604e2c..04d4a6bd8 100644 --- a/pkg/service/ingress.go +++ b/pkg/service/ingress.go @@ -110,6 +110,7 @@ func (s *IngressService) CreateIngressWithUrlPrefix(ctx context.Context, urlPref logger.Errorw("could not write ingress info", err) return nil, err } + s.telemetry.IngressCreated(ctx, info) return info, nil } @@ -254,5 +255,8 @@ func (s *IngressService) DeleteIngress(ctx context.Context, req *livekit.DeleteI } info.State.Status = livekit.IngressState_ENDPOINT_INACTIVE + + s.telemetry.IngressDeleted(ctx, info) + return info, nil } diff --git a/pkg/service/ioinfo.go b/pkg/service/ioinfo.go index 529eec8f9..41ff4541c 100644 --- a/pkg/service/ioinfo.go +++ b/pkg/service/ioinfo.go @@ -109,10 +109,37 @@ func (s *IOInfoService) loadIngressFromInfoRequest(req *rpc.GetIngressInfoReques } func (s *IOInfoService) UpdateIngressState(ctx context.Context, req *rpc.UpdateIngressStateRequest) (*emptypb.Empty, error) { + info, err := s.is.LoadIngress(ctx, req.IngressId) + if err != nil { + return nil, err + } + if err := s.is.UpdateIngressState(ctx, req.IngressId, req.State); err != nil { logger.Errorw("could not update ingress", err) return nil, err } + + if info.State.Status != req.State.Status { + info.State = req.State + + switch req.State.Status { + case livekit.IngressState_ENDPOINT_ERROR, + livekit.IngressState_ENDPOINT_INACTIVE: + s.telemetry.IngressEnded(ctx, info) + + if req.State.Error != "" { + logger.Infow("ingress failed", "error", req.State.Error, "ingressID", req.IngressId) + } else { + logger.Infow("ingress ended", "ingressID", req.IngressId) + } + + case livekit.IngressState_ENDPOINT_PUBLISHING: + s.telemetry.IngressStarted(ctx, info) + + logger.Infow("ingress started", "ingressID", req.IngressId) + } + } + return &emptypb.Empty{}, nil } diff --git a/pkg/telemetry/events.go b/pkg/telemetry/events.go index c75e4d828..1e831b9b5 100644 --- a/pkg/telemetry/events.go +++ b/pkg/telemetry/events.go @@ -429,6 +429,40 @@ func (t *telemetryService) EgressEnded(ctx context.Context, info *livekit.Egress }) } +func (t *telemetryService) IngressCreated(ctx context.Context, info *livekit.IngressInfo) { + t.enqueue(func() { + t.SendEvent(ctx, newIngressEvent(livekit.AnalyticsEventType_INGRESS_CREATED, info)) + }) +} + +func (t *telemetryService) IngressDeleted(ctx context.Context, info *livekit.IngressInfo) { + t.enqueue(func() { + t.SendEvent(ctx, newIngressEvent(livekit.AnalyticsEventType_INGRESS_DELETED, info)) + }) +} + +func (t *telemetryService) IngressStarted(ctx context.Context, info *livekit.IngressInfo) { + t.enqueue(func() { + t.NotifyEvent(ctx, &livekit.WebhookEvent{ + Event: webhook.EventIngressStarted, + IngressInfo: info, + }) + + t.SendEvent(ctx, newIngressEvent(livekit.AnalyticsEventType_INGRESS_STARTED, info)) + }) +} + +func (t *telemetryService) IngressEnded(ctx context.Context, info *livekit.IngressInfo) { + t.enqueue(func() { + t.NotifyEvent(ctx, &livekit.WebhookEvent{ + Event: webhook.EventIngressEnded, + IngressInfo: info, + }) + + t.SendEvent(ctx, newIngressEvent(livekit.AnalyticsEventType_INGRESS_ENDED, info)) + }) +} + // returns a livekit.Room with only name and sid filled out // returns nil if room is not found func (t *telemetryService) getRoomDetails(participantID livekit.ParticipantID) *livekit.Room { @@ -483,3 +517,12 @@ func newEgressEvent(event livekit.AnalyticsEventType, egress *livekit.EgressInfo Egress: egress, } } + +func newIngressEvent(event livekit.AnalyticsEventType, ingress *livekit.IngressInfo) *livekit.AnalyticsEvent { + return &livekit.AnalyticsEvent{ + Type: event, + Timestamp: timestamppb.Now(), + IngressId: ingress.IngressId, + Ingress: ingress, + } +} diff --git a/pkg/telemetry/telemetryfakes/fake_telemetry_service.go b/pkg/telemetry/telemetryfakes/fake_telemetry_service.go index 923916164..800160fb7 100644 --- a/pkg/telemetry/telemetryfakes/fake_telemetry_service.go +++ b/pkg/telemetry/telemetryfakes/fake_telemetry_service.go @@ -32,6 +32,30 @@ type FakeTelemetryService struct { flushStatsMutex sync.RWMutex flushStatsArgsForCall []struct { } + IngressCreatedStub func(context.Context, *livekit.IngressInfo) + ingressCreatedMutex sync.RWMutex + ingressCreatedArgsForCall []struct { + arg1 context.Context + arg2 *livekit.IngressInfo + } + IngressDeletedStub func(context.Context, *livekit.IngressInfo) + ingressDeletedMutex sync.RWMutex + ingressDeletedArgsForCall []struct { + arg1 context.Context + arg2 *livekit.IngressInfo + } + IngressEndedStub func(context.Context, *livekit.IngressInfo) + ingressEndedMutex sync.RWMutex + ingressEndedArgsForCall []struct { + arg1 context.Context + arg2 *livekit.IngressInfo + } + IngressStartedStub func(context.Context, *livekit.IngressInfo) + ingressStartedMutex sync.RWMutex + ingressStartedArgsForCall []struct { + arg1 context.Context + arg2 *livekit.IngressInfo + } NotifyEventStub func(context.Context, *livekit.WebhookEvent) notifyEventMutex sync.RWMutex notifyEventArgsForCall []struct { @@ -337,6 +361,138 @@ func (fake *FakeTelemetryService) FlushStatsCalls(stub func()) { fake.FlushStatsStub = stub } +func (fake *FakeTelemetryService) IngressCreated(arg1 context.Context, arg2 *livekit.IngressInfo) { + fake.ingressCreatedMutex.Lock() + fake.ingressCreatedArgsForCall = append(fake.ingressCreatedArgsForCall, struct { + arg1 context.Context + arg2 *livekit.IngressInfo + }{arg1, arg2}) + stub := fake.IngressCreatedStub + fake.recordInvocation("IngressCreated", []interface{}{arg1, arg2}) + fake.ingressCreatedMutex.Unlock() + if stub != nil { + fake.IngressCreatedStub(arg1, arg2) + } +} + +func (fake *FakeTelemetryService) IngressCreatedCallCount() int { + fake.ingressCreatedMutex.RLock() + defer fake.ingressCreatedMutex.RUnlock() + return len(fake.ingressCreatedArgsForCall) +} + +func (fake *FakeTelemetryService) IngressCreatedCalls(stub func(context.Context, *livekit.IngressInfo)) { + fake.ingressCreatedMutex.Lock() + defer fake.ingressCreatedMutex.Unlock() + fake.IngressCreatedStub = stub +} + +func (fake *FakeTelemetryService) IngressCreatedArgsForCall(i int) (context.Context, *livekit.IngressInfo) { + fake.ingressCreatedMutex.RLock() + defer fake.ingressCreatedMutex.RUnlock() + argsForCall := fake.ingressCreatedArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeTelemetryService) IngressDeleted(arg1 context.Context, arg2 *livekit.IngressInfo) { + fake.ingressDeletedMutex.Lock() + fake.ingressDeletedArgsForCall = append(fake.ingressDeletedArgsForCall, struct { + arg1 context.Context + arg2 *livekit.IngressInfo + }{arg1, arg2}) + stub := fake.IngressDeletedStub + fake.recordInvocation("IngressDeleted", []interface{}{arg1, arg2}) + fake.ingressDeletedMutex.Unlock() + if stub != nil { + fake.IngressDeletedStub(arg1, arg2) + } +} + +func (fake *FakeTelemetryService) IngressDeletedCallCount() int { + fake.ingressDeletedMutex.RLock() + defer fake.ingressDeletedMutex.RUnlock() + return len(fake.ingressDeletedArgsForCall) +} + +func (fake *FakeTelemetryService) IngressDeletedCalls(stub func(context.Context, *livekit.IngressInfo)) { + fake.ingressDeletedMutex.Lock() + defer fake.ingressDeletedMutex.Unlock() + fake.IngressDeletedStub = stub +} + +func (fake *FakeTelemetryService) IngressDeletedArgsForCall(i int) (context.Context, *livekit.IngressInfo) { + fake.ingressDeletedMutex.RLock() + defer fake.ingressDeletedMutex.RUnlock() + argsForCall := fake.ingressDeletedArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeTelemetryService) IngressEnded(arg1 context.Context, arg2 *livekit.IngressInfo) { + fake.ingressEndedMutex.Lock() + fake.ingressEndedArgsForCall = append(fake.ingressEndedArgsForCall, struct { + arg1 context.Context + arg2 *livekit.IngressInfo + }{arg1, arg2}) + stub := fake.IngressEndedStub + fake.recordInvocation("IngressEnded", []interface{}{arg1, arg2}) + fake.ingressEndedMutex.Unlock() + if stub != nil { + fake.IngressEndedStub(arg1, arg2) + } +} + +func (fake *FakeTelemetryService) IngressEndedCallCount() int { + fake.ingressEndedMutex.RLock() + defer fake.ingressEndedMutex.RUnlock() + return len(fake.ingressEndedArgsForCall) +} + +func (fake *FakeTelemetryService) IngressEndedCalls(stub func(context.Context, *livekit.IngressInfo)) { + fake.ingressEndedMutex.Lock() + defer fake.ingressEndedMutex.Unlock() + fake.IngressEndedStub = stub +} + +func (fake *FakeTelemetryService) IngressEndedArgsForCall(i int) (context.Context, *livekit.IngressInfo) { + fake.ingressEndedMutex.RLock() + defer fake.ingressEndedMutex.RUnlock() + argsForCall := fake.ingressEndedArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeTelemetryService) IngressStarted(arg1 context.Context, arg2 *livekit.IngressInfo) { + fake.ingressStartedMutex.Lock() + fake.ingressStartedArgsForCall = append(fake.ingressStartedArgsForCall, struct { + arg1 context.Context + arg2 *livekit.IngressInfo + }{arg1, arg2}) + stub := fake.IngressStartedStub + fake.recordInvocation("IngressStarted", []interface{}{arg1, arg2}) + fake.ingressStartedMutex.Unlock() + if stub != nil { + fake.IngressStartedStub(arg1, arg2) + } +} + +func (fake *FakeTelemetryService) IngressStartedCallCount() int { + fake.ingressStartedMutex.RLock() + defer fake.ingressStartedMutex.RUnlock() + return len(fake.ingressStartedArgsForCall) +} + +func (fake *FakeTelemetryService) IngressStartedCalls(stub func(context.Context, *livekit.IngressInfo)) { + fake.ingressStartedMutex.Lock() + defer fake.ingressStartedMutex.Unlock() + fake.IngressStartedStub = stub +} + +func (fake *FakeTelemetryService) IngressStartedArgsForCall(i int) (context.Context, *livekit.IngressInfo) { + fake.ingressStartedMutex.RLock() + defer fake.ingressStartedMutex.RUnlock() + argsForCall := fake.ingressStartedArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + func (fake *FakeTelemetryService) NotifyEvent(arg1 context.Context, arg2 *livekit.WebhookEvent) { fake.notifyEventMutex.Lock() fake.notifyEventArgsForCall = append(fake.notifyEventArgsForCall, struct { @@ -1152,6 +1308,14 @@ func (fake *FakeTelemetryService) Invocations() map[string][][]interface{} { defer fake.egressUpdatedMutex.RUnlock() fake.flushStatsMutex.RLock() defer fake.flushStatsMutex.RUnlock() + fake.ingressCreatedMutex.RLock() + defer fake.ingressCreatedMutex.RUnlock() + fake.ingressDeletedMutex.RLock() + defer fake.ingressDeletedMutex.RUnlock() + fake.ingressEndedMutex.RLock() + defer fake.ingressEndedMutex.RUnlock() + fake.ingressStartedMutex.RLock() + defer fake.ingressStartedMutex.RUnlock() fake.notifyEventMutex.RLock() defer fake.notifyEventMutex.RUnlock() fake.participantActiveMutex.RLock() diff --git a/pkg/telemetry/telemetryservice.go b/pkg/telemetry/telemetryservice.go index fdc6ff052..d3aaf1715 100644 --- a/pkg/telemetry/telemetryservice.go +++ b/pkg/telemetry/telemetryservice.go @@ -54,6 +54,10 @@ type TelemetryService interface { EgressStarted(ctx context.Context, info *livekit.EgressInfo) EgressUpdated(ctx context.Context, info *livekit.EgressInfo) EgressEnded(ctx context.Context, info *livekit.EgressInfo) + IngressCreated(ctx context.Context, info *livekit.IngressInfo) + IngressDeleted(ctx context.Context, info *livekit.IngressInfo) + IngressStarted(ctx context.Context, info *livekit.IngressInfo) + IngressEnded(ctx context.Context, info *livekit.IngressInfo) // helpers AnalyticsService From f5c5d4e079721451ab0219500ab765135dde2f47 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sat, 3 Jun 2023 14:26:26 +0530 Subject: [PATCH 203/324] Wait for a more stable measurement of sample rate. (#1764) --- pkg/sfu/buffer/rtpstats.go | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index 62fd5a960..51c18c4d7 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -17,12 +17,12 @@ import ( ) const ( - GapHistogramNumBins = 101 - NumSequenceNumbers = 65536 - FirstSnapshotId = 1 - SnInfoSize = 8192 - SnInfoMask = SnInfoSize - 1 - TooLargeOWDDelta = 400 * time.Millisecond + GapHistogramNumBins = 101 + NumSequenceNumbers = 65536 + FirstSnapshotId = 1 + SnInfoSize = 8192 + SnInfoMask = SnInfoSize - 1 + MeasurementWindowSecondsMin = float64(5.0) ) // ------------------------------------------------------- @@ -933,13 +933,16 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32, srFirst *RTCPSenderReportDat } } - rtpDiffSinceFirst := nowRTPExt - r.extStartTS - rate := float64(rtpDiffSinceFirst) / timeSinceFirst.Seconds() - pidOutput := r.pidController.Update( - float64(r.params.ClockRate), - rate, - now, - ) + pidOutput := float64(0.0) + if timeSinceFirst.Seconds() > MeasurementWindowSecondsMin { + rtpDiffSinceFirst := nowRTPExt - r.extStartTS + rate := float64(rtpDiffSinceFirst) / timeSinceFirst.Seconds() + pidOutput = r.pidController.Update( + float64(r.params.ClockRate), + rate, + now, + ) + } // monitor and log RTP timestamp anomalies var ntpDiffSinceLast time.Duration From 109620dfb6a3eb6e52824534e243a2aab50115e7 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Sat, 3 Jun 2023 23:51:29 -0700 Subject: [PATCH 204/324] Version 1.4.3 (#1767) --- CHANGELOG | 27 +++++++++++++++++++++++++++ version/version.go | 2 +- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index 43f763a1f..018bc8baf 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,6 +2,33 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.4.3] - 2023-06-03 + +### Added +- Send quality stats to prometheus. (#1708) +- Support for disabling publishing codec on specific devices (#1728) +- Add support for bypass_transcoding field in ingress (#1741) +- Include await_start_signal for Web Egress (#1759) + +### Fixed +- Handle time stamp increment across mute for A/V sync (#1705) +- Additional A/V sync improvements (#1712 #1724 #1737 #1738 #1764) +- Check egress status on UpdateStream failure (#1716) +- Start signal relay sessions with the correct node (#1721) +- Fix unwrap for out-of-order packet (#1729) +- Fix dynacast for svc codec (#1742 #1743) +- Ignore receiver reports that have a sequence number before first packet (#1745) +- Fix node stats updates on Windows (#1748) +- Avoid reconnect loop for unsupported downtrack (#1754) +- Perform unsubscribe in parallel to avoid blocking (#1760) + +### Changed +- Make signal close async. (#1711 #1722) +- Don't add nack if it is already present in track codec (#1714) +- Tweaked connection quality algorithm to be less sensitive to jitter (#1719) +- Adjust sender report time stamp for slow publishers (#1740) +- Split probe controller from StreamAllocator (#1751) + ## [1.4.2] - 2023-04-27 ### Added - VP9 codec with SVC support (#1586) diff --git a/version/version.go b/version/version.go index 3835a32bc..7f79727bd 100644 --- a/version/version.go +++ b/version/version.go @@ -1,3 +1,3 @@ package version -const Version = "1.4.2" +const Version = "1.4.3" From 7e5a7ae79f5b4d9bb9d0fce65d2200413d42b221 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Sun, 4 Jun 2023 00:17:25 -0700 Subject: [PATCH 205/324] Fixed windows build (#1768) --- pkg/telemetry/prometheus/node_windows.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/telemetry/prometheus/node_windows.go b/pkg/telemetry/prometheus/node_windows.go index d52e588ce..eb0ba48a9 100644 --- a/pkg/telemetry/prometheus/node_windows.go +++ b/pkg/telemetry/prometheus/node_windows.go @@ -18,6 +18,8 @@ package prometheus +import "github.com/mackerelio/go-osstat/loadavg" + func getLoadAvg() (*loadavg.Stats, error) { return &loadavg.Stats{}, nil } From 6e063896d0ba92a130a2a4a624c479af693f966f Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Mon, 5 Jun 2023 18:42:02 -0700 Subject: [PATCH 206/324] update psrpc (#1770) * update psrpc * update protocol --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 5481cd35e..d04dc76af 100644 --- a/go.mod +++ b/go.mod @@ -18,8 +18,8 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230523035537-27577c4e1646 - github.com/livekit/protocol v1.5.8-0.20230601212100-a186ecb11a98 - github.com/livekit/psrpc v0.3.1-0.20230528083849-53d664c6d912 + github.com/livekit/protocol v1.5.8-0.20230606012216-d8cc3581090d + github.com/livekit/psrpc v0.3.1 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 github.com/maxbrunsfeld/counterfeiter/v6 v6.6.1 @@ -98,7 +98,7 @@ require ( golang.org/x/sys v0.8.0 // indirect golang.org/x/text v0.9.0 // indirect golang.org/x/tools v0.6.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect google.golang.org/grpc v1.55.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 3a8f83488..4a044e114 100644 --- a/go.sum +++ b/go.sum @@ -122,10 +122,10 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20230523035537-27577c4e1646 h1:acGSGkWJdut7TUWozCDheHu4dwWFDqqRzv+SBbIY9Xo= github.com/livekit/mediatransportutil v0.0.0-20230523035537-27577c4e1646/go.mod h1:MRc0zSOSzXuFt0X218SgabzlaKevkvCckPgBEoHYc34= -github.com/livekit/protocol v1.5.8-0.20230601212100-a186ecb11a98 h1:OkQu5+sAgOzLqcpOH9WV5UyWsEcNwfvyi1KmpmXCKb0= -github.com/livekit/protocol v1.5.8-0.20230601212100-a186ecb11a98/go.mod h1:ZaOnsvP+JS4s7vI1UO+JVdBagvvLp/lBXDAl2hkDS0I= -github.com/livekit/psrpc v0.3.1-0.20230528083849-53d664c6d912 h1:q6i7ptxK6yZ220eyCWx5l1ZsUMQNyuYZ5feG91sM8EI= -github.com/livekit/psrpc v0.3.1-0.20230528083849-53d664c6d912/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= +github.com/livekit/protocol v1.5.8-0.20230606012216-d8cc3581090d h1:hSTJ80uM/suVDp40MyyCofWVUhtOz3qG3LwFD6/LoRU= +github.com/livekit/protocol v1.5.8-0.20230606012216-d8cc3581090d/go.mod h1:CnELWfDW0SvYy9II+YHsv83zz4gToYTbUdDM/7Fjxy4= +github.com/livekit/psrpc v0.3.1 h1:KfylgJHvoLQcc22t/oflwMOeSnx0c14G7cWsS+9MYS4= +github.com/livekit/psrpc v0.3.1/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= github.com/mackerelio/go-osstat v0.2.4/go.mod h1:Zy+qzGdZs3A9cuIqmgbJvwbmLQH9dJvtio5ZjJTbdlQ= github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= @@ -399,8 +399,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e h1:NumxXLPfHSndr3wBBdeKiVHjGVFzi9RX2HwwQke94iY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= From 076d8cad73d711f88142e1d30e72de45b3cae302 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 6 Jun 2023 11:20:57 +0530 Subject: [PATCH 207/324] Promote switch log to Infow (#1771) --- pkg/sfu/forwarder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index b2c4f4f27..32f8f2f30 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -1508,7 +1508,7 @@ func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer i } refTS += f.refTSOffset nextTS, explain := getNextTimestamp(lastTS, refTS, expectedTS, minTS) - f.logger.Debugw( + f.logger.Infow( "next timestamp on switch", "switchingAt", switchingAt.String(), "lastTS", lastTS, From 7ed3af193a3f1ec41e558f70dfa75e1a95d3d248 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 6 Jun 2023 11:28:13 +0530 Subject: [PATCH 208/324] No proof that this helps (#1772) --- pkg/config/config.go | 3 - pkg/rtc/mediatracksubscriptions.go | 1 - pkg/rtc/participant.go | 5 - pkg/rtc/types/interfaces.go | 2 - .../typesfakes/fake_local_participant.go | 65 -------- pkg/service/roommanager.go | 6 - pkg/sfu/buffer/rtpstats.go | 153 ++---------------- pkg/sfu/downtrack.go | 30 ++-- pkg/sfu/forwarder.go | 7 - pkg/sfu/rtpmunger.go | 46 +----- pkg/sfu/rtpmunger_test.go | 3 - 11 files changed, 26 insertions(+), 295 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index eded926ca..325eec68c 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -89,9 +89,6 @@ type RTCConfig struct { // force a reconnect on a subscription error ReconnectOnSubscriptionError *bool `yaml:"reconnect_on_subscription_error,omitempty"` - - // allow time stamp adjust to keep drift low, this is experimental - AllowTimestampAdjustment *bool `yaml:"allow_timestamp_adjustment,omitempty"` } type TURNServer struct { diff --git a/pkg/rtc/mediatracksubscriptions.go b/pkg/rtc/mediatracksubscriptions.go index 94b6f2b88..007428f50 100644 --- a/pkg/rtc/mediatracksubscriptions.go +++ b/pkg/rtc/mediatracksubscriptions.go @@ -104,7 +104,6 @@ func (t *MediaTrackSubscriptions) AddSubscriber(sub types.LocalParticipant, wr * sub.GetBufferFactory(), subscriberID, t.params.ReceiverConfig.PacketBufferSize, - sub.GetAllowTimestampAdjustment(), LoggerWithTrack(sub.GetLogger(), trackID, t.params.IsRelayed), ) if err != nil { diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index ecdfd2082..5c3cdef26 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -93,7 +93,6 @@ type ParticipantParams struct { SubscriberAllowPause bool SubscriptionLimitAudio int32 SubscriptionLimitVideo int32 - AllowTimestampAdjustment bool } type ParticipantImpl struct { @@ -233,10 +232,6 @@ func (p *ParticipantImpl) GetAdaptiveStream() bool { return p.params.AdaptiveStream } -func (p *ParticipantImpl) GetAllowTimestampAdjustment() bool { - return p.params.AllowTimestampAdjustment -} - func (p *ParticipantImpl) ID() livekit.ParticipantID { return p.params.SID } diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index c7c963372..2501d7013 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -341,8 +341,6 @@ type LocalParticipant interface { // down stream bandwidth management SetSubscriberAllowPause(allowPause bool) SetSubscriberChannelCapacity(channelCapacity int64) - - GetAllowTimestampAdjustment() bool } // Room is a container of participants, and can provide room-level actions diff --git a/pkg/rtc/types/typesfakes/fake_local_participant.go b/pkg/rtc/types/typesfakes/fake_local_participant.go index 3fbdcbc7d..e4ddfea5f 100644 --- a/pkg/rtc/types/typesfakes/fake_local_participant.go +++ b/pkg/rtc/types/typesfakes/fake_local_participant.go @@ -165,16 +165,6 @@ type FakeLocalParticipant struct { getAdaptiveStreamReturnsOnCall map[int]struct { result1 bool } - GetAllowTimestampAdjustmentStub func() bool - getAllowTimestampAdjustmentMutex sync.RWMutex - getAllowTimestampAdjustmentArgsForCall []struct { - } - getAllowTimestampAdjustmentReturns struct { - result1 bool - } - getAllowTimestampAdjustmentReturnsOnCall map[int]struct { - result1 bool - } GetAudioLevelStub func() (float64, bool) getAudioLevelMutex sync.RWMutex getAudioLevelArgsForCall []struct { @@ -1622,59 +1612,6 @@ func (fake *FakeLocalParticipant) GetAdaptiveStreamReturnsOnCall(i int, result1 }{result1} } -func (fake *FakeLocalParticipant) GetAllowTimestampAdjustment() bool { - fake.getAllowTimestampAdjustmentMutex.Lock() - ret, specificReturn := fake.getAllowTimestampAdjustmentReturnsOnCall[len(fake.getAllowTimestampAdjustmentArgsForCall)] - fake.getAllowTimestampAdjustmentArgsForCall = append(fake.getAllowTimestampAdjustmentArgsForCall, struct { - }{}) - stub := fake.GetAllowTimestampAdjustmentStub - fakeReturns := fake.getAllowTimestampAdjustmentReturns - fake.recordInvocation("GetAllowTimestampAdjustment", []interface{}{}) - fake.getAllowTimestampAdjustmentMutex.Unlock() - if stub != nil { - return stub() - } - if specificReturn { - return ret.result1 - } - return fakeReturns.result1 -} - -func (fake *FakeLocalParticipant) GetAllowTimestampAdjustmentCallCount() int { - fake.getAllowTimestampAdjustmentMutex.RLock() - defer fake.getAllowTimestampAdjustmentMutex.RUnlock() - return len(fake.getAllowTimestampAdjustmentArgsForCall) -} - -func (fake *FakeLocalParticipant) GetAllowTimestampAdjustmentCalls(stub func() bool) { - fake.getAllowTimestampAdjustmentMutex.Lock() - defer fake.getAllowTimestampAdjustmentMutex.Unlock() - fake.GetAllowTimestampAdjustmentStub = stub -} - -func (fake *FakeLocalParticipant) GetAllowTimestampAdjustmentReturns(result1 bool) { - fake.getAllowTimestampAdjustmentMutex.Lock() - defer fake.getAllowTimestampAdjustmentMutex.Unlock() - fake.GetAllowTimestampAdjustmentStub = nil - fake.getAllowTimestampAdjustmentReturns = struct { - result1 bool - }{result1} -} - -func (fake *FakeLocalParticipant) GetAllowTimestampAdjustmentReturnsOnCall(i int, result1 bool) { - fake.getAllowTimestampAdjustmentMutex.Lock() - defer fake.getAllowTimestampAdjustmentMutex.Unlock() - fake.GetAllowTimestampAdjustmentStub = nil - if fake.getAllowTimestampAdjustmentReturnsOnCall == nil { - fake.getAllowTimestampAdjustmentReturnsOnCall = make(map[int]struct { - result1 bool - }) - } - fake.getAllowTimestampAdjustmentReturnsOnCall[i] = struct { - result1 bool - }{result1} -} - func (fake *FakeLocalParticipant) GetAudioLevel() (float64, bool) { fake.getAudioLevelMutex.Lock() ret, specificReturn := fake.getAudioLevelReturnsOnCall[len(fake.getAudioLevelArgsForCall)] @@ -5519,8 +5456,6 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.debugInfoMutex.RUnlock() fake.getAdaptiveStreamMutex.RLock() defer fake.getAdaptiveStreamMutex.RUnlock() - fake.getAllowTimestampAdjustmentMutex.RLock() - defer fake.getAllowTimestampAdjustmentMutex.RUnlock() fake.getAudioLevelMutex.RLock() defer fake.getAudioLevelMutex.RUnlock() fake.getBufferFactoryMutex.RLock() diff --git a/pkg/service/roommanager.go b/pkg/service/roommanager.go index 061446b00..056b1b976 100644 --- a/pkg/service/roommanager.go +++ b/pkg/service/roommanager.go @@ -310,11 +310,6 @@ func (r *RoomManager) StartSession( if pi.SubscriberAllowPause != nil { subscriberAllowPause = *pi.SubscriberAllowPause } - // default do not allow timestamp adjustment - allowTimestampAdjustment := false - if r.config.RTC.AllowTimestampAdjustment != nil { - allowTimestampAdjustment = *r.config.RTC.AllowTimestampAdjustment - } participant, err = rtc.NewParticipant(rtc.ParticipantParams{ Identity: pi.Identity, Name: pi.Name, @@ -349,7 +344,6 @@ func (r *RoomManager) StartSession( SubscriberAllowPause: subscriberAllowPause, SubscriptionLimitAudio: r.config.Limit.SubscriptionLimitAudio, SubscriptionLimitVideo: r.config.Limit.SubscriptionLimitVideo, - AllowTimestampAdjustment: allowTimestampAdjustment, }) if err != nil { return err diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index 51c18c4d7..0c1952a69 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -17,12 +17,11 @@ import ( ) const ( - GapHistogramNumBins = 101 - NumSequenceNumbers = 65536 - FirstSnapshotId = 1 - SnInfoSize = 8192 - SnInfoMask = SnInfoSize - 1 - MeasurementWindowSecondsMin = float64(5.0) + GapHistogramNumBins = 101 + NumSequenceNumbers = 65536 + FirstSnapshotId = 1 + SnInfoSize = 8192 + SnInfoMask = SnInfoSize - 1 ) // ------------------------------------------------------- @@ -201,27 +200,17 @@ type RTPStats struct { srFirst *RTCPSenderReportData srNewest *RTCPSenderReportData - pidController *PIDController - nextSnapshotId uint32 snapshots map[uint32]*Snapshot } func NewRTPStats(params RTPStatsParams) *RTPStats { - r := &RTPStats{ + return &RTPStats{ params: params, logger: params.Logger, nextSnapshotId: FirstSnapshotId, snapshots: make(map[uint32]*Snapshot), - pidController: NewPIDController(params.Logger), } - - r.pidController.SetGains(2.0, 0.5, 0.25) - r.pidController.SetDerivativeLPF(0.02) - outMin, outMax := -0.025*float64(r.params.ClockRate), 0.025*float64(r.params.ClockRate) - r.pidController.SetOutputLimits(outMin, outMax) - r.pidController.SetIntegralLimits(outMin/2.0, outMax/2.0) - return r } func (r *RTPStats) Seed(from *RTPStats) { @@ -324,7 +313,6 @@ func (r *RTPStats) Seed(from *RTPStats) { func (r *RTPStats) SetLogger(logger logger.Logger) { r.logger = logger - r.pidController.SetLogger(logger) } func (r *RTPStats) Stop() { @@ -897,12 +885,12 @@ func (r *RTPStats) GetExpectedRTPTimestamp(at time.Time) (uint32, uint64, error) return uint32(expectedExtRTP), minTS, nil } -func (r *RTPStats) GetRtcpSenderReport(ssrc uint32, srFirst *RTCPSenderReportData, srNewest *RTCPSenderReportData) (*rtcp.SenderReport, float64) { +func (r *RTPStats) GetRtcpSenderReport(ssrc uint32, srFirst *RTCPSenderReportData, srNewest *RTCPSenderReportData) *rtcp.SenderReport { r.lock.Lock() defer r.lock.Unlock() if !r.initialized { - return nil, 0.0 + return nil } // construct current time based on monotonic clock @@ -933,17 +921,6 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32, srFirst *RTCPSenderReportDat } } - pidOutput := float64(0.0) - if timeSinceFirst.Seconds() > MeasurementWindowSecondsMin { - rtpDiffSinceFirst := nowRTPExt - r.extStartTS - rate := float64(rtpDiffSinceFirst) / timeSinceFirst.Seconds() - pidOutput = r.pidController.Update( - float64(r.params.ClockRate), - rate, - now, - ) - } - // monitor and log RTP timestamp anomalies var ntpDiffSinceLast time.Duration var rtpDiffSinceLast uint32 @@ -1012,7 +989,7 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32, srFirst *RTCPSenderReportDat RTPTime: nowRTP, PacketCount: r.getTotalPacketsPrimary() + r.packetsDuplicate + r.packetsPadding, OctetCount: uint32(r.bytes + r.bytesDuplicate + r.bytesPadding), - }, pidOutput + } } func (r *RTPStats) SnapshotRtcpReceptionReport(ssrc uint32, proxyFracLost uint8, snapshotId uint32) *rtcp.ReceptionReport { @@ -1922,115 +1899,3 @@ func AggregateRTPDeltaInfo(deltaInfoList []*RTPDeltaInfo) *RTPDeltaInfo { } // ------------------------------------------------------------------- - -type PIDController struct { - logger logger.Logger - - kp, ki, kd float64 - - tau float64 // low pass filter of D, time constant - - outMin, outMax float64 - isOutLimitsSet bool - - iMin, iMax float64 - isILimitsSet bool - - iVal, dVal float64 - - prevError, prevMeasurement float64 - prevMeasurementTime time.Time -} - -func NewPIDController(logger logger.Logger) *PIDController { - return &PIDController{ - logger: logger, - } -} - -func (p *PIDController) SetLogger(logger logger.Logger) { - p.logger = logger -} - -func (p *PIDController) SetGains(kp, ki, kd float64) { - p.kp = kp - p.ki = ki - p.kd = kd -} - -func (p *PIDController) SetDerivativeLPF(tau float64) { - p.tau = tau -} - -func (p *PIDController) SetOutputLimits(min, max float64) { - p.outMin = min - p.outMax = max - p.isOutLimitsSet = true -} - -func (p *PIDController) SetIntegralLimits(min, max float64) { - p.iMin = min - p.iMax = max - p.isILimitsSet = true -} - -func (p *PIDController) Update(setpoint, measurement float64, at time.Time) float64 { - errorTerm := setpoint - measurement - if p.prevMeasurementTime.IsZero() { - p.prevError = errorTerm - p.prevMeasurement = measurement - p.prevMeasurementTime = at - return 0 - } - - duration := at.Sub(p.prevMeasurementTime).Seconds() - if duration == 0 { - return 0 - } - - proportional := p.kp * errorTerm - - iVal := p.iVal + (0.5 * p.ki * duration * (errorTerm + p.prevError)) - boundIVal := iVal - if p.isILimitsSet { - if iVal > p.iMax { - boundIVal = p.iMax - } - if iVal < p.iMin { - boundIVal = p.iMin - } - } - p.iVal = boundIVal - - p.dVal = -(2.0*p.kd*(measurement-p.prevMeasurement) + (2.0*p.tau-duration)*p.dVal) / (2.0*p.tau + duration) - - output := proportional + p.iVal + p.dVal - boundOutput := output - if p.isOutLimitsSet { - if output > p.outMax { - boundOutput = p.outMax - } - if output < p.outMin { - boundOutput = p.outMin - } - } - - p.prevError = errorTerm - p.prevMeasurement = measurement - p.prevMeasurementTime = at - p.logger.Debugw( - "pid controller", - "setpoint", setpoint, - "measurement", measurement, - "errorTerm", errorTerm, - "proportional", proportional, - "integral", iVal, - "integralLimited", boundIVal, - "derivative", p.dVal, - "output", output, - "outputLimited", boundOutput, - ) - return boundOutput -} - -// ------------------------------------------------------------------- diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index b90d8ebb5..d8850ff64 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -185,8 +185,6 @@ type DownTrack struct { sequencer *sequencer bufferFactory *buffer.Factory - allowTimestampAdjustment bool - forwarder *Forwarder upstreamCodecs []webrtc.RTPCodecParameters @@ -254,7 +252,6 @@ func NewDownTrack( bf *buffer.Factory, subID livekit.ParticipantID, mt int, - allowTimestampAdjustment bool, logger logger.Logger, ) (*DownTrack, error) { var kind webrtc.RTPCodecType @@ -268,17 +265,16 @@ func NewDownTrack( } d := &DownTrack{ - logger: logger, - id: r.TrackID(), - subscriberID: subID, - maxTrack: mt, - streamID: r.StreamID(), - bufferFactory: bf, - allowTimestampAdjustment: allowTimestampAdjustment, - receiver: r, - upstreamCodecs: codecs, - kind: kind, - codec: codecs[0].RTPCodecCapability, + logger: logger, + id: r.TrackID(), + subscriberID: subID, + maxTrack: mt, + streamID: r.StreamID(), + bufferFactory: bf, + receiver: r, + upstreamCodecs: codecs, + kind: kind, + codec: codecs[0].RTPCodecCapability, } d.forwarder = NewForwarder( d.kind, @@ -1123,11 +1119,7 @@ func (d *DownTrack) CreateSenderReport() *rtcp.SenderReport { } srFirst, srNewest := d.receiver.GetRTCPSenderReportData(d.forwarder.GetReferenceLayerSpatial()) - sr, tsAdjust := d.rtpStats.GetRtcpSenderReport(d.ssrc, srFirst, srNewest) - if d.allowTimestampAdjustment { - d.forwarder.AdjustTimestamp(tsAdjust) - } - return sr + return d.rtpStats.GetRtcpSenderReport(d.ssrc, srFirst, srNewest) } func (d *DownTrack) writeBlankFrameRTP(duration float32, generation uint32) chan struct{} { diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 32f8f2f30..db2127065 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -1706,13 +1706,6 @@ func (f *Forwarder) GetRTPMungerParams() RTPMungerParams { return f.rtpMunger.GetParams() } -func (f *Forwarder) AdjustTimestamp(tsAdjust float64) { - f.lock.Lock() - defer f.lock.Unlock() - - f.rtpMunger.UpdateTsOffset(uint32(tsAdjust + 0.5)) -} - // ----------------------------------------------------------------------------- func getOptimalBandwidthNeeded(muted bool, pubMuted bool, maxPublishedLayer int32, brs Bitrates, maxLayer buffer.VideoLayer) int64 { diff --git a/pkg/sfu/rtpmunger.go b/pkg/sfu/rtpmunger.go index a80ad6000..452019507 100644 --- a/pkg/sfu/rtpmunger.go +++ b/pkg/sfu/rtpmunger.go @@ -52,14 +52,12 @@ func (r RTPMungerState) String() string { // ---------------------------------------------------------------------- type RTPMungerParams struct { - highestIncomingSN uint16 - lastSN uint16 - snOffset uint16 - highestIncomingTS uint32 - lastTS uint32 - tsOffset uint32 - tsOffsetAdjustment uint32 - lastMarker bool + highestIncomingSN uint16 + lastSN uint16 + snOffset uint16 + lastTS uint32 + tsOffset uint32 + lastMarker bool snOffsets [SnOffsetCacheSize]uint16 snOffsetsWritePtr int @@ -86,7 +84,6 @@ func (r *RTPMunger) GetParams() RTPMungerParams { highestIncomingSN: r.highestIncomingSN, lastSN: r.lastSN, snOffset: r.snOffset, - highestIncomingTS: r.highestIncomingTS, lastTS: r.lastTS, tsOffset: r.tsOffset, lastMarker: r.lastMarker, @@ -107,14 +104,12 @@ func (r *RTPMunger) SeedLast(state RTPMungerState) { func (r *RTPMunger) SetLastSnTs(extPkt *buffer.ExtPacket) { r.highestIncomingSN = extPkt.Packet.SequenceNumber - 1 - r.highestIncomingTS = extPkt.Packet.Timestamp r.lastSN = extPkt.Packet.SequenceNumber r.lastTS = extPkt.Packet.Timestamp } func (r *RTPMunger) UpdateSnTsOffsets(extPkt *buffer.ExtPacket, snAdjust uint16, tsAdjust uint32) { r.highestIncomingSN = extPkt.Packet.SequenceNumber - 1 - r.highestIncomingTS = extPkt.Packet.Timestamp r.snOffset = extPkt.Packet.SequenceNumber - r.lastSN - snAdjust r.tsOffset = extPkt.Packet.Timestamp - r.lastTS - tsAdjust @@ -131,10 +126,6 @@ func (r *RTPMunger) PacketDropped(extPkt *buffer.ExtPacket) { r.lastSN = extPkt.Packet.SequenceNumber - r.snOffset } -func (r *RTPMunger) UpdateTsOffset(tsAdjust uint32) { - r.tsOffsetAdjustment = tsAdjust -} - func (r *RTPMunger) UpdateAndGetSnTs(extPkt *buffer.ExtPacket) (*TranslationParamsRTP, error) { // if out-of-order, look up sequence number offset cache diff := extPkt.Packet.SequenceNumber - r.highestIncomingSN @@ -188,12 +179,6 @@ func (r *RTPMunger) UpdateAndGetSnTs(extPkt *buffer.ExtPacket) (*TranslationPara } } - // apply timestamp offset adjustment at the start of a frame only - if extPkt.Packet.Timestamp != r.highestIncomingTS && r.tsOffsetAdjustment != 0 { - r.tsOffset -= r.tsOffsetAdjustment - r.tsOffsetAdjustment = 0 - } - // in-order incoming packet, may or may not be contiguous. // In the case of loss (i.e. incoming sequence number is not contiguous), // forward even if it is a padding only packet. With temporal scalability, @@ -203,27 +188,8 @@ func (r *RTPMunger) UpdateAndGetSnTs(extPkt *buffer.ExtPacket) (*TranslationPara mungedSN := extPkt.Packet.SequenceNumber - r.snOffset mungedTS := extPkt.Packet.Timestamp - r.tsOffset - // with timestamp adjustment, it is possible that the adjustment causes munged timestamp to move backwards, - // detect that and adjust so that it does not move back - if extPkt.Packet.Timestamp != r.highestIncomingTS && (((mungedTS - r.lastTS) == 0) || (mungedTS-r.lastTS) > (1<<31)) { - adjustedMungedTS := r.lastTS + 1 - adjustedTSOffset := extPkt.Packet.Timestamp - adjustedMungedTS - r.logger.Debugw( - "adjust out-of-order timestamp offset", - "mungedTS", mungedTS, - "lastTS", r.lastTS, - "incomingTS", extPkt.Packet.Timestamp, - "offset", r.tsOffset, - "adjustedMungedTS", adjustedMungedTS, - "adjustedTSOffset", adjustedTSOffset, - ) - mungedTS = adjustedMungedTS - r.tsOffset = adjustedTSOffset - } - r.highestIncomingSN = extPkt.Packet.SequenceNumber r.lastSN = mungedSN - r.highestIncomingTS = extPkt.Packet.Timestamp r.lastTS = mungedTS r.lastMarker = extPkt.Packet.Marker diff --git a/pkg/sfu/rtpmunger_test.go b/pkg/sfu/rtpmunger_test.go index 68ea60716..b4d764ca3 100644 --- a/pkg/sfu/rtpmunger_test.go +++ b/pkg/sfu/rtpmunger_test.go @@ -28,7 +28,6 @@ func TestSetLastSnTs(t *testing.T) { r.SetLastSnTs(extPkt) require.Equal(t, uint16(23332), r.highestIncomingSN) - require.Equal(t, uint32(0xabcdef), r.highestIncomingTS) require.Equal(t, uint16(23333), r.lastSN) require.Equal(t, uint32(0xabcdef), r.lastTS) require.Equal(t, uint16(0), r.snOffset) @@ -54,7 +53,6 @@ func TestUpdateSnTsOffsets(t *testing.T) { extPkt, _ = testutils.GetTestExtPacket(params) r.UpdateSnTsOffsets(extPkt, 1, 1) require.Equal(t, uint16(33332), r.highestIncomingSN) - require.Equal(t, uint32(0xabcdef), r.highestIncomingTS) require.Equal(t, uint16(23333), r.lastSN) require.Equal(t, uint32(0xabcdef), r.lastTS) require.Equal(t, uint16(9999), r.snOffset) @@ -73,7 +71,6 @@ func TestPacketDropped(t *testing.T) { extPkt, _ := testutils.GetTestExtPacket(params) r.SetLastSnTs(extPkt) require.Equal(t, uint16(23332), r.highestIncomingSN) - require.Equal(t, uint32(0xabcdef), r.highestIncomingTS) require.Equal(t, uint16(23333), r.lastSN) require.Equal(t, uint32(0xabcdef), r.lastTS) require.Equal(t, uint16(0), r.snOffset) From b591140d66c19cb79cf418f2cb975cc6f7d9b149 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 6 Jun 2023 21:43:49 +0530 Subject: [PATCH 209/324] Ignore receiver report till initialized (#1773) --- pkg/sfu/buffer/rtpstats.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index 0c1952a69..574d7a543 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -527,7 +527,7 @@ func (r *RTPStats) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt uint32 r.lock.Lock() defer r.lock.Unlock() - if !r.endTime.IsZero() || !r.params.IsReceiverReportDriven || rr.LastSequenceNumber < r.extStartSN { + if !r.initialized || !r.endTime.IsZero() || !r.params.IsReceiverReportDriven || rr.LastSequenceNumber < r.extStartSN { // it is possible that the `LastSequenceNumber` in the receiver report is before the starting // sequence number when dummy packets are used to trigger Pion's OnTrack path. return From bc11419755250948bba06f50e0060cdb5c313805 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 6 Jun 2023 23:32:28 -0700 Subject: [PATCH 210/324] Update module github.com/hashicorp/golang-lru/v2 to v2.0.3 (#1774) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d04dc76af..3e398eeab 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/google/wire v0.5.0 github.com/gorilla/websocket v1.5.0 github.com/hashicorp/go-version v1.6.0 - github.com/hashicorp/golang-lru/v2 v2.0.2 + github.com/hashicorp/golang-lru/v2 v2.0.3 github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230523035537-27577c4e1646 diff --git a/go.sum b/go.sum index 4a044e114..761968655 100644 --- a/go.sum +++ b/go.sum @@ -87,8 +87,8 @@ github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUD github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/golang-lru/v2 v2.0.2 h1:Dwmkdr5Nc/oBiXgJS3CDHNhJtIHkuZ3DZF5twqnfBdU= -github.com/hashicorp/golang-lru/v2 v2.0.2/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/golang-lru/v2 v2.0.3 h1:kmRrRLlInXvng0SmLxmQpQkpbYAvcXm7NPDrgxJa9mE= +github.com/hashicorp/golang-lru/v2 v2.0.3/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/josharian/native v0.0.0-20200817173448-b6b71def0850/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= github.com/josharian/native v1.0.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= From 22813cd2be9b4daa99ae4e9276cee19973bca1fd Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 8 Jun 2023 01:40:07 +0530 Subject: [PATCH 211/324] Recreate channel observer irrespective of probe success/fail. (#1778) --- pkg/sfu/streamallocator/probe_controller.go | 27 ++++++++------------- pkg/sfu/streamallocator/streamallocator.go | 7 ++++-- 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/pkg/sfu/streamallocator/probe_controller.go b/pkg/sfu/streamallocator/probe_controller.go index f32c38b14..edb0dfefe 100644 --- a/pkg/sfu/streamallocator/probe_controller.go +++ b/pkg/sfu/streamallocator/probe_controller.go @@ -39,7 +39,7 @@ type ProbeController struct { probeTrendObserved bool probeEndTime time.Time - onProbeSuccess func() + onProbeDone func(isSuccessful bool) } func NewProbeController(params ProbeControllerParams) *ProbeController { @@ -51,11 +51,11 @@ func NewProbeController(params ProbeControllerParams) *ProbeController { return p } -func (p *ProbeController) OnProbeSuccess(f func()) { +func (p *ProbeController) OnProbeDone(f func(isSuccessful bool)) { p.lock.Lock() defer p.lock.Unlock() - p.onProbeSuccess = f + p.onProbeDone = f } func (p *ProbeController) Reset() { @@ -79,15 +79,11 @@ func (p *ProbeController) ProbeClusterDone(info ProbeClusterInfo, lowestEstimate if p.abortedProbeClusterId == ProbeClusterIdInvalid { // successful probe, finalize isSuccessful := p.finalizeProbeLocked() - - var onProbeSuccess func() - if isSuccessful { - onProbeSuccess = p.onProbeSuccess - } + onProbeDone := p.onProbeDone p.lock.Unlock() - if onProbeSuccess != nil { - onProbeSuccess() + if onProbeDone != nil { + onProbeDone(isSuccessful) } return } @@ -145,19 +141,16 @@ func (p *ProbeController) CheckProbe(trend ChannelTrend, highestEstimate int64) func (p *ProbeController) MaybeFinalizeProbe() { p.lock.Lock() + var onProbeDone func(bool) isSuccessful := false if p.isInProbeLocked() && !p.probeEndTime.IsZero() && time.Now().After(p.probeEndTime) { isSuccessful = p.finalizeProbeLocked() - } - - var onProbeSuccess func() - if isSuccessful { - onProbeSuccess = p.onProbeSuccess + onProbeDone = p.onProbeDone } p.lock.Unlock() - if onProbeSuccess != nil { - onProbeSuccess() + if onProbeDone != nil { + onProbeDone(isSuccessful) } } diff --git a/pkg/sfu/streamallocator/streamallocator.go b/pkg/sfu/streamallocator/streamallocator.go index 7cc35f84a..edea5051c 100644 --- a/pkg/sfu/streamallocator/streamallocator.go +++ b/pkg/sfu/streamallocator/streamallocator.go @@ -200,7 +200,7 @@ func NewStreamAllocator(params StreamAllocatorParams) *StreamAllocator { Prober: s.prober, Logger: params.Logger, }) - s.probeController.OnProbeSuccess(s.onProbeSuccess) + s.probeController.OnProbeDone(s.onProbeDone) s.resetState() @@ -916,7 +916,7 @@ func (s *StreamAllocator) allocateTrack(track *Track) { s.adjustState() } -func (s *StreamAllocator) onProbeSuccess() { +func (s *StreamAllocator) onProbeDone(isSuccessful bool) { highestEstimateInProbe := s.channelObserver.GetHighestEstimate() // @@ -931,6 +931,9 @@ func (s *StreamAllocator) onProbeSuccess() { // the send side is in full control of bandwidth estimation. // s.channelObserver = s.newChannelObserverNonProbe() + if !isSuccessful { + return + } // probe estimate is same or higher, commit it and try to allocate deficient tracks s.params.Logger.Infow( From 8235310a923c880903237f393aecb486ceaa6d8e Mon Sep 17 00:00:00 2001 From: David Colburn Date: Wed, 7 Jun 2023 16:27:37 -0700 Subject: [PATCH 212/324] don't save info after UpdateStream (#1779) --- pkg/service/egress.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pkg/service/egress.go b/pkg/service/egress.go index a05444077..29ce6f07a 100644 --- a/pkg/service/egress.go +++ b/pkg/service/egress.go @@ -255,12 +255,6 @@ func (s *EgressService) UpdateStream(ctx context.Context, req *livekit.UpdateStr } } - go func() { - if err := s.es.UpdateEgress(ctx, info); err != nil { - logger.Errorw("could not write egress info", err) - } - }() - return info, nil } From f518f5d7430d4109e42b70500318c9439c4e43f4 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 9 Jun 2023 12:13:06 +0530 Subject: [PATCH 213/324] Log head SN when packet cannot be fetched (#1780) --- go.mod | 2 +- go.sum | 4 ++-- pkg/sfu/buffer/buffer.go | 2 +- pkg/sfu/buffer/rtpstats.go | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 3e398eeab..b86a64fcd 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.3 github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 - github.com/livekit/mediatransportutil v0.0.0-20230523035537-27577c4e1646 + github.com/livekit/mediatransportutil v0.0.0-20230609061558-44a151ad2e1d github.com/livekit/protocol v1.5.8-0.20230606012216-d8cc3581090d github.com/livekit/psrpc v0.3.1 github.com/mackerelio/go-osstat v0.2.4 diff --git a/go.sum b/go.sum index 761968655..e3600dfea 100644 --- a/go.sum +++ b/go.sum @@ -120,8 +120,8 @@ github.com/lithammer/shortuuid/v4 v4.0.0 h1:QRbbVkfgNippHOS8PXDkti4NaWeyYfcBTHtw github.com/lithammer/shortuuid/v4 v4.0.0/go.mod h1:Zs8puNcrvf2rV9rTH51ZLLcj7ZXqQI3lv67aw4KiB1Y= github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkDaKb5iXdynYrzB84ErPPO4LbRASk58= github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= -github.com/livekit/mediatransportutil v0.0.0-20230523035537-27577c4e1646 h1:acGSGkWJdut7TUWozCDheHu4dwWFDqqRzv+SBbIY9Xo= -github.com/livekit/mediatransportutil v0.0.0-20230523035537-27577c4e1646/go.mod h1:MRc0zSOSzXuFt0X218SgabzlaKevkvCckPgBEoHYc34= +github.com/livekit/mediatransportutil v0.0.0-20230609061558-44a151ad2e1d h1:7yjO7MpM8e7FQKQ86XF+5yKgDcFIL4DhT7t6NbL8E3Y= +github.com/livekit/mediatransportutil v0.0.0-20230609061558-44a151ad2e1d/go.mod h1:MRc0zSOSzXuFt0X218SgabzlaKevkvCckPgBEoHYc34= github.com/livekit/protocol v1.5.8-0.20230606012216-d8cc3581090d h1:hSTJ80uM/suVDp40MyyCofWVUhtOz3qG3LwFD6/LoRU= github.com/livekit/protocol v1.5.8-0.20230606012216-d8cc3581090d/go.mod h1:CnELWfDW0SvYy9II+YHsv83zz4gToYTbUdDM/7Fjxy4= github.com/livekit/psrpc v0.3.1 h1:KfylgJHvoLQcc22t/oflwMOeSnx0c14G7cWsS+9MYS4= diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go index 575f2d433..4eabb04a1 100644 --- a/pkg/sfu/buffer/buffer.go +++ b/pkg/sfu/buffer/buffer.go @@ -440,7 +440,7 @@ func (b *Buffer) calc(pkt []byte, arrivalTime time.Time) { func (b *Buffer) patchExtPacket(ep *ExtPacket, buf []byte) *ExtPacket { n, err := b.getPacket(buf, ep.Packet.SequenceNumber) if err != nil { - b.logger.Warnw("could not get packet", err, "sn", ep.Packet.SequenceNumber) + b.logger.Warnw("could not get packet", err, "sn", ep.Packet.SequenceNumber, "headSN", b.bucket.HeadSequenceNumber()) return nil } ep.RawPacket = buf[:n] diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index 574d7a543..173d7f3a1 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -1493,7 +1493,7 @@ func (r *RTPStats) getIntervalStats(startInclusive uint16, endExclusive uint16) } if packetsNotFound != 0 { - r.logger.Warnw( + r.logger.Errorw( "could not find some packets", nil, "start", startInclusive, "end", endExclusive, From 72ed5b19f737d829aef5bbd1ac5ca09563aa02ea Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 9 Jun 2023 23:31:25 +0530 Subject: [PATCH 214/324] Use receiver report stats for loss/rtt/jitter. (#1781) * Use receiver report stats for loss/rtt/jitter. Reversing a bit of https://github.com/livekit/livekit/pull/1664. That PR did two snapshots (one based on what SFU is sending and one based on combination of what SFU is sending reconciled with stats reported from client via RTCP Receiver Report). That PR reported SFU only view to analytics. But, that view does not have information about loss seen by client in the downstream. Also, that does not have RTT/jitter information. The rationale behind using SFU only view is that SFU should report what it sends irrespective of client is receiving or not. But, that view did not have proper loss/RTT/jitter. So, switch back to reporting SFU + receiver report reconciled view. The down side is that when receiver reports are not receiver, packets sent/bytes sent will not be reported to analytics. An option is to report SFU only view if there are no receiver reports. But, it becomes complex because of the offset. Receiver report would acknowledge certain range whereas SFU only view could be different because of propagation delay. To simplify, just using the reconciled view to report to analytics. Using the available view will require a bunch more work to produce accurate data. (NOTE: all this started due to a bug where RTCP was not restarted on a track resume which killed receiver reports and we went on this path to distinguish between publisher stopping vs RTCP receiver report not happening) One optimisation to here here concerns the check to see if publisher is sending data. Using a full DeltaInfo for that is an overkill. Can do a lighter weight for that later. * return available streams * fix test --- pkg/sfu/connectionquality/connectionstats.go | 52 +++++---- .../connectionquality/connectionstats_test.go | 101 ++++++++++++------ 2 files changed, 95 insertions(+), 58 deletions(-) diff --git a/pkg/sfu/connectionquality/connectionstats.go b/pkg/sfu/connectionquality/connectionstats.go index 9cb5c716c..fb5c7e710 100644 --- a/pkg/sfu/connectionquality/connectionstats.go +++ b/pkg/sfu/connectionquality/connectionstats.go @@ -16,8 +16,6 @@ import ( const ( UpdateInterval = 5 * time.Second - processThreshold = 0.95 - noStatsTooLongMultiplier = 2 noReceiverReportTooLongThreshold = 30 * time.Second ) @@ -120,9 +118,9 @@ func (cs *ConnectionStats) updateScoreWithAggregate(agg *buffer.RTPDeltaInfo, at return mos } -func (cs *ConnectionStats) updateScoreFromReceiverReport(at time.Time) float32 { +func (cs *ConnectionStats) updateScoreFromReceiverReport(at time.Time) (float32, map[uint32]*buffer.StreamStatsWithLayers) { if cs.params.GetDeltaStatsOverridden == nil || cs.params.GetLastReceiverReportTime == nil { - return MinMOS + return MinMOS, nil } cs.lock.RLock() @@ -131,7 +129,7 @@ func (cs *ConnectionStats) updateScoreFromReceiverReport(at time.Time) float32 { if streamingStartedAt.IsZero() { // not streaming, just return current score mos, _ := cs.scorer.GetMOSAndQuality() - return mos + return mos, nil } streams := cs.params.GetDeltaStatsOverridden() @@ -143,12 +141,12 @@ func (cs *ConnectionStats) updateScoreFromReceiverReport(at time.Time) float32 { } if time.Since(marker) > noReceiverReportTooLongThreshold { // have not received receiver report for a long time when streaming, run with nil stat - return cs.updateScoreWithAggregate(nil, at) + return cs.updateScoreWithAggregate(nil, at), nil } // wait for receiver report, return current score mos, _ := cs.scorer.GetMOSAndQuality() - return mos + return mos, nil } // delta stat duration could be large due to not receiving receiver report for a long time (for example, due to mute), @@ -157,17 +155,27 @@ func (cs *ConnectionStats) updateScoreFromReceiverReport(at time.Time) float32 { if streamingStartedAt.After(cs.params.GetLastReceiverReportTime()) { // last receiver report was before streaming started, wait for next one mos, _ := cs.scorer.GetMOSAndQuality() - return mos + return mos, streams } if streamingStartedAt.After(agg.StartTime) { agg.Duration = agg.StartTime.Add(agg.Duration).Sub(streamingStartedAt) agg.StartTime = streamingStartedAt } - return cs.updateScoreWithAggregate(agg, at) + return cs.updateScoreWithAggregate(agg, at), streams } -func (cs *ConnectionStats) updateScore(streams map[uint32]*buffer.StreamStatsWithLayers, at time.Time) float32 { +func (cs *ConnectionStats) updateScore(at time.Time) (float32, map[uint32]*buffer.StreamStatsWithLayers) { + if cs.params.GetDeltaStats == nil { + return MinMOS, nil + } + + streams := cs.params.GetDeltaStats() + if len(streams) == 0 { + mos, _ := cs.scorer.GetMOSAndQuality() + return mos, nil + } + deltaInfoList := make([]*buffer.RTPDeltaInfo, 0, len(streams)) for _, s := range streams { deltaInfoList = append(deltaInfoList, s.RTPStats) @@ -185,7 +193,7 @@ func (cs *ConnectionStats) updateScore(streams map[uint32]*buffer.StreamStatsWit return cs.updateScoreFromReceiverReport(at) } - return cs.updateScoreWithAggregate(agg, at) + return cs.updateScoreWithAggregate(agg, at), streams } func (cs *ConnectionStats) maybeSetStreamingStart(at time.Time) { @@ -203,18 +211,9 @@ func (cs *ConnectionStats) clearStreamingStart() { } func (cs *ConnectionStats) getStat(at time.Time) { - if cs.params.GetDeltaStats == nil { - return - } + score, streams := cs.updateScore(at) - streams := cs.params.GetDeltaStats() - if len(streams) == 0 { - return - } - - score := cs.updateScore(streams, at) - - if cs.onStatsUpdate != nil { + if cs.onStatsUpdate != nil && len(streams) != 0 { analyticsStreams := make([]*livekit.AnalyticsStream, 0, len(streams)) for ssrc, stream := range streams { as := toAnalyticsStream(ssrc, stream.RTPStats) @@ -317,6 +316,13 @@ func toAggregateDeltaInfo(streams map[uint32]*buffer.StreamStatsWithLayers) *buf } func toAnalyticsStream(ssrc uint32, deltaStats *buffer.RTPDeltaInfo) *livekit.AnalyticsStream { + // discount the feed side loss when reporting forwarded track stats + packetsLost := deltaStats.PacketsLost + if deltaStats.PacketsMissing > packetsLost { + packetsLost = 0 + } else { + packetsLost -= deltaStats.PacketsMissing + } return &livekit.AnalyticsStream{ Ssrc: ssrc, PrimaryPackets: deltaStats.Packets, @@ -325,7 +331,7 @@ func toAnalyticsStream(ssrc uint32, deltaStats *buffer.RTPDeltaInfo) *livekit.An RetransmitBytes: deltaStats.BytesDuplicate, PaddingPackets: deltaStats.PacketsPadding, PaddingBytes: deltaStats.BytesPadding, - PacketsLost: deltaStats.PacketsLost, + PacketsLost: packetsLost, Frames: deltaStats.Frames, Rtt: deltaStats.RttMax, Jitter: uint32(deltaStats.JitterMax), diff --git a/pkg/sfu/connectionquality/connectionstats_test.go b/pkg/sfu/connectionquality/connectionstats_test.go index 57793ef85..dce11601d 100644 --- a/pkg/sfu/connectionquality/connectionstats_test.go +++ b/pkg/sfu/connectionquality/connectionstats_test.go @@ -12,19 +12,30 @@ import ( "github.com/livekit/protocol/logger" ) -func newConnectionStats(mimeType string, isFECEnabled bool, includeRTT bool, includeJitter bool) *ConnectionStats { +func newConnectionStats( + mimeType string, + isFECEnabled bool, + includeRTT bool, + includeJitter bool, + getDeltaStats func() map[uint32]*buffer.StreamStatsWithLayers, +) *ConnectionStats { return NewConnectionStats(ConnectionStatsParams{ MimeType: mimeType, IsFECEnabled: isFECEnabled, IncludeRTT: includeRTT, IncludeJitter: includeJitter, + GetDeltaStats: getDeltaStats, Logger: logger.GetLogger(), }) } func TestConnectionQuality(t *testing.T) { t.Run("quality scorer state machine", func(t *testing.T) { - cs := newConnectionStats("audio/opus", false, true, true) + var streams map[uint32]*buffer.StreamStatsWithLayers + getDeltaStats := func() map[uint32]*buffer.StreamStatsWithLayers { + return streams + } + cs := newConnectionStats("audio/opus", false, true, true, getDeltaStats) duration := 5 * time.Second now := time.Now() @@ -32,13 +43,13 @@ func TestConnectionQuality(t *testing.T) { cs.UpdateMute(false, now.Add(-1*time.Second)) // no data and not enough unmute time should return default state which is EXCELLENT quality - cs.updateScore(nil, now) + cs.updateScore(now) mos, quality := cs.GetScoreAndQuality() require.Greater(t, float32(4.6), mos) require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality) // best conditions (no loss, jitter/rtt = 0) - quality should stay EXCELLENT - streams := map[uint32]*buffer.StreamStatsWithLayers{ + streams = map[uint32]*buffer.StreamStatsWithLayers{ 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, @@ -47,7 +58,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(streams, now.Add(duration)) + cs.updateScore(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(4.6), mos) require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality) @@ -72,7 +83,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(streams, now.Add(duration)) + cs.updateScore(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(2.1), mos) require.Equal(t, livekit.ConnectionQuality_POOR, quality) @@ -90,7 +101,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(streams, now.Add(duration)) + cs.updateScore(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(4.1), mos) require.Equal(t, livekit.ConnectionQuality_GOOD, quality) @@ -106,7 +117,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(streams, now.Add(duration)) + cs.updateScore(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(4.1), mos) require.Equal(t, livekit.ConnectionQuality_GOOD, quality) @@ -122,7 +133,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(streams, now.Add(duration)) + cs.updateScore(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(4.6), mos) require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality) @@ -139,7 +150,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(streams, now.Add(duration)) + cs.updateScore(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(4.1), mos) require.Equal(t, livekit.ConnectionQuality_GOOD, quality) @@ -155,7 +166,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(streams, now.Add(duration)) + cs.updateScore(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(4.1), mos) require.Equal(t, livekit.ConnectionQuality_GOOD, quality) @@ -171,7 +182,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(streams, now.Add(duration)) + cs.updateScore(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(4.6), mos) require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality) @@ -188,7 +199,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(streams, now.Add(duration)) + cs.updateScore(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(2.1), mos) require.Equal(t, livekit.ConnectionQuality_POOR, quality) @@ -212,7 +223,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(streams, now.Add(duration)) + cs.updateScore(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(4.6), mos) require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality) @@ -228,7 +239,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(streams, now.Add(duration)) + cs.updateScore(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(2.1), mos) require.Equal(t, livekit.ConnectionQuality_POOR, quality) @@ -250,7 +261,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(streams, now.Add(duration)) + cs.updateScore(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(4.6), mos) require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality) @@ -274,7 +285,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(streams, now.Add(duration)) + cs.updateScore(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(4.1), mos) require.Equal(t, livekit.ConnectionQuality_GOOD, quality) @@ -298,7 +309,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(streams, now.Add(duration)) + cs.updateScore(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(4.1), mos) require.Equal(t, livekit.ConnectionQuality_GOOD, quality) @@ -324,7 +335,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(streams, now.Add(duration)) + cs.updateScore(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(4.1), mos) require.Equal(t, livekit.ConnectionQuality_GOOD, quality) @@ -349,14 +360,18 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(streams, now.Add(duration)) + cs.updateScore(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(4.1), mos) require.Equal(t, livekit.ConnectionQuality_GOOD, quality) }) t.Run("quality scorer dependent rtt", func(t *testing.T) { - cs := newConnectionStats("audio/opus", false, false, true) + var streams map[uint32]*buffer.StreamStatsWithLayers + getDeltaStats := func() map[uint32]*buffer.StreamStatsWithLayers { + return streams + } + cs := newConnectionStats("audio/opus", false, false, true, getDeltaStats) duration := 5 * time.Second now := time.Now() @@ -366,7 +381,7 @@ func TestConnectionQuality(t *testing.T) { // RTT does not knock quality down because it is dependent and hence not taken into account // at 2% loss, quality should stay at EXCELLENT purely based on loss. With high RTT (700 ms) // quality should drop to GOOD if RTT were taken into consideration - streams := map[uint32]*buffer.StreamStatsWithLayers{ + streams = map[uint32]*buffer.StreamStatsWithLayers{ 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, @@ -377,14 +392,18 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(streams, now.Add(duration)) + cs.updateScore(now.Add(duration)) mos, quality := cs.GetScoreAndQuality() require.Greater(t, float32(4.6), mos) require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality) }) t.Run("quality scorer dependent jitter", func(t *testing.T) { - cs := newConnectionStats("audio/opus", false, true, false) + var streams map[uint32]*buffer.StreamStatsWithLayers + getDeltaStats := func() map[uint32]*buffer.StreamStatsWithLayers { + return streams + } + cs := newConnectionStats("audio/opus", false, true, false, getDeltaStats) duration := 5 * time.Second now := time.Now() @@ -394,7 +413,7 @@ func TestConnectionQuality(t *testing.T) { // Jitter does not knock quality down because it is dependent and hence not taken into account // at 2% loss, quality should stay at EXCELLENT purely based on loss. With high jitter (200 ms) // quality should drop to GOOD if jitter were taken into consideration - streams := map[uint32]*buffer.StreamStatsWithLayers{ + streams = map[uint32]*buffer.StreamStatsWithLayers{ 1: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, @@ -405,7 +424,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(streams, now.Add(duration)) + cs.updateScore(now.Add(duration)) mos, quality := cs.GetScoreAndQuality() require.Greater(t, float32(4.6), mos) require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality) @@ -549,14 +568,18 @@ func TestConnectionQuality(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - cs := newConnectionStats(tc.mimeType, tc.isFECEnabled, true, true) + var streams map[uint32]*buffer.StreamStatsWithLayers + getDeltaStats := func() map[uint32]*buffer.StreamStatsWithLayers { + return streams + } + cs := newConnectionStats(tc.mimeType, tc.isFECEnabled, true, true, getDeltaStats) duration := 5 * time.Second now := time.Now() cs.Start(&livekit.TrackInfo{Type: livekit.TrackType_AUDIO}, now.Add(-duration)) for _, eq := range tc.expectedQualities { - streams := map[uint32]*buffer.StreamStatsWithLayers{ + streams = map[uint32]*buffer.StreamStatsWithLayers{ 123: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, @@ -566,7 +589,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(streams, now.Add(duration)) + cs.updateScore(now.Add(duration)) mos, quality := cs.GetScoreAndQuality() require.Greater(t, eq.expectedMOS, mos) require.Equal(t, eq.expectedQuality, quality) @@ -642,7 +665,11 @@ func TestConnectionQuality(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - cs := newConnectionStats("video/vp8", false, true, true) + var streams map[uint32]*buffer.StreamStatsWithLayers + getDeltaStats := func() map[uint32]*buffer.StreamStatsWithLayers { + return streams + } + cs := newConnectionStats("video/vp8", false, true, true, getDeltaStats) duration := 5 * time.Second now := time.Now() @@ -652,7 +679,7 @@ func TestConnectionQuality(t *testing.T) { cs.AddBitrateTransition(tr.bitrate, now.Add(tr.offset)) } - streams := map[uint32]*buffer.StreamStatsWithLayers{ + streams = map[uint32]*buffer.StreamStatsWithLayers{ 123: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, @@ -662,7 +689,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(streams, now.Add(duration)) + cs.updateScore(now.Add(duration)) mos, quality := cs.GetScoreAndQuality() require.Greater(t, tc.expectedMOS, mos) require.Equal(t, tc.expectedQuality, quality) @@ -729,7 +756,11 @@ func TestConnectionQuality(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - cs := newConnectionStats("video/vp8", false, true, true) + var streams map[uint32]*buffer.StreamStatsWithLayers + getDeltaStats := func() map[uint32]*buffer.StreamStatsWithLayers { + return streams + } + cs := newConnectionStats("video/vp8", false, true, true, getDeltaStats) duration := 5 * time.Second now := time.Now() @@ -739,7 +770,7 @@ func TestConnectionQuality(t *testing.T) { cs.AddLayerTransition(tr.distance, now.Add(tr.offset)) } - streams := map[uint32]*buffer.StreamStatsWithLayers{ + streams = map[uint32]*buffer.StreamStatsWithLayers{ 123: { RTPStats: &buffer.RTPDeltaInfo{ StartTime: now, @@ -748,7 +779,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(streams, now.Add(duration)) + cs.updateScore(now.Add(duration)) mos, quality := cs.GetScoreAndQuality() require.Greater(t, tc.expectedMOS, mos) require.Equal(t, tc.expectedQuality, quality) From 0e7bdeabcb10679e951a96b880669651a0160313 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sat, 10 Jun 2023 02:07:28 +0530 Subject: [PATCH 215/324] Simplify probe done handling. (#1782) * Simplify probe done handling. Seeing a case where the channel abserver is not re-created after an aborted probe. Simplifying probe done (no callbacks, making it synchronous). * log more --- pkg/sfu/streamallocator/probe_controller.go | 52 +++++++++------------ pkg/sfu/streamallocator/prober.go | 2 +- pkg/sfu/streamallocator/streamallocator.go | 13 ++++-- 3 files changed, 31 insertions(+), 36 deletions(-) diff --git a/pkg/sfu/streamallocator/probe_controller.go b/pkg/sfu/streamallocator/probe_controller.go index edb0dfefe..519d31832 100644 --- a/pkg/sfu/streamallocator/probe_controller.go +++ b/pkg/sfu/streamallocator/probe_controller.go @@ -38,8 +38,6 @@ type ProbeController struct { abortedProbeClusterId ProbeClusterId probeTrendObserved bool probeEndTime time.Time - - onProbeDone func(isSuccessful bool) } func NewProbeController(params ProbeControllerParams) *ProbeController { @@ -51,13 +49,6 @@ func NewProbeController(params ProbeControllerParams) *ProbeController { return p } -func (p *ProbeController) OnProbeDone(f func(isSuccessful bool)) { - p.lock.Lock() - defer p.lock.Unlock() - - p.onProbeDone = f -} - func (p *ProbeController) Reset() { p.lock.Lock() defer p.lock.Unlock() @@ -69,23 +60,18 @@ func (p *ProbeController) Reset() { p.clearProbeLocked() } -func (p *ProbeController) ProbeClusterDone(info ProbeClusterInfo, lowestEstimate int64) { +func (p *ProbeController) ProbeClusterDone(info ProbeClusterInfo, lowestEstimate int64) bool { p.lock.Lock() + defer p.lock.Unlock() + if p.probeClusterId != info.Id { - p.lock.Unlock() - return + p.params.Logger.Infow("not expected probe cluster", "probeClusterId", p.probeClusterId, "resetProbeClusterId", info.Id) + return false } if p.abortedProbeClusterId == ProbeClusterIdInvalid { // successful probe, finalize - isSuccessful := p.finalizeProbeLocked() - onProbeDone := p.onProbeDone - p.lock.Unlock() - - if onProbeDone != nil { - onProbeDone(isSuccessful) - } - return + return p.finalizeProbeLocked() } // ensure probe queue is flushed @@ -97,7 +83,15 @@ func (p *ProbeController) ProbeClusterDone(info ProbeClusterInfo, lowestEstimate } queueWait := time.Duration(queueTime+float64(ProbeSettleWait)) * time.Millisecond p.probeEndTime = p.lastProbeStartTime.Add(queueWait) - p.lock.Unlock() + p.params.Logger.Infow( + "setting probe end time", + "probeClusterId", p.probeClusterId, + "expectedDuration", expectedDuration, + "queueTime", queueTime, + "queueWait", queueWait, + "probeEndTime", p.probeEndTime, + ) + return false } func (p *ProbeController) CheckProbe(trend ChannelTrend, highestEstimate int64) { @@ -139,19 +133,15 @@ func (p *ProbeController) CheckProbe(trend ChannelTrend, highestEstimate int64) } } -func (p *ProbeController) MaybeFinalizeProbe() { +func (p *ProbeController) MaybeFinalizeProbe() (isHandled bool, isSuccessful bool) { p.lock.Lock() - var onProbeDone func(bool) - isSuccessful := false - if p.isInProbeLocked() && !p.probeEndTime.IsZero() && time.Now().After(p.probeEndTime) { - isSuccessful = p.finalizeProbeLocked() - onProbeDone = p.onProbeDone - } - p.lock.Unlock() + defer p.lock.Unlock() - if onProbeDone != nil { - onProbeDone(isSuccessful) + if p.isInProbeLocked() && !p.probeEndTime.IsZero() && time.Now().After(p.probeEndTime) { + return true, p.finalizeProbeLocked() } + + return false, false } func (p *ProbeController) DoesProbeNeedFinalize() bool { diff --git a/pkg/sfu/streamallocator/prober.go b/pkg/sfu/streamallocator/prober.go index a1b1a3205..66ef47b8a 100644 --- a/pkg/sfu/streamallocator/prober.go +++ b/pkg/sfu/streamallocator/prober.go @@ -175,7 +175,7 @@ func (p *Prober) Reset() { p.clustersMu.Lock() if p.activeCluster != nil { - p.logger.Debugw("resetting active cluster", "cluster", p.activeCluster.String()) + p.logger.Infow("prober: resetting active cluster", "cluster", p.activeCluster.String()) reset = true info = p.activeCluster.GetInfo() } diff --git a/pkg/sfu/streamallocator/streamallocator.go b/pkg/sfu/streamallocator/streamallocator.go index edea5051c..d13141e7d 100644 --- a/pkg/sfu/streamallocator/streamallocator.go +++ b/pkg/sfu/streamallocator/streamallocator.go @@ -200,7 +200,6 @@ func NewStreamAllocator(params StreamAllocatorParams) *StreamAllocator { Prober: s.prober, Logger: params.Logger, }) - s.probeController.OnProbeDone(s.onProbeDone) s.resetState() @@ -492,7 +491,7 @@ func (s *StreamAllocator) OnSendProbe(bytesToSend int) { }) } -// called when prober wants to send packet(s) +// called when prober finishes a probe cluster, could be called when prober is reset which stops an active cluster func (s *StreamAllocator) OnProbeClusterDone(info ProbeClusterInfo) { s.postEvent(Event{ Signal: streamAllocatorSignalProbeClusterDone, @@ -640,7 +639,10 @@ func (s *StreamAllocator) handleSignalEstimate(event *Event) { func (s *StreamAllocator) handleSignalPeriodicPing(event *Event) { // finalize probe if necessary - s.probeController.MaybeFinalizeProbe() + isHandled, isSuccessful := s.probeController.MaybeFinalizeProbe() + if isHandled { + s.onProbeDone(isSuccessful) + } // probe if necessary and timing is right if s.state == streamAllocatorStateDeficient { @@ -673,7 +675,10 @@ func (s *StreamAllocator) handleSignalSendProbe(event *Event) { func (s *StreamAllocator) handleSignalProbeClusterDone(event *Event) { info, _ := event.Data.(ProbeClusterInfo) - s.probeController.ProbeClusterDone(info, int64(math.Min(float64(s.committedChannelCapacity), float64(s.channelObserver.GetLowestEstimate())))) + isHandled := s.probeController.ProbeClusterDone(info, int64(math.Min(float64(s.committedChannelCapacity), float64(s.channelObserver.GetLowestEstimate())))) + if isHandled { + s.onProbeDone(true) + } } func (s *StreamAllocator) handleSignalResume(event *Event) { From 4805dec1f01dff8608b6b0d42d1567a21e3f93e5 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sat, 10 Jun 2023 10:54:55 +0530 Subject: [PATCH 216/324] Create channel observer on probe reset. (#1783) On a state change, it was possible an aborted probe was pending finalize. When probe controller is reset, the probe channel observer was not reset. Create a new non-probe channel observer on state change to get a fresh start. Also limit probe finalize wait to 10 seconds max. It is possible that the estimate is very low and we have sent a bunch of probes. Calculating wait based on that could lead to finalize waiting for a long time (could be minutes). --- pkg/sfu/streamallocator/probe_controller.go | 4 ++++ pkg/sfu/streamallocator/streamallocator.go | 2 ++ 2 files changed, 6 insertions(+) diff --git a/pkg/sfu/streamallocator/probe_controller.go b/pkg/sfu/streamallocator/probe_controller.go index 519d31832..6b476b6d5 100644 --- a/pkg/sfu/streamallocator/probe_controller.go +++ b/pkg/sfu/streamallocator/probe_controller.go @@ -12,6 +12,7 @@ const ( ProbeBackoffFactor = 1.5 ProbeWaitMax = 30 * time.Second ProbeSettleWait = 250 + ProbeSettleWaitMax = 10 * time.Second ProbeTrendWait = 2 * time.Second ProbePct = 120 @@ -82,6 +83,9 @@ func (p *ProbeController) ProbeClusterDone(info ProbeClusterInfo, lowestEstimate queueTime = 0.0 } queueWait := time.Duration(queueTime+float64(ProbeSettleWait)) * time.Millisecond + if queueWait > ProbeSettleWaitMax { + queueWait = ProbeSettleWaitMax + } p.probeEndTime = p.lastProbeStartTime.Add(queueWait) p.params.Logger.Infow( "setting probe end time", diff --git a/pkg/sfu/streamallocator/streamallocator.go b/pkg/sfu/streamallocator/streamallocator.go index d13141e7d..76bdddb50 100644 --- a/pkg/sfu/streamallocator/streamallocator.go +++ b/pkg/sfu/streamallocator/streamallocator.go @@ -743,6 +743,8 @@ func (s *StreamAllocator) setState(state streamAllocatorState) { // reset probe to enforce a delay after state change before probing s.probeController.Reset() + // a fresh channel observer after state transition to get clean data + s.channelObserver = s.newChannelObserverNonProbe() } func (s *StreamAllocator) adjustState() { From 3d696ac39fec89f568a5b9bf60706b9725c98f4f Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sat, 10 Jun 2023 11:38:46 +0530 Subject: [PATCH 217/324] Keep next timestamp on switch closer to ref. (#1784) If ref is coming in slow (due to pacing), it is possible that expected is ahead. Pulling next too far towards expected causes warps in a subsequent report. Keep switches closer to ref. --- pkg/sfu/forwarder.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index db2127065..cae2a6858 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -1842,7 +1842,7 @@ func getNextTimestamp(lastTS uint32, refTS uint32, expectedTS uint32, minTS uint switch { case rl && el && er: // lastTS < refTS < expectedTS - nextTS = uint32(float64(refTS) + 0.95*float64(expectedTS-refTS)) + nextTS = uint32(float64(refTS) + 0.05*float64(expectedTS-refTS)) explain = fmt.Sprintf("l < r < e, %d, %d", refTS-lastTS, expectedTS-refTS) case rl && el && !er: // lastTS < expectedTS < refTS nextTS = uint32(float64(expectedTS) + 0.5*float64(refTS-expectedTS)) @@ -1854,7 +1854,7 @@ func getNextTimestamp(lastTS uint32, refTS uint32, expectedTS uint32, minTS uint nextTS = lastTS + 1 explain = fmt.Sprintf("r < e < l, %d, %d", expectedTS-refTS, lastTS-expectedTS) case rl && !el && !er: // expectedTS < lastTS < refTS - nextTS = uint32(float64(lastTS) + 0.5*float64(refTS-lastTS)) + nextTS = uint32(float64(lastTS) + 0.75*float64(refTS-lastTS)) explain = fmt.Sprintf("e < l < r, %d, %d", lastTS-expectedTS, refTS-lastTS) case !rl && !el && !er: // expectedTS < refTS < lastTS nextTS = lastTS + 1 From ba513f521daed59716f9618b62da7d2d84219d0c Mon Sep 17 00:00:00 2001 From: David Zhao Date: Sat, 10 Jun 2023 20:15:41 -0700 Subject: [PATCH 218/324] Update protocol to ensure Room metadata updates are sent immediately (#1787) --- go.mod | 8 ++++---- go.sum | 19 ++++++++----------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index b86a64fcd..c7187f2ec 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230609061558-44a151ad2e1d - github.com/livekit/protocol v1.5.8-0.20230606012216-d8cc3581090d + github.com/livekit/protocol v1.5.8-0.20230611030650-7d128913f3bd github.com/livekit/psrpc v0.3.1 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 @@ -33,7 +33,7 @@ require ( github.com/pion/sdp/v3 v3.0.6 github.com/pion/transport/v2 v2.2.1 github.com/pion/turn/v2 v2.1.0 - github.com/pion/webrtc/v3 v3.2.8 + github.com/pion/webrtc/v3 v3.2.9 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.15.1 github.com/redis/go-redis/v9 v9.0.5 @@ -65,7 +65,7 @@ require ( github.com/google/subcommands v1.2.0 // indirect github.com/google/uuid v1.3.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/hashicorp/go-retryablehttp v0.7.2 // indirect + github.com/hashicorp/go-retryablehttp v0.7.4 // indirect github.com/josharian/native v1.1.0 // indirect github.com/klauspost/compress v1.16.5 // indirect github.com/lithammer/shortuuid/v4 v4.0.0 // indirect @@ -92,7 +92,7 @@ require ( go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.24.0 // indirect golang.org/x/crypto v0.9.0 // indirect - golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect + golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 // indirect golang.org/x/mod v0.8.0 // indirect golang.org/x/net v0.10.0 // indirect golang.org/x/sys v0.8.0 // indirect diff --git a/go.sum b/go.sum index e3600dfea..bd27f0818 100644 --- a/go.sum +++ b/go.sum @@ -83,8 +83,8 @@ github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9n github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= -github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUDDYFRKq/RAd0= -github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= +github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA= +github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru/v2 v2.0.3 h1:kmRrRLlInXvng0SmLxmQpQkpbYAvcXm7NPDrgxJa9mE= @@ -122,8 +122,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20230609061558-44a151ad2e1d h1:7yjO7MpM8e7FQKQ86XF+5yKgDcFIL4DhT7t6NbL8E3Y= github.com/livekit/mediatransportutil v0.0.0-20230609061558-44a151ad2e1d/go.mod h1:MRc0zSOSzXuFt0X218SgabzlaKevkvCckPgBEoHYc34= -github.com/livekit/protocol v1.5.8-0.20230606012216-d8cc3581090d h1:hSTJ80uM/suVDp40MyyCofWVUhtOz3qG3LwFD6/LoRU= -github.com/livekit/protocol v1.5.8-0.20230606012216-d8cc3581090d/go.mod h1:CnELWfDW0SvYy9II+YHsv83zz4gToYTbUdDM/7Fjxy4= +github.com/livekit/protocol v1.5.8-0.20230611030650-7d128913f3bd h1:pHX/MdFniQyvoQR55lOUeV8SrYMnBYmEb6cxjfzzLyg= +github.com/livekit/protocol v1.5.8-0.20230611030650-7d128913f3bd/go.mod h1:Y+jl7rD7u8ZMfIUzGr41DU7G5j+34rtgefTCrD/ApZc= github.com/livekit/psrpc v0.3.1 h1:KfylgJHvoLQcc22t/oflwMOeSnx0c14G7cWsS+9MYS4= github.com/livekit/psrpc v0.3.1/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= @@ -209,16 +209,14 @@ github.com/pion/stun v0.6.0/go.mod h1:HPqcfoeqQn9cuaet7AOmB5e5xkObu9DwBdurwLKO9o github.com/pion/transport v0.14.1 h1:XSM6olwW+o8J4SCmOBb/BpwZypkHeyM0PGFCxNQBr40= github.com/pion/transport v0.14.1/go.mod h1:4tGmbk00NeYA3rUa9+n+dzCCoKkcy3YlYb99Jn2fNnI= github.com/pion/transport/v2 v2.0.0/go.mod h1:HS2MEBJTwD+1ZI2eSXSvHJx/HnzQqRy2/LXxt6eVMHc= -github.com/pion/transport/v2 v2.0.2/go.mod h1:vrz6bUbFr/cjdwbnxq8OdDDzHf7JJfGsIRkxfpZoTA0= github.com/pion/transport/v2 v2.1.0/go.mod h1:AdSw4YBZVDkZm8fpoz+fclXyQwANWmZAlDuQdctTThQ= github.com/pion/transport/v2 v2.2.0/go.mod h1:AdSw4YBZVDkZm8fpoz+fclXyQwANWmZAlDuQdctTThQ= github.com/pion/transport/v2 v2.2.1 h1:7qYnCBlpgSJNYMbLCKuSY9KbQdBFoETvPNETv0y4N7c= github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g= github.com/pion/turn/v2 v2.1.0 h1:5wGHSgGhJhP/RpabkUb/T9PdsAjkGLS6toYz5HNzoSI= github.com/pion/turn/v2 v2.1.0/go.mod h1:yrT5XbXSGX1VFSF31A3c1kCNB5bBZgk/uu5LET162qs= -github.com/pion/udp/v2 v2.0.1/go.mod h1:B7uvTMP00lzWdyMr/1PVZXtV3wpPIxBRd4Wl6AksXn8= -github.com/pion/webrtc/v3 v3.2.8 h1:RmDEz7wjK3k0sAuCSMptfxp095pBYSkSSm5ySiJYIHI= -github.com/pion/webrtc/v3 v3.2.8/go.mod h1:6/7wF1P86AQAw4iTmKIgdzaevaQ8qh9SfrFyypqmN6w= +github.com/pion/webrtc/v3 v3.2.9 h1:U8NSjQDlZZ+Iy/hg42Q/u6mhEVSXYvKrOIZiZwYTfLc= +github.com/pion/webrtc/v3 v3.2.9/go.mod h1:gjQLMZeyN3jXBGdxGmUYCyKjOuYX/c99BDjGqmadq0A= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -286,8 +284,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= -golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc h1:mCRnTeVUjcrhlRmO0VK8a6k6Rrf6TF9htwo2pJVSjIU= -golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc= +golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= @@ -315,7 +313,6 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= From c91889edfdf9fd233bc62c0c1e4c03ecc99d7bcd Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Mon, 12 Jun 2023 15:07:47 +0800 Subject: [PATCH 219/324] Add dependency descriptor stream tracker for svc codecs (#1788) * Add dependency descriptor stream tracker for svc codecs * Solve comments --- pkg/rtc/mediatrackreceiver.go | 16 +- pkg/sfu/buffer/buffer.go | 14 +- pkg/sfu/receiver.go | 9 + pkg/sfu/streamtracker/interfaces.go | 14 + pkg/sfu/streamtracker/streamtracker.go | 2 + pkg/sfu/streamtracker/streamtracker_dd.go | 272 ++++++++++++++++++ .../streamtracker/streamtracker_dd_test.go | 83 ++++++ .../streamtracker_packet_test.go | 24 +- pkg/sfu/streamtrackermanager.go | 75 +++-- 9 files changed, 465 insertions(+), 44 deletions(-) create mode 100644 pkg/sfu/streamtracker/streamtracker_dd.go create mode 100644 pkg/sfu/streamtracker/streamtracker_dd_test.go diff --git a/pkg/rtc/mediatrackreceiver.go b/pkg/rtc/mediatrackreceiver.go index 0ae04a970..6a785ec9e 100644 --- a/pkg/rtc/mediatrackreceiver.go +++ b/pkg/rtc/mediatrackreceiver.go @@ -19,6 +19,7 @@ import ( "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" "github.com/livekit/livekit-server/pkg/telemetry" ) @@ -206,6 +207,15 @@ func (t *MediaTrackReceiver) SetupReceiver(receiver sfu.TrackReceiver, priority } func (t *MediaTrackReceiver) SetPotentialCodecs(codecs []webrtc.RTPCodecParameters, headers []webrtc.RTPHeaderExtensionParameter) { + // The potential codecs have not published yet, so we can't get the actual Extensions, the client/browser uses same extensions + // for all video codecs so we assume they will have same extensions as the primary codec except for the dependency descriptor + // that is munged in svc codec. + headersWithoutDD := make([]webrtc.RTPHeaderExtensionParameter, 0, len(headers)) + for _, h := range headers { + if h.URI != dependencydescriptor.ExtensionUrl { + headersWithoutDD = append(headersWithoutDD, h) + } + } t.lock.Lock() t.potentialCodecs = codecs for i, c := range codecs { @@ -217,8 +227,12 @@ func (t *MediaTrackReceiver) SetPotentialCodecs(codecs []webrtc.RTPCodecParamete } } if !exist { + extHeaders := headers + if !sfu.IsSvcCodec(c.MimeType) { + extHeaders = headersWithoutDD + } t.receivers = append(t.receivers, &simulcastReceiver{ - TrackReceiver: NewDummyReceiver(livekit.TrackID(t.trackInfo.Sid), string(t.PublisherID()), c, headers), + TrackReceiver: NewDummyReceiver(livekit.TrackID(t.trackInfo.Sid), string(t.PublisherID()), c, extHeaders), priority: i, }) } diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go index 4eabb04a1..0234be9f8 100644 --- a/pkg/sfu/buffer/buffer.go +++ b/pkg/sfu/buffer/buffer.go @@ -94,9 +94,8 @@ type Buffer struct { logger logger.Logger // dependency descriptor - ddExt uint8 - ddParser *DependencyDescriptorParser - maxLayerChangedCB func(int32, int32) + ddExt uint8 + ddParser *DependencyDescriptorParser paused bool frameRateCalculator [DefaultMaxLayerSpatial + 1]FrameRateCalculator @@ -175,9 +174,6 @@ func (b *Buffer) Bind(params webrtc.RTPParameters, codec webrtc.RTPCodecCapabili b.frameRateCalculator[i] = frc.GetFrameRateCalculatorForSpatial(int32(i)) } b.ddParser = NewDependencyDescriptorParser(b.ddExt, b.logger, func(spatial, temporal int32) { - if b.maxLayerChangedCB != nil { - b.maxLayerChangedCB(spatial, temporal) - } frc.SetMaxLayer(spatial, temporal) }) @@ -779,12 +775,6 @@ func (b *Buffer) GetAudioLevel() (float64, bool) { return b.audioLevel.GetLevel() } -// DD-TODO : now we rely on stream tracker for layer change, dependency still -// work for that too. Do we keep it unchanged or use both methods? -func (b *Buffer) OnMaxLayerChanged(fn func(int32, int32)) { - b.maxLayerChangedCB = fn -} - func (b *Buffer) OnFpsChanged(f func()) { b.Lock() b.onFpsChanged = f diff --git a/pkg/sfu/receiver.go b/pkg/sfu/receiver.go index 07c072501..80ef3f007 100644 --- a/pkg/sfu/receiver.go +++ b/pkg/sfu/receiver.go @@ -20,6 +20,7 @@ import ( "github.com/livekit/livekit-server/pkg/sfu/audio" "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/livekit-server/pkg/sfu/connectionquality" + dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" ) var ( @@ -216,6 +217,13 @@ func NewWebRTCReceiver( }) w.connectionStats.Start(w.trackInfo, time.Now()) + for _, ext := range receiver.GetParameters().HeaderExtensions { + if ext.URI == dd.ExtensionUrl { + w.streamTrackerManager.AddDependencyDescriptorTrackers() + break + } + } + return w } @@ -644,6 +652,7 @@ func (w *WebRTCReceiver) forwardRTP(layer int32) { len(pkt.Packet.Payload), pkt.Packet.Marker, pkt.Packet.Timestamp, + pkt.DependencyDescriptor, ) } diff --git a/pkg/sfu/streamtracker/interfaces.go b/pkg/sfu/streamtracker/interfaces.go index 3837f8470..934032f68 100644 --- a/pkg/sfu/streamtracker/interfaces.go +++ b/pkg/sfu/streamtracker/interfaces.go @@ -3,6 +3,8 @@ package streamtracker import ( "fmt" "time" + + "github.com/livekit/livekit-server/pkg/sfu/buffer" ) // ------------------------------------------------------------ @@ -40,3 +42,15 @@ type StreamTrackerImpl interface { Observe(hasMarker bool, ts uint32) StreamStatusChange CheckStatus() StreamStatusChange } + +type StreamTrackerWorker interface { + Start() + Stop() + Reset() + OnStatusChanged(f func(status StreamStatus)) + OnBitrateAvailable(f func()) + Status() StreamStatus + BitrateTemporalCumulative() []int64 + SetPaused(paused bool) + Observe(temporalLayer int32, pktSize int, payloadSize int, hasMarker bool, ts uint32, dd *buffer.DependencyDescriptorWithDecodeTarget) +} diff --git a/pkg/sfu/streamtracker/streamtracker.go b/pkg/sfu/streamtracker/streamtracker.go index b1d16dae6..a1b645fb2 100644 --- a/pkg/sfu/streamtracker/streamtracker.go +++ b/pkg/sfu/streamtracker/streamtracker.go @@ -7,6 +7,7 @@ import ( "go.uber.org/atomic" + "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/protocol/logger" ) @@ -175,6 +176,7 @@ func (s *StreamTracker) Observe( payloadSize int, hasMarker bool, ts uint32, + _ *buffer.DependencyDescriptorWithDecodeTarget, ) { s.lock.Lock() diff --git a/pkg/sfu/streamtracker/streamtracker_dd.go b/pkg/sfu/streamtracker/streamtracker_dd.go new file mode 100644 index 000000000..b6daee387 --- /dev/null +++ b/pkg/sfu/streamtracker/streamtracker_dd.go @@ -0,0 +1,272 @@ +package streamtracker + +import ( + "sync" + "time" + + "go.uber.org/atomic" + + "github.com/livekit/livekit-server/pkg/sfu/buffer" + dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" +) + +type StreamTrackerDependencyDescriptor struct { + lock sync.RWMutex + paused bool + generation atomic.Uint32 + params StreamTrackerParams + maxSpatialLayer int32 + maxTemporalLayer int32 + + onStatusChanged [buffer.DefaultMaxLayerSpatial + 1]func(status StreamStatus) + onBitrateAvailable [buffer.DefaultMaxLayerSpatial + 1]func() + + lastBitrateReport time.Time + bytesForBitrate [buffer.DefaultMaxLayerSpatial + 1][buffer.DefaultMaxLayerTemporal + 1]int64 + bitrate [buffer.DefaultMaxLayerSpatial + 1][buffer.DefaultMaxLayerTemporal + 1]int64 + + isStopped bool +} + +func NewStreamTrackerDependencyDescriptor(params StreamTrackerParams) *StreamTrackerDependencyDescriptor { + return &StreamTrackerDependencyDescriptor{ + params: params, + maxSpatialLayer: buffer.InvalidLayerSpatial, + maxTemporalLayer: buffer.InvalidLayerTemporal, + } +} +func (s *StreamTrackerDependencyDescriptor) Start() { +} + +func (s *StreamTrackerDependencyDescriptor) Stop() { + s.lock.Lock() + defer s.lock.Unlock() + + if s.isStopped { + return + } + s.isStopped = true + + // bump generation to trigger exit of worker + s.generation.Inc() +} + +func (s *StreamTrackerDependencyDescriptor) OnStatusChanged(layer int32, f func(status StreamStatus)) { + s.lock.Lock() + s.onStatusChanged[layer] = f + s.lock.Unlock() +} + +func (s *StreamTrackerDependencyDescriptor) OnBitrateAvailable(layer int32, f func()) { + s.lock.Lock() + s.onBitrateAvailable[layer] = f + s.lock.Unlock() +} + +func (s *StreamTrackerDependencyDescriptor) Status(layer int32) StreamStatus { + s.lock.RLock() + defer s.lock.RUnlock() + + if layer > s.maxSpatialLayer { + return StreamStatusStopped + } + + return StreamStatusActive +} + +func (s *StreamTrackerDependencyDescriptor) BitrateTemporalCumulative(layer int32) []int64 { + s.lock.RLock() + defer s.lock.RUnlock() + + if layer > s.maxSpatialLayer { + brs := make([]int64, len(s.bitrate[0])) + return brs + } + + return s.bitrate[layer][:] +} + +func (s *StreamTrackerDependencyDescriptor) Reset() { +} + +func (s *StreamTrackerDependencyDescriptor) resetLocked() { + // bump generation to trigger exit of current worker + s.generation.Inc() + + for i := 0; i < len(s.bytesForBitrate); i++ { + for j := 0; j < len(s.bytesForBitrate[i]); j++ { + s.bytesForBitrate[i][j] = 0 + } + } + for i := 0; i < len(s.bitrate); i++ { + for j := 0; j < len(s.bitrate[i]); j++ { + s.bitrate[i][j] = 0 + } + } +} + +func (s *StreamTrackerDependencyDescriptor) SetPaused(paused bool) { + s.lock.Lock() + if s.paused == paused { + s.lock.Unlock() + return + } + s.paused = paused + if !paused { + s.resetLocked() + } else { + s.lastBitrateReport = time.Now() + go s.worker(s.generation.Inc()) + + } + s.lock.Unlock() + +} + +func (s *StreamTrackerDependencyDescriptor) Observe(temporalLayer int32, pktSize int, payloadSize int, hasMarker bool, ts uint32, ddVal *buffer.DependencyDescriptorWithDecodeTarget) { + s.lock.Lock() + + if s.isStopped || s.paused || payloadSize == 0 || ddVal == nil { + s.lock.Unlock() + return + } + + var notifyFns []func(status StreamStatus) + var notifyStatus StreamStatus + if mask := ddVal.Descriptor.ActiveDecodeTargetsBitmask; mask != nil { + var maxSpatial, maxTemporal int32 + for _, dt := range ddVal.DecodeTargets { + if *mask&(1< buffer.DefaultMaxLayerSpatial { + maxSpatial = buffer.DefaultMaxLayerSpatial + s.params.Logger.Warnw("max spatial layer exceeded", nil, "maxSpatial", maxSpatial) + } + if maxTemporal > buffer.DefaultMaxLayerTemporal { + maxTemporal = buffer.DefaultMaxLayerTemporal + s.params.Logger.Warnw("max temporal layer exceeded", nil, "maxTemporal", maxTemporal) + } + + s.params.Logger.Debugw("max layer changed", "maxSpatial", maxSpatial, "maxTemporal", maxTemporal) + oldMaxSpatial := s.maxSpatialLayer + s.maxSpatialLayer, s.maxTemporalLayer = maxSpatial, maxTemporal + if oldMaxSpatial == -1 { + s.lastBitrateReport = time.Now() + go s.worker(s.generation.Inc()) + } + + if oldMaxSpatial > s.maxSpatialLayer { + notifyStatus = StreamStatusStopped + for i := s.maxSpatialLayer + 1; i <= oldMaxSpatial; i++ { + notifyFns = append(notifyFns, s.onStatusChanged[i]) + } + } else if oldMaxSpatial < s.maxSpatialLayer { + notifyStatus = StreamStatusActive + for i := oldMaxSpatial + 1; i <= s.maxSpatialLayer; i++ { + notifyFns = append(notifyFns, s.onStatusChanged[i]) + } + } + + } + + dtis := ddVal.Descriptor.FrameDependencies.DecodeTargetIndications + + for _, dt := range ddVal.DecodeTargets { + // we are not dropping discardable frames now, so only ingore not present frames + if dtis[dt.Target] == dd.DecodeTargetNotPresent { + continue + } + + s.bytesForBitrate[dt.Layer.Spatial][dt.Layer.Temporal] += int64(pktSize) + } + + s.lock.Unlock() + + for _, fn := range notifyFns { + if fn != nil { + fn(notifyStatus) + } + } +} + +func (s *StreamTrackerDependencyDescriptor) worker(generation uint32) { + tickerBitrate := time.NewTicker(s.params.BitrateReportInterval) + defer tickerBitrate.Stop() + + for { + <-tickerBitrate.C + if generation != s.generation.Load() { + return + } + s.bitrateReport() + } +} + +func (s *StreamTrackerDependencyDescriptor) bitrateReport() { + // run this even if paused to drain out bitrate if there are no packets coming in + s.lock.Lock() + now := time.Now() + diff := now.Sub(s.lastBitrateReport) + s.lastBitrateReport = now + + var availableChangedFns []func() + for spatial := 0; spatial < len(s.bytesForBitrate); spatial++ { + bytesForBitrate := s.bytesForBitrate[spatial][:] + bitrateAvailabilityChanged := false + bitrates := s.bitrate[spatial][:] + for i := 0; i < len(bytesForBitrate); i++ { + bitrate := int64(float64(bytesForBitrate[i]*8) / diff.Seconds()) + if (bitrates[i] == 0 && bitrate > 0) || (bitrates[i] > 0 && bitrate == 0) { + bitrateAvailabilityChanged = true + } + bitrates[i] = bitrate + bytesForBitrate[i] = 0 + } + + if bitrateAvailabilityChanged && s.onBitrateAvailable[spatial] != nil { + availableChangedFns = append(availableChangedFns, s.onBitrateAvailable[spatial]) + } + } + s.lock.Unlock() + + for _, fn := range availableChangedFns { + fn() + } +} + +func (s *StreamTrackerDependencyDescriptor) LayeredTracker(layer int32) *StreamTrackerDependencyDescriptorLayered { + return &StreamTrackerDependencyDescriptorLayered{ + StreamTrackerDependencyDescriptor: s, + layer: layer, + } +} + +// ---------------------------- +// Layered wrapper for StreamTrackerWorker +type StreamTrackerDependencyDescriptorLayered struct { + *StreamTrackerDependencyDescriptor + layer int32 +} + +func (s *StreamTrackerDependencyDescriptorLayered) OnStatusChanged(f func(status StreamStatus)) { + s.StreamTrackerDependencyDescriptor.OnStatusChanged(s.layer, f) +} + +func (s *StreamTrackerDependencyDescriptorLayered) OnBitrateAvailable(f func()) { + s.StreamTrackerDependencyDescriptor.OnBitrateAvailable(s.layer, f) +} + +func (s *StreamTrackerDependencyDescriptorLayered) Status() StreamStatus { + return s.StreamTrackerDependencyDescriptor.Status(s.layer) +} + +func (s *StreamTrackerDependencyDescriptorLayered) BitrateTemporalCumulative() []int64 { + return s.StreamTrackerDependencyDescriptor.BitrateTemporalCumulative(s.layer) +} diff --git a/pkg/sfu/streamtracker/streamtracker_dd_test.go b/pkg/sfu/streamtracker/streamtracker_dd_test.go new file mode 100644 index 000000000..6fdb6916d --- /dev/null +++ b/pkg/sfu/streamtracker/streamtracker_dd_test.go @@ -0,0 +1,83 @@ +package streamtracker + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/livekit/livekit-server/pkg/sfu/buffer" + dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + "github.com/livekit/protocol/logger" +) + +func createDescriptorDependencyForTargets(maxSpatial, maxTemporal int) *buffer.DependencyDescriptorWithDecodeTarget { + var targets []buffer.DependencyDescriptorDecodeTarget + var mask uint32 + for i := 0; i <= maxSpatial; i++ { + for j := 0; j <= maxTemporal; j++ { + targets = append(targets, buffer.DependencyDescriptorDecodeTarget{Target: len(targets), Layer: buffer.VideoLayer{Spatial: int32(i), Temporal: int32(j)}}) + mask |= 1 << uint32(len(targets)-1) + } + } + + dtis := make([]dd.DecodeTargetIndication, len(targets)) + for _, t := range targets { + dtis[t.Target] = dd.DecodeTargetRequired + } + + return &buffer.DependencyDescriptorWithDecodeTarget{ + Descriptor: &dd.DependencyDescriptor{ + ActiveDecodeTargetsBitmask: &mask, + FrameDependencies: &dd.FrameDependencyTemplate{ + DecodeTargetIndications: dtis, + }, + }, + DecodeTargets: targets, + } +} + +func checkStatues(t *testing.T, statuses []StreamStatus, expected StreamStatus, maxSpatial int) { + for i := 0; i <= maxSpatial; i++ { + require.Equal(t, expected, statuses[i]) + } + + for i := maxSpatial + 1; i < len(statuses); i++ { + require.NotEqual(t, expected, statuses[i]) + } +} + +func TestStreamTrackerDD(t *testing.T) { + ddTracker := NewStreamTrackerDependencyDescriptor(StreamTrackerParams{ + BitrateReportInterval: 1 * time.Second, + Logger: logger.GetLogger(), + }) + layeredTrackers := make([]StreamTrackerWorker, buffer.DefaultMaxLayerSpatial+1) + statuses := make([]StreamStatus, buffer.DefaultMaxLayerSpatial+1) + for i := 0; i <= int(buffer.DefaultMaxLayerSpatial); i++ { + layeredTrack := ddTracker.LayeredTracker(int32(i)) + layer := i + layeredTrack.OnStatusChanged(func(status StreamStatus) { + statuses[layer] = status + }) + layeredTrack.Start() + layeredTrackers[i] = layeredTrack + } + defer ddTracker.Stop() + + // no active layers + ddTracker.Observe(0, 1000, 1000, false, 0, nil) + checkStatues(t, statuses, StreamStatusActive, int(buffer.InvalidLayerSpatial)) + + // layer seen [0,1] + ddTracker.Observe(0, 1000, 1000, false, 0, createDescriptorDependencyForTargets(1, 1)) + checkStatues(t, statuses, StreamStatusActive, 1) + + // layer seen [0,1,2] + ddTracker.Observe(0, 1000, 1000, false, 0, createDescriptorDependencyForTargets(2, 1)) + checkStatues(t, statuses, StreamStatusActive, 2) + + // layer 2 gone, layer seen [0,1] + ddTracker.Observe(0, 1000, 1000, false, 0, createDescriptorDependencyForTargets(1, 1)) + checkStatues(t, statuses, StreamStatusActive, 1) +} diff --git a/pkg/sfu/streamtracker/streamtracker_packet_test.go b/pkg/sfu/streamtracker/streamtracker_packet_test.go index 2aee13ecb..a7beb2d3c 100644 --- a/pkg/sfu/streamtracker/streamtracker_packet_test.go +++ b/pkg/sfu/streamtracker/streamtracker_packet_test.go @@ -42,7 +42,7 @@ func TestStreamTracker(t *testing.T) { require.Equal(t, StreamStatusStopped, tracker.Status()) // observe first packet - tracker.Observe(0, 20, 10, false, 0) + tracker.Observe(0, 20, 10, false, 0, nil) testutils.WithTimeout(t, func() string { if callbackCalled.Load() { @@ -73,7 +73,7 @@ func TestStreamTracker(t *testing.T) { callbackStatusMu.Unlock() }) - tracker.Observe(0, 20, 10, false, 0) + tracker.Observe(0, 20, 10, false, 0, nil) testutils.WithTimeout(t, func() string { callbackStatusMu.RLock() defer callbackStatusMu.RUnlock() @@ -110,7 +110,7 @@ func TestStreamTracker(t *testing.T) { tracker.Start() require.Equal(t, StreamStatusStopped, tracker.Status()) - tracker.Observe(0, 20, 10, false, 0) + tracker.Observe(0, 20, 10, false, 0, nil) testutils.WithTimeout(t, func() string { if tracker.Status() == StreamStatusActive { return "" @@ -121,11 +121,11 @@ func TestStreamTracker(t *testing.T) { tracker.setStatusLocked(StreamStatusStopped) - tracker.Observe(0, 20, 10, false, 0) + tracker.Observe(0, 20, 10, false, 0, nil) tracker.updateStatus() require.Equal(t, StreamStatusStopped, tracker.Status()) - tracker.Observe(0, 20, 10, false, 0) + tracker.Observe(0, 20, 10, false, 0, nil) tracker.updateStatus() require.Equal(t, StreamStatusActive, tracker.Status()) @@ -135,7 +135,7 @@ func TestStreamTracker(t *testing.T) { t.Run("changes to inactive when paused", func(t *testing.T) { tracker := newStreamTrackerPacket(5, 60, 500*time.Millisecond) tracker.Start() - tracker.Observe(0, 20, 10, false, 0) + tracker.Observe(0, 20, 10, false, 0, nil) testutils.WithTimeout(t, func() string { if tracker.Status() == StreamStatusActive { return "" @@ -161,7 +161,7 @@ func TestStreamTracker(t *testing.T) { require.Equal(t, StreamStatusStopped, tracker.Status()) // observe first packet - tracker.Observe(0, 20, 10, false, 0) + tracker.Observe(0, 20, 10, false, 0, nil) testutils.WithTimeout(t, func() string { if callbackCalled.Load() == 1 { @@ -175,10 +175,10 @@ func TestStreamTracker(t *testing.T) { require.Equal(t, uint32(1), callbackCalled.Load()) // observe a few more - tracker.Observe(0, 20, 10, false, 0) - tracker.Observe(0, 20, 10, false, 0) - tracker.Observe(0, 20, 10, false, 0) - tracker.Observe(0, 20, 10, false, 0) + tracker.Observe(0, 20, 10, false, 0, nil) + tracker.Observe(0, 20, 10, false, 0, nil) + tracker.Observe(0, 20, 10, false, 0, nil) + tracker.Observe(0, 20, 10, false, 0, nil) tracker.updateStatus() // should still be active @@ -191,7 +191,7 @@ func TestStreamTracker(t *testing.T) { require.Equal(t, uint32(2), callbackCalled.Load()) // first packet after reset - tracker.Observe(0, 20, 10, false, 0) + tracker.Observe(0, 20, 10, false, 0, nil) testutils.WithTimeout(t, func() string { if callbackCalled.Load() == 3 { diff --git a/pkg/sfu/streamtrackermanager.go b/pkg/sfu/streamtrackermanager.go index cdb42f4ab..8c0784d40 100644 --- a/pkg/sfu/streamtrackermanager.go +++ b/pkg/sfu/streamtrackermanager.go @@ -41,7 +41,8 @@ type StreamTrackerManager struct { maxPublishedLayer int32 maxTemporalLayerSeen int32 - trackers [buffer.DefaultMaxLayerSpatial + 1]*streamtracker.StreamTracker + ddTracker *streamtracker.StreamTrackerDependencyDescriptor + trackers [buffer.DefaultMaxLayerSpatial + 1]streamtracker.StreamTrackerWorker availableLayers []int32 maxExpectedLayer int32 @@ -133,28 +134,58 @@ func (s *StreamTrackerManager) createStreamTrackerFrame(layer int32) streamtrack return streamtracker.NewStreamTrackerFrame(params) } -func (s *StreamTrackerManager) AddTracker(layer int32) *streamtracker.StreamTracker { +func (s *StreamTrackerManager) AddDependencyDescriptorTrackers() { + bitrateInterval, ok := s.trackerConfig.BitrateReportInterval[0] + if !ok { + return + } + s.lock.Lock() + var addAllTrackers bool + if s.ddTracker == nil { + s.ddTracker = streamtracker.NewStreamTrackerDependencyDescriptor(streamtracker.StreamTrackerParams{ + BitrateReportInterval: bitrateInterval, + Logger: s.logger.WithValues("layer", 0), + }) + addAllTrackers = true + } + s.lock.Unlock() + if addAllTrackers { + for i := 0; i <= int(buffer.DefaultMaxLayerSpatial); i++ { + s.AddTracker(int32(i)) + } + } +} + +func (s *StreamTrackerManager) AddTracker(layer int32) streamtracker.StreamTrackerWorker { bitrateInterval, ok := s.trackerConfig.BitrateReportInterval[layer] if !ok { return nil } - var trackerImpl streamtracker.StreamTrackerImpl - switch s.trackerConfig.StreamTrackerType { - case config.StreamTrackerTypePacket: - trackerImpl = s.createStreamTrackerPacket(layer) - case config.StreamTrackerTypeFrame: - trackerImpl = s.createStreamTrackerFrame(layer) - } - if trackerImpl == nil { - return nil + var tracker streamtracker.StreamTrackerWorker + s.lock.Lock() + if s.ddTracker != nil { + tracker = s.ddTracker.LayeredTracker(layer) } + s.lock.Unlock() + if tracker == nil { + var trackerImpl streamtracker.StreamTrackerImpl + switch s.trackerConfig.StreamTrackerType { + case config.StreamTrackerTypePacket: + trackerImpl = s.createStreamTrackerPacket(layer) + case config.StreamTrackerTypeFrame: + trackerImpl = s.createStreamTrackerFrame(layer) + } + if trackerImpl == nil { + return nil + } - tracker := streamtracker.NewStreamTracker(streamtracker.StreamTrackerParams{ - StreamTrackerImpl: trackerImpl, - BitrateReportInterval: bitrateInterval, - Logger: s.logger.WithValues("layer", layer), - }) + tracker = streamtracker.NewStreamTracker(streamtracker.StreamTrackerParams{ + StreamTrackerImpl: trackerImpl, + BitrateReportInterval: bitrateInterval, + Logger: s.logger.WithValues("layer", layer), + }) + } s.logger.Debugw("StreamTrackerManager add track", "layer", layer) tracker.OnStatusChanged(func(status streamtracker.StreamStatus) { @@ -213,6 +244,8 @@ func (s *StreamTrackerManager) RemoveAllTrackers() { s.availableLayers = make([]int32, 0) s.maxExpectedLayerFromTrackInfo() s.paused = false + ddTracker := s.ddTracker + s.ddTracker = nil s.lock.Unlock() for _, tracker := range trackers { @@ -220,9 +253,12 @@ func (s *StreamTrackerManager) RemoveAllTrackers() { tracker.Stop() } } + if ddTracker != nil { + ddTracker.Stop() + } } -func (s *StreamTrackerManager) GetTracker(layer int32) *streamtracker.StreamTracker { +func (s *StreamTrackerManager) GetTracker(layer int32) streamtracker.StreamTrackerWorker { s.lock.RLock() defer s.lock.RUnlock() @@ -270,7 +306,7 @@ func (s *StreamTrackerManager) SetMaxExpectedSpatialLayer(layer int32) int32 { // But, those conditions should be rare. In those cases, the restart will // take longer. // - var trackersToReset []*streamtracker.StreamTracker + var trackersToReset []streamtracker.StreamTrackerWorker for l := s.maxExpectedLayer + 1; l <= layer; l++ { if s.hasSpatialLayerLocked(l) { continue @@ -367,7 +403,8 @@ func (s *StreamTrackerManager) getLayeredBitrateLocked() ([]int32, Bitrates) { } } - if s.isSVC { + // accumulate bitrates for SVC streams without dependency descriptor + if s.isSVC && s.ddTracker == nil { for i := len(br) - 1; i >= 1; i-- { for j := len(br[i]) - 1; j >= 0; j-- { if br[i][j] != 0 { From 9809b8bc3a04d194e74d11677fb1a5c6f9d9f7dc Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 12 Jun 2023 13:01:02 +0530 Subject: [PATCH 220/324] Use nack queue params. (#1789) * Use nack queue params. * fix test --- go.mod | 2 +- go.sum | 4 ++-- pkg/sfu/buffer/buffer.go | 2 +- pkg/sfu/buffer/buffer_test.go | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index c7187f2ec..5f4e293ff 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.3 github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 - github.com/livekit/mediatransportutil v0.0.0-20230609061558-44a151ad2e1d + github.com/livekit/mediatransportutil v0.0.0-20230612070454-d5299b956135 github.com/livekit/protocol v1.5.8-0.20230611030650-7d128913f3bd github.com/livekit/psrpc v0.3.1 github.com/mackerelio/go-osstat v0.2.4 diff --git a/go.sum b/go.sum index bd27f0818..38db27e89 100644 --- a/go.sum +++ b/go.sum @@ -120,8 +120,8 @@ github.com/lithammer/shortuuid/v4 v4.0.0 h1:QRbbVkfgNippHOS8PXDkti4NaWeyYfcBTHtw github.com/lithammer/shortuuid/v4 v4.0.0/go.mod h1:Zs8puNcrvf2rV9rTH51ZLLcj7ZXqQI3lv67aw4KiB1Y= github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkDaKb5iXdynYrzB84ErPPO4LbRASk58= github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= -github.com/livekit/mediatransportutil v0.0.0-20230609061558-44a151ad2e1d h1:7yjO7MpM8e7FQKQ86XF+5yKgDcFIL4DhT7t6NbL8E3Y= -github.com/livekit/mediatransportutil v0.0.0-20230609061558-44a151ad2e1d/go.mod h1:MRc0zSOSzXuFt0X218SgabzlaKevkvCckPgBEoHYc34= +github.com/livekit/mediatransportutil v0.0.0-20230612070454-d5299b956135 h1:lWYbsondvqG69czxoACDwaJ/BoyD57BahCo70ZH+m4U= +github.com/livekit/mediatransportutil v0.0.0-20230612070454-d5299b956135/go.mod h1:MRc0zSOSzXuFt0X218SgabzlaKevkvCckPgBEoHYc34= github.com/livekit/protocol v1.5.8-0.20230611030650-7d128913f3bd h1:pHX/MdFniQyvoQR55lOUeV8SrYMnBYmEb6cxjfzzLyg= github.com/livekit/protocol v1.5.8-0.20230611030650-7d128913f3bd/go.mod h1:Y+jl7rD7u8ZMfIUzGr41DU7G5j+34rtgefTCrD/ApZc= github.com/livekit/psrpc v0.3.1 h1:KfylgJHvoLQcc22t/oflwMOeSnx0c14G7cWsS+9MYS4= diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go index 0234be9f8..bc5b45e20 100644 --- a/pkg/sfu/buffer/buffer.go +++ b/pkg/sfu/buffer/buffer.go @@ -230,7 +230,7 @@ func (b *Buffer) Bind(params webrtc.RTPParameters, codec webrtc.RTPCodecCapabili return } b.logger.Debugw("Setting feedback", "type", webrtc.TypeRTCPFBNACK) - b.nacker = nack.NewNACKQueue() + b.nacker = nack.NewNACKQueue(nack.NackQueueParamsDefault) } } diff --git a/pkg/sfu/buffer/buffer_test.go b/pkg/sfu/buffer/buffer_test.go index 68d8de3a9..1e22bd991 100644 --- a/pkg/sfu/buffer/buffer_test.go +++ b/pkg/sfu/buffer/buffer_test.go @@ -69,7 +69,7 @@ func TestNack(t *testing.T) { continue } if i < 14 { - time.Sleep(time.Duration(float64(rtt)*math.Pow(nack.BackoffFactor, float64(i))+10) * time.Millisecond) + time.Sleep(time.Duration(float64(rtt)*math.Pow(nack.NackQueueParamsDefault.BackoffFactor, float64(i))+10) * time.Millisecond) } else { time.Sleep(500 * time.Millisecond) // even a long wait should not exceed max retries } @@ -128,7 +128,7 @@ func TestNack(t *testing.T) { continue } if i < 14 { - time.Sleep(time.Duration(float64(rtt)*math.Pow(nack.BackoffFactor, float64(i))+10) * time.Millisecond) + time.Sleep(time.Duration(float64(rtt)*math.Pow(nack.NackQueueParamsDefault.BackoffFactor, float64(i))+10) * time.Millisecond) } else { time.Sleep(500 * time.Millisecond) // even a long wait should not exceed max retries } From afa773374840680e173a0b1a6776f25ad954fb38 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 12 Jun 2023 17:30:56 +0530 Subject: [PATCH 221/324] Promote switch logs to Infow. (#1790) --- pkg/sfu/forwarder.go | 29 +++++++++++++++++++++++++++-- pkg/sfu/streamtrackermanager.go | 2 +- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index cae2a6858..291d7e338 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -1461,6 +1461,13 @@ func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer i f.referenceLayerSpatial = layer f.rtpMunger.SetLastSnTs(extPkt) f.codecMunger.SetLast(extPkt) + f.logger.Infow( + "starting forwarding", + "sequenceNumber", extPkt.Packet.SequenceNumber, + "timestamp", extPkt.Packet.Timestamp, + "layer", layer, + "referenceLayerSpatial", f.referenceLayerSpatial, + ) } else { if f.referenceLayerSpatial == buffer.InvalidLayerSpatial { // on a resume, reference layer may not be set, so only set when it is invalid @@ -1483,7 +1490,7 @@ func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer i lastTS := f.rtpMunger.GetLast().LastTS refTS := lastTS expectedTS := lastTS - minTS := uint64(lastTS) + minTS := ^uint64(0) switchingAt := time.Now() if f.getReferenceLayerRTPTimestamp != nil { ts, err := f.getReferenceLayerRTPTimestamp(extPkt.Packet.Timestamp, layer, f.referenceLayerSpatial) @@ -1502,6 +1509,15 @@ func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer i timeSinceFirst := time.Since(f.preStartTime) rtpDiff = uint32(timeSinceFirst.Nanoseconds() * int64(f.codec.ClockRate) / 1e9) f.refTSOffset = f.firstTS + rtpDiff - refTS + f.logger.Infow( + "calculating refTSOffset", + "preStartTime", f.preStartTime.String(), + "firstTS", f.firstTS, + "timeSinceFirst", timeSinceFirst, + "rtpDiff", rtpDiff, + "refTS", refTS, + "refTSOffset", f.refTSOffset, + ) } expectedTS += rtpDiff } @@ -1511,8 +1527,11 @@ func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer i f.logger.Infow( "next timestamp on switch", "switchingAt", switchingAt.String(), + "layer", layer, "lastTS", lastTS, "refTS", refTS, + "refTSOffset", f.refTSOffset, + "referenceLayerSpatial", f.referenceLayerSpatial, "expectedTS", expectedTS, "minTS", minTS, "nextTS", nextTS, @@ -1647,6 +1666,12 @@ func (f *Forwarder) maybeStart() { f.rtpMunger.SetLastSnTs(extPkt) f.firstTS = extPkt.Packet.Timestamp + f.logger.Infow( + "starting with dummy forwarding", + "sequenceNumber", extPkt.Packet.SequenceNumber, + "timestamp", extPkt.Packet.Timestamp, + "preStartTime", f.preStartTime, + ) } func (f *Forwarder) GetSnTsForPadding(num int, forceMarker bool) ([]SnTs, error) { @@ -1679,7 +1704,7 @@ func (f *Forwarder) GetSnTsForBlankFrames(frameRate uint32, numPackets int) ([]S lastTS := f.rtpMunger.GetLast().LastTS expectedTS := lastTS - minTS := uint64(lastTS) + minTS := ^uint64(0) if f.getExpectedRTPTimestamp != nil { ts, min, err := f.getExpectedRTPTimestamp(time.Now()) if err == nil { diff --git a/pkg/sfu/streamtrackermanager.go b/pkg/sfu/streamtrackermanager.go index 8c0784d40..530fef826 100644 --- a/pkg/sfu/streamtrackermanager.go +++ b/pkg/sfu/streamtrackermanager.go @@ -577,7 +577,7 @@ func (s *StreamTrackerManager) GetReferenceLayerRTPTimestamp(ts uint32, layer in ntpDiff := srRef.NTPTimestamp.Time().Sub(srLayer.NTPTimestamp.Time()) rtpDiff := ntpDiff.Nanoseconds() * int64(s.clockRate) / 1e9 normalizedTS := srLayer.RTPTimestamp + uint32(rtpDiff) - s.logger.Debugw( + s.logger.Infow( "getting reference timestamp", "layer", layer, "referenceLayer", referenceLayer, From 2dd4e1365be64ff9ba44e63291ec95d4c582036f Mon Sep 17 00:00:00 2001 From: shishirng Date: Wed, 14 Jun 2023 18:56:07 -0400 Subject: [PATCH 222/324] Send EgressUpdated event (#1792) Signed-off-by: shishir gowda --- pkg/telemetry/events.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/telemetry/events.go b/pkg/telemetry/events.go index 1e831b9b5..26a215ba6 100644 --- a/pkg/telemetry/events.go +++ b/pkg/telemetry/events.go @@ -415,6 +415,7 @@ func (t *telemetryService) EgressUpdated(ctx context.Context, info *livekit.Egre Event: webhook.EventEgressUpdated, EgressInfo: info, }) + t.SendEvent(ctx, newEgressEvent(livekit.AnalyticsEventType_EGRESS_UPDATED, info)) }) } From 41bb1d8745bebf8ed3dd87235853e2d32c0553d9 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 14 Jun 2023 18:07:25 -0700 Subject: [PATCH 223/324] Update go deps (#1791) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 5f4e293ff..e002e600c 100644 --- a/go.mod +++ b/go.mod @@ -42,10 +42,10 @@ require ( github.com/thoas/go-funk v0.9.3 github.com/twitchtv/twirp v8.1.3+incompatible github.com/ua-parser/uap-go v0.0.0-20211112212520-00c877edfe0f - github.com/urfave/cli/v2 v2.25.5 + github.com/urfave/cli/v2 v2.25.6 github.com/urfave/negroni/v3 v3.0.0 go.uber.org/atomic v1.11.0 - golang.org/x/sync v0.2.0 + golang.org/x/sync v0.3.0 google.golang.org/protobuf v1.30.0 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go.sum b/go.sum index 38db27e89..575b10354 100644 --- a/go.sum +++ b/go.sum @@ -260,8 +260,8 @@ github.com/twitchtv/twirp v8.1.3+incompatible h1:+F4TdErPgSUbMZMwp13Q/KgDVuI7HJX github.com/twitchtv/twirp v8.1.3+incompatible/go.mod h1:RRJoFSAmTEh2weEqWtpPE3vFK5YBhA6bqp2l1kfCC5A= github.com/ua-parser/uap-go v0.0.0-20211112212520-00c877edfe0f h1:A+MmlgpvrHLeUP8dkBVn4Pnf5Bp5Yk2OALm7SEJLLE8= github.com/ua-parser/uap-go v0.0.0-20211112212520-00c877edfe0f/go.mod h1:OBcG9bn7sHtXgarhUEb3OfCnNsgtGnkVf41ilSZ3K3E= -github.com/urfave/cli/v2 v2.25.5 h1:d0NIAyhh5shGscroL7ek/Ya9QYQE0KNabJgiUinIQkc= -github.com/urfave/cli/v2 v2.25.5/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= +github.com/urfave/cli/v2 v2.25.6 h1:yuSkgDSZfH3L1CjF2/5fNNg2KbM47pY2EvjBq4ESQnU= +github.com/urfave/cli/v2 v2.25.6/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/urfave/negroni/v3 v3.0.0 h1:Vo8CeZfu1lFR9gW8GnAb6dOGCJyijfil9j/jKKc/JhU= github.com/urfave/negroni/v3 v3.0.0/go.mod h1:jWvnX03kcSjDBl/ShB0iHvx5uOs7mAzZXW+JvJ5XYAs= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= @@ -324,8 +324,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= -golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= From 12db4692972ca4892ce781e1c7352cd7bc75382c Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 15 Jun 2023 12:53:34 +0530 Subject: [PATCH 224/324] Better tracking of signalling connection. (#1794) * Better tracking of signalling connection. - Reason for closing signaling channel. - ConnectionID attached to request source/response sink * Tests --- pkg/routing/interfaces.go | 2 + pkg/routing/localrouter.go | 10 +-- pkg/routing/messagechannel.go | 21 ++++-- pkg/routing/messagechannel_test.go | 2 +- pkg/routing/redis.go | 20 +++++- pkg/routing/redisrouter.go | 6 +- pkg/routing/routingfakes/fake_message_sink.go | 66 +++++++++++++++++++ .../routingfakes/fake_message_source.go | 66 +++++++++++++++++++ pkg/routing/signal.go | 8 ++- pkg/rtc/participant.go | 20 ++++-- pkg/rtc/participant_internal_test.go | 2 +- pkg/rtc/participant_signal.go | 5 +- pkg/rtc/room.go | 2 +- pkg/rtc/types/interfaces.go | 45 ++++++++++++- .../typesfakes/fake_local_participant.go | 21 ++++-- pkg/service/roommanager.go | 4 +- pkg/service/rtcservice.go | 11 ++-- pkg/service/signal.go | 13 ++-- 18 files changed, 275 insertions(+), 49 deletions(-) diff --git a/pkg/routing/interfaces.go b/pkg/routing/interfaces.go index 566d2a2b4..3bc54d9c9 100644 --- a/pkg/routing/interfaces.go +++ b/pkg/routing/interfaces.go @@ -23,6 +23,7 @@ type MessageSink interface { WriteMessage(msg proto.Message) error IsClosed() bool Close() + ConnectionID() livekit.ConnectionID } //counterfeiter:generate . MessageSource @@ -31,6 +32,7 @@ type MessageSource interface { ReadChan() <-chan proto.Message IsClosed() bool Close() + ConnectionID() livekit.ConnectionID } type ParticipantInit struct { diff --git a/pkg/routing/localrouter.go b/pkg/routing/localrouter.go index da43566db..0b604a73d 100644 --- a/pkg/routing/localrouter.go +++ b/pkg/routing/localrouter.go @@ -38,7 +38,7 @@ func NewLocalRouter(currentNode LocalNode, signalClient SignalClient) *LocalRout signalClient: signalClient, requestChannels: make(map[string]*MessageChannel), responseChannels: make(map[string]*MessageChannel), - rtcMessageChan: NewMessageChannel(localRTCChannelSize), + rtcMessageChan: NewMessageChannel(livekit.ConnectionID("local"), localRTCChannelSize), } } @@ -93,7 +93,7 @@ func (r *LocalRouter) StartParticipantSignalWithNodeID(ctx context.Context, room logger.Errorw("could not handle new participant", err, "room", roomName, "participant", pi.Identity, - "connectionID", connectionID, + "connID", connectionID, ) } return @@ -103,7 +103,7 @@ func (r *LocalRouter) WriteParticipantRTC(_ context.Context, roomName livekit.Ro r.lock.Lock() if r.rtcMessageChan.IsClosed() { // create a new one - r.rtcMessageChan = NewMessageChannel(localRTCChannelSize) + r.rtcMessageChan = NewMessageChannel(livekit.ConnectionID("local"), localRTCChannelSize) } r.lock.Unlock() msg.ParticipantKey = string(ParticipantKeyLegacy(roomName, identity)) @@ -121,7 +121,7 @@ func (r *LocalRouter) WriteNodeRTC(_ context.Context, _ string, msg *livekit.RTC r.lock.Lock() if r.rtcMessageChan.IsClosed() { // create a new one - r.rtcMessageChan = NewMessageChannel(localRTCChannelSize) + r.rtcMessageChan = NewMessageChannel(livekit.ConnectionID("local"), localRTCChannelSize) } r.lock.Unlock() return r.writeRTCMessage(r.rtcMessageChan, msg) @@ -254,7 +254,7 @@ func (r *LocalRouter) getOrCreateMessageChannel(target map[string]*MessageChanne return mc } - mc = NewMessageChannel(DefaultMessageChannelSize) + mc = NewMessageChannel(livekit.ConnectionID(key), DefaultMessageChannelSize) mc.OnClose(func() { r.lock.Lock() delete(target, key) diff --git a/pkg/routing/messagechannel.go b/pkg/routing/messagechannel.go index de5bd9075..bf914e1aa 100644 --- a/pkg/routing/messagechannel.go +++ b/pkg/routing/messagechannel.go @@ -3,24 +3,27 @@ package routing import ( "sync" + "github.com/livekit/protocol/livekit" "google.golang.org/protobuf/proto" ) const DefaultMessageChannelSize = 200 type MessageChannel struct { - msgChan chan proto.Message - onClose func() - isClosed bool - lock sync.RWMutex + connectionID livekit.ConnectionID + msgChan chan proto.Message + onClose func() + isClosed bool + lock sync.RWMutex } -func NewDefaultMessageChannel() *MessageChannel { - return NewMessageChannel(DefaultMessageChannelSize) +func NewDefaultMessageChannel(connectionID livekit.ConnectionID) *MessageChannel { + return NewMessageChannel(connectionID, DefaultMessageChannelSize) } -func NewMessageChannel(size int) *MessageChannel { +func NewMessageChannel(connectionID livekit.ConnectionID, size int) *MessageChannel { return &MessageChannel{ + connectionID: connectionID, // allow some buffer to avoid blocked writes msgChan: make(chan proto.Message, size), } @@ -71,3 +74,7 @@ func (m *MessageChannel) Close() { m.onClose() } } + +func (m *MessageChannel) ConnectionID() livekit.ConnectionID { + return m.connectionID +} diff --git a/pkg/routing/messagechannel_test.go b/pkg/routing/messagechannel_test.go index b79a00795..25bf7fdf1 100644 --- a/pkg/routing/messagechannel_test.go +++ b/pkg/routing/messagechannel_test.go @@ -11,7 +11,7 @@ import ( func TestMessageChannel_WriteMessageClosed(t *testing.T) { // ensure it doesn't panic when written to after closing - m := routing.NewMessageChannel(routing.DefaultMessageChannelSize) + m := routing.NewMessageChannel(livekit.ConnectionID("test"), routing.DefaultMessageChannelSize) go func() { for msg := range m.ReadChan() { if msg == nil { diff --git a/pkg/routing/redis.go b/pkg/routing/redis.go index db781cbe9..054613e61 100644 --- a/pkg/routing/redis.go +++ b/pkg/routing/redis.go @@ -98,16 +98,24 @@ func publishSignalMessage(rc redis.UniversalClient, nodeID livekit.NodeID, conne type RTCNodeSink struct { rc redis.UniversalClient nodeID livekit.NodeID + connectionID livekit.ConnectionID participantKey livekit.ParticipantKey participantKeyB62 livekit.ParticipantKey isClosed atomic.Bool onClose func() } -func NewRTCNodeSink(rc redis.UniversalClient, nodeID livekit.NodeID, participantKey livekit.ParticipantKey, participantKeyB62 livekit.ParticipantKey) *RTCNodeSink { +func NewRTCNodeSink( + rc redis.UniversalClient, + nodeID livekit.NodeID, + connectionID livekit.ConnectionID, + participantKey livekit.ParticipantKey, + participantKeyB62 livekit.ParticipantKey, +) *RTCNodeSink { return &RTCNodeSink{ rc: rc, nodeID: nodeID, + connectionID: connectionID, participantKey: participantKey, participantKeyB62: participantKeyB62, } @@ -137,6 +145,12 @@ func (s *RTCNodeSink) OnClose(f func()) { s.onClose = f } +func (s *RTCNodeSink) ConnectionID() livekit.ConnectionID { + return s.connectionID +} + +// ---------------------------------------------------------------------- + type SignalNodeSink struct { rc redis.UniversalClient nodeID livekit.NodeID @@ -177,3 +191,7 @@ func (s *SignalNodeSink) IsClosed() bool { func (s *SignalNodeSink) OnClose(f func()) { s.onClose = f } + +func (s *SignalNodeSink) ConnectionID() livekit.ConnectionID { + return s.connectionID +} diff --git a/pkg/routing/redisrouter.go b/pkg/routing/redisrouter.go index c5da2a54b..d31a13638 100644 --- a/pkg/routing/redisrouter.go +++ b/pkg/routing/redisrouter.go @@ -174,7 +174,7 @@ func (r *RedisRouter) StartParticipantSignal(ctx context.Context, roomName livek // set up response channel before sending StartSession and be ready to receive responses. resChan := r.getOrCreateMessageChannel(r.responseChannels, string(connectionID)) - sink := NewRTCNodeSink(r.rc, livekit.NodeID(rtcNode.Id), pKey, pKeyB62) + sink := NewRTCNodeSink(r.rc, livekit.NodeID(rtcNode.Id), connectionID, pKey, pKeyB62) // serialize claims ss, err := pi.ToStartSession(roomName, connectionID) @@ -199,7 +199,7 @@ func (r *RedisRouter) WriteParticipantRTC(_ context.Context, roomName livekit.Ro return err } - rtcSink := NewRTCNodeSink(r.rc, livekit.NodeID(rtcNode), pkey, pkeyB62) + rtcSink := NewRTCNodeSink(r.rc, livekit.NodeID(rtcNode), livekit.ConnectionID("ephemeral"), pkey, pkeyB62) msg.ParticipantKey = string(ParticipantKeyLegacy(roomName, identity)) msg.ParticipantKeyB62 = string(ParticipantKey(roomName, identity)) return r.writeRTCMessage(rtcSink, msg) @@ -216,7 +216,7 @@ func (r *RedisRouter) WriteRoomRTC(ctx context.Context, roomName livekit.RoomNam } func (r *RedisRouter) WriteNodeRTC(_ context.Context, rtcNodeID string, msg *livekit.RTCNodeMessage) error { - rtcSink := NewRTCNodeSink(r.rc, livekit.NodeID(rtcNodeID), livekit.ParticipantKey(msg.ParticipantKey), livekit.ParticipantKey(msg.ParticipantKeyB62)) + rtcSink := NewRTCNodeSink(r.rc, livekit.NodeID(rtcNodeID), livekit.ConnectionID("ephemeral"), livekit.ParticipantKey(msg.ParticipantKey), livekit.ParticipantKey(msg.ParticipantKeyB62)) return r.writeRTCMessage(rtcSink, msg) } diff --git a/pkg/routing/routingfakes/fake_message_sink.go b/pkg/routing/routingfakes/fake_message_sink.go index ec53c4309..9359b2ac2 100644 --- a/pkg/routing/routingfakes/fake_message_sink.go +++ b/pkg/routing/routingfakes/fake_message_sink.go @@ -5,6 +5,7 @@ import ( "sync" "github.com/livekit/livekit-server/pkg/routing" + "github.com/livekit/protocol/livekit" "google.golang.org/protobuf/reflect/protoreflect" ) @@ -13,6 +14,16 @@ type FakeMessageSink struct { closeMutex sync.RWMutex closeArgsForCall []struct { } + ConnectionIDStub func() livekit.ConnectionID + connectionIDMutex sync.RWMutex + connectionIDArgsForCall []struct { + } + connectionIDReturns struct { + result1 livekit.ConnectionID + } + connectionIDReturnsOnCall map[int]struct { + result1 livekit.ConnectionID + } IsClosedStub func() bool isClosedMutex sync.RWMutex isClosedArgsForCall []struct { @@ -62,6 +73,59 @@ func (fake *FakeMessageSink) CloseCalls(stub func()) { fake.CloseStub = stub } +func (fake *FakeMessageSink) ConnectionID() livekit.ConnectionID { + fake.connectionIDMutex.Lock() + ret, specificReturn := fake.connectionIDReturnsOnCall[len(fake.connectionIDArgsForCall)] + fake.connectionIDArgsForCall = append(fake.connectionIDArgsForCall, struct { + }{}) + stub := fake.ConnectionIDStub + fakeReturns := fake.connectionIDReturns + fake.recordInvocation("ConnectionID", []interface{}{}) + fake.connectionIDMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeMessageSink) ConnectionIDCallCount() int { + fake.connectionIDMutex.RLock() + defer fake.connectionIDMutex.RUnlock() + return len(fake.connectionIDArgsForCall) +} + +func (fake *FakeMessageSink) ConnectionIDCalls(stub func() livekit.ConnectionID) { + fake.connectionIDMutex.Lock() + defer fake.connectionIDMutex.Unlock() + fake.ConnectionIDStub = stub +} + +func (fake *FakeMessageSink) ConnectionIDReturns(result1 livekit.ConnectionID) { + fake.connectionIDMutex.Lock() + defer fake.connectionIDMutex.Unlock() + fake.ConnectionIDStub = nil + fake.connectionIDReturns = struct { + result1 livekit.ConnectionID + }{result1} +} + +func (fake *FakeMessageSink) ConnectionIDReturnsOnCall(i int, result1 livekit.ConnectionID) { + fake.connectionIDMutex.Lock() + defer fake.connectionIDMutex.Unlock() + fake.ConnectionIDStub = nil + if fake.connectionIDReturnsOnCall == nil { + fake.connectionIDReturnsOnCall = make(map[int]struct { + result1 livekit.ConnectionID + }) + } + fake.connectionIDReturnsOnCall[i] = struct { + result1 livekit.ConnectionID + }{result1} +} + func (fake *FakeMessageSink) IsClosed() bool { fake.isClosedMutex.Lock() ret, specificReturn := fake.isClosedReturnsOnCall[len(fake.isClosedArgsForCall)] @@ -181,6 +245,8 @@ func (fake *FakeMessageSink) Invocations() map[string][][]interface{} { defer fake.invocationsMutex.RUnlock() fake.closeMutex.RLock() defer fake.closeMutex.RUnlock() + fake.connectionIDMutex.RLock() + defer fake.connectionIDMutex.RUnlock() fake.isClosedMutex.RLock() defer fake.isClosedMutex.RUnlock() fake.writeMessageMutex.RLock() diff --git a/pkg/routing/routingfakes/fake_message_source.go b/pkg/routing/routingfakes/fake_message_source.go index acfe7606c..40c48eb56 100644 --- a/pkg/routing/routingfakes/fake_message_source.go +++ b/pkg/routing/routingfakes/fake_message_source.go @@ -5,6 +5,7 @@ import ( "sync" "github.com/livekit/livekit-server/pkg/routing" + "github.com/livekit/protocol/livekit" "google.golang.org/protobuf/reflect/protoreflect" ) @@ -13,6 +14,16 @@ type FakeMessageSource struct { closeMutex sync.RWMutex closeArgsForCall []struct { } + ConnectionIDStub func() livekit.ConnectionID + connectionIDMutex sync.RWMutex + connectionIDArgsForCall []struct { + } + connectionIDReturns struct { + result1 livekit.ConnectionID + } + connectionIDReturnsOnCall map[int]struct { + result1 livekit.ConnectionID + } IsClosedStub func() bool isClosedMutex sync.RWMutex isClosedArgsForCall []struct { @@ -61,6 +72,59 @@ func (fake *FakeMessageSource) CloseCalls(stub func()) { fake.CloseStub = stub } +func (fake *FakeMessageSource) ConnectionID() livekit.ConnectionID { + fake.connectionIDMutex.Lock() + ret, specificReturn := fake.connectionIDReturnsOnCall[len(fake.connectionIDArgsForCall)] + fake.connectionIDArgsForCall = append(fake.connectionIDArgsForCall, struct { + }{}) + stub := fake.ConnectionIDStub + fakeReturns := fake.connectionIDReturns + fake.recordInvocation("ConnectionID", []interface{}{}) + fake.connectionIDMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeMessageSource) ConnectionIDCallCount() int { + fake.connectionIDMutex.RLock() + defer fake.connectionIDMutex.RUnlock() + return len(fake.connectionIDArgsForCall) +} + +func (fake *FakeMessageSource) ConnectionIDCalls(stub func() livekit.ConnectionID) { + fake.connectionIDMutex.Lock() + defer fake.connectionIDMutex.Unlock() + fake.ConnectionIDStub = stub +} + +func (fake *FakeMessageSource) ConnectionIDReturns(result1 livekit.ConnectionID) { + fake.connectionIDMutex.Lock() + defer fake.connectionIDMutex.Unlock() + fake.ConnectionIDStub = nil + fake.connectionIDReturns = struct { + result1 livekit.ConnectionID + }{result1} +} + +func (fake *FakeMessageSource) ConnectionIDReturnsOnCall(i int, result1 livekit.ConnectionID) { + fake.connectionIDMutex.Lock() + defer fake.connectionIDMutex.Unlock() + fake.ConnectionIDStub = nil + if fake.connectionIDReturnsOnCall == nil { + fake.connectionIDReturnsOnCall = make(map[int]struct { + result1 livekit.ConnectionID + }) + } + fake.connectionIDReturnsOnCall[i] = struct { + result1 livekit.ConnectionID + }{result1} +} + func (fake *FakeMessageSource) IsClosed() bool { fake.isClosedMutex.Lock() ret, specificReturn := fake.isClosedReturnsOnCall[len(fake.isClosedArgsForCall)] @@ -172,6 +236,8 @@ func (fake *FakeMessageSource) Invocations() map[string][][]interface{} { defer fake.invocationsMutex.RUnlock() fake.closeMutex.RLock() defer fake.closeMutex.RUnlock() + fake.connectionIDMutex.RLock() + defer fake.connectionIDMutex.RUnlock() fake.isClosedMutex.RLock() defer fake.isClosedMutex.RUnlock() fake.readChanMutex.RLock() diff --git a/pkg/routing/signal.go b/pkg/routing/signal.go index fab96ef7c..fad31191b 100644 --- a/pkg/routing/signal.go +++ b/pkg/routing/signal.go @@ -105,8 +105,9 @@ func (r *signalClient) StartParticipantSignal( Writer: signalRequestMessageWriter{}, CloseOnFailure: true, BlockOnClose: true, + ConnectionID: connectionID, }) - resChan := NewDefaultMessageChannel() + resChan := NewDefaultMessageChannel(connectionID) go func() { r.active.Inc() @@ -230,6 +231,7 @@ type SignalSinkParams[SendType, RecvType RelaySignalMessage] struct { Writer SignalMessageWriter[SendType] CloseOnFailure bool BlockOnClose bool + ConnectionID livekit.ConnectionID } func NewSignalMessageSink[SendType, RecvType RelaySignalMessage](params SignalSinkParams[SendType, RecvType]) MessageSink { @@ -348,3 +350,7 @@ func (s *signalMessageSink[SendType, RecvType]) WriteMessage(msg proto.Message) } return nil } + +func (s *signalMessageSink[SendType, RecvType]) ConnectionID() livekit.ConnectionID { + return s.SignalSinkParams.ConnectionID +} diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 5c3cdef26..5971d00e9 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -688,7 +688,7 @@ func (p *ParticipantImpl) Close(sendLeave bool, reason types.ParticipantCloseRea p.updateState(livekit.ParticipantInfo_DISCONNECTED) // ensure this is synchronized - p.CloseSignalConnection() + p.CloseSignalConnection(types.SignallingCloseReasonParticipantClose) p.lock.RLock() onClose := p.onClose p.lock.RUnlock() @@ -741,7 +741,7 @@ func (p *ParticipantImpl) MaybeStartMigration(force bool, onStart func()) bool { onStart() } - p.CloseSignalConnection() + p.CloseSignalConnection(types.SignallingCloseReasonMigration) // // On subscriber peer connection, remote side will try ICE on both @@ -1349,7 +1349,7 @@ func (p *ParticipantImpl) setupDisconnectTimer() { func (p *ParticipantImpl) onAnyTransportFailed() { // clients support resuming of connections when websocket becomes disconnected - p.CloseSignalConnection() + p.CloseSignalConnection(types.SignallingCloseReasonTransportFailure) // detect when participant has actually left. p.setupDisconnectTimer() @@ -2068,7 +2068,17 @@ func (p *ParticipantImpl) IssueFullReconnect(reason types.ParticipantCloseReason }, }, }) - p.CloseSignalConnection() + + scr := types.SignallingCloseReasonUnknown + switch reason { + case types.ParticipantCloseReasonPublicationError: + scr = types.SignallingCloseReasonFullReconnectPublicationError + case types.ParticipantCloseReasonSubscriptionError: + scr = types.SignallingCloseReasonFullReconnectSubscriptionError + case types.ParticipantCloseReasonNegotiateFailed: + scr = types.SignallingCloseReasonFullReconnectNegotiateFailed + } + p.CloseSignalConnection(scr) // on a full reconnect, no need to supervise this participant anymore p.supervisor.Stop() @@ -2101,7 +2111,7 @@ func (p *ParticipantImpl) onSubscriptionError(trackID livekit.TrackID, fatal boo if p.params.ReconnectOnSubscriptionError && fatal { p.params.Logger.Infow("issuing full reconnect on subscription error", "trackID", trackID) - p.IssueFullReconnect(types.ParticipantCloseReasonPublicationError) + p.IssueFullReconnect(types.ParticipantCloseReasonSubscriptionError) } } diff --git a/pkg/rtc/participant_internal_test.go b/pkg/rtc/participant_internal_test.go index adbe5862c..8dce027b5 100644 --- a/pkg/rtc/participant_internal_test.go +++ b/pkg/rtc/participant_internal_test.go @@ -229,7 +229,7 @@ func TestOutOfOrderUpdates(t *testing.T) { func TestDisconnectTiming(t *testing.T) { t.Run("Negotiate doesn't panic after channel closed", func(t *testing.T) { p := newParticipantForTest("test") - msg := routing.NewMessageChannel(routing.DefaultMessageChannelSize) + msg := routing.NewMessageChannel(livekit.ConnectionID("test"), routing.DefaultMessageChannelSize) p.params.Sink = msg go func() { for msg := range msg.ReadChan() { diff --git a/pkg/rtc/participant_signal.go b/pkg/rtc/participant_signal.go index e09ff85ab..192ef0b48 100644 --- a/pkg/rtc/participant_signal.go +++ b/pkg/rtc/participant_signal.go @@ -11,6 +11,7 @@ import ( "github.com/livekit/psrpc" "github.com/livekit/livekit-server/pkg/routing" + "github.com/livekit/livekit-server/pkg/rtc/types" ) func (p *ParticipantImpl) getResponseSink() routing.MessageSink { @@ -280,10 +281,10 @@ func (p *ParticipantImpl) writeMessage(msg *livekit.SignalResponse) error { } // closes signal connection to notify client to resume/reconnect -func (p *ParticipantImpl) CloseSignalConnection() { +func (p *ParticipantImpl) CloseSignalConnection(reason types.SignallingCloseReason) { sink := p.getResponseSink() if sink != nil { - p.params.Logger.Infow("closing signal connection") + p.params.Logger.Infow("closing signal connection", "reason", reason, "connID", sink.ConnectionID()) sink.Close() p.SetResponseSink(nil) } diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index 0441a56b4..4344c6787 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -400,7 +400,7 @@ func (r *Room) GetParticipantRequestSource(identity livekit.ParticipantIdentity) func (r *Room) ResumeParticipant(p types.LocalParticipant, requestSource routing.MessageSource, responseSink routing.MessageSink, iceServers []*livekit.ICEServer, reason livekit.ReconnectReason) error { r.ReplaceParticipantRequestSource(p.Identity(), requestSource) // close previous sink, and link to new one - p.CloseSignalConnection() + p.CloseSignalConnection(types.SignallingCloseReasonResume) p.SetResponseSink(responseSink) p.SetSignalSourceValid(true) diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index 2501d7013..d9b887041 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -86,6 +86,7 @@ const ( ParticipantCloseReasonMigrationRequested ParticipantCloseReasonOvercommitted ParticipantCloseReasonPublicationError + ParticipantCloseReasonSubscriptionError ) func (p ParticipantCloseReason) String() string { @@ -130,6 +131,8 @@ func (p ParticipantCloseReason) String() string { return "OVERCOMMITTED" case ParticipantCloseReasonPublicationError: return "PUBLICATION_ERROR" + case ParticipantCloseReasonSubscriptionError: + return "SUBSCRIPTION_ERROR" default: return fmt.Sprintf("%d", int(p)) } @@ -160,7 +163,7 @@ func (p ParticipantCloseReason) ToDisconnectReason() livekit.DisconnectReason { return livekit.DisconnectReason_SERVER_SHUTDOWN case ParticipantCloseReasonOvercommitted: return livekit.DisconnectReason_SERVER_SHUTDOWN - case ParticipantCloseReasonNegotiateFailed, ParticipantCloseReasonPublicationError: + case ParticipantCloseReasonNegotiateFailed, ParticipantCloseReasonPublicationError, ParticipantCloseReasonSubscriptionError: return livekit.DisconnectReason_STATE_MISMATCH default: // the other types will map to unknown reason @@ -170,6 +173,44 @@ func (p ParticipantCloseReason) ToDisconnectReason() livekit.DisconnectReason { // --------------------------------------------- +type SignallingCloseReason int + +const ( + SignallingCloseReasonUnknown SignallingCloseReason = iota + SignallingCloseReasonMigration + SignallingCloseReasonResume + SignallingCloseReasonTransportFailure + SignallingCloseReasonFullReconnectPublicationError + SignallingCloseReasonFullReconnectSubscriptionError + SignallingCloseReasonFullReconnectNegotiateFailed + SignallingCloseReasonParticipantClose +) + +func (s SignallingCloseReason) String() string { + switch s { + case SignallingCloseReasonUnknown: + return "UNKNOWN" + case SignallingCloseReasonMigration: + return "MIGRATION" + case SignallingCloseReasonResume: + return "RESUME" + case SignallingCloseReasonTransportFailure: + return "TRANSPORT_FAILURE" + case SignallingCloseReasonFullReconnectPublicationError: + return "FULL_RECONNECT_PUBLICATION_ERROR" + case SignallingCloseReasonFullReconnectSubscriptionError: + return "FULL_RECONNECT_SUBSCRIPTION_ERROR" + case SignallingCloseReasonFullReconnectNegotiateFailed: + return "FULL_RECONNECT_NEGOTIATE_FAILED" + case SignallingCloseReasonParticipantClose: + return "PARTICIPANT_CLOSE" + default: + return fmt.Sprintf("%d", int(s)) + } +} + +// --------------------------------------------- + //counterfeiter:generate . Participant type Participant interface { ID() livekit.ParticipantID @@ -249,7 +290,7 @@ type LocalParticipant interface { GetBufferFactory() *buffer.Factory SetResponseSink(sink routing.MessageSink) - CloseSignalConnection() + CloseSignalConnection(reason SignallingCloseReason) UpdateLastSeenSignal() SetSignalSourceValid(valid bool) diff --git a/pkg/rtc/types/typesfakes/fake_local_participant.go b/pkg/rtc/types/typesfakes/fake_local_participant.go index e4ddfea5f..235994fa9 100644 --- a/pkg/rtc/types/typesfakes/fake_local_participant.go +++ b/pkg/rtc/types/typesfakes/fake_local_participant.go @@ -131,9 +131,10 @@ type FakeLocalParticipant struct { closeReturnsOnCall map[int]struct { result1 error } - CloseSignalConnectionStub func() + CloseSignalConnectionStub func(types.SignallingCloseReason) closeSignalConnectionMutex sync.RWMutex closeSignalConnectionArgsForCall []struct { + arg1 types.SignallingCloseReason } ConnectedAtStub func() time.Time connectedAtMutex sync.RWMutex @@ -1429,15 +1430,16 @@ func (fake *FakeLocalParticipant) CloseReturnsOnCall(i int, result1 error) { }{result1} } -func (fake *FakeLocalParticipant) CloseSignalConnection() { +func (fake *FakeLocalParticipant) CloseSignalConnection(arg1 types.SignallingCloseReason) { fake.closeSignalConnectionMutex.Lock() fake.closeSignalConnectionArgsForCall = append(fake.closeSignalConnectionArgsForCall, struct { - }{}) + arg1 types.SignallingCloseReason + }{arg1}) stub := fake.CloseSignalConnectionStub - fake.recordInvocation("CloseSignalConnection", []interface{}{}) + fake.recordInvocation("CloseSignalConnection", []interface{}{arg1}) fake.closeSignalConnectionMutex.Unlock() if stub != nil { - fake.CloseSignalConnectionStub() + fake.CloseSignalConnectionStub(arg1) } } @@ -1447,12 +1449,19 @@ func (fake *FakeLocalParticipant) CloseSignalConnectionCallCount() int { return len(fake.closeSignalConnectionArgsForCall) } -func (fake *FakeLocalParticipant) CloseSignalConnectionCalls(stub func()) { +func (fake *FakeLocalParticipant) CloseSignalConnectionCalls(stub func(types.SignallingCloseReason)) { fake.closeSignalConnectionMutex.Lock() defer fake.closeSignalConnectionMutex.Unlock() fake.CloseSignalConnectionStub = stub } +func (fake *FakeLocalParticipant) CloseSignalConnectionArgsForCall(i int) types.SignallingCloseReason { + fake.closeSignalConnectionMutex.RLock() + defer fake.closeSignalConnectionMutex.RUnlock() + argsForCall := fake.closeSignalConnectionArgsForCall[i] + return argsForCall.arg1 +} + func (fake *FakeLocalParticipant) ConnectedAt() time.Time { fake.connectedAtMutex.Lock() ret, specificReturn := fake.connectedAtReturnsOnCall[len(fake.connectedAtArgsForCall)] diff --git a/pkg/service/roommanager.go b/pkg/service/roommanager.go index 056b1b976..4e3997d15 100644 --- a/pkg/service/roommanager.go +++ b/pkg/service/roommanager.go @@ -485,7 +485,7 @@ func (r *RoomManager) rtcSessionWorker(room *rtc.Room, participant types.LocalPa false, ) defer func() { - pLogger.Debugw("RTC session finishing") + pLogger.Debugw("RTC session finishing", "connID", requestSource.ConnectionID()) requestSource.Close() }() @@ -511,7 +511,7 @@ func (r *RoomManager) rtcSessionWorker(room *rtc.Room, participant types.LocalPa case <-tokenTicker.C: // refresh token with the first API Key/secret pair if err := r.refreshToken(participant); err != nil { - pLogger.Errorw("could not refresh token", err) + pLogger.Errorw("could not refresh token", err, "connID", requestSource.ConnectionID()) } case obj := <-requestSource.ReadChan(): // In single node mode, the request source is directly tied to the signal message channel diff --git a/pkg/service/rtcservice.go b/pkg/service/rtcservice.go index cd6bc93da..3888dfc24 100644 --- a/pkg/service/rtcservice.go +++ b/pkg/service/rtcservice.go @@ -285,6 +285,7 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { return case msg := <-cr.ResponseSource.ReadChan(): if msg == nil { + pLogger.Infow("nothing to read from response source", "connID", cr.ConnectionID) return } res, ok := msg.(*livekit.SignalResponse) @@ -323,16 +324,15 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { // handle incoming requests from websocket for { req, count, err := sigConn.ReadRequest() - // normal closure if err != nil { + // normal/expected closure if err == io.EOF || strings.HasSuffix(err.Error(), "use of closed network connection") || websocket.IsCloseError(err, websocket.CloseAbnormalClosure, websocket.CloseGoingAway, websocket.CloseNormalClosure, websocket.CloseNoStatusReceived) { - pLogger.Debugw("exit ws read loop for closed connection", "connID", cr.ConnectionID) - return + pLogger.Infow("exit ws read loop for closed connection", "connID", cr.ConnectionID, "wsError", err) } else { pLogger.Errorw("error reading from websocket", err) - return } + return } if signalStats != nil { signalStats.AddBytes(uint64(count), false) @@ -374,8 +374,7 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { } if err := cr.RequestSink.WriteMessage(req); err != nil { - pLogger.Warnw("error writing to request sink", err, - "connID", cr.ConnectionID) + pLogger.Warnw("error writing to request sink", err, "connID", cr.ConnectionID) } } } diff --git a/pkg/service/signal.go b/pkg/service/signal.go index a8ce00811..c6907dadb 100644 --- a/pkg/service/signal.go +++ b/pkg/service/signal.go @@ -136,17 +136,18 @@ func (r *signalService) RelaySignal(stream psrpc.ServerStream[*rpc.RelaySignalRe l := logger.GetLogger().WithValues( "room", ss.RoomName, "participant", ss.Identity, - "connectionID", ss.ConnectionId, + "connID", ss.ConnectionId, ) - reqChan := routing.NewDefaultMessageChannel() + reqChan := routing.NewDefaultMessageChannel(livekit.ConnectionID(ss.ConnectionId)) defer reqChan.Close() sink := routing.NewSignalMessageSink(routing.SignalSinkParams[*rpc.RelaySignalResponse, *rpc.RelaySignalRequest]{ - Logger: l, - Stream: stream, - Config: r.config, - Writer: signalResponseMessageWriter{}, + Logger: l, + Stream: stream, + Config: r.config, + Writer: signalResponseMessageWriter{}, + ConnectionID: livekit.ConnectionID(ss.ConnectionId), }) err = r.sessionHandler(ctx, livekit.RoomName(ss.RoomName), *pi, livekit.ConnectionID(ss.ConnectionId), reqChan, sink) From 0dab55556d14bc544d7cfc8126d5c00fe3c53f9b Mon Sep 17 00:00:00 2001 From: paulwe Date: Thu, 15 Jun 2023 11:56:41 -0700 Subject: [PATCH 225/324] add drain function to rtc service --- pkg/service/rtcservice.go | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/pkg/service/rtcservice.go b/pkg/service/rtcservice.go index 3888dfc24..5af97a78c 100644 --- a/pkg/service/rtcservice.go +++ b/pkg/service/rtcservice.go @@ -5,14 +5,17 @@ import ( "errors" "fmt" "io" + "math/rand" "net/http" "os" "strconv" "strings" + "sync" "time" "github.com/gorilla/websocket" "github.com/ua-parser/uap-go/uaparser" + "golang.org/x/exp/maps" "github.com/livekit/livekit-server/pkg/utils" "github.com/livekit/protocol/livekit" @@ -37,6 +40,9 @@ type RTCService struct { limits config.LimitConfig parser *uaparser.Parser telemetry telemetry.TelemetryService + + mu sync.Mutex + connections map[*websocket.Conn]struct{} } func NewRTCService( @@ -58,6 +64,7 @@ func NewRTCService( limits: conf.Limit, parser: uaparser.NewFromSaved(), telemetry: telemetry, + connections: map[*websocket.Conn]struct{}{}, } // allow connections from any origin, since script may be hosted anywhere @@ -250,6 +257,16 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } + s.mu.Lock() + s.connections[conn] = struct{}{} + s.mu.Unlock() + + defer func() { + s.mu.Lock() + delete(s.connections, conn) + s.mu.Unlock() + }() + // websocket established sigConn := NewWSSignalConnection(conn) if count, err := sigConn.WriteResponse(initialResponse); err != nil { @@ -442,6 +459,23 @@ func (s *RTCService) ParseClientInfo(r *http.Request) *livekit.ClientInfo { return ci } +func (s *RTCService) DrainConnections(interval time.Duration) { + s.mu.Lock() + conns := maps.Clone(s.connections) + s.mu.Unlock() + + // jitter drain start + time.Sleep(time.Duration(rand.Int63n(int64(interval)))) + + t := time.NewTicker(interval) + defer t.Stop() + + for c := range conns { + c.Close() + <-t.C + } +} + type connectionResult struct { Room *livekit.Room ConnectionID livekit.ConnectionID From f71544e27a2edceadd5a0f0e76a7eba50b8d2421 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Thu, 15 Jun 2023 15:39:04 -0700 Subject: [PATCH 226/324] Do not send ParticipantJoined webhook if connection was resumed (#1795) * Do not send ParticipantJoined webhook if connection was resumed * isResume -> isMigration --- pkg/rtc/room.go | 13 +++++++++---- pkg/telemetry/events.go | 15 +++++++++------ pkg/telemetry/events_test.go | 4 ++-- .../telemetryfakes/fake_telemetry_service.go | 18 ++++++++++-------- pkg/telemetry/telemetryservice.go | 4 ++-- 5 files changed, 32 insertions(+), 22 deletions(-) diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index 4344c6787..8d343a98f 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -280,10 +280,15 @@ func (r *Room) Join(participant types.LocalParticipant, requestSource routing.Me // start the workers once connectivity is established p.Start() - r.telemetry.ParticipantActive(context.Background(), r.ToProto(), p.ToProto(), &livekit.AnalyticsClientMeta{ - ClientConnectTime: uint32(time.Since(p.ConnectedAt()).Milliseconds()), - ConnectionType: string(p.GetICEConnectionType()), - }) + r.telemetry.ParticipantActive(context.Background(), + r.ToProto(), + p.ToProto(), + &livekit.AnalyticsClientMeta{ + ClientConnectTime: uint32(time.Since(p.ConnectedAt()).Milliseconds()), + ConnectionType: string(p.GetICEConnectionType()), + }, + false, + ) } else if state == livekit.ParticipantInfo_DISCONNECTED { // remove participant from room go r.RemoveParticipant(p.Identity(), p.ID(), types.ParticipantCloseReasonStateDisconnected) diff --git a/pkg/telemetry/events.go b/pkg/telemetry/events.go index 26a215ba6..71d8c05b7 100644 --- a/pkg/telemetry/events.go +++ b/pkg/telemetry/events.go @@ -91,14 +91,17 @@ func (t *telemetryService) ParticipantActive( room *livekit.Room, participant *livekit.ParticipantInfo, clientMeta *livekit.AnalyticsClientMeta, + isMigration bool, ) { t.enqueue(func() { - // consider participant joined only when they became active - t.NotifyEvent(ctx, &livekit.WebhookEvent{ - Event: webhook.EventParticipantJoined, - Room: room, - Participant: participant, - }) + if !isMigration { + // consider participant joined only when they became active + t.NotifyEvent(ctx, &livekit.WebhookEvent{ + Event: webhook.EventParticipantJoined, + Room: room, + Participant: participant, + }) + } worker, ok := t.getWorker(livekit.ParticipantID(participant.Sid)) if !ok { diff --git a/pkg/telemetry/events_test.go b/pkg/telemetry/events_test.go index 852701180..77529b211 100644 --- a/pkg/telemetry/events_test.go +++ b/pkg/telemetry/events_test.go @@ -69,7 +69,7 @@ func Test_OnParticipantLeft_EventIsSent(t *testing.T) { participantInfo := &livekit.ParticipantInfo{Sid: partSID} // do - fixture.sut.ParticipantActive(context.Background(), room, participantInfo, &livekit.AnalyticsClientMeta{}) + fixture.sut.ParticipantActive(context.Background(), room, participantInfo, &livekit.AnalyticsClientMeta{}, false) fixture.sut.ParticipantLeft(context.Background(), room, participantInfo, true) time.Sleep(time.Millisecond * 500) @@ -159,7 +159,7 @@ func Test_OnParticipantActive_EventIsSent(t *testing.T) { ClientConnectTime: 420, } - fixture.sut.ParticipantActive(context.Background(), room, participantInfo, clientMetaConnect) + fixture.sut.ParticipantActive(context.Background(), room, participantInfo, clientMetaConnect, false) time.Sleep(time.Millisecond * 500) require.Equal(t, 2, fixture.analytics.SendEventCallCount()) diff --git a/pkg/telemetry/telemetryfakes/fake_telemetry_service.go b/pkg/telemetry/telemetryfakes/fake_telemetry_service.go index 800160fb7..f85e03527 100644 --- a/pkg/telemetry/telemetryfakes/fake_telemetry_service.go +++ b/pkg/telemetry/telemetryfakes/fake_telemetry_service.go @@ -62,13 +62,14 @@ type FakeTelemetryService struct { arg1 context.Context arg2 *livekit.WebhookEvent } - ParticipantActiveStub func(context.Context, *livekit.Room, *livekit.ParticipantInfo, *livekit.AnalyticsClientMeta) + ParticipantActiveStub func(context.Context, *livekit.Room, *livekit.ParticipantInfo, *livekit.AnalyticsClientMeta, bool) participantActiveMutex sync.RWMutex participantActiveArgsForCall []struct { arg1 context.Context arg2 *livekit.Room arg3 *livekit.ParticipantInfo arg4 *livekit.AnalyticsClientMeta + arg5 bool } ParticipantJoinedStub func(context.Context, *livekit.Room, *livekit.ParticipantInfo, *livekit.ClientInfo, *livekit.AnalyticsClientMeta, bool) participantJoinedMutex sync.RWMutex @@ -526,19 +527,20 @@ func (fake *FakeTelemetryService) NotifyEventArgsForCall(i int) (context.Context return argsForCall.arg1, argsForCall.arg2 } -func (fake *FakeTelemetryService) ParticipantActive(arg1 context.Context, arg2 *livekit.Room, arg3 *livekit.ParticipantInfo, arg4 *livekit.AnalyticsClientMeta) { +func (fake *FakeTelemetryService) ParticipantActive(arg1 context.Context, arg2 *livekit.Room, arg3 *livekit.ParticipantInfo, arg4 *livekit.AnalyticsClientMeta, arg5 bool) { fake.participantActiveMutex.Lock() fake.participantActiveArgsForCall = append(fake.participantActiveArgsForCall, struct { arg1 context.Context arg2 *livekit.Room arg3 *livekit.ParticipantInfo arg4 *livekit.AnalyticsClientMeta - }{arg1, arg2, arg3, arg4}) + arg5 bool + }{arg1, arg2, arg3, arg4, arg5}) stub := fake.ParticipantActiveStub - fake.recordInvocation("ParticipantActive", []interface{}{arg1, arg2, arg3, arg4}) + fake.recordInvocation("ParticipantActive", []interface{}{arg1, arg2, arg3, arg4, arg5}) fake.participantActiveMutex.Unlock() if stub != nil { - fake.ParticipantActiveStub(arg1, arg2, arg3, arg4) + fake.ParticipantActiveStub(arg1, arg2, arg3, arg4, arg5) } } @@ -548,17 +550,17 @@ func (fake *FakeTelemetryService) ParticipantActiveCallCount() int { return len(fake.participantActiveArgsForCall) } -func (fake *FakeTelemetryService) ParticipantActiveCalls(stub func(context.Context, *livekit.Room, *livekit.ParticipantInfo, *livekit.AnalyticsClientMeta)) { +func (fake *FakeTelemetryService) ParticipantActiveCalls(stub func(context.Context, *livekit.Room, *livekit.ParticipantInfo, *livekit.AnalyticsClientMeta, bool)) { fake.participantActiveMutex.Lock() defer fake.participantActiveMutex.Unlock() fake.ParticipantActiveStub = stub } -func (fake *FakeTelemetryService) ParticipantActiveArgsForCall(i int) (context.Context, *livekit.Room, *livekit.ParticipantInfo, *livekit.AnalyticsClientMeta) { +func (fake *FakeTelemetryService) ParticipantActiveArgsForCall(i int) (context.Context, *livekit.Room, *livekit.ParticipantInfo, *livekit.AnalyticsClientMeta, bool) { fake.participantActiveMutex.RLock() defer fake.participantActiveMutex.RUnlock() argsForCall := fake.participantActiveArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4 + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5 } func (fake *FakeTelemetryService) ParticipantJoined(arg1 context.Context, arg2 *livekit.Room, arg3 *livekit.ParticipantInfo, arg4 *livekit.ClientInfo, arg5 *livekit.AnalyticsClientMeta, arg6 bool) { diff --git a/pkg/telemetry/telemetryservice.go b/pkg/telemetry/telemetryservice.go index d3aaf1715..e304980aa 100644 --- a/pkg/telemetry/telemetryservice.go +++ b/pkg/telemetry/telemetryservice.go @@ -22,8 +22,8 @@ type TelemetryService interface { // ParticipantJoined - a participant establishes signal connection to a room ParticipantJoined(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, clientInfo *livekit.ClientInfo, clientMeta *livekit.AnalyticsClientMeta, shouldSendEvent bool) // ParticipantActive - a participant establishes media connection - ParticipantActive(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, clientMeta *livekit.AnalyticsClientMeta) - // ParticipantResumed - there has been an ICE restart or connection resume attempt + ParticipantActive(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, clientMeta *livekit.AnalyticsClientMeta, isMigration bool) + // ParticipantResumed - there has been an ICE restart or connection resume attempt, and we've received their signal connection ParticipantResumed(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, nodeID livekit.NodeID, reason livekit.ReconnectReason) // ParticipantLeft - the participant leaves the room, only sent if ParticipantActive has been called before ParticipantLeft(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, shouldSendEvent bool) From d0cda7c1478a74374ce6bd520d5de015832a88a7 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 15 Jun 2023 21:00:19 -0700 Subject: [PATCH 227/324] Update go deps (#1793) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 18 +++++++++--------- go.sum | 32 +++++++++++++++++++------------- 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/go.mod b/go.mod index e002e600c..417fc48c6 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/livekit/psrpc v0.3.1 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 - github.com/maxbrunsfeld/counterfeiter/v6 v6.6.1 + github.com/maxbrunsfeld/counterfeiter/v6 v6.6.2 github.com/mitchellh/go-homedir v1.1.0 github.com/olekukonko/tablewriter v0.0.5 github.com/pion/dtls/v2 v2.2.7 @@ -35,7 +35,7 @@ require ( github.com/pion/turn/v2 v2.1.0 github.com/pion/webrtc/v3 v3.2.9 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.15.1 + github.com/prometheus/client_golang v1.16.0 github.com/redis/go-redis/v9 v9.0.5 github.com/rs/cors v1.9.0 github.com/stretchr/testify v1.8.4 @@ -86,18 +86,18 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect - github.com/prometheus/procfs v0.9.0 // indirect + github.com/prometheus/procfs v0.10.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.24.0 // indirect - golang.org/x/crypto v0.9.0 // indirect + golang.org/x/crypto v0.10.0 // indirect golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 // indirect - golang.org/x/mod v0.8.0 // indirect - golang.org/x/net v0.10.0 // indirect - golang.org/x/sys v0.8.0 // indirect - golang.org/x/text v0.9.0 // indirect - golang.org/x/tools v0.6.0 // indirect + golang.org/x/mod v0.11.0 // indirect + golang.org/x/net v0.11.0 // indirect + golang.org/x/sys v0.9.0 // indirect + golang.org/x/text v0.10.0 // indirect + golang.org/x/tools v0.9.3 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect google.golang.org/grpc v1.55.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 575b10354..331cc5c2b 100644 --- a/go.sum +++ b/go.sum @@ -134,8 +134,8 @@ github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/Qd github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/maxbrunsfeld/counterfeiter/v6 v6.6.1 h1:9XE5ykDiC8eNSqIPkxx0EsV3kMX1oe4kQWRZjIgytUA= -github.com/maxbrunsfeld/counterfeiter/v6 v6.6.1/go.mod h1:qbKwBR+qQODzH2WD/s53mdgp/xVcXMlJb59GRFOp6Z4= +github.com/maxbrunsfeld/counterfeiter/v6 v6.6.2 h1:CEy7VRV/Vbm7YLuZo3pGKa5GlPX4zzric6dEubIJTx0= +github.com/maxbrunsfeld/counterfeiter/v6 v6.6.2/go.mod h1:otjOyjeqm3LALYcmX2AQIGH0VlojDoSd8aGOzsHAnBc= github.com/mdlayher/ethtool v0.0.0-20210210192532-2b88debcdd43/go.mod h1:+t7E0lkKfbBsebllff1xdTmyJt8lH37niI6kwFk9OTo= github.com/mdlayher/genetlink v1.0.0/go.mod h1:0rJ0h4itni50A86M2kHcgS85ttZazNt7a8H2a2cw0Gc= github.com/mdlayher/netlink v0.0.0-20190409211403-11939a169225/go.mod h1:eQB3mZE4aiYnlUsyGGCOpPETfdQq4Jhsgf1fk3cwQaA= @@ -177,7 +177,7 @@ github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/onsi/gomega v1.26.0 h1:03cDLK28U6hWvCAns6NeydX3zIm4SF3ci69ulidS32Q= +github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc= github.com/pion/datachannel v1.5.5 h1:10ef4kwdjije+M9d7Xm9im2Y3O6A6ccQb0zcqZcJew8= github.com/pion/datachannel v1.5.5/go.mod h1:iMz+lECmfdCMqFRhXhcA/219B0SQlbpoR2V118yimL0= github.com/pion/dtls/v2 v2.2.7 h1:cSUBsETxepsCSFSxC3mc/aDo14qQLMSL+O6IjG28yV8= @@ -221,14 +221,14 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= -github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= +github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= +github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= -github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= -github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= +github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= +github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= github.com/redis/go-redis/v9 v9.0.5 h1:CuQcn5HIEeK7BgElubPP8CGtE0KakrnbBSTLjathl5o= github.com/redis/go-redis/v9 v9.0.5/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= @@ -282,14 +282,16 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= -golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= +golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc= golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= +golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -315,8 +317,9 @@ golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= +golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -364,8 +367,9 @@ golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -382,16 +386,18 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190422233926-fe54fb35175b/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= +golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From 6946d0a3a1f725add08f71731262c8bd5a668458 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 16 Jun 2023 12:08:01 +0530 Subject: [PATCH 228/324] Do not mute forwarder when paused to bandwidth congestion. (#1796) * Do not mute forwarder when paused to bandwidth congestion. Detailed notes in code. * remove word --- pkg/sfu/forwarder.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 291d7e338..8929cf736 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -364,6 +364,25 @@ func (f *Forwarder) Mute(muted bool) (bool, buffer.VideoLayer) { return false, f.vls.GetMax() } + // Do not mute when paused due to bandwidth limitation. + // There are two issues + // 1. Muting means probing cannot happen on this track. + // 2. Muting also triggers notification to publisher about layers this forwarder needs. + // If this forwarder does not need any layer, publisher could turn off all layers. + // So, muting could lead to not being able to restart the track. + // To avoid that, ignore mute when paused due to bandwidth limitations. + // + // NOTE: The above scenario refers to mute getting triggered due + // to video stream visibility changes. When a stream is paused, it is possible + // that the receiver hides the video tile triggering subscription mute. + // The work around here to ignore mute does ignore an intentional mute. + // It could result in some bandwidth consumed for stream without visibility in + // the case of intentional mute. + if muted && f.isDeficientLocked() && f.lastAllocation.PauseReason == VideoPauseReasonBandwidth { + f.logger.Infow("ignoring forwarder mute, paused due to congestion") + return false, f.vls.GetMax() + } + f.logger.Debugw("setting forwarder mute", "muted", muted) f.muted = muted From 908b7a9bb14b0c64e10eb23ec84737178181ccf8 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 16 Jun 2023 19:00:17 +0530 Subject: [PATCH 229/324] Promote some migration logs to Infow (#1798) --- pkg/rtc/participant.go | 7 ++++--- pkg/sfu/buffer/rtpstats.go | 8 ++++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 5971d00e9..3d0c573d1 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -565,7 +565,7 @@ func (p *ParticipantImpl) handleMigrateMutedTrack() { } if len(pti.trackInfos) > 1 { - p.params.Logger.Warnw("too many pending migrated tracks", nil, "count", len(pti.trackInfos), "cid", cid) + p.params.Logger.Warnw("too many pending migrated tracks", nil, "trackID", pti.trackInfos[0].Sid, "count", len(pti.trackInfos), "cid", cid) } ti := pti.trackInfos[0] @@ -638,6 +638,7 @@ func (p *ParticipantImpl) SetMigrateInfo( p.supervisor.SetPublicationMute(livekit.TrackID(ti.Sid), ti.Muted) p.pendingTracks[t.GetCid()] = &pendingTrackInfo{trackInfos: []*livekit.TrackInfo{ti}, migrated: true} + p.params.Logger.Infow("pending track added (migration)", "trackID", ti.Sid, "track", ti.String()) } p.pendingTracksLock.Unlock() @@ -787,7 +788,7 @@ func (p *ParticipantImpl) SetMigrateState(s types.MigrateState) { return } - p.params.Logger.Debugw("SetMigrateState", "state", s) + p.params.Logger.Infow("SetMigrateState", "state", s) p.migrateState.Store(s) p.dirty.Store(true) @@ -1678,7 +1679,7 @@ func (p *ParticipantImpl) mediaTrackReceived(track *webrtc.TrackRemote, rtpRecei } func (p *ParticipantImpl) addMigrateMutedTrack(cid string, ti *livekit.TrackInfo) *MediaTrack { - p.params.Logger.Debugw("add migrate muted track", "cid", cid, "track", ti.String()) + p.params.Logger.Infow("add migrate muted track", "cid", cid, "trackID", ti.Sid, "track", ti.String()) rtpReceiver := p.TransportManager.GetPublisherRTPReceiver(ti.Mid) if rtpReceiver == nil { p.params.Logger.Errorw("could not find receiver for migrated track", nil, "trackID", ti.Sid) diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index 173d7f3a1..0bb269ed6 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -1546,14 +1546,18 @@ func (r *RTPStats) getDrift() (packetDrift driftResult, reportDrift driftResult) packetDrift.rtpDiffSinceFirst = getExtTS(r.highestTS, r.tsCycles) - r.extStartTS packetDrift.driftSamples = int64(packetDrift.rtpDiffSinceFirst - uint64(packetDrift.timeSinceFirst.Nanoseconds()*int64(r.params.ClockRate)/1e9)) packetDrift.driftMs = (float64(packetDrift.driftSamples) * 1000) / float64(r.params.ClockRate) - packetDrift.sampleRate = float64(packetDrift.rtpDiffSinceFirst) / packetDrift.timeSinceFirst.Seconds() + if packetDrift.timeSinceFirst.Seconds() != 0 { + packetDrift.sampleRate = float64(packetDrift.rtpDiffSinceFirst) / packetDrift.timeSinceFirst.Seconds() + } if r.srFirst != nil && r.srNewest != nil && r.srFirst.RTPTimestamp != r.srNewest.RTPTimestamp { reportDrift.timeSinceFirst = r.srNewest.NTPTimestamp.Time().Sub(r.srFirst.NTPTimestamp.Time()) reportDrift.rtpDiffSinceFirst = r.srNewest.RTPTimestampExt - r.srFirst.RTPTimestampExt reportDrift.driftSamples = int64(reportDrift.rtpDiffSinceFirst - uint64(reportDrift.timeSinceFirst.Nanoseconds()*int64(r.params.ClockRate)/1e9)) reportDrift.driftMs = (float64(reportDrift.driftSamples) * 1000) / float64(r.params.ClockRate) - reportDrift.sampleRate = float64(reportDrift.rtpDiffSinceFirst) / reportDrift.timeSinceFirst.Seconds() + if reportDrift.timeSinceFirst.Seconds() != 0 { + reportDrift.sampleRate = float64(reportDrift.rtpDiffSinceFirst) / reportDrift.timeSinceFirst.Seconds() + } } return } From cadf3bf6495443a60f1aecd43964fd877c8e6a5a Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 16 Jun 2023 22:00:38 +0530 Subject: [PATCH 230/324] Simulate muted audio track publish on migration. (#1799) Till now only video was using simulated publish when migrating on mute. But, with `pauseUpstream() + replaceTrack(null)`, it is possible that client does not send any data when muted. I do not think there is a problem to do this (even when cleint is actually using mute which sends silence frames). --- pkg/rtc/participant.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 3d0c573d1..112e660c6 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -569,7 +569,7 @@ func (p *ParticipantImpl) handleMigrateMutedTrack() { } ti := pti.trackInfos[0] - if ti.Muted && ti.Type == livekit.TrackType_VIDEO { + if ti.Muted { mt := p.addMigrateMutedTrack(cid, ti) if mt != nil { addedTracks = append(addedTracks, mt) From 552e3758d5535a41218944359320a758e9ba9e9d Mon Sep 17 00:00:00 2001 From: Benjamin Pracht Date: Fri, 16 Jun 2023 10:58:49 -0700 Subject: [PATCH 231/324] Add IngressUpdated event (#1775) --- pkg/service/ioinfo.go | 5 +++ pkg/telemetry/events.go | 6 +++ .../telemetryfakes/fake_telemetry_service.go | 41 +++++++++++++++++++ pkg/telemetry/telemetryservice.go | 1 + 4 files changed, 53 insertions(+) diff --git a/pkg/service/ioinfo.go b/pkg/service/ioinfo.go index 41ff4541c..f3bd6cac7 100644 --- a/pkg/service/ioinfo.go +++ b/pkg/service/ioinfo.go @@ -137,6 +137,11 @@ func (s *IOInfoService) UpdateIngressState(ctx context.Context, req *rpc.UpdateI s.telemetry.IngressStarted(ctx, info) logger.Infow("ingress started", "ingressID", req.IngressId) + + case livekit.IngressState_ENDPOINT_BUFFERING: + s.telemetry.IngressUpdated(ctx, info) + + logger.Infow("ingress buffering", "ingressID", req.IngressId) } } diff --git a/pkg/telemetry/events.go b/pkg/telemetry/events.go index 71d8c05b7..b1e731bdc 100644 --- a/pkg/telemetry/events.go +++ b/pkg/telemetry/events.go @@ -456,6 +456,12 @@ func (t *telemetryService) IngressStarted(ctx context.Context, info *livekit.Ing }) } +func (t *telemetryService) IngressUpdated(ctx context.Context, info *livekit.IngressInfo) { + t.enqueue(func() { + t.SendEvent(ctx, newIngressEvent(livekit.AnalyticsEventType_INGRESS_UPDATED, info)) + }) +} + func (t *telemetryService) IngressEnded(ctx context.Context, info *livekit.IngressInfo) { t.enqueue(func() { t.NotifyEvent(ctx, &livekit.WebhookEvent{ diff --git a/pkg/telemetry/telemetryfakes/fake_telemetry_service.go b/pkg/telemetry/telemetryfakes/fake_telemetry_service.go index f85e03527..fd3ae6ff5 100644 --- a/pkg/telemetry/telemetryfakes/fake_telemetry_service.go +++ b/pkg/telemetry/telemetryfakes/fake_telemetry_service.go @@ -56,6 +56,12 @@ type FakeTelemetryService struct { arg1 context.Context arg2 *livekit.IngressInfo } + IngressUpdatedStub func(context.Context, *livekit.IngressInfo) + ingressUpdatedMutex sync.RWMutex + ingressUpdatedArgsForCall []struct { + arg1 context.Context + arg2 *livekit.IngressInfo + } NotifyEventStub func(context.Context, *livekit.WebhookEvent) notifyEventMutex sync.RWMutex notifyEventArgsForCall []struct { @@ -494,6 +500,39 @@ func (fake *FakeTelemetryService) IngressStartedArgsForCall(i int) (context.Cont return argsForCall.arg1, argsForCall.arg2 } +func (fake *FakeTelemetryService) IngressUpdated(arg1 context.Context, arg2 *livekit.IngressInfo) { + fake.ingressUpdatedMutex.Lock() + fake.ingressUpdatedArgsForCall = append(fake.ingressUpdatedArgsForCall, struct { + arg1 context.Context + arg2 *livekit.IngressInfo + }{arg1, arg2}) + stub := fake.IngressUpdatedStub + fake.recordInvocation("IngressUpdated", []interface{}{arg1, arg2}) + fake.ingressUpdatedMutex.Unlock() + if stub != nil { + fake.IngressUpdatedStub(arg1, arg2) + } +} + +func (fake *FakeTelemetryService) IngressUpdatedCallCount() int { + fake.ingressUpdatedMutex.RLock() + defer fake.ingressUpdatedMutex.RUnlock() + return len(fake.ingressUpdatedArgsForCall) +} + +func (fake *FakeTelemetryService) IngressUpdatedCalls(stub func(context.Context, *livekit.IngressInfo)) { + fake.ingressUpdatedMutex.Lock() + defer fake.ingressUpdatedMutex.Unlock() + fake.IngressUpdatedStub = stub +} + +func (fake *FakeTelemetryService) IngressUpdatedArgsForCall(i int) (context.Context, *livekit.IngressInfo) { + fake.ingressUpdatedMutex.RLock() + defer fake.ingressUpdatedMutex.RUnlock() + argsForCall := fake.ingressUpdatedArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + func (fake *FakeTelemetryService) NotifyEvent(arg1 context.Context, arg2 *livekit.WebhookEvent) { fake.notifyEventMutex.Lock() fake.notifyEventArgsForCall = append(fake.notifyEventArgsForCall, struct { @@ -1318,6 +1357,8 @@ func (fake *FakeTelemetryService) Invocations() map[string][][]interface{} { defer fake.ingressEndedMutex.RUnlock() fake.ingressStartedMutex.RLock() defer fake.ingressStartedMutex.RUnlock() + fake.ingressUpdatedMutex.RLock() + defer fake.ingressUpdatedMutex.RUnlock() fake.notifyEventMutex.RLock() defer fake.notifyEventMutex.RUnlock() fake.participantActiveMutex.RLock() diff --git a/pkg/telemetry/telemetryservice.go b/pkg/telemetry/telemetryservice.go index e304980aa..5b4c4b9c1 100644 --- a/pkg/telemetry/telemetryservice.go +++ b/pkg/telemetry/telemetryservice.go @@ -57,6 +57,7 @@ type TelemetryService interface { IngressCreated(ctx context.Context, info *livekit.IngressInfo) IngressDeleted(ctx context.Context, info *livekit.IngressInfo) IngressStarted(ctx context.Context, info *livekit.IngressInfo) + IngressUpdated(ctx context.Context, info *livekit.IngressInfo) IngressEnded(ctx context.Context, info *livekit.IngressInfo) // helpers From 395f403132497aa66b0d0c46f470ec5c448310b7 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sat, 17 Jun 2023 12:35:29 +0530 Subject: [PATCH 232/324] Small stream allocator tweaks. (#1800) 1. Probe end time needs to include the probe cluster running time also. 2. Apply collapse window only within the sliding window. This is to prevent cases of some old data declaring congestion. For example, an estimate could have fallen 15 seconds ago and there might have been a bunch of estimates at that fallen value. And the whole sliding window could have that value at some point. But, a further drop may trigger congestion detection. But, that might be acting too fast, i. e. on one instance of value fall. Change it so that we detect if there is a fall within the sliding window and apply collapse based on that. --- pkg/sfu/streamallocator/probe_controller.go | 2 +- pkg/sfu/streamallocator/trenddetector.go | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/pkg/sfu/streamallocator/probe_controller.go b/pkg/sfu/streamallocator/probe_controller.go index 6b476b6d5..28fda6ac7 100644 --- a/pkg/sfu/streamallocator/probe_controller.go +++ b/pkg/sfu/streamallocator/probe_controller.go @@ -86,7 +86,7 @@ func (p *ProbeController) ProbeClusterDone(info ProbeClusterInfo, lowestEstimate if queueWait > ProbeSettleWaitMax { queueWait = ProbeSettleWaitMax } - p.probeEndTime = p.lastProbeStartTime.Add(queueWait) + p.probeEndTime = p.lastProbeStartTime.Add(queueWait + info.Duration) p.params.Logger.Infow( "setting probe end time", "probeClusterId", p.probeClusterId, diff --git a/pkg/sfu/streamallocator/trenddetector.go b/pkg/sfu/streamallocator/trenddetector.go index 2c5bf4850..9f96b6298 100644 --- a/pkg/sfu/streamallocator/trenddetector.go +++ b/pkg/sfu/streamallocator/trenddetector.go @@ -49,7 +49,6 @@ type TrendDetector struct { lowestValue int64 highestValue int64 - hasFallen bool lastSampleAt time.Time direction TrendDirection @@ -70,7 +69,6 @@ func (t *TrendDetector) Seed(value int64) { t.values = append(t.values, value) t.lastSampleAt = time.Now() - t.hasFallen = false } func (t *TrendDetector) AddValue(value int64) { @@ -91,22 +89,26 @@ func (t *TrendDetector) AddValue(value int64) { // the reaction is not too fast, i. e. reacting to falling values too quick could mean a lot of re-allocation // resulting in layer switches, key frames and more congestion. // - // But, on the flip side, estimate could fall once or twice withing a sliding window and stay there. - // In those cases, using a collapse window to record value even if it is duplicate. By doing that, + // But, on the flip side, estimate could fall once or twice within a sliding window and stay there. + // In those cases, using a collapse window to record a value even if it is duplicate. By doing that, // a trend could be detected eventually. If will be delayed, but that is fine with slow changing estimates. lastValue := int64(0) if len(t.values) != 0 { lastValue = t.values[len(t.values)-1] } if lastValue == value && t.params.CollapseThreshold > 0 { - if !t.hasFallen || (!t.lastSampleAt.IsZero() && time.Since(t.lastSampleAt) < t.params.CollapseThreshold) { + hasFallen := false + for idx := 1; idx < len(t.values); idx++ { + if t.values[idx] < t.values[idx-1] { + hasFallen = true + break + } + } + if !hasFallen || (!t.lastSampleAt.IsZero() && time.Since(t.lastSampleAt) < t.params.CollapseThreshold) { return } } - if lastValue > value { - t.hasFallen = true - } t.lastSampleAt = time.Now() if len(t.values) == t.params.RequiredSamples { From 2383234f6e56dbfa6399edbab89f8e8c91a4ff84 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sat, 17 Jun 2023 18:56:38 +0530 Subject: [PATCH 233/324] Simplify sliding window collapse. (#1802) * Simplify sliding window collapse. Keep the same value collapsing simple. Add it to sliding window as long as same value is received for longer than collapse threshold. But, add a prune with three conditions to process the siliding window to ensure only valid samples are kept. * flip the order of validity window and same value pruning * increase collapse threshold to 0.5 seconds during non-probe --- pkg/sfu/streamallocator/channelobserver.go | 2 + pkg/sfu/streamallocator/streamallocator.go | 4 +- pkg/sfu/streamallocator/trenddetector.go | 105 +++++++++++++-------- 3 files changed, 71 insertions(+), 40 deletions(-) diff --git a/pkg/sfu/streamallocator/channelobserver.go b/pkg/sfu/streamallocator/channelobserver.go index 1960b6c2d..9c99e90da 100644 --- a/pkg/sfu/streamallocator/channelobserver.go +++ b/pkg/sfu/streamallocator/channelobserver.go @@ -60,6 +60,7 @@ type ChannelObserverParams struct { EstimateRequiredSamples int EstimateDownwardTrendThreshold float64 EstimateCollapseThreshold time.Duration + EstimateValidityWindow time.Duration NackWindowMinDuration time.Duration NackWindowMaxDuration time.Duration NackRatioThreshold float64 @@ -87,6 +88,7 @@ func NewChannelObserver(params ChannelObserverParams, logger logger.Logger) *Cha RequiredSamples: params.EstimateRequiredSamples, DownwardTrendThreshold: params.EstimateDownwardTrendThreshold, CollapseThreshold: params.EstimateCollapseThreshold, + ValidityWindow: params.EstimateValidityWindow, }), nackTracker: NewNackTracker(NackTrackerParams{ Name: params.Name + "-nack", diff --git a/pkg/sfu/streamallocator/streamallocator.go b/pkg/sfu/streamallocator/streamallocator.go index 76bdddb50..dba10d7f2 100644 --- a/pkg/sfu/streamallocator/streamallocator.go +++ b/pkg/sfu/streamallocator/streamallocator.go @@ -46,6 +46,7 @@ var ( EstimateRequiredSamples: 3, EstimateDownwardTrendThreshold: 0.0, EstimateCollapseThreshold: 0, + EstimateValidityWindow: 10 * time.Second, NackWindowMinDuration: 500 * time.Millisecond, NackWindowMaxDuration: 1 * time.Second, NackRatioThreshold: 0.04, @@ -55,7 +56,8 @@ var ( Name: "non-probe", EstimateRequiredSamples: 8, EstimateDownwardTrendThreshold: -0.5, - EstimateCollapseThreshold: 250 * time.Millisecond, + EstimateCollapseThreshold: 500 * time.Millisecond, + EstimateValidityWindow: 10 * time.Second, NackWindowMinDuration: 1 * time.Second, NackWindowMaxDuration: 2 * time.Second, NackRatioThreshold: 0.08, diff --git a/pkg/sfu/streamallocator/trenddetector.go b/pkg/sfu/streamallocator/trenddetector.go index 9f96b6298..2ea7ba0e2 100644 --- a/pkg/sfu/streamallocator/trenddetector.go +++ b/pkg/sfu/streamallocator/trenddetector.go @@ -32,12 +32,20 @@ func (t TrendDirection) String() string { // ------------------------------------------------ +type trendDetectorSample struct { + value int64 + at time.Time +} + +// ------------------------------------------------ + type TrendDetectorParams struct { Name string Logger logger.Logger RequiredSamples int DownwardTrendThreshold float64 CollapseThreshold time.Duration + ValidityWindow time.Duration } type TrendDetector struct { @@ -45,12 +53,10 @@ type TrendDetector struct { startTime time.Time numSamples int - values []int64 + samples []trendDetectorSample lowestValue int64 highestValue int64 - lastSampleAt time.Time - direction TrendDirection } @@ -63,12 +69,11 @@ func NewTrendDetector(params TrendDetectorParams) *TrendDetector { } func (t *TrendDetector) Seed(value int64) { - if len(t.values) != 0 { + if len(t.samples) != 0 { return } - t.values = append(t.values, value) - t.lastSampleAt = time.Now() + t.samples = append(t.samples, trendDetectorSample{value: value, at: time.Now()}) } func (t *TrendDetector) AddValue(value int64) { @@ -92,30 +97,16 @@ func (t *TrendDetector) AddValue(value int64) { // But, on the flip side, estimate could fall once or twice within a sliding window and stay there. // In those cases, using a collapse window to record a value even if it is duplicate. By doing that, // a trend could be detected eventually. If will be delayed, but that is fine with slow changing estimates. - lastValue := int64(0) - if len(t.values) != 0 { - lastValue = t.values[len(t.values)-1] + var lastSample *trendDetectorSample + if len(t.samples) != 0 { + lastSample = &t.samples[len(t.samples)-1] } - if lastValue == value && t.params.CollapseThreshold > 0 { - hasFallen := false - for idx := 1; idx < len(t.values); idx++ { - if t.values[idx] < t.values[idx-1] { - hasFallen = true - break - } - } - if !hasFallen || (!t.lastSampleAt.IsZero() && time.Since(t.lastSampleAt) < t.params.CollapseThreshold) { - return - } + if lastSample != nil && lastSample.value == value && t.params.CollapseThreshold > 0 && time.Since(lastSample.at) < t.params.CollapseThreshold { + return } - t.lastSampleAt = time.Now() - - if len(t.values) == t.params.RequiredSamples { - t.values = t.values[1:] - } - t.values = append(t.values, value) - + t.samples = append(t.samples, trendDetectorSample{value: value, at: time.Now()}) + t.prune() t.updateDirection() } @@ -127,10 +118,6 @@ func (t *TrendDetector) GetHighest() int64 { return t.highestValue } -func (t *TrendDetector) GetValues() []int64 { - return t.values -} - func (t *TrendDetector) GetDirection() TrendDirection { return t.direction } @@ -141,17 +128,57 @@ func (t *TrendDetector) ToString() string { return fmt.Sprintf("n: %s, t: %+v|%+v|%.2fs, v: %d|%d|%d|%+v|%.2f", t.params.Name, t.startTime.Format(time.UnixDate), now.Format(time.UnixDate), elapsed, - t.numSamples, t.lowestValue, t.highestValue, t.values, kendallsTau(t.values)) + t.numSamples, t.lowestValue, t.highestValue, t.samples, kendallsTau(t.samples), + ) +} + +func (t *TrendDetector) prune() { + // prune based on a few rules + // 1. If there are more than required samples + if len(t.samples) > t.params.RequiredSamples { + t.samples = t.samples[len(t.samples)-t.params.RequiredSamples:] + } + + // 2. drop samples that are too old + if len(t.samples) != 0 && t.params.ValidityWindow > 0 { + cutoffTime := time.Now().Add(-t.params.ValidityWindow) + cutoffIndex := -1 + for i := 0; i < len(t.samples); i++ { + if t.samples[i].at.After(cutoffTime) { + cutoffIndex = i + break + } + } + if cutoffIndex >= 0 { + t.samples = t.samples[cutoffIndex:] + } + } + + // 3. If all sample values are same, collapse to just the last one + if len(t.samples) != 0 { + sameValue := true + firstValue := t.samples[0].value + for i := 0; i < len(t.samples); i++ { + if t.samples[i].value != firstValue { + sameValue = false + break + } + } + + if sameValue { + t.samples = t.samples[len(t.samples)-1:] + } + } } func (t *TrendDetector) updateDirection() { - if len(t.values) < t.params.RequiredSamples { + if len(t.samples) < t.params.RequiredSamples { t.direction = TrendDirectionNeutral return } // using Kendall's Tau to find trend - kt := kendallsTau(t.values) + kt := kendallsTau(t.samples) t.direction = TrendDirectionNeutral switch { @@ -164,15 +191,15 @@ func (t *TrendDetector) updateDirection() { // ------------------------------------------------ -func kendallsTau(values []int64) float64 { +func kendallsTau(samples []trendDetectorSample) float64 { concordantPairs := 0 discordantPairs := 0 - for i := 0; i < len(values)-1; i++ { - for j := i + 1; j < len(values); j++ { - if values[i] < values[j] { + for i := 0; i < len(samples)-1; i++ { + for j := i + 1; j < len(samples); j++ { + if samples[i].value < samples[j].value { concordantPairs++ - } else if values[i] > values[j] { + } else if samples[i].value > samples[j].value { discordantPairs++ } } From 40f5902d36fbfe5cad52057635dd13be393a7bbd Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sat, 17 Jun 2023 21:02:02 +0530 Subject: [PATCH 234/324] Consistently use connID as log tag (#1801) --- pkg/routing/redisrouter.go | 5 +++-- pkg/routing/signal.go | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/routing/redisrouter.go b/pkg/routing/redisrouter.go index d31a13638..f10f77859 100644 --- a/pkg/routing/redisrouter.go +++ b/pkg/routing/redisrouter.go @@ -160,8 +160,9 @@ func (r *RedisRouter) StartParticipantSignal(ctx context.Context, roomName livek return } - // create a new connection id - connectionID = livekit.ConnectionID(utils.NewGuid("CO_")) + if connectionID == "" { + connectionID = livekit.ConnectionID(utils.NewGuid("CO_")) + } pKey := ParticipantKeyLegacy(roomName, pi.Identity) pKeyB62 := ParticipantKey(roomName, pi.Identity) diff --git a/pkg/routing/signal.go b/pkg/routing/signal.go index fad31191b..aa94b12dc 100644 --- a/pkg/routing/signal.go +++ b/pkg/routing/signal.go @@ -80,7 +80,7 @@ func (r *signalClient) StartParticipantSignal( "room", roomName, "reqNodeID", nodeID, "participant", pi.Identity, - "connectionID", connectionID, + "connID", connectionID, ) l.Debugw("starting signal connection") From a6d091a8107cf36d3dda847bba5842dfaaa2d5a3 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Sun, 18 Jun 2023 18:13:34 -0700 Subject: [PATCH 235/324] update protocol (#1803) --- go.mod | 2 +- go.sum | 2 ++ pkg/rtc/participant.go | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 417fc48c6..de8d5dbb8 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230612070454-d5299b956135 - github.com/livekit/protocol v1.5.8-0.20230611030650-7d128913f3bd + github.com/livekit/protocol v1.5.8-0.20230619005042-089430752e03 github.com/livekit/psrpc v0.3.1 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 diff --git a/go.sum b/go.sum index 331cc5c2b..316da55a0 100644 --- a/go.sum +++ b/go.sum @@ -124,6 +124,8 @@ github.com/livekit/mediatransportutil v0.0.0-20230612070454-d5299b956135 h1:lWYb github.com/livekit/mediatransportutil v0.0.0-20230612070454-d5299b956135/go.mod h1:MRc0zSOSzXuFt0X218SgabzlaKevkvCckPgBEoHYc34= github.com/livekit/protocol v1.5.8-0.20230611030650-7d128913f3bd h1:pHX/MdFniQyvoQR55lOUeV8SrYMnBYmEb6cxjfzzLyg= github.com/livekit/protocol v1.5.8-0.20230611030650-7d128913f3bd/go.mod h1:Y+jl7rD7u8ZMfIUzGr41DU7G5j+34rtgefTCrD/ApZc= +github.com/livekit/protocol v1.5.8-0.20230619005042-089430752e03 h1:Ma+BKzQHCZhXeklDN72ahkERSixANOwlargLT4mrsPo= +github.com/livekit/protocol v1.5.8-0.20230619005042-089430752e03/go.mod h1:B6hJiuXT84dHsUgaKHBo+ZLPX4XhklptYA2UbANSiNg= github.com/livekit/psrpc v0.3.1 h1:KfylgJHvoLQcc22t/oflwMOeSnx0c14G7cWsS+9MYS4= github.com/livekit/psrpc v0.3.1/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 112e660c6..79e9dafb5 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -2093,7 +2093,7 @@ func (p *ParticipantImpl) onPublicationError(trackID livekit.TrackID) { } func (p *ParticipantImpl) onSubscriptionError(trackID livekit.TrackID, fatal bool, err error) { - signalErr := livekit.SubscriptionError_SE_UNKOWN + signalErr := livekit.SubscriptionError_SE_UNKNOWN switch { case errors.Is(err, webrtc.ErrUnsupportedCodec): signalErr = livekit.SubscriptionError_SE_CODEC_UNSUPPORTED From 9e3f30d457f809412cc72dfc6a0d6f7a4697ce80 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Sun, 18 Jun 2023 20:37:40 -0700 Subject: [PATCH 236/324] go mod tidy (#1805) --- go.mod | 4 +++- go.sum | 7 +++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index de8d5dbb8..722aa7538 100644 --- a/go.mod +++ b/go.mod @@ -45,6 +45,7 @@ require ( github.com/urfave/cli/v2 v2.25.6 github.com/urfave/negroni/v3 v3.0.0 go.uber.org/atomic v1.11.0 + golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 golang.org/x/sync v0.3.0 google.golang.org/protobuf v1.30.0 gopkg.in/yaml.v3 v3.0.1 @@ -68,6 +69,7 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.4 // indirect github.com/josharian/native v1.1.0 // indirect github.com/klauspost/compress v1.16.5 // indirect + github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/lithammer/shortuuid/v4 v4.0.0 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect @@ -89,10 +91,10 @@ require ( github.com/prometheus/procfs v0.10.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect + github.com/zeebo/xxh3 v1.0.2 // indirect go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.24.0 // indirect golang.org/x/crypto v0.10.0 // indirect - golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 // indirect golang.org/x/mod v0.11.0 // indirect golang.org/x/net v0.11.0 // indirect golang.org/x/sys v0.9.0 // indirect diff --git a/go.sum b/go.sum index 316da55a0..088209c9c 100644 --- a/go.sum +++ b/go.sum @@ -108,6 +108,8 @@ github.com/jxskiss/base62 v1.1.0 h1:A5zbF8v8WXx2xixnAKD2w+abC+sIzYJX+nxmhA6HWFw= github.com/jxskiss/base62 v1.1.0/go.mod h1:HhWAlUXvxKThfOlZbcuFzsqwtF5TcqS9ru3y5GfjWAc= github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= @@ -122,8 +124,6 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20230612070454-d5299b956135 h1:lWYbsondvqG69czxoACDwaJ/BoyD57BahCo70ZH+m4U= github.com/livekit/mediatransportutil v0.0.0-20230612070454-d5299b956135/go.mod h1:MRc0zSOSzXuFt0X218SgabzlaKevkvCckPgBEoHYc34= -github.com/livekit/protocol v1.5.8-0.20230611030650-7d128913f3bd h1:pHX/MdFniQyvoQR55lOUeV8SrYMnBYmEb6cxjfzzLyg= -github.com/livekit/protocol v1.5.8-0.20230611030650-7d128913f3bd/go.mod h1:Y+jl7rD7u8ZMfIUzGr41DU7G5j+34rtgefTCrD/ApZc= github.com/livekit/protocol v1.5.8-0.20230619005042-089430752e03 h1:Ma+BKzQHCZhXeklDN72ahkERSixANOwlargLT4mrsPo= github.com/livekit/protocol v1.5.8-0.20230619005042-089430752e03/go.mod h1:B6hJiuXT84dHsUgaKHBo+ZLPX4XhklptYA2UbANSiNg= github.com/livekit/psrpc v0.3.1 h1:KfylgJHvoLQcc22t/oflwMOeSnx0c14G7cWsS+9MYS4= @@ -270,6 +270,9 @@ github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRT github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= From f11a7a229f4b887d07c6937ae8ec6863116b60ab Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 19 Jun 2023 16:40:05 +0530 Subject: [PATCH 237/324] Remove unnecessary check (#1806) --- pkg/routing/redisrouter.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/routing/redisrouter.go b/pkg/routing/redisrouter.go index f10f77859..5ad1a5c1a 100644 --- a/pkg/routing/redisrouter.go +++ b/pkg/routing/redisrouter.go @@ -160,9 +160,7 @@ func (r *RedisRouter) StartParticipantSignal(ctx context.Context, roomName livek return } - if connectionID == "" { - connectionID = livekit.ConnectionID(utils.NewGuid("CO_")) - } + connectionID = livekit.ConnectionID(utils.NewGuid("CO_")) pKey := ParticipantKeyLegacy(roomName, pi.Identity) pKeyB62 := ParticipantKey(roomName, pi.Identity) From 27051e99993cb2b0960a9dd30d5dc9f2b184c5c8 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 20 Jun 2023 11:58:01 +0530 Subject: [PATCH 238/324] It is possible that pipe is closed before blank frame send, do not warn (#1807) --- pkg/sfu/downtrack.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index d8850ff64..03eaa1a24 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -1194,7 +1194,9 @@ func (d *DownTrack) writeBlankFrameRTP(duration float32, generation uint32) chan pktSize, err := writeBlankFrame(&hdr, frameEndNeeded) if err != nil { - d.logger.Warnw("could not write blank frame", err) + if err != io.ErrClosedPipe { + d.logger.Warnw("could not write blank frame", err) + } close(done) return } From 583648a1eda0c3fe1c045619915179f8bd91bf3b Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 20 Jun 2023 19:06:01 +0530 Subject: [PATCH 239/324] Avoid closure to reduce life span of objects. (#1809) A subscription in subscription manager could live till the source track goes away even though the participant with that subscription is long gone due to closure on source track removal. Handle it by using trackID to look up on source track removal. Also, logging SDPs when a negotiation failure happens to check if there are any mismatches. --- pkg/rtc/subscriptionmanager.go | 29 ++++++++++++++++++++--------- pkg/rtc/transport.go | 7 +++++++ 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/pkg/rtc/subscriptionmanager.go b/pkg/rtc/subscriptionmanager.go index 18a3581ad..468ea678a 100644 --- a/pkg/rtc/subscriptionmanager.go +++ b/pkg/rtc/subscriptionmanager.go @@ -436,7 +436,8 @@ func (m *SubscriptionManager) subscribe(s *trackSubscription) error { return ErrSubscriptionLimitExceeded } - res := m.params.TrackResolver(m.params.Participant.Identity(), s.trackID) + trackID := s.trackID + res := m.params.TrackResolver(m.params.Participant.Identity(), trackID) s.logger.Debugw("resolved track", "result", res) if res.TrackChangedNotifier != nil && s.setChangedNotifier(res.TrackChangedNotifier) { @@ -444,18 +445,18 @@ func (m *SubscriptionManager) subscribe(s *trackSubscription) error { // we set the observer before checking for existence of track, so that we may get notified // when the track becomes available res.TrackChangedNotifier.AddObserver(string(m.params.Participant.ID()), func() { - m.queueReconcile(s.trackID) + m.queueReconcile(trackID) }) } if res.TrackRemovedNotifier != nil && s.setRemovedNotifier(res.TrackRemovedNotifier) { res.TrackRemovedNotifier.AddObserver(string(m.params.Participant.ID()), func() { // re-resolve the track in case the same track had been re-published - res := m.params.TrackResolver(m.params.Participant.Identity(), s.trackID) + res := m.params.TrackResolver(m.params.Participant.Identity(), trackID) if res.Track != nil { // do not unsubscribe, track is still available return } - s.handleSourceTrackRemoved() + m.handleSourceTrackRemoved(trackID) }) } @@ -474,7 +475,7 @@ func (m *SubscriptionManager) subscribe(s *trackSubscription) error { // that we discover permissions were denied permChanged := s.setHasPermission(res.HasPermission) if permChanged { - m.params.Participant.SubscriptionPermissionUpdate(s.getPublisherID(), s.trackID, res.HasPermission) + m.params.Participant.SubscriptionPermissionUpdate(s.getPublisherID(), trackID, res.HasPermission) } if !res.HasPermission { return ErrNoTrackPermission @@ -493,8 +494,8 @@ func (m *SubscriptionManager) subscribe(s *trackSubscription) error { if err != nil { s.logger.Infow("failed to bind track", "err", err) s.maybeRecordError(m.params.Telemetry, m.params.Participant.ID(), err, true) - m.UnsubscribeFromTrack(s.trackID) - m.params.OnSubscriptionError(s.trackID, false, err) + m.UnsubscribeFromTrack(trackID) + m.params.OnSubscriptionError(trackID, false, err) return } s.setBound() @@ -516,7 +517,7 @@ func (m *SubscriptionManager) subscribe(s *trackSubscription) error { go m.params.OnTrackSubscribed(subTrack) } - m.params.Logger.Debugw("subscribed to track", "trackID", s.trackID, "subscribedAudioCount", m.subscribedAudioCount.Load(), "subscribedVideoCount", m.subscribedVideoCount.Load()) + m.params.Logger.Debugw("subscribed to track", "trackID", trackID, "subscribedAudioCount", m.subscribedAudioCount.Load(), "subscribedVideoCount", m.subscribedVideoCount.Load()) // add mark the participant as someone we've subscribed to firstSubscribe := false @@ -529,7 +530,7 @@ func (m *SubscriptionManager) subscribe(s *trackSubscription) error { m.subscribedTo[publisherID] = pTracks firstSubscribe = true } - pTracks[s.trackID] = struct{}{} + pTracks[trackID] = struct{}{} m.lock.Unlock() if changedCB != nil && firstSubscribe { @@ -558,6 +559,16 @@ func (m *SubscriptionManager) unsubscribe(s *trackSubscription) error { return nil } +func (m *SubscriptionManager) handleSourceTrackRemoved(trackID livekit.TrackID) { + m.lock.Lock() + sub := m.subscriptions[trackID] + m.lock.Unlock() + + if sub != nil { + sub.handleSourceTrackRemoved() + } +} + // DownTrack closing is how the publisher signifies that the subscription is no longer fulfilled // this could be due to a few reasons: // - subscriber-initiated unsubscribe diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index c1913bf7a..a57910560 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -1610,6 +1610,13 @@ func (t *PCTransport) setupSignalStateCheckTimer() { failed := t.negotiationState != NegotiationStateNone if t.negotiateCounter.Load() == negotiateVersion && failed { + t.params.Logger.Infow( + "negotiation timed out", + "localCurrent", t.pc.CurrentLocalDescription(), + "localPending", t.pc.PendingLocalDescription(), + "remoteCurrent", t.pc.CurrentRemoteDescription(), + "remotePending", t.pc.PendingRemoteDescription(), + ) if onNegotiationFailed := t.getOnNegotiationFailed(); onNegotiationFailed != nil { onNegotiationFailed() } From 84994b39abef4306285d0150da897d77de089c68 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 21 Jun 2023 11:35:38 +0530 Subject: [PATCH 240/324] Make the samples string more readable. (#1810) --- pkg/sfu/streamallocator/trenddetector.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/pkg/sfu/streamallocator/trenddetector.go b/pkg/sfu/streamallocator/trenddetector.go index 2ea7ba0e2..2a5281230 100644 --- a/pkg/sfu/streamallocator/trenddetector.go +++ b/pkg/sfu/streamallocator/trenddetector.go @@ -125,10 +125,23 @@ func (t *TrendDetector) GetDirection() TrendDirection { func (t *TrendDetector) ToString() string { now := time.Now() elapsed := now.Sub(t.startTime).Seconds() - return fmt.Sprintf("n: %s, t: %+v|%+v|%.2fs, v: %d|%d|%d|%+v|%.2f", + samplesStr := "" + if len(t.samples) > 0 { + firstTime := t.samples[0].at + samplesStr += "[" + for i, sample := range t.samples { + suffix := ", " + if i == len(t.samples)-1 { + suffix = "" + } + samplesStr += fmt.Sprintf("%d(%d)%s", sample.value, sample.at.Sub(firstTime).Milliseconds(), suffix) + } + samplesStr += "]" + } + return fmt.Sprintf("n: %s, t: %+v|%+v|%.2fs, v: %d|%d|%d|%s|%.2f", t.params.Name, t.startTime.Format(time.UnixDate), now.Format(time.UnixDate), elapsed, - t.numSamples, t.lowestValue, t.highestValue, t.samples, kendallsTau(t.samples), + t.numSamples, t.lowestValue, t.highestValue, samplesStr, kendallsTau(t.samples), ) } From c12e15ff522a4a19d73bb9bcbaceaa01faa189c2 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 21 Jun 2023 00:09:43 -0700 Subject: [PATCH 241/324] Update module github.com/urfave/cli/v2 to v2.25.7 (#1804) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 722aa7538..f45065249 100644 --- a/go.mod +++ b/go.mod @@ -42,7 +42,7 @@ require ( github.com/thoas/go-funk v0.9.3 github.com/twitchtv/twirp v8.1.3+incompatible github.com/ua-parser/uap-go v0.0.0-20211112212520-00c877edfe0f - github.com/urfave/cli/v2 v2.25.6 + github.com/urfave/cli/v2 v2.25.7 github.com/urfave/negroni/v3 v3.0.0 go.uber.org/atomic v1.11.0 golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 diff --git a/go.sum b/go.sum index 088209c9c..886ed55c3 100644 --- a/go.sum +++ b/go.sum @@ -262,8 +262,8 @@ github.com/twitchtv/twirp v8.1.3+incompatible h1:+F4TdErPgSUbMZMwp13Q/KgDVuI7HJX github.com/twitchtv/twirp v8.1.3+incompatible/go.mod h1:RRJoFSAmTEh2weEqWtpPE3vFK5YBhA6bqp2l1kfCC5A= github.com/ua-parser/uap-go v0.0.0-20211112212520-00c877edfe0f h1:A+MmlgpvrHLeUP8dkBVn4Pnf5Bp5Yk2OALm7SEJLLE8= github.com/ua-parser/uap-go v0.0.0-20211112212520-00c877edfe0f/go.mod h1:OBcG9bn7sHtXgarhUEb3OfCnNsgtGnkVf41ilSZ3K3E= -github.com/urfave/cli/v2 v2.25.6 h1:yuSkgDSZfH3L1CjF2/5fNNg2KbM47pY2EvjBq4ESQnU= -github.com/urfave/cli/v2 v2.25.6/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= +github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= +github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/urfave/negroni/v3 v3.0.0 h1:Vo8CeZfu1lFR9gW8GnAb6dOGCJyijfil9j/jKKc/JhU= github.com/urfave/negroni/v3 v3.0.0/go.mod h1:jWvnX03kcSjDBl/ShB0iHvx5uOs7mAzZXW+JvJ5XYAs= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= From 24380584749a0bbaf788496ffcbfe604e5c7ed17 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 21 Jun 2023 14:11:17 +0530 Subject: [PATCH 242/324] Drop error logs due to pipe close (#1813) --- pkg/rtc/participant.go | 9 +++++---- pkg/service/rtcservice.go | 2 +- pkg/sfu/downtrack.go | 10 +--------- 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 79e9dafb5..e9bf6bbc8 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -2,7 +2,6 @@ package rtc import ( "context" - "io" "os" "strconv" "strings" @@ -1390,7 +1389,7 @@ func (p *ParticipantImpl) subscriberRTCPWorker() { pkts = append(pkts, &rtcp.SourceDescription{Chunks: sd}) } if err := p.TransportManager.WriteSubscriberRTCP(pkts); err != nil { - if err == io.EOF || err == io.ErrClosedPipe { + if IsEOF(err) { return } p.params.Logger.Errorw("could not send down track reports", err) @@ -1407,7 +1406,7 @@ func (p *ParticipantImpl) subscriberRTCPWorker() { pkts = append(pkts, &rtcp.SourceDescription{Chunks: sd}) } if err := p.TransportManager.WriteSubscriberRTCP(pkts); err != nil { - if err == io.EOF || err == io.ErrClosedPipe { + if IsEOF(err) { return } p.params.Logger.Errorw("could not send down track reports", err) @@ -1980,7 +1979,9 @@ func (p *ParticipantImpl) publisherRTCPWorker() { } if err := p.TransportManager.WritePublisherRTCP(pkts); err != nil { - p.params.Logger.Errorw("could not write RTCP to participant", err) + if !IsEOF(err) { + p.params.Logger.Errorw("could not write RTCP to participant", err) + } } } } diff --git a/pkg/service/rtcservice.go b/pkg/service/rtcservice.go index 5af97a78c..653ff80b8 100644 --- a/pkg/service/rtcservice.go +++ b/pkg/service/rtcservice.go @@ -347,7 +347,7 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { websocket.IsCloseError(err, websocket.CloseAbnormalClosure, websocket.CloseGoingAway, websocket.CloseNormalClosure, websocket.CloseNoStatusReceived) { pLogger.Infow("exit ws read loop for closed connection", "connID", cr.ConnectionID, "wsError", err) } else { - pLogger.Errorw("error reading from websocket", err) + pLogger.Errorw("error reading from websocket", err, "connID", cr.ConnectionID) } return } diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 03eaa1a24..848595ae4 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -221,9 +221,6 @@ type DownTrack struct { deltaStatsSnapshotId uint32 deltaStatsOverriddenSnapshotId uint32 - // for throttling error logs - writeIOErrors atomic.Uint32 - isNACKThrottled atomic.Bool activePaddingOnMuteUpTrack atomic.Bool @@ -610,12 +607,7 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { _, err = d.writeStream.WriteRTP(hdr, payload) if err != nil { - if errors.Is(err, io.ErrClosedPipe) { - writeIOErrors := d.writeIOErrors.Inc() - if (writeIOErrors % 100) == 1 { - d.logger.Errorw("write rtp packet failed", err, "count", writeIOErrors) - } - } else { + if !errors.Is(err, io.ErrClosedPipe) { d.logger.Errorw("write rtp packet failed", err) } return err From 00558dee5c3d403c731a5db373477e05d5a6fcfb Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 22 Jun 2023 10:09:10 +0530 Subject: [PATCH 243/324] Close participant on full reconnect. (#1818) * Close participant on full reconnect. A full reconnect == irrecoverable error. Participant cannot continue. So, close the participant when issuing a full reconnect. That should prevent subscription manager reconcile till the participant is finally closed down when participant is stale. * format --- pkg/rtc/participant.go | 20 +++---- pkg/rtc/room.go | 10 ++-- pkg/rtc/types/interfaces.go | 2 +- .../typesfakes/fake_local_participant.go | 18 +++--- pkg/rtc/types/typesfakes/fake_participant.go | 18 +++--- pkg/service/roommanager.go | 55 +++++++++++++++---- 6 files changed, 79 insertions(+), 44 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index e9bf6bbc8..889619c76 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -650,13 +650,13 @@ func (p *ParticipantImpl) Start() { }) } -func (p *ParticipantImpl) Close(sendLeave bool, reason types.ParticipantCloseReason) error { +func (p *ParticipantImpl) Close(sendLeave bool, reason types.ParticipantCloseReason, isExpectedToResume bool) error { if p.isClosed.Swap(true) { // already closed return nil } - p.params.Logger.Infow("participant closing", "sendLeave", sendLeave, "reason", reason.String()) + p.params.Logger.Infow("participant closing", "sendLeave", sendLeave, "reason", reason.String(), "isExpectedToResume", isExpectedToResume) p.clearDisconnectTimer() p.clearMigrationTimer() @@ -680,10 +680,10 @@ func (p *ParticipantImpl) Close(sendLeave bool, reason types.ParticipantCloseRea p.pendingTracksLock.Unlock() for _, t := range closeMutedTrack { - t.Close(!sendLeave) + t.Close(isExpectedToResume) } - p.UpTrackManager.Close(!sendLeave) + p.UpTrackManager.Close(isExpectedToResume) p.updateState(livekit.ParticipantInfo_DISCONNECTED) @@ -699,7 +699,7 @@ func (p *ParticipantImpl) Close(sendLeave bool, reason types.ParticipantCloseRea // Close peer connections without blocking participant Close. If peer connections are gathering candidates // Close will block. go func() { - p.SubscriptionManager.Close(!sendLeave) + p.SubscriptionManager.Close(isExpectedToResume) p.TransportManager.Close() }() @@ -760,7 +760,7 @@ func (p *ParticipantImpl) MaybeStartMigration(force bool, onStart func()) bool { p.migrationTimer = time.AfterFunc(migrationWaitDuration, func() { p.clearMigrationTimer() - if p.isClosed.Load() || p.IsDisconnected() { + if p.IsClosed() || p.IsDisconnected() { return } // TODO: change to debug once we are confident @@ -1338,11 +1338,11 @@ func (p *ParticipantImpl) setupDisconnectTimer() { p.disconnectTimer = time.AfterFunc(disconnectCleanupDuration, func() { p.clearDisconnectTimer() - if p.isClosed.Load() || p.IsDisconnected() { + if p.IsClosed() || p.IsDisconnected() { return } p.params.Logger.Infow("closing disconnected participant") - _ = p.Close(true, types.ParticipantCloseReasonPeerConnectionDisconnected) + _ = p.Close(true, types.ParticipantCloseReasonPeerConnectionDisconnected, false) }) p.lock.Unlock() } @@ -2082,8 +2082,8 @@ func (p *ParticipantImpl) IssueFullReconnect(reason types.ParticipantCloseReason } p.CloseSignalConnection(scr) - // on a full reconnect, no need to supervise this participant anymore - p.supervisor.Stop() + // a full reconnect == client should connect back with a new session, close current one + p.Close(false, reason, false) } func (p *ParticipantImpl) onPublicationError(trackID livekit.TrackID) { diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index 8d343a98f..df9db3764 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -487,7 +487,7 @@ func (r *Room) RemoveParticipant(identity livekit.ParticipantIdentity, pID livek // close participant as well r.Logger.Debugw("closing participant for removal", "pID", p.ID(), "participant", p.Identity()) - _ = p.Close(true, reason) + _ = p.Close(true, reason, false) r.leftAt.Store(time.Now().Unix()) @@ -622,7 +622,7 @@ func (r *Room) Close() { r.lock.Unlock() r.Logger.Infow("closing room") for _, p := range r.GetParticipants() { - _ = p.Close(true, types.ParticipantCloseReasonRoomClose) + _ = p.Close(true, types.ParticipantCloseReasonRoomClose, false) } r.protoProxy.Stop() if r.onClose != nil { @@ -705,18 +705,18 @@ func (r *Room) SimulateScenario(participant types.LocalParticipant, simulateScen case *livekit.SimulateScenario_Migration: r.Logger.Infow("simulating migration", "participant", participant.Identity()) // drop participant without necessarily cleaning up - if err := participant.Close(false, types.ParticipantCloseReasonSimulateMigration); err != nil { + if err := participant.Close(false, types.ParticipantCloseReasonSimulateMigration, true); err != nil { return err } case *livekit.SimulateScenario_NodeFailure: r.Logger.Infow("simulating node failure", "participant", participant.Identity()) // drop participant without necessarily cleaning up - if err := participant.Close(false, types.ParticipantCloseReasonSimulateNodeFailure); err != nil { + if err := participant.Close(false, types.ParticipantCloseReasonSimulateNodeFailure, true); err != nil { return err } case *livekit.SimulateScenario_ServerLeave: r.Logger.Infow("simulating server leave", "participant", participant.Identity()) - if err := participant.Close(true, types.ParticipantCloseReasonSimulateServerLeave); err != nil { + if err := participant.Close(true, types.ParticipantCloseReasonSimulateServerLeave, false); err != nil { return err } case *livekit.SimulateScenario_SwitchCandidateProtocol: diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index d9b887041..7370bfa1b 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -237,7 +237,7 @@ type Participant interface { IsRecorder() bool Start() - Close(sendLeave bool, reason ParticipantCloseReason) error + Close(sendLeave bool, reason ParticipantCloseReason, isExpectedToResume bool) error SubscriptionPermission() (*livekit.SubscriptionPermission, utils.TimedVersion) diff --git a/pkg/rtc/types/typesfakes/fake_local_participant.go b/pkg/rtc/types/typesfakes/fake_local_participant.go index 235994fa9..f6882fe93 100644 --- a/pkg/rtc/types/typesfakes/fake_local_participant.go +++ b/pkg/rtc/types/typesfakes/fake_local_participant.go @@ -119,11 +119,12 @@ type FakeLocalParticipant struct { claimGrantsReturnsOnCall map[int]struct { result1 *auth.ClaimGrants } - CloseStub func(bool, types.ParticipantCloseReason) error + CloseStub func(bool, types.ParticipantCloseReason, bool) error closeMutex sync.RWMutex closeArgsForCall []struct { arg1 bool arg2 types.ParticipantCloseReason + arg3 bool } closeReturns struct { result1 error @@ -1368,19 +1369,20 @@ func (fake *FakeLocalParticipant) ClaimGrantsReturnsOnCall(i int, result1 *auth. }{result1} } -func (fake *FakeLocalParticipant) Close(arg1 bool, arg2 types.ParticipantCloseReason) error { +func (fake *FakeLocalParticipant) Close(arg1 bool, arg2 types.ParticipantCloseReason, arg3 bool) error { fake.closeMutex.Lock() ret, specificReturn := fake.closeReturnsOnCall[len(fake.closeArgsForCall)] fake.closeArgsForCall = append(fake.closeArgsForCall, struct { arg1 bool arg2 types.ParticipantCloseReason - }{arg1, arg2}) + arg3 bool + }{arg1, arg2, arg3}) stub := fake.CloseStub fakeReturns := fake.closeReturns - fake.recordInvocation("Close", []interface{}{arg1, arg2}) + fake.recordInvocation("Close", []interface{}{arg1, arg2, arg3}) fake.closeMutex.Unlock() if stub != nil { - return stub(arg1, arg2) + return stub(arg1, arg2, arg3) } if specificReturn { return ret.result1 @@ -1394,17 +1396,17 @@ func (fake *FakeLocalParticipant) CloseCallCount() int { return len(fake.closeArgsForCall) } -func (fake *FakeLocalParticipant) CloseCalls(stub func(bool, types.ParticipantCloseReason) error) { +func (fake *FakeLocalParticipant) CloseCalls(stub func(bool, types.ParticipantCloseReason, bool) error) { fake.closeMutex.Lock() defer fake.closeMutex.Unlock() fake.CloseStub = stub } -func (fake *FakeLocalParticipant) CloseArgsForCall(i int) (bool, types.ParticipantCloseReason) { +func (fake *FakeLocalParticipant) CloseArgsForCall(i int) (bool, types.ParticipantCloseReason, bool) { fake.closeMutex.RLock() defer fake.closeMutex.RUnlock() argsForCall := fake.closeArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 } func (fake *FakeLocalParticipant) CloseReturns(result1 error) { diff --git a/pkg/rtc/types/typesfakes/fake_participant.go b/pkg/rtc/types/typesfakes/fake_participant.go index c32b8a6ed..a660644cb 100644 --- a/pkg/rtc/types/typesfakes/fake_participant.go +++ b/pkg/rtc/types/typesfakes/fake_participant.go @@ -20,11 +20,12 @@ type FakeParticipant struct { canSkipBroadcastReturnsOnCall map[int]struct { result1 bool } - CloseStub func(bool, types.ParticipantCloseReason) error + CloseStub func(bool, types.ParticipantCloseReason, bool) error closeMutex sync.RWMutex closeArgsForCall []struct { arg1 bool arg2 types.ParticipantCloseReason + arg3 bool } closeReturns struct { result1 error @@ -260,19 +261,20 @@ func (fake *FakeParticipant) CanSkipBroadcastReturnsOnCall(i int, result1 bool) }{result1} } -func (fake *FakeParticipant) Close(arg1 bool, arg2 types.ParticipantCloseReason) error { +func (fake *FakeParticipant) Close(arg1 bool, arg2 types.ParticipantCloseReason, arg3 bool) error { fake.closeMutex.Lock() ret, specificReturn := fake.closeReturnsOnCall[len(fake.closeArgsForCall)] fake.closeArgsForCall = append(fake.closeArgsForCall, struct { arg1 bool arg2 types.ParticipantCloseReason - }{arg1, arg2}) + arg3 bool + }{arg1, arg2, arg3}) stub := fake.CloseStub fakeReturns := fake.closeReturns - fake.recordInvocation("Close", []interface{}{arg1, arg2}) + fake.recordInvocation("Close", []interface{}{arg1, arg2, arg3}) fake.closeMutex.Unlock() if stub != nil { - return stub(arg1, arg2) + return stub(arg1, arg2, arg3) } if specificReturn { return ret.result1 @@ -286,17 +288,17 @@ func (fake *FakeParticipant) CloseCallCount() int { return len(fake.closeArgsForCall) } -func (fake *FakeParticipant) CloseCalls(stub func(bool, types.ParticipantCloseReason) error) { +func (fake *FakeParticipant) CloseCalls(stub func(bool, types.ParticipantCloseReason, bool) error) { fake.closeMutex.Lock() defer fake.closeMutex.Unlock() fake.CloseStub = stub } -func (fake *FakeParticipant) CloseArgsForCall(i int) (bool, types.ParticipantCloseReason) { +func (fake *FakeParticipant) CloseArgsForCall(i int) (bool, types.ParticipantCloseReason, bool) { fake.closeMutex.RLock() defer fake.closeMutex.RUnlock() argsForCall := fake.closeArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 } func (fake *FakeParticipant) CloseReturns(result1 error) { diff --git a/pkg/service/roommanager.go b/pkg/service/roommanager.go index 4e3997d15..bd09448dd 100644 --- a/pkg/service/roommanager.go +++ b/pkg/service/roommanager.go @@ -194,7 +194,7 @@ func (r *RoomManager) Stop() { for _, room := range rooms { for _, p := range room.GetParticipants() { - _ = p.Close(true, types.ParticipantCloseReasonRoomManagerStop) + _ = p.Close(true, types.ParticipantCloseReasonRoomManagerStop, false) } room.Close() } @@ -229,11 +229,35 @@ func (r *RoomManager) StartSession( if pi.Identity == "" { return nil } + participant := room.GetParticipant(pi.Identity) if participant != nil { - // When reconnecting, it means WS has interrupted by underlying peer connection is still ok - // in this mode, we'll keep the participant SID, and just swap the sink for the underlying connection + // When reconnecting, it means WS has interrupted but underlying peer connection is still ok in this state, + // we'll keep the participant SID, and just swap the sink for the underlying connection if pi.Reconnect { + if participant.IsClosed() { + // Send leave request if participant is closed, i. e. handle the case of client trying to resume crossing wires with + // server closing the participant due to some irrecoverable condition. Such a condition would have triggered + // a full reconnect when that condition occurred. + // + // It is possible that the client did not get that send request. So, send it again. + logger.Infow("cannot restart a closed participant", + "room", roomName, + "nodeID", r.currentNode.Id, + "participant", pi.Identity, + "reason", pi.ReconnectReason, + ) + _ = responseSink.WriteMessage(&livekit.SignalResponse{ + Message: &livekit.SignalResponse_Leave{ + Leave: &livekit.LeaveRequest{ + CanReconnect: true, + Reason: livekit.DisconnectReason_STATE_MISMATCH, + }, + }, + }) + return errors.New("could not restart closed participant") + } + logger.Infow("resuming RTC session", "room", roomName, "nodeID", r.currentNode.Id, @@ -244,20 +268,27 @@ func (r *RoomManager) StartSession( if iceConfig == nil { iceConfig = &livekit.ICEConfig{} } - if err = room.ResumeParticipant(participant, requestSource, responseSink, - r.iceServersForRoom(protoRoom, iceConfig.PreferenceSubscriber == livekit.ICECandidateType_ICT_TLS), - pi.ReconnectReason); err != nil { + if err = room.ResumeParticipant( + participant, + requestSource, + responseSink, + r.iceServersForRoom( + protoRoom, + iceConfig.PreferenceSubscriber == livekit.ICECandidateType_ICT_TLS, + ), + pi.ReconnectReason, + ); err != nil { logger.Warnw("could not resume participant", err, "participant", pi.Identity) return err } r.telemetry.ParticipantResumed(ctx, room.ToProto(), participant.ToProto(), livekit.NodeID(r.currentNode.Id), pi.ReconnectReason) go r.rtcSessionWorker(room, participant, requestSource) return nil - } else { - participant.GetLogger().Infow("removing duplicate participant") - // we need to clean up the existing participant, so a new one can join - room.RemoveParticipant(participant.Identity(), participant.ID(), types.ParticipantCloseReasonDuplicateIdentity) } + + // we need to clean up the existing participant, so a new one can join + participant.GetLogger().Infow("removing duplicate participant") + room.RemoveParticipant(participant.Identity(), participant.ID(), types.ParticipantCloseReasonDuplicateIdentity) } else if pi.Reconnect { // send leave request if participant is trying to reconnect without keep subscribe state // but missing from the room @@ -356,7 +387,7 @@ func (r *RoomManager) StartSession( } if err = room.Join(participant, requestSource, &opts, r.iceServersForRoom(protoRoom, iceConfig.PreferenceSubscriber == livekit.ICECandidateType_ICT_TLS)); err != nil { pLogger.Errorw("could not join room", err) - _ = participant.Close(true, types.ParticipantCloseReasonJoinFailed) + _ = participant.Close(true, types.ParticipantCloseReasonJoinFailed, false) return err } if err = r.roomStore.StoreParticipant(ctx, roomName, participant.ToProto()); err != nil { @@ -598,7 +629,7 @@ func (r *RoomManager) handleRTCMessage(ctx context.Context, roomName livekit.Roo case *livekit.RTCNodeMessage_DeleteRoom: room.Logger.Infow("deleting room") for _, p := range room.GetParticipants() { - _ = p.Close(true, types.ParticipantCloseReasonServiceRequestDeleteRoom) + _ = p.Close(true, types.ParticipantCloseReasonServiceRequestDeleteRoom, false) } room.Close() case *livekit.RTCNodeMessage_UpdateSubscriptions: From c38791ff0a77c8012b681299bf06f87ee17e612f Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Thu, 22 Jun 2023 07:09:34 -0700 Subject: [PATCH 244/324] stop retrying signal connection if the request context is closed (#1820) --- pkg/service/rtcservice.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/service/rtcservice.go b/pkg/service/rtcservice.go index 653ff80b8..32dfca410 100644 --- a/pkg/service/rtcservice.go +++ b/pkg/service/rtcservice.go @@ -199,6 +199,10 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { var cr connectionResult var initialResponse *livekit.SignalResponse for i := 0; i < 3; i++ { + if err = r.Context().Err(); err != nil { + break + } + connectionTimeout := 3 * time.Second * time.Duration(i+1) ctx := utils.ContextWithAttempt(r.Context(), i) cr, initialResponse, err = s.startConnection(ctx, roomName, pi, connectionTimeout) From 1f6efedd31b976f681075f218c53b60179e56c30 Mon Sep 17 00:00:00 2001 From: Benjamin Pracht Date: Thu, 22 Jun 2023 09:20:58 -0700 Subject: [PATCH 245/324] Send updated events on state updates (#1819) --- go.mod | 2 +- go.sum | 4 ++-- pkg/service/ioinfo.go | 7 +++++++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f45065249..56c64c670 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230612070454-d5299b956135 - github.com/livekit/protocol v1.5.8-0.20230619005042-089430752e03 + github.com/livekit/protocol v1.5.8-0.20230620161627-ce9e603cfda8 github.com/livekit/psrpc v0.3.1 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 diff --git a/go.sum b/go.sum index 886ed55c3..966bd9126 100644 --- a/go.sum +++ b/go.sum @@ -124,8 +124,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20230612070454-d5299b956135 h1:lWYbsondvqG69czxoACDwaJ/BoyD57BahCo70ZH+m4U= github.com/livekit/mediatransportutil v0.0.0-20230612070454-d5299b956135/go.mod h1:MRc0zSOSzXuFt0X218SgabzlaKevkvCckPgBEoHYc34= -github.com/livekit/protocol v1.5.8-0.20230619005042-089430752e03 h1:Ma+BKzQHCZhXeklDN72ahkERSixANOwlargLT4mrsPo= -github.com/livekit/protocol v1.5.8-0.20230619005042-089430752e03/go.mod h1:B6hJiuXT84dHsUgaKHBo+ZLPX4XhklptYA2UbANSiNg= +github.com/livekit/protocol v1.5.8-0.20230620161627-ce9e603cfda8 h1:pri2aylzPrDDTjBKQQdcYsYqwjv9J7W8CnEkrbnF0lU= +github.com/livekit/protocol v1.5.8-0.20230620161627-ce9e603cfda8/go.mod h1:B6hJiuXT84dHsUgaKHBo+ZLPX4XhklptYA2UbANSiNg= github.com/livekit/psrpc v0.3.1 h1:KfylgJHvoLQcc22t/oflwMOeSnx0c14G7cWsS+9MYS4= github.com/livekit/psrpc v0.3.1/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= diff --git a/pkg/service/ioinfo.go b/pkg/service/ioinfo.go index f3bd6cac7..612020916 100644 --- a/pkg/service/ioinfo.go +++ b/pkg/service/ioinfo.go @@ -143,6 +143,13 @@ func (s *IOInfoService) UpdateIngressState(ctx context.Context, req *rpc.UpdateI logger.Infow("ingress buffering", "ingressID", req.IngressId) } + } else { + // Status didn't change, send Updated event + info.State = req.State + + s.telemetry.IngressUpdated(ctx, info) + + logger.Infow("ingress updated", "ingressID", req.IngressId) } return &emptypb.Empty{}, nil From c21f275ab233081881cee5d4d7ec1e673580b3ea Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 23 Jun 2023 00:55:44 -0700 Subject: [PATCH 246/324] Update module github.com/hashicorp/golang-lru/v2 to v2.0.4 (#1817) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 56c64c670..d1166415a 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/google/wire v0.5.0 github.com/gorilla/websocket v1.5.0 github.com/hashicorp/go-version v1.6.0 - github.com/hashicorp/golang-lru/v2 v2.0.3 + github.com/hashicorp/golang-lru/v2 v2.0.4 github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230612070454-d5299b956135 diff --git a/go.sum b/go.sum index 966bd9126..32f45f0bf 100644 --- a/go.sum +++ b/go.sum @@ -87,8 +87,8 @@ github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZn github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/golang-lru/v2 v2.0.3 h1:kmRrRLlInXvng0SmLxmQpQkpbYAvcXm7NPDrgxJa9mE= -github.com/hashicorp/golang-lru/v2 v2.0.3/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/golang-lru/v2 v2.0.4 h1:7GHuZcgid37q8o5i3QI9KMT4nCWQQ3Kx3Ov6bb9MfK0= +github.com/hashicorp/golang-lru/v2 v2.0.4/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/josharian/native v0.0.0-20200817173448-b6b71def0850/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= github.com/josharian/native v1.0.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= From 8ac394c5bb9b0abc931b68403d135c73a9b5b131 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 23 Jun 2023 14:18:55 +0530 Subject: [PATCH 247/324] Removing commented out short cut path, don't need more debug data. (#1822) --- pkg/sfu/streamtrackermanager.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/sfu/streamtrackermanager.go b/pkg/sfu/streamtrackermanager.go index 530fef826..ef92d6618 100644 --- a/pkg/sfu/streamtrackermanager.go +++ b/pkg/sfu/streamtrackermanager.go @@ -548,11 +548,9 @@ func (s *StreamTrackerManager) GetReferenceLayerRTPTimestamp(ts uint32, layer in return 0, fmt.Errorf("invalid layer, target: %d, reference: %d", layer, referenceLayer) } - /* TODO-RESTORE-AFTER-DEBUG - this is just fast path, below calculations should yield same if layer == referenceLayer { return ts, nil } - */ var srLayer *buffer.RTCPSenderReportData if int(layer) < len(s.senderReports) { From 81f41aca20e32be1c672bb625df85e8568222794 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sat, 24 Jun 2023 19:18:05 +0530 Subject: [PATCH 248/324] Full reconnect on publication mismatch on resume. (#1823) * Full reconnect on publication mismatch on resume. It is possible that publications mismatch on resume. An example sequence - Client sends `AddTrack` for `trackA` - Server never receives it due to signalling connection breakage. - Client could do a resume (reconnect=1) noticing signalling connection breakage. - Client's view thinks that `trackA` is known to server, but server does not know about it. - A subsequence offer containing `trackA` triggers `trackInfo not available before track publish` and the track does not get published. Detect the case of missing track and issue a full reconnect. * UpdateSubscriptions from sync state a la cloud * add missing shouldReconnect --- pkg/rtc/room.go | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index df9db3764..dcd5a6b1c 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -526,6 +526,46 @@ func (r *Room) UpdateSubscriptions( } func (r *Room) SyncState(participant types.LocalParticipant, state *livekit.SyncState) error { + pLogger := participant.GetLogger() + pLogger.Infow("setting sync state", "state", state) + + shouldReconnect := false + pubTracks := state.GetPublishTracks() + existingPubTracks := participant.GetPublishedTracks() + for _, pubTrack := range pubTracks { + // client may not have sent TrackInfo for each published track + ti := pubTrack.Track + if ti == nil { + pLogger.Warnw("TrackInfo not sent during resume", nil) + shouldReconnect = true + break + } + + found := false + for _, existingPubTrack := range existingPubTracks { + if existingPubTrack.ID() == livekit.TrackID(ti.Sid) { + found = true + break + } + } + if !found { + pLogger.Warnw("unknown track during resume", nil, "trackID", ti.Sid) + shouldReconnect = true + break + } + } + if shouldReconnect { + pLogger.Warnw("unable to resume due to missing published tracks, starting full reconnect", nil) + participant.IssueFullReconnect(types.ParticipantCloseReasonPublicationError) + return nil + } + + r.UpdateSubscriptions( + participant, + livekit.StringsAsTrackIDs(state.Subscription.TrackSids), + state.Subscription.ParticipantTracks, + state.Subscription.Subscribe, + ) return nil } From 95f360bbce3d5dad4b9d1307ecbef237788502c3 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sun, 25 Jun 2023 09:26:14 +0530 Subject: [PATCH 249/324] Do not process events after participant close. (#1824) * Do not process events after participant close. Avoid processing transport events after participant/transport close. It causes error logs which are not really errors, but distracting noise. * correct comment --- pkg/rtc/participant.go | 6 +++++- pkg/rtc/transport.go | 5 +++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 889619c76..685a470ef 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -538,6 +538,10 @@ func (p *ParticipantImpl) HandleAnswer(answer webrtc.SessionDescription) { } func (p *ParticipantImpl) onPublisherAnswer(answer webrtc.SessionDescription) error { + if p.IsClosed() || p.IsDisconnected() { + return nil + } + p.params.Logger.Debugw("sending answer", "transport", livekit.SignalTarget_PUBLISHER) answer = p.configurePublisherAnswer(answer) if err := p.writeMessage(&livekit.SignalResponse{ @@ -1290,7 +1294,7 @@ func (p *ParticipantImpl) onDataMessage(kind livekit.DataPacket_Kind, data []byt } func (p *ParticipantImpl) onICECandidate(c *webrtc.ICECandidate, target livekit.SignalTarget) error { - if c == nil || p.IsDisconnected() { + if c == nil || p.IsDisconnected() || p.IsClosed() { return nil } diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index a57910560..63145bf2c 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -1370,6 +1370,11 @@ func (t *PCTransport) postEvent(event event) { func (t *PCTransport) processEvents() { for event := range t.eventCh { + if t.isClosed.Load() { + // just drain the channel without processing events + continue + } + err := t.handleEvent(&event) if err != nil { t.params.Logger.Errorw("error handling event", err, "event", event.String()) From 352bb1d2042b5881d0df1cc3486faf0c7131287b Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 26 Jun 2023 23:15:53 +0530 Subject: [PATCH 250/324] Add GetClientInfo interface, to be used to decide migration vs full-reconenct (#1827) --- pkg/rtc/participant.go | 6 ++ pkg/rtc/types/interfaces.go | 1 + .../typesfakes/fake_local_participant.go | 65 +++++++++++++++++++ 3 files changed, 72 insertions(+) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 685a470ef..b87623375 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -279,6 +279,12 @@ func (p *ParticipantImpl) ConnectedAt() time.Time { return p.connectedAt } +func (p *ParticipantImpl) GetClientInfo() *livekit.ClientInfo { + p.lock.RLock() + defer p.lock.RUnlock() + return p.params.ClientInfo.ClientInfo +} + func (p *ParticipantImpl) GetClientConfiguration() *livekit.ClientConfiguration { p.lock.RLock() defer p.lock.RUnlock() diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index 7370bfa1b..c903d0034 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -285,6 +285,7 @@ type LocalParticipant interface { IsDisconnected() bool IsIdle() bool SubscriberAsPrimary() bool + GetClientInfo() *livekit.ClientInfo GetClientConfiguration() *livekit.ClientConfiguration GetICEConnectionType() ICEConnectionType GetBufferFactory() *buffer.Factory diff --git a/pkg/rtc/types/typesfakes/fake_local_participant.go b/pkg/rtc/types/typesfakes/fake_local_participant.go index f6882fe93..b4c11ad79 100644 --- a/pkg/rtc/types/typesfakes/fake_local_participant.go +++ b/pkg/rtc/types/typesfakes/fake_local_participant.go @@ -212,6 +212,16 @@ type FakeLocalParticipant struct { getClientConfigurationReturnsOnCall map[int]struct { result1 *livekit.ClientConfiguration } + GetClientInfoStub func() *livekit.ClientInfo + getClientInfoMutex sync.RWMutex + getClientInfoArgsForCall []struct { + } + getClientInfoReturns struct { + result1 *livekit.ClientInfo + } + getClientInfoReturnsOnCall map[int]struct { + result1 *livekit.ClientInfo + } GetConnectionQualityStub func() *livekit.ConnectionQualityInfo getConnectionQualityMutex sync.RWMutex getConnectionQualityArgsForCall []struct { @@ -1849,6 +1859,59 @@ func (fake *FakeLocalParticipant) GetClientConfigurationReturnsOnCall(i int, res }{result1} } +func (fake *FakeLocalParticipant) GetClientInfo() *livekit.ClientInfo { + fake.getClientInfoMutex.Lock() + ret, specificReturn := fake.getClientInfoReturnsOnCall[len(fake.getClientInfoArgsForCall)] + fake.getClientInfoArgsForCall = append(fake.getClientInfoArgsForCall, struct { + }{}) + stub := fake.GetClientInfoStub + fakeReturns := fake.getClientInfoReturns + fake.recordInvocation("GetClientInfo", []interface{}{}) + fake.getClientInfoMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) GetClientInfoCallCount() int { + fake.getClientInfoMutex.RLock() + defer fake.getClientInfoMutex.RUnlock() + return len(fake.getClientInfoArgsForCall) +} + +func (fake *FakeLocalParticipant) GetClientInfoCalls(stub func() *livekit.ClientInfo) { + fake.getClientInfoMutex.Lock() + defer fake.getClientInfoMutex.Unlock() + fake.GetClientInfoStub = stub +} + +func (fake *FakeLocalParticipant) GetClientInfoReturns(result1 *livekit.ClientInfo) { + fake.getClientInfoMutex.Lock() + defer fake.getClientInfoMutex.Unlock() + fake.GetClientInfoStub = nil + fake.getClientInfoReturns = struct { + result1 *livekit.ClientInfo + }{result1} +} + +func (fake *FakeLocalParticipant) GetClientInfoReturnsOnCall(i int, result1 *livekit.ClientInfo) { + fake.getClientInfoMutex.Lock() + defer fake.getClientInfoMutex.Unlock() + fake.GetClientInfoStub = nil + if fake.getClientInfoReturnsOnCall == nil { + fake.getClientInfoReturnsOnCall = make(map[int]struct { + result1 *livekit.ClientInfo + }) + } + fake.getClientInfoReturnsOnCall[i] = struct { + result1 *livekit.ClientInfo + }{result1} +} + func (fake *FakeLocalParticipant) GetConnectionQuality() *livekit.ConnectionQualityInfo { fake.getConnectionQualityMutex.Lock() ret, specificReturn := fake.getConnectionQualityReturnsOnCall[len(fake.getConnectionQualityArgsForCall)] @@ -5475,6 +5538,8 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.getCachedDownTrackMutex.RUnlock() fake.getClientConfigurationMutex.RLock() defer fake.getClientConfigurationMutex.RUnlock() + fake.getClientInfoMutex.RLock() + defer fake.getClientInfoMutex.RUnlock() fake.getConnectionQualityMutex.RLock() defer fake.getConnectionQualityMutex.RUnlock() fake.getICEConnectionTypeMutex.RLock() From 2896aeb126cc2b8c8faa8d1f5f72e80941d78401 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 27 Jun 2023 04:34:41 +0530 Subject: [PATCH 251/324] Set potential codecs for tracks without simulcast codecs. (#1828) When migrating muted track, need to set potential codecs. For audio, there may not be `simulcast_codecs` in `AddTrack`. Hence when migrating a muted track, the potential codecs are not set. That results in no receivers in relay up track (because all this could happen before the audio track is unmuted). So, look at MimeType in TrackInfo (this will be set in OnTrack) and use that as potential codec. --- pkg/rtc/participant.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index b87623375..2832fed24 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -1707,6 +1707,24 @@ func (p *ParticipantImpl) addMigrateMutedTrack(cid string, ti *livekit.TrackInfo } } } + // check for mime_type for tracks that do not have simulcast_codecs set + if ti.MimeType != "" { + for _, nc := range parameters.Codecs { + if strings.EqualFold(nc.MimeType, ti.MimeType) { + alreadyAdded := false + for _, pc := range potentialCodecs { + if strings.EqualFold(pc.MimeType, ti.MimeType) { + alreadyAdded = true + break + } + } + if !alreadyAdded { + potentialCodecs = append(potentialCodecs, nc) + } + break + } + } + } mt.SetPotentialCodecs(potentialCodecs, parameters.HeaderExtensions) for _, codec := range ti.Codecs { From ad5133935733af6e3ef34fb817267949dc8d1e4f Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 26 Jun 2023 16:48:09 -0700 Subject: [PATCH 252/324] Update go deps (#1826) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index d1166415a..0bef11416 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.18 require ( github.com/bep/debounce v1.2.1 - github.com/d5/tengo/v2 v2.16.0 + github.com/d5/tengo/v2 v2.16.1 github.com/dustin/go-humanize v1.0.1 github.com/elliotchance/orderedmap/v2 v2.2.0 github.com/florianl/go-tc v0.4.2 @@ -45,9 +45,9 @@ require ( github.com/urfave/cli/v2 v2.25.7 github.com/urfave/negroni/v3 v3.0.0 go.uber.org/atomic v1.11.0 - golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 + golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df golang.org/x/sync v0.3.0 - google.golang.org/protobuf v1.30.0 + google.golang.org/protobuf v1.31.0 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go.sum b/go.sum index 32f45f0bf..63da1ff1d 100644 --- a/go.sum +++ b/go.sum @@ -14,8 +14,8 @@ github.com/cilium/ebpf v0.8.1/go.mod h1:f5zLIM0FSNuAkSyLAN7X+Hy6yznlF1mNiWUMfxMt github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/d5/tengo/v2 v2.16.0 h1:HEpo2Rk8fIiXmTGtkRMJZ4RTQdpgiL8n/tNrRUDj75c= -github.com/d5/tengo/v2 v2.16.0/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8= +github.com/d5/tengo/v2 v2.16.1 h1:/N6dqiGu9toqANInZEOQMM8I06icdZnmb+81DG/lZdw= +github.com/d5/tengo/v2 v2.16.1/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -290,8 +290,8 @@ golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= -golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc= -golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME= +golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= @@ -419,8 +419,8 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 5b975af55fdf7d813991bd9d306f54980ee929b1 Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Tue, 27 Jun 2023 15:11:06 +0800 Subject: [PATCH 253/324] Refine dependency descriptor based selection forwarder (#1808) * Don't update dependency info if unordered packet received * Trace all active svc chains for downtrack * Try to keep lower decode target decodable * remove comments * Test case * clean code * solve comments --- pkg/sfu/buffer/buffer.go | 2 +- pkg/sfu/buffer/dependencydescriptorparser.go | 90 +++-- pkg/sfu/buffer/fps_test.go | 2 +- pkg/sfu/forwarder.go | 4 +- pkg/sfu/receiver.go | 16 +- pkg/sfu/streamtracker/interfaces.go | 2 +- pkg/sfu/streamtracker/streamtracker.go | 2 +- pkg/sfu/streamtracker/streamtracker_dd.go | 5 +- .../streamtracker/streamtracker_dd_test.go | 7 +- pkg/sfu/videolayerselector/base.go | 6 + pkg/sfu/videolayerselector/decodetarget.go | 56 +++ .../dependencydescriptor.go | 280 +++++++------ .../dependencydescriptor_test.go | 377 ++++++++++++++++++ pkg/sfu/videolayerselector/framechain.go | 114 ++++++ .../selectordecisioncache.go | 91 ++++- .../videolayerselector/videolayerselector.go | 2 + 16 files changed, 851 insertions(+), 205 deletions(-) create mode 100644 pkg/sfu/videolayerselector/decodetarget.go create mode 100644 pkg/sfu/videolayerselector/dependencydescriptor_test.go create mode 100644 pkg/sfu/videolayerselector/framechain.go diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go index bc5b45e20..b242662dd 100644 --- a/pkg/sfu/buffer/buffer.go +++ b/pkg/sfu/buffer/buffer.go @@ -42,7 +42,7 @@ type ExtPacket struct { Payload interface{} KeyFrame bool RawPacket []byte - DependencyDescriptor *DependencyDescriptorWithDecodeTarget + DependencyDescriptor *ExtDependencyDescriptor } // Buffer contains all packets diff --git a/pkg/sfu/buffer/dependencydescriptorparser.go b/pkg/sfu/buffer/dependencydescriptorparser.go index c52e86c83..71b4aac2e 100644 --- a/pkg/sfu/buffer/dependencydescriptorparser.go +++ b/pkg/sfu/buffer/dependencydescriptorparser.go @@ -7,6 +7,7 @@ import ( "github.com/pion/rtp" dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + "github.com/livekit/livekit-server/pkg/sfu/utils" "github.com/livekit/protocol/logger" ) @@ -17,6 +18,11 @@ type DependencyDescriptorParser struct { logger logger.Logger onMaxLayerChanged func(int32, int32) decodeTargets []DependencyDescriptorDecodeTarget + + wrapAround *utils.WrapAround[uint16, uint64] + structureExtSeq uint64 + activeDecodeTargetsExtSeq uint64 + activeDecodeTargetsMask uint32 } func NewDependencyDescriptorParser(ddExtID uint8, logger logger.Logger, onMaxLayerChanged func(int32, int32)) *DependencyDescriptorParser { @@ -25,16 +31,19 @@ func NewDependencyDescriptorParser(ddExtID uint8, logger logger.Logger, onMaxLay ddExtID: ddExtID, logger: logger, onMaxLayerChanged: onMaxLayerChanged, + wrapAround: utils.NewWrapAround[uint16, uint64](), } } -type DependencyDescriptorWithDecodeTarget struct { - Descriptor *dd.DependencyDescriptor - DecodeTargets []DependencyDescriptorDecodeTarget +type ExtDependencyDescriptor struct { + Descriptor *dd.DependencyDescriptor + + DecodeTargets []DependencyDescriptorDecodeTarget + StructureUpdated bool + ActiveDecodeTargetsUpdated bool } -func (r *DependencyDescriptorParser) Parse(pkt *rtp.Packet) (*DependencyDescriptorWithDecodeTarget, VideoLayer, error) { - // DD-TODO: make sure out-of-order RTP packets do not update decode targets +func (r *DependencyDescriptorParser) Parse(pkt *rtp.Packet) (*ExtDependencyDescriptor, VideoLayer, error) { var videoLayer VideoLayer ddBuf := pkt.GetExtension(r.ddExtID) if ddBuf == nil { @@ -52,49 +61,53 @@ func (r *DependencyDescriptorParser) Parse(pkt *rtp.Packet) (*DependencyDescript return nil, videoLayer, err } + extSeq := r.wrapAround.Update(pkt.SequenceNumber).ExtendedVal + if ddVal.FrameDependencies != nil { videoLayer.Spatial, videoLayer.Temporal = int32(ddVal.FrameDependencies.SpatialId), int32(ddVal.FrameDependencies.TemporalId) } - if ddVal.AttachedStructure != nil && !ddVal.FirstPacketInFrame { - // r.logger.Debugw("ignoring non-first packet in frame with attached structure") - return nil, videoLayer, nil + + extDD := &ExtDependencyDescriptor{ + Descriptor: &ddVal, } if ddVal.AttachedStructure != nil { - r.structure = ddVal.AttachedStructure - r.decodeTargets = ProcessFrameDependencyStructure(ddVal.AttachedStructure) - if len(r.decodeTargets) != 0 { - r.logger.Debugw(fmt.Sprintf("update decode targets: %v", r.decodeTargets)) - r.onMaxLayerChanged(r.decodeTargets[0].Layer.Spatial, r.decodeTargets[0].Layer.Temporal) + r.logger.Debugw(fmt.Sprintf("parsed dependency descriptor\n%s", ddVal.String())) + if extSeq > r.structureExtSeq { + r.structure = ddVal.AttachedStructure + r.decodeTargets = ProcessFrameDependencyStructure(ddVal.AttachedStructure) + r.structureExtSeq = extSeq + extDD.StructureUpdated = true + extDD.ActiveDecodeTargetsUpdated = true + // The dependency descriptor reader will always set ActiveDecodeTargetsBitmask for TemplateDependencyStructure is present, + // so don't need to notify max layer change here. } } - if ddVal.AttachedStructure != nil && ddVal.FirstPacketInFrame { - r.logger.Debugw(fmt.Sprintf("parsed dependency descriptor\n%s", ddVal.String())) - } - - if mask := ddVal.ActiveDecodeTargetsBitmask; mask != nil { - var maxSpatial, maxTemporal int32 - for _, dt := range r.decodeTargets { - if *mask&(1< r.activeDecodeTargetsExtSeq { + r.activeDecodeTargetsExtSeq = extSeq + if *mask != r.activeDecodeTargetsMask { + r.activeDecodeTargetsMask = *mask + extDD.ActiveDecodeTargetsUpdated = true + var maxSpatial, maxTemporal int32 + for _, dt := range r.decodeTargets { + if *mask&(1< maxSpatial { - maxSpatial = dt.Layer.Spatial - } - if dt.Layer.Temporal > maxTemporal { - maxTemporal = dt.Layer.Temporal - } if dt.Layer.Spatial <= layer.Spatial && dt.Layer.Temporal <= layer.Temporal { activeBitMask |= 1 << dt.Target } } - if layer.Spatial == maxSpatial && layer.Temporal == maxTemporal { - // all the decode targets are selected - return nil - } return &activeBitMask } diff --git a/pkg/sfu/buffer/fps_test.go b/pkg/sfu/buffer/fps_test.go index 206041acf..b090d2770 100644 --- a/pkg/sfu/buffer/fps_test.go +++ b/pkg/sfu/buffer/fps_test.go @@ -31,7 +31,7 @@ func (f *testFrameInfo) toVP8() *ExtPacket { func (f *testFrameInfo) toDD() *ExtPacket { return &ExtPacket{ Packet: &rtp.Packet{Header: f.header}, - DependencyDescriptor: &DependencyDescriptorWithDecodeTarget{ + DependencyDescriptor: &ExtDependencyDescriptor{ Descriptor: &dependencydescriptor.DependencyDescriptor{ FrameNumber: f.framenumber, FrameDependencies: &dependencydescriptor.FrameDependencyTemplate{ diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 8929cf736..75e00baa9 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -1409,9 +1409,7 @@ func (f *Forwarder) CheckSync() (locked bool, layer int32) { f.lock.RLock() defer f.lock.RUnlock() - layer = f.vls.GetRequestSpatial() - locked = layer == f.vls.GetCurrent().Spatial || f.vls.GetParked().IsValid() - return + return f.vls.CheckSync() } func (f *Forwarder) FilterRTX(nacks []uint16) (filtered []uint16, disallowedLayers [buffer.DefaultMaxLayerSpatial + 1]bool) { diff --git a/pkg/sfu/receiver.go b/pkg/sfu/receiver.go index 80ef3f007..26ca98ad7 100644 --- a/pkg/sfu/receiver.go +++ b/pkg/sfu/receiver.go @@ -645,6 +645,14 @@ func (w *WebRTCReceiver) forwardRTP(layer int32) { } } + w.downTrackSpreader.Broadcast(func(dt TrackSender) { + _ = dt.WriteRTP(pkt, spatialLayer) + }) + + if redPktWriter != nil { + redPktWriter(pkt, spatialLayer) + } + if spatialTracker != nil { spatialTracker.Observe( pkt.Temporal, @@ -655,14 +663,6 @@ func (w *WebRTCReceiver) forwardRTP(layer int32) { pkt.DependencyDescriptor, ) } - - w.downTrackSpreader.Broadcast(func(dt TrackSender) { - _ = dt.WriteRTP(pkt, spatialLayer) - }) - - if redPktWriter != nil { - redPktWriter(pkt, spatialLayer) - } } } diff --git a/pkg/sfu/streamtracker/interfaces.go b/pkg/sfu/streamtracker/interfaces.go index 934032f68..a9135e631 100644 --- a/pkg/sfu/streamtracker/interfaces.go +++ b/pkg/sfu/streamtracker/interfaces.go @@ -52,5 +52,5 @@ type StreamTrackerWorker interface { Status() StreamStatus BitrateTemporalCumulative() []int64 SetPaused(paused bool) - Observe(temporalLayer int32, pktSize int, payloadSize int, hasMarker bool, ts uint32, dd *buffer.DependencyDescriptorWithDecodeTarget) + Observe(temporalLayer int32, pktSize int, payloadSize int, hasMarker bool, ts uint32, dd *buffer.ExtDependencyDescriptor) } diff --git a/pkg/sfu/streamtracker/streamtracker.go b/pkg/sfu/streamtracker/streamtracker.go index a1b645fb2..1be7dc0fc 100644 --- a/pkg/sfu/streamtracker/streamtracker.go +++ b/pkg/sfu/streamtracker/streamtracker.go @@ -176,7 +176,7 @@ func (s *StreamTracker) Observe( payloadSize int, hasMarker bool, ts uint32, - _ *buffer.DependencyDescriptorWithDecodeTarget, + _ *buffer.ExtDependencyDescriptor, ) { s.lock.Lock() diff --git a/pkg/sfu/streamtracker/streamtracker_dd.go b/pkg/sfu/streamtracker/streamtracker_dd.go index b6daee387..2d7301f7c 100644 --- a/pkg/sfu/streamtracker/streamtracker_dd.go +++ b/pkg/sfu/streamtracker/streamtracker_dd.go @@ -123,7 +123,7 @@ func (s *StreamTrackerDependencyDescriptor) SetPaused(paused bool) { } -func (s *StreamTrackerDependencyDescriptor) Observe(temporalLayer int32, pktSize int, payloadSize int, hasMarker bool, ts uint32, ddVal *buffer.DependencyDescriptorWithDecodeTarget) { +func (s *StreamTrackerDependencyDescriptor) Observe(temporalLayer int32, pktSize int, payloadSize int, hasMarker bool, ts uint32, ddVal *buffer.ExtDependencyDescriptor) { s.lock.Lock() if s.isStopped || s.paused || payloadSize == 0 || ddVal == nil { @@ -133,7 +133,7 @@ func (s *StreamTrackerDependencyDescriptor) Observe(temporalLayer int32, pktSize var notifyFns []func(status StreamStatus) var notifyStatus StreamStatus - if mask := ddVal.Descriptor.ActiveDecodeTargetsBitmask; mask != nil { + if mask := ddVal.Descriptor.ActiveDecodeTargetsBitmask; mask != nil && ddVal.ActiveDecodeTargetsUpdated { var maxSpatial, maxTemporal int32 for _, dt := range ddVal.DecodeTargets { if *mask&(1< d.targetLayer.Spatial || dt.Layer.Temporal > d.targetLayer.Temporal { + continue + } + + frameResult, err := dt.OnFrame(extFrameNum, fd) + if err != nil { + d.decodeTargetsLock.RUnlock() + // dtis error, dependency descriptor might lost + d.logger.Debugw(fmt.Sprintf("drop packet for frame detection error, incoming: %v", + incomingLayer, + ), "err", err) + d.decisions.AddDropped(extFrameNum) + return + } + + // Keep forwarding the lower spatial with temporal layer 0 to keep the lower frame chain intact, + // it will cost a few extra bits as those frames might not be present in the current target + // but will make the subscriber switch to lower layer seamlessly without pli. + if frameResult.TargetValid { + if highestDecodeTarget.Target == -1 { + highestDecodeTarget = dt.DependencyDescriptorDecodeTarget + dti = frameResult.DTI + } else if dt.Layer.Spatial < highestDecodeTarget.Layer.Spatial && dt.Layer.Temporal == 0 && + frameResult.DTI != dede.DecodeTargetNotPresent && frameResult.DTI != dede.DecodeTargetDiscardable { + dti = frameResult.DTI + } + } + } + d.decodeTargetsLock.RUnlock() + + // DD-TODO : we don't have a rtp queue to ensure the order of packets now, + // so we don't know packet is lost/out of order, that cause us can't detect + // frame integrity, entire frame is forwareded, whether frame chain is broken. + // So use a simple check here, assume all the reference frame is forwarded and + // only check DTI of the active decode target. + // it is not effeciency, at last we need check frame chain integrity. + + if highestDecodeTarget.Target < 0 { + // no active decode target, do not select + // d.logger.Debugw(fmt.Sprintf("drop packet for no target found, decodeTargets %v, tagetLayer %v, incoming %v", + // d.decodeTargets, + // d.targetLayer, + // incomingLayer, + // )) + d.decisions.AddDropped(extFrameNum) + return + } + + // // DD-TODO : if bandwidth in congest, could drop the 'Discardable' frame + if dti == dede.DecodeTargetNotPresent { + // d.logger.Debugw(fmt.Sprintf("drop packet for decode target not present, highestDecodeTarget %d, incoming %v, fn: %d/%d", + // highestDecodeTarget, + // incomingLayer, + // dd.FrameNumber, + // extFrameNum, + // )) + d.decisions.AddDropped(extFrameNum) + return + } + // check decodability using reference frames isDecodable := true for _, fdiff := range fd.FrameDiffs { @@ -90,122 +170,14 @@ func (d *DependencyDescriptor) Select(extPkt *buffer.ExtPacket, _layer int32) (r continue } - if sd, _ := d.decisions.GetDecision(extFrameNum - uint64(fdiff)); sd != selectorDecisionForwarded { + // use relaxed check for frame diff that we have chain intact detection and don't want + // to drop packet due to out-of-order packet or recoverable packet loss + if sd, _ := d.decisions.GetDecision(extFrameNum - uint64(fdiff)); sd == selectorDecisionDropped { isDecodable = false break } } if !isDecodable { - // DD-TODO START - // Not decodable could happen due to packet loss or out-of-order packets, - // Need to figure out better ways to handle this. - // - // 1. Should definitely check if this frame is not part of current decode target OR discardable. - // In that case, forwarding can proceed without disruption. - // 2. Add a packet queue and try to de-jitter for some time. Safest is to packet copy to local queue on - // all down tracks. - // 3. Force a PLI and wait for a key frame. - // DD-TODO END - d.decisions.AddDropped(extFrameNum) - return - } - - // DD-TODO should not update for out-of-order RTP packets - if dd.AttachedStructure != nil { - // update decode target layer and active decode targets - // DD-TODO : these targets info can be shared by all the downtracks, no need calculate in every selector - d.updateDependencyStructure(dd.AttachedStructure) - } - - // DD-TODO : we don't have a rtp queue to ensure the order of packets now, - // so we don't know packet is lost/out of order, that cause us can't detect - // frame integrity, entire frame is forwareded, whether frame chain is broken. - // So use a simple check here, assume all the reference frame is forwarded and - // only check DTI of the active decode target. - // it is not effeciency, at last we need check frame chain integrity. - - activeDecodeTargets := dd.ActiveDecodeTargetsBitmask - if activeDecodeTargets != nil { - d.logger.Debugw("active decode targets", "activeDecodeTargets", *activeDecodeTargets) - } - - // find decode target closest to targetLayer - highestDecodeTarget := buffer.DependencyDescriptorDecodeTarget{ - Target: -1, - Layer: buffer.InvalidLayer, - } - for _, dt := range ddwdt.DecodeTargets { - if dt.Layer.Spatial > d.targetLayer.Spatial || dt.Layer.Temporal > d.targetLayer.Temporal { - continue - } - - if activeDecodeTargets != nil && ((*activeDecodeTargets)&(1< 0, each Decode target MUST be protected by exactly one Chain. + if structure.NumChains > 0 { + chainIdx := structure.DecodeTargetProtectedByChain[dt.Target] + if chainIdx >= len(d.chains) { + // should not happen + d.logger.Errorw("DecodeTargetProtectedByChain chainIdx out of range", nil, "chainIdx", chainIdx, "NumChains", len(d.chains)) + } else { + chain = d.chains[chainIdx] + } + } + newTargets = append(newTargets, NewDecodeTarget(dt, chain)) + } + d.decodeTargetsLock.Lock() + d.decodeTargets = newTargets + d.decodeTargetsLock.Unlock() +} + +func (d *DependencyDescriptor) updateActiveDecodeTargets(activeDecodeTargetsBitmask uint32) { + for _, chain := range d.chains { + chain.BeginUpdateActive() + } + + d.decodeTargetsLock.RLock() + for _, dt := range d.decodeTargets { + dt.UpdateActive(activeDecodeTargetsBitmask) + } + d.decodeTargetsLock.RUnlock() + + for _, chain := range d.chains { + chain.EndUpdateActive() + } +} + +func (d *DependencyDescriptor) CheckSync() (locked bool, layer int32) { + layer = d.GetRequestSpatial() + if d.GetParked().IsValid() { + return true, layer + } + + d.decodeTargetsLock.RLock() + defer d.decodeTargetsLock.RUnlock() + for _, dt := range d.decodeTargets { + if dt.Active() && dt.Layer.Spatial == layer && dt.Valid() { + return true, layer + } + } + return false, layer } diff --git a/pkg/sfu/videolayerselector/dependencydescriptor_test.go b/pkg/sfu/videolayerselector/dependencydescriptor_test.go new file mode 100644 index 000000000..21e416691 --- /dev/null +++ b/pkg/sfu/videolayerselector/dependencydescriptor_test.go @@ -0,0 +1,377 @@ +package videolayerselector + +import ( + "sort" + "testing" + + "github.com/livekit/livekit-server/pkg/sfu/buffer" + dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + "github.com/livekit/protocol/logger" + "github.com/pion/rtp" + "github.com/stretchr/testify/require" +) + +func TestDecodeTarget(t *testing.T) { + target := buffer.DependencyDescriptorDecodeTarget{ + Target: 1, + Layer: buffer.VideoLayer{Spatial: 1, Temporal: 2}, + } + + t.Run("No Chain", func(t *testing.T) { + dt := NewDecodeTarget(target, nil) + require.True(t, dt.Valid()) + // no indication found + _, err := dt.OnFrame(1, &dd.FrameDependencyTemplate{ + DecodeTargetIndications: []dd.DecodeTargetIndication{}, + }) + require.Error(t, err) + + ret, err := dt.OnFrame(1, &dd.FrameDependencyTemplate{ + DecodeTargetIndications: []dd.DecodeTargetIndication{dd.DecodeTargetNotPresent, dd.DecodeTargetRequired}, + }) + require.NoError(t, err) + require.True(t, ret.TargetValid) + require.Equal(t, dd.DecodeTargetRequired, ret.DTI) + }) + + t.Run("With Chain", func(t *testing.T) { + decisions := NewSelectorDecisionCache(256, 80) + chain := NewFrameChain(decisions, 1, logger.GetLogger()) + dt := NewDecodeTarget(target, chain) + chain.BeginUpdateActive() + dt.UpdateActive(1 << dt.Target) + chain.EndUpdateActive() + require.True(t, dt.Active()) + require.False(t, dt.Valid()) + + // chain intact + frame := &dd.FrameDependencyTemplate{ + DecodeTargetIndications: []dd.DecodeTargetIndication{dd.DecodeTargetNotPresent, dd.DecodeTargetRequired}, + ChainDiffs: []int{0, 0}, + } + chain.OnFrame(1, frame) + require.True(t, dt.Valid()) + ret, err := dt.OnFrame(1, frame) + require.NoError(t, err) + require.True(t, ret.TargetValid) + require.Equal(t, dd.DecodeTargetRequired, ret.DTI) + + }) +} + +func TestFrameChain(t *testing.T) { + decisions := NewSelectorDecisionCache(256, 3) + chain := NewFrameChain(decisions, 0, logger.GetLogger()) + require.True(t, chain.Broken()) + + // chain intact + frameNoDiff := &dd.FrameDependencyTemplate{ + ChainDiffs: []int{0}, + } + // not active + require.False(t, chain.OnFrame(1, frameNoDiff)) + + chain.BeginUpdateActive() + chain.UpdateActive(true) + chain.EndUpdateActive() + + require.True(t, chain.OnFrame(1, frameNoDiff)) + decisions.AddForwarded(1) + + frameDiff1 := &dd.FrameDependencyTemplate{ + ChainDiffs: []int{1}, + } + + require.True(t, chain.OnFrame(2, frameDiff1)) + decisions.AddForwarded(2) + + // frame 5 arrives first , but frame 4 can be recovered by NACK + require.True(t, chain.OnFrame(5, frameDiff1)) + decisions.AddForwarded(5) + + // frame 4 arrives, chain remains intact + require.True(t, chain.OnFrame(4, frameDiff1)) + decisions.AddForwarded(4) + + // frame 3 missed by out of nack range, chain broken + decisions.AddForwarded(7) + require.True(t, chain.Broken()) + + // recovery by non-diff frame + require.True(t, chain.OnFrame(1000, frameNoDiff)) + require.False(t, chain.Broken()) + decisions.AddForwarded(1000) + + // broken by dropped frame + require.True(t, chain.OnFrame(1002, frameDiff1)) + decisions.AddDropped(1001) + require.True(t, chain.Broken()) + + // recovery by non-diff frame + require.True(t, chain.OnFrame(2000, frameNoDiff)) + decisions.AddForwarded(2000) + decisions.AddDropped(2001) + require.False(t, chain.OnFrame(2002, frameDiff1)) + require.True(t, chain.Broken()) +} + +func TestDependencyDescriptor(t *testing.T) { + ddSelector := NewDependencyDescriptor(logger.GetLogger()) + targetLayer := buffer.VideoLayer{Spatial: 1, Temporal: 2} + ddSelector.SetTarget(targetLayer) + ddSelector.SetRequestSpatial(1) + + // no dd ext, dropped + ret := ddSelector.Select(&buffer.ExtPacket{}, 0) + require.False(t, ret.IsSelected) + require.False(t, ret.IsRelevant) + + // non key frame, dropped + ret = ddSelector.Select(&buffer.ExtPacket{ + KeyFrame: false, + DependencyDescriptor: &buffer.ExtDependencyDescriptor{ + Descriptor: &dd.DependencyDescriptor{ + FrameNumber: 1, + FrameDependencies: &dd.FrameDependencyTemplate{ + SpatialId: int(targetLayer.Spatial), + TemporalId: int(targetLayer.Temporal), + }, + }, + }, + }, 0) + require.False(t, ret.IsSelected) + require.True(t, ret.IsRelevant) + + frames := createDDFrames(buffer.VideoLayer{Spatial: 2, Temporal: 2}, 3) + // key frame, update structure and decode targets + ret = ddSelector.Select(frames[0], 0) + require.True(t, ret.IsSelected) + require.Equal(t, ddSelector.GetCurrent(), ddSelector.GetTarget()) + sync, _ := ddSelector.CheckSync() + require.True(t, sync) + + // forward frame belongs to target layer + // drop frame exceeds target layer (not present in target layer or lower layer) + // forward frame not present in target layer but present in lower layer + var ( + belongTargetCase bool + exceedTargetCase bool + lowerTargetCase bool + ) + idx := 1 + var frameForwarded, frameDropped []*buffer.ExtPacket + for ; idx < len(frames); idx++ { + fd := frames[idx].DependencyDescriptor.Descriptor.FrameDependencies + ret = ddSelector.Select(frames[idx], 0) + switch { + case fd.SpatialId == int(targetLayer.Spatial) && fd.TemporalId == int(targetLayer.Temporal): + require.True(t, ret.IsSelected) + belongTargetCase = true + frameForwarded = append(frameForwarded, frames[idx]) + case fd.SpatialId < int(targetLayer.Spatial) && fd.TemporalId == 0: + require.True(t, ret.IsSelected) + lowerTargetCase = true + frameForwarded = append(frameForwarded, frames[idx]) + case fd.SpatialId > int(targetLayer.Spatial) || fd.TemporalId > int(targetLayer.Temporal): + require.False(t, ret.IsSelected) + exceedTargetCase = true + frameDropped = append(frameDropped, frames[idx]) + } + + if belongTargetCase && exceedTargetCase && lowerTargetCase { + break + } + } + + require.True(t, belongTargetCase && exceedTargetCase && lowerTargetCase) + + // select frame already forwarded + ret = ddSelector.Select(frameForwarded[0], 0) + require.True(t, ret.IsSelected) + + // drop frame already dropped + ret = ddSelector.Select(frameDropped[0], 0) + require.False(t, ret.IsSelected) + + // drop frame present but not decodable (dependency frame missed) + idx++ + for ; idx < len(frames); idx++ { + fd := frames[idx].DependencyDescriptor.Descriptor.FrameDependencies + ret = ddSelector.Select(frames[idx], 0) + if fd.SpatialId == int(targetLayer.Spatial) && fd.TemporalId == int(targetLayer.Temporal) { + break + } + } + notDecodableFrame := frames[idx] + notDecodableFrame.DependencyDescriptor.Descriptor.FrameDependencies.FrameDiffs = []int{ + int(notDecodableFrame.DependencyDescriptor.Descriptor.FrameNumber - frameDropped[0].DependencyDescriptor.Descriptor.FrameNumber), + } + ret = ddSelector.Select(notDecodableFrame, 0) + require.False(t, ret.IsSelected) + + // target layer broken + idx++ + for ; idx < len(frames); idx++ { + fd := frames[idx].DependencyDescriptor.Descriptor.FrameDependencies + ret = ddSelector.Select(frames[idx], 0) + if fd.SpatialId == int(targetLayer.Spatial) && fd.TemporalId == int(targetLayer.Temporal) { + break + } + } + brokenFrame := frames[idx] + brokenFrame.DependencyDescriptor.Descriptor.FrameDependencies.ChainDiffs[targetLayer.Spatial] = + int(notDecodableFrame.DependencyDescriptor.Descriptor.FrameNumber - frameDropped[0].DependencyDescriptor.Descriptor.FrameNumber) + ret = ddSelector.Select(brokenFrame, 0) + require.False(t, ret.IsSelected) + + // switch to lower layer, forward frame + idx++ + var switchToLower bool + for ; idx < len(frames); idx++ { + ret = ddSelector.Select(frames[idx], 0) + if ret.IsSelected { + require.True(t, targetLayer.GreaterThan(ddSelector.GetCurrent())) + switchToLower = true + break + } + } + require.True(t, switchToLower) + + // not sync with requested layer + ddSelector.SetRequestSpatial(targetLayer.Spatial) + locked, layer := ddSelector.CheckSync() + require.False(t, locked) + require.Equal(t, targetLayer.Spatial, layer) + + // request to current layer, sync + ddSelector.SetRequestSpatial(ddSelector.GetCurrent().Spatial) + locked, _ = ddSelector.CheckSync() + require.True(t, locked) +} + +func createDDFrames(maxLayer buffer.VideoLayer, startFrameNumber uint16) []*buffer.ExtPacket { + var frames []*buffer.ExtPacket + var activeBitMask uint32 + var decodeTargets []buffer.DependencyDescriptorDecodeTarget + var decodeTargetsProtectByChain []int + for i := 0; i <= int(maxLayer.Spatial); i++ { + for j := 0; j <= int(maxLayer.Temporal); j++ { + decodeTargets = append(decodeTargets, buffer.DependencyDescriptorDecodeTarget{ + Target: i*int(maxLayer.Temporal+1) + j, + Layer: buffer.VideoLayer{Spatial: int32(i), Temporal: int32(j)}, + }) + decodeTargetsProtectByChain = append(decodeTargetsProtectByChain, i) + activeBitMask |= 1 << uint(i*int(maxLayer.Temporal+1)+j) + } + } + sort.Slice(decodeTargets, func(i, j int) bool { + return decodeTargets[i].Layer.GreaterThan(decodeTargets[j].Layer) + }) + + chainDiffs := make([]int, len(decodeTargets)) + dtis := make([]dd.DecodeTargetIndication, len(decodeTargets)) + for _, dt := range decodeTargets { + dtis[dt.Target] = dd.DecodeTargetSwitch + } + + templates := make([]*dd.FrameDependencyTemplate, len(decodeTargets)) + + for _, dt := range decodeTargets { + templates[dt.Target] = &dd.FrameDependencyTemplate{ + SpatialId: int(dt.Layer.Spatial), + TemporalId: int(dt.Layer.Temporal), + ChainDiffs: chainDiffs, + DecodeTargetIndications: dtis, + } + } + keyFrame := &buffer.ExtPacket{ + KeyFrame: true, + DependencyDescriptor: &buffer.ExtDependencyDescriptor{ + Descriptor: &dd.DependencyDescriptor{ + FrameNumber: startFrameNumber, + FrameDependencies: &dd.FrameDependencyTemplate{ + SpatialId: 0, + TemporalId: 0, + ChainDiffs: chainDiffs, + DecodeTargetIndications: dtis, + }, + AttachedStructure: &dd.FrameDependencyStructure{ + NumDecodeTargets: int((maxLayer.Spatial + 1) * (maxLayer.Temporal + 1)), + NumChains: int(maxLayer.Spatial) + 1, + DecodeTargetProtectedByChain: decodeTargetsProtectByChain, + Templates: templates, + }, + ActiveDecodeTargetsBitmask: &activeBitMask, + }, + DecodeTargets: decodeTargets, + StructureUpdated: true, + ActiveDecodeTargetsUpdated: true, + }, + Packet: &rtp.Packet{ + Header: rtp.Header{ + SSRC: 1234, + }, + }, + } + + frames = append(frames, keyFrame) + + chainPrevFrame := make(map[int]int) + for i := 0; i <= int(maxLayer.Spatial); i++ { + chainPrevFrame[i] = int(startFrameNumber) + } + startFrameNumber++ + for i := 0; i < 10; i++ { + for j := len(decodeTargets) - 1; j >= 0; j-- { + dt := decodeTargets[j] + frameChainDiffs := make([]int, len(chainDiffs)) + for i := range frameChainDiffs { + frameChainDiffs[i] = int(startFrameNumber) - chainPrevFrame[i] + } + + frameDtis := make([]dd.DecodeTargetIndication, len(dtis)) + for k := range frameDtis { + if k >= dt.Target { + if dt.Layer.Temporal == 0 { + frameDtis[k] = dd.DecodeTargetRequired + } else { + frameDtis[k] = dd.DecodeTargetDiscardable + } + } else { + frameDtis[k] = dd.DecodeTargetNotPresent + } + } + + frame := &buffer.ExtPacket{ + KeyFrame: true, + DependencyDescriptor: &buffer.ExtDependencyDescriptor{ + Descriptor: &dd.DependencyDescriptor{ + FrameNumber: startFrameNumber, + FrameDependencies: &dd.FrameDependencyTemplate{ + SpatialId: int(dt.Layer.Spatial), + TemporalId: int(dt.Layer.Temporal), + ChainDiffs: frameChainDiffs, + DecodeTargetIndications: frameDtis, + }, + }, + DecodeTargets: decodeTargets, + }, + Packet: &rtp.Packet{ + Header: rtp.Header{ + SSRC: 1234, + }, + }, + } + + startFrameNumber++ + + if dt.Layer.Temporal == 0 { + chainPrevFrame[int(dt.Layer.Spatial)] = int(startFrameNumber) + } + + frames = append(frames, frame) + } + } + + return frames +} diff --git a/pkg/sfu/videolayerselector/framechain.go b/pkg/sfu/videolayerselector/framechain.go new file mode 100644 index 000000000..60cb4ea73 --- /dev/null +++ b/pkg/sfu/videolayerselector/framechain.go @@ -0,0 +1,114 @@ +package videolayerselector + +import ( + dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + "github.com/livekit/protocol/logger" +) + +type FrameChain struct { + logger logger.Logger + decisions *SelectorDecisionCache + broken bool + chainIdx int + active bool + updatingActive bool + + expectFrames []uint64 +} + +func NewFrameChain(decisions *SelectorDecisionCache, chainIdx int, logger logger.Logger) *FrameChain { + return &FrameChain{ + logger: logger, + decisions: decisions, + broken: true, + chainIdx: chainIdx, + active: false, + } +} + +func (fc *FrameChain) OnFrame(extFrameNum uint64, fd *dd.FrameDependencyTemplate) bool { + if !fc.active { + return false + } + + // A decodable frame with frame_chain_fdiff equal to 0 indicates that the Chain is intact. + if fd.ChainDiffs[fc.chainIdx] == 0 { + if fc.broken { + fc.broken = false + fc.logger.Debugw("frame chain intact", "chanIdx", fc.chainIdx) + } + fc.expectFrames = fc.expectFrames[:0] + return true + } + + if fc.broken { + return false + } + + prevFrameInChain := extFrameNum - uint64(fd.ChainDiffs[fc.chainIdx]) + sd, err := fc.decisions.GetDecision(prevFrameInChain) + if err != nil { + fc.logger.Debugw("could not get decision", "err", err, "frame", extFrameNum, "prevFrame", prevFrameInChain) + } + + var intact bool + switch { + case sd == selectorDecisionForwarded: + intact = true + + case sd == selectorDecisionUnknown: + // If the previous frame is unknown, means it has not arrived but could be recovered by NACK / out-of-order arrival, + // set up a expected callback here to determine if the chain is broken or intact + if fc.decisions.ExpectDecision(prevFrameInChain, fc.OnExpectFrameChanged) { + intact = true + fc.expectFrames = append(fc.expectFrames, prevFrameInChain) + } + } + + if !intact { + fc.broken = true + fc.logger.Debugw("frame chain broken", "chanIdx", fc.chainIdx, "sd", sd, "frame", extFrameNum, "prevFrame", prevFrameInChain) + } + return intact +} + +func (fc *FrameChain) OnExpectFrameChanged(frameNum uint64, decision selectorDecision) { + for i, f := range fc.expectFrames { + if f == frameNum { + if decision != selectorDecisionForwarded { + fc.broken = true + } + fc.expectFrames[i] = fc.expectFrames[len(fc.expectFrames)-1] + fc.expectFrames = fc.expectFrames[:len(fc.expectFrames)-1] + break + } + } +} + +func (fc *FrameChain) Broken() bool { + return fc.broken +} + +func (fc *FrameChain) BeginUpdateActive() { + fc.updatingActive = false +} + +func (fc *FrameChain) UpdateActive(active bool) { + fc.updatingActive = fc.updatingActive || active +} + +func (fc *FrameChain) EndUpdateActive() { + active := fc.updatingActive + fc.updatingActive = false + + if active == fc.active { + return + } + + // if the chain transit from inactive to active, reset broken to wait a decodable SWITCH frame + if !fc.active { + fc.broken = true + } + + fc.active = active +} diff --git a/pkg/sfu/videolayerselector/selectordecisioncache.go b/pkg/sfu/videolayerselector/selectordecisioncache.go index b83bf9559..c98d13984 100644 --- a/pkg/sfu/videolayerselector/selectordecisioncache.go +++ b/pkg/sfu/videolayerselector/selectordecisioncache.go @@ -33,18 +33,23 @@ func (s selectorDecision) String() string { // ---------------------------------------------------------------------- type SelectorDecisionCache struct { - initialized bool - base uint64 - last uint64 - masks []uint64 - numEntries uint64 + initialized bool + base uint64 + last uint64 + masks []uint64 + numEntries uint64 + numNackEntries uint64 + + onExpectEntityChanged map[uint64][]func(entity uint64, decision selectorDecision) } -func NewSelectorDecisionCache(maxNumElements uint64) *SelectorDecisionCache { +func NewSelectorDecisionCache(maxNumElements uint64, numNackEntries uint64) *SelectorDecisionCache { numElements := (maxNumElements*2 + 63) / 64 return &SelectorDecisionCache{ - masks: make([]uint64, numElements), - numEntries: numElements * 32, // 2 bits per entry + masks: make([]uint64, numElements), + numEntries: numElements * 32, // 2 bits per entry + numNackEntries: numNackEntries, + onExpectEntityChanged: make(map[uint64][]func(entity uint64, decision selectorDecision)), } } @@ -57,19 +62,39 @@ func (s *SelectorDecisionCache) AddDropped(entity uint64) { } func (s *SelectorDecisionCache) GetDecision(entity uint64) (selectorDecision, error) { - if !s.initialized || entity > s.last || entity < s.base { + if !s.initialized || entity < s.base { + return selectorDecisionMissing, nil + } + + if entity > s.last { return selectorDecisionUnknown, nil } offset := s.last - entity if offset >= s.numEntries { // asking for something too old - return selectorDecisionUnknown, fmt.Errorf("too old, oldest: %d, asking: %d", s.last-s.numEntries+1, entity) + return selectorDecisionMissing, fmt.Errorf("too old, oldest: %d, asking: %d", s.last-s.numEntries+1, entity) } return s.getEntity(entity), nil } +func (s *SelectorDecisionCache) ExpectDecision(entity uint64, f func(entity uint64, decision selectorDecision)) bool { + if !s.initialized || entity < s.base { + return false + } + + if entity < s.last { + offset := s.last - entity + if offset >= s.numEntries { + return false // too old + } + } + + s.onExpectEntityChanged[entity] = append(s.onExpectEntityChanged[entity], f) + return true +} + func (s *SelectorDecisionCache) addEntity(entity uint64, sd selectorDecision) { if !s.initialized { s.initialized = true @@ -90,16 +115,60 @@ func (s *SelectorDecisionCache) addEntity(entity uint64, sd selectorDecision) { } for e := s.last + 1; e != entity; e++ { - s.setEntity(e, selectorDecisionMissing) + s.setEntity(e, selectorDecisionUnknown) } + + // update [last+1-nack, entity-nack) to missing + missingStart := s.last + if missingStart > s.numNackEntries+s.base { + missingStart -= s.numNackEntries + } else { + missingStart = s.base + } + missingEnd := entity + if missingEnd > s.numNackEntries+s.base { + missingEnd -= s.numNackEntries + } else { + missingEnd = s.base + } + if missingEnd > missingStart { + for e := missingStart; e != missingEnd; e++ { + s.setEntityIfUnknown(e, selectorDecisionMissing) + } + } + s.setEntity(entity, sd) s.last = entity + + for e, fns := range s.onExpectEntityChanged { + if e+s.numEntries < s.last { + delete(s.onExpectEntityChanged, e) + for _, f := range fns { + f(e, selectorDecisionMissing) + } + } + } +} + +func (s *SelectorDecisionCache) setEntityIfUnknown(entity uint64, sd selectorDecision) { + if s.getEntity(entity) == selectorDecisionUnknown { + s.setEntity(entity, sd) + } } func (s *SelectorDecisionCache) setEntity(entity uint64, sd selectorDecision) { index, bitpos := s.getPos(entity) s.masks[index] &= ^(0x3 << bitpos) // clear before bitwise OR s.masks[index] |= (uint64(sd) & 0x3) << bitpos + + if sd != selectorDecisionUnknown { + if fns, ok := s.onExpectEntityChanged[entity]; ok { + delete(s.onExpectEntityChanged, entity) + for _, f := range fns { + f(entity, sd) + } + } + } } func (s *SelectorDecisionCache) getEntity(entity uint64) selectorDecision { diff --git a/pkg/sfu/videolayerselector/videolayerselector.go b/pkg/sfu/videolayerselector/videolayerselector.go index fa1e71d83..b108976bf 100644 --- a/pkg/sfu/videolayerselector/videolayerselector.go +++ b/pkg/sfu/videolayerselector/videolayerselector.go @@ -32,6 +32,8 @@ type VideoLayerSelector interface { SetRequestSpatial(layer int32) GetRequestSpatial() int32 + CheckSync() (locked bool, layer int32) + SetMaxSeen(maxSeenLayer buffer.VideoLayer) SetMaxSeenSpatial(layer int32) SetMaxSeenTemporal(layer int32) From cea41e4189c6f26d7c61692e2a2c0dea9b87f5d0 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 27 Jun 2023 17:44:53 +0530 Subject: [PATCH 254/324] Discount out-of-order packets in downstream score. (#1831) * Discount out-of-order packets in downstream score. More notes inline. * correct comment * clean up comment --- pkg/sfu/buffer/rtpstats.go | 22 +++++++--- pkg/sfu/connectionquality/connectionstats.go | 1 + pkg/sfu/connectionquality/scorer.go | 43 +++++++++++++++----- 3 files changed, 50 insertions(+), 16 deletions(-) diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index 0bb269ed6..ce702409b 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -60,6 +60,7 @@ type IntervalStats struct { bytesPadding uint64 headerBytesPadding uint64 packetsLost uint32 + packetsOutOfOrder uint32 frames uint32 } @@ -77,6 +78,7 @@ type RTPDeltaInfo struct { HeaderBytesPadding uint64 PacketsLost uint32 PacketsMissing uint32 + PacketsOutOfOrder uint32 Frames uint32 RttMax uint32 JitterMax float64 @@ -106,6 +108,7 @@ type SnInfo struct { pktSize uint16 isPaddingOnly bool marker bool + isOutOfOrder bool } type RTCPSenderReportData struct { @@ -412,7 +415,7 @@ func (r *RTPStats) Update(rtph *rtp.Header, payloadSize int, paddingSize int, pa isDuplicate = true } else { r.packetsLost-- - r.setSnInfo(rtph.SequenceNumber, uint16(pktSize), uint16(hdrSize), uint16(payloadSize), rtph.Marker) + r.setSnInfo(rtph.SequenceNumber, uint16(pktSize), uint16(hdrSize), uint16(payloadSize), rtph.Marker, true) } } @@ -431,7 +434,7 @@ func (r *RTPStats) Update(rtph *rtp.Header, payloadSize int, paddingSize int, pa r.clearSnInfos(r.highestSN+1, rtph.SequenceNumber) r.packetsLost += uint32(diff - 1) - r.setSnInfo(rtph.SequenceNumber, uint16(pktSize), uint16(hdrSize), uint16(payloadSize), rtph.Marker) + r.setSnInfo(rtph.SequenceNumber, uint16(pktSize), uint16(hdrSize), uint16(payloadSize), rtph.Marker, false) if rtph.SequenceNumber < r.highestSN && !first { r.cycles++ @@ -490,7 +493,7 @@ func (r *RTPStats) maybeAdjustStartSN(rtph *rtp.Header, pktSize uint64, hdrSize beforeAdjust := r.extStartSN r.extStartSN = uint32(rtph.SequenceNumber) - r.setSnInfo(rtph.SequenceNumber, uint16(pktSize), uint16(hdrSize), uint16(payloadSize), rtph.Marker) + r.setSnInfo(rtph.SequenceNumber, uint16(pktSize), uint16(hdrSize), uint16(payloadSize), rtph.Marker, true) for _, s := range r.snapshots { if s.extStartSN == beforeAdjust { @@ -1132,7 +1135,6 @@ func (r *RTPStats) DeltaInfoOverridden(snapshotId uint32) *RTPDeltaInfo { } intervalStats := r.getIntervalStats(uint16(then.extStartSNOverridden), uint16(now.extStartSNOverridden)) - packetsMissing := intervalStats.packetsLost packetsLost := now.packetsLostOverridden - then.packetsLostOverridden if int32(packetsLost) < 0 { packetsLost = 0 @@ -1173,7 +1175,8 @@ func (r *RTPStats) DeltaInfoOverridden(snapshotId uint32) *RTPDeltaInfo { BytesPadding: intervalStats.bytesPadding, HeaderBytesPadding: intervalStats.headerBytesPadding, PacketsLost: packetsLost, - PacketsMissing: packetsMissing, + PacketsMissing: intervalStats.packetsLost, + PacketsOutOfOrder: intervalStats.packetsOutOfOrder, Frames: intervalStats.frames, RttMax: then.maxRtt, JitterMax: maxJitterTime, @@ -1409,7 +1412,7 @@ func (r *RTPStats) getSnInfoOutOfOrderPtr(sn uint16) int { return (r.snInfoWritePtr - int(offset) - 1) & SnInfoMask } -func (r *RTPStats) setSnInfo(sn uint16, pktSize uint16, hdrSize uint16, payloadSize uint16, marker bool) { +func (r *RTPStats) setSnInfo(sn uint16, pktSize uint16, hdrSize uint16, payloadSize uint16, marker bool, isOutOfOrder bool) { writePtr := 0 ooo := (sn - r.highestSN) > (1 << 15) if !ooo { @@ -1427,6 +1430,7 @@ func (r *RTPStats) setSnInfo(sn uint16, pktSize uint16, hdrSize uint16, payloadS snInfo.hdrSize = hdrSize snInfo.isPaddingOnly = payloadSize == 0 snInfo.marker = marker + snInfo.isOutOfOrder = isOutOfOrder } func (r *RTPStats) clearSnInfos(startInclusive uint16, endExclusive uint16) { @@ -1474,6 +1478,9 @@ func (r *RTPStats) getIntervalStats(startInclusive uint16, endExclusive uint16) intervalStats.packets++ intervalStats.bytes += uint64(snInfo.pktSize) intervalStats.headerBytes += uint64(snInfo.hdrSize) + if snInfo.isOutOfOrder { + intervalStats.packetsOutOfOrder++ + } } if snInfo.marker { @@ -1822,6 +1829,7 @@ func AggregateRTPDeltaInfo(deltaInfoList []*RTPDeltaInfo) *RTPDeltaInfo { packetsLost := uint32(0) packetsMissing := uint32(0) + packetsOutOfOrder := uint32(0) frames := uint32(0) @@ -1860,6 +1868,7 @@ func AggregateRTPDeltaInfo(deltaInfoList []*RTPDeltaInfo) *RTPDeltaInfo { packetsLost += deltaInfo.PacketsLost packetsMissing += deltaInfo.PacketsMissing + packetsOutOfOrder += deltaInfo.PacketsOutOfOrder frames += deltaInfo.Frames @@ -1893,6 +1902,7 @@ func AggregateRTPDeltaInfo(deltaInfoList []*RTPDeltaInfo) *RTPDeltaInfo { HeaderBytesPadding: headerBytesPadding, PacketsLost: packetsLost, PacketsMissing: packetsMissing, + PacketsOutOfOrder: packetsOutOfOrder, Frames: frames, RttMax: maxRtt, JitterMax: maxJitter, diff --git a/pkg/sfu/connectionquality/connectionstats.go b/pkg/sfu/connectionquality/connectionstats.go index fb5c7e710..5acc6c0da 100644 --- a/pkg/sfu/connectionquality/connectionstats.go +++ b/pkg/sfu/connectionquality/connectionstats.go @@ -108,6 +108,7 @@ func (cs *ConnectionStats) updateScoreWithAggregate(agg *buffer.RTPDeltaInfo, at stat.packetsExpected = agg.Packets + agg.PacketsPadding stat.packetsLost = agg.PacketsLost stat.packetsMissing = agg.PacketsMissing + stat.packetsOutOfOrder = agg.PacketsOutOfOrder stat.bytes = agg.Bytes - agg.HeaderBytes // only use media payload size stat.rttMax = agg.RttMax stat.jitterMax = agg.JitterMax diff --git a/pkg/sfu/connectionquality/scorer.go b/pkg/sfu/connectionquality/scorer.go index 56e008a10..4d333e005 100644 --- a/pkg/sfu/connectionquality/scorer.go +++ b/pkg/sfu/connectionquality/scorer.go @@ -29,14 +29,15 @@ const ( // ------------------------------------------ type windowStat struct { - startedAt time.Time - duration time.Duration - packetsExpected uint32 - packetsLost uint32 - packetsMissing uint32 - bytes uint64 - rttMax uint32 - jitterMax float64 + startedAt time.Time + duration time.Duration + packetsExpected uint32 + packetsLost uint32 + packetsMissing uint32 + packetsOutOfOrder uint32 + bytes uint64 + rttMax uint32 + jitterMax float64 } func (w *windowStat) calculatePacketScore(plw float64, includeRTT bool, includeJitter bool) float64 { @@ -59,7 +60,28 @@ func (w *windowStat) calculatePacketScore(plw float64, includeRTT bool, includeJ delayEffect = (effectiveDelay - 120.0) / 10.0 } - actualLost := w.packetsLost - w.packetsMissing + // discount out-of-order packets from loss to deal with a scenario like + // 1. up stream has loss + // 2. down stream forwards with loss/hole in sequence number + // 3. down stream client reports a certain number of loss + // 4. while processing that, up stream could have retransmitted missing packets + // 5. those retransmitted packets are forwarded, + // - server's view: it has forwarded those packets + // - client's view: it had not seen those packets when sending RTCP RR + // so those retransmitted packets appear like down stream loss to server. + // + // retransmitted packets would have arrived out-of-order. So, discounting them + // will account for it. + // + // Note that packets can arrive out-of-order in the upstream during regular + // streaming as well, i. e. without loss + NACK + retransmit. Those will be + // discounted too. And that will skew the real loss. For example, let + // us say that 40 out of 100 packets were reported lost by down stream. + // These could be real losses. In the same window, 40 packets could have been + // delivered out-of-order by the up stream, thus cancelling out the real loss. + // But, those situations should be rare and is a compromise for not letting + // up stream loss penalise down stream. + actualLost := w.packetsLost - w.packetsMissing - w.packetsOutOfOrder if int32(actualLost) < 0 { actualLost = 0 } @@ -102,12 +124,13 @@ func (w *windowStat) calculateBitrateScore(expectedBitrate int64) float64 { } func (w *windowStat) String() string { - return fmt.Sprintf("start: %+v, dur: %+v, pe: %d, pl: %d, pm: %d, b: %d, rtt: %d, jitter: %0.2f", + return fmt.Sprintf("start: %+v, dur: %+v, pe: %d, pl: %d, pm: %d, pooo: %d, b: %d, rtt: %d, jitter: %0.2f", w.startedAt, w.duration, w.packetsExpected, w.packetsLost, w.packetsMissing, + w.packetsOutOfOrder, w.bytes, w.rttMax, w.jitterMax, From 2b0a4704744af8003a547db0ca8320dbd5bda800 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 28 Jun 2023 12:48:38 +0530 Subject: [PATCH 255/324] Less flapping in probe. (#1834) - Increase max interval between probes to 2 minutes. - Use a minimum probe rate of 200 kbps. This is to ensure that the probe rate is decent and can produce a stronger signal. --- pkg/sfu/streamallocator/probe_controller.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/sfu/streamallocator/probe_controller.go b/pkg/sfu/streamallocator/probe_controller.go index 28fda6ac7..9fc80ad53 100644 --- a/pkg/sfu/streamallocator/probe_controller.go +++ b/pkg/sfu/streamallocator/probe_controller.go @@ -10,7 +10,7 @@ import ( const ( ProbeWaitBase = 5 * time.Second ProbeBackoffFactor = 1.5 - ProbeWaitMax = 30 * time.Second + ProbeWaitMax = 2 * time.Minute ProbeSettleWait = 250 ProbeSettleWaitMax = 10 * time.Second ProbeTrendWait = 2 * time.Second @@ -178,7 +178,11 @@ func (p *ProbeController) InitProbe(probeGoalDeltaBps int64, expectedBandwidthUs p.lastProbeStartTime = time.Now() // overshoot a bit to account for noise (in measurement/estimate etc) - p.probeGoalBps = expectedBandwidthUsage + ((probeGoalDeltaBps * ProbePct) / 100) + desiredIncreaseBps := (probeGoalDeltaBps * ProbePct) / 100 + if desiredIncreaseBps < ProbeMinBps { + desiredIncreaseBps = ProbeMinBps + } + p.probeGoalBps = expectedBandwidthUsage + desiredIncreaseBps p.abortedProbeClusterId = ProbeClusterIdInvalid From eaf70d5549ac02bc53b725f85ceb5e0de6797d49 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 28 Jun 2023 13:22:44 +0530 Subject: [PATCH 256/324] Pacer in down stream path. (#1835) * Pacer interface to send packets * notify outside lock * use select * use pass through pacer * add error to OnSent * Remove log which could get noisy * Starting TWCC work (#1727) * add packet time * WIP commit * WIP commit * WIP commit * minor comments * Some measurements (#1736) * WIP commit * some notes * WIP commit * variable name change and do not post to closed channel * unlock * clean up * comment * Hooking up some more bits for TWCC (#1752) * wake under lock * Pacer in down stream path. Splitting out only the pacer from a feature branch to introduce the concept of pacer. Currently, there should be no difference in functionality as a pass through pacer is used. Another implementation exists which is just put it in a queue and send it from one goroutine. A potential implementation to try would be data paced by bandwidth estimate. That could include priority queues and such. But, the main goal here is to introduce notion of pacer in the down stream path and prepare for more congestion control possibilities down the line. * Don't need peak detector * remove throttling of write IO errors --- pkg/rtc/mediatracksubscriptions.go | 1 + pkg/rtc/participant.go | 5 + pkg/rtc/transport.go | 32 +- pkg/rtc/transportmanager.go | 5 + pkg/rtc/types/interfaces.go | 3 + .../typesfakes/fake_local_participant.go | 66 +++ pkg/sfu/downtrack.go | 410 +++++++++--------- pkg/sfu/pacer/base.go | 83 ++++ pkg/sfu/pacer/no_queue.go | 80 ++++ pkg/sfu/pacer/pacer.go | 31 ++ pkg/sfu/pacer/packet_time.go | 22 + pkg/sfu/pacer/pass_through.go | 24 + 12 files changed, 537 insertions(+), 225 deletions(-) create mode 100644 pkg/sfu/pacer/base.go create mode 100644 pkg/sfu/pacer/no_queue.go create mode 100644 pkg/sfu/pacer/pacer.go create mode 100644 pkg/sfu/pacer/packet_time.go create mode 100644 pkg/sfu/pacer/pass_through.go diff --git a/pkg/rtc/mediatracksubscriptions.go b/pkg/rtc/mediatracksubscriptions.go index 007428f50..8887176cd 100644 --- a/pkg/rtc/mediatracksubscriptions.go +++ b/pkg/rtc/mediatracksubscriptions.go @@ -104,6 +104,7 @@ func (t *MediaTrackSubscriptions) AddSubscriber(sub types.LocalParticipant, wr * sub.GetBufferFactory(), subscriberID, t.params.ReceiverConfig.PacketBufferSize, + sub.GetPacer(), LoggerWithTrack(sub.GetLogger(), trackID, t.params.IsRelayed), ) if err != nil { diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 2832fed24..a89315c10 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -23,6 +23,7 @@ import ( "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/livekit-server/pkg/sfu/connectionquality" + "github.com/livekit/livekit-server/pkg/sfu/pacer" "github.com/livekit/livekit-server/pkg/sfu/streamallocator" "github.com/livekit/livekit-server/pkg/telemetry" "github.com/livekit/livekit-server/pkg/telemetry/prometheus" @@ -231,6 +232,10 @@ func (p *ParticipantImpl) GetAdaptiveStream() bool { return p.params.AdaptiveStream } +func (p *ParticipantImpl) GetPacer() pacer.Pacer { + return p.TransportManager.GetSubscriberPacer() +} + func (p *ParticipantImpl) ID() livekit.ParticipantID { return p.params.SID } diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index 63145bf2c..a164b7eac 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -27,6 +27,7 @@ import ( "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/rtc/types" + "github.com/livekit/livekit-server/pkg/sfu/pacer" "github.com/livekit/livekit-server/pkg/sfu/streamallocator" "github.com/livekit/livekit-server/pkg/telemetry" "github.com/livekit/livekit-server/pkg/telemetry/prometheus" @@ -185,6 +186,9 @@ type PCTransport struct { // stream allocator for subscriber PC streamAllocator *streamallocator.StreamAllocator + // only for subscriber PC + pacer pacer.Pacer + previousAnswer *webrtc.SessionDescription // track id -> description map in previous offer sdp previousTrackDescription map[string]*trackDescription @@ -232,9 +236,7 @@ type TransportParams struct { } func newPeerConnection(params TransportParams, onBandwidthEstimator func(estimator cc.BandwidthEstimator)) (*webrtc.PeerConnection, *webrtc.MediaEngine, error) { - directionConfig := params.DirectionConfig - - me, err := createMediaEngine(params.EnabledCodecs, directionConfig) + me, err := createMediaEngine(params.EnabledCodecs, params.DirectionConfig) if err != nil { return nil, nil, err } @@ -305,21 +307,7 @@ func newPeerConnection(params TransportParams, onBandwidthEstimator func(estimat ir := &interceptor.Registry{} if params.IsSendSide { - isSendSideBWE := false - for _, ext := range directionConfig.RTPHeaderExtension.Video { - if ext == sdp.TransportCCURI { - isSendSideBWE = true - break - } - } - for _, ext := range directionConfig.RTPHeaderExtension.Audio { - if ext == sdp.TransportCCURI { - isSendSideBWE = true - break - } - } - - if isSendSideBWE { + if params.CongestionControlConfig.UseSendSideBWE { gf, err := cc.NewInterceptor(func() (cc.BandwidthEstimator, error) { return gcc.NewSendSideBWE( gcc.SendSideBWEInitialBitrate(1*1000*1000), @@ -376,6 +364,7 @@ func NewPCTransport(params TransportParams) (*PCTransport, error) { Logger: params.Logger, }) t.streamAllocator.Start() + t.pacer = pacer.NewPassThrough(params.Logger) } if err := t.createPeerConnection(); err != nil { @@ -414,6 +403,10 @@ func (t *PCTransport) createPeerConnection() error { return nil } +func (t *PCTransport) GetPacer() pacer.Pacer { + return t.pacer +} + func (t *PCTransport) SetSignalingRTT(rtt uint32) { t.signalingRTT.Store(rtt) } @@ -898,6 +891,9 @@ func (t *PCTransport) Close() { if t.streamAllocator != nil { t.streamAllocator.Stop() } + if t.pacer != nil { + t.pacer.Stop() + } _ = t.pc.Close() diff --git a/pkg/rtc/transportmanager.go b/pkg/rtc/transportmanager.go index e34cb3e71..78dd8dee2 100644 --- a/pkg/rtc/transportmanager.go +++ b/pkg/rtc/transportmanager.go @@ -17,6 +17,7 @@ import ( "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/sfu" + "github.com/livekit/livekit-server/pkg/sfu/pacer" "github.com/livekit/livekit-server/pkg/sfu/streamallocator" "github.com/livekit/livekit-server/pkg/telemetry" "github.com/livekit/protocol/livekit" @@ -283,6 +284,10 @@ func (t *TransportManager) WriteSubscriberRTCP(pkts []rtcp.Packet) error { return t.subscriber.WriteRTCP(pkts) } +func (t *TransportManager) GetSubscriberPacer() pacer.Pacer { + return t.subscriber.GetPacer() +} + func (t *TransportManager) OnPrimaryTransportInitialConnected(f func()) { t.onPrimaryTransportInitialConnected = f } diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index c903d0034..443a0a3d5 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -15,6 +15,7 @@ import ( "github.com/livekit/livekit-server/pkg/routing" "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/livekit-server/pkg/sfu/pacer" ) //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate @@ -383,6 +384,8 @@ type LocalParticipant interface { // down stream bandwidth management SetSubscriberAllowPause(allowPause bool) SetSubscriberChannelCapacity(channelCapacity int64) + + GetPacer() pacer.Pacer } // Room is a container of participants, and can provide room-level actions diff --git a/pkg/rtc/types/typesfakes/fake_local_participant.go b/pkg/rtc/types/typesfakes/fake_local_participant.go index b4c11ad79..deb07909f 100644 --- a/pkg/rtc/types/typesfakes/fake_local_participant.go +++ b/pkg/rtc/types/typesfakes/fake_local_participant.go @@ -9,6 +9,7 @@ import ( "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/livekit-server/pkg/sfu/pacer" "github.com/livekit/protocol/auth" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" @@ -252,6 +253,16 @@ type FakeLocalParticipant struct { getLoggerReturnsOnCall map[int]struct { result1 logger.Logger } + GetPacerStub func() pacer.Pacer + getPacerMutex sync.RWMutex + getPacerArgsForCall []struct { + } + getPacerReturns struct { + result1 pacer.Pacer + } + getPacerReturnsOnCall map[int]struct { + result1 pacer.Pacer + } GetPublishedTrackStub func(livekit.TrackID) types.MediaTrack getPublishedTrackMutex sync.RWMutex getPublishedTrackArgsForCall []struct { @@ -2071,6 +2082,59 @@ func (fake *FakeLocalParticipant) GetLoggerReturnsOnCall(i int, result1 logger.L }{result1} } +func (fake *FakeLocalParticipant) GetPacer() pacer.Pacer { + fake.getPacerMutex.Lock() + ret, specificReturn := fake.getPacerReturnsOnCall[len(fake.getPacerArgsForCall)] + fake.getPacerArgsForCall = append(fake.getPacerArgsForCall, struct { + }{}) + stub := fake.GetPacerStub + fakeReturns := fake.getPacerReturns + fake.recordInvocation("GetPacer", []interface{}{}) + fake.getPacerMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) GetPacerCallCount() int { + fake.getPacerMutex.RLock() + defer fake.getPacerMutex.RUnlock() + return len(fake.getPacerArgsForCall) +} + +func (fake *FakeLocalParticipant) GetPacerCalls(stub func() pacer.Pacer) { + fake.getPacerMutex.Lock() + defer fake.getPacerMutex.Unlock() + fake.GetPacerStub = stub +} + +func (fake *FakeLocalParticipant) GetPacerReturns(result1 pacer.Pacer) { + fake.getPacerMutex.Lock() + defer fake.getPacerMutex.Unlock() + fake.GetPacerStub = nil + fake.getPacerReturns = struct { + result1 pacer.Pacer + }{result1} +} + +func (fake *FakeLocalParticipant) GetPacerReturnsOnCall(i int, result1 pacer.Pacer) { + fake.getPacerMutex.Lock() + defer fake.getPacerMutex.Unlock() + fake.GetPacerStub = nil + if fake.getPacerReturnsOnCall == nil { + fake.getPacerReturnsOnCall = make(map[int]struct { + result1 pacer.Pacer + }) + } + fake.getPacerReturnsOnCall[i] = struct { + result1 pacer.Pacer + }{result1} +} + func (fake *FakeLocalParticipant) GetPublishedTrack(arg1 livekit.TrackID) types.MediaTrack { fake.getPublishedTrackMutex.Lock() ret, specificReturn := fake.getPublishedTrackReturnsOnCall[len(fake.getPublishedTrackArgsForCall)] @@ -5546,6 +5610,8 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.getICEConnectionTypeMutex.RUnlock() fake.getLoggerMutex.RLock() defer fake.getLoggerMutex.RUnlock() + fake.getPacerMutex.RLock() + defer fake.getPacerMutex.RUnlock() fake.getPublishedTrackMutex.RLock() defer fake.getPublishedTrackMutex.RUnlock() fake.getPublishedTracksMutex.RLock() diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 848595ae4..b47d9dd33 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -22,6 +22,7 @@ import ( "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/livekit-server/pkg/sfu/connectionquality" dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" + "github.com/livekit/livekit-server/pkg/sfu/pacer" ) // TrackSender defines an interface send media to remote peer @@ -187,17 +188,17 @@ type DownTrack struct { forwarder *Forwarder - upstreamCodecs []webrtc.RTPCodecParameters - codec webrtc.RTPCodecCapability - rtpHeaderExtensions []webrtc.RTPHeaderExtensionParameter - absSendTimeID int - dependencyDescriptorID int - receiver TrackReceiver - transceiver *webrtc.RTPTransceiver - writeStream webrtc.TrackLocalWriter - rtcpReader *buffer.RTCPReader - onCloseHandler func(willBeResumed bool) - onBinding func(error) + upstreamCodecs []webrtc.RTPCodecParameters + codec webrtc.RTPCodecCapability + absSendTimeExtID int + transportWideExtID int + dependencyDescriptorExtID int + receiver TrackReceiver + transceiver *webrtc.RTPTransceiver + writeStream webrtc.TrackLocalWriter + rtcpReader *buffer.RTCPReader + onCloseHandler func(willBeResumed bool) + onBinding func(error) listenerLock sync.RWMutex receiverReportListeners []ReceiverReportListener @@ -232,6 +233,8 @@ type DownTrack struct { bytesSent atomic.Uint32 bytesRetransmitted atomic.Uint32 + pacer pacer.Pacer + // update stats onStatsUpdate func(dt *DownTrack, stat *livekit.AnalyticsStat) @@ -249,6 +252,7 @@ func NewDownTrack( bf *buffer.Factory, subID livekit.ParticipantID, mt int, + pacer pacer.Pacer, logger logger.Logger, ) (*DownTrack, error) { var kind webrtc.RTPCodecType @@ -272,6 +276,7 @@ func NewDownTrack( upstreamCodecs: codecs, kind: kind, codec: codecs[0].RTPCodecCapability, + pacer: pacer, } d.forwarder = NewForwarder( d.kind, @@ -471,13 +476,14 @@ func (d *DownTrack) SubscriberID() livekit.ParticipantID { return d.subscriberID // Sets RTP header extensions for this track func (d *DownTrack) SetRTPHeaderExtensions(rtpHeaderExtensions []webrtc.RTPHeaderExtensionParameter) { - d.rtpHeaderExtensions = rtpHeaderExtensions for _, ext := range rtpHeaderExtensions { switch ext.URI { case sdp.ABSSendTimeURI: - d.absSendTimeID = ext.ID + d.absSendTimeExtID = ext.ID + case sdp.TransportCCURI: + d.transportWideExtID = ext.ID case dd.ExtensionUrl: - d.dependencyDescriptorID = ext.ID + d.dependencyDescriptorExtID = ext.ID } } } @@ -561,14 +567,6 @@ func (d *DownTrack) keyFrameRequester(generation uint32, layer int32) { // WriteRTP writes an RTP Packet to the DownTrack func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { - var pool *[]byte - defer func() { - if pool != nil { - PacketFactory.Put(pool) - pool = nil - } - }() - if !d.bound.Load() || !d.connected.Load() { return nil } @@ -581,12 +579,16 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { return err } - payload := extPkt.Packet.Payload + var payload []byte + pool := PacketFactory.Get().(*[]byte) if len(tp.codecBytes) != 0 { incomingVP8, _ := extPkt.Payload.(buffer.VP8) - pool = PacketFactory.Get().(*[]byte) payload = d.translateVP8PacketTo(extPkt.Packet, &incomingVP8, tp.codecBytes, pool) } + if payload == nil { + payload = (*pool)[:len(extPkt.Packet.Payload)] + copy(payload, extPkt.Packet.Payload) + } if d.sequencer != nil { d.sequencer.push( @@ -602,45 +604,28 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { hdr, err := d.getTranslatedRTPHeader(extPkt, tp) if err != nil { d.logger.Errorw("write rtp packet failed", err) - return err - } - - _, err = d.writeStream.WriteRTP(hdr, payload) - if err != nil { - if !errors.Is(err, io.ErrClosedPipe) { - d.logger.Errorw("write rtp packet failed", err) + if pool != nil { + PacketFactory.Put(pool) } return err } - // STREAM-ALLOCATOR-TODO: remove this stream allocator bytes counter once stream allocator changes fully to pull bytes counter - d.streamAllocatorBytesCounter.Add(uint32(hdr.MarshalSize() + len(payload))) - d.bytesSent.Add(uint32(hdr.MarshalSize() + len(payload))) - - if tp.isSwitchingToMaxSpatial && d.onMaxSubscribedLayerChanged != nil && d.kind == webrtc.RTPCodecTypeVideo { - d.onMaxSubscribedLayerChanged(d, tp.maxSpatialLayer) - } - - if extPkt.KeyFrame { - d.isNACKThrottled.Store(false) - d.rtpStats.UpdateKeyFrame(1) - d.logger.Debugw("forwarding key frame", "layer", layer, "rtpsn", hdr.SequenceNumber, "rtpts", hdr.Timestamp) - } - - if tp.isSwitchingToRequestSpatial { - locked, _ := d.forwarder.CheckSync() - if locked { - d.stopKeyFrameRequester() - } - } - - if tp.isResuming { - if sal := d.getStreamAllocatorListener(); sal != nil { - sal.OnResume(d) - } - } - - d.rtpStats.Update(hdr, len(payload), 0, extPkt.Arrival) + d.pacer.Enqueue(pacer.Packet{ + Header: hdr, + Extensions: []pacer.ExtensionData{{ID: uint8(d.dependencyDescriptorExtID), Payload: tp.ddBytes}}, + Payload: payload, + AbsSendTimeExtID: uint8(d.absSendTimeExtID), + TransportWideExtID: uint8(d.transportWideExtID), + WriteStream: d.writeStream, + Metadata: sendPacketMetadata{ + layer: layer, + arrival: extPkt.Arrival, + isKeyFrame: extPkt.KeyFrame, + tp: tp, + pool: pool, + }, + OnSent: d.packetSent, + }) return nil } @@ -704,23 +689,23 @@ func (d *DownTrack) WritePaddingRTP(bytesToSend int, paddingOnMute bool, forceMa CSRC: []uint32{}, } - err = d.writeRTPHeaderExtensions(&hdr) - if err != nil { - return bytesSent - } - payload := make([]byte, RTPPaddingMaxPayloadSize) // last byte of padding has padding size including that byte payload[RTPPaddingMaxPayloadSize-1] = byte(RTPPaddingMaxPayloadSize) - _, err = d.writeStream.WriteRTP(&hdr, payload) - if err != nil { - return bytesSent - } - - if !paddingOnMute { - d.rtpStats.Update(&hdr, 0, len(payload), time.Now()) - } + d.pacer.Enqueue(pacer.Packet{ + Header: &hdr, + Payload: payload, + AbsSendTimeExtID: uint8(d.absSendTimeExtID), + TransportWideExtID: uint8(d.transportWideExtID), + WriteStream: d.writeStream, + Metadata: sendPacketMetadata{ + isPadding: true, + disableCounter: true, + disableRTPStats: paddingOnMute, + }, + OnSent: d.packetSent, + }) // // Register with sequencer with invalid layer so that NACKs for these can be filtered out. @@ -734,6 +719,7 @@ func (d *DownTrack) WritePaddingRTP(bytesToSend int, paddingOnMute bool, forceMa bytesSent += hdr.MarshalSize() + len(payload) } + // STREAM_ALLOCATOR-TODO: change this to pull this counter from stream allocator so that counter can be update in pacer callback return bytesSent } @@ -1123,16 +1109,16 @@ func (d *DownTrack) writeBlankFrameRTP(duration float32, generation uint32) chan return } - var writeBlankFrame func(*rtp.Header, bool) (int, error) + var getBlankFrame func(bool) ([]byte, error) switch d.mime { case "audio/opus": - writeBlankFrame = d.writeOpusBlankFrame + getBlankFrame = d.getOpusBlankFrame case "audio/red": - writeBlankFrame = d.writeOpusRedBlankFrame + getBlankFrame = d.getOpusRedBlankFrame case "video/vp8": - writeBlankFrame = d.writeVP8BlankFrame + getBlankFrame = d.getVP8BlankFrame case "video/h264": - writeBlankFrame = d.writeH264BlankFrame + getBlankFrame = d.getH264BlankFrame default: close(done) return @@ -1177,24 +1163,24 @@ func (d *DownTrack) writeBlankFrameRTP(duration float32, generation uint32) chan CSRC: []uint32{}, } - err = d.writeRTPHeaderExtensions(&hdr) + payload, err := getBlankFrame(frameEndNeeded) if err != nil { - d.logger.Warnw("could not write header extension for blank frame", err) + d.logger.Warnw("could not get blank frame", err) close(done) return } - pktSize, err := writeBlankFrame(&hdr, frameEndNeeded) - if err != nil { - if err != io.ErrClosedPipe { - d.logger.Warnw("could not write blank frame", err) - } - close(done) - return - } - - d.streamAllocatorBytesCounter.Add(uint32(pktSize)) - d.bytesSent.Add(uint32(pktSize)) + d.pacer.Enqueue(pacer.Packet{ + Header: &hdr, + Payload: payload, + AbsSendTimeExtID: uint8(d.absSendTimeExtID), + TransportWideExtID: uint8(d.transportWideExtID), + WriteStream: d.writeStream, + Metadata: sendPacketMetadata{ + isBlankFrame: true, + }, + OnSent: d.packetSent, + }) // only the first frame will need frameEndNeeded to close out the // previous picture, rest are small key frames (for the video case) @@ -1209,22 +1195,17 @@ func (d *DownTrack) writeBlankFrameRTP(duration float32, generation uint32) chan return done } -func (d *DownTrack) writeOpusBlankFrame(hdr *rtp.Header, frameEndNeeded bool) (int, error) { +func (d *DownTrack) getOpusBlankFrame(_frameEndNeeded bool) ([]byte, error) { // silence frame // Used shortly after muting to ensure residual noise does not keep // generating noise at the decoder after the stream is stopped // i. e. comfort noise generation actually not producing something comfortable. payload := make([]byte, len(OpusSilenceFrame)) copy(payload[0:], OpusSilenceFrame) - - _, err := d.writeStream.WriteRTP(hdr, payload) - if err == nil { - d.rtpStats.Update(hdr, len(payload), 0, time.Now()) - } - return hdr.MarshalSize() + len(payload), err + return payload, nil } -func (d *DownTrack) writeOpusRedBlankFrame(hdr *rtp.Header, frameEndNeeded bool) (int, error) { +func (d *DownTrack) getOpusRedBlankFrame(_frameEndNeeded bool) ([]byte, error) { // primary only silence frame for opus/red, there is no need to contain redundant silent frames payload := make([]byte, len(OpusSilenceFrame)+1) @@ -1235,18 +1216,13 @@ func (d *DownTrack) writeOpusRedBlankFrame(hdr *rtp.Header, frameEndNeeded bool) // +-+-+-+-+-+-+-+-+ payload[0] = opusPT copy(payload[1:], OpusSilenceFrame) - - _, err := d.writeStream.WriteRTP(hdr, payload) - if err == nil { - d.rtpStats.Update(hdr, len(payload), 0, time.Now()) - } - return hdr.MarshalSize() + len(payload), err + return payload, nil } -func (d *DownTrack) writeVP8BlankFrame(hdr *rtp.Header, frameEndNeeded bool) (int, error) { +func (d *DownTrack) getVP8BlankFrame(frameEndNeeded bool) ([]byte, error) { blankVP8, err := d.forwarder.GetPadding(frameEndNeeded) if err != nil { - return 0, err + return nil, err } // 8x8 key frame @@ -1256,15 +1232,10 @@ func (d *DownTrack) writeVP8BlankFrame(hdr *rtp.Header, frameEndNeeded bool) (in payload := make([]byte, len(blankVP8)+len(VP8KeyFrame8x8)) copy(payload[:len(blankVP8)], blankVP8) copy(payload[len(blankVP8):], VP8KeyFrame8x8) - - _, err = d.writeStream.WriteRTP(hdr, payload) - if err == nil { - d.rtpStats.Update(hdr, len(payload), 0, time.Now()) - } - return hdr.MarshalSize() + len(payload), err + return payload, nil } -func (d *DownTrack) writeH264BlankFrame(hdr *rtp.Header, frameEndNeeded bool) (int, error) { +func (d *DownTrack) getH264BlankFrame(_frameEndNeeded bool) ([]byte, error) { // TODO - Jie Zeng // now use STAP-A to compose sps, pps, idr together, most decoder support packetization-mode 1. // if client only support packetization-mode 0, use single nalu unit packet @@ -1279,11 +1250,7 @@ func (d *DownTrack) writeH264BlankFrame(hdr *rtp.Header, frameEndNeeded bool) (i offset += len(payload) } payload := buf[:offset] - _, err := d.writeStream.WriteRTP(hdr, payload) - if err == nil { - d.rtpStats.Update(hdr, len(payload), 0, time.Now()) - } - return hdr.MarshalSize() + offset, err + return payload, nil } func (d *DownTrack) handleRTCP(bytes []byte) { @@ -1416,14 +1383,6 @@ func (d *DownTrack) retransmitPackets(nacks []uint16) { return } - var pool *[]byte - defer func() { - if pool != nil { - PacketFactory.Put(pool) - pool = nil - } - }() - src := PacketFactory.Get().(*[]byte) defer PacketFactory.Put(src) @@ -1443,11 +1402,6 @@ func (d *DownTrack) retransmitPackets(nacks []uint16) { Attempts: meta.nacked, }) - if pool != nil { - PacketFactory.Put(pool) - pool = nil - } - pktBuff := *src n, err := d.receiver.ReadRTP(pktBuff, uint8(meta.layer), meta.sourceSeqNo) if err != nil { @@ -1471,41 +1425,38 @@ func (d *DownTrack) retransmitPackets(nacks []uint16) { pkt.Header.SSRC = d.ssrc pkt.Header.PayloadType = d.payloadType - payload := pkt.Payload + var payload []byte + pool := PacketFactory.Get().(*[]byte) if d.mime == "video/vp8" && len(pkt.Payload) > 0 { var incomingVP8 buffer.VP8 if err = incomingVP8.Unmarshal(pkt.Payload); err != nil { d.logger.Errorw("unmarshalling VP8 packet err", err) + PacketFactory.Put(pool) continue } if len(meta.codecBytes) != 0 { - pool = PacketFactory.Get().(*[]byte) payload = d.translateVP8PacketTo(&pkt, &incomingVP8, meta.codecBytes, pool) } } - - var extraExtensions []extensionData - if d.dependencyDescriptorID != 0 && len(meta.ddBytes) != 0 { - extraExtensions = append(extraExtensions, extensionData{ - id: uint8(d.dependencyDescriptorID), - payload: meta.ddBytes, - }) - } - err = d.writeRTPHeaderExtensions(&pkt.Header, extraExtensions...) - if err != nil { - d.logger.Errorw("writing rtp header extensions err", err) - continue + if payload == nil { + payload = (*pool)[:len(pkt.Payload)] + copy(payload, pkt.Payload) } - if _, err = d.writeStream.WriteRTP(&pkt.Header, payload); err != nil { - d.logger.Errorw("writing rtx packet err", err) - } else { - d.streamAllocatorBytesCounter.Add(uint32(pkt.Header.MarshalSize() + len(payload))) - d.bytesRetransmitted.Add(uint32(pkt.Header.MarshalSize() + len(payload))) - - d.rtpStats.Update(&pkt.Header, len(payload), 0, time.Now()) - } + d.pacer.Enqueue(pacer.Packet{ + Header: &pkt.Header, + Extensions: []pacer.ExtensionData{{ID: uint8(d.dependencyDescriptorExtID), Payload: meta.ddBytes}}, + Payload: payload, + AbsSendTimeExtID: uint8(d.absSendTimeExtID), + TransportWideExtID: uint8(d.transportWideExtID), + WriteStream: d.writeStream, + Metadata: sendPacketMetadata{ + isRTX: true, + pool: pool, + }, + OnSent: d.packetSent, + }) } d.totalRepeatedNACKs.Add(numRepeatedNACKs) @@ -1528,38 +1479,6 @@ func (d *DownTrack) retransmitPackets(nacks []uint16) { } } -type extensionData struct { - id uint8 - payload []byte -} - -// writes RTP header extensions of track -func (d *DownTrack) writeRTPHeaderExtensions(hdr *rtp.Header, extraExtensions ...extensionData) error { - // clear out extensions that may have been in the forwarded header - hdr.Extension = false - hdr.ExtensionProfile = 0 - hdr.Extensions = []rtp.Extension{} - - for _, ext := range extraExtensions { - hdr.SetExtension(ext.id, ext.payload) - } - - if d.absSendTimeID != 0 { - sendTime := rtp.NewAbsSendTimeExtension(time.Now()) - b, err := sendTime.Marshal() - if err != nil { - return err - } - - err = hdr.SetExtension(uint8(d.absSendTimeID), b) - if err != nil { - return err - } - } - - return nil -} - func (d *DownTrack) getTranslatedRTPHeader(extPkt *buffer.ExtPacket, tp *TranslationParams) (*rtp.Header, error) { tpRTP := tp.rtp hdr := extPkt.Packet.Header @@ -1571,18 +1490,6 @@ func (d *DownTrack) getTranslatedRTPHeader(extPkt *buffer.ExtPacket, tp *Transla hdr.Marker = tp.marker } - var extension []extensionData - if d.dependencyDescriptorID != 0 && len(tp.ddBytes) != 0 { - extension = append(extension, extensionData{ - id: uint8(d.dependencyDescriptorID), - payload: tp.ddBytes, - }) - } - err := d.writeRTPHeaderExtensions(&hdr, extension...) - if err != nil { - return nil, err - } - return &hdr, nil } @@ -1739,20 +1646,24 @@ func (d *DownTrack) sendSilentFrameOnMuteForOpus() { CSRC: []uint32{}, } - err = d.writeRTPHeaderExtensions(&hdr) + payload, err := d.getOpusBlankFrame(false) if err != nil { - d.logger.Warnw("could not write header extension for blank frame", err) + d.logger.Warnw("could not get blank frame", err) return } - payload := make([]byte, len(OpusSilenceFrame)) - copy(payload[0:], OpusSilenceFrame) - - _, err := d.writeStream.WriteRTP(&hdr, payload) - if err != nil { - d.logger.Warnw("could not write blank frame", err) - return - } + d.pacer.Enqueue(pacer.Packet{ + Header: &hdr, + Payload: payload, + AbsSendTimeExtID: uint8(d.absSendTimeExtID), + TransportWideExtID: uint8(d.transportWideExtID), + WriteStream: d.writeStream, + Metadata: sendPacketMetadata{ + isBlankFrame: true, + disableRTPStats: true, + }, + OnSent: d.packetSent, + }) } numFrames-- @@ -1763,3 +1674,88 @@ func (d *DownTrack) sendSilentFrameOnMuteForOpus() { func (d *DownTrack) HandleRTCPSenderReportData(_payloadType webrtc.PayloadType, _layer int32, _srData *buffer.RTCPSenderReportData) error { return nil } + +type sendPacketMetadata struct { + layer int32 + arrival time.Time + isKeyFrame bool + isRTX bool + isPadding bool + isBlankFrame bool + disableCounter bool + disableRTPStats bool + tp *TranslationParams + pool *[]byte +} + +func (d *DownTrack) packetSent(md interface{}, hdr *rtp.Header, payloadSize int, sendTime time.Time, sendError error) { + spmd, ok := md.(sendPacketMetadata) + if !ok { + d.logger.Errorw("invalid send packet metadata", nil) + return + } + + if spmd.pool != nil { + PacketFactory.Put(spmd.pool) + } + + if sendError != nil { + return + } + + headerSize := hdr.MarshalSize() + if !spmd.disableCounter { + // STREAM-ALLOCATOR-TODO: remove this stream allocator bytes counter once stream allocator changes fully to pull bytes counter + size := uint32(headerSize + payloadSize) + d.streamAllocatorBytesCounter.Add(size) + if spmd.isRTX { + d.bytesRetransmitted.Add(size) + } else { + d.bytesSent.Add(size) + } + } + + if !spmd.disableRTPStats { + packetTime := spmd.arrival + if packetTime.IsZero() { + packetTime = sendTime + } + if spmd.isPadding { + d.rtpStats.Update(hdr, 0, payloadSize, packetTime) + } else { + d.rtpStats.Update(hdr, payloadSize, 0, packetTime) + } + } + + if spmd.isKeyFrame { + d.isNACKThrottled.Store(false) + d.rtpStats.UpdateKeyFrame(1) + d.logger.Debugw( + "forwarding key frame", + "layer", spmd.layer, + "rtpsn", hdr.SequenceNumber, + "rtpts", hdr.Timestamp, + ) + } + + if spmd.tp != nil { + if spmd.tp.isSwitchingToMaxSpatial && d.onMaxSubscribedLayerChanged != nil && d.kind == webrtc.RTPCodecTypeVideo { + d.onMaxSubscribedLayerChanged(d, spmd.tp.maxSpatialLayer) + } + + if spmd.tp.isSwitchingToRequestSpatial { + locked, _ := d.forwarder.CheckSync() + if locked { + d.stopKeyFrameRequester() + } + } + + if spmd.tp.isResuming { + if sal := d.getStreamAllocatorListener(); sal != nil { + sal.OnResume(d) + } + } + } +} + +// ------------------------------------------------------------------------------- diff --git a/pkg/sfu/pacer/base.go b/pkg/sfu/pacer/base.go new file mode 100644 index 000000000..fef7b413e --- /dev/null +++ b/pkg/sfu/pacer/base.go @@ -0,0 +1,83 @@ +package pacer + +import ( + "errors" + "io" + "time" + + "github.com/livekit/protocol/logger" + "github.com/pion/rtp" +) + +type Base struct { + logger logger.Logger + + packetTime *PacketTime +} + +func NewBase(logger logger.Logger) *Base { + return &Base{ + logger: logger, + packetTime: NewPacketTime(), + } +} + +func (b *Base) SendPacket(p *Packet) error { + var sendingAt time.Time + var err error + defer func() { + if p.OnSent != nil { + p.OnSent(p.Metadata, p.Header, len(p.Payload), sendingAt, err) + } + }() + + sendingAt, err = b.writeRTPHeaderExtensions(p) + if err != nil { + b.logger.Errorw("writing rtp header extensions err", err) + return err + } + + _, err = p.WriteStream.WriteRTP(p.Header, p.Payload) + if err != nil { + if !errors.Is(err, io.ErrClosedPipe) { + b.logger.Errorw("write rtp packet failed", err) + } + return err + } + + return nil +} + +// writes RTP header extensions of track +func (b *Base) writeRTPHeaderExtensions(p *Packet) (time.Time, error) { + // clear out extensions that may have been in the forwarded header + p.Header.Extension = false + p.Header.ExtensionProfile = 0 + p.Header.Extensions = []rtp.Extension{} + + for _, ext := range p.Extensions { + if ext.ID == 0 || len(ext.Payload) == 0 { + continue + } + + p.Header.SetExtension(ext.ID, ext.Payload) + } + + sendingAt := b.packetTime.Get() + if p.AbsSendTimeExtID != 0 { + sendTime := rtp.NewAbsSendTimeExtension(sendingAt) + b, err := sendTime.Marshal() + if err != nil { + return time.Time{}, err + } + + err = p.Header.SetExtension(p.AbsSendTimeExtID, b) + if err != nil { + return time.Time{}, err + } + } + + return sendingAt, nil +} + +// ------------------------------------------------ diff --git a/pkg/sfu/pacer/no_queue.go b/pkg/sfu/pacer/no_queue.go new file mode 100644 index 000000000..b34b994ae --- /dev/null +++ b/pkg/sfu/pacer/no_queue.go @@ -0,0 +1,80 @@ +package pacer + +import ( + "sync" + + "github.com/gammazero/deque" + "github.com/livekit/protocol/logger" +) + +type NoQueue struct { + *Base + + logger logger.Logger + + lock sync.RWMutex + packets deque.Deque[Packet] + wake chan struct{} + isStopped bool +} + +func NewNoQueue(logger logger.Logger) *NoQueue { + n := &NoQueue{ + Base: NewBase(logger), + logger: logger, + wake: make(chan struct{}, 1), + } + n.packets.SetMinCapacity(9) + + go n.sendWorker() + return n +} + +func (n *NoQueue) Stop() { + n.lock.Lock() + if n.isStopped { + n.lock.Unlock() + return + } + + close(n.wake) + n.isStopped = true + n.lock.Unlock() +} + +func (n *NoQueue) Enqueue(p Packet) { + n.lock.Lock() + defer n.lock.Unlock() + + n.packets.PushBack(p) + if n.packets.Len() == 1 && !n.isStopped { + select { + case n.wake <- struct{}{}: + default: + } + } +} + +func (n *NoQueue) sendWorker() { + for { + <-n.wake + for { + n.lock.Lock() + if n.isStopped { + n.lock.Unlock() + return + } + + if n.packets.Len() == 0 { + n.lock.Unlock() + break + } + p := n.packets.PopFront() + n.lock.Unlock() + + n.Base.SendPacket(&p) + } + } +} + +// ------------------------------------------------ diff --git a/pkg/sfu/pacer/pacer.go b/pkg/sfu/pacer/pacer.go new file mode 100644 index 000000000..3be8a8a86 --- /dev/null +++ b/pkg/sfu/pacer/pacer.go @@ -0,0 +1,31 @@ +package pacer + +import ( + "time" + + "github.com/pion/rtp" + "github.com/pion/webrtc/v3" +) + +type ExtensionData struct { + ID uint8 + Payload []byte +} + +type Packet struct { + Header *rtp.Header + Extensions []ExtensionData + Payload []byte + AbsSendTimeExtID uint8 + TransportWideExtID uint8 + WriteStream webrtc.TrackLocalWriter + Metadata interface{} + OnSent func(md interface{}, sentHeader *rtp.Header, payloadSize int, sentTime time.Time, sendError error) +} + +type Pacer interface { + Enqueue(p Packet) + Stop() +} + +// ------------------------------------------------ diff --git a/pkg/sfu/pacer/packet_time.go b/pkg/sfu/pacer/packet_time.go new file mode 100644 index 000000000..3dce57e3a --- /dev/null +++ b/pkg/sfu/pacer/packet_time.go @@ -0,0 +1,22 @@ +package pacer + +import ( + "time" +) + +type PacketTime struct { + baseTime time.Time +} + +func NewPacketTime() *PacketTime { + return &PacketTime{ + baseTime: time.Now(), + } +} + +func (p *PacketTime) Get() time.Time { + // construct current time based on monotonic clock + return p.baseTime.Add(time.Since(p.baseTime)) +} + +// ------------------------------------------------ diff --git a/pkg/sfu/pacer/pass_through.go b/pkg/sfu/pacer/pass_through.go new file mode 100644 index 000000000..ccbefbd61 --- /dev/null +++ b/pkg/sfu/pacer/pass_through.go @@ -0,0 +1,24 @@ +package pacer + +import ( + "github.com/livekit/protocol/logger" +) + +type PassThrough struct { + *Base +} + +func NewPassThrough(logger logger.Logger) *PassThrough { + return &PassThrough{ + Base: NewBase(logger), + } +} + +func (p *PassThrough) Stop() { +} + +func (p *PassThrough) Enqueue(pkt Packet) { + p.Base.SendPacket(&pkt) +} + +// ------------------------------------------------ From 2668073c292560b27f5b791786eef56e2de95d6a Mon Sep 17 00:00:00 2001 From: Juan Navarro Date: Thu, 29 Jun 2023 01:52:43 +0200 Subject: [PATCH 257/324] Honor bind address passed as `--bind` also for RTC ports (#1815) * Use net.JoinHostPort to build "host:port" strings for `net.Listen` net.JoinHostPort provides a unified way of building strings of the form "Host:Port", abstracting the particular syntax requirements of some methods in the `net` package (namely, that IPv4 addresses can be given as-is to `net.Listen`, but IPv6 addresses must be given enclosed in square brackets). This change makes sense because an address such as `[::1]` is *not* a valid IPv6 address; the square brackets are just a detail particular to the Go `net` library. As such, this syntax shouldn't be exposed to the user, and configuration should just accept valid IPv6 addresses and convert them as needed for usage within the code. * Use '--bind' CLI flag to also filter RTC bind address The local address passed to a command such as livekit-server --dev --bind 127.0.0.1 was being used as binding address for the TCP WebSocket port, but was being ignored for RTC connections. With `--dev`, the conf.RTC.UDPPort config is set to 7882, which enables "UDP muxing" mechanism. Without interface or address filtering, Pion would try to bind to port 7882 on *all* interfaces. This was failing on a system with IPv6 enabled, when trying to bind to an IPv6 address of the `docker0` interface. It seems to make sense that the user-passed bind addresses are also honored for the RTC port bindings. --- cmd/server/main.go | 5 ++++- pkg/service/server.go | 5 +++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/cmd/server/main.go b/cmd/server/main.go index 29b8a6588..5e93b7cd2 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -208,9 +208,12 @@ func getConfig(c *cli.Context) (*config.Config, error) { if conf.BindAddresses == nil { conf.BindAddresses = []string{ "127.0.0.1", - "[::1]", + "::1", } } + for _, bindAddr := range conf.BindAddresses { + conf.RTC.IPs.Includes = append(conf.RTC.IPs.Includes, bindAddr + "/24") + } } } return conf, nil diff --git a/pkg/service/server.go b/pkg/service/server.go index 69e3d404c..c21b98fe0 100644 --- a/pkg/service/server.go +++ b/pkg/service/server.go @@ -10,6 +10,7 @@ import ( _ "net/http/pprof" "runtime" "runtime/pprof" + "strconv" "time" "github.com/pion/turn/v2" @@ -177,14 +178,14 @@ func (s *LivekitServer) Start() error { listeners := make([]net.Listener, 0) promListeners := make([]net.Listener, 0) for _, addr := range addresses { - ln, err := net.Listen("tcp", fmt.Sprintf("%s:%d", addr, s.config.Port)) + ln, err := net.Listen("tcp", net.JoinHostPort(addr, strconv.Itoa(int(s.config.Port)))) if err != nil { return err } listeners = append(listeners, ln) if s.promServer != nil { - ln, err = net.Listen("tcp", fmt.Sprintf("%s:%d", addr, s.config.PrometheusPort)) + ln, err = net.Listen("tcp", net.JoinHostPort(addr, strconv.Itoa(int(s.config.PrometheusPort)))) if err != nil { return err } From 7be9e2258d213521010b322a1689436d2704c5c3 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Wed, 28 Jun 2023 16:53:58 -0700 Subject: [PATCH 258/324] Upgrade to Pion 3.0.11, disable active TCP (#1836) --- go.mod | 8 ++++---- go.sum | 14 ++++++++------ pkg/rtc/config.go | 3 +++ pkg/rtc/transport.go | 1 + 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 0bef11416..dbeb57513 100644 --- a/go.mod +++ b/go.mod @@ -26,14 +26,14 @@ require ( github.com/mitchellh/go-homedir v1.1.0 github.com/olekukonko/tablewriter v0.0.5 github.com/pion/dtls/v2 v2.2.7 - github.com/pion/ice/v2 v2.3.6 + github.com/pion/ice/v2 v2.3.8 github.com/pion/interceptor v0.1.17 github.com/pion/rtcp v1.2.10 github.com/pion/rtp v1.7.13 github.com/pion/sdp/v3 v3.0.6 github.com/pion/transport/v2 v2.2.1 - github.com/pion/turn/v2 v2.1.0 - github.com/pion/webrtc/v3 v3.2.9 + github.com/pion/turn/v2 v2.1.2 + github.com/pion/webrtc/v3 v3.2.11 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.16.0 github.com/redis/go-redis/v9 v9.0.5 @@ -84,7 +84,7 @@ require ( github.com/pion/randutil v0.1.0 // indirect github.com/pion/sctp v1.8.7 // indirect github.com/pion/srtp/v2 v2.0.15 // indirect - github.com/pion/stun v0.6.0 // indirect + github.com/pion/stun v0.6.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect diff --git a/go.sum b/go.sum index 63da1ff1d..73021ab72 100644 --- a/go.sum +++ b/go.sum @@ -184,8 +184,8 @@ github.com/pion/datachannel v1.5.5 h1:10ef4kwdjije+M9d7Xm9im2Y3O6A6ccQb0zcqZcJew github.com/pion/datachannel v1.5.5/go.mod h1:iMz+lECmfdCMqFRhXhcA/219B0SQlbpoR2V118yimL0= github.com/pion/dtls/v2 v2.2.7 h1:cSUBsETxepsCSFSxC3mc/aDo14qQLMSL+O6IjG28yV8= github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= -github.com/pion/ice/v2 v2.3.6 h1:Jgqw36cAud47iD+N6rNX225uHvrgWtAlHfVyOQc3Heg= -github.com/pion/ice/v2 v2.3.6/go.mod h1:9/TzKDRwBVAPsC+YOrKH/e3xDrubeTRACU9/sHQarsU= +github.com/pion/ice/v2 v2.3.8 h1:/4vM7uFPJez3PhNhlqUcJhboYaDNWo+R8oAuMj2cKsA= +github.com/pion/ice/v2 v2.3.8/go.mod h1:DoMA9FvsfNTBVnjyRf2t4EhUkSp9tNrH77fMtPFYygQ= github.com/pion/interceptor v0.1.17 h1:prJtgwFh/gB8zMqGZoOgJPHivOwVAp61i2aG61Du/1w= github.com/pion/interceptor v0.1.17/go.mod h1:SY8kpmfVBvrbUzvj2bsXz7OJt5JvmVNZ+4Kjq7FcwrI= github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= @@ -206,8 +206,9 @@ github.com/pion/sdp/v3 v3.0.6/go.mod h1:iiFWFpQO8Fy3S5ldclBkpXqmWy02ns78NOKoLLL0 github.com/pion/srtp/v2 v2.0.15 h1:+tqRtXGsGwHC0G0IUIAzRmdkHvriF79IHVfZGfHrQoA= github.com/pion/srtp/v2 v2.0.15/go.mod h1:b/pQOlDrbB0HEH5EUAQXzSYxikFbNcNuKmF8tM0hCtw= github.com/pion/stun v0.4.0/go.mod h1:QPsh1/SbXASntw3zkkrIk3ZJVKz4saBY2G7S10P3wCw= -github.com/pion/stun v0.6.0 h1:JHT/2iyGDPrFWE8NNC15wnddBN8KifsEDw8swQmrEmU= github.com/pion/stun v0.6.0/go.mod h1:HPqcfoeqQn9cuaet7AOmB5e5xkObu9DwBdurwLKO9oA= +github.com/pion/stun v0.6.1 h1:8lp6YejULeHBF8NmV8e2787BogQhduZugh5PdhDyyN4= +github.com/pion/stun v0.6.1/go.mod h1:/hO7APkX4hZKu/D0f2lHzNyvdkTGtIy3NDmLR7kSz/8= github.com/pion/transport v0.14.1 h1:XSM6olwW+o8J4SCmOBb/BpwZypkHeyM0PGFCxNQBr40= github.com/pion/transport v0.14.1/go.mod h1:4tGmbk00NeYA3rUa9+n+dzCCoKkcy3YlYb99Jn2fNnI= github.com/pion/transport/v2 v2.0.0/go.mod h1:HS2MEBJTwD+1ZI2eSXSvHJx/HnzQqRy2/LXxt6eVMHc= @@ -215,10 +216,11 @@ github.com/pion/transport/v2 v2.1.0/go.mod h1:AdSw4YBZVDkZm8fpoz+fclXyQwANWmZAlD github.com/pion/transport/v2 v2.2.0/go.mod h1:AdSw4YBZVDkZm8fpoz+fclXyQwANWmZAlDuQdctTThQ= github.com/pion/transport/v2 v2.2.1 h1:7qYnCBlpgSJNYMbLCKuSY9KbQdBFoETvPNETv0y4N7c= github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g= -github.com/pion/turn/v2 v2.1.0 h1:5wGHSgGhJhP/RpabkUb/T9PdsAjkGLS6toYz5HNzoSI= github.com/pion/turn/v2 v2.1.0/go.mod h1:yrT5XbXSGX1VFSF31A3c1kCNB5bBZgk/uu5LET162qs= -github.com/pion/webrtc/v3 v3.2.9 h1:U8NSjQDlZZ+Iy/hg42Q/u6mhEVSXYvKrOIZiZwYTfLc= -github.com/pion/webrtc/v3 v3.2.9/go.mod h1:gjQLMZeyN3jXBGdxGmUYCyKjOuYX/c99BDjGqmadq0A= +github.com/pion/turn/v2 v2.1.2 h1:wj0cAoGKltaZ790XEGW9HwoUewqjliwmhtxCuB2ApyM= +github.com/pion/turn/v2 v2.1.2/go.mod h1:1kjnPkBcex3dhCU2Am+AAmxDcGhLX3WnMfmkNpvSTQU= +github.com/pion/webrtc/v3 v3.2.11 h1:lfGKYZcG7ghCTQWn+zsD+icIIWL3qIfclEjBGk537+s= +github.com/pion/webrtc/v3 v3.2.11/go.mod h1:fejQio1v8tKG4ntq4u8H4uDHsCNX6eX7bT093t4H+0E= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/pkg/rtc/config.go b/pkg/rtc/config.go index f3dbc2402..c47bcecbc 100644 --- a/pkg/rtc/config.go +++ b/pkg/rtc/config.go @@ -51,6 +51,9 @@ func NewWebRTCConfig(conf *config.Config) (*WebRTCConfig, error) { return nil, err } + // we don't want to use active TCP on a server, clients should be dialing + webRTCConfig.SettingEngine.DisableActiveTCP(true) + if rtcConf.PacketBufferSize == 0 { rtcConf.PacketBufferSize = 500 } diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index a164b7eac..d924a141e 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -612,6 +612,7 @@ func (t *PCTransport) onPeerConnectionStateChange(state webrtc.PeerConnectionSta } t.maybeNotifyFullyEstablished() + t.logICECandidates() } case webrtc.PeerConnectionStateFailed: t.params.Logger.Infow("peer connection failed") From 4952c641b3a759a562f0664059738d32f63da876 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Wed, 28 Jun 2023 20:22:59 -0700 Subject: [PATCH 259/324] Fix regression when bind-address is not explicitly provided (#1837) when bind address is set to loopback, it would break RTC IP discovery. --- cmd/server/main.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/cmd/server/main.go b/cmd/server/main.go index 5e93b7cd2..1092bd539 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -3,6 +3,7 @@ package main import ( "fmt" "math/rand" + "net" "os" "os/signal" "runtime" @@ -204,15 +205,27 @@ func getConfig(c *cli.Context) (*config.Config, error) { conf.Keys = map[string]string{ "devkey": "secret", } + shouldMatchRTCIP := false // when dev mode and using shared keys, we'll bind to localhost by default if conf.BindAddresses == nil { conf.BindAddresses = []string{ "127.0.0.1", "::1", } + } else { + // if non-loopback addresses are provided, then we'll match RTC IP to bind address + // our IP discovery ignores loopback addresses + for _, addr := range conf.BindAddresses { + ip := net.ParseIP(addr) + if ip != nil && !ip.IsLoopback() { + shouldMatchRTCIP = true + } + } } - for _, bindAddr := range conf.BindAddresses { - conf.RTC.IPs.Includes = append(conf.RTC.IPs.Includes, bindAddr + "/24") + if shouldMatchRTCIP { + for _, bindAddr := range conf.BindAddresses { + conf.RTC.IPs.Includes = append(conf.RTC.IPs.Includes, bindAddr+"/24") + } } } } From 3de1e49165d022ecbe88670897984c24bf67cc5e Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 29 Jun 2023 20:31:17 -0700 Subject: [PATCH 260/324] Update module github.com/livekit/protocol to v1.5.8 (#1733) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index dbeb57513..9153ffb53 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230612070454-d5299b956135 - github.com/livekit/protocol v1.5.8-0.20230620161627-ce9e603cfda8 + github.com/livekit/protocol v1.5.8 github.com/livekit/psrpc v0.3.1 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 diff --git a/go.sum b/go.sum index 73021ab72..89c61e2f4 100644 --- a/go.sum +++ b/go.sum @@ -124,8 +124,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20230612070454-d5299b956135 h1:lWYbsondvqG69czxoACDwaJ/BoyD57BahCo70ZH+m4U= github.com/livekit/mediatransportutil v0.0.0-20230612070454-d5299b956135/go.mod h1:MRc0zSOSzXuFt0X218SgabzlaKevkvCckPgBEoHYc34= -github.com/livekit/protocol v1.5.8-0.20230620161627-ce9e603cfda8 h1:pri2aylzPrDDTjBKQQdcYsYqwjv9J7W8CnEkrbnF0lU= -github.com/livekit/protocol v1.5.8-0.20230620161627-ce9e603cfda8/go.mod h1:B6hJiuXT84dHsUgaKHBo+ZLPX4XhklptYA2UbANSiNg= +github.com/livekit/protocol v1.5.8 h1:Q8Ctyq3rhv+trg0Hnd2rKncKAa9QAMlQO7zAUxvz4TU= +github.com/livekit/protocol v1.5.8/go.mod h1:B6hJiuXT84dHsUgaKHBo+ZLPX4XhklptYA2UbANSiNg= github.com/livekit/psrpc v0.3.1 h1:KfylgJHvoLQcc22t/oflwMOeSnx0c14G7cWsS+9MYS4= github.com/livekit/psrpc v0.3.1/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= From 69a1e572be76dbf3e4e42119f191e448926d9c21 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 30 Jun 2023 11:09:46 +0530 Subject: [PATCH 261/324] Attempt to reduce disruption due to probe. (#1839) * Make congestion controller probe config * Wait for enough estimate samples * fixes * format * limit number of times a packet is ACKed * ramp up probe duration * go format * correct comment * restore default * add float64 type to generated CLI --- pkg/config/config.go | 55 ++++++- pkg/sfu/sequencer.go | 3 +- pkg/sfu/streamallocator/channelobserver.go | 4 + pkg/sfu/streamallocator/probe_controller.go | 167 ++++++++++++-------- pkg/sfu/streamallocator/prober.go | 20 +-- pkg/sfu/streamallocator/streamallocator.go | 41 +++-- pkg/sfu/streamallocator/trenddetector.go | 4 + 7 files changed, 192 insertions(+), 102 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 325eec68c..77bfe4adf 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -105,12 +105,31 @@ type PLIThrottleConfig struct { HighQuality time.Duration `yaml:"high_quality,omitempty"` } +type CongestionControlProbeConfig struct { + BaseInterval time.Duration `yaml:"base_interval,omitempty"` + BackoffFactor float64 `yaml:"backoff_factor,omitempty"` + MaxInterval time.Duration `yaml:"max_interval,omitempty"` + + SettleWait time.Duration `yaml:"settle_wait,omitempty"` + SettleWaitMax time.Duration `yaml:"settle_wait_max,omitempty"` + + TrendWait time.Duration `yaml:"trend_wait,omitempty"` + + OveragePct int64 `yaml:"overage_pct,omitempty"` + MinBps int64 `yaml:"min_bps,omitempty"` + MinDuration time.Duration `yaml:"min_duration,omitempty"` + MaxDuration time.Duration `yaml:"max_duration,omitempty"` + DurationOverflowFactor float64 `yaml:"duration_overflow_factor,omitempty"` + DurationIncreaseFactor float64 `yaml:"duration_increase_factor,omitempty"` +} + type CongestionControlConfig struct { - Enabled bool `yaml:"enabled"` - AllowPause bool `yaml:"allow_pause"` - UseSendSideBWE bool `yaml:"send_side_bandwidth_estimation,omitempty"` - ProbeMode CongestionControlProbeMode `yaml:"padding_mode,omitempty"` - MinChannelCapacity int64 `yaml:"min_channel_capacity,omitempty"` + Enabled bool `yaml:"enabled"` + AllowPause bool `yaml:"allow_pause"` + UseSendSideBWE bool `yaml:"send_side_bandwidth_estimation,omitempty"` + ProbeMode CongestionControlProbeMode `yaml:"padding_mode,omitempty"` + MinChannelCapacity int64 `yaml:"min_channel_capacity,omitempty"` + ProbeConfig CongestionControlProbeConfig `yaml:"probe_config,omitempty"` } type AudioConfig struct { @@ -269,6 +288,23 @@ func NewConfig(confString string, strictMode bool, c *cli.Context, baseFlags []c Enabled: true, AllowPause: false, ProbeMode: CongestionControlProbeModePadding, + ProbeConfig: CongestionControlProbeConfig{ + BaseInterval: 3 * time.Second, + BackoffFactor: 1.5, + MaxInterval: 2 * time.Minute, + + SettleWait: 250 * time.Millisecond, + SettleWaitMax: 10 * time.Second, + + TrendWait: 2 * time.Second, + + OveragePct: 120, + MinBps: 200_000, + MinDuration: 200 * time.Millisecond, + MaxDuration: 20 * time.Second, + DurationOverflowFactor: 1.25, + DurationIncreaseFactor: 1.5, + }, }, }, Audio: AudioConfig{ @@ -596,6 +632,13 @@ func GenerateCLIFlags(existingFlags []cli.Flag, hidden bool) ([]cli.Flag, error) Usage: generatedCLIFlagUsage, Hidden: hidden, } + case reflect.Float64: + flag = &cli.Float64Flag{ + Name: name, + EnvVars: []string{envVar}, + Usage: generatedCLIFlagUsage, + Hidden: hidden, + } case reflect.Slice: // TODO continue @@ -647,6 +690,8 @@ func (conf *Config) updateFromCLI(c *cli.Context, baseFlags []cli.Flag) error { configValue.SetUint(c.Uint64(flagName)) case reflect.Float32: configValue.SetFloat(c.Float64(flagName)) + case reflect.Float64: + configValue.SetFloat(c.Float64(flagName)) // case reflect.Slice: // // TODO // case reflect.Map: diff --git a/pkg/sfu/sequencer.go b/pkg/sfu/sequencer.go index c3c2e4302..4a8b31f9c 100644 --- a/pkg/sfu/sequencer.go +++ b/pkg/sfu/sequencer.go @@ -11,6 +11,7 @@ import ( const ( defaultRtt = 70 ignoreRetransmission = 100 // Ignore packet retransmission after ignoreRetransmission milliseconds + maxAck = 3 ) func btoi(b bool) int { @@ -187,7 +188,7 @@ func (s *sequencer) getPacketsMeta(seqNo []uint16) []packetMeta { continue } - if seq.lastNack == 0 || refTime-seq.lastNack > uint32(math.Min(float64(ignoreRetransmission), float64(2*s.rtt))) { + if (seq.lastNack == 0 || refTime-seq.lastNack > uint32(math.Min(float64(ignoreRetransmission), float64(2*s.rtt)))) && seq.nacked < maxAck { seq.nacked++ seq.lastNack = refTime diff --git a/pkg/sfu/streamallocator/channelobserver.go b/pkg/sfu/streamallocator/channelobserver.go index 9c99e90da..0e094bdad 100644 --- a/pkg/sfu/streamallocator/channelobserver.go +++ b/pkg/sfu/streamallocator/channelobserver.go @@ -120,6 +120,10 @@ func (c *ChannelObserver) GetHighestEstimate() int64 { return c.estimateTrend.GetHighest() } +func (c *ChannelObserver) HasEnoughEstimateSamples() bool { + return c.estimateTrend.HasEnoughSamples() +} + func (c *ChannelObserver) GetNackRatio() float64 { return c.nackTracker.GetRatio() } diff --git a/pkg/sfu/streamallocator/probe_controller.go b/pkg/sfu/streamallocator/probe_controller.go index 9fc80ad53..1d6bfd352 100644 --- a/pkg/sfu/streamallocator/probe_controller.go +++ b/pkg/sfu/streamallocator/probe_controller.go @@ -4,26 +4,14 @@ import ( "sync" "time" + "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/protocol/logger" ) -const ( - ProbeWaitBase = 5 * time.Second - ProbeBackoffFactor = 1.5 - ProbeWaitMax = 2 * time.Minute - ProbeSettleWait = 250 - ProbeSettleWaitMax = 10 * time.Second - ProbeTrendWait = 2 * time.Second - - ProbePct = 120 - ProbeMinBps = 200 * 1000 // 200 kbps - ProbeMinDuration = 20 * time.Second - ProbeMaxDuration = 21 * time.Second -) - // --------------------------------------------------------------------------- type ProbeControllerParams struct { + Config config.CongestionControlProbeConfig Prober *Prober Logger logger.Logger } @@ -31,19 +19,23 @@ type ProbeControllerParams struct { type ProbeController struct { params ProbeControllerParams - lock sync.RWMutex - probeInterval time.Duration - lastProbeStartTime time.Time - probeGoalBps int64 - probeClusterId ProbeClusterId - abortedProbeClusterId ProbeClusterId - probeTrendObserved bool - probeEndTime time.Time + lock sync.RWMutex + probeInterval time.Duration + lastProbeStartTime time.Time + probeGoalBps int64 + probeClusterId ProbeClusterId + doneProbeClusterInfo ProbeClusterInfo + abortedProbeClusterId ProbeClusterId + goalReachedProbeClusterId ProbeClusterId + probeTrendObserved bool + probeEndTime time.Time + probeDuration time.Duration } func NewProbeController(params ProbeControllerParams) *ProbeController { p := &ProbeController{ - params: params, + params: params, + probeDuration: params.Config.MinDuration, } p.Reset() @@ -57,45 +49,21 @@ func (p *ProbeController) Reset() { p.lastProbeStartTime = time.Now() p.resetProbeIntervalLocked() + p.resetProbeDurationLocked() p.clearProbeLocked() } -func (p *ProbeController) ProbeClusterDone(info ProbeClusterInfo, lowestEstimate int64) bool { +func (p *ProbeController) ProbeClusterDone(info ProbeClusterInfo) { p.lock.Lock() defer p.lock.Unlock() if p.probeClusterId != info.Id { p.params.Logger.Infow("not expected probe cluster", "probeClusterId", p.probeClusterId, "resetProbeClusterId", info.Id) - return false + return } - if p.abortedProbeClusterId == ProbeClusterIdInvalid { - // successful probe, finalize - return p.finalizeProbeLocked() - } - - // ensure probe queue is flushed - // STREAM-ALLOCATOR-TODO: ProbeSettleWait should actually be a certain number of RTTs. - expectedDuration := float64(info.BytesSent*8*1000) / float64(lowestEstimate) - queueTime := expectedDuration - float64(info.Duration.Milliseconds()) - if queueTime < 0.0 { - queueTime = 0.0 - } - queueWait := time.Duration(queueTime+float64(ProbeSettleWait)) * time.Millisecond - if queueWait > ProbeSettleWaitMax { - queueWait = ProbeSettleWaitMax - } - p.probeEndTime = p.lastProbeStartTime.Add(queueWait + info.Duration) - p.params.Logger.Infow( - "setting probe end time", - "probeClusterId", p.probeClusterId, - "expectedDuration", expectedDuration, - "queueTime", queueTime, - "queueWait", queueWait, - "probeEndTime", p.probeEndTime, - ) - return false + p.doneProbeClusterInfo = info } func (p *ProbeController) CheckProbe(trend ChannelTrend, highestEstimate int64) { @@ -111,7 +79,7 @@ func (p *ProbeController) CheckProbe(trend ChannelTrend, highestEstimate int64) } switch { - case !p.probeTrendObserved && time.Since(p.lastProbeStartTime) > ProbeTrendWait: + case !p.probeTrendObserved && time.Since(p.lastProbeStartTime) > p.params.Config.TrendWait: // // More of a safety net. // In rare cases, the estimate gets stuck. Prevent from probe running amok @@ -133,41 +101,87 @@ func (p *ProbeController) CheckProbe(trend ChannelTrend, highestEstimate int64) "goal", p.probeGoalBps, "highest", highestEstimate, ) + p.goalReachedProbeClusterId = p.probeClusterId p.StopProbe() } } -func (p *ProbeController) MaybeFinalizeProbe() (isHandled bool, isSuccessful bool) { +func (p *ProbeController) MaybeFinalizeProbe( + isComplete bool, + trend ChannelTrend, + lowestEstimate int64, +) (isHandled bool, isNotFailing bool, isGoalReached bool) { p.lock.Lock() defer p.lock.Unlock() - if p.isInProbeLocked() && !p.probeEndTime.IsZero() && time.Now().After(p.probeEndTime) { - return true, p.finalizeProbeLocked() + if !p.isInProbeLocked() { + return false, false, false } - return false, false + if p.goalReachedProbeClusterId != ProbeClusterIdInvalid { + // finalise goal reached probe cluster + p.finalizeProbeLocked(ChannelTrendNeutral) + return true, true, true + } + + if (isComplete || p.abortedProbeClusterId != ProbeClusterIdInvalid) && p.probeEndTime.IsZero() && p.doneProbeClusterInfo.Id != ProbeClusterIdInvalid && p.doneProbeClusterInfo.Id == p.probeClusterId { + // ensure any queueing due to probing is flushed + // STREAM-ALLOCATOR-TODO: CongestionControlProbeConfig.SettleWait should actually be a certain number of RTTs. + expectedDuration := float64(9.0) + if lowestEstimate != 0 { + expectedDuration = float64(p.doneProbeClusterInfo.BytesSent*8*1000) / float64(lowestEstimate) + } + queueTime := expectedDuration - float64(p.doneProbeClusterInfo.Duration.Milliseconds()) + if queueTime < 0.0 { + queueTime = 0.0 + } + queueWait := (time.Duration(queueTime) * time.Millisecond) + p.params.Config.SettleWait + if queueWait > p.params.Config.SettleWaitMax { + queueWait = p.params.Config.SettleWaitMax + } + p.probeEndTime = p.lastProbeStartTime.Add(queueWait + p.doneProbeClusterInfo.Duration) + p.params.Logger.Infow( + "setting probe end time", + "probeClusterId", p.probeClusterId, + "expectedDuration", expectedDuration, + "queueTime", queueTime, + "queueWait", queueWait, + "probeEndTime", p.probeEndTime, + ) + } + + if !p.probeEndTime.IsZero() && time.Now().After(p.probeEndTime) { + // finalisze aborted or non-failing but non-goal-reached probe cluster + return true, p.finalizeProbeLocked(trend), false + } + + return false, false, false } func (p *ProbeController) DoesProbeNeedFinalize() bool { p.lock.RLock() defer p.lock.RUnlock() - return p.abortedProbeClusterId != ProbeClusterIdInvalid + return p.abortedProbeClusterId != ProbeClusterIdInvalid || p.goalReachedProbeClusterId != ProbeClusterIdInvalid } -func (p *ProbeController) finalizeProbeLocked() bool { +func (p *ProbeController) finalizeProbeLocked(trend ChannelTrend) (isNotFailing bool) { aborted := p.probeClusterId == p.abortedProbeClusterId p.clearProbeLocked() - if aborted { + if aborted || trend == ChannelTrendCongesting { // failed probe, backoff p.backoffProbeIntervalLocked() + p.resetProbeDurationLocked() return false } - // reset probe interval on a successful probe + // reset probe interval and increase probe duration on a upward trending probe p.resetProbeIntervalLocked() + if trend == ChannelTrendClearing { + p.increaseProbeDurationLocked() + } return true } @@ -178,13 +192,15 @@ func (p *ProbeController) InitProbe(probeGoalDeltaBps int64, expectedBandwidthUs p.lastProbeStartTime = time.Now() // overshoot a bit to account for noise (in measurement/estimate etc) - desiredIncreaseBps := (probeGoalDeltaBps * ProbePct) / 100 - if desiredIncreaseBps < ProbeMinBps { - desiredIncreaseBps = ProbeMinBps + desiredIncreaseBps := (probeGoalDeltaBps * p.params.Config.OveragePct) / 100 + if desiredIncreaseBps < p.params.Config.MinBps { + desiredIncreaseBps = p.params.Config.MinBps } p.probeGoalBps = expectedBandwidthUsage + desiredIncreaseBps + p.doneProbeClusterInfo = ProbeClusterInfo{Id: ProbeClusterIdInvalid} p.abortedProbeClusterId = ProbeClusterIdInvalid + p.goalReachedProbeClusterId = ProbeClusterIdInvalid p.probeTrendObserved = false @@ -194,8 +210,8 @@ func (p *ProbeController) InitProbe(probeGoalDeltaBps int64, expectedBandwidthUs ProbeClusterModeUniform, int(p.probeGoalBps), int(expectedBandwidthUsage), - ProbeMinDuration, - ProbeMaxDuration, + p.probeDuration, + time.Duration(float64(p.probeDuration.Milliseconds())*p.params.Config.DurationOverflowFactor)*time.Millisecond, ) return p.probeClusterId, p.probeGoalBps @@ -203,18 +219,31 @@ func (p *ProbeController) InitProbe(probeGoalDeltaBps int64, expectedBandwidthUs func (p *ProbeController) clearProbeLocked() { p.probeClusterId = ProbeClusterIdInvalid + p.doneProbeClusterInfo = ProbeClusterInfo{Id: ProbeClusterIdInvalid} p.abortedProbeClusterId = ProbeClusterIdInvalid + p.goalReachedProbeClusterId = ProbeClusterIdInvalid } func (p *ProbeController) backoffProbeIntervalLocked() { - p.probeInterval = time.Duration(p.probeInterval.Seconds()*ProbeBackoffFactor) * time.Second - if p.probeInterval > ProbeWaitMax { - p.probeInterval = ProbeWaitMax + p.probeInterval = time.Duration(p.probeInterval.Seconds()*p.params.Config.BackoffFactor) * time.Second + if p.probeInterval > p.params.Config.MaxInterval { + p.probeInterval = p.params.Config.MaxInterval } } func (p *ProbeController) resetProbeIntervalLocked() { - p.probeInterval = ProbeWaitBase + p.probeInterval = p.params.Config.BaseInterval +} + +func (p *ProbeController) resetProbeDurationLocked() { + p.probeDuration = p.params.Config.MinDuration +} + +func (p *ProbeController) increaseProbeDurationLocked() { + p.probeDuration = time.Duration(float64(p.probeDuration.Milliseconds())*p.params.Config.DurationIncreaseFactor) * time.Millisecond + if p.probeDuration > p.params.Config.MaxDuration { + p.probeDuration = p.params.Config.MaxDuration + } } func (p *ProbeController) StopProbe() { diff --git a/pkg/sfu/streamallocator/prober.go b/pkg/sfu/streamallocator/prober.go index 66ef47b8a..be774f9b2 100644 --- a/pkg/sfu/streamallocator/prober.go +++ b/pkg/sfu/streamallocator/prober.go @@ -354,7 +354,7 @@ type ProbeClusterId uint32 const ( ProbeClusterIdInvalid ProbeClusterId = 0 - bucketDuration = time.Second + bucketDuration = 100 * time.Millisecond bytesPerProbe = 1000 minProbeRateBps = 10000 ) @@ -423,14 +423,14 @@ func NewCluster(id ProbeClusterId, mode ProbeClusterMode, desiredRateBps int, ex } func (c *Cluster) initBuckets(desiredRateBps int, expectedRateBps int, minDuration time.Duration) { - // split into 1-second bucket + // split into granular buckets // NOTE: splitting even if mode is unitform numBuckets := int((minDuration.Milliseconds() + bucketDuration.Milliseconds() - 1) / bucketDuration.Milliseconds()) if numBuckets < 1 { numBuckets = 1 } - expectedRateBytes := (expectedRateBps + 7) / 8 + expectedRateBytesPerSec := (expectedRateBps + 7) / 8 baseProbeRateBps := (desiredRateBps - expectedRateBps + numBuckets - 1) / numBuckets runningDesiredBytes := 0 @@ -447,13 +447,13 @@ func (c *Cluster) initBuckets(desiredRateBps int, expectedRateBps int, minDurati if bucketProbeRateBps < minProbeRateBps { bucketProbeRateBps = minProbeRateBps } - bucketProbeRateBytes := (bucketProbeRateBps + 7) / 8 + bucketProbeRateBytesPerSec := (bucketProbeRateBps + 7) / 8 // pace based on bytes per probe - numProbes := (bucketProbeRateBytes + bytesPerProbe - 1) / bytesPerProbe - sleepDurationMicroSeconds := int(float64(1_000_000)/float64(numProbes) + 0.5) + numProbesPerSec := (bucketProbeRateBytesPerSec + bytesPerProbe - 1) / bytesPerProbe + sleepDurationMicroSeconds := int(float64(1_000_000)/float64(numProbesPerSec) + 0.5) - runningDesiredBytes += bucketProbeRateBytes + expectedRateBytes + runningDesiredBytes += (((bucketProbeRateBytesPerSec + expectedRateBytesPerSec) * int(bucketDuration.Milliseconds())) + 999) / 1000 runningDesiredElapsedTime += bucketDuration c.buckets = append(c.buckets, clusterBucket{ @@ -537,10 +537,10 @@ func (c *Cluster) Process(pl ProberListener) { if bytesShortFall < 0 { bytesShortFall = 0 } - // cap short fall to limit to 8 packets in an iteration + // cap short fall to limit to 5 packets in an iteration // 275 bytes per packet (255 max RTP padding payload + 20 bytes RTP header) - if bytesShortFall > (275 * 8) { - bytesShortFall = 275 * 8 + if bytesShortFall > (275 * 5) { + bytesShortFall = 275 * 5 } // round up to packet size bytesShortFall = ((bytesShortFall + 274) / 275) * 275 diff --git a/pkg/sfu/streamallocator/streamallocator.go b/pkg/sfu/streamallocator/streamallocator.go index dba10d7f2..475b38e94 100644 --- a/pkg/sfu/streamallocator/streamallocator.go +++ b/pkg/sfu/streamallocator/streamallocator.go @@ -2,7 +2,6 @@ package streamallocator import ( "fmt" - "math" "sort" "sync" "time" @@ -199,6 +198,7 @@ func NewStreamAllocator(params StreamAllocatorParams) *StreamAllocator { } s.probeController = NewProbeController(ProbeControllerParams{ + Config: s.params.Config.ProbeConfig, Prober: s.prober, Logger: params.Logger, }) @@ -555,7 +555,7 @@ func (s *StreamAllocator) processEvents() { } func (s *StreamAllocator) ping() { - ticker := time.NewTicker(500 * time.Millisecond) + ticker := time.NewTicker(100 * time.Millisecond) defer ticker.Stop() for { @@ -641,9 +641,14 @@ func (s *StreamAllocator) handleSignalEstimate(event *Event) { func (s *StreamAllocator) handleSignalPeriodicPing(event *Event) { // finalize probe if necessary - isHandled, isSuccessful := s.probeController.MaybeFinalizeProbe() + trend, _ := s.channelObserver.GetTrend() + isHandled, isNotFailing, isGoalReached := s.probeController.MaybeFinalizeProbe( + s.channelObserver.HasEnoughEstimateSamples(), + trend, + s.channelObserver.GetLowestEstimate(), + ) if isHandled { - s.onProbeDone(isSuccessful) + s.onProbeDone(isNotFailing, isGoalReached) } // probe if necessary and timing is right @@ -677,10 +682,7 @@ func (s *StreamAllocator) handleSignalSendProbe(event *Event) { func (s *StreamAllocator) handleSignalProbeClusterDone(event *Event) { info, _ := event.Data.(ProbeClusterInfo) - isHandled := s.probeController.ProbeClusterDone(info, int64(math.Min(float64(s.committedChannelCapacity), float64(s.channelObserver.GetLowestEstimate())))) - if isHandled { - s.onProbeDone(true) - } + s.probeController.ProbeClusterDone(info) } func (s *StreamAllocator) handleSignalResume(event *Event) { @@ -925,7 +927,7 @@ func (s *StreamAllocator) allocateTrack(track *Track) { s.adjustState() } -func (s *StreamAllocator) onProbeDone(isSuccessful bool) { +func (s *StreamAllocator) onProbeDone(isNotFailing bool, isGoalReached bool) { highestEstimateInProbe := s.channelObserver.GetHighestEstimate() // @@ -939,18 +941,23 @@ func (s *StreamAllocator) onProbeDone(isSuccessful bool) { // NOTE: With TWCC, it is possible to reset bandwidth estimation to clean state as // the send side is in full control of bandwidth estimation. // + channelObserverString := s.channelObserver.ToString() s.channelObserver = s.newChannelObserverNonProbe() - if !isSuccessful { + s.params.Logger.Infow( + "probe done", + "isNotFailing", isNotFailing, + "isGoalReached", isGoalReached, + "committedEstimate", s.committedChannelCapacity, + "highestEstimate", highestEstimateInProbe, + "channel", channelObserverString, + ) + if !isNotFailing { return } - // probe estimate is same or higher, commit it and try to allocate deficient tracks - s.params.Logger.Infow( - "successful probe, updating channel capacity", - "old(bps)", s.committedChannelCapacity, - "new(bps)", highestEstimateInProbe, - ) - s.committedChannelCapacity = highestEstimateInProbe + if highestEstimateInProbe > s.committedChannelCapacity { + s.committedChannelCapacity = highestEstimateInProbe + } s.maybeBoostDeficientTracks() } diff --git a/pkg/sfu/streamallocator/trenddetector.go b/pkg/sfu/streamallocator/trenddetector.go index 2a5281230..ed0ae1d71 100644 --- a/pkg/sfu/streamallocator/trenddetector.go +++ b/pkg/sfu/streamallocator/trenddetector.go @@ -122,6 +122,10 @@ func (t *TrendDetector) GetDirection() TrendDirection { return t.direction } +func (t *TrendDetector) HasEnoughSamples() bool { + return t.numSamples >= t.params.RequiredSamples +} + func (t *TrendDetector) ToString() string { now := time.Now() elapsed := now.Sub(t.startTime).Seconds() From 496656627e3f5afa4ef562cc6f871c48c03354e0 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 30 Jun 2023 11:59:53 +0530 Subject: [PATCH 262/324] Logging more to understand layer transition leak better. (#1840) --- pkg/sfu/connectionquality/connectionstats.go | 27 ++++++++++++++++++-- pkg/sfu/connectionquality/scorer.go | 5 ++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/pkg/sfu/connectionquality/connectionstats.go b/pkg/sfu/connectionquality/connectionstats.go index 5acc6c0da..107be9678 100644 --- a/pkg/sfu/connectionquality/connectionstats.go +++ b/pkg/sfu/connectionquality/connectionstats.go @@ -81,18 +81,42 @@ func (cs *ConnectionStats) OnStatsUpdate(fn func(cs *ConnectionStats, stat *live } func (cs *ConnectionStats) UpdateMute(isMuted bool, at time.Time) { + /* TODO-RESTORE + if cs.done.IsBroken() { + return + } + */ + cs.scorer.UpdateMute(isMuted, at) } func (cs *ConnectionStats) AddBitrateTransition(bitrate int64, at time.Time) { + /* TODO-RESTORE + if cs.done.IsBroken() { + return + } + */ + cs.scorer.AddBitrateTransition(bitrate, at) } func (cs *ConnectionStats) UpdateLayerMute(isMuted bool, at time.Time) { + /* TODO-RESTORE + if cs.done.IsBroken() { + return + } + */ + cs.scorer.UpdateLayerMute(isMuted, at) } func (cs *ConnectionStats) AddLayerTransition(distance float64, at time.Time) { + /* TODO-RESTORE + if cs.done.IsBroken() { + return + } + */ + cs.scorer.AddLayerTransition(distance, at) } @@ -253,10 +277,9 @@ func (cs *ConnectionStats) updateStatsWorker() { tk := time.NewTicker(interval) defer tk.Stop() - done := cs.done.Watch() for { select { - case <-done: + case <-cs.done.Watch(): return case <-tk.C: diff --git a/pkg/sfu/connectionquality/scorer.go b/pkg/sfu/connectionquality/scorer.go index 4d333e005..b84d27777 100644 --- a/pkg/sfu/connectionquality/scorer.go +++ b/pkg/sfu/connectionquality/scorer.go @@ -252,6 +252,8 @@ func (q *qualityScorer) AddLayerTransition(distance float64, at time.Time) { q.lock.Lock() defer q.lock.Unlock() + // TODO-REMOVE-AFTER-DEBUG + q.params.Logger.Debugw("adding layer transition", "at", at, "distance", distance) q.layerTransitions = append(q.layerTransitions, layerTransition{ startedAt: at, distance: distance, @@ -262,6 +264,9 @@ func (q *qualityScorer) Update(stat *windowStat, at time.Time) { q.lock.Lock() defer q.lock.Unlock() + // TODO-REMOVE-AFTER-DEBUG + q.params.Logger.Debugw("running update", "at", at, "stat", stat) + // always update transitions expectedBitrate := q.getExpectedBitsAndUpdateTransitions(at) expectedDistance := q.getExpectedDistanceAndUpdateTransitions(at) From 06f9b574cb611c97e28370c3614b620b4a12773b Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 30 Jun 2023 20:44:57 +0530 Subject: [PATCH 263/324] Delete down track from receiver in close always. (#1842) * Delete down track from receiver in close always. I think with the parallel close in goroutines, it so happens that peer connection can get closed first and unbind the track. The delete down track and RTCP reader close was inside if `bound` block. So, they were not running leaving a dangling down track in the receiver. * fix tests * fix test --- pkg/rtc/wrappedreceiver.go | 8 ++++++++ pkg/sfu/downtrack.go | 12 ++++++------ test/singlenode_test.go | 1 - 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/pkg/rtc/wrappedreceiver.go b/pkg/rtc/wrappedreceiver.go index 84d4d9910..545ac1b78 100644 --- a/pkg/rtc/wrappedreceiver.go +++ b/pkg/rtc/wrappedreceiver.go @@ -103,6 +103,14 @@ func (r *WrappedReceiver) Codecs() []webrtc.RTPCodecParameters { return codecs } +func (r *WrappedReceiver) DeleteDownTrack(participantID livekit.ParticipantID) { + if r.TrackReceiver != nil { + r.TrackReceiver.DeleteDownTrack(participantID) + } +} + +// -------------------------------------------- + type DummyReceiver struct { receiver atomic.Value trackID livekit.TrackID diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index b47d9dd33..5e7644449 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -842,13 +842,13 @@ func (d *DownTrack) CloseWithFlush(flush bool) { d.bound.Store(false) d.logger.Debugw("closing sender", "kind", d.kind) - d.receiver.DeleteDownTrack(d.subscriberID) + } + d.receiver.DeleteDownTrack(d.subscriberID) - if d.rtcpReader != nil && flush { - d.logger.Debugw("downtrack close rtcp reader") - d.rtcpReader.Close() - d.rtcpReader.OnPacket(nil) - } + if d.rtcpReader != nil && flush { + d.logger.Debugw("downtrack close rtcp reader") + d.rtcpReader.Close() + d.rtcpReader.OnPacket(nil) } d.bindLock.Unlock() diff --git a/test/singlenode_test.go b/test/singlenode_test.go index 8a00e6a8d..8d9acbdc7 100644 --- a/test/singlenode_test.go +++ b/test/singlenode_test.go @@ -584,7 +584,6 @@ func TestSubscribeToCodecUnsupported(t *testing.T) { for _, t := range tracks { if strings.EqualFold(t.Codec().MimeType, "video/vp8") { return "" - } } return "did not receive track with vp8" From e3954d1d643b6bf0a91008a8ded984b192c6b5df Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sat, 1 Jul 2023 10:21:15 +0530 Subject: [PATCH 264/324] Use timed aggregator. (#1843) * Use timed aggregator. For aggregate bitrate and average distance from desired. Also, clean up debug added to track leak. * update deps --- go.mod | 2 +- go.sum | 4 +- pkg/sfu/connectionquality/connectionstats.go | 8 - pkg/sfu/connectionquality/scorer.go | 150 +++---------------- 4 files changed, 24 insertions(+), 140 deletions(-) diff --git a/go.mod b/go.mod index 9153ffb53..2a5b7b9e9 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230612070454-d5299b956135 - github.com/livekit/protocol v1.5.8 + github.com/livekit/protocol v1.5.9-0.20230701042848-e5323cdb4ab3 github.com/livekit/psrpc v0.3.1 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 diff --git a/go.sum b/go.sum index 89c61e2f4..1c1db1c6c 100644 --- a/go.sum +++ b/go.sum @@ -124,8 +124,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20230612070454-d5299b956135 h1:lWYbsondvqG69czxoACDwaJ/BoyD57BahCo70ZH+m4U= github.com/livekit/mediatransportutil v0.0.0-20230612070454-d5299b956135/go.mod h1:MRc0zSOSzXuFt0X218SgabzlaKevkvCckPgBEoHYc34= -github.com/livekit/protocol v1.5.8 h1:Q8Ctyq3rhv+trg0Hnd2rKncKAa9QAMlQO7zAUxvz4TU= -github.com/livekit/protocol v1.5.8/go.mod h1:B6hJiuXT84dHsUgaKHBo+ZLPX4XhklptYA2UbANSiNg= +github.com/livekit/protocol v1.5.9-0.20230701042848-e5323cdb4ab3 h1:OUWOLcsgEJ3o5p5NAlebqHzN0xJHBWl1HBUwMe/PZv4= +github.com/livekit/protocol v1.5.9-0.20230701042848-e5323cdb4ab3/go.mod h1:B6hJiuXT84dHsUgaKHBo+ZLPX4XhklptYA2UbANSiNg= github.com/livekit/psrpc v0.3.1 h1:KfylgJHvoLQcc22t/oflwMOeSnx0c14G7cWsS+9MYS4= github.com/livekit/psrpc v0.3.1/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= diff --git a/pkg/sfu/connectionquality/connectionstats.go b/pkg/sfu/connectionquality/connectionstats.go index 107be9678..2b2601b74 100644 --- a/pkg/sfu/connectionquality/connectionstats.go +++ b/pkg/sfu/connectionquality/connectionstats.go @@ -81,41 +81,33 @@ func (cs *ConnectionStats) OnStatsUpdate(fn func(cs *ConnectionStats, stat *live } func (cs *ConnectionStats) UpdateMute(isMuted bool, at time.Time) { - /* TODO-RESTORE if cs.done.IsBroken() { return } - */ cs.scorer.UpdateMute(isMuted, at) } func (cs *ConnectionStats) AddBitrateTransition(bitrate int64, at time.Time) { - /* TODO-RESTORE if cs.done.IsBroken() { return } - */ cs.scorer.AddBitrateTransition(bitrate, at) } func (cs *ConnectionStats) UpdateLayerMute(isMuted bool, at time.Time) { - /* TODO-RESTORE if cs.done.IsBroken() { return } - */ cs.scorer.UpdateLayerMute(isMuted, at) } func (cs *ConnectionStats) AddLayerTransition(distance float64, at time.Time) { - /* TODO-RESTORE if cs.done.IsBroken() { return } - */ cs.scorer.AddLayerTransition(distance, at) } diff --git a/pkg/sfu/connectionquality/scorer.go b/pkg/sfu/connectionquality/scorer.go index b84d27777..5084d6ce8 100644 --- a/pkg/sfu/connectionquality/scorer.go +++ b/pkg/sfu/connectionquality/scorer.go @@ -8,6 +8,7 @@ import ( "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/utils" ) const ( @@ -139,16 +140,6 @@ func (w *windowStat) String() string { // ------------------------------------------ -type bitrateTransition struct { - startedAt time.Time - bitrate int64 -} - -type layerTransition struct { - startedAt time.Time - distance float64 -} - type qualityScorerParams struct { PacketLossWeight float64 IncludeRTT bool @@ -173,14 +164,20 @@ type qualityScorer struct { maxPPS float64 - bitrateTransitions []bitrateTransition - layerTransitions []layerTransition + aggregateBitrate *utils.TimedAggregator[int64] + layerDistance *utils.TimedAggregator[float64] } func newQualityScorer(params qualityScorerParams) *qualityScorer { return &qualityScorer{ params: params, score: maxScore, + aggregateBitrate: utils.NewTimedAggregator[int64](utils.TimedAggregatorParams{ + CapNegativeValues: true, + }), + layerDistance: utils.NewTimedAggregator[float64](utils.TimedAggregatorParams{ + CapNegativeValues: true, + }), } } @@ -207,10 +204,7 @@ func (q *qualityScorer) AddBitrateTransition(bitrate int64, at time.Time) { q.lock.Lock() defer q.lock.Unlock() - q.bitrateTransitions = append(q.bitrateTransitions, bitrateTransition{ - startedAt: at, - bitrate: bitrate, - }) + q.aggregateBitrate.AddSampleAt(bitrate, at) if bitrate == 0 { if !q.isLayerMuted() { @@ -230,14 +224,8 @@ func (q *qualityScorer) UpdateLayerMute(isMuted bool, at time.Time) { if isMuted { if !q.isLayerMuted() { - q.bitrateTransitions = append(q.bitrateTransitions, bitrateTransition{ - startedAt: at, - bitrate: 0, - }) - q.layerTransitions = append(q.layerTransitions, layerTransition{ - startedAt: at, - distance: 0.0, - }) + q.aggregateBitrate.AddSampleAt(0, at) + q.layerDistance.AddSampleAt(0, at) q.layerMutedAt = at q.score = maxScore } @@ -252,24 +240,22 @@ func (q *qualityScorer) AddLayerTransition(distance float64, at time.Time) { q.lock.Lock() defer q.lock.Unlock() - // TODO-REMOVE-AFTER-DEBUG - q.params.Logger.Debugw("adding layer transition", "at", at, "distance", distance) - q.layerTransitions = append(q.layerTransitions, layerTransition{ - startedAt: at, - distance: distance, - }) + q.layerDistance.AddSampleAt(distance, at) } func (q *qualityScorer) Update(stat *windowStat, at time.Time) { q.lock.Lock() defer q.lock.Unlock() - // TODO-REMOVE-AFTER-DEBUG - q.params.Logger.Debugw("running update", "at", at, "stat", stat) - // always update transitions - expectedBitrate := q.getExpectedBitsAndUpdateTransitions(at) - expectedDistance := q.getExpectedDistanceAndUpdateTransitions(at) + expectedBitrate, _, err := q.aggregateBitrate.GetAggregateAndRestartAt(at) + if err != nil { + q.params.Logger.Warnw("error getting expected bitrate", err) + } + expectedDistance, err := q.layerDistance.GetAverageAndRestartAt(at) + if err != nil { + q.params.Logger.Warnw("error getting expected distance", err) + } // nothing to do when muted or not unmuted for long enough // NOTE: it is possible that unmute -> mute -> unmute transition happens in the @@ -402,100 +388,6 @@ func (q *qualityScorer) getPacketLossWeight(stat *windowStat) float64 { return packetRatio * packetRatio * q.params.PacketLossWeight } -func (q *qualityScorer) getExpectedBitsAndUpdateTransitions(at time.Time) int64 { - if len(q.bitrateTransitions) == 0 { - return 0 - } - - var startedAt time.Time - var totalBits float64 - for idx := 0; idx < len(q.bitrateTransitions)-1; idx++ { - bt := &q.bitrateTransitions[idx] - btNext := &q.bitrateTransitions[idx+1] - - if bt.startedAt.After(q.lastUpdateAt) { - startedAt = bt.startedAt - } else { - startedAt = q.lastUpdateAt - } - totalBits += btNext.startedAt.Sub(startedAt).Seconds() * float64(bt.bitrate) - } - - // last transition - bt := &q.bitrateTransitions[len(q.bitrateTransitions)-1] - if bt.startedAt.After(q.lastUpdateAt) { - startedAt = bt.startedAt - } else { - startedAt = q.lastUpdateAt - } - totalBits += at.Sub(startedAt).Seconds() * float64(bt.bitrate) - - // set up last bit rate as the starting bit rate for next analysis window - q.bitrateTransitions = []bitrateTransition{{ - startedAt: at, - bitrate: bt.bitrate, - }} - - return int64(totalBits) -} - -func (q *qualityScorer) getExpectedDistanceAndUpdateTransitions(at time.Time) float64 { - if len(q.layerTransitions) == 0 { - return 0 - } - - var startedAt time.Time - var totalDistance float64 - totalDuration := time.Duration(0) - for idx := 0; idx < len(q.layerTransitions)-1; idx++ { - lt := &q.layerTransitions[idx] - ltNext := &q.layerTransitions[idx+1] - - if lt.startedAt.After(q.lastUpdateAt) { - startedAt = lt.startedAt - } else { - startedAt = q.lastUpdateAt - } - dur := ltNext.startedAt.Sub(startedAt) - totalDuration += dur - - dist := lt.distance - if dist < 0.0 { - // negative distances are overshoot, that does not compensate for shortfalls, so use optimal, i. e. 0 distance when overshooting - dist = 0.0 - } - totalDistance += dur.Seconds() * dist - } - - // last transition - lt := &q.layerTransitions[len(q.layerTransitions)-1] - if lt.startedAt.After(q.lastUpdateAt) { - startedAt = lt.startedAt - } else { - startedAt = q.lastUpdateAt - } - dur := at.Sub(startedAt) - totalDuration += dur - - dist := lt.distance - if dist < 0.0 { - dist = 0.0 - } - totalDistance += dur.Seconds() * dist - - // set up last distance as the starting distance for next analysis window - q.layerTransitions = []layerTransition{{ - startedAt: at, - distance: lt.distance, - }} - - if totalDuration == 0 { - return 0 - } - - return totalDistance / totalDuration.Seconds() -} - func (q *qualityScorer) GetScoreAndQuality() (float32, livekit.ConnectionQuality) { q.lock.RLock() defer q.lock.RUnlock() From 869f23a054d739b8b48ec4be29c9a04107755ab5 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sat, 1 Jul 2023 12:31:51 +0530 Subject: [PATCH 265/324] Close subscriptions promptly (#1845) * Close subscriptions promptly Two things: ----------- 1. Because the desired is not changed, the notifiers are not notified that the subscription is not observing any more. So, that holds a refernce to the subscription manager. Address the above by setting `setDesired` to false on all subscriptions when subscription manager closes. That will remove observer from the notifiers. 2. When subscription manager is closed, the down track close is invoked which flows back (with onClose callback of downtrack) to subscription manager "handleSubscribedTrackClose". That callback handler sets the subscribed track to nil for that subscription. A couple of scenarios here a. Without the above change, desired could have been true and it would have looked that the track needs to try subscription again because `needsSubscribe == true` (desired == true && subscribedTrack == nil) b. Even with the change above, there is a new condition of `desired == false && subscribedTrack == nil` and there was no handler for that condition in the reconciler. Address this by adding a `needsCleanup` function and delete subscription from the map. Note that the reconciler may not be running to execute this action as subscription manager would have closed the `closeCh`, but doing the code in the interest of proper clean up. * clean up --- pkg/rtc/subscriptionmanager.go | 52 ++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/pkg/rtc/subscriptionmanager.go b/pkg/rtc/subscriptionmanager.go index 468ea678a..fbf83dd73 100644 --- a/pkg/rtc/subscriptionmanager.go +++ b/pkg/rtc/subscriptionmanager.go @@ -104,6 +104,7 @@ func (m *SubscriptionManager) Close(willBeResumed bool) { subTracks := m.GetSubscribedTracks() downTracksToClose := make([]*sfu.DownTrack, 0, len(subTracks)) for _, st := range subTracks { + m.setDesired(st.ID(), false) dt := st.DownTrack() // nil check exists primarily for tests if dt != nil { @@ -133,17 +134,19 @@ func (m *SubscriptionManager) isClosed() bool { } func (m *SubscriptionManager) SubscribeToTrack(trackID livekit.TrackID) { - m.lock.Lock() - sub, ok := m.subscriptions[trackID] - if !ok { + sub, desireChanged := m.setDesired(trackID, true) + if sub == nil { sLogger := m.params.Logger.WithValues( "trackID", trackID, ) sub = newTrackSubscription(m.params.Participant.ID(), trackID, sLogger) + + m.lock.Lock() m.subscriptions[trackID] = sub + m.lock.Unlock() + + sub, desireChanged = m.setDesired(trackID, true) } - desireChanged := sub.setDesired(true) - m.lock.Unlock() if desireChanged { sub.logger.Infow("subscribing to track") } @@ -153,17 +156,13 @@ func (m *SubscriptionManager) SubscribeToTrack(trackID livekit.TrackID) { } func (m *SubscriptionManager) UnsubscribeFromTrack(trackID livekit.TrackID) { - m.lock.Lock() - sub, ok := m.subscriptions[trackID] - m.lock.Unlock() - if !ok { + sub, desireChanged := m.setDesired(trackID, false) + if sub == nil || !desireChanged { return } - if sub.setDesired(false) { - sub.logger.Infow("unsubscribing from track") - m.queueReconcile(trackID) - } + sub.logger.Infow("unsubscribing from track") + m.queueReconcile(trackID) } func (m *SubscriptionManager) GetSubscribedTracks() []types.SubscribedTrack { @@ -255,6 +254,18 @@ func (m *SubscriptionManager) WaitUntilSubscribed(timeout time.Duration) error { return context.DeadlineExceeded } +func (m *SubscriptionManager) setDesired(trackID livekit.TrackID, desired bool) (*trackSubscription, bool) { + m.lock.RLock() + defer m.lock.RUnlock() + + sub, ok := m.subscriptions[trackID] + if !ok { + return nil, false + } + + return sub, sub.setDesired(desired) +} + func (m *SubscriptionManager) canReconcile() bool { p := m.params.Participant if m.isClosed() || p.IsClosed() || p.IsDisconnected() { @@ -267,7 +278,7 @@ func (m *SubscriptionManager) reconcileSubscriptions() { var needsToReconcile []*trackSubscription m.lock.RLock() for _, sub := range m.subscriptions { - if sub.needsSubscribe() || sub.needsUnsubscribe() || sub.needsBind() { + if sub.needsSubscribe() || sub.needsUnsubscribe() || sub.needsBind() || sub.needsCleanup() { needsToReconcile = append(needsToReconcile, sub) } } @@ -375,6 +386,12 @@ func (m *SubscriptionManager) reconcileSubscription(s *trackSubscription) { m.params.OnSubscriptionError(s.trackID, true, ErrTrackNotBound) } } + + if s.needsCleanup() { + m.lock.Lock() + delete(m.subscriptions, s.trackID) + m.lock.Unlock() + } } // trigger an immediate reconciliation, when trackID is empty, will reconcile all subscriptions @@ -837,7 +854,6 @@ func (s *trackSubscription) setRemovedNotifier(notifier types.ChangeNotifier) bo func (s *trackSubscription) setRemovedNotifierLocked(notifier types.ChangeNotifier) bool { if s.removedNotifier == notifier { - return false } @@ -963,3 +979,9 @@ func (s *trackSubscription) needsBind() bool { defer s.lock.RUnlock() return s.desired && s.subscribedTrack != nil && !s.bound } + +func (s *trackSubscription) needsCleanup() bool { + s.lock.RLock() + defer s.lock.RUnlock() + return !s.desired && s.subscribedTrack == nil +} From 7e96c98dc387a802542e1e674a0ff8c60834e593 Mon Sep 17 00:00:00 2001 From: lukasIO Date: Mon, 3 Jul 2023 17:32:28 +0200 Subject: [PATCH 266/324] Select highest layer of equal dimensions (#1841) * Select highest layer of equal dimensions * clean up test --- pkg/rtc/mediatrack_test.go | 61 +++++++++++++++++++++++++++++++++++ pkg/rtc/mediatrackreceiver.go | 6 ++-- 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/pkg/rtc/mediatrack_test.go b/pkg/rtc/mediatrack_test.go index 5def4e0b0..2eb172fdc 100644 --- a/pkg/rtc/mediatrack_test.go +++ b/pkg/rtc/mediatrack_test.go @@ -98,4 +98,65 @@ func TestGetQualityForDimension(t *testing.T) { require.Equal(t, livekit.VideoQuality_MEDIUM, mt.GetQualityForDimension(800, 500)) require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(1000, 700)) }) + + t.Run("highest layer with smallest dimensions", func(t *testing.T) { + mt := NewMediaTrack(MediaTrackParams{TrackInfo: &livekit.TrackInfo{ + Type: livekit.TrackType_VIDEO, + Width: 1080, + Height: 720, + Layers: []*livekit.VideoLayer{ + { + Quality: livekit.VideoQuality_LOW, + Width: 480, + Height: 270, + }, + { + Quality: livekit.VideoQuality_MEDIUM, + Width: 1080, + Height: 720, + }, + { + Quality: livekit.VideoQuality_HIGH, + Width: 1080, + Height: 720, + }, + }, + }}) + + require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(120, 120)) + require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(300, 300)) + require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(800, 500)) + require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(1000, 700)) + require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(1200, 800)) + + mt = NewMediaTrack(MediaTrackParams{TrackInfo: &livekit.TrackInfo{ + Type: livekit.TrackType_VIDEO, + Width: 1080, + Height: 720, + Layers: []*livekit.VideoLayer{ + { + Quality: livekit.VideoQuality_LOW, + Width: 480, + Height: 270, + }, + { + Quality: livekit.VideoQuality_MEDIUM, + Width: 480, + Height: 270, + }, + { + Quality: livekit.VideoQuality_HIGH, + Width: 1080, + Height: 720, + }, + }, + }}) + + require.Equal(t, livekit.VideoQuality_MEDIUM, mt.GetQualityForDimension(120, 120)) + require.Equal(t, livekit.VideoQuality_MEDIUM, mt.GetQualityForDimension(300, 300)) + require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(800, 500)) + require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(1000, 700)) + require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(1200, 800)) + }) + } diff --git a/pkg/rtc/mediatrackreceiver.go b/pkg/rtc/mediatrackreceiver.go index 6a785ec9e..8fc33f20f 100644 --- a/pkg/rtc/mediatrackreceiver.go +++ b/pkg/rtc/mediatrackreceiver.go @@ -681,11 +681,13 @@ func (t *MediaTrackReceiver) GetQualityForDimension(width, height uint32) liveki }) } - // finds the lowest layer that could satisfy client demands + // finds the highest layer with smallest dimensions that still satisfy client demands requestedSize = uint32(float32(requestedSize) * layerSelectionTolerance) for i, s := range layerSizes { quality = livekit.VideoQuality(i) - if s >= requestedSize { + if i == len(layerSizes)-1 { + break + } else if s >= requestedSize && s != layerSizes[i+1] { break } } From ab7cad4aab4e15af8327dcba5fa7151acb28a509 Mon Sep 17 00:00:00 2001 From: Jonas Schell Date: Mon, 3 Jul 2023 20:16:01 +0200 Subject: [PATCH 267/324] update readme (#1847) --- .github/banner_dark.png | Bin 157432 -> 131711 bytes .github/banner_light.png | Bin 51437 -> 48704 bytes README.md | 2 +- 3 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/banner_dark.png b/.github/banner_dark.png index f96a81f14ff17b4c72d3068d537a45d0ccb5c139..d19c687ce870844cf5403bb614645024b0a408b0 100644 GIT binary patch delta 96687 zcmaI8bySo6|39u1BvepRrIb`cL`0fR1xX3%RuK?KhzyXy*gX{$ksKjC0qKyIQ33*z z8##KC8;lSbUB7GIpZk8lKcDkGzyIJI&i1;VuV+2tV!Mm!=ndmnt^j2GOU)MpTZae7 zrLFEgC|YpWE^wDh@9ET%?Umfj-^@Sxm~HA9%O~eckItNBFLgu-Lo7}(ML;es{)&KD zoSqS(dwp5rH=FXO9}zV}Gmu}r^&ig+wYIiwbQk4%3>4)0=Pegt%V892(th{T47=RG zEn`zD#PW)-^$2LZFrTITAw2FB)ehQZj?#AxErDb?-ZByEj2N0J+DP7YpI0<+qYNw7 z*FC?v=T$eEy#7OtoxGUji)De0d1AcOl2ZAE@y=8_ETVw*qq#+Zva1FY^&t~9qqaqE4; z3*?&~ep}`y8&P7=&qr_t{((Ac3U%d$LdWAq8AL#}ZYCH8M}FiOl_4-T(FdW(j?$0$ z-s8Z_@4+Q3HMG}9XlpslPX=?M>iS1aVqG`_tOIk^qT6mhOYK5Fxu{_nwF;)(^w|p( zwNf^xD%I#wZknmaJNc?T|5R$-SIWe9;=La{j4*ikY=AGu1)pl>#jrq$SWj4SdMOV<mcz5S%3LlcYWYrj;js*GSqD9Q z^b;g&!8tx7jjKyiK6zJ1R4PE5PLz`j8(Tck0Go4?sYM?ksUC|)u`YU$Qp_#i*eY*5{pI~K{$5>vnxE*+uB-9QY^uA zwc8JbSS^z_pdx*f*O?0#mT*F=?ScAB>pNb}YKILLLsIkcANdzq%jXP)@bf!HiiT|D zJCRcFbKZau?b7&pD+#2BAE(ml_tLIcBXND+(23>eXYJ#=TQIlM-9+)P{bq~Cv6q!e z2cUx_gn^MNNgK8o=NfSeX8yR^R!m8U6mUKJ1oM#v5P$aM^}{v^n4T*y*+y0h?}F@Q zge5$^U1V@Ns|qL0+&xeRN)|Yg0K3di?hh8$!K^Qh9(eX}pCtz&vo$U;{E7Gz;7H+n zCT-uUUXBE(a5D%_Gb?M^-~|Sqo#qyGo_N~KhhK*`|2}r?(lKrI+a`7F!fBtriWR$B zVbC|79|U@R#loiPxch@LHxajQ&clVbIe0w+)Kr~Y}%sut1sIc>%(j?MVwdg z$zzh`@`@<6>^j@Jr}peX5H%$6$+K)pB&FVt4te64zrw&*@Y>>Qz3#fQ*{ANA@WV^@ zTrcMd<23^4>onQJJ}qXnlfJc<{-Rg1bx5NR{3cQ}F7nVR@Vez?$sRu>O3yDWPtBa0 zJekBMfSvwv$ETZt9!?czIFeE9xUA4>K#2JzH`@9PA&2))D}Z%}yh7$vlZt}7cYtS< zD#ZjAu-b3AxRN(UH%#}A8041MyUXnY3o|WGs44`bM0MK=aj^OrCK%TFcW=^pSeQ^i z(pmEMEM0xlU&pT>JBb!;K99W4c6z4rBz6rx^a1)Xi<2zeHQr7L>JX&gb>AxvJ7!av zH&9vxk+#Bcm#`>Ro`jE$d~5LjHG8Kk)gkI*jbTQ4t>cXXCvYtLbve!NhC>By^e^rS z_vAz+w|SAgcSX3#q)=v*Ok{Qyn(!$Cxy?tNxvJN z03k|Pj@oHtj1~~`Ld@zB-$zDO=MpjB8CH2jaPx>MdhE_>1idsq=b2?okyaY|(}p>w zGJ{mpH1n{ebGlF43NNA5OrRJ8Gz45EjF;%+o4t1VW-vq)KcCWY z3YKvGO4M%-S}xN4n~WTn@QT4H!vcex^v_1*@Ow-5&2#Yro#}Z4wOsusp((8w$+uB; z68ab(yA!I`@q*{XKd*Y6KTCcT+#RCMW^p6_9>u$w;)~`D%f9|)K{s9x+N(qybI@)l zGf`hRUB-(@ja|*wxZ&Sfx9q7*FmdT>LDiZ^Lx?e-Az7H<8_3AV6_OxT;okHwN~_Kt zurU9Nte+J~F9byJ=-KMaObFC9>|sZ@--6+!{fTjD+nx0~;>dHwFCQLjVoHDhN(w(m z-u$Mgaf&)6YWSsQ&Y;xsUI(fzyo%omK2tjwn4?FzwY6&%zAhO0*nd7r!ujc)w|bf0IYHE4p2C6KsXz{d z>FJ8&eVZAnO?Oa%m+dgtD{2J=ss|h~Q(sAhGQIE=^qF z9o&oBr*C+6zY_P@DJOXku?4R`e8+U;H=%V2KOhYMN}&r4(Mv1B33y-l;;1%cH6q!W zwEiwRRVsvTH|RVnFjpOIR#S^m~IIvbYp;LDWC!cg@4W`v;?U|ah*)r=E@xH&Y}<2b_e1?-#m5J~Zi13GV-Bz5 zHjz{byy@sSc@BCkTtl2m=hn5d*aE%G)yG3IElGMpOl>o#m%{ZjjlL-Ok2$SG+h7R& zbi0%1QHA2SG!%_F$$bf+{O+NYN|Dcc zN&Y=Awt@?6Q`J*CepH2NaWldAg1gr%{FfsPYkK{Eqg7%@84gZosgaTiU+H&$sSYWE zrOuM~cfPm$)vfv{ke+b#;u^6>%QR@236{`*j)Ru72JeR!?B^4YHNRRGaLs=LLX?-z zN;MS+^yHdxy_seVV}|WVy|RCRj{M1oJb&S2$+PLg^>-V<-vK#W1PdBy4ZsVu$A7(a z;Q^0HLrHdb=dfGAu!Giz>~=3clp3c5KJfu+g8vzcU%JPX9v%^NC4P|E1Z+Gyhxw~m zLQ66J-ebipeNS)D6EppC5SN5axUBaO;qlNyPq0U;N%~vD-7s!)7ZrrKl>GwyxN)T_1PFmp_4A!DO|u2>JEaV@=_*^q` z?iAJb0V=Qp$ZVoGED~ZKLpDaWv2*lKBnXQ(^A|qru|1`jo11`^_OFf`h0Z*{L)XB{ zWHSi-T(nEcGZWxp2Jf;U`O^y_VQgeX4g`F)pdU!5x{}l)y16)mh>ms3*HVov?{H-x)pmB*L+Abcg`yH%}m*wsPKLcV=PF!U6R#5+yD`+)S9$5H%s zE@s%>0n&7p?GcX-Tk`VcewBt^-$6X-Q|vD?{c@gg#SXOl-qsrrJ@T$T!4H0oc=eJF zM%is=YnNY>pp|frI+$!*DW;A0PY3bUC;f_JM>n9j&}R`0$a^1-o1)w^WA@&xic>Rf zFbqFj3^I#R`_JvKM3rws547UBOhXsEC4Ld^vM|FA zm=`fg-tlGwtRGDfM3}?%^bFw5J(J z=C!8RKwT3k%{0NZ(wB>dN6qc~cj@^qdRVM|xr+;=k4x6Bq z61MRGi7gqFzJLAAXDY%U)eq9mZ}nqO{&itB3gG7vOb3rls~f?mgDH87i6u|R}> zHQnes*Tq+Nu^jQe@}5FH{|jWof|pUDlD7e7%SryE-?L#5dWqHa*V?OR4y)1rr)B#D zSATjVFcn)iVLa#Dy3;p@O=8v?7qJL_BSX2I_VTx;t)gt?rPqyw^sp0!kkooT|I@*I z3opN$f8T4KR`hRdws9B`>#$U>y_NV>eY)Gg=8^67JJXRSNtz!othTne1sASk+3#AI zIL{S?E^B{&`DLZp>pt&(6_|U$bg@NqB2<)ai6p1#yNJ8gl3g5 zlOos5d)!tAzJ-HXax%T{ zhE(rl&dgV_;}tXFLC8IIG3lL{p<)MkklH zH8`KAy|Mk2Qyk~u#ZgT6O8*;)j5%?lO*c8$YbPV}wBQS=jX9Q&!qCLL_v>_bRExC; zRV*1K)p5T9uW^G*I6uomwj6c1YN?2gY=%R^u(!(Y&ROzX@F&sXOZbpON08B5-qVQ? zWWyPGE0>C+I#}C?63lPX&w%|bxjFne`dkYa(hH>OA_A9U57w9RunaM`!kO4xiA63( zX>HUc+|aGsRgp%ye`6kh=>7F@$XU84n$Egq%|;O%{3V)Qcj>iz$TKAa?d&{$&9bul zLeDmW3iKCyPw%Yj2zEJ$34A&q@`UhK4{J;(I+#%|J>!=>HOBp6>t(Q> z^v!auR2^j*9qCK>GsvI%CATrt+e^2sv)&~X1OL~sR3%yJGz-dZ z|2+Cq2{%$Fm_I_e%klS_a34BcF?=_pJD~svk%RN_XUyrKZL%mx zhh|c1NX3=mSUpUhE-Xd`k8SFo_|)CnHF1>)ZJG>1WT4N;;9SW@DKEWmMfzzfqg=7-Y*F>Ha%`f8mzwOIt=* zlRydD6#z#X4E2jF;gI@;p73&hIQg=~{5>3^(?LL<&EY>!Kp$LJ3mX*c+!|chOlttK%ZfRt}%y<|J=n#R}%H=Bk-L zvAsRA@WD!`w<7zeRcBWI4Fg%OdxaOSI>J-7&dgG3g^zaERB{vu=y`Vf5Ng`;0WCY8 zhymHL8p0wFILya_3LGvr9%Mys-vy3nN+GDtUlo#__$NI$ktkO>xLi6oE#&C0Wk%{( zb)qb?-!7}{G*Zi)lLVLgHmx4_p;NpRVj$Yn_r63=A<``Uq=;9-;L+S{RU{Mr*w$-M za(B0fG#a=Bf?gRNwOG4OSX_|2{F!i_XL{* zO;wpnVMV!b-}Q+)15AGMuek9Sr>O480qow{QDQ}}YY}_{s_C9|86P5mz81@k!~zIH zhZ-P`|8OpS0U2O$%^W6U*x9pfr`+;Q&`=9{6ITZF#>Bybg&$KX3 zekxp%B4a3!s7{Cm6<&rDHsH*DDU}N5ar5jsYOMXb$}%8Y^I8<{BAHLHjozZ0BWeDjt10Pqa{(7U3&n7+ z+2?xiV>g-l~}@cFxTvtl+mUCD$E1c6X!#O{Wb_mr`EogDCYH*sdUggeuP%7vNOGv`BX&yrh^ zZCd|y7;80@nv7}b;rS_Pu=AH;wXG_gHM(yjlFuI4vRjX#c+p06?8xsw8L1yd;AMt5 z>*opQZlGI45^~xt)_cV*zmU7bX4Y&-Dq6jTwnl?3+E({bi3KJCp#aWi-a0ky;E?vL z{*eN#`q+;dJEIg$UC&zF38yD;DU=p>xJ%u%7^?|}=P)Aq$!kcfw9>X?WdP`)NgnY- zrxdY~XJ1OcekELrjt;)+@Ymnt1rm;$JVy?@*=2BJPC~xRR8BN*9&Qw^5T^@kc}FHD zON_2Z&N+>icQ_VYj%Ub8c@UfUY_H(ft=%bN|AquYIKJk|y`-!I$tPCX1hz0J*K`(A z+VS}R$zmodp?>u~FzLcP0-BzAb7=I_@L>Vg%5dOrY0X$`|bKD zzk>Rd&$$bqnoVz#5YF+TN!_2yg$qDHJ%y=+RE<;2jTd z&5u{<;3xho7ROC6+|u2uOgWN>Ge2Q`#daAt%{gZ?fB2Ogbm$~xaZdU%Qnz!xI3qvR zD;QiMtbfQCM$D`G!9TkPPTjs0HP-(oGN9RS-}#*kd(8YjQSB#SCZLk8YyIa8m+`5v z@r^;DaFxd*<8?;|E7p`fmc>u2Zy+b715cm*c^n=2ogJAi`_xyZUL?M-peay z{K8MKE~aCd>hR(z8*>a#H0iF;3thnob`75Vtf^Ri&$XStG*}unqrF_E5vt4LwtmL! zh&AqJ(UnkdQHO65^?yO+Pip~Pww8>(Sd6CMZE0v4MA~D4uKtr@K}|?>>xWoFT9ww{yEBa$4lJwArr4snX7&@#h@u7mAJP;Yi2u zjbxy|>R@}|^9EIiq*FLsBiij8KOYst%&G+>az*aC1~z?dWaey4e%Re|6ER3tGFa3okm>_pKjtOTFt0cKo8_9dsr2rg;Q8 zxVdWQ3xi;l+?6gN4XhI<)@q}wPKD^lCZqjz>|+$K0Xl?yrCw5Cpg89|Pe)qo8IWod z$^dAT+=voEp9F~5>RwDyz<^F^?VPvA6%}UBc#XPI{%BE;TgOLP;6Lxy}6g`uT$NrB; z;;e9xtwvIwHcSr1FNMpJ{Q&<09muoIv$A|2LCyuKVs~u}Ql>A4@qT&;sRx0TZ?im7 zi|si-r@C{Tll)d1uYKzhBWz6A6WwWmG0+>xuix&#S-MngA%ERHFUttK{ufjps8|l< z@e^3&%kEsaY8yQspe?mGxsB+VnAbr7Cg*B~}}b z(9cx_D|dl1A72*B+Tk;K#`fi6%D<6oHnwjo$Df+`54ySypE-2i_afJZN!mXb%4M_F z#AuqOYUcBt@w1~M9*MQ;LxaG~MpBYMC9*pGoqf;CrW#GbF5}x%aaKhE8_;}fkAo7P z%5k6Sm7~=C{;Z`|yhAv@a=8QD?mLg&_c2a;rQyili^L!vp;|s$;>^GA?FSzlqH~55feg`wnqCuxt)qK__-K z#i<%LA0;(_oQ%g6+k0`)?hLL4Z@QU4(58NBhusF2@QjQ zs#JRS5+3)hXfzwIHYbMP@(!<>N_rw=I=p4Y;}fK+Q92^8(01D`^#dE3CqxV}Wh0DR zvQYDI+bAk?a#D(%3b=Qc$}foO2Dh-2*Ds@%b*PebaMxLY8k<*7D7bP2?e5bZ@nh$387to~=-0wqS&8!3y60I(T^4QNN$pgZQ?co#!^0 zJaEsgov`U1_7Q?y)eRIeRjb<_C>ox&qw^5cPwtr@R@N=CmJvy*?$zVs)0-oN`r@wu zeaicXhyI@;McdXfAiwKJlN?<}3$JzaUHAk;JpF9QTEktgDKCQOPM{1<1a&N7d?psq zpdWjp++^*%%8gfciQG+-Tiv3aCIDt7dpn{6AFh^__YPNJyJ|Nr`kuU6I=o<=BEZkM zw6T-kK9%&!PT~qq1C$&jgqMCxhiwWBuuG@M<|CA?D!GwJ0SLg4Ka|v90W$hTG%#C= zENW-vmTJbStpZmULE9OWLc#=*P&?`Su|~e!+Q8baurx9QcQ?cNDn50W>YUNuAYZ>D zl{a81`B9z`#z;*Yy89o-{oi(kO1^~e5W_j-b)V7llKZ^O1Vw{Xtz!L=D@A1+>pSFKvUD#YRLW|km9iu4hf31)_otS*YUP- z@L`{um0$JN#9n@I4$|r}cO5H{+Bz~Futps=Sx*6Uf&u8wKnJ(FHnunh+7w_{LEVS+V zNQYfdfk{Rtb7Wq1R!s%;=HI*PDz$D^%FR zdRoyQbIeL=GFGl!HdjmQkuHh-DqrE20qQNJmvG6qw7 z%e2n9Tn)|zI`}bPI(ULrC1U&KN)?G8iXfsIJ!;O9&yqFJ$OJ}fx|)rB`txj488Zlp(0oh(xo|6>BO_i#%`BrPT%Z6W5NqiCfWSre z8Ff2@Qak${!1?~6&ykZ?qf`D7gLx~{`-Tc?q)oo)fnvZFuWKb1H(KUN`q1B}d?NJZ zj2f1-kzNLLzKMJJ0RJY>B55}Xom@V2jhu?+>#tB-7}C1Le!`@oJ1n$rqW)#$p=FYm z^PY|a?xmooo`WW|xNqBN;;EGXhpy(n!2pNwNjW^H*qhT~@h_UgysJt%uXgiQ_*R!B zg_T-Tbm^2%k*Xe{CMuLdO%p}@5t|J=w{5wuAjNvM8FT32J%oEeA36CQqM^tD;CO>i z0P4e{tD7_pW=gufMih{%G9!CCo8SO*algR%UXeUWo~2Vu#>4-!HD&!AMQj~l@Lp)v zKb3Pq{r^=tMjT}90BJ+ds?vC%Olmc@+wv4rSm%o%c9v0H|2>1qqK;z+%LXX$;uoaqP{vZHu-Zkjd1yq-juxi(FFsJ{yWK{R=k3q-DxY~ z$oU+fI?#9u;}mbD??>)}=HUZ2zvZ9LbK5J$xQ7ZxrAN>ZK-CrM$!SdDCf{@s`eYxt zA8W-^3_{$wi%CcL_iWM80@uPea}3z=6=*xYH1f!kU~5&C-bzBO?u`@u^ES_e=N4K; z;6-9KdovH()+I%-+9-V~Lj5{t@`|Wh9fLps9y`?;)H`Nz84rLWv{4op@@+6mG8adQ zFLxI!USOoI4<0Ui=BS9xKdmg{*8=AXv!0^5nX~by>5!X^2S@gepx(<7QlD6*cbZ+R z>+&V>l{Tor2Cle6?`rGFV*lmem=_aFb?sIf7=rkCgh?hdwEeg#CM3hFkCUv#HjeHw z0%tvY(Jpo>_ZDe(XJQ#CqrClLZ)Yyy_t2}gBw_i`Qd<`yczSO*1yJ5cu?+M|-n`H2 zd}b66QrGT<%ShoJ+0~y6kP1dh3^08!+h!E$WjfUrP5b+{`sv_e!uVk;@(=26qoa@h zEA*wOU|_4RixmRU#&xrYw2m!y6Tm1b`uYM&a1{djp848UG26UkI`4S-C^FPvURMln zovcCWR<4iV{s#23R@?O-55@r&&i(v6JQM#+|6S%w;iBi{?NB&X&u!OZomL8`psMH*AeR+zml|AE-fZnZw) zzlIa+_od5O8R@b0XAtchBSNnRt)17BX3C!NXMac_IGH$|GhB z%82iNz!KWm{a28+gx&tG)M7hoRNOG-$9h`g4;93UX;EN-_ukTx>zCoDwm!ztw8XRI z^0}l9VB?k>kWro8O_d%Gy#V3^)jmCW?&om;^!+0+|2^&=TAcrxg{aP1@EV*-$jy%b z2B_%F^h)Kvq`YXy62)t-7cN+^T4|)MgOJHj()IUzRZ;xWhk-lVkNb|!ll*R80L%=f zv1Dz)sF9mHH?q*d-)c@%!$VbKF_4w+z2*^X;2``+g)y`hK zd423?@KIjhG5>&(+BT2%?%=L_ z3h;e98q!j;_v!wBqM=R?Z&(_Q2lV#NAAIE}Sj{L(ueQHQ@a3c_Aazn?&t2V9l+MUe z#ePu7X=2`Ol^YY6M(wRmDer%a9T!9l(J2uFCwTgvQ%4ggW`Y|id5lzGB-nPh3*seY z*Xy_r`_El|wzRJ1&3*o-S8j{*nvDTQdYev3Mud&rpOBjTu0O%v8dJ^g_4Uwibqn;T z?L3&HwHjRsvkP?Cmx9MklS&aHlah%X5C=99yo;pP^U>lA z?wcu=fT1s}lK{2lEou>I>@XxW3T&1LNoEYX`V9>h$o zte;K>n4|-NZ6?DU56pkIiuB8X<`2!G+r8seB!zn>yAJ#aY`}M4MT+C%YbagITTImE zVMHM;Blodo^Q|^btI^?erQ9)Tk3#0wn5hHZTm<;uOLa$t!~=}LMaE@eP;yIW8HBaa zg_wO!N5Hz2@OzVAmvcZIa1j+}UcyuE(P8uZK=mGbTw#{)s*ux|S-Mwf@Jm!(@yp%< zwUzQ)HkVU;&cM!5%8+H??oX+uEL8n>UJNL^S5TEdi>2Z$kVwz(yt|J%B5c(;AN+m? zHVH#+x0^3{XL{f9P0bX)PdVIMA+%4X9_`paNy&3op;V1H>@AYi>?x{RZz%4AGvZg|9Dz5+L@ELF2?gq&7bfX zIaVL)mwPxkt_4esCP`+CR4@C?_kJ0phmS19TGv@@E1HU;3VCG9V>bg4lOdH{DF$fi zST1CG$M@%jDE&2efceOS7j_xOctS57src!*q8|tz9Hfe0y4e(g)R&n5fxF5!Xqr``1k`Ju0?84YSsI@{&-&gQn3>D*d*q!iYQRvz zUvn8{nR-ofD35Bs|vz82T+aMl>aF= z+Pe7s4o%OSRJk>JYOskFoFAjfR%e-fCA@krcw~AvD()a*jbhAMwd}JsPVA;mMLbd6 z9Cfc&CCK6X-21D3bynr+Ae(|9e^bUZ#y0x+13YYWN=-zGUhidellZug^owPm}Ua4{?mDR)L&C-`rH5YxU-Qj8RKEAE;ebB+H zue(W1uvleLd<+4sNe^c#JovwWZ^|a{Fjo5M?COO7O2;B-M{^~xX!BcPfrk^eZa2hk zt)$a~5%4Q%*9ekKQHQkAH`tYWm=91?4Xl9=`IKJd_sg6AFY%?Di##ua)#2QwMwf=k zql<;BwJl1$({rTC^-3OWV8!sPS{kLLCjoCbV1q_X-f*^$?H=v57EkAV1FwN>y?Iv9 zyU~WLYH^=*a0zug{FaSCaIM)0_X*L%-)Q-^bPn5FW!c5P3vD(A+yw{Y><|2|0pFO1 z^A=W`4yZ#`EiW@P>-bAVhi3<~*U9yqprn^R9fTjs;Uz*ENiP9-U!J`?c8-iFh7iA8 z!WSR<{Xe5J@w<`Rg|xP~qv`k}vWAr6U$!|-v9G^N*#LX5JHKEbuq+BnFKaU|21&eX6?#Z1ks9!{8|Q30ROE>{k|Sy%HzvkFgE>UF~9tl?+?OSLIF>0rN86@ zn6xho_Ep440b1d-no|zqyI6cW9agNCuS#p6$rV|#=j?Vh2Yc_OVfKm{7g_J7eRuGj z#G+ioPHH9PaKS68EA*MPv1hktZ3)@TQkNcnghb!(Dpk(Z z5dAxANY&Tk%E#)iS~Cxydg&ye-};WKrY2~0l0#ipYo&q7h0Yb(`)8l?S2`8IDgvg) z{KnUPOuy+pPGlq1K%A#y){y+=c5uT>EA}eO%d*WWJ01BuE&R%_&{uD98?^GDlEy>z zFk(?8rE%{4d}FA$9i}hi-HF$j{p~d57u6Ku;dJ%gvNQ;*v^Bm&S^oF_`x(Vq{VRCW zTwG6GEgb>zXRka>`nNx=1q3b~?WdbvJDM)6{+KFU1oj)}DQl&!O=rc}A5Enf4h9~W z*Y1{Aji1$ePjQXAX4~#|hU$T!5XVU9{lhKfQN3A0sIW@gcFDNwzp`~x^Iw2k)Z1=b zp6vf3I4~PID5kGXHuB$kNbhB+5cPvg`-PgB@@V%+k9*#_UG|6#R~c8O)^h1t)pv2r zY3*>)Udu$dnInH;3h^{whLp5lMP&LFH#xD^xN)p) zwhGW9$dGH(B!$L}sHE1fCKmN}lTb_P_gbl`^@rXCTSv>jN4k^^?YF=- zG`@}{iwxJNx9ZPc!(TNcY>qAs^||)5q>MDi&1ces{>`RwabV!!T<^nD)#E~708Y{n zG8N1#%o`Udvtre8g4cf8yuDW}Y&ov8BrfeZoo=}_8bcbTUffhm(Y=Njxsxp3|GHH} zhA`&9w-)S6@?+pK7?ql>R5;hw;e@z4aaKJVva*|7(7d!WdjkQ_m2w{kzo&U7K0spz z*Qtj{y@v!RuA4J{C2c~2gE`Ihh3fXH^(rMzudZZksDeMg5*2#SZyrCdKtlaxtmbRd z(=96CDw%TA5J5AW)B<8Q09N1pp(PCfTGBQJSI`s(C9Rty4>(&|O6yBB_YROptP_zt zyQG3DJ9q9tt3Dgw^zy!tvt*9Zci|T-x5l?lNObwEXMv%iE8;i?mH*w6WFp?F8xVR` z^*iBfrXIm=m2T z4#p3&em>6{X8hw@Jja5|-N4hL%~eS|8EoT|x19wwBm^7LQn2%pZC)m++z38R8qb~o z7Y+RCKLtXrs+8pcU$-t>+xaOJZjl@$A~e0TV_k>YnAn=|PnDA~?MR9*@$f*5j-KM< zBy$i{4y`Dy;6u;c>6ha?fms~#&i^am-Dk;7mwUT($&WKUb{(|W0uvf-^2*90{i&;Ro9I- zLvt5GxX2toZjUU{%a2)XpE(cW6FBz4=`4^*$FHL;vRILCfI zes#@!-z?YRbJ$4(?DFm%k6VE2>g#J(Wwgv&PUMHYdj=AHavJatxm4Gt;W+Z6F-<)5g+VTHst*d^BE&qwE z7-Op=cy6zrTr6@w5%_HC(qXo`X>BbK7a9E#?G5NPfWk?^L#K}S6Ujmo0jCP z((@y7F;4n=qqjOs`U?g?3Y7bsZAzj+ls{|}b-U(}DrbQ2rpSdKRsEROqI~Lu&smz+ z+6VLx3|?_}WG8b791OJ@^YsC=`om-e#Hor1o||+^Y8)I5e1Pqb1NrgB2^S4`LN1K+ zLSAWiMd8bjInTNuA%2seEk?hxFO}PS)1W2NU!Lx+pY-mHpVeTWl>Nhd*?IcehDAf+ zR$IISBn|sM8pm^9HtFBIY9+fA6^v3- zw5@dC;cnyxglw9U@5ZZ=wATgXmpn4y7#;dJf&&>*t}ZXw%Wk&Na?0YhX=jdz-^mvW zr3k{gIVH-dW_Y^Y7q|(_KuV%kQmfAT`H9f?fDAN<6PQt4zP-&0I@#i5@gKJGKD|sq zVhW6xxLX0n+lf^JybF)4BeC5ArEdi7wzS`AH#g=R^{rdC&8sgFBb<^0d?ttTnt$M$ za>#LZLDk@gH?Q=`7P{Z!-dQR2b4iIak@r=3?CBonB^yBMC@WD2w^-+m0m4!JCmbDr z1`vA!kj3U5OB)7z^h;xQo|});Jom$^m)mX5XFbYX9L=*k7(M;q|SfAuWdTUNj?L-VIUGBe(AKx){SLm{UT>lXs?kHZM*FbukmjfVMx`)z)XBdAzCHP_{Kei| z`8rGg)0N@8ef&*4_WLDcpQ$1oVQy z-;)eznU4T50O=7#NN(lA(09f^^2+lKuT;WQT8-7!UZ_qMKN0ONov(>tcN} zkM6cBa6k7HXfyin5aCuxhYMT-nE(q!79Hd!?PPRUSGSJaz5<< zTy2k9Ca@fYj-3(oK0BtuwFaiyV;Q(HC$_#@0%|I2K^83TN z=H50JKe4$V=0+|z<_-Vo;_xvpdPO*9_b};K-Cwj8YItGz-`QZRT2B)}_v@(Ufwnsv z%BOT`t|QUT-q?Jd>*Q0zIkN;3ye*KO06{?u; zp9y*!Lz(?Fm@+tAw@*N(?~mXDm)E@nz*e$0zW$Wa3A^oP2NOD_W^`e-o&S{jbN^l*!!`zj*Ydr%@%y!aVHB%2n=JrO1Xi=Zw+J zHxi!zmFqQj^!(_=2?yot|0V&UW3{M$U%q@z_i?m%hpF9OU+PZ&AoQr!*7u83rW~P9 zxTRznOIdD|=1&J&tsTu9HnL=i^RpBElLjXY=w6_EnmCadt$2PR+xt4tg-UMij7-jl zr$3dkE4(h$HEA91AgBwARN?I)^st-({RT~YgvW4huc<(W)CQNYQrM{4S?6xaHOm4`ZhF_ z#CQ@^`%;b8@nsq~kfkWiNgi#DXptBo(JOQ{GGF`gg_dBi)s&}sil)| zXm0I|b-fs((x{l4pVuPPZhbqokH6ZBo8RqCB6sbK(oV zY2XBZ`vrT1KgyB{Z(t41l80Y}{o~3(2Una%0#d{HE6vdF`fE+3H{FMkK-m0TtH|Ce;fBDvgR)}wSHV;2drT}AOCh=96RCo5L3h1 z)xiPq>yixo6wbH;n ze3bbENB*Jdjo|$%$B_T?{xQwC_qT>~{p0CqONBchxp1x7bZ=e){ze+ z=-J)m6?kQFETzc5Y2|6{;K3GAV3bWwdu&$(;esI8`to?praZatGZIyZw7ZAWzrB_y z{PGW!duhxK9eCMrrrz~{lI;7ed787(?{QkNfYK^J;obGGf8Q$EdI4w}mzxF9kXU-8 z!*Q{7@DTT3LVUd1x7Zp?s;pr(zuxjVgs z;x=~yVF$4F#I(tX6x^v_su&H`re^H@9;~cY9oUOC!WbL!_>8+VG=z>cILk~5irZ~> zNDofnvTTm1om&*Z%9K-UUxYZ|@d9-vn1eH?KusMDYm z7&XJDso$uG#YvHc%FSG$>A9hJJtZETBaYCy-^kBOctRGWE zbL>_vO3=XrZ2;>b?eR2YBHR9XypNz{@rE2h8|UiPW@+7xeIC2*G#mZyjHUkdG57j| z)X;)jHCij7b<>5KTHwNEi3W_+fGVIQfS2p}X~)+R7mEB-7<_YX=joN2?e}NxU4Yl1 zl1fQx8!kA!TFS(lmAlx}coA%{H|`y9>HC8cNM+Mye!rP>6yPB%FVi7MTX*vK3n$W~?J z+25bv@s|`H=0I0FX1QJ>duy(`y1Re;ua)UX=Jn9y++^KhZRc`P}tS zH1mN-tsEW-lx@zZ3+%~KWX!?(dlnTe_I5*UI=#Eb^2iqP7qpR?GQ0}hb81qdb!EV0 zNY0C0V%#-L9tKt(qC5cP{`=_G;&&7OIr=C1-J-)03ERM@t^V<8)0Cp`{8txB!$v;gX z|H%L5UnzuGUF_&sd(fSTEc(rU>)S2(pkT82L=GZ=zf~4^L@A-)D@TV&}u&=|MAoV|2yRzExl>o z?^H}9pmNx~y8T~Cg*N=bd`@SQ_C=5Of2i+X**#CQeH%6MqyIm)-a0I*uX`V+Q$ZLI zMY@I%5R^_4kWT4TL_`pT8DfSQLZy@xkeV3~1?d`)7AXN`=q{-lx`wXz_<5dh{odyv zE)K(7=bXLwT6?W~t^3|((KL)er`LlNQYr#B^J4sZJ4*3?hC6Alm5Sk!rZ5oTJmBl; zBqRQ6f2RHV-*PSg+>#lX5I$f+?91+O_Q>_oG#w?;{o54-HwEbk0S|*rx`(y-;KDr| z^OpJFpUt(>GTZ^$#FeSuT7As8wyJ^S{GU&l?mUyF%vY5W>>Z3+d(n~b-l$W9I_YsY zSCZpTS{6rl^}qLvyoa?;ap4{29*9wS{5Mkh_uJgaS4Gp;$c}<07eJVy1dP!`4SBI& z-|mDb^>>0c280%O{=q=0(jh8DuOJe z&P+eLW;`3nh`VChu8hjnC4nWFlpa3=<2e3-vbej6t-s|Z;#P8No0vSccy4_~KFW0p zBFr1Q{}Tz`yx*Y;r_QR;61I*^l~bN6*Ar2DyEFTY zQL#Cb<+tqL_i<9-qXc?V)_Nd8UP$$ihO9JpUy*OLBGko^kbjE0Cwg&>se(CYQ%)e~CV_WWWIEJLP=Vn4ji6Wta7cDqW z09=1S=bNc2!;Rf;lw6+w#bgb15?aYk3$9b_= zocTlvK}U;id=hK?9_XtEVxGUYsi20=OTcxt!z|N#p4Qq%)_~MAEm^vtN(o5woKIN+ zJ2+U0D{j4NZ!Bl2`VE#yf2d5!_C}d!Yqol+QAtapYh+kr9>Cu zW@1p`_BQtK7E;O$tw7!GBqT4YA@Ldx;g?OV!>hyMjWlKaYrh1l-NbtS(qe zo)5nB>!0An5r_;Op*M|^ajks270C-$8pe{MjeU9}n|1yM-GYOADa%U>y@KHqY}d1z z154Oxj^96KXKciu-XoFAf2aO;#tjE6>;ST(?%rZW;4^Qr&yVQ435z_yKlyk3D>J!{ zD7$;Z=%r|u`SOynkE@Q6ly_tfdBfxEv|28#!|L+vh;s2g@l7@nwK81K;|n##lamk2 z>F6H{z*8PRBl9-8*~Rv*N)@Y#T#JIF-Aj$~O3D@(YByVXoPf+yzeFwAN6J%td?)E` zb#Tt!{xiL@;{BhpjsJ1rcam;(TtP^XdI8MDJ`wA%TZs#%LSahHe2Gf*U#-NfJeK_{ zEybtb-4hT0#jZh7`O}3X$$P$(Q@A#^Q1caXEh zF!4qirN$8Qr3Tu0?96g5+x%q#yN*!D>abofpN&T@u_q~ZxQ!hGJ~JfAq=^v`A{V;J z5}@5~DpAq!AkoH%3gM)V9yk?-aFq43Ar+i}I~PH+Fk_xf(RUz<23 zQaf~4tW*DO&U5kikFc7FEK+}XW(%N@;pOA*?b|t2bb}Qxnz1ec)5f*FKfP~1R;*xT zKeYZQyQCqjOTVEzS)tS@OHT`V7;Rzt*0LmnaiyOz$i3k@O`1CA0*~OZKF#l{#LC-G zq&;TCK@8nSHMnh zXON^u36I@1>v>y#hh=PPpm0u%iv%;|4fXDa(Rj;*60!dNZg!N$HjL>C-($x`;XeH; z4?_{`r?y&$^qAMM(OA})sP>~d4lI&Wbm)@?oH0sS*}%Bwfk1y};2##eh8BTKRy&_J z8!~J0M(4sfWktBtEcdodyR8+N^ydMmWaCjlQbIlm;Rvqrz&UsSf=(7#)Q4LVG<=?k zVWu+DU&u))6^gF-I&+nLnfho((`{H>;rP(G?xpQpef^rEm6d9CNvrEt{r2%~dgkmL zji`N{(HD;betbl}$Wt`${!_#rr7kZ%sG%~*gc~%iG^>a@frD!P78$O^Y`0Gth;6TjD zQ$yCX1@$LNfP-Wo}>D=exbJD$|-BRTk zE8hX#%Hs4C{<0GOB|%d8K`p1u$wLpZuHY|4&Dl0_G&}aO$yJJ)&b`6Lq!yn~yD=cja2i}+5BX3S%DdwI{!ld~m;IQ{BoO1On(L`v^?f45P@q(6*7?gka0hM=#E28Nut)@rPeDr!88@8XB|AKv|oLUz36qvRFo``xt%kn@7PIO3NjavCJYQ#Fe$ z>ARp!l1lcuF$MkLPM3{WK;3#k8?B($=2 z`rF9*7hh+CR<5TnLbf&7{?q2m30piJ`eOjiEtaj8q-;A&0tvgrN z7hXgFjWGeB(&pl(CK!Pgrxt4O$Pa~wmIw~Lugu5Vrcb3K(M|_&o>I-xWc>3bdqN`s z+CO?LOv#Jj?OQ0s>i0kp!?n?}^g56voB<80`d+rOe!c8??osCEa5>H9_|r+y{-@|MqnX}M)&4tjQvA%r|? z>VdAyhGph73%nsVcZnSC=UbX<_AQnTSm#Qi!DHfmv>x0Z_Gl3V)qbwtlHhMpkN2O~ znJJoW4o*)`i+KBu-vFEOt-qTw4<*nPv><#ZsT*R|<6EwnYLP+t8D^9MRGvntaB}y* za(q~kOz5rgrPm>Rr3U-zzve#?v&l^YmIVQ>nlsApmP=VUa2}>4YhITP{<vejC1UTcG(OE9hU9%O9S(SgGPyfew936pXXvYrak;^7 z-IWaf?%OXD`wkk253F`VZjoaaF~5HCD9pE{{MkmD%8Q4E)l8K$zA3v?lbrHR!?h30 zKaJ>$_70kB$`tm#CZ}1g%5Dqi!;gB%t_^M4=@yh~Ix+GSgIbfazl`HUMcHWz& z(%DDrrA87|rA~geHAvl!$2F*MV_UaB!&m#~%Bl2e~Xk4*%(`DM+6c zVA(^PR$1_Gt#xmKLZX(EdE4EROU%;JXuEdP5UOQcK!>{J!xZs}a=$;8YAeq6U%()| z#Jsm~tso0#Kkq!(XTmv5{1zbSHDJaEpKbv3+6uf4+OzH^uf(!DC#~Gu)O-pj+05Bn zx#Nn;U>(g|NsBeRsVM$?F*6NdRLWOglW^_KXyA=bYY8SR1rM5vp4JKc5-RA^eHEvP z0LCt%w}tK+-l_+pJq#<7Wue|HyWy$78GEJt>0uAlF2O#HD|9mzkeA%U=;fS_Y98{z z!;@BxlUI;Tz7tKSt+PU^N{`erb@3^*)@ix=!q`rX+7N5=Gxgkz#0nq;JG3J>DHLD+wzo<`W2dD`bKl_3!O75-3078%Q4bYb4 zKM^M`8|N<`U4O8f8MhR#O0O&SqS2tV{otYbS7oW`>I2_6E5xbb^&iF=R*}CL*eQD$ z+Re9gW?A<>L^vA794UV0-+3Tv&U}=Yvz`y9T-71? z!|Y3;#X+>4S$O&KSvr?@kwFRgWO585568I*wCRdx)r2>F+4?PWK-HV|s&A1t!w5@&?}$Zji?-M>iMG+&~w32Z0fl+npo*T*y@oI|9+Kd7k&E6qPmRF;NLp5OIgp@Gl*Wi-`7jJWFBB{RJ>YCpTt)u*so)j6Vg zMY2TQp4IVYejh9-9vc;n-dwj2a32e1Teow?to9rr7Vvd#vgkecrdp2iZuXZ`F-;D< z@!{P+mmKlr@PXC6d>?F@VvS9*NDsub7-C#_P$o+sy%lnM7W3?(g233q4--3q!B!r? zc!qa03g!Q3K_BK~ZIVN^QgsQ{syRvb4lTX_5?0TDLp?VLXjM;j^>;%|Rq=79>nSnM z?d~IPVN|laNPgJ~Hjk9h8+Mu6GReD%nCWxi~d@#y?>2$v3iJAp^p#lfVcv0#s=Gji{Kn^_(t;FLsX2t&%vi_e<_Y-d) zrQ4*iISN=v1{o{QFu1~afvB58eR7XA%EuKpy1A_b6m^IO*5u+EpIg{cEHP{Mp!0KH zjA3e9tEs@_{!f>3Zaw-fcXyEPjUG^RaNfBN#?y;t)vQ({6jL#d9+RhKUF70}+Hi6W z{@H@Gi7r!&5G|u;Ai1PjQz>Ow+cUW9O___z?C8DXi<_tCUtb^(VlK9*3HHk*S!5Cp zQHEPsvT6r(jK+z2SUkofuZz1$$G^Vd*j)?ViE*1%^81;*$h6KjeJell3;Hlh9-(z@ zkIf|b3}k}HN#)1i8&Pz{QrpVowCDD4O?&4_=_E?9ZIY~lb{Nb6mgfdq>H+uFim^N@ z?!_f?JQ$u`+k3b%x=<=Ksf)`^@>C}Yyqx#Sd+^|rW_CZ~pucfb3u!1ft>nkYHxNZS zpH(n0j!9A3Iyb|6x&3SQaerqYDtF^%D8l2*iQ;W+8X(bwW!DyE#*sXp@p7`e7QjF2 z;D=&iTGjiqxAj|H-fhDkWm)k8q+6n1KI}nkoO{p3rNN z3sC-^*6|K0!3<^(M|2K*2wnG&nsBkqDu1%MuNpq3Ap2T1-w{0`1L4FSDAq;OV5yJz&OhOObI|c|0k9C zccDy=#I86fpcRDIZBVsH!p-le;o3-)xO&q*u3-)!qS2-CG>R`^zjGNe&Ux2=-G^yx zq#yfkJR7wEeRUY*1-Dtste=D@8Z(7rBZd^&i#}uD8xiN|Ny=+nTzYIpK zg&;Cr=dZ5KOXRB^`XRMuI(`z6xRrjL`pb$=aQi-LK*UV34Kj6tYZ9Z7|iF2t3WZ=V|K{1C@#{3%_IE5xUArZ&tg z#S|>(uLcZD?^igPW(Ce+F|y5d4J&t9uo;ZN#+Un8_R`tus_sqY?bpgYtrMGjWOd%UA0d&Iv35e53rW|-D>%b_0j4q)C5ONsbOv>4Iz*p4y68I zkyQTOO!B1<8&2-+nCX%!1~i}3Id)G?AI%xf@Hf%TRRnUZ60K0N>$vs6>{UpU52EHB zxQQ`K_)8_wa-Ea(=(_&y*eWb}X7glSjuu)MtD$c&SE{k6zx7QpzmP&4Yn4aa{nE(e zuGeO_r$Mag!*|jFH-ZzE+Tivy3Zj z7^*8dd9ikLxW-auXvXDSX8|JxWxuo_VW4t#clHS%n6Z!~hj5|>$V*XkJ z9vIXi%KLPGjIMWTk3e7t15p5+1BjB*^pq-k=(6Q`^0i zX8Nnxp}@*IQf1v<=K1>5QPg#x!8!GS#=4)>1fRpxd!awpg3IyMuV(>WUOV3P&j_Xl z=}iFuPU+t{rue5vx?+&5&SM`2P$BGnIGAwfb`iyucMU6%=0TtKyE`1LWH1+~{hi+M zuk*iums&O3UZVGpx_gSlO)ZYqj{W#QrXv4zMyW%~$d$x( z9$;~c$46sD?ER-gbft+ZA8&0PovEbuz>GEO8@!dy<^Qa{Tx-p-_3z*%UV-ibi1NIcp+A;IlTtiZkH%m(g+{QrT81dEd3!?@Ob&eLldqz~v z6Y#=-wst==d7aiH6NMK|@)k1BgWN9m0p{+(-Bk)l9b_Rb!Rl&gaIbjZRV@PFFiwXA zp~U5v6-!A6u6iB%4wOFE60@O5CJ)-r(dMYSE_16j`5@)=+>tJkOf>BpxKE}I{Gkvl zl>fONL1Ji$tVHjjeCnRdmjD1u1~{xhzwGccc6%ouosWBtg$28v z^qadS3im7hHPQri&Y~FQnaTp_6lucdPpBa}U>MJb#l4K`B|E7?3kRhd7VN1_pTeU% zc)3Iy4^DQ`onLif_69M*0*nKaLI7{G?IS43>7rCf{j|eS%n@|O644ZB%bDtl15`wiwP-<$Kcca6$FC#t+Se);lKg~> zKEBR2HUpB{q-bbSggxP$=|6-AMI=Fre^~KWR0LTG92I}bn+nlnr6`THio2mr2x)-J z(RUnekaev%yqA_>blah7vOU!l9n)UX=o>)edh=qg+c+e9DBp z!4@467gI_A$Z4EHef0a-x?M!QLj9=U^c;BEDBx5u6yTS0=(Qwo zJTvDb`&e0Sgys!mmxz<}YmffDIWXsIgaDlVLbm;fvUYNlfz2zJ4FpDxX)p^KdP>@V z7zVo6aJKEPE`ea_%o2kHYvvp%$w4l5_IrYc3r@<6fHd#RUtE~|FE0GQ9P~dIf-$P4 zUl{RgJhZe=kd1cNAbKeBP#w61z8{uxmK2PRKZ&isY zo{s-mhv%vuHuv5q4^{4@CL{@w*Nt|ti%#u~UqwxS|9*oEUI$QMqX&Eu?rXk=$V29( zT2r43d>YPr%Ppwpju*!}nR^$#^2+Dxq3Va$bqZ@ahrsM@R{8jm0f2Dh8V!M_PzxY= zf_g_G7a#H8B$j9&K8X(Q6_APiamjjLXp3c!hnhWQgiz|BRk`)1aJfB4TCl5ele?veDf9a*`I>oODDt`^TmvE#G!Nt05F?$Q!+bf@JhGW{k$pq&t z>>0{a84jSg5r(~=GLc-&{YQSxYNawJ1~@EmQ`aF|HMe z`Teq@oea^YEW|erV{P}z;EfZYvawzxbfE>>rJWBr*CE^I(g;7xjo?<`R_c);dQHf&*n!q_^@jg)f>dH zCJCPwxbp(!4K2j#?8%1R$d9&V)tplD5e?jA-6198G-yTdi-lcU6TT!#`R)uxc!5yL zJGgP93RYv&2x%EHKKzlCmfdL(jR-%VVn&8;=YDkf|MQptB=ab0D=i=|YBArXj13WF9nN)Kq zt~r$YwRI0J&_d2x1si1=$cv@ZMiD~XB7P>n=@PtrHa~Nzv|j*EGh*o?AfP;UpBa@b z-^A(wUZwfpQPKahxzNCycf|X$e5bF5li1nr%dwkM>m)Dpl^uw(`F)mZ7I>A9ksM_l z|9atmi{UTlYBK=3I%B=gRG9T(Vpq`s9WbwWTi_EpTzNNe6deuhMTtLzv47fY^3yW` z6)1KD!-VGsJvAh20ZMi1mSgc=d! zWMeT)E(CbeNx?^JzW}7nAlOr=6u`fVqR8Pur$wa{peV9aONvrz`EE%pH})8K>L3m# ze9qr!yS`@gxMH|$={Wl7?>(P27h!RPsBL^U3HV# za%i0f4dh^^TXRyPyug6#o!grlIhcA%a=k^HVnzZsd|rhe5}TcsoK0F^-#)*k##nj9i|B4!mDoAIc z4SG{>LX|C6pvMvgHH(Th-)x`N{SVm5gGRa&v0>>=AGhl2MJmFDl?_SMgkhl$v>Zs4 z@l}W>Igk}!+j_URfu*T)u^u`}{fWs7=h#Ivn`;0K`K3j$WE95p&&RDGKm%@f1ElT4 z4ijU0j5qg5zrr0t}brPqb7l6*&<{{ur7IgrDJyec8x1C2 zWdZu%!OUd~JKoZu^w)QOvO6q!Hjji&%FhOrz9Qo=;-5zo-%RVD3jkV&@+1Oj>h#ovkNXkFXkiN8p+yk7tM2}JOD^j2 zJ!Z%Hvj`J@s3jD^*5zf0-tuZ{zRiedAt4Ep&Q?9#qG#PaE7uF6@M)oq+ef?7JuXy$i1D%Y*c@KS!B6qfBX*6oI0 zzG02h0q+JkXAtY|fJlVOw-Lotf{7{ppnTd^CO#9 zIzS<>laf7fjQs!EK^?(&;{j2W&yE1fbzW&++Q^+8;>BYp+-K*-EIP2^^I9%#$!yN! zwQ%G48d_niXYo393#)iPDPzN=ZB08bMw9NsR-kYERC*miyK)l(p%YexZ68GX5ZKB4 zW<|no;g$JjnV2zve7TfL7B~Zxn_Jx9@Aw939LXjoJbJHdRwG=h+u7O!TaAskMSF_S z5a7{Tz$qaHJbmXl&BE$M1Zw}4Ytb{YKG*rqU>h`680h4KU7>;ECYZd$ zylTG~u=HZW&9WVO*=r!Zpa_J=>B0BitL}*g7|Hl41~YuIyguXZqMZJ&W0ts}duY=- z3!ZQRXuZw4{B>?OZfyW3SsZNG54$T=oJ@UH+`77WoLK6k(6yIh18eGp2X8dK(cP}{ zg0BjBFxT-8Sm_=zKg^A*tllib)bkDs$$S7mL@U6Y)^aIF^-A?+5w-iTz|Zl7>8&vvfQdmOjz7ioxYn1cy?5!@IiCn?QbmySX9g_GnH9M z6y`V11~(qr7Je^iTj9oWn;<>LQ`d-`In6N(PS<3xu@g{kY z_>JN229dHb$?31VS7@Q7!b9TKM_fX^2#ZL`(1)`qxAp3{go)dGIgQel*M$1AQhNVf z-+=y7t{tDbLj5q&o`7V!M;sz-CuGSiIoa$OtTm$1rBcv=kTp1k%><9gd25S~v1p>4g_ zUs9T4gg)h$H-qnG_FE2?n~aIJ)}q+)St&pkQ<_o!Gt53lDOJFS46&(ay+aEfk$VgP z-a;tgsH{S+&=P_EtiVB^7%abjNjAi*srH#e5*=jM)_3|F1_7vvR`-a{2mY*&Zu@W;~)OsNNzpSj;Gt!?uJ7kU*U+WcYa>=*Pb^L}K?jD8Lfk zS>p+W`t;QvU|8_uWve$mo**yNwM(m8Nw^Iipx9M9PLicw$2+KxFY29E1JJSq)u+1kp3F(7z0!`9m+EcDwlWp4h+pknyV zCcdeWMDb(zV3!|+IMe0L?;_;CAn-A=n*pzH^B-b}Ph%ZfEci$KIy`Sv@p{UMA6IIm z9}GmRGmFcou`QC<3Ci!Ag|>OCUoVCjgvF*>@iBZVbW*C-KkbSs`9>9e2mT&7_JyTc z9CHJAnA9e7xu=OyxfW}Hf!W$4EaS^Sx+>tUNA0Y?6gc@Vhn9Gg8h55C!tQtK&P%;; zNThk&doyKEu+W7c9_$NW*-%-v)IR>J`S|Q{8ING8J?!6`4O!M zzwd1)me=D8p(anVy`!w}rt>a|Xw{$@S_^fXI=aPOzgoIUv3-W{F=qJg-@6walD-f- zB;T+-bTwc9&U;t*Wg-EtW2KlEx?MoDD0uF9>^~G*x*N3IbOOjRoiE!t?_~`f&rnm$ zM3DtwlV&T^48=btn!dU;O*Bi^Bs^VgGC?D~P_C`IN!dBQ;Cv!D2C<*5#cD90E$(sP zf+Q4O3?jGT1}X?m*u`QS%`{{tx51%qMz1S?tDc{kZtxtm;>86YGTD68p|rJS&G?=e z7NV%i!9^cW`IB^!c$Wyc?6cvZBF0PT);_QvAX*_&R`Z^?TFm9lMxh_>D9`s{lD~)@ zujumz{PR=FXU$hi9o$JI4)!0(%dVI-1Z(@=AiebN{enQBDX`@s=Z8~=_TVJTx*nNf5twkLK|F+VqYu>-LlwH7^lclNfdYrY*mcro26are0h{_NO{2cN+(44@fYeZ z+}h%yRbdAgLhz76svn)OFiS&l502T9-N`4UNNx3)TI{lm!h8K!b5@ngti|nYPXozU zc?F2`p5*}3U-FMedqM)Z6}A>Qj3jM?JLr+8$%Pi%qgfRe8^naYY?Q7H+i2%6tdl)< z1*F_iu&>>$v2BY&;T9Ju*~{t$9&9h%o#Y~e^XKEeO%#80%8CTh!9Rf0p06BlKy^;zwv z8+FNi5w8|a3hy2IRq8A~t32p=HY`3dcOd(zl3whMKTG|2u&7J-xa@NYcr1U}7VP>t zc7YfgGR6oE123F3;g4XUg;rUw7`s;*ZsqbJd$*^Y7kl`)*mVjK59T>GZ=GYlN`sgv zNJ8cHa)<@#mgXp53L;44(<8j;z}Tr5~^ z5Qpn2#j@EWs*X($CRQWH-X~LzKR(8|eOoNmML>fju?Ni z@8C4MepwT%N`L|NzXyb1F(SZAQ954XO786$YyGIF@Z!b1lljNx?`*nF3r=LKg5*Ox zFd=fe-=jpm8C#}XW@V-&qfC^@pd>*`WWnnOlI)5+1vM!3R|zxm&^+pgg(Sgyg!0O^ z&Z8QWX^SMalk&CGO}KsTnen&MW#zTgLIGO;$A$Hrjns&taQM3($+geQAB5{S#n?H> zel-Sl{eD{A)a#_^aoqBs`}Qutq3TZmkKg!v$>tfk85YhFi5OnL-4;!K zy^?6YTotAnZ=0HR_udv+*{`da4@vO*bfi$o6k%q645vKr<5(rPU9F_`CEiDX0I5+* z@5tEMl=9lTNh&n7tjy&4Lygy-B+CS^w0*9hBazQmDX=Y@aMSJg&br zizPEh6kj@w+q@93XdKWTN9gT9Z*6#eRJNO1N*CcMvk{3ab758KVnR1Jo|s8nT0RZ8 zW(?{}cOp%p7+{YSlj;GpqKP!sY7cKVk;z7>vZwBrUnuG{j_xTp8SFt;eW+C%<1*5Z z+=5Jtx*qR;U@@|Jxkz$wCK0hf)xw{f`ikum>CGjl=&Kq$%Qs8kgb+Vd5)7n^y&Csm z*3U}XK|ISd1uMh+NO{&yc!t(;GL_g5-gMgFLE`OsGte(_!jAeo{q40R@<+*}5@H<# zYa)c`Nt-&Ao$^fuqekq9mEn)bJ=LI<+Nd$4DKq;oZJvu{wlEuB|AEy|E+yIbGaNZC zC^hVlH&21aEhys`7pW-lNiMm_ zgAjwyQvv&z-(FC3pWe`F_HOH(8zUy5cM%(Jex74LeejeS#8n=o>8b83VX)iEs~asa zSuIqR-}w8_ly~3YL2q6X_p!igPl&~+`yW$l6XcY|M0aj*%L@$Iys;wY_Zs)+c;Mhe zeS1f)`7meu+@oc0SLuZkO=Lkcv*1f=LE%)Yub&Af1RA|hT*jrzUs<;NRP!mCVbO@G z{@fk5StKvnQgM@O$=hoUZ#&82%5e>qvm+l&mipm>O=2 zZ;Hii>KCkqne3Z1MAR4ReH-7Kd`)qUS!>YC|`x7!yv+O%uJERm5NA7s2)bt9wGGmGxvU%U4 z*IJmkC)>14Mx%Fp8cq;&g{nl(bTPqaX0`cPN&nf1{~!D6dr4153{6&xBk7+^f!5S# zm>XOjh8S1+g~F~G8BmE&$CZb zND#}5&)PqCXks70l@=){vl=}}^JR}F{Se0^qlK!tPt{K!UAi1~aGU5vSJ$wtak<8O=3Ck1bXK*>q9Qd;8=iN zMMWrY2p(Ii!9X8K#y&Jou@mP?pVaAE0L z3p-;$$H>GrxPv#wLD;oafGS^__-^{Einrz?TiIEMFBbDA6Q6l?T`SqzlaQx<*07o!XCVb)YH@NCC?jEpH?!&$}?6a z8ucyu*LWb(y4*7PkCE0ZsFIOBb-&Y>e%|k+1}T@leC0^Sct6TSFj>5Ii+3KeUrAp> z__po4=^sDECNmp;R(}(N4dutO$58WVH%?wf_+oymQ-;(TU*+vM3h1ohAVq`Ju?wg2 zLWmn%Rl9Fm$B9-OI?KfO{mtRl53i#yb#y{FN-t{-k9`xAIjdO4Du1gOkmqG%VK-z^ zd&bwaP}R)plrJY_D#WXT6IjuRqGO!<7)eM|8*$x08117C(D0V1erbwz-Hf&%FVk@7 zH|rm65L=p#AG4LpKX455y&so$vpOpRFN)k$_9aiezqVSK@u=KxnJOWZRoy9wC+&LA z25_q)o!fFVz70ktzqud`ETf4awYn<<=<9u|y)|=!JA(QNeArAEHf~ zrVck1Jfb8s+5uS&tX5N!S+nWCZ|2rFxV=f%AZ+E?o$<);q|!z#-Xpg^T(>SDAg5lp z_MvWejmScYVQvl+{9?A}KUyLh%ng4ocj)U%pt>`?`5QU0iCk!;?x_DDy~Q8tfy>?` zB@83VNzDu2eW9#=$dn1*GDA%jT1~kxh|>Km{T~>A|J$lRM{cfP)L<0%l3EjeTcD zMt6W2GjyA98YIb?wGQS>Vmf)}^Y*G$k2@lsc24dGlx`U2>PNJkV^Jry^=6f>tj&H^ z@O-C%wu3h_6~)Pv7-E7jc=>R4IgB>3n`4vq3A^{#Ibl^-I(@mw%4|noL zDTgSC3SQBEPI-wDbl=F*?mi7_aP-ru>=fihJ^t?T&81o^T&6tE)W26Tb=gY?n-=A# zWGBwKrpnIWGf~(DnxSU0HHbT=_GCp&*W?CFnQX3fe*>xJapLfl*8j<+UIRxVIdUJ8 ztx9COBhihAA9y2`chLA(@VJ3zmsS_V#wD5k&Z_yrE)A@%NV|MI=s9an^2M(S&IX z?!nygIF$((JCO_((aAfC{=8zlgRjf8UnuYvI)0Up-QGNaQV&ENFYrr0x^~Ad1R8() z&_nCM7)KHk6}QvK_^i9xtlQ3kG7#CbhkUc$N%Cy|z{`~=P}FB#)fBu`lKQptP&*`A z2mwpGK^a146lfmuDv*PtrmpIj{QgVYF2$ua z{Yf*^D)yg~omV{lJ2N>EyI2(ukAwMh+S7h!&hbn4x(Am>X2@l+iSy=*Gzgk z3wXl(*LQCovK|VWn-ZbA3TSrA3o3|ocMz6=BRk0L*v-mNEhOnj4|p#``90(-%plDF zQ$e^+hpwP_cHQ`dDzPtn{u=1UFUc*&9K*M&(8|mJOQFMUCi(sQ*wcXI{~TlL)2ko1 z*z@H%oIY0$qqUuh{5|=)3gH4}YcQAp8ZwtYv6(_B!t(fj<6IPTBBMU)>I|Pt? z0V{I)3J?K~IgWlHKz}-XH7irlh&F}&fjOI*;m7${L(g}krO>v~orb_LB6+n8!Ezxc_iWSLZN~0)O~O<@eiSGvfXwMlN}^ zPQ>GWUiN$I-?hm3`Jc<=HVTY}Gdj-fHc`UMb<l*nb4p#Az^cC4c8QV2#$nhbU!-;Exn^C|!6met;x=2UEEy}e?e5S3*G}@N*7h`_L zIo_=K)$m=#8(;p0i4O4Dm&vaz_}llNBC2NM0#!dqrIy)#sTfwLfZ5LMeTs|74E*Dd z-MM9O_FbEESHfRQNo5^ocQm5Cwm^Uu&JfNWvQrde}4N3;EBB(ZlB>xFLGt}sH(e3CZ#CHosoW+wmXxYMa zjbh^%A{S=AeTa)VNvRqoM(c`}b04!A>V4`c()TSV2>AMK46c`-)QnTIsyxHi1b1u} z2DK!QAS}?cgBdf!qc+6mm1N)OzBN+syN(c9&d7YCMkOiLn0b062TJ$E2AoD0n(XJ$ zgh*3=cV$3-u6JG{f#gtxx#JoXW*#AvFWAe?kDY35E5s`@y0;ulOB7o-i-S@cD=$P3 zLObHd_I{(hut62aqwC@CRi18@TuKK#+?cH#MVP7l5c@Fm^I6egEDn03G#nm+?^gu> zX;_v+MS*&AX+)>RbO(Dn;k2ZUqxCgY3d+(t^)_fB>Bm`5Mw4f|XG~7n0vayt4qBEC z$7p{yFE9(`qXHlZD~H3508^hW?yC=g*uehd#X&_qPS9K^?AHD*3Jk zi*LR{XYli3St|VQzdYD_9_Ng4LmI3ufGGNGsEO+Ao2F^|u*vqRG-ws0Q*U+u;JL(O zc@pSrynw1q5*{%)!jVzoJD!Va{Zhm(?fMd;3jFZWXObRQe;o6kMSAR8V@t>SRp}R{ z3M3)Q-H&-`7*{@#6LZs234DHg&M|ppf0PL?4f~iEEmJxA2>G~q;;8k}!U^UA@p$XS z8Ko}-o z8|WuV7a_KFSiZAg<_``)EU1ukY<=3_dL>B9;qP~T`fXG5V=!0sCFw!#HAN1wq8Fc= z&E#z_CyD2S?k5`A@}u;%l(ArW#gz$ZmUW9h<(QqYE)sv`SL<0cRK0vx2#J#r{N0ad z<*WWT0)s6Pa8Z=(uG(7j=iMos^<{>V(J8BfU0_5nDSJ8uaqS2aCtXeF{*^C&Z>L*Y9B+6i$zD) z2^x19r1u|}0lbsc3lYq>e<8?z88QFdQ7Pc3y$VN2ZK@-z^TU!Hb{rJCEk$oMy}}2h zB!VNEnPbWH;+ozI5-AC*)4vXesr7S+e$Hb0m`7US)$9^L#+dhEN^|B;gWH13eqwW@ zvT`+%9Z6adaox-%8_|jN&I_Hs(^B_-#G4Kzl*r;HQtG+UrsfGUvHm zutGCp0hFq=pddDi2-2G<2q+0ila7c9QbGv?Lc7az?tbq%@7?=DKIg-lbCl=#jWG>* zUzoH<^j2HWW=OEiDOK%vxJY*9m=7k$Twcrusl5MN9sb;^u?~NJ?)QD0$!3sf?nGGa z2y3q&(UlSbh$Gdiy$)H2N2fEK`pm)Q*4hd^eooyCxa_hpNU+lhNO=%Zy50O~gSX93 zhwr4_k;d2D3=P)s-PXXh{YO0C8m!*=6F4iewNr608zBy!ojMdD7+u)7S-KhBwIh~09bbf!( zPvp)oQk|mhT%Ht?jxnTSEV87nFp+=wcLM}X83SdD1UK;wG&tL9Nx=sWb8fSPE7KB0L9Euf)fRwp5x}PLnZ?leW9imk&=e{j;G~yl@Bd(M zapxkI;}re-vx&TzDKfO^*Byo@_o*~s>x5Nqhny~!b;vv8{GzC?sv>{EKur1P7UR{u zNJjb@S9+`7O&@=f;l)OI@N3hr{Z@p7c}{`ZlL_))eyQ8m(K$ptEII~S%wIaQZhq-C?pKm3B5;c(rF|J+Zt_x8y-4<>S%+KJNV9v1AAzQhx5 zjhL=&t@&;;!ZkDC>4=;w-Kkz(|5$5#SyI6{Plw!}GWtgs&G{v`Fj75PzPmMqQT6Jh zCgIvT0)wFYHU;}0Hfm7><4mc~=~ubQds+j9U+A@krO{a@HmZpxueYL(+8Fe`duwz{ zVo>gsGZM*oCSCpHq?JocmM~)-q(1)d1pdGCcjR+sq@287bW-Hm3FQ1il)0>xF}IFYZZ-Y> zrYgsu27#$jjAhg*qvkYW3UzS5X@%S=djONf;LkG=#sHic!NFfQaF@C<<3p9@U=xfZ zexy)vy8>^CwLl4+zU`b=-uYq9d$y{@ss?^^gmZf)ZMLNwf$~a8Oi??1Y&22tZ=(aR zD2YiXbZ=ETgo{W-9$SM`nvbEr-~iq5(aYD_{q@PAu9SYQ)u~kCnBABUx06`eg(_h~ zWbp)nWjONMUO)A@Rp7SU6<)ARuyI*N@#gq1lzst2Cwbv*T~m?TSSyt%m5nH7{!she zc2Kn>Aw|44+cqN5KG@;f%&NJ~9NOs(CoGolxck&#G@v`IxMjo0nvS-VA70)@DRhbd zn~Zn117SvRJ;OA@Ux?5*D&=EjQryG8JONcIrw0aFXU|&QSVl#&T)1bQ&GQshn#O!< ze>xG$J#uFw@e6FXZsx1-`m?&+{RESd>~ObhnyGx@S^HloJ^^eLN!g+l^i5#P0B0SypS66uRjjPwyt=iQyb^WIj%8#u$>DwhxmnW5F z%7SdzuV9>vbxomX#qadmnfL4BTW7|Cgny*n56R!3#=TMd{k~%qj{CdV4=k+AO|JJ^ zwWO;fj%c}huT^S{eZkeV){m;W13KuT$2GV+y8;)5=!9@$)G5(jP`u{m(f6I-` zU3aCV^T>@ccS#0D_-#K5=4V(k1Kt-r3t5rt5v`S-Coi47Z8= z=Y>n&>_mErMcw95R(+Zb(Le)B&u0pF-|COJ^ejmK;!aVLO}6rE-u@)V_5|&^5AvqS zH3OwwJgn@CMjl_zS=l3JUqy_iC+=Kb28%_QLs==Fv8|uhmArb^pj+tKiEAIMCwwlo>Em*lH;Df8*>Gtmh-(qz)1xdvFh+vDtNnxx) z-jH+=crYY%y1+_v=bp*{8NPJTG(-U|s_C4>r5;oqeSS?(5KI1~js5=Q5_!kkNPj!-kV|b+e;!&0_yutN8v2mqsMykL=)(ruEMle&u_F`7$VQ? z>7;%jn*R(Mc8V|^KA4mm;8)#V*?M>abxgr-YTIO2T+6?lhW~2!z<;%WlBW~14J?>0 zAM1wP4A|0JHL$N9h?vgEs)5NLla0fI^;?g>bnVH!Zn?3pKY^zMlpN0*L|5}f6_WD@^BZi{DRwIH&7bMlI0i1m>n$`P5mpVsy+C(|1 zjNe;bi|<_eFv5Q%lkdAs52#@#s*^1W%@-oa=jzUsE z<_Hh-^FG~6X|5i1-ii1bB!jh&o3c+C%L#S1c98}-z9tPmF=I=M$|qTHuP1^vF;o-j z5T(<_hS7C3?<&~kOU-d?ffn%=vy<0c1dTHb%p~LukP+K%lL@!kcIhszTQ3JZWxoZs zhX7v#baKOd--iai_G#w0Kji9sRui^ z%=PCV=J`6%sD1Wn$4@-2!AOJ42=3uk5IH@P$#rF3TAp?sU?i3jPfxWu$qaisLKXhy z9nG5OODFV{>Yu9Y1yLB1s%$kE1EB7vc<>YZKHAJ%8sP{D=L7D>o8gq^=CL?gVA|m) z2CDx@qn}Z zd#%C@n?fdTaS9%8&KutrjOr4g?CE;%%$L(@IR9Kqb{Hh%2;D?{-l|~X&&c42we`W^ zT0Xq_N?_H-p?j3lPceZ8aggcmdOe67@VvTt?@&8H8OyqDBLJ|f(b$l*04QQZbxAT* zu_6`!y>_qf&UwT&^8{EqB8&H=RB6fgyN7lk2kJ2T%w3|OS?V7{-}dAA*?Ue|`9D;M z$^8*_z$87}tJ_#Cxh?D~{AFvE^fiKX`XYC*FEbq4^EoV4+7k|8z zT^aO}uC>3}e94lPZ~kY@0k4OohoV?<`x(ZD)6L+hyH*PHv_%j%oh?uZy5;z8vA@>U zI@g|dIvdf8VaaoZ%Is=WZ`Sy$6zGUiyd^1CXXc|V0RyIVlW&qXhv6bZe}I#YFm;WU)rjTIx}a9kc23U>wWG2hxfzq4I>ygnfa zo!4rfu@UwJsD{zV2N5io7?y0)@ApPP5j_ceZdyl#Gr!hSwR?lC)xqTu- zHS_w$XZ!aA@0@SDl2skZmecqrFUcOC%j*6JQac9kf5Pu4MHD92zO53YnBT2$l6pzh zSmb2;Y{vm(-Y0oK$vUfw{Qjd6PVDKr^XJF6*!)(jAyCaK=)wKIFpbsl%_cu3)~G6T zI;%JDo|e5)n`-Llr_{p3@pQ;LAqMA%su7-L`*uh7e19mte@+-jbx%PiPwIGQWGO2! zol))2lw-IU+e8{4aQ&uX^Vk}ws6T!(2Ii=`uLWr zorfP%bt4S+ddvqBj69dYx~E*3=DV?0+%&OWBvi>lTMN?@-m5E5y3c&Qv$v*hN%e!T zEA_e#w)OEm&;_GVmqc}s>^YICIveID|Dt~K39 z0Uj|(n4TFwFzD$ex`4oU~hE*S^wqj`HvXc$a zd1=P>_t`Hl(=8?*e7D7LSvwiu=I-5{J?WjU-8$!yaJ#FxwxrF)K0H7g*8 zWifz_q?eKgsV5>n2G2T%<@4yYaT&ZIptnaowNj)kg68v!M41zYwsCl1s-{zJkKF`J zmh*w={d%sh7~Gb&Wgcg}NAoab&OuTM0`-3-=>LA-y+o_k0;WXZ82lJHnr_AgQ3(4SwOrm_?ho_VUe_cQY! zI71e|^r9$gx&Q`$2JwYZwEn%v|Ft9De#@vN@Uxc85i!7-KRct`nwMr57qXq67HSVT z8=~7*sy<$FS@GIDKKJww|@O%O2BIYmwqn8a|SUm-_yjmzpc8W0+m0 z`GdTFM`y9VHh1Ztf)pUWVft)3>z!xAi52bo+D#WtKW1QL&h^;w#mo-E(n`<_&+65K zBXW1?8Ag*L05~jiVHo0RECgup6Oj}XX&f8^SUd#&=U3Ptt$gfXFv0&|rzF+TS7ANx z$&9%1Fd1#Cgemt<<52cK=wz;HkK11R zq)V}T-&b+Mj7HT-DD^J_G|x>~bs;3J)V2JcL##gO+X25~{6BX=@^_PJ+I(>1dS*6O zEps9rB6U+3!&U;S<)OD5>Z?b{>74$f)}Kv8tyR4VXUfXYr!}T2Qh3{TQE(4^CSOY%v1G z8>mx%Ax|T}LgLZke*aTB0so~)Lm##=U>R$Zm@WD&geRMQOdB0_;msuZ9e@|UI~r`^ zvAK}L{gq`E_yYKLBhv0r_T_RuO|9DISl|%;k4E=V&&-=X^wUO+Uofjnk*u#1VVj@c zF$Y52oD#^#ba#$2dn4HyUufZ!!uBG0-lFNbS8tp|w%!ba8%eM**BlIYrPOUTS1E%T zi(}&6DxSKLxyfssT@7jF%tunlJjJYC&sN!72v!7wk`_ereaHjHp}I@}&da(MSxDJb zCgiS!RGAc2;{7I#mT-qxO~vH91)kx`A6bjZkEw9|RtiUx=1xp67qfPbJC+ywC-QB5 zF&4i1M;)|<@;q2!M`~_>l#sjSq(qqk3_%k#lQX zqo!eX&uVEqVZeRS@`H12${6+coQX@f%P5kkFh3gxxn5fTzY_fa@K+SJF~0lud;q`+ zh@l$-#L^7FKuJeTnmES^NUmwZ0Oiy>3h)3-le!eeertuXF#D z)*#wEJl@*I2c-HttIcFN55??q#~&SPmvJB0N2#kR_SEvw6gTX%=*{losr~qh*LG`> zOJ3|H&=SG2x#xXi(`E0TyqX5W(Aam#7Yk!ApF-fCoMdugr7QZwq$iF&U$k!@;1wRRTnFg3As!V7fnb6>(uT?nx^t608_fzH7 zs>k>0F4)g-y;xhPV>jY;r@S6Biw{+zq+UkoEE?8AzLF;#VaoA6lksbD^-sSIhJIsX zPq;9ggW3#^R}t-e7Z!RXd$L|xazPWu|2iX^V7~JC!p1w_F`4*|Ii0$zkaePRJqHVd zfh$SjCwa>9#7k2L63f5vyyfom`}RJn40bDDpmo68QTl`e$DLjqrT^Pwn|l4p?PL%7zKc_XYL4f^jaBu>Z?1q_y&t`#Tr4 zS-$9T&_4p31#Fvf)AifePDM_4E|y1Lhlpki|EIG0YI~3lh58?r^P74WBsXmip|8XL zockP6?^-geG;Be+8J2E$&g7lDZq(10Y*l@7Uwok2Gp%K3k)y$QvA{b(*d!=gN91Op~0tXy=4>-Ls?pMrgu+3$cY6}^t zX!}(cbr`o&xoF2>4D)AbjK&|Q0(W1{>Bs(-b?lL?UfqnMuOM9b4bLlI0?+Rf zdPQ()cS(7hhCP0mgo<14_*?@C>CFmNmNx**gf?+0qcCG;h??IH!MZg@vsX zEuTtG=p3o|AxECF8M+l^R1sZTqE_3j3F(jQI5tcn(2W2rP%=w$!@34N)jmXIhi)bG zHXIz+;n!D`rp_4lDB>L8C1RijRYgp-w76{hZwCLcB>O$^at7BE%d@2H`mvfGhwSw^Hk`$k9b?CUlo&@eoF}yX8FEw=}=& zxYNS(a3OWq{W{+^s-u8G?xq*pgQB40z7hIUNr$PN*zDit&kHcXA#Uq#zl?PRi!mLD z0e~ZtzF(I;TqO;B8h44`?|p|TYHB&67ErHU+S+Oym0G@NvR$u4dU09m${UZ3aH9GL z;Std-?`zY-lD97H-uPM27Fj;f7#;zJW)vpm z9FIqAXYU)lGR5n5-nh(I^LSYsl}}QH*`Zpd8~o)=#(hbfui%_jfCzB2@$bO zdeFlY^du9OIFTyRd-Q{LX~zeJuamGst4w-7?!E44@B@Iy3hlcPm`AuC_!G`nuynf1 zuj)A5zfkpXak<5?DWX%*=lvP*+Aur&=<&*_VMn(NV|MCQHbrCG>5XO_@s;x=xBAgr z8RH50kB-+8xgvfGgS+~jK%ZlMr(|p=m}HN{`lsXUE`|sWI2Bc4?Ef`_7G7fbEddr@ zXF%Me5tLH+FlTZo+WUy9B;`J}af^TcT8S!4s0dkPw{TIYEX3qzZi^(?uv5i!4&8|& zd{H(k3U>MW)8)mO$3(=cl~Cn113FkA5F;l+-Dl_o(%( z)py-cL{P+R1QbniRs|*OY8M8+?*1Ldz?8dA60?o@DLK94 zI$o~&33cra=|9Dpg%GH-uq>GwoqCH~YtPqHIuz$x`zTAk6RsB{>|am1ma47E&3_aA zc?pJdXuUYG-zsYZfeV!q9vhN;Fw;1Up!qyMZd4T`Z#QULf$6~W(4rR%-@ylSNj7h%6H_l>oVW9$W%Z28eaOq9$ zqHRq`s|M}`s~@r_jefup?rSJ}+`8^B(iB4iB$sXodAMKgk5r3Dbo{ti=z+w80P*3- zIONj%64mrPipjwWAUPBU{sA&1boZ?7dlfs8Mcay8i6Ke>Ru{aug$B{jsuJY+Wq6dx z^k4V1)fDto;OaB0@9-mI6%kMOJJ7T5a(8*8F5EeBcs6M)z?9h}-gGqfp zje9qDUlf+=eAIwW5i_3LYRad*uTHCCV&?jgB|p&dHL&W9boZ!F9}pjlZSQZY- z<&^E#KVz?R0bP9&tXT2xZtZYqmo}cNt}TeYXve#5DV8L>Iqz)4MT-XTxfpDYN3`TK zwbERGWcUKcGBe#ZG*nAoT$$P!uze_f9o|@-OE{dfTL{y*ZfKPmxF?{k2^w@k-l9gF ze(jT0?s|=6L832ci&G|-*9Yk~kj{eG%LX75OUxKKGBic+wkjH$)|xH+9Pdg|N2+0~^Q+jU@2 z^b4+T8q^~(UP3%_KW8V_^}9#x$_1cK7KU$o7FUo9`@`ag-m$MA;qK&~U5Nr<>dt(SS9G2MYXbor zAa}ch`0mEteM=f#-@!u{+uNv*rhTV$sNR0|)ax*6K@3n5& z*G?Y4{q!5k{p6)F=>Eq2jp`UlXGH? zxsJfd$w=FC46e37Om@drEv%E|^Y@*<5NZ&9QtS%?$T;ecT-$Ws-p8LP7k!jPot}k! z3q(kfyd^dT>mLJX1-!WYs`)+uXPDZI{~+FbAa+mJ*|Edhf=hm_1*mC^n? zGed|*zDdO7oaF#w;M;ymO+x>nE@_7%=ad^c`-c5HF>CN9osWLn(z@e!7D<7IQ1e1< z20K}HXZ`)6y}znpC^OSRr1|B8H4NmO#=r|(_$W77(WzhG$y2@4Hd-QFFR?PUxM@q4 zG5Tn#?kka*7#CRS*J#+ZqQ>UKR=LK7r)!VTRtQkgJq3cH@spCfs1?K_JMGc@yZy?U zu9Z?^v8?tz{yBI0o0->q=5HLGcVF581-1ybFUFhREExtvq{g}289vtaJTAP(QEaOr z1E94+Co;LN`NjR3Th*1VmU|$t@i4U%Y5|h>-wC>`B{~N6Ked7<{S94*J>Scqj-$1) zV9-?bD>nrNs#<7F;}YBv?oea~4I||!zf+A=1lZI9C3lp3`M0t<+bf^tj>imt{jL|wfzmxye~YL)Tb&2;Amw^dhtTUgt~%9 zwGXy4UQ36R3s~EsD{WGzL94BbACF_D9G74WhRz{YCYuQk)V^xO?n+S-c3$R@iB#_06|M%qilYYgLoqp)4Kl^99D9X`1A>JR5Z zF-iac;A>hfF^Nw@N5+PHuj(WC+U}N z3>^C%m2DDs-R`M)W6!0FLHK5Y1xM()#B_hD%@2a==d2RoSj0|>Q{6-N@U+$vxzNO% z{NF>XShj7M)WAQ*c9kxyrDe-~frZIhfFEu3nskzr)K(1ZC+Ij z-TI0vatJv5&Z;sBoP1u$jd<29C#!`ei&kt)i62fE&n@VIY0eDU6nk&=tPr!R+Z7ep zjDO*;Jz&oGdvsQ8>N&M?h)D=S_Az^Az!*4H*FWdIJQ|6W&MjMa8Blb&9*!Fe$%zIy zZp1zvt#p=AN9&1~W1EAOJLSrDhtYvKp~ZGP8^_ip2o&&n@pyG|g!yQ^%;1(eTXWd; zH#wwAZF6%M8xLowl3W5{xEK@sUy~a-2;h!P&T}(t;S{Qx|?c$j$KmGW7ujq zxcR=v|A@D--_o_2#{lDH)ia@;4U(egSO)xh#Iwb;TOHL;e17x~9NO6L`g788AT{XW zjA;KiFg}z1+0*ie*7LO)t;LnPitU;>_HK*z5XW_04I{@=}1zAK@O1F z^Q(FvgJ^kZEYjfnTEI_Y&6_2CtCYad&W=)qw!QqIVxd9NA%Vw#XwojMxjPY4vG4Ui zXXA#*k_z7AP8STIv)>5AKV2EK-#WlCU;7F!W^rfNoL;pt5E=}fHXQjU ztb6gYIu^X7+|ZQXB*&|T^&3MFQ?2Cq;S4)LN*WaH@jtmBSIsi<-*GU12V}>+3TOmw zO2-9nML?Hycr6!t|BzvQNQ~1?#jXwusd- zNa+P<5v2FhST=e0c2v+z+ysR^J$)YVoS3oO&>Fma>|FN-#ze<$36K*W0W9vN8TPg8 z-Z+5fl9KpR98${HefGiuT_FkrcMp;k&3w*H8`Q$uV^^A1^%T`RpMT@_*rfc4$w9+Z zXF!8O*++w&(KK(vuhPBkC7zFYOfMb=gl3Qf`8XLA<+%USneP@t&BTLmFGs4r^v|_d zAxcv@Ao~wC1r}sQ0cH4o!Qx21I@u>aQH!!?)g#?J zIl8Kq($mF}!+5;71uZ0QfN41Q3j6(H>`O7@vs0O z5Fw37+k=%ZUom84z@OE?II0=Dg`A^~#$5B%$-E;s6Wc8@kd=ad7ot^9g$3_-15;z? zi|Q%kRR|iI3v2WX*sA)SL$A-d^yhuy&NVpeSs&;a{qzQ)OMXHmJOVhO5GPi=kJ%64 zpSAFL};(1+Hm*T$Z1?GB4G3 z8QlwhgF>)O_0oPXswI+^TLErWmha}LAx!|?3%^M1k^>0hFu>R&O`2C3`x}s@?f!$y zHNF-`v4?th=@0!nSf<5}yAaa4o_U^^rswj}xv)aD`*(!F71#^-U4SByUsj5uO)ML$ z^FA&_nMxV?)&4&3{HKoC+#w7Sa`Tm$MT-6-PsJTHY-;Bo|C$5vw164;hp;c}nD?$x z`yW@$Up=uV*Z%H_%!u^krvQfF+KON_$r#+cEdmowO-bwDBd0xtz^Snu_v<{DkFbi3 zG85NRoHMH2=O1s^fCIt;^cj%7KzeD+9J1u4ZM{Cm#W%$JMUyx2w;;HRblP}*pwal* zhNE@IE*jQn&}`VF*rz;aoSQqlATzMTOI zRe`m44BEHaYnwF{DY}stl}FLSGbP|ne}mo7Jb&K&cM1VHCS7=;R_!W%Ixw7C*0CJ0 z+6H5Mcfdz{;%?5Z2Fb2qn6r7}k(?GbRWOJDNHnwl;O_PLboE;asmJdq zzK4>pLi41TCxnx8!5L zeSkBx4Z70TXWt43V+6pUv)&DU$=e6mdVUtP3`9R)&1}WP4c1%~sW}m<%}rIJjW$oJ z$txNC#ZUd=3&}ZSOMt|T>ut(m)tFW;-h-1dUhch*Q*v!?BVl0Y|sud z%WcajtdTzejWh&RZ5;F_=B*d-QdU^F5p zhm2scriNa{5dPCl9ZBE64A5oy1wo-$m{k7-rqGPV4o^p{tc4xO+Cb1V0H@R|xknhD88u{q#m3eU?_ih|8r!S$s0_YjT~#(8i! zca#8N-YA?JOM?Wq>^@$+S;!9wXNAz`qhcQB*fVuDF}0Q(Gv8aPBH5;-PDfc@2EYG> zkGDv({1jJcwS&JEl*CHvWJstv&CqksyxYMhSCq7hny+z9hq@vX(Me~b70hRJ&YT3FHVSrk#tjfpKl_Jw9p`nUmAk+O~7q2Td!E0a%dzIxe>x)k!`z zOsQ7=n|6g`oV@LzJBlfPPp~qu~FqRCg^g1U`OAa zl)aN6{BJS8zi^rMWdPGYd$M$IyH()B=Y1!GI=kFFQV}iy4`5+UuX~F1@poR}m2s>9 zs%^pRy1fa@4>f_h(cL6eD^UtV(){#K%?g0cLSYQltpIN(XZDz^w{?Na6FV)nz!N3SY;FjiN+^f55x zfK-u4sHOZ)irX^*09VJ2@CAZUuARacZ8 zhad3LqBkgK4^Pf#oa~p;ReMqd!*Tae161~QzB>$w_#WlVtNiTxZjPrkDSnF@67R0# zLn|T@&cZmD{DOfz0gA_KFnkC$B?5Hj^Oq!JF&ZzTSEKl7T_HoWce;m!nUiT7;Gy=i z!=dwN+Me0EixEnTt58}!uIbt4jXlDRZM~qo3f1WA2LBae@}-N8zV{(1M!HxaX;x6L z;o#nEmIKv*ZoF7OaX#Gpov`xeE$Sg6w4p`)|25db|9d)!RgJ{wq4Bd)j4TAh`nUER zdN~ymXP?1XC(jtrV1`(;tKtVCadMWhs{}ZXVl?BNx6nO_;`ZeyT65-!xQF3;^Stp!wv|Nx{G>s)gt#! z%F8?5TV^oXw-W6U9YMx?zg@T(kSE|VC*bsb>3w1Zw9A|YIQyUnPk-5p{G|FoPmQ3e zeCf;56wM=};2pO=H&(f$BKkV{gCum6#i2BQ4@W|6h;mVLmG7V;XL{cOZhVVIY5!sg zc9|_1gPJ{=Q$Bp5I=6FTG;Ve1O?b)qfBa{t&8(;Erk_PPw_=YhI zKB<0GMx2skDM<4>Y!IJ6v|$;LXxP?N#l?`2%!br5Q9~p!5;?~W&~5lB)Jq$$THLAi z&dE2^7W8>39%D-oL1kK72$`4D0yMf4u+`a4zRf<{$AN5ST@=l}a0U(noYEvi>0b+y z8#!76+5x67s0Le9OsJu7ZaAUi`wmocU8sk2kxT!f9MW!=Vw@&HYc(KKK#Am%!P@GQ zi1V4gP@zJz8KzG(|@w12}h$FJu9$31Eum3|vD|L%w|qgR59AN(m1c z77Tscg_&m%*@u#>&=$@7EMP|{d{BKkDYK+1jBM?`Qs7qB`)0* zKkCLt!hev0zaXHcS*{6KmJ&d`#(IF@fqh)xS`ZwUHNaZ&P>roo_N$@DFFAIcVbEd= zRxP?&0-|Y=XN=I+B9D~A znK$tY_5ulI=!K+weBAZVhl1te&D3)~k<2&=@ z2(0?OWnhlmt*m47OVz^iEmK=?H>I*WiH$+$La%)j z>|E)dRo)Hi9MeZ1?m`**Hz4PemLNsfc=WXE_{Ku^bRF5>OJx+%A4aQFKw+i*o@{&q zVWpa?<$vxn9T?Zf3l6%xnH#Ull#5c4QC%j^XkuHW+s%Yn&scxM=Q~p0nR?6^9aD z`@Fqu&y98c4uzMC^~6yG?R&Cc8nl%v#ScACwA`XjBM)-A67{j*|6Zx~-!xD>3Y%cl zf;smaE*c|U4r^T7S}@S7OcAEE?(B9k;vqqEO^BW^!r86d4BcU_^huJP{l2-WuH!tn z`e3@ejgitolW@0k6^1MBI~=F2rThbLXk+HR6VcrT!;6P^CrC71ll1rfc!2am0c1(@ zk|^+)Go%dbPDeOSDQomTFrr^Kk3h!;+icu`T=1S;N)O$Im%`zQ(Vgq1EZ=dY< zLw5EOpz2Pfd&mjd*x|9!G+qmi{M!G|3n^Q+3rW>>jq-3&ATr7KVk;_(Z7fY_U}JB) z)+60?Su?&N%s2sB?$_D~3;%}Y2WNG*%P-#psK$^lz)Mq9TW)i?1jP@#<50p&ab_`h zhaK^Mn~X(sF%DklV+SB82-E#Xj$8@zA=t7q_Fnr@=&b26_wLe687L5gU?l;PNk8Xo z8BIc%@!S}Oz*FFAw5X{TjB|2ApXrg!Q(srT1Vhp2lBD<{XXQ-l-^>KfoBeZgV^_us zLIc%U{rocmf=w|tOPcI+!9)U3?VF0Eve9&+uFcyKJP#rw~Q6**b z^cY>-(EJz(3iK($IQulItx`Br>0exC07YsR70eo4uJViM*!5* z>%m2+6u*S-jbTCB^jYD5zX9m)F}mu|GI9@2)5|!C^oo~nQ+rn!XD+3UB6f{TwLeF& z>Q0SYlj#(2v9ACO7!PoSraTC*8XIh7z~?*%INr z<5V?+T^oFHUxWQcC`TI3D5&YAz;vM}zh(Gau7c_Iafz*NQd-Gx`X@wRRGp_O4jugx zz>{;b>K3S^^~Hd%WOrn_SogHQSfAcN`ig;Ki63x5_Mh1ZLAN5q3D}4H=9}p+qvQ)d z2Zw@w%LdhF#+~BWD)HH?vEA<7if(VMGCf!Rgt{$B79CW&f$G0m0O-E>YA3-RvQ8X5 z?lc2dd~;M;1AI*L^~N)bJq$}g9n$%9C+_UP1_fC4uvFQ=%L{iN`Yevl4H6o~eEm^u zBZst_pE0Tn1cx6%4Wm<~ODAbDQlq1!jUIu_=y^s1?`Pv}Xv^nmE>5PnMW%{_|aqcc1V|6%{P&?Y61eIcfX znnQ9lvMA&VR8RFIiFZaWL?~$}4XKVkRgbNwZd^i~>Nn=pLf`I1hLAHH7rHh2$J2I> zE%Amr5d&Z-%c30PzrpV@-FQ_mZ9x+I`vmB#wg=l#&Qi*Vsh>It&~~Ro^IvBR-*?}@9qrX-dWP&VnBIx>Tf4!F{#)hw_k*k)5${_0%&{mTbJelYFZ|1os??m z=LKLCMb59ipvq&(1n@NPJ7a%lKW5yn|<}< zNdjr!j8{Ngrb)z%x)OZWC zSGr`RQQ30m>H90r-r>TC2bjjE(8+I1skPKTl~enLdi1U56}J=ttRHmo2f=|{L>*MI zuZgq!3x2o}Tb}SfCmc|Uo5_9&GjlZUZC(cULdNe}6|LiveaFH2h?vn|x@Ab%1u}v6 ztw5@}rE-AowBCd8}yNCev{y|*(`5CyW#4kU{tYw;zd^w{kAs=gbM0S@!biAGBkYS=5n9l{sEVLdL6}Y zCVopwc&jWCwg=ZD5rY$k07G$0H0JCPsZm3w~nci0W&cvj1$b?p1Y4BjWIFR6u(f;Sp+khN4 zf&mO}3C~!&e==}ycMOh@{aOCHh{(EQq>lu9+Yh#|b;9?y@X8ntpKU~|AC-Hvk-DMx zxMDx8MNHvsqd`Q8VTaVN z=?gkQ^j+QJAr+adT=LHG8LD_O*tgZI#vv3X4+pmUq2Fb6*tC0B(|sRjPeQdVG_ilE z2)u=M!=n^^W?$6q09dl72rtb(bUiqRejlaJ4C$r=v=!ER-npDWlInMQ^P_J1jgAVC zzJ>WuOEP{%KA(tu2^csZVxPQo2uBjsl}wf&$(QL8!%(PtGjV8I-GUQ8=^kI#`O{>2 zxkX214a`_5xhby*-t3!R|&+oJc)^UVla zC-~3&wQ-e@4wB3hz?m@6)?$gW`cCz(Wn=WcTgy9%mHTJGpT@xN`fmvZzFh7x$UzSEYGc!nN_gLti{rF3YU@$V?Q3n`)s z7xHrj%v<-oGMlm^kG1cQPCZqz`a^4DM`dw!)Ouaq5vNZ!7YaQWX1Ojn+Tr?sD9rNZ z%P`B(hitzz6^tY`?(s@$#9Mf#i8-$#F+STIk|@0v!cWf-i`u~ow)b=(A8llM`Eni$ zYV6OWu?0(W>tMlu%mOReDADRHze6Ew0TPtENPbJFUq*S&GYYl)d5^I9WgvH_xu&hL zN+8?Zc%RpR)!L!~2AlBcB$}HMA!M+Na+m^dYqGs8?ZkIl#%v$RbJ%VvZNA;i^IIAnelo)(wKk!3$kWa+Axr1e@}>+A@1Prv1NY7 z%I{G60xSvl5g$z5Jm|} z6xbU33@vOGBF)>k)EQvO$L9YZy52jUt^fb~H>;{escK89)~LN|@6p<$_KYf_YVR2_ zs;U&VXKY%#)E25ON)V%HYSv1u1PLiIuJis}-|z4Dy?)p4{pa~Fx07>T=Q$pa`vci_ zE-j@AC=&1R2lSd?BY`RfB%^9+Y|lGJY$3>zbwdELJhaPn7vDe0CPw=o0Zm=`QGQL4 zem4j+n!G&8`9i*d)`}KrAJi(3?+~C`jiTfy-8&0jU-T#3=j0#UOq|@! zqLp}RO-{H*-{(jR{%sC@Nj+Y%EJwD<1p5Y1qGkcB!me(at*uABkM!M$mU`f6AkHXf zhd>v+t0O4}t;*}+^1{P;bZ@+Nn;kq)pRVjGJ-rYn|K8_1sWt44ky|ju$Q^?4hmWfc zDF5ynhj=;Q*<<`aTFa=irc=8F^jYE8GQ!U!$s7k<_S3Io0f5z zL^8Zp>EptSvvP4Me%&wyh}SCZv%6nBqxPF3q@&7xUSOdNNg+jtTdzz@Ic9A6uRXqm z4t;p}NnbgQ?9GRvSB>uu>$Yu~b{B-D`94vV8JqnE|CO`OZ~v-bLb)MDWrs#TCd&Vk z-k!Nx@7J~K?KhX(|Dq+1LQK?9?(+>i(%%PZOr~$E<1aHVIMfT!@2+iW&_}V2O@v%X zuz3nMMh`qPg8FwYxjuZhP$w{t$U+%IvWm1+o^f!8sC;oIEK7)H)9;F{ka^oLJA~GK z>$vhV?ho`r4A(5SNr|1ZVv2n4;GTLTMd9N^L8;$%HIBqSuX$W)<+$Doh@2Rxn01?e zrm*Rh?{+Z_&`nxE#zTiqvV4mkofI!Qko;aooeyjcLQ&=LyUe9@?c1sW_I)1%^xxu| zwO%f1w;*l%^C4j-lvfp>$z))STCkL)mOGl3#S-ikVHWbD_ep;Fy;=7x39TK6fJECT(OJ-wZszqlz~*oP*ZUWn zgVr=C1LFq|~Ek`9PvcTuEmE zYW22BU1S*JVY^JER^M^V`HCAGKO*8L+W0?7arasArBi{KtN*@J?9bk z=c^2Fs1Dwck74hn`p;f~!*~GRD~9a(eQ{B7Y4RU;l~Hjg*Ga%-&FoFvQ~^1xVdVo= zt!ME{Jxit+yJn;-N{!!iUjoAwR6H9rE#$1o`%ImXYXL`)n~kw@3(cnW#8IA~vO}+t zdITM}B0u^h*Zs2ml?H*v3D0Gt{Prbmn?&~q8-KQ3)!SyW6|h^YICShbn*`csk@#Fw zGD*2{%^bPRJH;w7Jag2FG*4XKwKo`Vg%S#|8tY1ba6+-zNIJ0VUCnr>hN%R9%iM{;85NmzB5U;Ka7d zYR0fVq}RRUAMxTRE!m7xg9d|Ja0-Dt2K42VqYqhmqRLaQZCbiN=t^63ReIE2-KX(~ z`Y8hKuVawz?Cpx<)Pc-NL|G^|E>x<^Pc$}i%gac^JykYR8K6wB(~ZZT6q7Q2q3*lD zI9DEX_KtJ^d-W@fi(D{sxn2&?(@DJ%(MwNU+lIFwD;JTo<|->0KeG~^p_*?;pI9Hg zJs_EQt{x3{{fhC-`VJFh_c5kNgldsiLhA;*^b6^fKo?#sXCa>sq5iKn$b{?e(mXfx z^K8}$8C%@o`8!2MaIes6WbZPqhEPwp-6xY*#e?aQfl*`TjQ8wM-=&DZA9L)aM4E>N zQ*CbVAnF`=7tziA3V+*+Z|FlmpVyf&(~9|C+q=cbyUZPD?|MLyJw<0%@p-Dibv*gZ z1oTmr!go_}&|%DJ8p@>B!}^iHsd?GsWRgdoBOPMX}rtP!Ke6+p?NCdhIrOdZYrw z_&46WpDFVIm9^o0HnL7Ay;OSkeI%!Ba+`&^+C4%N{hf3yS!0B5vn%cR+)2$l;(fiI zUOL8x?BR9ST#!~&B!B613PQ5^9So25VjW3cdT}SNnx_Bq;LqJ~Ywjp)ht^$NfR@?W z^{A0{d{h~8d3=?iCWC0#NPBec$K)Y^usv8D0OpJglVAx;Q8(!Bt`6jNot5RU^mj{0 zXHJ@9A;!wB|(O*3xqPX$}BU5s7Ampsi zcU_raQ@VnLJF#9X=8FYxtf)W0GBAlL1(7BHA%pm@+-Ivra#+8}uo(SKwcTJ)aWi9B zTFrh@OyuM*8baT6)E&xkO1(?B0$U}ocheZ%(){cj1O@&`7ZgF?@tF3&W&Zs7S&e(mf{1V8pFd?dVT_pyMrEe^L&`YYB0)ivxTC3o6(+T)5gu+ z;SK#g(v@)2L*{N|m<4}mbGckO^pjS51ikJ{N==%-1unL@T0pd{^D2+To>oTSz`0WD z%}_$eBcvbmhmHf#1VAcHmQ!^#B>DFW(#R+3Tr{E0zmCwDE)O`@f3|(UM9TV`_RWue z3;=pLzQ6X>$^Ci>;4?||_&^2~ZF1c2rk~V09qvpVoXx} zSkTC_I29TrNnn3b6Cykv1T^3j#iH%Ya^=V>^UWabBG#BGbY|-&-r~iHqAt##0>I+o z{B@k7-QsJ^aSGDcc*}pDQWY*UT+4|IkA_fX9DxGYfXZb;fqZ;5+qY<=Vs`3F z?LEkc{lT5r2M~{rkRh7#8g6XHrAC=VnNQi4i-^V9zwi+vdgDMr_UHG5Niqp>Xy6Fx zkpfSPI7_$d5=^HfgkCbVBW$OEo@WcK`7un$QM71wc-oez-0$4o1r?F^tiirFA@jSqkddHy5 zi8h3}{DEpwYmmvi>rj0YpJ#0?2$1cu@k${4(}^#i{WU!y%brnQ4>@1QLH&#Ep@2U_ z+1X3^zx}gyy@PD7x6eKJkVX6Q>ubV%CqyZ{=eXHyF0%dnTBLsZX-);&@RP4dPHi+S z|Dnex1JZ`soXuyXfA7^B^Ly-5=xa1Eqb3{2c|Hhs7s6rD1BWs;M|9EuP`zV;mXi+=s}mbM2<^ zA6;5^iZ8m%$dqh20bdX{>el6Pk@jyA2%c;)VwzjH4gpwGQOK#~?rNml{) zIQYnFwQI*|wD*UQ8wV90?B=QOOu2T$?b-XwSw44x@}Cgz-U|!s0)$oXvWH%4!goj_ z(Unuob0$i^&~^Sh2`Anai`W_gBL$w{;)jWh%#Q{xp7DB02e} zZ^xHQCuFIVnD1aH)Df0FzX7$)`@W?3UPRa5N#HK2oWTve5HF`a=!XTMA_XD64OxpT zs-9%?z*Sm^z3Z+5BfH-I_g<Vy5UjUoFXH3MJtefZ zd-^?F?l0w;Ms4TeTe~pfo_X0IGPGaQCEZZ3DImmm5_)%8PCL`na<7|fxkCmhSIUT? z1q3;ALifLa)VM?Lf%${4l~xJra4a$?*}j2sheiTyAHFXxe#Txr#xurnQ=UX4`MoV= zZQMqCyulG>{(DL~&9p%yhyC%=6(2b~L0}>B~*;d5}*xEBgpF*#g{^Pt*>%~7@{p_rwXEjGG^dn$g;45@$l**{Z z4nncNpfyS6GOvI>OEK{gh5oBHrsrNh&yo>=(jq4bRpb~M7ET}pQ(hK_MueIg~WQ$GfG9BTFOA`ajuGF@N%|N#R6Bt`S37_8AqI}b_jST%H5P+PcwF#(>*+q zZ|vBFE%?>Y6>fyY#$G4=!k=cDjK!=3wDSC^=#K6t2kfT>vlO@D;1O+Ce|A&6L=HDa zSU5m>&?J|0beh^{S6+PPL>M>^w{Hq5bV`soT`74L7EDpd zHtSF#_t!&xoLnT&Y!^ME3$ie{XO)i|Rpb*uL6sg-V3hd-?iLFIZ&+hemrPvGvqIM; z%tdc5Sqo^gZe7d#{x5#F`d4E1-IqQ-;|e()9-hj@BC>6JosDG9qu?zkfX-w$aYyOh zk|U@0Ik@j{`fO}&~ps+dauu_^fSH|Y!pg-bnoWmB9n`Q$YNoLQp!wk|G=(6J+0@^ zpo5;n>fZ;(SrsGXtttL{$rP#Q*a2_Xt$5_43)XPUGtv%h%k@|6VT1n#_`U2bAL9s$ z#>Sk|dQ&ybKt$#h0I%!io|Y~#iH$)+tvc_fqfL1gMgL}c8Ko(EYZ?}rUf{c-`v2xF zU)_#=>0LFxTML~xvguL!M{+h<_66@ucDF<=r4QelezQ~jcm{eb94ho*&tFA--j>FRg1hNqxzyhQe9CM-Y?#b)@@yz!#08n>Mh0%in3 zwfE#RG_-O)GWEGwy9r5oX3Wp%`Oa1ST0LAEIg>5j>ia#5`Ug(!Av8)a#+>yb(BSe| z$yIso4nTCEo%`UC!!zL?Dj(%R8;HM0v3J&CqQqR|4tg&li*${nO;$S|G$WBI0*&f? zcqT%sV(@(<5?=m|iR&cW<{~3x$#S=b-J`eV=y+|=A#$7?<8*W>xtB5t;DOw;HhNBn zV)KZB($9*!;#^IJ;kKO}6oaBQcb3HkkwdF?u1?hJ@OMAS&WyK1l~;=tMU$GV85D^| zMXO3bU!k8O+L#rAKInh*h5s-8A257NqiLeVENsUumfQ%lyw}O8%@FBbU_#t2QIj_2 zXJOnD+mq8}L~~aYd3iOpbA!n=Ii!|z;51>J1!yc>JCg3)@XD#-E`?oN*sWTvc=S23 znwfT-E*kO=le|r@M5gXGy;RB1zu6}jOY_h^I~$-G5y6S?a>ur4w#H!oDptnHJQfI< z(xyhDTv2p&y@<1r-BBwewQvTMR}C~=_xO1#qKio3uGqqbbeYR-D54E`1HApYLjf@KP7KZ@8?rlC{4HM z=X2YYz!AVARgm(vovtqgp+hHjeh?wce2Yqd1O3V<#LAMHbLSd>G~o$Wc|4&sZItMA zDdnRRdtkDW!OvUFFv{M#Dk_Zs3^@%3_`uezh1$oWr==_AnhlHCh({cM>;L6h29P(< zef*cFRG7`a$F^kSzHd0{+aW7mlyxS_zO3o#@JpWBaOq2KSaMX9*1?O0J_0rxU(4qM zE*m2cQgH1JR||Rc0^3&4e0988zr^Sze*je;*}G;T{JiIa{n=GsXlRoqPs8XVF_HaH zYmiMv%wUU|7`IC=0Fl5BvT&1q@t=YAP-i)y3v3cAVm?1a{rd|=B!WEx#@B(x*I65DfS zs_dQjAMR_BkU!$vJ^tbJU*y8P;_BvRt9~EV20r}K!3vv|3IXwJ3NB!0Ns0z1ia6Y@ ze^Dd?*=i091Tj_I-;_O+Mt~q|2%t|_QcfBnJ76r|As1WN9=?B=f6Wf))o5f>ppuFV zdmtb^NW&PO(||6c1U%ZZX=F<5rIrfgG)gvC?s9eNUeb`mGss-%VT(oTu^$w{+%YUa z%@REHUFm$V+^(|}CVd|slbA?mcyB)y^Xsc)2MWtyI_o@9DP29H;&BQN&P5xFOCRH?vd zUp`3Hd6bwSw&JIk_(F($I9hUP^q!-6N-n#{U$&;!t_9ub2=Z9gku9 z+8_^8de~FVXDgtH`U?T|sbSG?xD;JpZER@1bNtgKMd~!0D{7&qlGTDrJqn>G$^hWH zy?LPp4q?cDT(J88dT$9941_(-KMMKJ=%Qcp|9KMTvnaUNold5t(Wp^;K$Yc)T$acA z#ZggU3VSRM@2KMHbZjPif=$#9pSwbeuBHNL^YWxF7UKb>M{!=N0lCnBmVlHaRLh4} zX9^vZT@Sty~NwUqjzQN6s@Q+dmg zlu^lF%{psi#)b41;Ly73<$}e@-~|?anm5YNgd2%6=cd)PmvvOSfC<%;T^4B%gs;Uo z`{3;&t*BaF(VVYH9|zqf-CtgHp9%p?kM53Zoy@FqH1pHXw3H7OXSg7Km;M^y3kVA@ zUjhKt<>PWu#rU<*3dVV2xBcc3AR^H%o=UMKtg`%XJmCLz{<6zOEyfW-gVXa0jc5mA zS>Y7ORds@g>Qjq*g_Awjebq0>1gblO?Pc)MX#0Yn?=SwkgkBv4iotlawK{$$Dy(dK z5!bC^Xh#7z1<%9z11?Okpf`)&qfLdAp8sTR_{9fDo|tSG{c1VTsBVA;Yu~>`UiB(V z&_>69n!zqc?-eHJvc)szFgfqWSu^ESk}vSMIYy2=pzk)n!$fA;Hv%N?SUvwnZ6)&d z0NrS}j0hk*i=6S9+(>6H{?^2m6-YnM#dEdM^yDge*tflf8}VqXEVgi4!5opW4?w0~ zRDe4SUV7j>JIFcU@wpgE)^R7CJ-Fl1v!@>84f{?E?MPGxj$`OAgO23b>|m5a8wF6Vs5FQ^I6em%TawrR~E>% zGQ1VGULff|mMb}je9{V5#Y~5jOFzFJDJ*tGCvneOh{AT@QY?=x)Jj_Ya=zdLU-G6` z&bLKUsVV-j+1lOnMR0yGFXbz}?=?7`4H&1rxH`p?D>z;byyqkPC;+cMa5=mA`Jp4m_ z4h&%S)o>5?7JVl(q!Ku$6AJQI${`O^Zml5OH8-HKy(lfHF|L>ju z=OQ|OKy`Wt{}YaEGlyA1-*fVRZ*&o9X8|ZP8e4^qol$52E1EeK%Ues3bx-a z>Ddpo3s)MXC0Nm$W>*=?W!3N3KA4!xMjPzs*uX4V(9^HN7Kn8Vb;~NFHXY3r@KGT` zCljPQxaH<1v?^2Rm&!CLA2oQUGCW*QrzXNJLDIEuc$@J^BTEblwRtEi9NA!eZpYu| zdF9CA=)&4U7rq&O!9W9*3Gcm_8$O&{7akh<#JIjK2N3NdvHzR2OEm?B1X}2DQp90oQkLED2F@e zoNRL3dmgyNrY9aF`=m@Jd$=o|f0rfda1s5R0zN2#e0+3Fiwni3q8%Z~8rP!4tAy4i zQ^s5Aj^A!T+k3Bi#45@ths-L!Xt+@Kt2{X1mwaWu@3x9Wz^VHrec z@pb#pS?>tZmz-6jL-@+(3pTbvg-o|@9o`yR!srjY*!!MM4SkZW7_N0Y>m)qr;QS1a`PVDK0*(&$O*h=}^>mCwmdJE6IE_!u3kIonRmK=qz zj|KP2%tSdSyJpwa>TC>HY{+`=;1n!TFVP(Kb?)U-RS3M30gMxSW4)s*pvxWxj2cL| z_m`8%)8Bt#hMHxguMw3RxO0DJ=mdhp+9T+Y@(k@ulv2qVVv$)Mr^5peGgXaDM~jD= zi1|0Sf;Doy!$wSK354z7=1CuB3MEjf);l{_ z(T^*oKbl&ZRh>Jr!!gAXskQ`YlN_O}>X1Tsy-N*Sy!wZRznsdz4FU4J1xbnDdL3-G z3Uf%vCl`F-r{00*g*)goD6fLkq#as8^uil#$bM^i9Z<**kGm?G1FTMhd)}fno)ol+a1YK3VJdYy7#0#3PhU4~R zw*!-#v5vHsY>64``!u#;e=FX_(mxJ`_=vfDa*H+$zrE}9-M)oVwdaLy(_piuQL(o= zjKmX`jzW{b?)O$8Bj?b_o1`9Iv{fx#*jTvGxCVD8|M)lEjzK+w>EqI_=843#108Ae zn5NI7L?B7!s27|D_(C^@eGVi6)^|e)_;;;m51Y1~0F?_qd|`{y5y( z`HFk)w@{Q@S}q~NeWJcQtNV#BPB=0bEGsJf7@z-GQDopRtN3xF57AshVV}94XqSa(P?QIricv6Hal27v zyAS+f2Q2)m^Ty`Ogy8GyrKZS$79;uWCcDy_Bwa&(%_pCOmQ{1Zcai!Y4gyYNgZ7u+ zsc`z=@jmKQhyd^O|Mh<_gpQ#FOKW8xucDS_{rP+RvGzK*)gV#3Tc$HOSV%>hd7dn_*|B>s z2r>q0=xM|R8?rkw^4=My);zYlgeZeoa)%x64DSB;Y0*$txXl$RlGAA)8PxFcoy*zp zK{vcLW^+AsT#<%uRRGV`upV*n5}pki+bG+9=>)a%q&K;m6F;9wv_xDj`xSnV z0wVh2ZpOX?WKnNvR(GS#Uu?nwC#k>a=NqaS&z7TDsX>1_PI=LcaTBHHg@EzH+?4_%$CmHO^;Z-{2YcxG zhtcj&BMYFoy$Nx;6Xl_?WS(C{d{K=_Mq-e_XdNOziP`#1fRv=YO9dN^l2Adl_3?2D zrIf8Au!lddSl&3ukKg$g`B{YD$YFG!)yb}b17r+XH}}x1$Oo@ZZ1mCi?w-~d0xJ#q zjY};a;prVUA6ohcqoC#udal^7zDV>HZ#4bJ&u`aXws!0Uj-eQ@`udK=CgK%mNQHW^ z!3-~kQDrR#!qJk^f4|C3#gQAwhJpu*b|MvNeI+q#k445+Tw}3{+1%imkI4Z{lLnXh zL<%>k|D@_lf7gyTJ9r4DC9cIwVttJ-?QKtq3ccBE*I!CYT8bS-1Pqs|a40*9Z`I{x zVn}mb*XsPQH})19d6!Y0fJ%4#!yssog}+dq>_fA4$k(9-UjLXGV5Bb53IyEmi~f0L8NK}C7ylwoogznN&L4;qf)2PzDO zBRX!2Z4!Klzcnnh^7OuACxwCs5zxU5v{RxY=X4hO;nLzciI2kRzdDNlQRKGcu71Bv z0Bs*AYKnPUo2Z`$pVHfx${$i`^EHQ5(@7O2}H^B%$hl9uIWhPIE^?tIyE zdCv(P+d4-l1VOsz^4Itbz~@;A1Zq#fp(w=_cPi1GteKLEHr%*l)9aYq`iWveWTNJW z|2C*qEg{e5gO*MJlH*ssYMUP6b2#qdpKy|uU^C3`olo@ndgK*CfHmfu-3{|Bj6$U* z5nN`g*(oKtATkqrqd5i-{n1~U8vOE-7_Q9Yih;o^<OQ^;p(ZD=j=y<^u48p? z*2y%#^sJw$bElq~s0LhX2~ypM$Pk>I7w|Tc(i!_+!MJg3=jY4+`18X|;pWK4NI!vo zuj~y|-Z2LJa?I)SL=ts2+6#foDpG@G$o0ulw~%XnPjZMSstSZ7 zisU-bh(F%ev9_GfNpG4TV7Ilsxk2Una!*b?*H%6!9kn5`ZRc0|7B7FmP60n7s!|R7 zkFF&B>X$1}1pX)-Cc{5g^VTuHRw2K8goCr!NevRA(f`!rSGlc7p<%D{nF!Eam%(C-3t;{9kcAuM`p=<^u{yh8^QA<*u z-hn!!6!I2WncGTPWs!U9#Lk~!xqOuq4hCxhTj6ZsrpSuVV?4afi*`}|Zp!8{&te(q=@{r&_;bX5PiIETEM~l`H4Erv~{nzOIrxa{&lb}OViuD*c+QV z@Wi>3&$P#rwga`Fa2+GX9rj956d~UrdewB-Js&kST;uk0VSB2ZAw=$-;>aIok@lqU zZp8+XiEVHZ&ir@qH}r{^*mJasvlrL6e>7Hb-3u3w&BhE`B-Fc`R~GSIuPsH?|=Tty3RXzvCK&`|M%|S zJ|$KouNh@Q>))Bez1pnuTz@hoUY8&njeU>dEgFL@`5qL^Sk884_t)72@aYG%GQ!Bj z=|>Uq(K{r#D1BYHHS{f|#o;2`3ZmV+Lv@J(6&znd64_;2_~zzV&mc&p#Gq7kwnR2P zGYDtIjDkxBbqL?=O0!+dvgve#2fVG@n<5W;h8~v;Jr{oD-}mwGN7h#pyWStkz&G%% zYKu&BF24|Um63;E2S1f(q|x?1BQ#rO3TnAnHL@o>UlZj05=tAS)K!3ezag5^^2Myb zV&pUv)hj_~=R$_K>Mt_Jz+rH}xDt_a-gC1+-|r?z**T;%A)tl3?MCHk%e4FZ+rtuq zh3Xdi#&VgpTO(JF;lp9usTIrA8YZvN`)_DCgMAsJUCs2=ga(Qb{^Cdd-{=Ug;(QVs zwvp2fYUkEqGvW0(&Z}ax#lyC@{nhj6W_}fM9_LIkP?ftFW};(07Sy%KHatL1kh=eK zc&1;Ey8+b?1wAgV%0iczq$@AOu8K4vP_{F{8Qh;PGYKPERho<5-`a;nofH1X?;M}H z-aH%A36h8?P|Cl^fXkf?tQ>B+2T5L}8qip(eO(*cXV{TT+*e|b*rYNIef?RU_i^+X zPu~#6E%M?<`gxpxE$^9RkZ40z(ZPOb^T}|ew=W=Ey1`(E%^x7RoSRDDwad~iU;VSX zbFu~rf*d#n$%npIEgb}_{T?{_0L@cUKJOVPhEJ-v5})|WM~=O^nz;{YIhEohWg}8{ z!b_TRbe$8?&d+f^X>|k0X3HAnamnSktcVnkb34d)x3SH(??S3vsScz5ww zUFgkt1fpzb@7Y(IJL9;?z&Gxt+hyp|L)HiU2_4(V;jQP^6>%-jkdD8Z?^s8H#?>|5 z5;Zy$AJ~lq2`O(-6vE>Ds|JtP%ta>P_u*-Z4mj{x;*Q^N$93}u1o+PA$?_}PTUonc z@6YgBJEloH6tRi(zFzz6(wus=+qg{#RQf_Lr8*pSfEft=@HS@^dBMz9B=IC3&N4@Q zG4H0rOw>qX!k_yl39o_-V3z3^E((Zo_|k4bC2Aj|b3|C9X?-$H{5&_!D&6{sp)gj* z7se1e-Wl5ZH=(Yqich7fsPLVGi|EbXhd8`LQ`Mk~%Qx(#1aY7sD0?RqlQ#kl^py%H zicyCJCWugVcJYO|Nr#oxcIj1iNk$^nSqATSnP=yjq|U`vX8d?8#}6#FI@*lOH@fes z8{DoVdLAVjNHN>tMn^T+DIO{tzO@ZyRPAtGRrc3$*S2mE3eWCoB4svpd{ZCR%RLnQ zUQvb&fH{ir$o4Om?aTiyA_hYTIuw_xdYc)c(!H*i19`R9-F0(CnkK<6o!voP;N5~N z7M{0{BWi!jWvzZp_Qg%aG`kSqScdo4p!PiG`-%8{OUZ`y!)oy6H2NpnYOHHfYu<`8 zOX>rM;=Z`$GODX}r<)}=A3YKXoN*ij#u{(@MG%QAitiMO+1}nHI3;DYc)IV<4QRaJ zcz7mt*x&eV$J+_qWO2>{12Vg@zG(#_@4r6gfbR6KS&-}{vsTfOn<26jny}m7u=jOn z*BC=1mv3O+UcIyMf*33Z*=6TTd(-o&cK}&B)}z9l2A0Uv`RI>lU#b$7kud zFS+BtQ8`-oa5FhnH>Jg$M?7k6{sZpF*b$jo?LLPq&ZGsx7cnPSP$hhvpQZR^@zg^m zRlf*t@r}yJs#R8-*DVkS(!eXnYIqX;tqkzoq)p(i>*rt(U6^EaeNv)9F%-%63lSFI zTpGe@_adcC>Fc1tT@I$Cv;Y>O#tJBj=Wd=AhmsQ#oIGfM9m7vtfULulZV82_xQ1`9 zI7wqh{M|&H(yoOi?u?>jII5Nas=W95((IBc!_zlX8#bH;J5MLbUoXHgnCF4B>v+Z;hS<9)J6VYh)c!;5!#wr~CTsR*)vX|xF6 zi*MM%<(mMbvSBe1mvVgS?csypbOjp+oj+>sRG+RpDImjAGQ^j<-t+JuySa); zMN63Z!ay*M;Wg?nm^Dj zn$`5a-1TpV;v_OVlGs_2Uhpt|xpBpf{{6iKE%|q-z%|Do@*?B?kVk^ZT?+dv(bE(D zyFJ&l8Ge)l^&J%0_a;1%8M*vj-v z5nVEzv>Ab?=2fo=zuhX0da$X>!uSwsYksDj(A>1+U|u}sND;UL5`ltEY)1zkQOjrY z$n-w~>nX$F!k2x4plvc|>mM0z9rVM0ZxGehWz1mts~X#@&rkB|%)>YiUCS0f;=}Z_ zG+$p#?<*Wd5V{Ew>XrEg?`11zZ>Zf@MyfhQKEF+JNnLIQ({yQu<;u!{9YvtyM!i-? z5-a1s5GSs2Lbhm7(AFZt@U_tuck^rFu;-#(_K`DGVOx=y9_78Q5(?8|QTy;*DZLA= z-A`QMxwzu55dpjGD%I`97B)HbqChWMI~lcpk49V$TeL(Ot^DDP!w3i7ihS(!gof>n zp8P#Ed^(bq`HzG39A(1X3 zVGC7_^5Zxx#}*tW7kDSXId_V#&a#%OeCB^dXCkTn;;o1KTv3JbO_OG}dr}o1*-+{J zQjvDBH&((ef5|Db=q1YCG?leM58j*1q`nx$@stzKJcF=vK0k2|6&Z!LfTq-6(~%oa z+6`cXPnOo5fI51R?jx3Qt3zyB?`wYUaA-m>YgMIjnw_9zGScGEAAjk?|>TRIiI3g|y3* zE$F4JXv1O@3AXA_DmgUB0V!-`4wg;wWVir)X^yY7nYsoH;f zB*7cB#~=Ei2zTZ6(1O?qi~IMePI$RHbKhap zFlzS~()Mk657ad|nS)>dG<*C!XMByqWHbnTyP;#bnbxTHr_kHc)tLDV)Q`7GX5s2M zn{v;D_Lj_E>4w2>OQ!7j6sXH~!wT}oRCiGaNP2EZTuJ2eh*IJswD|B5M*^c?`bF;G zhrQr)*<*$KdiJ9(mKwdv_T6~}TeW_mX~vzmy)_56M^=`Iq-`dvRh{Fmk$RZifCJ0W zLGWV>c!)5HxZpg#b5YN+@z7;)P&}Avv|!EC;_H0vmx8)ST?RQdT-VD($Fj8w;$2`5 z{%5UPifHpzB+Z}qbs9X%<*esCwEP{v!9`KXoqj5pEs!Bzr+a5>^{&7Kf?7V?m=w4c znyTv1Q$LrxyxQ;XGe~e| z%-p>Rr^lWAQ?U|}!Y0UEmf8eGX!dvk^<@K|RrB=F&iQ7&QR{?X8PJ&qqbr`Cc1h!s z{hli`J1h`8zoOwYoA2vtBFM*Y&R3frg>$^_Zq>mT;-2Uip!a{={ALHMfufh#Hh`E# zd4zz4`W3U>Op%z(U}BR%sIYF}ZS@Gmv84twC>V@INZ_9XHV4PJkK*SN!%pv%as6n^ z@LoFlz?&l)6rV;=q3+2Y6yBx6Q5Nf86dOyL!J@O+G5L5!?+Xb1f0cHvRAC=~)}}%@ zPWt5I1Tsz*IxOX%9^1{)Y!ehSX!%|Z0hoAtgy92nQ<3q7`;gvJ#aZeT8QyW7sE;f0 z68A=v{FMZaRH9UKM_Pf(?rF)}jYt6z7x;O?m zEdIng``4;M@7^1qPDAzd{B&^<9eRpy`A7x%0pEV-Pmk?ts zLYt0fjG9D&WG<`a`Ju3mgrz@@s*b&oF{J zjFsM!MbIb7DofGZ{T`9~H^W0Ki<9f%3Z%_V%=mdrXrYAC{a5LIa zjYYNqU(aqLUc=BnLqip&szL}!sLu*WxxNWT^*Bl(M<_P>pi5a3+st{LuO7v1JTiV! zjrufo<+*9*UL@2}5W7F_pK=%Bo)=H?mgqL1Ns`RYw9C=U{g`FcmZ&GG4Xfc=Yrn0q z?Q=E{s4r5ta<{5Amn3;QWp+jsHyf*Z-u88FqX3Jomfus$eT6ms|K zx|H5Ou0ip5!zCdnUoOWK+`&$l-e$Z(#ip=5UeO`Et#}2ls0Hvc#$y>s>3?FFp~ZKj zl{br;lYoeKhftx=?t_6U#Sh&wkYdBH#%GZn?sCQ>-4Am4C1V;~aJ5Yd%D*bpepRia zYj*^Oj0Al3QP7TuHZ+H`7AjV*lSG2vdEd$WexE#bW)grG;vf1*wi&?4KOTDQ6S>qX z?lZ8Ny))gZ*{D~X4{t{fhzd7Bb$ZdO6HcC|Fj%+f_dpKaBEpn6>M8EC*b}>Dy!(9} zXP4)b{%^voOu7%LJC*1T`VX9!-M z<-9AMb#bs7UXXI)szw0;en5g-V1)LEd^NnG8YwA6NF3bL`kNt_2&0FxV7wA~tfRfs zzi)to`hWGB5t|MpU9jMnS#=%R!rkF0y^hLE_()q(CLLjmNonm_sz6z&!es==f)Qg* zlz`N_dFGgaJZ($rDa9in#h;>&zHtO`tmlkNF|V542X$*z>Y@Aa#uphaD!zH#8Ex|6OmK}JZ|%*&7U9@Um?$@P9t+0 zr7G%-vbGxP&h3vbwJiwtE{W6S?$*53QySUvHU(XYKE^0O-*VB3{CIrTYNs)kqVQa# zLjIz|{Jm^NMp z!LPF7&jYP8JFFS~!&qYP`}c=utsfg~97JwxPwu!}c7ypdagA;N@HZ`9)nNKR2bt+g zeV3upK~(aPdmydiDVh~RUl`8ELa#U-BQ80Wwd!hvLMx8#s9C=OYNz!EPtIz|Ts279 zG*ct_WC9(*0e?^rrm$aZzP(OSNGuDY3y^$aXr0xr0n0BH77&7baf{HtQ*%E_;{28N zee$v+ipt*Y(mGvEf>^Yn^_<-XL)jel2PnnD+SjY*Rz@_eY7~W3-bt!xrTTB&l65M+ zN`v3{4BhG%HELDcKCv z#UyCp<~Cj?y4S7&#LU9}k3z$-8V0kDD;9EhC0dAQEjz)8$-5uvBx8g$EZ#!h3n< zW%wo*P#CWJaDZAc-2HrSDdl=zb-)dHlfhFD)@8H^1;lDSZ`-cXk>)jac79MUDfQ;`_6xR1;$a053 zUnpWyA8YeF=BrleJ3^&fZ_MB)rtU=>Zi_xC_NPTavzu}uDHU?ev(3)ASTA2-M3`=7 z8v4MZ2nnUXQY~9zPbb)5w>Srfsb2JCPVT-0zy1n(5{?T~NtC<#^E>F~{c)$Jad;g+ z{a5PF8IF4>5O3^B%H02A>pc9~?)v|KRjJu(DcVa>yLM5tLTc2m+B;fXQMF1E>>m)exK{QzxVfd|9+4AKltS2ocDQ;=j(N5%!-|sOA;Tv+^I<(wmk?LgNr_ zt*IlhoBK~ZRhG%&=gJrM$GUCe1cmQE+6oT2`6+DAERoDFnByBC9dQB-4az)~urf_3 z)M`w9Z4kf3&PzU3H9qW?r7E1!wWJ$F@J9a}t92M;@XVMRaKfZ1+))t!HEMRAWtW#e z?LH50svGZjVB}p^NL`3)vqsoPhd|&mkrUQZy)Ji4=^n^DBN%NO1Y1-6Up(_T|2oMLHT1~?Q}Y=wxVJYl-AmbkaW4z zOmiKt-{b3uYCL;H_uux-=LH6jX6k0!7guOg>alkY#`T=%cg)2iE9h>{Vgj+FhGep-T)2Jqlv6JeYw(@aX1 zTb~KSa_`hFM4t~5rp{w|Z~2GctLb8J5k3!xlKRpRMQ!4Sgk9Zl`?9*~d)*a%9jYJwU{Pi*Q94j|l?AtZg{{qp4kcK;S^v2kof? z5@GnI>;b$`&sot#otz6e(*xbsgL!!!acn@sA(#;nCsEGmG@q(DTvz{SWw!HH%gHQ` zg%FqghBdWT?m8hOkWkb*xF_Ju zY^S2&;{{E-Fe*tm=Bfs`Sc-k>O7$Sq`epaxnoQb#jiC=6 z&37a?SzSY$DD3!0e+UjaEGr;jIgMwZX#d;49vH~*Km@C+@(g_!TT)j#!`OJ2Jjqo~ zx_8+usEYG$)@q*M5aXn!!K}@fRxcBjW`E%yNagLEe$iL;Tqo=9lRKFfontl+AN&@I z1mhjV(a}<3+PCN=HNW*h=oGWS=1|_pgr|s0POn+}R;1gM@81xs)tUqD0H~{iuXWtf zO^U1oFF;5lxd9jDD7}u8Ub73-{mf+|`l~X9U(nK4BQqN1F^TQC#667%Hl35ZjE4lf zJA67L37%#s!>w@zHb#-fmHled$+kLG|HZb`b8_a%3186kryEYT@dOer@y!|UBw{FM>$KhdA#sitRebM0qol#V$J3>?@KGFdH?Tbm1D7B{=6 zmOD15#lxyp@~qCX>=#I%mJa*?w-dgfiudlOiP%|(D&T@~E|m+#@_!c^XN>=KPG)jN zl@j(|-GCQs%T=iYI_AE3mX#l_i!x?)M!fgSmHM(hP`9&>|5W}BnC?reqHx!r@Q+5S z*MEM}S8m47*q_;uY>GV zPd~L^!Myy89VZKBs2XpzdnUq_*W6M!55g;2Riv4xU+0`G9FRrl%cgyQnEg4M0vd7( zl%(4;y0K&vJF9kTqx_`LSB8k2?ElWDaL`N4H6cZua)2DtHREZm-?p~!Q#MuLTTVIH z5}nsh{30kzCFMjxqSw2u<$X=_6_x>>^ZCDd&N>n7Qvc_~yI0R%Wqp6vjyucoO1&`bW3eN<$m=!pRJT<1hM+E>e!>1@B5GJk!|wLla;AqZCl<& zi3m0;sN=fa9Ud$>p28O6#guglQ_Y8Z+ls}#yrT#I-lUmr{FMKXuk+EJ!|ZGXAU#tG zln~iKJ{b-m{OE66#e6Y^?Bs$Qx1Bt4M)_e=5LO8+P|%78J3Vf8&uWt@v;7cc+vnly zUw#+cE;aAzp4Wekq8A#>d>1P;yC923jBh|}9ZKs+6*1I8*o+TAaP`a4@{$_E36k`x zZ}U8e_8AqB@Gr64>=5H?sbFgq<0`i^>Z%LOX3;)$tMa2`CeY?*YA7{1@wTs)O+O5XTd%)s6`LdEvnbL( zUyIQ3U8ntFTjjC$VYE?6+K!DdjBl@k|E42iOejE`U%DLUMCJsrjG`1-92A6sdNLdT zdZ&ZdPz#`5Lq5?fOUr1`^IUJKP+kq%6SiGK2b_D!alTn7fVF8etM-7?t^ z)=7D)>CETMwh@SR#p7SBgq*$Vo71Z%R2{NrLR2SDR?7Ep-X^^HqULk@7j6A8;=1n; zyqAr{n(%982Gs#xta_(`itg5*CQqq9e70+UZ0I(&$NjmFftjE~pFk~wJ!1m3=EG&; zIfVu0_9_!o(uV&%-oMv%raCPkYpXhzJ!d7b8{{*5b4ymiiz~yzyNur2I?i<=n6fad z!~ffMfue6E!^z$oinjWR2fp5mclKJhNYek)&ISUpyCor>tIGJ;+O+tCR-RFF;yJY8 zmhHZB;R~~L?yT`=PR%jh@yicygPekW(fFNuM*N`E_@bKs$^9HDM-5R68+U$~wh7=o zVrV**++TEqdA!oYR)F_fQ8}qm7y(aVS~f|y;CQof9Jk3?QcmpsEy>Qjz95<%5tdMf zP-+@TYWqcD+xvy@_7>Bt-IZD6-aD3ind$;g6l6+(luB4O;miamx#{e1+az~WDn+Ud z?}`D?;MMBK5nQg7CeZC6qc~(wdf5^5#4JPwGeK?R zPf3!bJyh{?pTJqPhd?`Q0P0=eBBocf$^^rXv)w1~n^~a-Ixi$eaA#HO;5#O zNzHUkv8tlTe3CJFmTg&m+%4PnR>!a9TqMhA)aka&2<<;^@y!gp)`G4^Y0 z&Ed_PtLIv`SHwf66|WBc007qo0c18Jmr>D6WdemH5jb>va{9Z;b}Ekx$$ao1J5%b_UZMxpIQ*CMhll42=kOb@2~AlK zzuB02UxUt{!`Ueu5+ze^A|@zdI*||w>5uuD+4>^2j2Z@BDarR>=_y)*2}W^o?TsPD zmCmNK$`*nnCB6GhYkbRhoVd+c2u@5T<9$XOfJKaJ&%>(IU+uDM7X~OBkzd#dR?B(Ly{HF(~-j;;WoM6WA_Q z2!Tlnw^G@8_pz3a=Ja~FRm@+ZQfq6^)IDCe00^+=nK*_MeeD-3{`=Qzwe>QmnG&8q zXpL|uxgKt%fBa;vJINfA2U-D5&UAjws{&MtY`%x}1zS#KA2A-iQfvC&;)gXg2fbaA zzpp=aSjsO~X)6o^BO1N?cfwu7%zyMJ1z$|8<2F4`5|S?tBllaKdXB;k)4uW!Gj38V zl-{}1tU?j&niZh9?5*KREy|R2I6nQ;2P0U4>t9Q4b!^v1F^~OCtlF2{$$eV-=IIKR z-NDULVdK|Y+h1vkuQg!IX>GrJBp$n_V;a@ev(Su?u94d1@Hoat8?Q?<# zd&g$ux7xe8|Es)g2z(&5y{6j~f%x(S^dt@@ViuBG@@zErM5 z4U3hJyvd8*;GBHr^l9G+R|+=Q&EjJ87wG>EMRH4MBCAsdO-?d(po!lb57-#5GS#Zy zH7pF8z;i#>cTii>#66Bch##H!==$OUu@=T%67}AkAoHrwByU zb$+RTvgZv8VS*1P6Th9B`22R;CqvB_%`mT7r$Ql_rA|mwmA;^-Z5rlWjiQ0(?wjdLMKQ}*H$qVu`oMyu3#!Bki^Iy#0c?DF%l#2Ae zDO!GvN%gklQhr76mj<#my~pP0ZR@?w68|YeDC$x0xRcibWBoQR_#*1Qfh=pr4ZJ^P zv*%prI~$oiE0)ktMpc6#50$lf7KzcaDNWWDh6T_%OF!7g%%}Yb+wr!hxf$^3 zNA!!C!n9%U4W>UDqSsCU!}7ljY~3om22YC+LHgK}cN`0f$#PLrWN{keqocNrpI;&k zP~BYT!sz?^KdG7x@*{flANbVJMw)ASq)XM_doaiUJ8M|IP>fXj+0`Mv5mJaHo;RzcAoiPE83h_|9}lDe5XqHBI~-~fHB>ke07F!G5Yf~?;Ptm`w!g1Q=dac zsBC?6>Qx^kreSNa>|pF}vpH(yJUS^HA~52BZ&?8K z;@=&W?wzyLn0>ETLwQ-`H;V1rhYO^ppYQ!zF+$tAZL}D)43zVDWw50Q#8DwRJKc_2 zzYmyZ;q{Db&c&t7;~Mp)B7C-vs_3}P{_i&Uk6qySvrUJ{8|NoQ1)D*X*LgH{T>oWm8{5i`n2R3`PMDVY2@hV^HVdn zb!xKaaZn_IgVRer$tzkEEw`OMrE*2w#79{O3W+r5#J}h|B@^w_u55>Oym#`n&fsn0mhYY#GimEqdx&Uz|0o zuk7r87w9a|wJ%($i##p~x!P8Z7Km?*XcQhen-_cj_dtE5hXwH)T!VeX-@J1<@a%HH zLc~@^ijb-F*y5XoMgijo#~*a^HFeBt)dMsvZC>wGPviBIn6I(O&oiwW-k#Cd?nzHu zyZUYq1HWY&=EW?=yiW7QGx}Z#`=4V1^IS%>el6+-VxzX`8QA{gX4}wBAj8`(99A5niaT7>q0@SwYqo-y7*}Wh z_j_O2QNbP-{5X!-JZGGkjW_P-qJ%L^VQL&J`0l>#?QfLh#X2nO&-tuc^93Yol6e}| z-YFM9(`wh}STB)dI|(svCFSy5|d( znQ_kX)A>5gG~pNZ?Y{0{xe|2)_7>BXV~9U6g3Iy@w0i?aAIA;12nZQR544DTZv5~X z05uAU=b@kLb}6kKH4a)N!xt!Ny*{jXA_9?L05NJCGvZ6fkSbbmWGPGs!#P<(X)o+n z&n@)P(I#l`p89-#KbR!tAHnaN=u7_a9a>RMY4l=}b*2PJ z;`w|nblvdw^g{Q>D=fr1YR~-**z1u|gV^uVr~!e3ErBj7d&%lR{rg=)1=|g~S5|*> zUstB2NW2t$6Y&a72@8i1o?8-R!d&X=N{!EDlJnW?1V%IBZ~Au{Qb0L7jjgF^)1&K1 z#HR(&+!X2|Ofl?uO&I>(f9H2E7OdUO!4xc=MvyEZs%=XKDff}SI2SA@F-Sw? zvm_va?L)n8s*%ELm74FSBjgpK6?an;_^)Up$A-8=S9z0dUXYb#A3Do)i)m$hsb<0t zBZWs>-vZccp5FTio=OyI%7xfj{$fbzD;Q*oSrNB3h zhxkAKH^cgu&qd;V4~^^Z4*-lON(^HPE>m~ViRTo#W*oV^hbyycmX{?fZQk!XVa&^NDURbAi+_;CrTJ9CZ=W zx39Cb*dv9Our+oRk0{Sd(FyX>LE%#H4!U&;MOQ0CdiF#I+vt$7h>uGuE`NW!`wxdJ zGl}>LTEDWpDp4EenfIkJrQpc;dy?Nu)0BU*ClOe7FeT@BPc7nL%1?t1*D=eA z{Knx0JmXZ>x35S6TE4vfLUmvQJ!Mzes9tH#v^nuz`vvWE5*W<~AX~7CHq6t{<9UrvvT|d%y(l=CfvjI5GuXcnytskww5s?as^r46zuL5 z-O7{3 zhedQ-qC4_@@5eAzMo=@e0(Vu#yK3U1QfGam;P6JW>%#j7zz}3<1^Vs*T41CD=eqfW z^PYQowh~Sw5bc|Awb_?7g4)5$!z|y+s5k@8-QM^c0}GVMgI|;Hhh^O+(;??iRLuT4 zj+}@FOO%|@a9OGs{+p;xtfy4%xR7B&q_b_|#2{a}RP4~@%k`ZfG;0*|q3<(!wf&Tv z24AM;)Zoe_tbSy+GN#y!eRVpA8^j@|-Z9L2*`#maZ0i-j%{N5vJC3%7bk!JCHE+v3 z&Njb+_h!PB^j+`RwD-wCS9cPcLzQrNH7$8>E3(&<_Uc>ysRc}+a6PoZs2uN@;G5{p z*9U2xiAy`|2n{{hs+#Mey%EgHXEl_nu)NR`&WF;caYLlraQm z5KFux_4YXrh&VG!S}(vMs*ie>-w6$804I9Y&-h!=xV4jDN0I1BR$}^{uBiR1z57;+ ztxbCiZj>+>bUp>l<_nVyuAm`i&_G|Z;r*##bZMNvAN%U#^r+dUcYo6Y%Kiqbld8r6 zbP@Z4g7|c8Ddzy~Y{2@aAcp`}!RtDq;=+ODpNpLX@eIf;oBhaLE6~!-{jCJNM!XS* z+9&FFU-zFilFJ{!?{8`w7G~Rf`(aY>HYmHndFTE-7txkv`_XpSr>O4Uyg|pbepK~! zk~9}+qOz&9=S4FPnkgSQN57d+m+cy3E#XfIqk{sXatBJ-k5*Pn*jmU~HsJgK+1`8AUM2 zoB%BE=btL#|19qhkt^#iIYo*1gPLL8mFdTQ(zt%btEqz}lxp8^39Qo&Zi@j1Csf+? zaz()Gv?cPpYrtU_=9k$X?(~I zC2VlhHwq9Z1;yx};62`}t`0q9i&)xxT|h@<5t;LhYnGG3EFysh3i6zPBVe_Ftd~~I z>Bly`v=ksp_+U8x=Pa5RYuwg7;pMQ%Lg<6@{2i!1{wq*%0)`Vo9eY6%vgB6UM+tXxmk4%7MV9qmT3nNOPsRH&4TxCfn!5M}gUUDt6|BQXCV0kY?Ya z>4l1ikv@tX9yrxDmkmbOps5G!#J>sZ#lEHWi+k#IR|I5I2=rs`Lv2aeAyZFe_F^#V zM7h>Z02@P#p`_rrQ22^?@^{!o1b(2vxIwdluYt?=D^idB(VMH;D6PxW56@Zx#8O6< z*&{o%j?%#&5mhRa+y2aZtz|ZG3J|}X*HC{szUCQn=4UrsAL~#_|G-R+O>!-1@U9tw zR2=7Jm_DnT>bt@unmkq7EfzyRCm55cTeF>7c^tJEBxV4gbAZ3~0h@M7Va(9Ji>+-h zC_oR$N-r-e>r!#o2@}^2j_`4Z<_!TtJgv>(oGOp4TWQ;g1d|X;0Xx zkBBL;#q}f4$(^AspY>PiwnOudfN;tYIPT7WANj!Qqh#7Y#J^n*JO1?^lwFh7>HD8se<2N8EFBo6kH%YE-_3#6MOwEY!= z6X!C6da^k$dSI~GN|&xmJT7S9<*JJ0ae36IvV&kiT=$*RPGAI>u}*CmW$Hs@ct#@T zG={EH#pLzT*`hyRfr?37brUR@@Oae;%bE>6Dek5%3OII`vJAODWTrV>W8QBtBhzLP z%%^iO$e-nY$8912n((zUw6p80ESKCqqlfr3E|GEO7X_2w>eSPx+FOO! z5d8bE#cb|y0d2}^i79xyxx~UY z6QO_CS60Qol0fbOcO#zs&S6gcieyc^9!!nczIs{TIj1NdFW#%Fnxb4+7#MU4N(?Ln zOubkm_y+W7AMw1=PPEAl1{9~%2h#_%_1+inF|oo%1E?H>D|oXK)dbAZjL~NNt!ES& z;jMB^)CbZ?e@pNq`;j{;vj-m?0O{i+W&%EO_S6M%2uX6z9g}~Dn;=INC00F14Mnhz zkXl8x7+98%6}5ayeRJQ6|JA29c$Z&RdzD-%g04bQM#>wfmvOJgR1V6rB}iw<*k#7p zd8ni8^z87yno?Qci*}j|JNiSFBr1>5zuY0c;sQv!w$*m2ukJd2ih7AELpAu{GKUT! z!K$dT4IVR;dA~BcIVWO!1E4;55oAS+TCr>FNxi-^dDDT4t5J?hpq<%TFHswcEPG{G z0;T}EH*i)qf%j*HOY&x^*MD?R=-K=u+~uOkT@L3KqN3xvVHUskqAGhg_AKI>@aM?{ zUkT$w)!>EWMVeq0`-sFOJW5SseIpj`GOy>S_>=#TQjuV)Vr}~h&q}CcU`%r`xb$~U z;QBu~ffW-0en1JUAKhA^uw`!BU~CeSjyq_0P2(p}%@Fyxy`#1eAxv|%??-K9T?i@v zu0gg~i1T!0lGBUx_fmBFCAqZ33G>5CAX=CG%O`@}DvqfptmZd(VZG}oHDG2!(SsDb zn&`EP6i>mfjo#fjkJfST+)aq&lLyZ0!}p|T5d0&W7~`$BC?ENlFpjE$ksZsovN3|; zeb+dyU0-0U23mhN34(coP+5K_{nE4X4H~~hHyC6wp{A@}(x}pD%GxzIYVy*qk0SKz zMa#mJtRyaIBRZT0q_3C(MJ(E-@;6hCACAJHn_1%dxMaa z>@h?;D2L|!BHa@%FTqSG+m$}6-u#J$AD#m*Ttfd+O^HlpD50>uIAh6_ zveI~_JCL=F!8CvDEK;WY%2~FhhFX}F@ulxAscvwTsNnG78)k!i{VlW>XuI8 z(d~^y>FoQDN}Lm)#Jo4|GjSTO4q?!~QlBYJB@sS^Ku9)2>@GVkmRK&Rtl~oaPlpl> zEeu!*6lWqEwz_C-Tz(3hfE!;|bk}2>3lGGrl3o#EW6_Yux7RA?3FftO&_lkMc>ViD zN!9D0kSI<9^`&YR``htCl+84KPh{Zp#)m@5uCnCx+V`9FRXm^EH}nnK)kb&yPL+zg zeO$Sg^y&R1UA})(aPcb+|Ndu%1}PoyuE|n)E+c{S!^qg(B$Oj$MYb@b-%Ere>?#m^ z?8K=P3ee+h!5JJcRdX3hAz7=D-Vc7L0D@$p^Uf`m+3Gk6m`7}c_UFlT1dQ5Mrd=-s zL`TSzIJoS^7Yk#mhnS1vf}d;kgG}ed`GVg437p3a;9Hw#-0Qb>U(?HUhRwEa26ea4 zq{)+!%qgHc+y5k_O2>EMK*>3xOzh|YOSYOmh{h|e=e&U5-EwFk$)_{>tDxS9WiS!^ ziuF{mRX}ZU!BzJ^MG9hV`8a@Gcp=-Wh5*pX78AAAD-ma_)CWl{#m+{?)9x98pWe6H z*nc3b56YxH2F04*c#N5f2E_?Z_Yz9P2VnGDcqo@G?^u@81E3|JjAp%E(g2w<)u<7` z;%feqVs%1z7&f1=Gb=4BvgNV&U;1nD*62YW(~gg_iNbIn1o5z+0Z{&{00U6B?5w!P zo^tn;l7a3It3`Si;WQPVECfH-x0H==k0AGR_qJ8W(8<$$zd^SCAeD*y^5#`j0D#pK z+o_-{s^r-Y|&-_xPrQi#%L+eeFM# zdQR1hYh%kS(c1eJQr{_?joNaAZ9eXR{J2a^A1SQ{9({)%_3&eZ6BN}8jp->3M?&d| zK5ijT)lg+P&7H{50N6#{mfV9>dll5NZWHb^rQ#etMrEdy^X}#U%*x(TU;wu;P_GmN z6Jh)^NIl>1X;-l@_F;r{8Mr8f(2GK=$HKIi%D^gl?(zKC zoB?%blKG+4;)#o%$8S-q1f0Kmt^QM5wqPO^g(^KWQ8#(*s z&unw|7x4R!FmDF+5Iq9wo^{{_v=EkYky6pr_+30cH_T@~F5vvvJiQ-N?u5jwVkHdM8_TQASwqE8N(~jaUuTez00wL%M%CCqyBmZ2*s4e(4=_rJhpJHUeRX zBL&DHT-&ds)ezU_W`zI@Ke(zH+&GNSTnz?Ijz`Q20I#$v?9%#~wb>pk#NZBksk@@y zvGdPkFXbO>RCY-)AkXD5rnmMIlONH&Sa|Az_?1Ecrl2+(&H&hG%1oeUzfPbYe~~NQ zUsnEw|TL{MSKEB5xB(TGdIX^CL89zBb zOxm;)U`DX4_18pc2rQq#-XRq!vce*~7=-El`iG5aiPJxYtx(2h^)p3~t4tx|yjfwe z&nE0tbbi5lLhdcG;39@$;6vScAEP$h&&iJN^McsU^rAqY(;;E3*Q$z}d-$Q0p}#8T z92iIhak^qUX^0>8wj@%yE>pn_bH~p4A^rT;)3)Rm#T_DPoZ>9#M2gEj;YS+?-~-(T zCUW|ZRvC7i{eA~8 z-}u_NF>f*aiJxR6EBpNZO!WE0Du5l_&6j+a#Vh0LnM&(tfA8-Vg`~#&PqrGM%G}S> z_LQ2z7uWA$b2RCQpZB)vB2GvLi<}an+mDibiZ(Ysg0d}G_Mb>Ky=3(B73bpW@3_wA z7z}?9)1s?}+Rr{T%It?%x1&pz)J*AZ_z5R9L8BPRbTkw0rP{7B{`yfH$aA03hJrFBWS98;5tEy~Ct z&W-E$`LKIA9ORXOiOraS$1>SPw8*wb0L!9`3S`h$m0aDN$DuU}+C3>ihtI$s9|Cg+ z4gpv;NoFTF=u=p$_}9m0#wqpMi<1H)Zn6E#(0AZCF0bmse)uyorDA)^^G+GW#gGX0 zK;|MCo4%u7ejExmhmgZHP_=uf4(;G!(!gVKj@$%2NYC*tji0Zw<;o?^%F_Yq~I0f8D0!bXI0LsVIm_xA2)5-{8R;_J`sZ(SBJBE>A zle(vE%z`8O$dJIh)*{85pJ>A18yCg@?p{1Xi7fq~0U^nGS*>Gv3a zIEY75h7%^XmWiZ1Gvw31*QmQ`f%B*s`V88s(0)Dy$rz$edDO(v{2^5#2}E zdXs4VQlHgVkQdWcd@A7$s|FJ?DzR4k4IcjfgE%&Vi}hXXn6}@->FnV>CIT9eDA@bu z4d8}{K+UshZYwx|FwsXPelSDK@7&Rh_N^8K&cHe=f!YiiVlAAdkK7g>+)iAb#Jr~^ za?!zop43UnHxP%#r;S{a(VyH{HcdH}Yg=Jk3RK88PWl(p6hLF@-A@&se>JAu0T%q^ zwezP8Ae$@nMt)V9Dq0}**3I&3(~3W1)x@RiJ>&%St}yiL?e{RlICd>gun2a_W?kTN zKA+*mb}J*ddsgwP8qOv)9`grEzZv@TCaDyAneflwdomLqN!jY4-^jSDDlOix>xzwDfdo;wgjnlv4#Iey#S(ABF?P~!xzB10BA${b_L#}7@u zhc54>Z$(ERC=bk^b=xOo|ztr1N7Vy|pQV{LB@VKFE{OO2x!TatLz%$ssg1xJ{K+ zk(r?9-2A=#N4Jo*e2#&vUFrSkYL}fSm2InVN4%z^2mTe_Y#%Uj#}D z`q=t{@*s1nejld#_}}q7*0Q46-FtDMun}PJAX0U@u=FmX`qbis=<%>Y2j$5PW`f-v zKt**o!=a1vL~(XV5X;B|FsZ`{<5bNvaqzKez}Eqyk=rPQ)qZV^_<0sAZaatorqi|X zo9IV4tHVse4nb&_-~n%;f`?HyS-n{o6`|t?R6CE{&%@mZMIMbBR~{Q!w!_P{rx(*N zG-jn?5z<0WaTK=ij7?GW&`JP=3n&A7@snG11Yi>oLPsvRsFY@Cz*%U-jU#He5R|oQ z3!>4OUS5Ar9cAccdVMhuab9wGi|M<#+Y>ZUaSvT9|9Cn@1>5CRrEC_?M8__yEUAt* zZC}bP#tbCPg-F2?xW4z5!R2cvgH7)jzMGa?{J;Tt)1>cROc39S^{v|F0!|VouLh#Ft=~yCx_$ei#Q5!ml=RS~bg$ z)tIzM)5~K}li^KPpL%hc8EnyhM`aiy?35`)AXBBimLG~`DDi1INyvHvsoKa3>E)Z4 z$T2=PH*UB;-4Mbd!w%?h(X7ajbi@br--_I6VXV2`1MRz1Cpa4vmpUD>bEH=Fuu1r< zY!)IIB7hAJP1(-S2`clJNo0z4%sSpbCNu-mnxJ#v>;kA{lha;C&3QMb)&olP$ifKm zk0aIUGDdA3gN(2iK+ChLUNyK=NK5ugq{!^0+rw6JIN-{SC_gjwN=rop{UVnQ{~pG7 zPe1U{m*j(SYBh;98EeND5-+M(#o!SLVJzCf4+$U;@!VJyHcFVqA#`l76?kqlP91PF zb@tV-e&(G8vi2S7+>Z0Oun>L(M+=Wc8u+)lW>v&yJ9T%6OfJnQ;Q{IA*S-XiIC#_P z$E+u=+vtzCil9c}AWX{iLQp9!ku>DtkqQoobcO|+?v0+Sxxf8*^U=n1gQ2X^juZKK z{wt$>Bq*XQZl23L5id%>i00?FPPOa>yB?RMD=kE)5+3Xnf?Ze?3ei(!h5FZ@)+w(O zBF~>U*NzRJ_Yt^Q!vYWi)hxpiz+9ZWlO2JK_y#TpOQiUC6%KecS>!Rc1Oshp-sG@Q zVV{E!-&TSaQ@`<1dN6e{Fku^N+9cST!EB3~Y?C%9-(xQ}fNh z4{L&T>1nJpW1_{hJiKK?K6!(8;FSkl_0&{aeoO>m`1+-5BRYQ#uI%GP&n0&w4lld| ztHf#m>{vXZI280r2i4Q6xS__d=UczpOf z>(W8c5S8Lw9PH#XEIL;`*)IY?o%nM%x2;#X z`n37F`&ymBsCMcuF~~&O=2ZOPWkWQ)(J}Y^z7+Ctt_RkZGSXMZmE)UoC4F}ThqhcL31)nW1}r(7PzbQv{o*5T;zI0GvIsxIx{S7u zL#69Q!9f?_$q5{=bA!7^t(pb5-ZqI`_Bl-1idpr??(_j~NQyc<@y&L5XaJ$&%D+MW zV=lFc`UZJDyO!&7YSXp*zcB*{(AZc0S^NauEgr3^h&#)-4^x1x@{#ONaak#xM5~74 zdW9TXGM-fUc#||eCUKoma&CCyLXAM*ZDoT?H2mj(_U;B@mVcyQDq5GnnI}R>I$i3a zP;IhfAdV2X*{3e%oXUw`c~x{qKgl>PXup{|J;B@XW(#~7EUSClaMV0CV&Iyu<|~%K+0{#sqD=s zT2U=cpBC+5ik(-YuODiakHJCJ<2wV~SYxKsH^km^399L{B5DNOJsXe%0+`et1Y-9Mh6VUMB!GVUiQx8$Etv;X2kRM*ROE+?+}(|T+zpk=5MX*&r`QR zziaJk@MENjJ~qDcvdH}}X9YZL@O!_qe{$s^YA)ALDS&kSaXv$Inz~WxTu;I@s@xr z|6rMUKFu6;DCu&4TUCxx?qe5b#?c<%vb&5^8=$l`{b*v<3*K43d%2aOUUo1qUD4!2 zDwENiF8Lsp()PwMvt}fKCf#5hS(_gV-8`r7Xm6MKk3ic8E>{nTCOY$-ywz#OC4A?0 z8tACKo%WYDk40&+hynLVcd&5$nS4to<~hbbT&W1sX)-kDk7(P_ne<|33g%t>Vd~A( zR-ks$chis@YFAM) zT>SYAkvVuJwLGhD@bt|j`4s5dR?8a*jsi^RIWIAecpmg9@B((E*|2zg4#wLJo}c}? zaE(!!I>!ADx~1|v^Zcd{r?B?{B2#ktwn6cjAXF73`P-e-hM9>7LDEs z7#t-b9ekw`=mRA-z?oX=b1~m=BC$DKJb81j$jE5sR0k0o-|V$KX#*4f-HvMwRBiCj5Z zbC!p8Txad+o97a-=Q$n$E}vP3NyDUbqeorQsGMKN_d|YCAvP|PFC7}UB7S!01X<8t z`F|4}Ah$t9fl1LxWbP3;!O@H)KqzCMrFNwhEa-i1zuSKZh}m;fr4zwM$9+)|+f;5& zF#lb)$!>++6EPONXU~4p&Lx(az;WT(g?x(zibvTm>=>p5Gd^so31C&KeYSXt5uv^F zU+?K;7AfR2pL<<(MB$FQ@m%t!%v6$R2L2a=6;NdwzdN`7kg(vIVtn-%4RztnIg0MT ztaY{)@nKRVC-9LQpRC1)#`^Y&OmoW0XWL5bfUM3Cu_87N5g(2j-D3Zs=9wfhA0)qQ zv*odmm@>P@KE=i4R0N3ewiRd+@fj?5AtylFlKoP=6EbTv`PpTD_QR4`^r5XMr{*-O z@!8169Zl_hA8*drPmB8674Bk1&hva-igM19M^r>T0O zl&iR&sbCO0TtLtK|1&s7TxQfpEht1n)CwtG_+gITVgMICoEZJFi?3NxSrUZWc&CwS zQX&}AU{G0j!yodPLc=s&Z@*3o6#mr9Wxjv)nRV?@S-T(~+uGGW;%gkll_+=XYTqGd zoiuH-5{;Go3yeOB8v>p@F+IYb+pV9i04w?*CfbQsJvR}TGSm(D^*BADtT35KaShme zz0`JQo|KGUCA0umEdS+K9`9Nb?_%Q37R}~G&V0xy>HlTc7=WT zkn#48;^Ge_dDES{rblqK%^zk^E-g94iZ6Q|wO7H-1?TB`uczraCxdxjN?0dC{bKs- zf(Xsd%KR49zY3a9-5dz;o+wf03~NkMie=_DJoBcv#5F$+(6nl@F-OVqjPUZTw+Z(a z1|maLmZ1~xaelddLIZ=r_HqR_Xr6IiN4pq*RuQ^4ozQ&XGF=^{J2>G1=@b^Fw>a9j zw(%`Yhq|iu?9`c^<({5%V?|DRPu9-6fr!+2;^4SmfuP%Y4)Qi`!7A$+syx6)4suKDH zSwWe8dvQ7DF}r<`^j@HGO|aky=Hwbaf6CzR9m9KKw{{T|`a7iQBF^id82xXUI&u0d zi_=k9uQL@4K~D*z(7q$qU*_n%@y##5fdZ7JzTj>G3}*#i+JzL|`1|dLFqBW<9;Ib} zeK@S?riV&d1FlLQP0miWD)cISq1`gl#^9xRa`pf8cJ1*{u3g(%IfNonhDt~dQ6k5o zB9T)L2{Ss%c8-P_GaiRZsEBgNY1$PDgQgJ1u;m;xl7^fTnvt^^hGE9{RQr8*@4MUk z{r>pg?;n2rW}f@L*IM^l>sjl%uiJd!C7%;v&XIVN;B^5zD*twwB>A=Bc3vzxiot`) zmx_XD%4e#a8ajFj;n)sJ{|3{&J&3| z3A57Ydc$v4k5y|Mp&`1OuV<@v*vhfT&W}oemJke4Ed?>?ER!?e8@6^c;Jcm4D%y3P>t9vL|(PWh2|lV7wB zA#)ANM#cAZbk8V@)4*0CCzvPYZtM=XI)WGW4&!(lhGRF5l#7Dw>(YI{yVf}sb9w7k zkk`A}(gU_U3*!Gq+sgxT^~5ASt3$0fIBy$XYD8ArflLk+ew%!(JmFLRmM;|g<#jaP zOm4iof$?5f<~wra%&H3c$QTlTu`*@R*oAmVBdySq;dj`xR4b}h#}&NGXA;CV-82*% z-dV zcm_v5dK=pszCD@hXEhR2*ZUMoIaZ}=$1ER-G#y{PH9^hBIj)vFDn3I&<@vg2$d!lp6WB zhGdj(^l$x(zVL%MrtKTRA_TZ+UF$9>n&TI0%-$oDIfGZ9yt+3g zm@}8KI7g`+{#4V3DOc-^*<|~=>Cy>K>~V%B`$L{#391Eq<~?R$ z@yX;YtS$C8QxAa$XB9(43WXVD5XQEJ5Nf6JJ_TSNo@rWpk~v!~-LHEi`}Xwi5IGS1 z7O?pry~_@(_~N_NOtl8T<8<+nzR5SwBbe8Y)u{CFPhhKC%nA2(^aWEsXs>$5jG5jzQXf{hNo0BEWw2f=Y7%K=m~~L2v!M}n(=qWr&6|%km_o>uzIL{x zXQ^`Fey>J>8{8~(X8OYl?sJVVb9tT;3k{Gv!N>g_pH!_v%9N(|j0Y@=jHb22@w)kN zSCR1cFAgEVy+I8VuM!t{QQ5Y^qS=xMYR0)WZT>E~u5T9@-_oF=t+U7|vgnk6uOGEV{&Oe9^Up8Dw7mWoSQf znq=X%Zv!t6Mp%jayG_DB?rfoteX7Y*vsEW|_=Y6+M?~SQK5y$n8xvj%G4wwfjtw2h zCd|9u$xS;kth5IO;(IAz_w*vjqi43ZU)w?BNU8}7s`gIL@Ij)GhDx=N4Iwqu zkHd1HQw>odb8{c!`NtK4;D1uyLDK|jy&(Q!sn$NrLk~G9aT+{rVYWUZC94yz-=Vew8Mfq1#l%97L-VEML9{x-}!85pzAhvsVoMU3U zd19+sq7<#gI3Qfi0xT8Pg{miVYm>RpPKtl8Z*F(;G~i?9fuLCNVkJ~z+UJNTU<*=K zQO%dcYeex!0+6YC#M5id@h^>3x2o4@Ygy|J4 zc6%FMzI8HV$SCU~p&`Q}uYZp|8$`BUy|l23Cof(VjT%~l-4+MgkE3SGLqw9IYWY(l zzZFsBfKoyqwWE#Rj1T?FpFVYo6J=?udeR2inw)xp4zlio$N)wl)gCt7jvqx0OwK+Dy9AQk`;h2k%dthl zdbcGff&}@jy=|_nH-0*A47`)iS^hOf>>Dl>#0P3he5Oy$&sR_NEKsJV`zj1+vuSvW z_04WfrfJ6X!A+8Qkg|S!fwHEPs~*wG8a(VHJQv**!DhP@+Vta3p>{!Rv2(uCUgemKjNvfUxtjn6Uy1k5&e zoxJBB0rJa}V7f7_%aON&H-Au!@)#{_Dy?R02;ro6w+c^}#>gB@&9>p)JeWkiH_kpc zpRj1Ga?_;@JeHOZb=k9wW+;Y-tAad=$6+aqL;NhTowYds>8fSqFs}Bp$a6h*+vld# z+r}W=SL|?+m`cmXUD?_nUV>YWq28M)v5j~C+@PEs?*;I@f#UB@9C&;d`)Vq<#E`;= za_?ILN&5Nc+q7d7hs%oGly*qTptR8r8syddRO5H8N(UJwNR4JZ_)9^$?JJZSQV>N_Z2HNC#PCV0P|3)r>+8C%HU+;!*ZN#vrBCmxR zC&A*x65d$v+=8tX?p~Mml~XOW&Jo{i54GoUfVvQ)q0Ct$6GZ+XXHJ8a++Pf>&fK5PwcYjPJEFi7caF7wE+ zeA!gU-Rm}!H$T6qxn*#ovVv&48$9Oyv07dJpVsg-$xZ30mjQfh(5v97g6V~|x?N<; z%g@ZxqQ~-7{R-r(GS1?J8Tbo4*^l?EXP}FRM#>OIirU~cb*kjF2Ts_ZhpkP7o!$#Dp#_iVlIODc~upE!vY;jlsKd-04k4h~Sd7((Q!jsrWMIb_BPo0ND8uLRc zp|$7vl;la!&+l@6lJ{UI8oc5{L)Ws_86@#L6y%)__F>DR`C8BRk@p%5-}fV-T1!d- zdfj@q&?dtV@#}jeKEN1H!%wq@Y@frJifL4^rhLx5!x*frmwkmKNtha@ZVK+@(>J&n zlQ4VgI@dhAu&>%3%QR?4x`x^#O-m8>+MVarykQ_B2TK9ZWQA**e~V3_=zsOqAobX- zlDc(nHs{=dpW&}keP3W^3R}Xcw$<7`_xDN_4oMzU^?F@dsoCbKDHH;0$~N-|Xc*cI zYD#%ommhrRuKuT|l?RSuNdjpv`B<%UU?bA8w&o5LKrO^5@qlXkwPz2RYGE5(#y+=F zRt^OU?|XshyS813jf|9$QTMGU8~t-NLP;^9eCd zCG<+w?-RI*B)kJLEyqFlzvbp`pIfV53&zgy;_?1t18-qC^3#Kx@N8=02{_ZI1e(97_I`U@I3Q3p71R05t~6ef1>4EoJXHw# zSuO#6u`7Sl;#Zio^#sTxgj9>Tlud@oK^oXliArV#}9+A9T+@@RXu8Af^vGbRWpx=VG+#B$n4rW zW0eB{FnvM7vTVKu$pTU9C_oz(XM5D-%6MU$8^jyAFheU;`Y(N4>`BMSh-Bk~NzB7M z93(f&@ME$dZT?Irl!>I#leh&oZ=s1-O6;%W;uZPHj}Bf&xD`Jw)<>AUdo5FEWyeD= zh_8j7fhIIJ3)mq~G0n;tRDcpu44k%MV!Yo=dL(_cCqOu#MCUT%8GqWWJX_c>Z-3g5 zxC=`3mc-iLpf&1S2Zov5{(bCD3QUrRKh>+WIGecY_ocDOKx<0t`O6T?p#YVSlm5XY_>GLS7?ym4G| zJi0vpqLDuoq);G(rDU7Iji4km#)|3Xc@ai>XEvy-$ppB0J%aE4_(y~C$0TCJd?^}% zbk+j-kf8uF;($~ZRcdypnBMv1a9Cgt1>c<;K%#SemVv1;>iyrwdWB3(w%0^N8k<-X zRShKr>0`5Pv#mS~U&2;FTHnhYNVz!HrPnf4a$ELo4sm3jGukF@WH!a9@~14$AC1fr z>>CATwZ6p5P^-}tpf$=#G;yPrauRBX+|jvT0wCCCf1WZ72K+6vG|fjdKE(i+l$<>s z1souUmu@&A_M`p%v05y^jq30lu;6AD?PWYuc%ctX^sA&}%eO`6ann+W7H3DVW_5WO zkFq}X(?};$y8GI%O-Mkc^T~yIU*L~ZzTkNoXg3zqg?I<|@+B8x*U`iSZ!N?; zvz$yK?FTp}Vr1qCZdUW;u=&o=wRHp!V=A;6u|Cd)Dba??&xVML9h0Q&ie7j&-G5#* zehY0rV7}t?CUP^-&J&VbVH_ai>8SvXt-B2_6!_V~5slCWzX4#%e>+o9iW}v4ZFpRq zG2TZBN?#tEJ)4|$(!aPiR2vjk`u$2IvBR94$=DUd9RA3{XbjNoop@)dqr^j`*{=~(@K0IHcrMJ_=5G5IIjL&? zd-?C^`qmmZY;BD<4a3lx%S;z4+`hO~C(lL>3Smeq6o7 z+9p>Xn1$P3qk`YMollZ?Jeey@t|nUSGxGE-$=S_9oa z62>yh-3|Dv*TCNOC(KTS$Hh;-NSM`8ikIf6Hll!)k~ndr=+9wLg_0OTLP2gExz_nc5`Y9ps^ybuti1XSNs?bE$*y-_ja#@Y~QVkdTJS$>h zHdkbYuAu*0w|+emW}&6CwoHl>C!E$Y_26Wvsjv&}8xM^kTh_3WSj}glX~fPfheR$T zrL}H6i2OBvQ`0%Yg$cSm)M8U_=bHv8Ie<|geW`)-@qLw>n`n&d1`S>e4qtOvDLv-Y z=+B>G!S`+>8OZ5lh{&x_m_O-}F35s|i4B`GBq zp92}IqzJM@y~jE~+bXC$oudOQ z>CPsHBePL6NT6Q0ZTrF9h->+sCd_ygj$opSy-D^SEk# zL7I24$ma8V(<_G!jZlYcP|F&k_0kx?>9T847`yk2;SlLj|AF$%>z@l_o2>$_!tBdV zqO*t*5e(Hq2!}DZGN;lCbeab&DAXyH;AeD{Tl?mwXkaHP_CA$B0ydog6m^S$e_>lT zW8ak@)^fg!)KgH!i8qjaUDbFHaCbAjECJVCdZKDr^1v8w7&WxWw)$$@irGtbd{ri9 zz_mYV`jC(X|RgZ`^aGqcTL(Awr}FZNfK^=&M&U*wSDQIpGij*@uE zaa=~P9teSay>jURmg$BDp8f`o0DKX1+wV=`a_xJ_4%)~OnfA)etL&4%1ygfZi{^*7 zi|kL@K;9X+`-k-GkMI~7HK^?q{3Mc&1#vnY97_!Q1epY15__sYi1tOe&b8Ylb^{kC}mKdk2WEEJ4>CV5@$rl`YExIUMkmP&xv z^O}*ds~wh89KK15eFC4I#5^1;UCEiw6{y)7*{WwWulazk_tPH3!S^Bi_xHYqq(f)x zcd7g-vHd<)lt-pHse$x}T{tpd4~)K;(M+$(7BTb6p7&T>EH|6ixjwFLd)Y;BvYl^g zG}n#8+T`|ZNvXS9_`OSzo{1j(kfAPS2Hy+-`|F^)eh=e6-9EH)vV>iHemmC&e2Zo8 zRcwQloQ-EUi+T);={#$P5=?XgL@xo0A*=w0u$coSF9>M5N`I|uZhG>Jh_&`<$(!P`3-#R#LxQ|7f-9^RdV|5Vp@y8oR2- za0RneUN7w5J=b5BV3RZy7>}~w(D!9!)K8!_foD7-c4|_rjUx3_^YBCB`&i8fnT0_A zY*3(vA=8zLzr!scV8!^zZq9tcxw|f(1U>g>+xRADL|>hL4CBjOa8OwFih2uWOH#lg zI~d%L#Y!b|Sd-P8&KG9*d{}KQ2p&N)nGRoVyZnV94+EkW{g(IN@@nAszk3M^m_Ex- zi|?0iw6F`q6Mm6o*i;ofJ9K0bcUy;vAfE{I?_a{r7JS?EmVdwL1Jhy{;e_nRL+;Sp z{nq~KdNY=GoY7yOjd+>X)efvqe{eGRJfkj`Wk9c41;U?!a}qF`jp8B*CSbp&#Xqy9 znUJKR%c_KqkN5_zSBNC{NBWxvGU3AdEgzA!n6jL>Rb4eDF1^a^i7M( zGeuoZuT_ttE^w~^4;n^O&6rEdExH~FB?O-%`@-FRW#)BO*V?Ln&BC4-iT|xnnBkki zH!x1}=S*K~@rZ5myFTMVB#OeB@nTzxe1GBH!j7inW)r?i4f!*veUHE2gK&6%HD!Y# z1!6tpfF1A?O)Xj^k<5nkJvUx)?k&Fi zYM-9uv!JNBsy1r<#kMulQov3{i2m-Vf120*FvB0bc~Z#GzX=d@Vn2Ur1_RHZKKx(! z?B9(4ZDRlJNdW(v4*ze(_J1Y`;NSG|fA--2^xRviuMu2a+(3}!QB%i%n!tZs+<%fb z@Xw+Du*3h)N&Ia5?}o?Fgnsh?KR1a#W;+)V8?Nwo`NoeP-CuvUew`aWOWF1S$GNeMX z%vdMuB+HC_nPJS>es4XW&+~k~zwHnIdA;s?&V9~xu5+F1y%)*3*~L{R9EPZW(1n#n8>PebUdx3pRr49UgO2!g z^ip5}e9pSwg6?5}UlQSgHdVN3_sqE{6YT4@BDtY=yp!0ql`G%r+teN zw>loX_J$x)9JKpMg`4(q{fDXM;`E@NLBXRFBu9)enqN6vLK+uuKi@>X_&I}=jy9(p{|-jr#&SCXc)AJ>BQ2K8#|@6p z|KwlbtNKQ!8o5PY#c(VRG3+*rzN`z_R&!izYvZBGmWHc4@r}yd|6-PILV5Q-sM5r8kjt>G8lDRN-dSEi-jDM?aK? zAbMk)Y%V_3>#d{uBhkTKR5hX_$T3O@%Sc&BH;{Lx_MK~%QQ@we@!^4{E94jq%D06( zrXyXm9w`N6L6TgHTG;9gQdUty$5=A;-QIISUCXRFcm?JzI*S}KMGARj5~0coe8f7)1B&hclq%(g(+OLu4j|)4!k$`Si}y; zr-Sq7Jd)pM3wNE?*cmiFr5t9;HX}rVjxOxD8xVykG;ACga(}r$#q%1t5_5r|m^OH4 zEkyFxi-PN~QP=H?15bd?BNm5pA1;ngmJ=u3MWTZ@#-Z0H-oN?@Ck`!kFLKguW!|@I z(UIVyOtkm%F#P-xApGbk&slQY`u1FQIX|t<3Y!Ywq(9EM) z^uBjF`WeU*I0kCw$vbKn6$u(sEP6LpPCoLL#ASz3#|}_7Cs{JzOQ|mM{&gs*`AWyv zAEV-s?_ONczaZtyJ(1FKIt6Q5^Q)&HAuZgl)qS?Cj-=PKb)G&frz9hb&{lnY3hyvL zX0qs|S^xe@(=EU5vY*=e?H6_PZfQmS+VpgejvrzvDp@Qq71S&7b`{=sOD2X{2Y1;k z5j@k_cgpp(milVn5tpl~xAqL~Kg{B5g|AmR-aSIwlsS}sI#(oXHlj#?0>%Hz)lxft z!emYJBw@9KD?;la>#YMQ##0f5%<&JtN1%)AW$MJwDvAWt^r&QP3@DeICbsLwdhk;R z)=ssFWvmJuQo*ZTa|k{-{Ao4l*MOmUr<$p;t3vPaIFQIRi4Lwe_atvAY!6NeXi1|e zHtpS0uodkhbr8j)gqwCHi36W+_akY(rMr7Y_w@;KQ5ID^)$_HpyA@*aCdTqFZrUk5 zWTR7gH7@Q7-3g0V2`$30jxXg3eqm$Re?dAAvsgwc6Be`hY*$z^3sY>jf5>P@JCw8F zfHY*4<3Q;R;<;J$=F`d2t2YPK@mw^R2}}5gL2#BFGQ!Rbqmj2Y>|%Id0C^SDr}q>@ zIj?qIZsL?AArtWd)%Qdap~+9lFj8IQn@?w{qKHt6a69j8-Wj~fw=XhL{!k!-d#O&C zWC=fOhH-Lb#W|;C^D9PEN!5}Blt3mZTBgkA?5n|fUgENxR)*E>s{^S(MBW88*0v=k zsR>dzqAp>S96woKkGZ??M$asDvo`Olcd+=+&|EHBy8yd3yLl4ARH!oOD1|Eltuhd8 z0Lcp+?PI)t8K^?{WYL2(j=VerqExIxg1W4uxc!enZ`7?3cf3L%-%r^d<6z0W%tZ`) z4yNNfZ0f)v8py(5Zw!+4y4$cOa)fv}wmRo*syBu{(oSvpreXepXQa%`@l#(Nx!8TH zcXTOI(*w(ajAMsbtaNap%D;;tq`y9n180dDA{%8~9iG^4;+ zJ50@Cc5Q?5+qw1jr}|Q~>z@ryHeAWkYzTH&@VIaI{A!-^4W8NN$t&9{1|6yd*0dCR zjO9H}T8$fW&DJG9Ao`xGiWBh+ z)Nz~N8SJ*>Yu@;`WwL_gRbIj0LF3&g3F|WOhIzJ3P_hxJ>Y?u4q!h+2Z_moIz%Gd+ z5!EV!6!KX%X1fR%&1-4h+8^1Nr%ITGcWOhjQlkWapI{kd za5%69960p(!0`h>Jsf_F>TDB0fc4nmVk4j4azYao8f>_RugXghy?tH?b)1^#03D@# z#L}W*)q6P*Wa&sNuH+P9UCGMydIi-xMZ22i@w!HUrFVH3PyKRtKD{A*fjKpuAp=WLpd%#=Yde0y*F9H zSHu_6vr;&ryYO4&$EA9n5wepAz0~FVphClVHf{5{7d*80Mr3)T6OK5;HwRjtEJ-Xl z7qBrmcHiCqD{h@S{=n@9H+=jCG7e(3V#fWSlmI*UjGFJ3p)E0%d0e#C#E;B3e9TK) zg2FPF(w{2c6s|l+jSz)w98Z+0TN^dPSh{o5;;bU0l|^2{&A5*9>DQWTJYDC-u5cz#FS$;rQ8Ue#$hcUq{wetPAr_pqGlq{X*%^V! z>QE*;E#+%gi$>W8a}HErj%%+5Th0_&o9#F2*^YevTe z>t{kZJ4CM6KXk=ZszsHu-oTi&nHF9k$RS?S(4*eq<@#G9g5l#^=c z(=6durwI7|D;RsHXF#IY34|+NfIWFK?1cLE{%5b zJnwU%*Ehborx%LY9;5WYzav)N&I^6_WC^!DMNqZ2o9kfHh795DS*RWIAEp@@Ey5Fv zhAf$U+QM*O-TQ}cV{3X077%{Vc+#&3MsF(I4>%Y)UfL4#>b+jeA&uY#$4}o=XLkmJ z5wSVBoV=$YDmJLTx?jM{P5NXR2rPlvqc6c#?|hA4iqP{fb*#AP--T$y?wm%VgBJU$ zjD#N(<2B9kL> z$K%UQMS`+zR=oX%-I^*H?Xj;6@0d4vTj%g<%FX2h_1MIpGi@2+W~tod$kq@M@@#Eb z{^RpV@sHCaW!<>eD+1}X>hm=nSuz)wUvWdX1D-%EP&dAcA&#Fq1ZA(wv|AD!EF$Ot zTt_h9pJ9%89dG~b$CJA;bz<9vgWf9_r99kr%Lv8}as>Cb4mRd(IfA^&HH`NaXfha| zL}3+AuQZD=BNbDPuJtf%+dpnBRD}G@x_^{Qq|q>$-8`6b-rm~X(NW#2zvMB5PfLZ+ zrBpTPI>lbAh2n(jff-f#RP3@EN2pC4xPoMPppjry^1^nlL4^f(INHKHHHrmSepjaVIG>FD1yG!lyNGQ{qGlX6UL4|)I?Uas zWH~8R`0>Q!YoF!a9VOGZ!qmN%AD0t=UJ<8oFzPd02L=I*UkFwjM=3>>0wJ)TCm&KAA#e+_8 zvNOM2ZY96|`7VwljJ-iY97~gT?yRZdsmQoI>|Ggc3>{}gD;4t?{|=iHBA+71Zo2uA zqg!Ep(I-#e^AK(TGSRd}DLmfAd+ir?hhdcea#J0fxN_dpsQM#^3$?i3sDYKMLnD(l z95((#xjUwVW_LTkDfei}U71oj?D(njWuRDsaD461Kw0Lw zaC4ys#iJ;>^l4q830WokS)b+gPk4Ra!aNU@=>|6AhyUpmL}7+rwoPi^|kY8VG&0v%~#~3B~SO`(3eq2GWXVErywj^GJkNNciBr{Liyzi1+a~~V*sDrP{rGej9W$|O& zP$ydo>|0*+6%D~wxRstxoy9clUYu15_)PSQh#2&iB+Sz)s1$wP%J6+@?)cD?1P`A; zF@N->4`kt4xk<49c#I2CZZd)w*4+`1sUXwql&d_Et*e%w`)YC)9|f-2Za$Acgj`~# zav6yQoO!^q7>ONK{ZC@KZJ~b2KgRrh*ZzLBvo?-POp{kkF|FlLj*j zv^bN>i(J}0f%1PNj5vNZS{xA!x zlNLAFacQSjbAIUhoc^xSZtU;NZCQHiQImr1J4L+uEd_j4ZmmA@I$pNViCvB$#6K(C zMl-5OYo0g6IPJd9^mqzxQ`O#d4|&AKw6%;Vr2lk_hru-9OV()S#{^9>KV=DzQi~Ht zAaA^;L=n3=zhuN;D$2(d6qM?zS&90K7a8{Rzm%3GD4Wp?!`^>n)<|`!=e{+Ex!Qo+ zp9~(K$<>MhzyQ`~Z4+MFUibW?x=@wf`;sCBr%q7uS=9iQ;@H2~-QmgfBa5r935Y-# z+CJlLJB_PNYHR=}!7{cMwV*4?rX3;K)2v$m^&~-6+rM{fm)^8IJ@&zCXV^p6`);=e zTisS53fU%%z??wpJM~KROe4U2Y;ddWWoOMLthHm5!Tb4_ftA9e4%ucV_XZp=WEADH zk;ADBg`yjiiM3be5nF4cKO$L{w)~M6L`vzim~AfB;S5o8{(`H+yvVyX!b5GDXtVms zDCQ{T73gvs*M)njdV}-Ozx?I9eE{q#kiu=&_8BO&@l#R^?vjTj39B)@5knz>7de+gt07CL?Q@bYX5{zf?G_|W=boUr)-RA9X#^qN; zN%K}gXsmSCr$LW{mlDvgXRBNFgYN46PZalqxGqdFD;mLV9c^NmW4~`a*Jy-Ib)Wku zgr99N_SVs^=IrUBYK41k^zDF__bj*tMOwC$)ulD}QZAQj=U~}EpNCVwnP+b4Je!@X zKV@TE>q`2rKU0B6?$)-e=V>0l)Y-ajcM#@UrAbVk6?~@8_t8s7jpg6x{Bsc@%Hq-! z`atN?>dObqTR3|Bl-6@La0r4Im9TOQao{lJ-1X}i&})kAPPwkbZ97O$L6b#!G@Ga) z41JO?f8EwG?k*O-S2X02`O-0>sjLX#r?a){La)A%j#N4vU6s#7+ z2uLgHW@POhLCO!o+2ON#L3fNC=+A#uog^qDzIRt%#7u(3wkw=0krq!jvAaw@zXquh z2@-QQL9glGBDh*aD6z{NDlsGqV&u-2ch21sT+R0V>g+4)sW$~vulObhnRVgG%UE<8 z$|f~x;uAkkb2*Fl;(U9@D^AB_6EZ9A1(yd}uWh=WIW!wjtP{+#TJurpu6z%xwZf%o zF6WU4>}n1csc|8FcT!{jWd5H}2~q^D&6kDoyW6V3wIJIq5co)*lXPgrKiMQLO)yKL z%o^gnnmtXvKd4mcd0^W*oNC8Vyj&AJl}eT)UPK2krdM1r>lNW14ajBQk4!sF`~ z6?UpZmJ=r{lukw=vBHSQf3bd*Ca`)sX5KBf^Bpeo>QwFP`jV_^Zw76RS|F7ujT9x! z1dmh|xtGYMaTK^-PZ{yrDhcuJY-uW4EqXMS)^OJ(1GIDcuG0sNKs51ecNPPe#17y( z-w^BqCv-deDI|DamOxso4gqQ^v~fcnI|31BP`UTPR`N9}@XayA88^vgiPq_guT$|8 zK|++JFTSC_Dk9-x-)f!F9bb3n>ji$bmvUM^OOYw76|c`M46OZBrR+_OUN5}!sPKo9 zzN%e7f57oWWSJUGRVp=JIc@y(BMYDL{LbYxpun4d{F5va$lVH2U)swDq;32jP1wHL zFFKGL$@*W|KjU@)IK2fXbFwNI%bw;dHRy6R(B+1&Sq6Eesp38C#!=U2V0@`I@o4lw z3^gMFSEyjtOSDJ=f#x>k#gnAleaw#+*kyPif{$0;Nb_}1G-9A6B?-z;ll}|aHR3Eg ze-T}l^;W+Lx{m+d^1bm$yt>dlINb0Ozjr%S)N1Vcc~uR|U^($O5^f1@MNztdX52Cj6ZhA*lN>c$>4}yi zzZ<3AvFm|EE8tIc4%C)zkux03$TbmyGC(lCCjPkg}M+~i>v@Rf`SjGm}?&nP2b-up>C;TJvy z<9FC{%GI?5mBHOke1khA>f-f?1F!;&Ph-p6-bCWek&-}{A6gm3D!>(3|GvTv>FNqF zDP-jXodi;O1gmM2K#Yd|u+Ey%MW_JS2i!oC110rX5OIA;qUvp2AwGsPP)DeCU3fNI zxAZ-87?uk|>RuZ(%NF(5R*osX_Yi+dhR|1p@0+Ss5HX)F=~@*_o@XXDcp;r?R9FV8 zzDYECQH|eG9T_swPddcG7^mk_f5?=f@dI0AV_kXQ)?iKJ;`G>k9L>XJ&zW2sDD)pF z19|BkGPXyjLfYpYU#ChAlw75r)Li-^#0ZwHYdp_qvy z1T`WC%?ndyJLeH7bOaW`W^S>@zACvp`nPSTn$b9+m6To zgXVY7y{5+xtf!C%YCZ=mN5w?Dms|~c^&#s#@inEkrFJKm!>YDQq09Pm-EE;d*Y(<3 zT@qYC&9ZDXHt~dNVhrs=(`ku;z`n9z6_+dxPMUya82Sjy*m}#$pI$7P-tX*2LUb^APcoqdxs&Q7qv6#Qg%4cSL!)^@wdC* zqL&!At;IF&&ESSwpGZ_k!p8>}&h_m)geYA3WlXjduU@+)M+o{>i#8$(QfAk}>fYa1 zPiYujNYog)c`bob$D)U0j$T8%9X8>8aORdP5`vbDIwPSpd%{kSaS(t{4vMa*B%8Kx z;wMu(d4hai`gqn30|EdTA8GAI#yL*tnaAa#sEg5pi06+F_8I7=4pal{w|ND;=>>5Y zoub(8g_n|IIXk_AUq`m0aMIrPa1&xF&2DC6rto{~G7X(X0R6rB3|%3(;y2H9r57S$n2evB#VZ?zIm4QP=aDT)NETq*d*x zw0+}+gU3OnFXQgapHz69BSbD`-dJw6-+4l3YBaXjQ@d54`4e42c=q;aK3S_0m*Vvz zeov(wrATaT%KzPEJ^jK2G>m`9DzXyBAgGIPMG$1ib9KYPi}zT@bb~u3jzD4}3rlwN zZLVNIJU<+q)2RuT82M*);nBVjOh|gs_&%eSYCdXeai!1_Py6^xO9C1ZUy=m^C{v35 z1rG$*xLw=rPnrn@(MSoqbfDB$)9q)TmNw4iRwTW{RxPH)q2UYBeonC z^6o_VM7QdGZ%!uPqc>hMCZ1~HiP#{=lE^y}o3D@JAvDJ>le{6lHoGXIR$9W|EbLbtSI~!PDFLKF4k-$ zINV?S`{#!FOJ>jD3%Cz@b;`)bpF`7&t_?r=yvQX6r@)PVUPyEeeB?Bx3Z|A3wjA)q zLI^I79PbsTaHLqmM0dW9TY1zVNKbVC57!IFdB+5D!%IHt0y45N5^Rvg zPdT_X1TzOw3O5$-k1g%m>OThae*SzDc>=)ELka+nj<47aYP_OutQu@aRvugd+N@tg zQxjvb#tG)35(JCD|bmDN0rwh{8{ymj^Klt9sFi2u+ouGJ9=7H08 zIrRdR&=i$b9nkkjX!IO>E5(xAcA}rfz%0Z1yeLm&m_fA~|8Lr|Eog*P zk2r|pQqSU2zM-XvcOeP#e?NHq__@x5R%~_d_t$`N_2Sa*YSz-(A0tos^ry_Psb?Mz z14>Zhcow6Xgp&l9 zwW!2?A;kB`?{rCauYf-rbRdhJ{d4>U;f8x|~4w&}E~uWV?ar>V(hSZaiNjFGsN74qlm<%w^eWK+r@^7C>w;In2@ZPx1S2_on5~+bIHX zCqA`AgzSm7*5+K8vE_RyvuHa0_5o>k10htZ-BfgFv8rGVSG?ZS!B0s=YftK}eV49{ zrz1JUSArL{er>n%Ac5IE{o?UKp>PUx~i9oE)SUrOA zUZP}3_99#Fd{>PIE|aGX_to2j?V@P_{qv7get{Ej60Un#0X~3;k)H|>%zwALKCDQ6 z{?DSgOS>WycbtEl04{`3$cL+GY>J2n9QG=7qG)~=M@ciDaq;YcF?)zim-Nww{-Mc5 zgj3z=Sk{hI9_7)mS<2QKw5P*^`C_76cket_{5e$KLj<(SbwrA6?@(iJt%q)cW?rC) zcX=Npr=iDj_ry*;ZF-B67!K*rd`ZeI56aFr|u>L7F=5O_d5K@zH{-?#UPH8w+0jB6|5(?1NY(Z{vBWMu+7@R@&;u zbrkee1X|uwO|0=C950ufK$F83{!unYcneQ^C)zZ5s`0Kl_bEjAyYWDic@%L35@7oNp?+y1hpBE2#&7!tyvL^rK%$xFtXZ zZ{0d=!n@Dr_i1(f8eM`u=8>cW4;y$&^25!`O$Kf!6=kK&>akMbfRe#n;%*a#nkhY-4W0xO;J6ra_m4 zLqNaqHU5rxS9waW`jf7FyE%CX`w^n2y#0Xu`*MNL7~mPu=4FF$tYz$4!gAVv3R4QX zqzdP#zJV_M6lkN(TE>&(_p&n-=Vn9UEI0tAA1Fibz=^*vM_BwuN!~XA$Y%WXe)ovg z>ipBd_en=g`>b@EY5^@#sS>1&?N)>=;~@BO=Z~(;zEVbk z<5&hehb z<-sM$L)=CZW_>VCS6-{+Ek|MY3_Z8O|BOn`cBa;r%;Yw1$sPh3Vh{Ya;y<|qFs@y~X84tBpaNaQO0yvcZp3%5nEu1Q zsTCrR293$g%!c|sntY3^U*7e=em9=47;zLFdk*#sp24Xi7)sCLW*vwIHNhJ9_FEkB zFc7FeEw1d$b@Q!YrvY+ti@NQNbN)5`L&`#w{_#w#m0kNx@rwK12YUC#qPqeA$}71= zQWqQCnHjT&`j)r(<`?YzUtn26M+ybT&g3mDoEe@g5^uB5UUt+(!%qXd?DI8k@bB;Q&v}=R3=*{4QI>IH*F&Z`qmT&^oMcqGgtyT+hx`DUA`-uy33I4zc zD6<2H&&LKF00?S!J21G%XodYka>TCDr?K0Y+(EmlJ3r=Q@ynJ5RK#CRe9^NFzcapl zitd-IjUa=Xrl!aA!$&-TReg5%ytLmnRQ8kNr|?rs|N8rY$B2Kp%>FVR zc~poJH&SyBX(5mnt>eWZjvYM0wkrD$`u^qb^RRBC_jBZi*LfQhZ2_DJ5wrF|c_J{T9JRyL=HNXnhuGQNkDsx>CY;#SYh0 zu7O{!24>03DqMjf2H=hN`2QWQ00#lex0C~6d2qwELGd(<@%AmS-R0 zLOzdZAv#l+ZGt+UfO?y~2QFedxb+>S{K&%43&=}$NQ=d}oj`-vr2aMiEHFrk zkYg&H4PI8)_!4NbxH!#%OFl_(A0U4K23Nm77~T0NZtwn1o$V|G{}O**$aZ3uYWM*e z)4gwkmd6)~HGN)*P($bQFR4*AEht{j&nnK|Cs`JhGA`O4vpb|=8L_uDyb6p2FPpkQn!Z`; zv&os%+g|xdNGSslSttundSS7)DO({BuPSe!uQBPza}dLrcb2X|Dpxf zWJ~Zw2>&)-hn@mtCIS8h(#~FmL(8r;R1SMdZ<~AHZsB<2biePj%$2V zT4Jbcv0KTEr)}4-Ydk|tO~TVy!W99-(swrIT{t_O$t17ujwrEc0X22&TH^jpb_|~-%CR?3n=lS~9t#yX#=&WuQ=M?JemtzRyROEy+Yv$BgClvAl z?%RDm**`>bPYQO`r`Vk(b4o&$t6aX^a3tDSL8aL?V zq6uDwiTeTAPAi+P-4LPlz|b7W~7qTw1w-XT9cJMee~E@BYk3f67CE5!8j%aUv}e=1iS$r3JShQUuCqnMim zi0=GfL=W6Y^e_8mDNo&8AUAy0j5k>CBPL(qyl%VA7qP;cv%@HA+I`zwgpaWc9~qY@ z?3Iv1?w-SkH(wvs{8ywzMJ66aNS`7EUWFp-Ao+9*sD70`D4Q<=$ezA+|n%X7{{z-_vH!@plo~H@xQ?)Z3t9*qOnji>%$J+`m+bAqeu&+q;@s|=4f*!^@v+;lS~ z=$-Cjde&KeEH^+-lZRFHkW+-Ml0~(iVtGOV zJ6w{0hvPO5$-kIPQ10zxuT$vRTI^c+bYCpcwjL%xIEyTm#}AO@7Q=G6p~9yIC^Gd! zyQNUNVxXna$TjrVe7RDbi7Ww-7;V=^ixJ#><5MftAWZoNCJSYkbq?7b(8gA`!2}>1+x+wYJ_oAcry=93 zupC|e&0m*tkGyW$+vO9qi5!4*ZD5WXWuNN?A++ni>|lOU#I879hYDxTy!>l3 zlYbtCtQh_@a(@~v>qc(qVz?Q0cvt(<00(P9b|?;U=2&FEZ|1k#+iKo8)+cu@1SttL z-YF?tzRDe~26$^~Vx4zn+cPGw53<{c+LCdmkr+tl{JmloGBewDqAV%~rVCjW;brAY5nq~?80 z=?JW?X!OkJ2-zL1FX4NhnyKy9AknlsQ8zxkK8{tNB=mOVP02uQ6aP20~D{Xxy@O_mo1oy)L8qDc&9c?fj@-HL2WL zkPR!S>;RSaGs8)1xm+;7;`U{DCd2uC%FnPMUBHCWFSC6LkU*2UCv&@f{QyqDe^F-> zQ-M}2G|i@#6NXML2-+T^Ze0^(d(!5!?=E_x$>&Nhh6t4={;d9{rAkP9l#6J{FG&># zeFroFE_&J>zz8^Q7@9A!+P*IYxQp|s65d|JkQ2#rV<4l!f+}rQf@kkn&}c~eLGK~Q7jhYR-XB`wORt@2wO}>asmyeSz9M_*W zySa+aAD*;Pz^ob|A)B%d?|U=~>qSOsdCTT$ZLoEja&YV=d9I&Gl|h z@JhdMqs1W&p??#~?w!0W+|xUoN5Dn7x1cZ`^vWi>xW05_4l@9EKHE26vd+tlmu(lZ z^TMxLUC`04EQ*+B+=d@PNMJiv2||XMiv5#qm(@r^vtmcTg|0NPcI0oTtVL-a?drG} zcxpVwC>;zDC@h&?WDJ}QhZoBGjTyXf;Q=v(h}tFOS;S7-XW#DABHyWDD3iOz)-ed{ z=tl2}-~GHi6&T_5?MUwQSjkXlw>$}F4gm|s=#9%dZ^O#`7-x9~0As__GW(xyHoZT? zyFc33Cg;M$wc+Ld`iNP3SC7T8Qs0I15qqTFOnOS=E-WM>w*r|pcwS@d{vs5_lKFe( zc#q)t{?<{cVr(}J4+FR&<1xp-d7^))@HM5yvCboU=t^UH$vORVebfqk2g7n{HPMFFg3Cvf0*%o^bungJ-z(?l(D;8BTYLx z-el^uM&s~inCtpfe#)>^W;L;C)jYW+%8zkhR(ZyU3;MFp-gR1!Ny!0RTdWe>4}E+a z|Fs+_%l5o-s?*@9h7yHl*`Gn9zl=WDtv)KZby8?@m6{YV{(yTu`pQoGxFH=h|RAO!rF19u=x>h|G?+}v)FrZ1gU>QYOFCbSUNmL z!VFD6cWkC^;!5893u?d=Ogwr#So5M0ipvcax3Rd?2HP)+)~o+}7kEoD`BfnE3kdgq zrFMZPjN4zFQo;?^dgATBTpQCUEDTX5oOn$DMnK8Z78>+dKC>EY;dahtDTk?K z1?>dJ;-~wtxKe?8bBzO+G1kW$HiDX5DK|glG@mQ2AD`_y1Y9t9%C~% zXUDF;2YtAzs_`V;)DPl{-5Eg5D979$+|bkW_>Es5?jnQ8lOhS0*&xB$_4_bCMf^#E zKfYB@^zixT`ukl;FdYyH@!j)V1}i6XGJ2Ur>sJ=<0vQ}xo6?i?2m7OYd2MKu#!IJ6 zHtm+s@vTO~HtN2W%bsWNr+W7xFHWci%;e7k{0#hXXNpvBn-c%*0CyX$sxih0)<&q7 z>XJX7U97m?NhnU&@Fb%7r*$W5<>Hc>!jM9x)&Xx)biM}?-8BCIz0mXlmazcCeFxkS zY-iOQ?VCaLj#$eN%rbFo%NRVTT0gh5jVj$R_Ah!1&flEM_f>{oi`QLy5i(nVI0sZ1 z;GDbu>xs_)o4$VB{m{#oG%4Fz?X!b-GkvUA-`p5*Zf5?P9r(%4-gMi0gDd97l@FpO z9cWEUPHJ{B$f@yj+;w8+^>>i+tm>sd`9Aw6-yQzm@a+qPmDH;PQ+x*7Wj9OA_m_Ers8+U{;1;6%NP zIEKJHi1xkwo?P7DC0r(4^oGZ5OK7cEyEgSXGm(U^7T!h?EX3zM4G#awCGvr z42A5j*>j7H)HJU9_yxIn!CBWeOi+kKVT4o6K~X8a_*)+p`pimwt3d11x&i*a@qs#F_8lLqH7y;eCz64r_Wir$e*9DZ>j9xVcwM|^O=^}_*1Ya+TR1&g zB{t@aHGT1jUs#|ryY^*Zi!~OyOym)h(93eoFMNBBOt_z@3+4n45V z?HF+dC17U~hf6oB`&t`5+*P$alPVhQ)B^JEbA>*+ax}qL+u>Wpak8qkU(d}6_=4fz zf?V;;KN$qCUHnhUE7SmsSDjb*i3vCPSp38*EZp3^b2NmsvpYN+#Oq}au}4~56q>Rz z1sWpX)!@SW*xz|<$T&WO%@x)^@AJ9p3h;TJ;#l7G8tFD46UJ6a7I~HB#w%w$>~wB4 zH9%X{=Ie9g-ScaQU~yXl`fnd2%a8t>BopLXLNVIY1})E7qV3c(a?{2Ti^Li zji=C6W6YWFgu78P%b3JAulPFgylO>H`rOBA_)skp9D1y zibZK1#r>s3{SMhcWSQc?TjBWp2mZcAR6~c1<1?1$LVz85oBQ8y^MBl;A`5YOHehRc zZ@0r3yJs0P=JFzKU!`4D6EpA+nDXVf<$=C5v&>u-hfN&?1j%;GF-N#BFe6ffxmu+{Q4lV_cVM09`#&r zpBmlM(to)5$v-Y@YdfneY^=bzEtJ+{^2(L&vfMbtt+79Ww6=Y$9eLnGuZ!|VTCAqj z;JzwV?5so?&`KHY-X-KiY%F7XS%8NcgaC^@V}7{29^f3Zv%EIQh5h}g@OaZ)GDj%m z>(BUozYO7L;{G}>g8;Y>>tN`DwUN79C`%HWkxty6vJ}m#SHkN7r((m_y<v|G4 zM|{XTV#FZdHp3~T*VzD6ir-n8;{78l6FywtZ@}{V-f&+uxdyjW6C1GoRAT!x_Kam{ z7hf2lFJc*ht@h^R^Y@jr!4qHDn|4CeSifb4q}#U|0KQ<}Dsa{tFL({2wKy3O7D{=y z42z0KmsMNxQ>Ize;(%{9_H&~jiDcTH=L<}d?^w)ms-H-oU{>md)n57=q%!_(b2Bjg zAZCU?8nK<8pWi6Q<&RptB(eL#HrSW(z0J$^S;VfXz34K2hmJT$-`eK{gu5|K%=sq3 z*-Bz`hu~a7`5cENIB5aL_o*Tfhl6KO3Xk~^bDiMz#V;L+^}g>ZXTJB`5SVtCpntGD zzb`!ldke4Rf7BjwM^(YfE{?SKc&*C9)qilaVF#b}X|T2|Z*emkRaKy`Dn*-@A)Fc> zV_zUG#-k*v9D77P+$EjgWky?i!OuM zS8vY1j8TtUWDtP;qO*OQZ-f1p!XXp%E2A^EOLcLnV-MYCR>ia;qE}aUmbS(r-1e3L z=ZK|GS6F+z$F5glcVc?xN15_9$pP13MTF(Ld0%ew-_zJf$*U3i^GtG5BV^yKG`Rz(I*qg1c`of+k5LcX`CXB%DP4aWCNDJ@ zuLluTL|PaoDmKTw`~-+cui|Gt&|6=6Rjvy3z+~YG+z(%pn^szC$G4g}Bu<}RnO7v7 zQURRmfAa2f^@ssg%kNI1L;J237-Ns}`14Ji)P0Pa)!zZWJ^tJrW4%2g5&q*oq>(ug zDu7-6VZWE%cR7;dwhyR^?|c~?|VO<1<--*%p?4#C8ibqC<_j*^53^L zMYa;ifA~ifOwosqNf((x(Ao_yz?TfRNr>WYsgfL{eD&diu7%0=73u?+#lHafU)F1`}YsY|mzpB%C`xphV zpN)%U?HRty*rh+?8E<@QQnCzyxz0Zka0DXocgF`Ja1Xg_*0imi6As^;h{BP!cKD9# zksG&`LL!>2bO9)3Dg&>yk8=%OV_Rl=rN9Bly1Eh61Dgqzi3?0eO9_fB89K!*dHkY|RmII4?=!$_8g zGyTD&cXEolf{rB!0iZUdcwZu}zz6YunC z4%Yt^L^Q-Wfi?v`%`!OujTF>*D25E40O@}jb8?g+bRqOP?<>$#-VZP`{bwsF|Ng_j zRgYdj+B!#7laSI?1{>3K_tPI<|Ix{@p~AgAk@w~?=@K9UF!WJxucj~MDt<@~Y{`KxX9g$x%)vRZ%4x{ikm1pXPZImaxsYAh?Ek47G-@ z!}18XcCTrez`se*_{2SUSu`ke*nYe%6t`)mC63L7&yiyaEuPUt{8v^uoG`kAsZ<fT`+1xiR3W(o{R_gF}=%85!$=W=g21V-gSkp}LNS zAstCm(OxheqFHUNF94g2$?-QI^$sL*Z2FXb+dvP-&ye>`<_2#0RG*vO=Q@+F9iVM9~&D!=-+D>?1Bw z$H%=AO07=BDRM=B)5jxlo?xx`>@h~izLj`24eh2iE&JTdT^&BfeRdx~cfJHc*;8y^ zdZp{HC-$Kt)Zg2i-;9Bm%%K|pa{&Hx`u^cZe(nCY8`W#j!jMS#FHWeuf^~Hv;@dE4g~lnjo>#m5$$Dy#WDy}Y*3;9gG-RU3 zI~BQ?lOmr(0Uor=i&qdsO%>rxOmj)0e_CHP9fuLaDLyoM?Ek$7G_(bD&T7G~+C&D- z_rtw3{9{+838(d(w`PDBu=j}TQVXwi6MQKMmGQ9@-(($QI^RUojOT`WPB#8gBg%Zv{Iht(!!_nYXzYVUpfwlII>(vZioN2sADNNxv^> z*X$98f+{FLKG2;;fl>v$J4>yTunKGNO&dVzxyH}LIq2tbuilpISo>gu7(g;rcrA}# zQXg!RFFIGeiKB+?QyS|e)JVv>U17Dv{p9}z5Ds$xb!A8P&pLzuUi1GfH0)hoU^1Sd zrd3Pl_nM`G8i;s#M2pav;vKt>r?~gXe;XgBY`CVc^5Et~RH>s!A}Jscq$90%NDbW! zwfCJ7(u18cW*UdEb&!xrcscw;{dA>f%cL*W-$H(zTj}}h!zpT-4NXINU<{_uA>rS>KT9Z1oQ7~I8Vos zxG_wQ-}>%1S7BZ}&;Vc1k+SkOGheKF)96%Lg3VV19<{U6v*|n4GE>T+{iBzuwy=d~ zodt3yW!|d+b*g{W)~RF|2c}lGZLNCK#60Gr2GH^MR*&K-bLE&k`Ba{fxPC88@DFVN zUwTU;4UO%GDk}ts5A2|p18mAc;?U+$sAF3&ciIbFS zW1|AV18VRoztL*rugb_hoEG$qYjVVhhih~ytVP{tOwC%C4AzgJ*j7fpYU|Dz>)+7# z&wW8H4;sEC<5qt%vVK1 zEhN8B_iz$DmYh|2+239yzjnLkf32ML`ShCzgKQ#zHDo2@3%`hgoV1zbPcm zgzXX+n4bcET}7()BQX76VPQVv6ra__SnQzrWlvjq6?nWo`9iSRLSyc%Fo=bPezF9% z)6>5C@}_k>eZFrdXU46Fmtt`NC!hQe#aga>FBE>H2)=87kyJ&BvWFFaHQ6h#A%;aSXR9Xu1fK9EnVZwZ?10+q~rp+8%g9{(r~Tm}ou8ch1Ves`QJk zc*N@}xMG?iD`hM_KxESYvbxDP=4@LW2=N=C`|^|?+H#mlNc72gxceuV;FN0}RC7Lw zk~4Z0Ip{*_NW=X+Q!rymU!5W6Yd2 z?`nSt&02`s>F)V~1G)LV@N^jMA~9JjoX6BJS6PR6q_0Ii%P#sD^4>j^8*!K}s)e${b@<8) zWU4hIl^9cZ6jjo8@b>G^3hUqnGttrRf{plO)T=P4t+X~hm@9g;SDaBZukJy`>1&Dd z(7W65ROF@x{~=jAVmszS>R8G&XQ;5H>KSxt<|To74a(c&n#CN9@_X~X17ZJvcvz_8 z6fCrHB@o^2vEgI}9|HGSpKQ=nwAv zj^Gv$_`rU<_B zHqULEk4%2yA4wx#tFv4hqxdt{ZJ{fSC4s!DKm_+(f4@;Zh2WnDZgtOmliZZeqtaIe zPQdsT8;}_!mZihl9l|&Kn(b=15quS5CEX!p&N0b6u?XJ$Z|VEbt@s=3)Nz2O8q}#E zeLeGea=kr*?+rmsJTEk;*QYfWw-@HAQ#bdk1gS$hOo+>=SGU&QCJD+%dZS*9xZ6%g zVEh?7E-UBw$zV51H|w+GKwk#={BElzeuHOzVVt2ouPdj&(| z9sTsPl)6@MZqNG#%YQP`-{AEtG?)h^ghsZct)#7A>IJ_WtB$s*R&borNLvtJ5}MIy zVjbCG0;nW)3mqHx73S<7DRRP%r;(*f-T`B;m~i9rN4r(=Pdzq3CaEoZXBitp0aDDp zFKp9SYbM4*#@$+6s9J>COKH0hrFS?es0OnWCg z@2V$z>u4>>wE{}Pq@2X(Q96{DDQ0MxWru$lA-_(xZOkIw^2%2*nva%{-1?QW6}aw` z;P1Xpk~L|2y?t*_F}t45hkwU&8m(Zyn2z?j2S(;-f~ZvFZcYX??YQl_vw=y{3f(~uK`#Ga|BfM@g!~e>zDDj_C#p_jk7NbX(*u7 zKLJ#r8CaT1tEOH5cWr{779}l9sPrMxoQNYq*`;Z6ykk+mE-t$j^15ABh_#JRZ}N8) z+e_GS9R8_;!|)BWjH}F5^AX4m?Ay-xN zpmP6zTFCD%X`XS7y~fgu7Shr%YWciQWt(vpb6$|z-B)lt@HjOFs74bF$n~G=vVfIE zd5u{yZ|2}1hgho?tHRTf_8a@x_bAxCG{uR3$X3EqA4X+f5N4pC z9;3kXe4{Q51Q|dBQ-pfvrX=xA#;TsS;uFw2=)&j(5)qG&xz-D%5@w(1N!uCRbA*E| zp19i5Z%04-z}8K8igYJTK5!p(YrKeF0=zOURD0{YWvoh#D@UY2JDHd~lj>kmI8b*f2Lc8j$&d8v8Pt@wE&` zTAlUbtTFSdE-EN5Y8J+lOFNmphQtDcEmD=VQ^GjN=c7N3G&9DvYQn`usS&oHGl%BE zKY}JX?cgwQ%xQ%0k(F`OT;|nx4>v~`ZSQGLO<^{VyxHq;!4k^P&=1z& ziG&}(u~wq&tbO09DQQ`~TAqf2#iZV0LwZOjwDLzX9AcjI<5aJDrAC(+ar={f5|qhz zgMzwHNY=?ylmpi|NNYq}8BVm8*lZ}6A&y?i6w4t+wdsn*vH25F{d}T^K|;oRrH+$k z7AKf56#>F-jh1@pr0hcFo~d;lOAQ_m-c26hJ@f4!IEa>AE>6r-=GsZo|GF!-PJyAv zyS-6asDC_Cc{-7bS1a-}wyVo`#9=XQ4rNg%bEL2v_HEP_SXI{x(lK_ey|4E$-+q%^ zjx8~8+%72Hxia!;%ujf&8!kT#co_A2Yt3^VczA5aH<6+%U86ikY2)i5^a(a;CKAmm9;|2x*@*>&3uvnzF_|nF=!Qy0?BKaZYrJMtQ*KLuvwG1vpmDBaTc(t@lhy+pe{M{lS%aIb=~jfzhT zX9IKzjv~cTWfEVbzHT%=*9Zzq`7tij9P=E&2 zD;Vfj)yqp~9;-9e-D&zVlTfIdK!_~0dcoJZ(8A(FZ;`R34Bnhj4>rQne8D9RA{Nsg@hv8p@M5ado#(*s=-WBsu6nv?xXVf3tI3SP8n>&;4b za^JNIwx$Y?0=q>Jo=XU?#dgY(SWRS8-H)wbeP`P2aL_6~Niry6Q1#-Nz)a$+svE`f z{fgt)yd;smOB#FW`JB*V_?RMIPe^($!rY5@2s|qMjE7!;=4SQS42AqTMnMjcQob?L zjXjt0$)AsRo(oC8E-Z&lT-}6==|mq;m|<#vNKc@oscbJXF>D-jDGA%Cf|C^plxAl%O$%1?8$`mAU(6+l6C8 zH6)F7h%l;bYj>O)d9@Dj{*Ce`8yV%f6w1JIE@|0*^X@5-UG>i^(>z*wJMhzc&4SL6@!6LGs#$+gET_@MrnEl-ki^ zrRsW&ML9?zA$D5`mpP;#+-K`G1cuOBeEYF+wXQTq0v`ETDNpwAnJ*7~@&)I3$ z64l8+c(gvY^`H@m^*z+)ow0eVS(?jTgx4*`;f4ea{i~hnj zQ!iH~Cwh9*cv32Hm1l;C)d!+kMJOb#Tv($21s9$r<`AuRx{g)E6I``T`leCtCB-jP za(?0=+VUyNwItz`h9k-KAkgjs@-k*P6;%Qx#ali-iu~Wq!aCJy&V2Qu_~!kP7kM-H zVn>c#S2OsHK2JyE>tKk68NNchq#UV{rie)yhKSMXKC7za7MKB_iaz+2A~Bp0W-CIF^v z0|G!*j&pp}2$VBHnx>s@STtFS4{;~4lG!HhqXwI#U(F{c8?B z;k%Xe4oW`XbrRAcTWXd7gJW(nL|?497re0(d>&n)MSfI7bLrV+{8dn7Nk$69+mGEg z>u4q#%-Z4{WCn3E%+V+Wu?@45ftQG_7X?BPeMPOh#oWp$+n}j4%D5ICc3%%?eKp|O zHBu;;@6!_;YaJ_RA*x@#)-cWC`Q$p|RWliRhnXx@Z{Cr*eURwBxAFn#%MUYs1$vjo z#Rr+t|ErNdGmobvSHtHM2~X*SVwA>FTY5WdCidZ3B~f3h5?D7g>X-*bZLmsu$SgZd z?mNGK|KN#=k?YmsjP!nFC{{Y((C75b?`D=3t_O_x32q=)H)eUs<)u?Elz#8ag4vLf zTbm|`VUL7<`TUC2qOC>OD2KZ8)BZ3H1=X|Coe;;>NWSdUAs}J<>lypsYVpD!D7Qq} zB<7pf0Ms(x;NZVKUz=&|G%0NY;8|9y=#0C8{^Qz}B2_kMpQB}sx}Le<;LOYTSZ{r7 zcM}a?4i`Pay@!PqC9}d3HQ<%RuzPoQN0mp5e#((-eRfn>{oO5@MlF|l-jHpJ8RX>m zaM2(jW-9~LVd&fh>So#Dc?6m}#Cew5?&t__Ez*J0dshO!_iOfcaqg#IHm7z!Y+0T+ z&?L!`gjB0Tlk{MblsnN4ze3NZzrZ|e=%sETG;Cv-Eu(}!7NtYnQ?`{$Ek*3N?0h~+ z308UudHW(D3t{d#jIThVGS(|U7$O;GN2m*P<(+j=Er+m7 zV*9ops;5}=al_A0?ekS3aCvTPc#NZPaFy7r5p_RFa&v76qNVtE6EPf7o2UwZKPd!w z1=#H@k1q2Bgf>2alz z0R9pkNr6b`;2qDU_=D#u|4o*az7Glc)@#d{T6JJD5~WbTtbHlj`65a+2e;d>l=Fe# zpsMa{+L-$i+szlubNz&WDf1&DT;gWFT5qh`b;_h;%I8TGDnP5iD=X^4oWH+1l)kE^ zII4QG>VfV7;xowIi3X$!EbsT5(MamD_?jI`rM}T7+fs1yHi%;|Q@kzwyxz;+pp=kM z=0HJXL*9YRuwj8smA76U>E8orJq*OD?81DZBfWm+Y^~tHzK;95s!x7P&jbvs^iUP! z-M=&JO4YR#DRKd&LoREm7eUW3wZPVGpEEi}Ay0f`84AMlyM`t7gV}44tQSAz=kt!; zp;LbhX+<-w4eE{B#`}6`y;ht0VJhpmr*dn&lG_|L!c5Dvs;t7PSa9*;i>pvBc(jc& z6B62X>W`y&4KMD_5SE_0cp8eJ%*+uM7Cu0w?PAUOXpV!QRv*4H9v#{E!gdmzCUXi1 z=%L()bL2$MC}QAWnE=G%C}pYDyn5S={o9wBUog+yP=j`!Lh{^l9wHXL3FRX?YGgZ2 z`kkd&N)E;Rb0V`6AMbyEi2~le@X$bAMLSXRlT9i0$P7 zYR)SirCM0c5vp`8=11qpDB5a zCf!3el1|wC2K5@IeUT|oR^hI-t>Sp@;y9ufUqugen&MT930;w3^*(f0iaE z=aLla%nr?9+cXjV5wix?9TyLxHGT1eW$V)BS=sA;{XpjtR=c-FRXX(*dS8cN4dsbd zn|s?=0AZ&MNP2qBd+REbzsbL)f)6auDe-fTLxPNU{M0My7>argJ$i6+S0Xr{GL&zOrWeTN66@LhsgT!Q@O)LHlLFm zU`8l z!5Z?rXRKl>y`7q;u#ow#sgr~Xi!FL{$sOE)*UP5%2cOL(2l57}Wg`7aBQ-eNQQ}+8 zGO3qsPd%C^B-gEJHuPS!^7>ZuY}mr2PkI*uv-WC4NL2WP>zRkOOaq?9DDx^ika~9)kES- z;l*^Vl`YYix9+i44z5jL*V#ZivzG5WjWc)s6V1J1c{|Seb=(c9l7DjNEh3nWcQnDPy!6eOpo=D%R9NZ}m>{OD0o z$ibE0iBVb2adX%=4AFo?3@T-F?H+AEGof(GoJb4fU+9O2;4wBmz5jFv4cJq8reGMV z4m3#JH*ChmkalG3VoTaSis}VRb`?w9JXwA!W8xja^c(kBBs0wuL0D@?bR9BDTp8m(5KB)u_uZn*j09;8R)Z9l6vtn1XQ#Mkv15aOs zNQ*w7P7#uRKH0aA{|AI%^Zn?;&M7d!@$4uig+rwOmrXy~jIXQ=Rf8~C!L1-+|1yK4 z;59V8L3@iLY5jtB`ee$cb$cnb;h{)<_VbOwlZ&?c-?obf5xFhx6zDNSKvf{TYv^AE z{|3Qn^DZIwwF%p}=Z0Zd1KD7iPhzt=g$G+gLKT$0SKO_!Cc^}E_Em&4l_s^gsAx)9e;i-Z9%Dyv2A2HT!^(;MC$e^mi@s4CoyvMJ(e z;_0eo%6C&Vc~z@Bin$RsLv6T+%h;;W5G7Z#Mhv0-O7~;?iJOCK0?M;`-P)#F`Jh|O z=0@(O7hzpfXIj3R1N-ECnxKTpwT|raLP}|BUx43oe{rxN z##dgoy5f8mtRRM{lXr`Ur5G8}C#9*aVCOvt$sO%}Q@bV(qY6J4X4RmScO(VUHGdwu zgfERbZ+#b?5=azCcYiJU%?blMGU<5g zow%daugdLWI`5f&m0>JBq~DKuV@0uuqempzs6>C9jFUBZ#bVCq%o+`Oync*N2OcQG z+$0~u>6Nr9Y;dM7A60gTaE!1Db_%Qf3~#McO-RFxV)F&w+mM^*9FoSBbo>%b9qSVN zz}BXRj8~kq{%oRaH=uvk>?q3oyjp5(dzTqZf9Anvd}CRp^s);vz=>-^59LY+?Spy- zbN05c{e|dLk(YhsU^CQk0W~l|Y4-&_VFY|_to(4+qM4UnpSeF=g-1EbCioUxg0tco z%IcS3gS1i_!hyDR7Sa#7;O1L9jw!S?)6)^z-MrPY3`=pAYIK(&f^FtxZm}z{k3TQ1 zy(HyCCRF5}nRH4SHoV)vy7>wvv?xwFjfqw7)E90@DRF#D#kO=wU<309FH;hdXZID^ z&Rjt?^l}(Yr1!dH^+%=xCmvXXJ9S&-IOT^8SEp`-gLw&nEvnF;WU!gN7vf5#VHf3D zV{BNy25sl4K8}K#R(I9JVhxuAfiJXvClX6=jCn&PCo#E`N4~4$iJG6*%A|-k2DIxu z^uY4Nv+)yL{_!%CoOA^Ev-i{NXOWyi83J3;rdD;b)}InHLkJ1q^#9FE@lzBT@G199AC? zX1KRVx!FWK8=Yd|+tefPu&4ATn@dz-%ZR8L6xKI7qFkq_T`BilBh$a&n5l{P=A=37 zfl_x)zg+t)i_q-FnM@lvv3EJKDF;>^OL7cLmhEWURm$^LDUPt&`mscf_C}C)Uu;=k zgAW1c;N(pJ`2IqzfTZ_YO!=@wwVuSrulZggfL_1j<(2xGyy+zewPq=h^`66XAPRivCD;ntNhR4^q(zZiE_l5axa5;sL` zwQ^or^oyG8Z7uqQQRZial{aRf2acsXgFp($$W7obShsioGxWKQZvj`X+aAM53SRMV z964C@gdd8?&FKs`F7Vy0lA8@$#e(H?oO@a^*-Vf6`x&k>9xJyiMu=6F!P=*-?Xn8> zhm3W`ix1x3M`@g#-lX}VCAq~qVByP`AX#F1LcE7|Zaeu&!l89_jDTWlvxU%_&}LNM8h>rb(XESVBE8KoTrh?j+4+vK{lW1f$Oq!E`ewNLl2WWyi?+ z<_k{oH3jLThc`K~`YZmnU_t)zFO9zmpL846#b9Vvc#NbWZ?Vv7Vq{>+28No&P4qKW z+v;UXzf$oXCY3Dc87vdbJ}(~;cvyyMvniS{(o%2A^Yx8l>oka4b4NufqB=9>8JV9S z8CjC`f(k}JMrYEA@Ildewdarq9FZbw3p5X=1pTzH#b#2tL~Jsd$8Sqi$5g5Ou+X5iC;*WUOc#<*HV^ z;!>(r|J%+sEE-~$untZ~Y8DdiEry>pBR=o-UZHC6V{}yIRv<;)W`T7>myS3}+f)02 z0huhia&@zK>IZICy*K;3Q=plyUx-T~U7&@pS%3?VS1;h~pSJf;O(;KNmszE7T&=w5 zI}#1JRf}!@Y^b7Jk(E&Ho09yapRK@v4CcyEYl}bGT3o8sY9dg+fLthbtm?VrgLp6A zED8v5LW%MqXz#dwokRom3f8FYZdZx^8TA0MP!0;HFM6d`f;L+8^0<3J^=X$O5oo)h zY{GuU8YL%_{*7r}JHx}1;EaM9WKu>9){L_?Gih>Yf0EKC4WP-6*kdH7gW8R9|6_yv zD#*D>T%VGkH+?ip$9v>xDPL$uaO%j8@@=t5Ox+;>jfQv(o(tBk&bt{h;Jrz%Sv@HL zk5eF3gs^-Aqaj>l;=wYRT;m(7Ip+L&0i#X(21mVy1#kO+a~Qd5z~sSNHHJuo0`-HB z0p~H;5^>fJDHNAA_W2a_)YzWWD;wfFK>3!qAs3EO#jBA4_wA^poV+O#4(Tv`q+K{P zyt*cENfzgvVR5td4L##0iteK~oCF47hcN`38?86W&ga4l%VV4-hVBzeWovSFB^o!g zhvId;&2k&Ui+)oAkFim$m9DO!lt^I>mP@dbc${Zt`_ieKx6y%`TU-tD_ONnxBRL1# z)LOiA#zQYUhht;40D^Do-j#yVs6Mx$+%`_NycS}v^x)e|*mJKbAQwDxJX#04kK+Fs zdX*ompsW7M>XSb92}ay&y_L`n=qN6BN{JNa?Ndb33%lu2h1l~$OOC|MH=EoYxXC6n zud+?SCf<)6J6+CyH;x3qASoEaKV9+b9%F6c3zY7UK6AZnmnUMo&RNFENk>*lNSB(} zP49MJ%zQUEU?@NZXz7 zEj842ZQl8;=`U?eJ)mq7+w~H+gpwFE&+)`iEu7gRX6A{|lfH!V+e=udHma-HViTcW z+RwT2wMIqs9?zPLdZL*p8!m$9jGttwAbO=N*=8BNBdKv z=J9NfEES|jKY5oD^&LEa`P5Abl%JSb^o*D+dnzKSHy$<~x_xY)JQ^SV zSk=P(s#DS_B^`*-&MCkBUAHGc+HS!Kn&A6V-GZTT2|u?ZpVi7OeaaM=?7M7LoI8U~ z5BEGy4ZT!|<9l|4<$hMjOnSBB!pA9uCztE7dGo)k#q1s;5S**J&gcyKIVyW|p-k@s zCw`$w@7V!O7q@fSR_=8dA#>SxVB_%z%W? z!-cKH9WCBz$HJ&Df5zS3J4Vefptmek9Ov%(yE;QY*D}yZ1p6^d0^-rC+FSmk`Oc3e zTwaaDe(mg?G@4&=$Nj{W_eM@i0PV61cWV0MuI!0kX^ zI}B!Sfo|>}cPO;)J#eBsdgn@$Ng*jl#DT3@7!G;E&w`rJqpDtte^tGv!NFN5o?zUh~iPqg+yI+sKrAeV2BLO%Vcj`KMrin-p zY^(vxS92eSo&-b79v_pE^F?hBUaN+yw(H{~^)vjIT{2wkj7=j-?FTJ6-1dTyE;Zd)3_*M&%41nVg6U z%`~SDk#TfoqS;dR20xCQ;iD>_1A!RuroshAHKj|&-K9ORDFp{$RjwNgaNL72Fzins zRd3RvP7^TQFx|H4b*g@RS^3UtD$wIO%f6{u`%jp({$vcr3c0%+5#fWj5|dCUUCiDC z$YIpuz%ciwo91(a17rOl6>J34VYJBs?7L=2;12w-DIVilqR%YT*c4|~!lqh`8xr8T z8g;RdTTPi($P?J&nGL5;FzGr0v!H(OlMfHnKS{u$+;s{R@#EB)8-uNcp4A&`fM#BG zve%~|7Pf`UI$4xF_D=1c3EK2NVo2A#P<%Ggx51XIA=1m~rF3jo>P3=Hnvw8&b47yD zW&ofzEv|MFHxTP@u{?dn>BLjm$KU9726=J9;^w{}57x!qNh1Zy;rW2o)K9xX1)}%S zXM1m313s=fejJIlM!(~RFb>$@&gd|Sp4(viIlt(PPJTgNbaa#$m+#2TOKUovEPB0! zg@01BVtFnS?!4SPfbGwjaM_D_RmeW)Q_Xcdd`EfuMz;g{UyqRhR+M|* z6MUdMI73PX#yq|TSDK&S9EoTqQurF^eXffER>qR3Y^o$jIxXJ)5&yj;%abDv$(51^ zsZt<6o5F%nMBsJzqtyE$fZto}YgBVI|JEXFf%g;bZ7&_lCW>R+!E(+z`ZdZd=ztM# z`K8hJYJ1`h(*t0svuH>mSgW5?FE6tF3kE$@@KBtB=mpcbCsu2FfM5HyS+%DhOo|a) z!LO;Kmv_;$cWl|8&h#Mb=Qhuv;qm(-5Hp`TFZh-9a&YJ#ni$$>Cp1%-aKKoMwk-3{ zN803`zEzOVxi%>7Hy=j?A-F-QE zhXaB}{rvf9cyPO_3ij$koYS)RZQTAVu(!T}9><%m%LnUczt_^>Hj{r zb7>kp z%t2(A!+BhSk2=F3guyhAXcxKHE?X}{F+>=6iw^x?tzy|L7 z=P&Ji_;}|T9i%LuCAczz0NI^W4$RkY44Zy^CT^awK{JDv(HSz*KcpAQPw>~@r(~7y zK4lj)?Ms3Ayyi6=eT;*wWd0h`B=~;!Ps617Y$}B+T$b?0|JodXsVK`l$OLn07v9*0Zf~6!@;zKc?nb)$m{?#Bq^H16DxFJ9h%d8jT2^Up?KTC<{`T<*BWr0xrh2@uvzS2a4ciDSt z<&M<%oPG%uN{8Jj&n+`@_3h3n0S@^$5FQBL1_qUbfLK4>%8At&8MVSkS(k{-PK+!x z-ZNmJ%Ubw%w17shoPsRD%R98G@&!i|Tr(%qDV8Of>xM}=WE01Ofy)eEnSt%%&KFnS z)^u(=LQLRpTh)7U*OJ#%G$SFu?td<=Y3PZ0BEHH2O%42BB~8DSb0ku(99() zzjm|@SndTe1&d2JV$1Ef+X7}F%U_rx0LOI}m z+&-lw5pNOI^Oo})4~rt+CWjS}0hP3>WT5daX%^V$?z9K`B{^JLEs_X7zvtaN_9RhR z3he4l{CNz?;2#{ zE#e%V!|5MH3`5yTi@TJ)z6p_zcbi~9qK*@~#ObokQ$p|ifE&=p^YOZ}@wT_nCQKwe z1#L03tx&8Hjou7v@;G`0hK(GV=#%5qIwQ@qGt9f-;$)Y@r3AgC+8=+LlR&7qu?I%G zzGpFOs}PdiAEr&Tld|LSslG}oHelolVz|k>0Oux<_2&tGwf^`PurS_;p2hGcKg!POt~?glG&s zZ?t@Y5aaZQEYH->e!F3~-(G*1ycldcD_s44jP5&wDe20E8i_G6r@rN0g|4K$`}JKt z!y5Vb%a=3k0`B=SZ3ldZUTDE~k-2XYV&x8z&x2AB78rnhCaOL{gT6tSga#flU^?aK>Cie6ij@y@)K7h9BSuC z0*fAuK;OiYcY%8PxD?JACNRnO6P#LptJHjOz7EFh{=;6-MJwhx~9$VyNQ9 z;Ps2VCACxrX5X#{?`X3A|Af~JOhH^L;>RPw$LJDso(buA<2|skKg8;9-eh5Zn}f8&ClNx`*3n{dY zbn)kYdESKM35o|E0cLqzWAm8wF@ft+b#)(gz8%~ROOKcI!+X#IVA6631|(Tte^=4! zvaTR*APdInF}g3=nYd|FQz+uMHTCM>P-YsZ4^C?+Fht|hD`;zM4@@xaFEA|qX)b0p zy+bti$e;Q^1ZMpS+tcrE1RMSOrp&Oe=1i+D0%t{0-qCK6PzK%aR%TI6-#1I&HA~;o zPk>?HmNtZe*_QqrIowyqq56Ouc*k&2rTZIqjqlC7cPTjIckTJ#i5l(f#(%7Jse-Bj z5OBA6u40*3x?F-^py>t=+BdkU1!5kqg!U0b}s13r_oF{|BDar!#6N$&pX zzCZoy;gE|@?JBSJMSrp3$Z6kXgaleVk+3Aqd*BLO`W%6L8~>fYaR+ZKze&>d333f! zfUkX!N_EnYu?-}^nF8|iOEAB$)RQ!gf`ECJR_H|F!C{-5^BIpuP0e~nqLL*^TZ+v8{f+CugoOnly@kQ)yT9cTY~p* z8D*%zylNq5UrnU5>3t9iUn_RZ)rYX{Ml=`zX=i*=Re#z~0B)p!QQmKODk}N`92rYm zN&5?})P@Brl)kqJk)?0{3j{JSF(+_4(x!iOlyUq>!_VVij39kxBb!TE5Q*+98by3J z^27$6RMEBtZ6>ecH6{`boJ6BuIf~YvMW93#BojWU9bvQpK;#^Vivp0l6mn;-DWO_mfcENA|cz0r-lb2a@Ahe%N9a%=^zY8M7_TP z61I#Bg%%kV5d|^Sx@qGgD_OCS3+CuRF@AqDQvl-hL@kP#4-I@g?-s`^0T}wR>mcbi z4AsuH&U!b-Pse9stwE3O%Ze#y*H6Mg@e=+8L;Bu@>OnMCeEgXwk*huSwe~s&63AME zdP~!tbFV{xvx_-if^j=oKiMsslAUce`j(CzU=TIaGTLJ?{d5%Xp}0i?afeU z`Z8`8=88X&H8@`{Nb4@^Npr2j=l9y=rL-c~q+vK)7W_V-Q0X$hlS-2w(K7s-U!r%0 z#w|3u#Mdw9i?BUu3Y%lbZ15t3!W{iyg*BRcJKws1?Lk#@`&onn$y|U%nIQeml_<0o zvu56RorJbR-03Z2O$|IH45M$K(mmM72L%RPRyyW4!oi-y@bhs3zbcHH@=b26U+4t~ z1_wO)0k`W&>e3Ql?2vA8>0(BO2e5oY}M6sSG~X_lZBp&Aa*> zvGrDhX8WY2mcYxf%$0KcH+X?-GGH2@I7aK5pOF0}-zaAiIqyYVkWU z-*%oqmJe9w`O1~S9_)I%H{pi(M;rb}v0xTbiLSC%f$eGvt>vr3-=Wopi+CfZYD`Z@W7$+2-bwuGm$p%c2HmC_+r^*fKWh-G0ofnd zd)uR*>+ZMMN^doA`8Au_`KWyM*(m=n)of0M9rMEg z%pAa;TtHhvP<KN|CmsG#5hlhVhWY;~qXZ=6ucMG26VLG#+rH~YGqWMPN z_Xx?^a?3LXncP3QPAw@@U^XZ@=lyL`44K$GOWMcyMhQ+M8PDguxnI?H;_d?bo5#fu z+hQQdKPhzC;04*?e}rI+F~9rdfYvMVgWO4_;AKJ6fkjB%90tA3SDpIDc5B)kG?JZHd^1&f`P4q>M6mKh@QR}D_c@8*5X9~KXuilmH>1F0Xp zU1kw^pZ)aH>3x^lj)mXb#Yh42g4yhcY+nXMUM&=ey@- zAFC80Cyn8MLI4pUXT#BVY2S__vZ|b6DzuEx<%0ticxP(+L*m0EI%-R1xa_9WO&^M0 ze7pLy=?0r4I5k>kv6oQ-n~Hfc;07U1ZzD{~A7dgJ<$^|`kCqipFSSC>uiMIL@y={D z-y;Kr@}S>K;c;hDJ@eMZ52Dy8!O--{v$40xobGTLp@h16=(w_+)*lkZ#TKQB$XxC@ z%3v3yr-eedoD2P*tH3+gv&w<49ZAwL3h0xEfCe*h7Zdc^0!l9M$8Sy^KX>BxzX2hJ zWi&i4(tpHz_?y+T!8y=!QGZWApLi$SLUx-}0uWrJFW`688%3_3JJ8(dLhYlN088Yo zt&X7wy;nRV+&C-UymbEn(5l-Cusj$wSjmOAmGR9;8rr(`4@-A??c`dPz6Wr%FEtDe z$d9y~Dzxl>K6?dzzf4seDer3$#h{P8dw_DH_WQqZ<1cV#0x&VkdFQV*VqfmDYT{kQV$?|)6h?79mY(@&&J)f(UJQNg5r(4W|$g2kD9O8Vl|mis$E~= z@%+OzF^tn_)L&j)Au)kj+4E)^Ru$b_A)CYo`B2LK*Kn!z{2I55`IjG)M(q_9_eHr* z!2=wR@r`|ml^J&Z(Sy+f`mm9QcYlG4-|Dh73b{fWjs)#4q->vf9k%OZ$U&8Z zh(!lF6|&LD%ExWT-ec%d#av6f2E;C59g?p6jzqk^?9A``j&?VVH;#>b64Fo9%(I(+ z6lNr3J@U2x{8jo_POQ~r0qGGAN;e6xH%cS@NBQTmNWOi_%W3*|t37EiNcp-`O{GXf z%o=g)1EDhMNH(TxMU9nbywxq79SvQ@D}ZhUc7DtMqv}1Qnrgf4@12C+dy}dnN=HQ` zv;Zn9N)=E#qJne?MFL@m&Mi_b2nvMIq!$alg93^`=)FlvBm__hq4Upu&U@bRJfHF{ zVGORlueIj<&9(X{e^y1jgv{%Dk>yWP-7US%9I^2o_QTEqLkc$e+TJ0w)u?<1Oi`pR%#;vcCRCWd&2 z7)R{GkEro#FP&%!2DT_Wr#&N%Tz5erH* zkLHPFrVvdEuPwF_V&WZ3OXzAIRYqS5^!$j7v2&QtnyEABGaUXKep)l=7O)B>(7jNr zxRuH;czXy==&#T&ZuHDAWwx$iV>1?@APkx95E#pncGq&q|W@E=9MeF8Q_oU~k(E zXepy4O5_ez7sZofnEXFwdDeek0uMR1Cp)#+1|WU3a^m)=^Jf|EJGxQ8h5I6~(un(C zuO@0m31f9zkzp^cJYjGB>jW>-q&11fdUlo4FkJ1uIPhX?B*CRKX+b5T_Y)o2`3%w% z0)GIPQJ7pv6RTCPN(T~wrg;r1900s2=PK1(Ioj%CbhOjhWpoQ=ey2w0!_BC#!RRmp z_AoL-yDBZ>L+q~IQPcc0^4gy^#PwQ1a|C_Bw zvJ~}_hZ06me=M>vGnzK{2HjwRn_Itt2af{jO~b`cs^dV1PEpru7%V@oV4~&{qULsH zQQs=kb4FVcg*ALOTU=fw*an@0VHPqANU}_&Z6T%%{MZ zQBlbJl7u%Gi9GPoc#gEx@h%dMB20~Aacr38`&zB{i^T%jn1=<6xenaKflx{m>R^2A zHtap_$!w|Jx5bQdajQ!>P_M$HPOm(V{9|T=LIEb(0SkJ8OP33 zfiBQcGv;Wz^LXQW^NHS(>7rii=(@8$FrYLY_%ze5~xPEm2&iolTU2WB6UCVxOLD#Ba zQ&%Xo(+`S^Z4s33Tn>ZozO6_~QhYjp({0OWu^j@oDzY89PV0VKO=_Uga1#q>I%fO4 zU}3F)pwB#Ol!5{H{jd7R`Dn|V0UPb1=?zBR(=n%C?#0tD1Z!i$(irg37vD>vkpA{$ zB|CP3lpxiM#=k?Vx+AjbG-U3aK@`}=AlU+Gp*{O09G(K#QV;BZB_g0 z&WDPM&B2X<))b&5n!wkS5BRTF-0@|Dfbe#I6P0w|N6AL5qtU;4cDfgLEL^fr;b{^X zlU|g(Jv!a0=pHhjpAIl^{1x%`?D?A!awX*fDGLHZ);BM}6u!jLDN|FHp|6rS72=P# zAC)CtXa7(c-l|T&tUm4p(==%Cj(R<893rC7_JEyTIkk1KG?7-HDbadt_7*z@(8Bnk z{2088Cno0ih(7xO=Wyg133>h}*pe28DW%qrJP4h{t|aGHE_AoWKl-Isif;jY(ix`M zkJu5&obku9m7lxbByys<1RmDhO?4>MSs9PjU+5oCjv_r#Vl4}v7}W?yACT<8L9*mKiBVsf z4CI1p|568je%ZFWSE{Hy$mS!RR7gs$)}0Nb1guz2ko)rN#tHAkuPXo1Y^xzTzCeg} z)^(BAgOJ}M+7W(}v6i@tUW|8#9O38RXj~{|_qVTu@h?5_EMm4*l6>@OUXJ#BybXV@ zmiH)Dg^9Lir~$IG;gezZP43`blA6FpeIz5!cn{E`dnd@<9Rpku4H0lv3<r_P7+O`w)@=%ulFR>* z_Zf$Lg9GwgVHhnScrUp|ltvOD3+igyy-J8YvkuI;pGJFHRr4ue_oB>&9( zlKuvIB6kiIME{Rwn(XW|1lca#A{u_L;R@k*D~tEvm{tr7ztCVgOxd?#6&ovw)xq)qc(kQOzgcx#Qmh(q2{u%o(UV|B zeL=lC*nOWq5xN$-2&WUX1)L7%d7d#nDEvUVd3Ta}&NHNJ6~E&{IL)R_Q&We~c(Hx; z3Glsyk`2FdY)t&<>?CK*N=ahVF&Gt2aNj>P!G6vWY!#!9VR^1BF!iZ+UpKLtX?s?x zi~wV;i?QwZ%dwVnRX*}#M*rZIn^0y7md3?B6q(!6X*3;*b3q;R6gI$)m^1tA5#Q$) zY8hiyJ%U;`%icI{9oI3xlp-cdP-h>6JYTGhR1Jyl%yb^roXsy>qdiMt*Acf;C#Y9vkc86Hv`p02i#b3Mg z>&LtuEv4L(;2=Ou@=Byp_jj+YX0N}&yt%6NkNAFk^HWp!yX5~5p**kgCv33IT8xjX z?0qM(%pW{&l6ISZ)#Mt?M09Pwpwsdr@f=;B0o3z!J3*nYC3}(9&CI#`{cBOBhp~BG z#jKyVW!ctoc_z+4{60&d>z{a-F+vcKZQr4F9O$v_Pme8RBH7)I?*MkQ$#2!3&ZJK$ zB2wRee%4{H4>Krxq4p-Y!Hfj~0uXqcrHL#IM8uNMdJI4_bm8HXx3Zr-!Rr@)UydTz zf^(|uUxA2Lo_SE*+jMi8o7uGqxzwo1DiRt7fLmxwEY zOn3*CLnAsu57YPl!W8(o;@Qcoccn0J1eLeSs@UQEp+i38r;LHa=yNX!o*TO{PFp5! zlksN3ufbdh2l}jyap5k>XH~3O``BlzWL2ya%U)<{%CM$1>CJ&3k89UMyvc*<`tD2v z7rL(2*f}mkeLf0H?$UY4_zezXX>hcDFiIlU~&@;ovv+T0H@Kf%oo9VVhG|76rV zFL|#1+>CT939~30?JusHRIjpXF|-ppz7;_2));Zl&7zIq+dELliX0U`Wp2IgpJx6C z^-bkI@Acd3jc&F&KjDi4p0@ zUl_%rr`GZ`;KJ4CV@t>v?7bG3lb3DsF}%izguY`}1<&)!tb z9d&V?Wp$iT^T9IX)I!+sGrq5zNJahVPEzYrWU@y{veIPelfF>xRZ)dk=`+2mGkwD5 z^f?`xZ;Bk98a3!q_p&)#_r=a6>Mq8diaf=Flkw^cpvUKf=!0tv2Z1%z;O=_6b|j|t zI%Ge?68Aah^)Qi6b*a~Z!8Fv_YNxyk$b92y_WK=CV$}wX^!9+U3gKGHXgH1Xq3$PY zCoo^{6aj%p28bJTSngd~^L0vKBt)p;UJDcrJG?vnD04M}B^Wlv7>t7(>S>+a(9n6( z^N4Nz?iz>ySS#h1KKOtu(~dQ?rIjj=U{W8L!}{NwS#$alFL2 z2cGua&ymc1Rd;u7*<1I-+pQD3PGy~cV^AR!oN7()ljtkp%#L|{Bp+dTif%;VzE0Vd z;?$9zG94v$+puScHJiIPdz%WLl!Ul_sE zV!>haTnD+TIM~Ay6gvbStzhQI`ewQ=Wo6hI;Ibgk1<2;a> zNzL~R>wfU9>O7||C6HuO&tCg`4@>OL1Z_8Xj@Edib?+`milaEYFAzbLwuOnkXGz%kRb;ORxRGrDU0B^_n zCl#Cw#N-yBEFOzYJ5glU`Ox;Ppplgzh4A+=8ZWi8$Pzl48T|VvAvddo#I4_7VvCD< z?vD;DJc9l#+8+1l)zm}2$Z*g@8vA+uqMp^?0Uy=O4+#KuUUGiNFatJrJxJVY4%@^~ zCJ@m5*1qQ02UE@35{z~>G15V{y*v=YtbWDI#>ktUe%5zT`{CTSwi$_4V{XWq|?Pr_tKW5(74Iq0m50`VY_sio+e z%ln}CI4Yp@x3@wHaX+SH?fUBZ?z5UTmQ?~7hV)oVrS$Yc+L8k9XU>>_N(QylPkuU^ zoTKSTGdFfwRPt-s9wng6I;^W%WYUb1yG-w#8-CnuSJG6-nkZ_1nN8QT6IN53G-Ehw#p4@ z6C*@M2q-gDH+?xt_&qWkmL(HK5Xt!bHv^Snp0yPW>!;pPE|kU`U!s}M#zoFmTSD%q z6Y2G!KZ9PNn=aqqGkgse1g?&aWN2$M5KC`l6(zHX@vR{B;dhe!EI+*dR&S96bDcB) ztl!|q;cU=L>SF79V8@Z<(u1>TSHjqSR6OgA7Xj2$CVPSe=OrQfJ( zvall-qu$YHHz;6>&};ngibLcBQ#Xx6+I%%i{hTTeh^!|w`O#v#xE;By18nQ%D+ZN4aSbu=IPnBZ=EJ( zmZWi%tZ9@5)vid7)&;z@$v~2fwDZBTAfwseRUF79XTZ-0x}SF@(xRimlT+_#=9{_; z30!OO;{Sb16%`m~|A9qovX*)R=KlxRGe7!0d?13W$xs!=usdmZuwlkWqKE7aApT|m z7nSa>tIn^>UEYw=9CNNw54s^HKlgz{#mo5XF0uWtspKoQpd|3LPSTg&Yn+(+#h8I_ zyE0v$Cg>4KTulAE3lrlwcv8OOD}+XR};QmS7O)qh?G%KyGz;FoKe=`!Q?DjlXEZNDNGb6Ghf zT1r5NDS|U_R&9P7tAX-xs#6G@*JGN5(w%cHCr^U3lKOr_AbTId{Xk?`+_AA-5&@<# z5J}>Nww!b=JNRy7ej75*up=oaVoHEH18+&WrwN#dO2XTH7=AP0I*@tVH72&+u{#-( z7{$2`79`WN^(e^77TZtX5S)M@ zVfzBMe_lk!G=G>5j%UR#ty_ta%p_QSrO( zU8nW=8A{_hhtH(<@+&X<1W$ExM-mVlhLlv31zAxfS@DxlYMN-izJ2p+Sh(WGn%EtPOD|0B%yIhsI4yiY%iv%*A-ni%9 z;X+HB0!wh$Zm6UNNDJc#;HKOMWpC5I{1}|p`FkhZIrP!?o}{L6#1^ZVQ@y8_j{pRI zgHpPGo20}_pzR57d9q^t0C#puVf>+ViAjy8>Qd-p|5o*Ur$vp8WdGYg_qz3^vJ9K& z1h%@{<&6w)*%P+U_ogrx1c8iQEr< z$=nu~+3BC^x?ph1vP{T_j#@wVUtf*-WE0vG)-$fH1IT&K@%aLQw+D>d@)Z469w>(A&(}JfS-tZ9a zoGi1X%|_CY%dRTZfSszB%KCx~IN10R_DB3q5DWbmi>Au_<;@hwPinI5u%}ZcHO;pZ zvj3TIu?%rrEp{DOo82x313Py#{fIvvszMEGhRi{{?@Z~=$4VwT#IEsfN74=v7~Kv7){!#So0a^rPdi?Qin64+pQCK3XU(mv^if9#qE-|j zf1-blBZAo<2YWiEqhfZqHTf@?#S78-hOo+9p}QPQ{t(Bm8vU{8oH7wzd+s@U zE4#d^FI8Y4vT4nI+tYWZwyCiWthLKst6wg9`%MJQa%sv9=qnFOEUUBMFv(6xemAD! zmE}05QIi7bdRlJ?y^8fK-$HeHI@R8c67zj=$tVli7Z6bC7V^||qvl#A*wy_D@%)44 z7!w}mw#~-X#KvFmzA!u6*)(^e_BGD=|0Cr8HL|>-MZ5hvW@QJa^tWmCe5#2SVhYr+s4`pq14clv|1-SUBLd9J3AWNlYp zf0L%vXsGjVQ_|_;nXg0zftkd9#}{M9+h;2pdX{bIa5ZKS)L=8(i2BBMb8YGY+AAdM zlBNu%`2iH6vxyr0(VR)*hDe_OdWpGT6(@!9 zDr;i?y8aLc7X;3D{SGK`8^yncG}bw-m-Kf}&Q!@!z4Z=S!$&HHVX6D`<&$0F2p$~)`_PrieC_!#y$gM8WT1&;60nw6oManI8 zBqpx6K5B7NvIuko1Pu-Wc!3WhWBO`HG(`@=djnv zh=*jH!f@alZ$q{seuM7z>RfNhrEs0gHn4trL>C5Ws!{aTt>EWk%8%bX0DLLl0v`GN zHBcXzfY*vs;QK|o({ED`bPGWK99As>{xNHo=w=Z*qW{qY%rPym;U_USCUab`pqWa4 zDr{HsD2;(AqeqQkpu zp*8qqYulx^<#Wp4gX*hsVk)+2#jMZ2W;`8(PVILbm$sGWR*n@vgLSE?GEu=<4_=$v z%uWTZnAW*(8_voeGv-1(Lyi5{9UJ*A_Y=?jb5C*Vcs}sxR=M}B18??Giip}uX%Km| z5s#9oC^I67fesB?fBvII=Pu~DsW=zd~QLp^U%F*G${^#+54_WG6tusL>PO%Iv z3WkNsc&#~G<&Uen+sk*jwBlLpcJ!1=IBBf1cieyIhz~iYWz~}@=fc90v7E&OI+C5< zj_T0*NYy`8HGY-p5!Mv50 zw;BIrS5|c|MWD^{^PHbAuL`?&o;{4I=hA)mA%|mJPwhad}V)spk8f<%pOJu z8<#tH#xnjvW$}Iq)*>jO(XJ^FT*_VjK9D+c(0TmNe)le?X~KTOKP{FzWBaU>qR|a0 zAPlwEu&R)tRAR-}Vn`k>=VepVC9sgk-5cD>t4p4^AHFAVBqTw0KTV3(h^)}qc*}*K zo1(da)=kZlh?bxwG*e&A9b40Mc=c+k$~+}!Ec->O4Ol)}A~<|euCWA+b%RL3O9rG7 zI!*w*s7D%++W)&9RwOSG0i=03sV6GzDCtPXalBpIh3g}MWtkDb>m5fwZdGBDbOSh? z!P{k$$^~5}UolAwb*mSutkun3J<3Pt3M+i;y~Own-nA?RJQ6$Ydi?1qMIW&9V#aNW z5_Ys)Qp&6Bo?x1z_dtidlVI^@>yg{e7I9lf%GX>BsXZMP=H91{SUPPCK%=jz0inz_ z-fg=)-^P1ZZ;2&cwr_=Uymx=>%T~ob-+m4@oEdDxHz?_vdj0Y&Y2T&WVVn?qO2X1x zYX#AF?TS^$UhR!tp9 zdbll3IbNX-8+3ws+H?EPl!;LZb;(idm4;Q10++_28XJLs9^lqxQ)k+h6r(;51UG%o zdOFtd{SBqh0(^X6Nzy#$ulTI#GpU~OD~)A&T|e8_kH`M{vB|r-(7B~{t*@?+W7O%! z>h1Bw}#e-hX}hvK#L&PHSBa`PZ=l53?ICoGgW$VQoV0WN*Z>t25I-ZK#mKo zCURL6hoDYQ9R5oj?G^1t>0b_xtOLz}4+KV1BxIWL%08rT2!V0{c1(2x>b&>zIVFDeOs|CL3-i#+LMIo6(norrUoS3ASfr1X ztai};wGy7ssf*4B?+)!G{^ahRiAwsXxuD*vsDk**`ImTYUw_WcHrbQ4jXAADBg(_A z*mK(LD#!XwIy!9Y5??jC!a)0y?F(nGIiCN90i1Mup`IMfmBu3*=1FSW*?JC&PG=HO1!``zhSB>MIM-LAPfdx2MLE z!iw?Qb83IJqgF0n5sM4E-sHSSqV>_0n7N9XGD}2Tg2{d!Ks!w{(#Eb}VS`3J~^weCG!eq&vnVOAj; zrq)bd^!&xU*ka=vsy()!^b202?oD_uQ7kC+UgsW7Zsob{Z*!&J?Ryy@BNi5jEU z@giK3sb(Y{K6oeCG?@v-6|^C}q>??PFt_%c{6IU@L{a;CM)ZuI59AAPZf>&4_r&()DL5+*t` zhk~oL>|$2PorNGNs<=q7QJF>mKRL45X)Xpl5Zj|6#{a1Z5MRfv8f*yvOO{Rsq|K0Epd3BfKxTsFpzgc>!5ld9PQLa&`G{`X> zP2%}dPfC~ggp>XW9*hT;hJnHnXz!a>Qy5n z(3F+PqDCv_j>R4TD9bo3%h5hFc%5d}r%Nc{n0tez<+EGq z_t;_g2cj*veeUq_p0iDJBwLi-W<%)BqNCEMFEL|UfA|6SMf}@#gq#+p`{*a)T;wMbZl&ZW?G50?nB#!Jfa`?=?ALS_b^XOUnZxb4 zrr$#EZMNII*Dw4AT|OOo>%DUz`M_s=Awo=p+PWA|F<=#3`w~x0LL1wa>Ih)=OiaH1 zcodOYO=0v*D4zYo@$v@>#pbruc(6kYP?X8VdA)>eh6cLYc-|{7q`J6rV*oqY*y)46 z@BYjgpHD$gf!;$dQx6v56+wQz8g?O-_`Rs$2|L0PO2+R6vi!+b@*>a5bW4LYa=ka# z;pSCd<(rJVC(SELF5Pu4->zIMPYIzT9vw@Z`!8%;IgYEzO6_n9wiAA@s!PbnXLfOG zT7P7(S6B_xpvTZ^&vCP#WvD?R!c`5#k|j}rt&?)+twr+0m0E(4N$Oc-W+rAJy^LySXm#IJuaau_L)$taBSBL_o@o_pQGJ9O&ziRr!NvaK{m(6kXzKB{%&pS<`5jdI8MGx#cHV~|j-JAM= z@OgE)M%_J6pdN4vN@_Gm2mfl>W;+|z$>)fTLej-U-dwsGBXqA za~e1bU*O~#H#kKdI`>xAg@F&}O-KF!!A)86WkV(3cM$_(Xu6K^SpeP(0TduN_-ClB zlrg%6I#^#e8mQ*F8R5@BJXPDo1rSB)@E7C}26D#;DMs*%;eG#d^BFTpgNr=pGcOt}1Z@_t8gtI zvYk4@W<8HYG`w6~^*j3Wb4(HNua<3bS<;H`bge^8KFDIbLuldEB)rc%6Cav&#C8yz zP84_&(C`=*ZkB2(7rdk3O zYfmJ5OmE5lFFo*IHN``YCCKmJY9>kc^rOb(;*e04?ssV09-UZ&-1~|1jdXF^agt36 zcQ&MN_7f!WVe%Rd?njL8Kxt^#8Ca4NJzAJFX(}i>vzy$M70uX-jiBb57q}<^b@`Bb zi!4@5qME2Mu7_r^QfM+FhH$Bh+zyDnX=Dq_W69iq!9(|?!NSts=;Adf7YEmKCv|-p z7+zT0v;}FFbqV1TrWR4W zgIxVA+S&6dyI00Dvm&MIZ<#k##Ta~368#UW9q1D#UBEQGI(;^-d7~8>Dw-DJc!H}! zwD{4lFn7DN?x?z|{&?nYnpDPpIU_|jMEJTbEsv*p^|oZ*7?;>{_xhf!mP=@&np`SN zLT_e=aWHtsB#O23km7Zwc6LHFY1-CaqWEu^_nwU-I_21_HV($%UIBtAow!=PsF5u z0JH@H$%-!mX^3HOwkjtM>EIJX$U|g_OrFKqH+{zYl1JAsqCZ1cF@N7W_CcWGr?c2s zo!vjv2D7jYG8Y?LR4bcgmPf9+QMNvS7j&KowIqzL1XIbNdD)3!q!_zd!v5>|y$I)& zzUM2j07AFzTG9^l| z`S?{N5alg;bSL|$P6Dz|X2_7Jm?<|Jw$E<9`o|?-Y%9KZ&eS~xsRy|OpzFjCv0EQl z^@VUsbP=HE)CGjF?Va#rNx&Z{ab4ipJe2SJfsn<6m*#WYn$Q6dPU zB1j5Fn}2d}lKUUFA}w_t7&ohR(;LQn5xR2>q3YIh>j09K=dUR;4{ooAUULllB0%1A z^p;A~MjQ==(BrpuK5l5>-#iTZF&EgL7qF z3ULpa=mXbz8Qz=Uk<_eFv43E`sDDNN-%^mNM7tn;y@6&?+mjU|5=5{z2OF@c>({jp zj$gXGP=!bOvkqM-G=@(!B9mtt!o<5WA`jlR6GW~4)uYzYU`WoN3-`jTSs(wo-(lsP zR4z5{oM5~ocD5T}b*Zel5g*9#Oe6=HwZe}poC2;1t;dDesMMOmMysPm?ivlljY6r4NV`AOVb~ z(OPR%90c6vR0GZVAo&n~+09!>{JbSU8G!evurUO=eVa6n`t|M4oF&y+BYaBqeltIm zs2V}!JtHM}ir^zCH# z@rdw+`b9k30^AOWPf5DV{G23L?9Z{h2r9s zx5ij#e^%IpxqAw0opoZXsv`d%Z}Fqh^`pHm-jk%?pB38FAXfYDMyt!NkTPwz9-XW& zuJ*mG;fK5Q@OP_OFXHY;W^!&$KO>1PPp3Qg(F5=-hx=f&%Ac`10DiHPeAoEVbf!1eau$&^OrT$^# zE_ej%+e=CeyOUX_69w4CK!{a6ZbOkoHvFE7-I*^{v;q91lXZ(#1vT|*@RhvtZZAK& zS}Wi+;~krR6^~vpDvrGoR==UnBf9tE(MRv) z*$Tn;I+C%%XvJDbjxltGoO9n};jWq&%^t@wd-&Z70$eZ;;qb#_QJkN6)clVGjjsH^ z$jnz!A-ZIH?K?W~=mXP8O7q;d4vXWgDuO7MMnY*ail!aRYvf()69<30t>m=K8Q0Eg z075?|a>I$_h2X!M+Z=hEB5b{ak0F$cqwznt%x`@YdTj6B6x(UQML$|)EoiD05%8HW z%^tvJ`@70UPGoPppJ2&U!O%a*YRn`SWKw>xhQ5$VB5}c&A-boc28)TlGp_ zD-i#3g{aQ-g)p*FBN=8qqxRQ-{Rry%%jSX%MBsjoIcD%xPu}-pxV+!A$SI zd%fS!Fqgclwlgfq&8EjkPsE3$&F=+Je7?7G4%YyCXpE6881gie9T4lwEctcahw=G+ zHp}-tWsC!=j(K4bZdG~&$4wd-2_ep72lx>VJ#=^{MRK)E!0hg~?=7bN2`6wU zrKcmQsBATb57XB)AbqF6{j2Oiogkd5uT!#6=an+QbPh%tx|)g?HH;RIx7}>xV5xd! zThWrgu{QuJHmmRH{Ha%KbvVCPbQQB=#pT(m4`0buPW3AWn zQRCQU`^(V}*P7coAw6HhVCD)Iy)1%H|W+$4P_h1_0=E4%+iSY31znXJ?!^{bACe+z_@I`IPndj>rUaBU_7|w9O}~(Y%bn$Pr}FO~ zZc)b4uz8lh3v^MBIi+Zaq(1^Cwtc8LxmB;LWfZ@yf<74!iiQu0Ye4DBM6OE*k>8-c zZE75FuzmVU^V7ZFtdw3+yZs;vBusbgY0}>C7G#D_Y03v$=@$?{i>nuPQE2GP4e5TZ z!drW%pv2)25lk{2aZM4B^)I>X;#eNCBtlV_yUb|_5ZU&bvHGH??qS4T_1mtjXF~Sf zWD(vj3mjX2&GPSO>v2=I`(K=BP^t$i3at_j-%M7_Dy&U+0GUXxTo7ds+eZF+{~4H} zJN_xKL2E(Ye|Cm0moktik2CL)3PyCvDX_=cnF3i}M~w1r*?X5tzob47QM82a?uD=D z$kE}|ic?w18cW!!w1S84Ph%*w8`Jms*GB_FWluNNqFP4D0k#_)ZWXd?mpe#ef;}%U zMYtY?t5b{9wF%nyK}iCD>kzSiG{=nV(5!z`#0iJ^uSfnX8GFxn{Z|dn!2V$_*|CWa zXMgV62K!~>;w2QDF#`iGD1y)wetPfTjGE21dA_~CwSM1(m8k6HaYgrAnabLSY z_{fHv?G3%Rt~q=X`n~1J68**m-~3UUcOvL`T)YhV-J45$QR{d=#Awe3aQfG_F$Ja` z!H5O)h#D;;VccgH+WZ=eyw@z~yLa2Pp(ca4P&03*IR!+)Y=;7SpXYr{;wq9IulwCn z6xU*=?;HYq&;LqPCo8u`zKdj_<5@b`E0Dd6eJ?R-j@18>DKW|4)*pLtPl=sB67;GN z39D)T=oN{pqqpOdai^wo@Xl{FtN+m%KsBxP$ISMZF!ekLFw#nYT(PCgX~0!e|_RzbRBXk5h~OwT>^fGpYohmJ$y+M>hLAM0kr87A4Zme$eIuJAMZzU zAWn$1uq`R)B^^;n-<<(x?z#-BYq&n6y%oQfhY>G_2#F;Ppm^#K4+(jMgOs_X<{?P?9RgxY=z5i1Ov!FbGIQWj{w0uJKZn8?# zy4REzaQ7%WB;nFk#@z&WF}{Bm>d%d-v!Jl;b{ig=7hj!RZ5iA*bXoi8s2_=R+v7$m zlI};G>g_lqp(uzub&9Ht2=bghgV>;+z*T+;{om-Ve+R^I7ITVw=)jZhMKmRDrih_8 zcE@v0xhSEEK(C4ZnecMqDbkacq!e=!pScQgzC4-st&r3BN!m$=sa&wQ20VE-b5iq7 z4>yiBuVCUNBm6RC`zZuJ(JUr;*A&wnK}*Y+XVda8E~@ZwsTd*;&e$SR7{f6idJ(|O z6bo3_0hy&V<6Qd^-4En?ZoRn(!hstfDIZ-|uzi8D{?Xyb(O#C@k@!S@2m(k65TYCM zB@;S93X0B+*of>!i z9V84}FE`ekfOoVk%vyXf&NN+WwA>DZtaLZ%31rW^6&%J*1E;q81@^rvJx}yVIw^jl zToO-@O+4=@N<&a(S_l`zheKWVQ&ekAAJ@?}DDE}7A>iD37Q7YvwG`m!5CiiM?8DVD z&DXHvt=a)Q{cT?ZbRP|UQb6cfs)G*QR~oD!^osqepcpcwWnVT|uOJXA$OFt=)saLL z;I{e&HVU*bMmrnIb0#EuvyzVf4Y32G3tgKu1EDa)V78=hSlp~jRxp}!s2*U_V5|k- z_svlIv0H9P=~8_Wz4!&`e)@84^cYQ~$}-OqJIUn%>d6Bf^!8IJ?$O%d*6JqdHR)ZT z?XB`4>&Mvp8?L=eb);`A-h|mK8fUBjg$0LnoI=0JLgF}DyqC~MeZH7n_Ak+OrpWcO zTn97SH+Gl-s6tMZ0LF+*%bG@r-a!BVc+tOf(Yz&u+(f_m7_)sE(9#JNpHHiG49bw?JysgPdV`;X9A$TeabQ=c2li6LT%td~!M_gI);hotzC3BOx z5~Ek#Lw07pe1R*iFOPa|XwzWMtEj ztz@s`P?|_WnI$P?9eaHaGD=3JlzEI&w(Px%2F|gwNt|OIa;(GQ_vzl>ecZ?Q{>$T` zIG@k^{dzs;W8?1W;PSxRP_$ryYFudXnqM-HLiK_5iW#LPVBN0GCIoC3&;?6yvuO19anKI{8)e!>7f-qbY1mY$5u@fJKMg?sC zUE>ZR)Elku4vFS0?vAbruBUu;Ic_7ul#Nh}-q$)kD%ql*H^fra2pVg{IQ2QRixo(EnR$_p?PaM z?aD^<&qZ+I`B;gsXyCSIXjsN#TfkR2{&HyNLvb(JY@&-e(xtthR}W8R6A_#ap7KMZx6zyF^8Wl26*L5ASkB&_T~m_6!M0fb~1EcNv`f>s|B zDHmVZYPb1Np_~(LZGc=H5~L2$pEHtm3^)BIG#K03yFN91Y)8eBb$V5CLwM&mKChh6FgwD)FW*mEu^TT)IlzT6}2e?`A%Y`E8Zv zoT~ic=Y6Ry>M|_~Z*ime2z)>o6F3 zDr0|Y(VMQf{5TCsuHw`{Kf-b-qwSdC)b?VShHDUtsG!!$)&B>4Vy@Qi$rt&+KYixx zk(ze?Y@IGeurz|P*6AdSW-tN>kE_E0;k^qMTLUstQ-j<4OCQu2S;XUq-L~(XO1!SC zgG#K!xN#|#@Z;jSZ1jDLELDs6@|^~wSpc1w>z}l!v%u7T-tnEkf@r@Xu6=fW7XD1|I&90(QAXV%`h$g zc&At9WgmR)BPhA*+UNX0tAJCB*xiF`dv`gs`_75%f?q088Hvx?v+zZq~g(o?;q z{1DsOdm#gxIVG1%uXV>BY^oDje{KL&-xuG?Y2C+}h!zhI^l_8#e>5i#&w+vW_3%3 zDvU#i&Lc6M3uFgRZtD_AUNcA^C#}X`mTnE+DI&? zYrtd{2~?vzoV<&_U3j={A$jmaiq{_PLTepndT3g*qlADHXHxQ9B3a$!Ih%`#B+Pn} zj}5Gor`dj4%%?HOcFwn3#?)-{_YNh-{ODvr?~=`N^=i9D!QrFUOZGj**1RqmHWdEC zlBqUzzW>so{AY5`7a=ug5I-3dGb!k=^KDv--E>9Xc9jKB3L^`5R>TS(I<|C`LYvV; zd6vSc`q-ysrz&-V1R?-a-$9oT4)V7LN2U1miBhnPAMl0u?Hj8+ovme*)|yG78-2SJ zuVSNg_jCFipW@H{ek?wnqEqQ5+0pl>7)qQI6YVM&gcVo6`=X50LBUpiBSTL6VvE z{TCs;gKq+wA^$P=p>nTPNST)@D$JV?kGQ6GIZP228ZFX%Z)YXao>{u*c*cn@zr{)! z5GU>X3V}zosJCtuEe?hOCPc-L1+d-4b z1_xUA4GU4>H5<_w38Otnk_Wp_KxvJuPH-Tk%on45HU%tRus|0B`-YKn8N=9LW9dZ& z%x{$Jqe+GLCK}AO?Y`Lx{C1}M{E?sO(u7uhSVCB;8p&D(%lYMBWNihfeW&}e&7%*% z6rJ@#+^TTb6ZN%7NV@qdN1-4nYN|kfIMW159l&1+Eb*-#Etq%@QJHd#rFCd6TvoOBLNw2ZSwKqd=OH}K#?;4Il`5aN_gNQ(&{=``&{f4XmZXzSbdshext zGIZhkfl7E|iV0qoo5j08$Gzm=dc*#o>&;t)D#+o@+P5)U>xUq^U-TCN$UhC4iTz1f zo3=tYS9uUoD7-6cy8_ntPA8`^{Ijd&0FOQ8$Tedy+UOK~0Qpebeq;hMSEe+?&{=b> zM*eZ*PnT2^iEAavOKg1{X?}<x$#Y07OfW{Y>i?(Q7(+DFT zL7jm!;Uq=?4IUTE7)O`}+~;w!wFnwCf*w&RfPRk!Eh!m32~Zm`&^*J6#-E3}iDl&ck6j7nou{u%WS+B;4l z=hs3QW7;bdhT#ZjFU;*@1a7qMBU0>W7R_X17)$9acQ_mXJQ>L*iXB(5g7+E=PCMiU zT4k-P6;2sC^h9cM$_1ZR+D%wR1t;MXV!|jPOv>{69CkeT#UYC@k>yT0@5TC_U8S{>3;pV?T!6Q3O@d zK=;aKz>QO9s38jQcNpOP*>eTDs?0&7dZ34zE>|OP5P;|jc(eW;-rv3MgL^Omqsq5F zdb$=|qw~Jw;=GxIf1RRMbnv6+*OK9jQA@9C>`GUp#=DN>oG8yftJSZ3P0}RIXEhu9 zcF`nJZMYFmN)J=F=JberN4hv_D1eT}5yGsK2tKR>0pREPd!r5W1jm2~T~S)3K6rm5 zxd&)0r!`Ml#yQrQ_ZJ(hfs$iAjuU8V_H@IXO>+O~O!wRW5_@(5+H(|R)CV-;e@7v?cn85ksZq7J-Xz?Fb+9Gq+6j}47ZFsY)) zajxMcp;+6sjUDh{^zvrQxANL;hItO;tch$CmJ?a%BAGsoId^C2%%4EN!*6@6cT&gR zz>cP2_hPcu{dlqQ!fwYKlg|}8&m--Lb&&=Q48Iy-on-0OF z|3OH^s4McT6XVi@s<|4)&;+Jg^E}Q0aBDw+twt)7v8$TYbJxy-3FrH$^pThAg_SP5 zFP0gjU#B87<{_Ul8}1Kq<~XhA06|?|#4-Z_Z%@4YbeVg1Cf1(u`PbF9Q_mOH9Ugi% zDf{tuhUxvB{OuTQ$BS0dIG5wgC;yw&`s>`osI!riv-|J3)|wXP?<_iXSE+FAfZsyG zIyG7ON|OWL&_{arjy{|F)L0z0+X3xP_tu|bh1NDkyRrnn14FU~spc+9D1W2rx1|d8 z@e`3hotr8n6Q#34)0%Of)~G)sq?Lm{<&B2twL=#}V+E7utYep>SA67dN3)K9vF+PmM(MHf z1iSMNWs0zn->*8y1*|#^;ReO z@drLepIc1FdffS|Ylq&9f+?C#q_fM%38bZaNOA-NksyIKgfuJ6O6bfDpsvD8RjiSv zC2jf$0vol+G<5vqRe#%&FH@4ZP%S3nc0q}>xAmT~X_VlSuP8ppE4I*hs=sS0buV(> zUC_#ZCm8YVX2hhvv;VX9QSHchp2FTZIJ7JNAU^^r2yRoph94j#>nZ+ZGff@sYHzJ*)5OcZr_B)_K-$aTvIlTcI7nAXk(f; z{KHA~lo9|6o3(g|K#F4vQIW&$CX!<{ni5Q}gi^cIF!h}F1IIt~??wsv`z~8x!KtgW zhqhhX7Tr*s3@hTgCdK;tyP$`mYBu%F#Px_sW$Tiq<<%L>A*CrxINNqMx(hu~A8n8*9QA`G1Z!;%OQlE3ou7c8_y!d zvKp%cU9Nh-3N7l5B%Zv77LLnPiI;AJmVK2HfJ-y zcuxqZN)%m@otY7e+#t?NeEEI9jwkRo_MNVuH%$xIEv~kT@IHK-h5nrcoAHxC8w=(= zEpwC|y(F5Wc_cs2rXcT+^6_tTzS!IO66vIhs$!PML*6z?Tq*Tx+3*k(XpWKs$jIJ> z`A02ak1&kp%a3n&crs55;KXzL4`+r+dqdw&Q3Ia6MiC!RVMml+0~+JOIytsy|0b}& zPLY|fWwLvu-8;nv!FlsOIpHN4dO_5NsY3`qlicIdZ2YK@UtT$UOyFm1rC}wDaSM{J z#zwT6%V4-|G2pxTOygD%o_Js_8p;}m1=)$JNAPx@fX4Ey?@CG_b{1N>mn!2DK9(5m zJ4)HZf6H)#SzGyjn@Bvt5BhJ78$l`%PH{9Jo#2z0z8xusMs?z#5s=;nH1YuCV-WsR z1}(=%2yb!(v@53}r_(abgEs32Upd&BOr`Co_WdnPUXOhR<`@mG@&TZYL3(C{oEn$4 zbdq~QLhw1}b6*>{8Oe%M*5xfW4ICyLg4~C4+Xq_eG=`gs(>>LHc5a;R3f7-J^@JqT zQTYY#Et(C59J8!zj(+_eTcq3FXjkGA{Nv2`_p>a^l9ARZLW=U+;fBh^cNle;k%*~J z&+bK1n6<*Q`Lz1U^*erG5j#3BZMH2=P??nRaR2$8j~OPcfwtYO#2NQH%tXOYfbetx zlDr-j@~O;(4}({Cuz8}I<+S5~h2*km@7aKhHc z+m4wu4t~x8h{ux0>FWY@l?+AD0o7 zJop8=ZY4Qf*3~)}xvS932}nNYd|q8riYm2IL2oAvTT=5zE3HzuDi=bqt6~aFY25o! z8{#n4=cu%e&5?e85u=jHNAP)-nvVTM==0AXhSj-urDp|mm9+W^u=;ayC8BxPy1V`cd3i)}(gFQ@fYGHV^^<2R68%DA#I6KLX z&Rk?7f@b6>sb68g*>64}oA*A#schYmjLCqoONG-6u#;x*HVB26R2%z`rQFaBv{a6`(uc z#T~SKLk+wiT^iUvec0UQ5kAX3dl*HVe*lmRA^}vE_4Z3G!a=ul zu(dRRX6d7cpw<=x8vDZ6Vc<>G^yczicXyU;ir0aa@S&|J?ojwYvw0xfZ==w+JPd^q zzl{ljB$Bw=&x4usjJv;;k+7{0P;QVv0~0%y+b7qMz=%(HfFKVZT^XBGgMUFP#+WGL zfhahvG-RtN1a{>T5jKr5`*5`3+hiM;!-s9Zqq%#K8)!O#(*#%xi*wfmIB}7#5$*W_ zOln(&SG<_B03w86*%Mrgp2Du#nB;s=ZD4H9xf*|zyGNf~54(7I^9%@_|JHoBr26c0 z$~wbmr14Lu__%m&qW+5_l<$xWPch#kU3b^6$k(cX5$>fuN;sUvb@Yme{6v)bpA(a{ zVliK(=LNjt!ijNGb86zFR3`J4`JaIdkH1pjCwO`F78D2s~}` z^+sv4V)4GvzD|o3;{oW=lqDCD$*Ji6y$#>{V&P>=t(<*0FMMB^7%tAL6%D98CAmR` zIp-l`6z|?qS43g1&J1rn!Pr#;k{>^T`kAENc18GQ&OA?)_Q7@ktzTZs&a4|Beyv%i zCXRfO8pkj`JhHxtr)yewdQBSGCrD%WM8Jc&+F^o=->MiMG&UIxQCQ|4eNM1rTg3{X zGu=d#NKr`#d>+^OY26gLsFGXpWiM`bz4}urkdZ9j{0XHo$77wG=Ds_2io;V7tL%k`(Xx?b7!M?*uy|^~HL>;ek6-~?BJh$MJuD5uz(>;BDYvQy zWieD?b|Sc-`Te2@5O~&VBVx31s7ZKQ1@6}L4jh6(O==8(Lx-&d?Ih4R=XxrzRukT; zolE%S@S64GJ;L5Z@=}6JkVQ?a5jsr7%KmmPX@{@l&>+2VGLOS$`_n58@;aSAdhO1p zPB7v7Vxz`tIbk-4^1fww^+xny(BMY5=%2|vPB8AUj*-k|fka1lLgTD$SDP@6fikpe z2FW7)vIy(56FF0V0U87QyHS|;VnlqeB<^7PTeKD~v|7$GnXCe+SIt|`Q=lMVy{6q^`tc>FuG*$({923?k z+pi8MMB6uFIS|wG_K@Xz(@|?UK{@mT(J!rdQZy}yV`leD1GeOy9|lXviXT?hNe==c zg@$UzZ`%m#S=tSbe;0DUKn8RX!#W{FsvHKStQ6sj^**(wRx-!t+V-=cENyepeMt!)`DAI}Vck zR3G#}DGRsh)%F5t(ysRj27H7As%7I8D={ua^UF09Zt@C1_4e0EVIe^3hr`3`{|OIT zDWm>=BBB`Z<@uF-5KX}|PBv~*)ihfaM1#Y#OsbnRHNcjl!RssbY8`V6+UyPOxSuFB zrS3{G{}(v^!5pa;btJJ}Q9@{K#Pn?1md>Y1q{q+!cIBYk*j4ilHOqnjvXlsLA)B*B zjYzY6eq!@N#uaUq7!%5UzFflX3*ORb6nQ`ntzcOBT|CGik>oOr()2d6Pa|2Sf;~H7 zO9p2+Ve&sShVYkGk|KyMO*a41>bWlna4nkyVk*dA>L#;WQg!y}JG$ge! zf6t23iG^kA1L;}Xu6%(2@v?Z&A_Yp|PVEe6Uc$~E_MTbJ?VI3a%msLA{1ygjf?l8k z&V;-jDBBwWm8M(f!s)I{2`q7Q_^%d{p|XGm72TYzQaV*QcG$p%J8(vl{lp9u;Gvjx z^0!V1#gb%4+fhM4qrq@XSAl~V!l1FDgz+BH!28}P^2~HCGJuxcvSYX=A`IWZpA(i4 zMGAlg$}kaUwvXq&iVK3m985G{Ktry|DnPVQhMXQf`8R<9uIS33!H^Ti1%7b{;k0W@ zoO_|dm>GiYrje}u2=1dxOfeW$OIpdXqwIyg5k`TP7Vmc70L7Vmd@SwPOm0}=)C$YN|vNrguRC4CqAk1@TT$Hb0`y|qDyM`Tlq_{0}r0evwBQ_+iOfv(V1_# z&kw@CS4_gt3F^u2u7p96BY@2oI=XqU3R;rmA$Ybfsuf6wKHkGx268zg6jJ|*flRFi z&(SA@21)dT|I<7I5tQ=?a=N=~CK@-HdeF|rgb$~5GZTZ{X54hEKMl@jiIf31VFm!P z`(BYOOk+P(fDP_g0Q(+mkv{w~|LVwCb)%f;(K~m7{64j8Iy`b7zOn9W`qB?he$*)^ z7YF+m8Lys;XLJOMOP)t*efM&ezr?9;%7?#6Sy}}GH7*hI$4>$ms{RJsgoT$Yp^Y)k zE43S#^`OskZ=aMYIUg=WBw=0m-Xp#(okoS=IAC1LDydm;AAev$g_k-W%5e@%xbgSfA<~Ii7-HfDw6cxYe1EzuWxl=KX=`>z5~=-pEEKy?A2<5p3r{&{Ke?FE8Pf&Y~9aiZvuVE*xEYJXkU*E=L1fop*STp;l(!Np+n_8#Zmr>^LiwXsk?MK~Z(EAJ zoBX-0JeLxqU+?M}9Kb~EVFg9q>%{!H>D+xrT=B|SL7|Q@+?!@Kw51Eh^7Ju6Wf}iW zp8vO-2Z~LyCPvZyKQWbwl&qd*0L4?B0=~e@;jZ zEfed<5`>RH(mmy2=oz+AV-$y8MUp9Img{6k=*O-U61RkFjYgysK z0F9x@wY1@@XL@kE$DT>H^a-cmeX?D$bX`N9$N<%E34Z5&}GxUN4W& z>?s3=F2*e3=0lj{)9TWL>?V2Whqmg)^Bx63MK1QuCP$%ocM(Vl%}yNpFCLcLAo^fN z*A6m9>;4VxB@I^B4Ee4w5vBN3cyB~E2kzWvCo*Vl2XgUMc8Lt&CO`Ay?TnxZ8nQdo z`DkYt6WZV_%u@|9yta=zVbi}|>36$yza0rTMiMNZ0n}%##O|O1$*Tx*8j0zLbRnkN z*%}Eh5(R~;1-6plMCJ?yZzMTY1Z}4Zg!CcE5BoD%@g?d&x*0k31E&Upx|>qrsmm1t zMPwO|)=2()iLMWvF&Y*%;fxD5fh-)3o3EcXlXrb~UOzpwIz#li!Fs?p{Ks*#cfNUf z&-#Y%xca^s@W57<{lUNVC#LRXSOzC}7Om?D&`hP#Q1xy)#FrqT`Uj86!yhijZP;)* zq1j!aMV+*f-cI>OPq%wae_iJ82di@xQZtl7(9t0%-v}@3iv$wSh(`es- z=$;Dq1l8RWk-Cw@8KFxT zJZy1ID%XdWvw*Ai%$~sFeHht(FpcLZJnj4F7}S}z1W>6>%L{-oa9{J+$ymlZb|B+# zP?le@mdfXF1#Y~Z0zllroEb%ZyX>V)9L_eEZ1)Vmil{mD3-hehS%so@RTY`qPtuO0 zO~bss!0X?d6d?ZLXwi|}_nO-mofzDQ0?L+%i8nMZtodk+So`#_H$=lUsxU+%&-9sg zSKKl$eroVmOIijaab~X%(5M(gzpmrh8HxU4uPQ%tF3dY+82F>#%VgdE2#Z=ae)9&& zg|ADdL{=I!8DXQ-f}F0v$XT<3+kqX?5}Vtg#hhAX$sklu60ZU@3dLmHIT=13e9?+B zR{1r7iHKssXR0bbt1iSq*_F4*$EfteYxiP|7O{+r-t9bS^-pJyXK#| zE8VQZqS5A|Q2d}s>2&QQz19o&KIp3IqUHi8DSPN89vD#}K*$2X+i|ETJGknO?IIM= zdClugc)|yO$^rK-1>NnGArJq*wQUzeCp^ZvX<{NLNwZCtMI@f2MZ2@r1Un~6!O@umYL|AdIIdJmmgF0gOu8}Bi zQUvM|bD&BQ@3L7q#s3c7Ip(8-oAiLtlTdTzs^zlO2jGkj(@7-PaEW1!sP$xgU{Pfs ztZpoMkKFhA8f=4U`NiCJ4?OXz$;ulBV)8{Mq74%Rj%&X53CXQxv^A3B5pZPMYpBfE zhTDD&*>o$@NsDt2nhm}Q-M71xW!%iVkv`=5Ok2kFs|xb@)pu*-x4yJ%g$+OdJB2(} z2cVNfI>FYEu)8XjQ7-;XThfP>C&Z^mUqKEZyO)5*6C~MWdudw98z zlN((V^c}=)isW@9sZ`5gRQ$P7@NZwfD8NNn_~+Xm?Q)Tqp)CAZ}haYj_bj9zKf4L{pwn=It+?x0ul1 zRM$FZkY~G4XjWY`b{wxz8h#8$^Yq!+vSenVqGWMR^MH55f`XU^Kt(MY&mA;bd^o(& z>d82N8Ayk|$N-!TXr}y}(Cu8Q1wy#t;OVy>bz2>n*%~_pt@0?^!r1~B%|t8#lQ@X5 zl&d%nPshJm80{w$I~Woe$8k&k5%iox#gv?YO!$z@Rq-# z_w#|3=Z_xNRG!Vi&EF<~2HKeqPw|?NRXLgu2`@1g_J6&D?b#r!3@|`x|t-{QRp_SlzR8S->(65BbVFfo~3~iQ(u2yW*}tY`T+0S zNd{{T(PddcUEX1UvdbZsnW*^`(7qfZ0?>924OxzKi2XtT?>q4;IfRH>4b#zr`T#pG zxOZ8cLfd_3B+8uHM5A_qH{TR5u8Tkt1b%yAAVCkHFJ^J} zceX&`-Ub}=OXXMSMpa9AfG(Vx{&fSf!rJxdz{W(E^!@d3U?NzwB6gY%`zrrjb2LLn zlC?@9T4d~Fhj|MRAT?I4@7j|-00_r}XFnYVyh((-l2T9}noQJ_mX&!^xFX}JUDxx& z6JM&34-HpuSho9pWsT=O${J7Ol~i~30T%z#1CQY31MxRSsS^)Bl)J@sp59X4MtBFv z8<$LVObu!i1tW^%0w7!~CgaVROuu1aEt`vB|IHmPAQ0j|8&4TB5FhhE4mnWuUkd`p z?LEfsKF-!5Ce$DdTeAR_?EptVb;Zr?31OgUk=R0=&d2@RuLlw7kd!IH2%tAG5QiDi zQ=e>QsOcmZ_n}fPVLZ2q|A)1{Sg4^=nIfKE`+1|Lq!aw`;f+BS`XS!wdNU z`DivDitI!$I&w@+sfUA&2;n@E%Cty~gEvBMd_*eZnLBST&ARgW8swh3?S}9Me!D#g7qM5K>A2+iHR%0ysC>lV!`>U)RJA_?Dp?u{&jag?-^eocnl zhTg|>p&!T?M+`D$&mgagkw6633iH*pa z^OF(oZ3%9eaZ)|EbUC+HW}-rHD*toZb9ssVX$k!qMrR`K=~HvACbOg6ivryP+q?I@ z02*HPx26cL$t;D|Ac{_oWNI8@!I&`<9ffSw__}>n&cVEY86DkeX=)XDyBvP+$xU@= zSUdp}Iu#OsEQ7J@<*C40K->0wVzI<^aL@L$caOP{A`{nyJ7ANabX(g6eSSM)@_C%_ zANgiZtg?buw3tuD4JHzb-0@-DlBgLeWG)jn1M{vxSL1v#gQWzsHQjiBkupqyZg8+BaDvxNPGbnufxDF|jv^Fs@bY)6}%zKR2gWn$9{y`P# zQT=HyY&Is9|M#7_*Nsc#*%Rg!#_BlXVr8$#-x-LthK(9kakbn-Z@YXrPhq@G?q`~3 zM!=uSPB(~Zi87##tK1T*PvOcg3ruM;;;-`1&H%J!6Ik7itnxU`aA}5$8*%ZJBL>4_ z5Y?IiC(j=VFPx0~ofeq+>TfW3MAQb?GqnB{f-st3WGCoghq@#N5W?swk|vn6rzx63 z+eZV9mk65$mC;Ht>PBVtF@a9HfW2S{)4xPzHQQ28)S%3^JtJ<1JYdYrNM`Hu^RXUY zztp-?zrHrW#{i9Dt0=O}Cx8w`j^$lpAiL*-7?`)OpPCC$2c}qP<@wu&T@k)Tz7l79 zKiKVB)74|UXEp;^(1!JQ9n1bP7Heu~Nq;v94kG)dDK16Ga5hpCafQ1H?17O(1-Ea8 zJRa{iBeV71gavLsiy~3eLUpY6vdD9*Q@lMR3Vw%J3H0G4^-y!GGJDy7Y=9(-3;hNs z^RXbPPRbx!H}Z@k$3px_Rf$ z&o&$cS^mX~;;!7h=(MZD9LsdHU#H#wL&S!w^KX4@-e7GLcwBDN*APi8k1$6r#NEED z7b^HVV>1-#JaPAb#02&Vl3Ip-QIi>?LyeZxFZu8}6O|fv_?oQ{#(6sQs4YIf(HU-9 z@7|NdOx%Q4-0-qNl~t{hq_1ive1LwIG7>S&^N>4hu&Z1x?;0m{j4b;?6rb5U78Z;fvQkr&ScQ3JSU^U_5F4DB9JHk) zj^>^1bC&n-#EiC-IHeWPr`7qMny|)L)wt&{lFg3k?XrPZNL&UCA8`fcrqUN@Xo|iA zG}ayO9^iO{(@{0Rt!`l?p|}eY4;u9rj90aQYU1xe_Sa9u1$B&f0ZCooJBs8wBM%FN zuFthSUUZ2EfJUXde)h35{N@DEG(XrvyI|(Ze&_>SXz1T)EXWBE^`Rclz^etRWXPCl z*@}5W7s!)@HZhHsS6BTTq5-@GyQM4#F|!w#_~Ij@GkP#+ny_*}+mizQl;GY>QKVQn zUcs2((K!G}=OLHcK?l4vq>g<)p9Jx&8%AsULejmlaBumw+f;x$^6ZH=wEwlJ8|Hmp zgpn63S@fvIIR=IR1qP`qE=NML18Yq-v~jg3As?g4mHy1|+Dr_GuVW0%I}mb>vPX(< zs|FsveK>Q_-v~B~?%lZoB>@dc%4$B=QTR;q^TdTp-$_?HUSysNMc8^t`&t!dv@D%N zd!_Jk%?B1EU>bi?zpGoaLY3C6QdO`PL5~a$qHgbjUu`p~wPWKrpkcEST8F#7Kaqk( z660v&P-s_XdkRaBKxx&B5KUe#^~H_Qt(AbiOw?uoSaVktfBk=F8YXoQqL)1gqspqR z+P`Bp;c|0E;MOO0CeyTUbKyBdxot%dEms-Ifbx$gqzJe-vnp)(phZK3+Qu}fw z#9DC#15adgwKR*?V--3R( zFkqBgu9t2=rL;o}d(as`+N}C4R^1sKGG8-DI)1D>U|QW^IBXOtghG+=8E<@hng}?7 z;A4RBv9-TJ51vSb60>6)7B{{pt>3j|TVj^3tSxC=e3d@nE)^#A^&z~d*LxdoYvFPueznfpOQ%_vAppx(jjT9yPIEesU ziw>Qj4`t}VG=6pcbO_&+R}qP8B{^#&w4yCjwT*y~O~|Cjt!bKIVM+2?@%AAa-&R zp~#TNS8(ElExA-@F{2-l9OT!_f|W2Z9vs^ZEldTZHzTc&yFS}7fC5UeJ4q0u z5;rQeGO|l6yi?HPbK6c+7*{-bBBm!xc4AP#^uC?w<0j~D<{yWJC`QL4CSctbV;U$5 zy>4e2iOu8mjWW_Vy`Sn(m5WYZGl?Is}H zfo7%XxFBU#v}iGlFkb!k%cuo;Ahn+2bC zyOAjbMQvm-Idqp@-C1I6dHs4Jhlw|2ihU8Rol1-b4uTi&{xUYXzZH=RIC{nxP7d?{ z?U5u;E(V8l3GzpR*|t0*Y#N{3%$={&6Rmcb_KX6x80uYhlTyE?A%9(J(coA(IUc#v zctB|pcS*S>4;O{gq zzOXXymHk&}{)lBtyR|ScTJm-6qnznciUYx;7gA%&`QubR_9^dlJa#>`rGP4B{(37B zy9z>8a=Ux@uN?ye4VyN<>TQA<|F$uZN)KAAtUH;HCsH0&Ej3NfEjJ{Z5hGUE6^&=1 zkinHT9*!^sA}wf2pfVLs>{Um9oC#>e>h`-A4;3Y12dkSuFd>J0nl830i*^j42+<70 z1g1`A%OD#ZM<;t{APS&<(gD(;RxUA6mJ=^p1<=~KV0Apd90nmY)nZ`S0!9^(dc$i{ zYrrKOl#95JGAR~e*Ft$TIRSwT_y&jvhvESHWXB)lIPetL3IPO}ZMTQ-vw5@f?&>v& zrF^sz+9|js3YpiW_FdJWSX^A&`?9Hh(TflLKgL3?FxT)aH(=f|hF1WP418zKw!7(h z3$^`L=SI+V>dNGy&FH!w$LgS)lqGws`IIC#ps|Uu{Zbo4Jo zZs*#P1$+HI92#ahfI-FdrQ>Gg(3to|YqqS5kc!@(#$(KNU)QwJrY-nBF7)=t-z5eF zzff@5<-&+(&Or>{gQMr8wirmU2=BCZ$xlEB^x_pRowPB0eIg<-cS|N;?0Sn2AHH1X ztKbr0ZgQBf4ut+dsUo=0O5Ck|)w)CU*LzQaiDS9w1KlZbxADJUJ0#gUDR*?kgo5>@ z{_5LWsaSn1?ccMtf+UPHDNI`UY=d?eIs?9;nEg|;_}E9h8Ek@r2UGCS()!Y0+y9!aWa-eCEX%% z^L44tR)%FUunrFU>$)|})6YmCGln`S=pTPqQ%1>95yIrS@uD_$*;&K$v9+QZnhk+|={?L(RXoC8w z!f|;;fe}SsZ;Rm=eS0;URZVn5K+O~uu93-9p&+?6*fGQoccsYLd@TS zTNivg!q*aI9pq~*_(es`zFrZ9x7eQFms)Cckg;gIf-QZax*}aX>xZz6Ivcf)^y7S& z7Zl6KjZGN^-V)jZLDtA)0Ym3%eG?Z&hvFU7E}HNIgu8-fXMB3^kU{`qbiHvI2geDw zR8S$gdw9ZKQSVZQNjY{9U*b>8KdQx-z=D63-F@bD z%S--g^#X`5W9`k5V>)^_qKn2ZRJ-pcd_o73zmAio0(+UJ!`VTi%nzGbq!*K z5}X#wR>6efA?@ay;GIsFCDC}>hy-K9lE+%YjUU#t$jWh?1%}U1VvEK%T3mc+yG^vc zGCtXY)y=Z}*+?->2x|_cKVt9Ubnytifcka6{P)%s`iyDT_hjgWgVG^R1Ky^INBQ#% zfqMWEmU|N#V989q(FkIep%Su74p<0u%c6n%Y*M|jjM>?D4|BY#khq=;s!*ZR@)$RS zzug^l*&(}h2BPa+s;dqsbtTMSlw64}{-IAOi{hoj%sMjd2 zyd8u(VxeM%=UdX-yg$ZP7TK-Bid&0e-Xr?$6riGdgRvfqWDWrhl!negqSoE&oP z-)r?s{7{mlj7M~dd6vToMi4C69U%3Oww}-n(UI8O0*HuQrbLPVIb8lB-@hp8i9C3| za<7u4j~LW1u^BO|IxH?*K@&aIH+EbI-#Md>0(bMNC=elWeGN#gH|VeRtQ zl0ENjp2n1)|JDxCMWX1}TH5dK3YshUv7cerx{v#FlY2D%@PwJ~Pm8vI))b+!G zx)LG+?Kc`A;L85D^2J)%;vXP}wKM6B>N$w)k&^blor?y3wOCq&#nC@k_Kh);(}z9} zM%3E@X`oNFI0Vl8DMC*NDV5noLPRrsNUc7mJ>D04WH=&pH`!4xV%*D$@nj@B@p{YB z`VixmPDG!;P+;w<@u#+!1MKGH7Y|Z?iB06A%QF|tDrH&G(IGG0OA0n`QN#{68?m^yN`8wGYh{e<)?L znV{0o)Yr?wSNMSkZTMbRPrEAZ!RuwjCg=RYWQot=2L*KVZBSx+t9gK~)EE+=G=a3L zm;Pnacq}BJZUxa!yFH=?W4BE01y8AbDr@Ztu4nO-8@&M|w92PsDbYC^{BCX9s(|Kw zR)Yoi?cy3=WhJYR-H|yHW4{K^UpWrNxCf36jmityM~2>ju6etygbpt4564Zi|5p1- z1=`FmV5UeNoukcBfbip1Wi%;t9&d6AC;xc-6gqnMSs10?;VkapEPQUTf2Wa+I8({P zrd;jJrC8lEPiJd|h}H`{>JivY%lhX_S0+QEGWyw|7X_;s`LnnErk(70 zkue_Ipfmn-_2)}Hn-5X#HsJJ&VVO$C85b{+_EGGu5^XLsHvJ9$Fj=rz^U8#)6#l}@ zOP~WdJJOqL$w?Je+en7d&i+5P-aDMlKYsg8MC{etyPMj+_KKjY)2h8yTT0E?#1#~+ z)@YS#tyr~J?GdAjRuOy8Dv3>r*q-bA+{b-C&;31qf8aR$$tTzKd7tmsd7e!*|A62G zf;o-?>*7>8{aE&aDP$&$Sh&3`I@8jPYtbKmGG8xR%e zU2vm*fhGC~ug&rw^{%1>Bb(R3;flp!Q$c*+Q(>4`AmcgV5*Wfv2c#z)r9zd9(fmc{ zhksl5H^K|>3DBdua~+!hsO{heilP74Rs=#-)UU3dQU4kh+|+G8#@l}`(p2WPZyDsh zEHXnqg*g>$dWXHh+N&=t2T5ZP7&T*i z*o%AJnhcl$tbVD&v)`>eyB^AupVa|g74`V3iP1}}yJ_O5_3gU^FFfNx?DzUiiazQd z#}-E40FZN4^|!i=x4I-+(%kkA8SeZ`eg^3=PPP1y0wmk35g;o46HX+uWg=?-spJ%Y zxMik>12H04wD(}3FNn6UTk2D;0>p(`1?n}r&AdNepK$?AQcu*_yTikP)X5hIkOn1N zyS9uy{&K`!e+Gv&R3PzBYLSHEN9dTdB2}`(#dJ>dUm!K}!&Nwa{9+#%z$s{3AA0ON z_HR@exQEuL*HXBA9ZAhgj&YiV5>ZznM69OmyQIzEkeQ(?N}l=SdB14kzchSzF1fAu&?szUG0@*SN@2l`~a+4)4_zrJzgJ&NC$ea zc4*G6P+`m|m&ebopnDeoAcfc)K39%Y^Dlo`3AZr{)rH^8^RcJvEg3qX0fQ%~Xx>6m z`mKB4==OAceo^~Y$M0{*uxBQL7lY&duS*Sv@uC>Ff*$Z?8cjjmR)WpuCLWi7ZH9PK zkj0#b5%KgA&;VUY-F^kSMn(Hf*}$DsO9J19{b{beXK_GvG|+|Bz-5(`2T{9Z8T`=C0d zBty2~Mq+O5CQ8K&QVgqOluF;IgkHRkOQh7;brP8aGF_vrJOl^|}4I=_Y*2Oe*E$PSt7$kiSz ziC(OkartIZ!`oY}u%mi}2{3FWKyN%^+RU<6u4?C^^ap-nPL}GK2qFdSk-%yBn7Qx~ z#^hEOoL%n$sz z9?qn|pYSij07FKN*BoKv7_iwfDn*v{Wrc>{q*cLMwt`-`qK=saMEPcaoQ3?RVT8@_ zw^P7uBLseU4N17v`262_@uP@&dFoj(ALr zPMkpzXV&qm9tY3;16@X@V#GWpei6dI-dmthNBT_PKe5WKXR;ACR7(>Z;__I zVqSZqPKymnbazs9yot|LmkCAD4kVTYF(Q@KDA=g-nNUTFNWv8`f_AWZ2E5(&-K^S> z>a+mQKj;qAvxS4RV;os4Z{u5lWMKMFrqk%IZZ)O>>-f^+Tb6AQ>oA`I;404drn1^$ zq~SlVPa2zRhYd5<0|#gY-ybHpOsCO!ZbRrb*{k6%II<| z4PawjbsrR01R%!0_;qj zRa9SnR3fT)eWA)^ano&HFiO2g?%Ecte~K1@3ui&g96ctevm@|&)h$JI%x@Eecy5iM zw4n;Vu2LJ!R>`_7Y4n=uFVIE(XNcM0o1+O9DtC<}?JeIlO`U0y^(e0?ARmQ2Fi#CH z<=D39u&rzFP$33}*6e--6g%0rlCvm9zI$ripe$}IZnAlFDIt}k4_qWtr9X=QZ3XS4 zqdWygt@WC_N=*d`H!KxA3n;UUOId>?aaAm{zH)-l5bl+0=MWP7-GZHmU)9ASkGDu`@P|0K%v9 zN=R=C;)|K|97F~_2Z4V!w`rOYcMOEH*fabu3hDnSKTj#Vp#w#J^ta}iw^f;u_sMckf9T<`W$Xst8CtP3_Di`P&^TY6QH-G!_4VOdRW! z7r*;arNx7p?b37|$S1!>LDy3e5j+ID81k5l*Yr#RHU<@mx7c_C59|*qBJ1gHdBMJ4 zmsO{XOZ>&w{nu&|EPb3SwTjtyuFaa6EOF|JP7!e%j~n3~(1oYZA>k!eKnZ9CwT*JP zeLWBiR0*fWEwdND%bdTV89RAMJor(I$0K6)(yz5hNz-ijN`Qn;YPRr`a(7zvB)!=x^vrl0mkrW(aj|Xs5dfF}{k0LtfePSHhRE5T6n6Y_Z z`5%gsl=f6U`nC@?@ViC<257gV^hz}qouY5ZSAj=0qVl(d;&lssB3CGvS5#()5BzTu zj;dprMd?qTyxI)&Ui4m|5*?yVXDz;p@$>n+aB-jXTu;w=ZsC|H*wD=gn_|oSB+iE~tkWUt#`>l^X!fbwE`jC1d_I(WL_(ZDQ&Gc%2+)5n1#r=XO!UxBZA7r**RMC zrS$)$dv5Z*nq|(}f_We=sg@;C;eSO~O~U=(}7@m}9WG}-M)%)V26MMFB-i@(Mm zGOvxKI324h%JNV_c&$sdn4j0`1g2v^{s`7SPkBedKT~r8PQOBUW<+pI#_$*C4+Va4 zWK7ojN$_E*Q&d#QQ_zKjDD20{Z{Chz+Md5vog8sIuzE@xA7|e{u?Bzvfg@F6 zDCpo7~_N1#SXi8JBp);7S!sS zh}h#1ntruzzXF;kW-s*3cmdxL?y$zrAY& z^B%d9=~XO+6yl_#$;EJ1-4jGqOYLb(09&)M6g#{HHHJgDhcUCUVr^R3#Tto!k^%-AVlgGxd>&&B<%&8M@E zzms-HM7v9;#+`lYA3!MBH}^VIYi|Ivx7xpOxD5@o7|>A1)>nI&h<=4) z@8^G-iw-_%^{lV`NJo+B=slSHmI@Qo^%?g7Gpo>Y^v7m-9v=t!y&=?i^3TW`WxxBg zw@|P-ER|}X^_t*FgTAiYCiPhOPlhhp)oL+TWL${>6e|STJ45$d^ODVVfCw8lEX?C4 z*zLlsd~@O&(s9jqh#HJ^f5tXzXr*F2w6d`bJmF)F~cbt=sn)m)gDf4NPIO<7IABo2!3CJ&e;HwLP##OB6bK z)3eZd$t<`4G*vsoRQ)jzRUkV`Al$iH@s+jL*tgsVO5Fn&6E@g-o?MDlZu2(D+$DX2 zDyxP4R@y>nNwm#f9OQC2-s$|=eiyQ?zIEN;TYYu=ccVFE9wp&bKuL7Dr66Gh2qu!{dD!kq}NIwVSG2Q zMiA4`bofPwkBnGCIOvgbw-?*@r3=eCh42kQk6tJ84!j83ZPm=q8gMPSHB?gl4QetW zpURc-07?e%X`UAq;hEUBAlJ5Q&{#7E<-$5 z4CFyNd9bl2TXHp@0TTnY7`lEiyn{7O-B976xAzQHE!c;{f!vnNe%uXq9?QOj4b_=D zgp^hJ2e*kBsY9`Egewk5FV;RjK5eaTs1B5#>^B)04l=jU$zH7vEQn83{SImzMp77U zSf@B9POuYp`Ar}bZI+*7ci|5|Esv0HG)0Le;JTu62XfXhc z%o$nN?frEx)Vd3KfN3;5!JzKadiXVj(!b9|8U{LKP6B3QXIMci%hbqPZn}%GqJREj zzM4A?woKXnH*|EoqT4uC==049cbY|Pc1xc zfE#(3jM#eC``U7}C65%NdAxmV?ded8O&;lwk@?aZcJ`BDM|n znSM{8*>E@c&(lbsXmnOG#^amZJ+C~ zKK1hw@E2esE}qW`|7ZNOJ7B@ddA&mL#ivuJ$6drSmm={+n(lxcN0yBXyF(%3BX81M z-fIm5`S{zo(P%nEGXndE3GUDyd&)S_rH%lr;SgQNko+y?6p~7ics@TyxWQzm0-`oc zo#w{I1AbhX8@_sy^ZUW_56@1S5ptn4I<$W;iPU}zw5Db%A%r|ir!IrZM8-sI$kEy= zzHp;e_4*~fZr`xw7o2#O3^J`)7B^OX<_rvpJxyByV%$ar-#oj(_TEROeRuZd${4+~ zhkgX_8!!Sx+@miZRKGg%UzZ(2j=r9uO)_iY$fyJ4ZlWPxt|)kJ=1AE>8OBnpab!Nl zr~$zD^qUO;u|W4V$E-5)S?TcU;mWLfesxF1FvkWPS5~<4iDS^YY3A1PP9M-3G7OPX zxYfo~tPr)Qe9eY4iNvg{W_#ctr`K#zrq;A6_NKnsP|lS;x~c#oCq+`4)9O${?oPaR zWshP<<|P;Kjh+@d!WA=0_+pm}S*99BfY8Vlk2LLMw-+RcW_;_8)a9@! z5tJ$D{qILUJus)4*i}>pM#r$ebzCaReLK>sr{Vonk`aRD)=<7Vs6KYsK7H&F^cOea zyi^qp$P5rB4{LoY_Bl#oyU?uNGOwPZwbIBPzJ7y2D+xsYqrSX1{!qx(~v}`^~kjH+UyR50gxqWDG+RW{N z#3YhT8v*N8z6QlZ6T>(YHdSh~66ldZl?R}(aX~m5^s#nQ)L3BWYv>Qq6tj||9!!}V zSuS1WG}&(*&Mmgwv%nneQ0(dmiTQS!bCW`#aoTU+Q?-GPpVaKCjwwF5JQ?w7pah3A z2WQaEK5+hoB$`F&wmvWMk~r$yI~OKdH9|eM>1Whq5rbz>y|4M3uH_QZz3%qF+G~PJ ziz%1DGtY3TRkI;osI`>r4+Yn2(R<^|2Zm(5Y=`P&(&W|58$D${3^m1hT~S}6OQ0<3 zlZWI6f9hq~&1_m3rj+9K=uoGDc3roS#_a~j)O>DzL0`O6ONHSIVk`#x%C zp!g;0?LN1p_qnXHMI*jjNrEFhxN_$6C+_6>0yzRg?(g${5qYtY-Ue1eYoq zwzmn{-}v|Rou~iz^r4?FfkBJX_y6vN5SKLf0M+hY6)2V&_F|cOX@t)}-bgU%>COyO z({1KK9i`3Z9Rls5Cw0fz2FB6KF8`np%<9a$B^j4>8q=*9A!@A1GG&O`NGI<1WF07H zB^VKD`eW!$htlRGg0p>7A^amA%Dz7Cd)Z(;8%Ta%{LB3+Z1yS2VEd@NuRj2S`tIM5wwuWuWA%( zAc~Hkm(N#_Q57)8eg1i4w(jouY2x99-?tYl24h=nr(nN)lSdNhurgeZcVg$QV0+gY z`%~C0KCU=6ayHOL5_3i=4=BYYZhNLKz9K6CeZpaK^RlO(9=@R+xt--pB|Zz-uAUh$ zY*88SHh#8^(S6vg3*FtN0bgg~6lbc+HYfUvtWNFLojU?WGkMX^jD9g&0sg(YO0g(w zC5_->mY$PllQfttwVj(lt+2?AFGW$rfDyrBu}bax3=lVncdO(FWi>O z$XY`y(fX0((rlmQ7sS(&0wO?{UHV*g-+N~9cSSHErrd5b(0vo!}( zU>{nNRaL=N;`i$A*>*4sDF6h@6?WVI3nDQ+X5teAh00g6+%TjdTUISIB~s9PdKa?$ zLf?sR+uf*qeLiyLypB&wX+6EWxk+5qZ|im)V6*ypvC>LVNlcK`BzURo$V#EhJl|+4 z>H;LNkOH>>CkFh#bbUs{Rmf2(lYZv~WJ?R!GvyJNnXy9MdpervY*OHGnJY&tH9Z); zi9T4_1af)L5)%eWCKu~#ci-)oVMWs0icPu~dr^^u;s@EhYc!nt=FBZZ8s7_u;=rSh z!1kj+S)gIJaR7{EdOJIJB)^YCSCBvEhIHi~S(Z4+!zqOr?_hL1*!0o03#tes7VKPU ziz)2rI|~8coUlt>JfacN+vdD}MMTfGY53H|=Beyh+Zp75QezipkFhXJ=xL?3=o%Q=aQ7Enh(6l_y}BsQAQ>I@mF|fNf#H=QM=(# zjdl<;3Tf^26452@j9}Ryza;rguc|0%gKc{M!uAhL+T=r)J1s_Do3QZ{O+s8Yvpj$Z z)Z679r#_0<1Vooc>Ob8PTE{d?*~zKtXic>670Qf#f9o1KR2)8n(0r-Oq1>y@?M`Z{pZaIGz{1%WW)Un6pDJNEjp{P5 z``309xQ`&l(&0YmXx0X0!fvS(Aplx890l=EX&|q?1H+4I19{oMQvH3WJo?gbMI=$V z{b3jpf{fFCv$~TwtIqlLeH8D!T}i4RFS9J-iBT_&Q}5Y7Qp^3$j^%}%mjWgXM%y2r zduE(#xNWk2h;)E+xyG7@Q=U$nMYh)mI6+$v5TY%Vbo+8KKU;eh-TkGQ`I z@(7bvF7(&a7GY5zs?i{t{sPQ*wt^n8rx||>m3Vc)>M1ipMBQ+KUVUyzW52)i1`ow0 zYwpqRQwoSc5x&|mILvj*JU~L@&`>rPb2$7m@GaDvyYyn(aZwoUUecP=$B=LiA5EJep>{aC(y zBN`e;ZZMAW6-uG=O!UDI>_37OuT-5{D^B>v&*y{J>p$L{dtY|<_1N%*R?;ehYHb2o zs0U*~vY~+{ynf9CM9|k?(BBwKhdyG05cO(9-UZ^EJ({!AVaS%U{u;l+T9(g4`?=ap z2Hh8}a?A|r%qhxrvb8CRYYIE-PTTb}T3~Yf_gRqxSg87*hZ$*(M$8WB-n5SGN_trGPM z;q!yBsko*n{?xY2(cIAIcWfHbV*_yBL`OZAsXLpt1%ORg)rq4d5u3$%(llGtPPLwvHz80*`@xs% z^hlDUr8-xT?#5i|aj8kVpVX+h|Ez8(q4X$qitPe&Yl(;k#n;6%!T(|HH$&I|Cj3n^ z#E!_Krbe~Fs?OtN?D1A7KXux@AGq;7gg5+(3sPxb`VTQAd?g!3c*^OK$l4BDua+hR z@x62mUj_vp4G5Y`ev>}OjZMUxcHoc-L?ppdl`flNuBidlscsYBtg9oBaSeWj3K#Y+ z{7Jy#8M8mgnGES{y{RjE_Vf67OP_nKD=FEP2mVsg8VCAACn2?bW1|!Jo{WPEE~&J=d#;Hq3RV zT@LK&#OXyW2( zvZ56(R;{WljouA1KKH5VZUn<>^38E7E9i5d**gKQA}fA)BChCP%q4TYB24^0_}KF$eP7# zx9IfEjP9}?e$(LU<1t0fr$O}DG>9)F6&9zz@ccS7rtA#p!3@#<+JIzF+}SIes}J8@ zEYm9AhXDOX+NKT0Q1ZpOn^3HvVLacwQur z=3?nav9d?wJHs&ea}<4F%bS>Z=Cs=}eSAJYoR@7Hc&@iOXIqqRIHppc{+Fbyu^3JH z(|6&6XTpM}qJ&pil8_KRiLM2O+H9PxNt?0(`Ymp+@wB7FZgfNNzYH^|G5_PTsYigg zqOcHEWOqN*8NP7a)1%pT{!j`U_}Weg)7{t~iF+Ah}iMkA0Q350}LYwlT4@sh>0`rNOX!;Y+hG-Qvy!wP*Wb$R?_Iny6hv`L_*H4* z{z&$~Gm_?B$EIMMnxJBMNG z#%2aoU1l3&p?vRGlHMLvJ!R%(FnZ8d6KmqW_M1&Yf1%!(4?9$I2jtktPJ$Ibo0Mff z^d?Dt`)dv*T%su-S`3Hd4b(B<0O~NM8|_8$8`7mjtfs>3f{%~P2Ks(ZJ{3QTJ%;tW zIND3mY-xi^f9@?V7~;uNixtbJS)@tsMIdAWf(T@Cy+)(|mBOEv|MFXr@hTOmLc8)6 zDxaf-UiY|DMWv80f!vjl%zWxx54=_5DBr2Zx@VBK^NAJjq4)wv?5V6ijy?1M@1 zMKRM=WBxrJc$=cavjqdFZ%8o62M-2i!H`{=*K!fH8TadDZ^D5FRx*cIXI^a@cH zrlDLk@et7S7QB@w7xYNW8}h^57183ZaRfre92+BU&yCA$FNlq?=+G9keUs*7(5F!; zv;(_oPG`o|gHpRr95&`|>AZ@#k&;qG>P?^kW=%$ z*5JqD&JM-5#VLs=er?f#*Wz^np<-thH)kR2Gz(Byg2Aj5-|5PCv$vm6^xExxq<(Kq zxoPv8GLs9!mc0KMe%Eb>?VJgBqqQ=z_CpkTG~-oBDaP=u4?G++1cn+{1t#7m#BXI{ zC}CxEqa!ebnx4FKe$M(IlbpUha?W&h7Y##nJ+Gabhyo|+R|pXdFbfoSX6R5$Ml%1? z1q}qnW4?RS7om?sIyTBX>k=+%qulu*=y+1e%V|+~x?}zRzNC{V)sTvofosKR$k{R!uzaTJY(I=>1bdS2kCuMfwSF{~fum$aUo!xWfmZSc*Ev7>c4oye#TML`Yhp?J)ICL_!P6? z&GOFYL1)R}ZtRr)8>VPZzH4y~gJuH3BkpH5ZSBz2s~KGpgk3ubx?fdnD!rkn;sFo- z3JdyVJJ{^dOkNdF-?s}TWsQ}rHvc&HTJE-nf_Y3^e@rp~1$HFayk6$tS(yLQ6=CC> zH8fdz5YSrSG!<23D)%Q_$7KASauPPf5ag02#J|Lls#nOW&efBrBq5TYi@w;mlj8l4 z09BEj58vk2ovFYuHhC*Y((K63+N;9?`_duJyE>pwTkC9`XS-!Q>-Gn`w|_+{N;fmev}qr-P*jx9Ai z-c|ZaSN7*3Ypsf}jR{DMFH}d(KI$-Y*5!l|)#K<;(d?j=s6wvdqTJ5AjcQsT@VA4H zl5;@|Yx%q`g)je;p^d`9m@A+{Y&&$&e`j}_7YDGF7!PN2`8|4d4UCYt`u=8~re>Fv zfNZGm8xII*@&$e<^<3pmM-9A_kh=Gha9BL$iV+9|v9a^i|BAl`kiDHY6rr=xCtHn~ zl20Bf0yv*qJ!n&~83}0KQGEr$sU}<9z=#SgM*UfYeB3|bZ{qZ^NM~YBXSO_5ukuvY zzqe?x`~-nP@c5@Q=EOC;dhR8dN^YdXI#zma1wsQk*c+I$;O zUROrc;0%>=+xWB)+2s!YdGi90{p5}+gVIV8K%)L!_SQQXH1mHLX(B3>U_?WCtMReM z@|I65GX`*ZxoNelF}>Kfz-z4@IpBY7b`+1jS6RFmSaJN(sRV?!pOYwsoUTBlZ#u_! zt$z~{^(t88h{=_lrm^PQV>LT$MW0jgR9+SRxVp9;{-Lwzz z$aU3Zt-LNFc5K@InH?SZ83i7+p6ijWYe(;Fhfb5&g*$cNzYgDF(kk91LG%h86pA>r z$K=vA>PQZ0=Pc-g-Mj&XpmXt|<};_`Ir>K5NCH*xeJLh=K7ln>l!3!zCaAK8cOV#v zMUIO1SC#kF2DujLRc8czSrBO2PJHRu74p1$b>xPPqHLi%DC5 z%PmArS>^DCXR+o$s($gcH(ou;3@!{MT2|NEJ4CkJ&}w7^;vZ51O*btTkbFcR_WFA8 z?8`6~l1!T}>vk;^>P@PoTK^D5^@SCwfTzv+WwRnBWPcL%4!FCMzN~QT4f~_<;EynOTyb-H7%8<<1l zdCvTQy%_<}%_`N*Hy(w?qcpyFe#^;{Qd0^3RjQ2lX!7A6x{^x1$J-L(V8nrx?CP~k zt|4brty061;{toSDTqt>xdF{~@KCp1K03vOtk%7mONadzswmSaoF>)M`5~?ofnCPy!_0w}=EGA8-${<&x8_lZS$Uuw;Yvvg&@yb;xtDelg6__r%^=x(&5Ir4IbyZ@XF=HoXsbM1^!G2U`Y zCO|^Gcg2xOLSjK`%Yf;Hb2)SULO5_z&ir99el4Z1X>xeCARMnGKOCIUw#zsh4)I`` zXMFrnO~_bOLy@8*ZIue~k8D!Sz!8c`R;cJXQzRNp>Age}udc;bUnrTp@XsW!uimeo z{@Zmv3jfX9T8JpTa6f2nqaB(UodLO4!!M_iFNx1xYRt;n}>x(eyz~xKxCTugc42C?&AB9VWkpOO0;agTxfUD zkJ`c+U1YmYrU`X?U-Vv0_C29wx4_&X>3os_C!=e?`4?=3VTm{&#O2VHocu7@8%~(w7b^LYy3;N`ekubz5LW;;AT&K8UJ=_dlPC@ zH(S|p3B(!pc|JW1K)gCq1OSjDYy3IQMAx z798fw$z@!S?GG7!bmc-}+=VQH33Aht6mn=+_kefb#No2@L-AX^w=t`zwa}?0BnAn-5~!r!$9SthP!K8eZ&_9(lGtE38IEZbov~js(^|e36Bx7mZa5_)eJrbZto`DTKPI-Z>;>656xjj2Ff1Xv`vSI8Hz{aCACSi3yAeKKGFMWCk zesw4KQnZs6S0CO-%I`E56IQJ=4I(^@-o!o z1Z5^wLI~jokik*xGgSQUXE&;GtKJ~v-?sz=J$*j*lE&fxu%1GPK<6^53VPS0VtzKNC}aqMUOIZO;{BY$9wBs(9~o8Q7vKp?&>V z=Q?e5(#+eWz{bN)sXS}Hx#|bSjsas;zon|rxtKl#`OzZFo_?l2{i}W+uoYyzh6>4B ztqinDHN$y2!uHNY_V#Ru6UUj2ENa>lhqqhwyoj;m(@;b@ZOZlu4d;c_;ym4!lc0Lqx|+P z@1AZ2#)zkj?VE*9(+U!neOq2?#B3meCr~SfKh6Cx)xFq7QONjZMe~q^K4;SJ^z@k* zm&@$6G%rU&S;E;F%G1;Z)k#2U-r`W4h$B%+P)0P3@qq`b+e9^GSJM7nDAQi5x{$i) zLit&#nUt(=ixyQ8r7K*%Nd0=#KXm?bOxxT~b0xak6(m?r)LdO`22$U}U5wwI9^Acy zrW&q1v2YAK65Bob3Q;V*TV%jFDwL}a-WO23)7(#sOFn#O8UslLR zPHQBK8QU~exmetd`!ih%Ta7#rX}KuK(6iDKF~K2jq(K~q;P#ppOuzM6JBpO zfg@r1?7tJ?*}30;C&JDW)v~vs1Q2!qY1@1;=1(#eQ6XCwbyw?s2&RC>`!TVz^)+_L zkU{YSoR=z;>1kW`KjPE(CHv~aVQj5=qi*&{UFI))8`%dXaROh$qeC(qV+LGG04B~X zy-j8(q8canXl+b=0t~c$-d#lczFm*;-#hGo5DDDk;|XGKz9f7wLyVM%dYT1c;<+){YPD5D28=5xwC)d>&g zC+1P8JmVpQ>_>g(u%iwUY+eW;kHJR-pEwPhGpLy3o`yfD)v26gO0`!hTE6tQa12ar z*-0vNpl!T9EVQJGpuR43QXGB5>14D=(IgkW-%u;uW^DE=3>wLoM4>)PZ3U&y;2XbLu}cBl2*U#QSK=< z-7=-$Ikfq^mD&`Axynn^IE@6x7q5h)kZvi2MEd@XdN@j_PyF+voK(3s5fds;if_hk zHt3OTuO&SA;F7|d+~e_(*eGIWqg-=ls1cCIbu+5^wJE_oF@Izzl85kYe{tNWhYhAN zZkHNX-nf-bP!3cUPfP%BGEVGf48kqjb_a{scI475X~x%-*sHh*u=;cqG-~79B@6a> zRQlL$Yq2+x&s6W(^S=1}SO9l!?6dLH`)%{$2Nu%?@BA$Giv2yqv2;29Cg1TxH^SGd zCqlXPq02XxcPcJ_10}&8R9|i6k#+F!VOI(K+5-jot*nWdrlqTPwVV95jWQO-^ zi_6ObY7!#XE?z0~GihYU_3ik#j}IxP>?mm8|LWh3i#;y$OQ|pQ+ppfV`;%I+PiE)i z_L$}3R@I{dS8tZ;|3NOn1WjRRpv(HUD!m<#xa?25{)Es3`gE$OhtGh={m&_}H|f2B zJ5%RZOg7u~!6!dsUlyf6o`{M=PT`Nd?9t)J-p(5~3_n)9{ys75o;%z2YHrLW5Ec$A zSEMRwZV5utg_ft|m)+%aH?WWy=7HT-)Ru+3<|_+%eLx-*Gz zm?-|JqyrYxH&jB7%8iJniH%dtGmZd3z*PQO%R(u)6u`r1BY-Bu<$Eg$CE zjQO^TDtn;6UW$FGwC}=?Uo!-5a#O2+^EC@aF+~zCrh>)S+d&tx0TCEQt-qfmmt}rC zMy^m5x)3Fh4JS-PylL}KMIS9ZI{KWJb5Pwb?(~}T_PdMYicUY|W~^J5l%LfKZgW;8 zlV;d<%0(&J?;K9utXuoyeef#uAzb0%mLt{fK7PmLBfZ#+FRkyp561afz?;P1F3hRw zU|d7aq@Ud7kLHJsFQYv%%Q0NC8E<%4cdyW-k1Z&{Py=9acCFw|=9t29#jD%d6+wO* zGc#da=29P0^X|HezVP608BNZ*yuuWoIi_6AQMkq**NEFM+O{G*F)(a;)#H;|LBDVi zC{z}pTFdmfZzn(@0<1Bf0jl3qjMcOG+pNLxLqY$pe>Tsb|M$_mpXhU({9-DKwr@Ag zUa;6QLrs9+M)3_49b6fft$5Sp+-fg za-R|~hoOrS|EP>!sd$Il5(WlN1@xwK$*GpbP^6O^BbORWulF(^oNh#*xT2Qz=FtCs z3K~C^OtU5urC$j|{bTOtPrk@zSxS2Tm(gy*a>6A3ZgsK2!#t$^06F>id#A?iSHNF( z;Ol%k;~@)!g+kKxRv9`UOZK%#j8pp77SjzJGn4W`VOM%DPE+kuzKBYsQn73yVd>Qa2oNPemDJYV!%yO<~b5^XgG zv8)HrFcNKbP~y-*YDl`t)_@Dom-em4$s3d9vl!iR1;JZ-7L`L|E_%=pFknpjB3qk8 zzC6l;qv8h##oGTv)_J(I*}o4vc5Aj2Z7HQjQLAe2ReMy8+M5!4Hi%GFMU7T%BGhQD z*n2CgMvM|fY=YRa=bPvFjrV!K?{VCJz@6OJ@fp`R&j3XV1}_Jmx3>PJKK2!pza)P7 zp%{KqxXxb1TUT!ZQ*Z90ySPsz&~^nLrEgp~ZMSnpk)_;R^{9A%DLzQQ5BQa`4*+w> zywHdzEf4*S3)|3eCs$bqU(3iA^E)O zIfF;HVY4wAeIEDErMSs+v%JNZSp%CEd{g~ev4vK;H^g>iqK%`A*!RkmISB1Rn{#S=Iv8_!w7}KA;AJjegfoe^JcP1EJL)cJ@e&gZCe&r!4*pV zX7EdzZ@T*|Srq1sElBtX#E)#sys{8~-V4LX4>Zb-{8R0G5eBu4gbw{dQ;d>@J>Snn z=Tm>4zau83T%5O+*|~!l^O>oGKm)?HW^`-~w#XNF7N7)6#M_h&PxvfCca&tC`<~5f zbB#nLR9vO8i+Kn?f(*mt`=Lx$WD|nnq(RWXyVE}sMPE3oz9{)nt7OJY)Z<&`%DF0t zexqjq=<95far?Ve9wejjYUe%6_q1ANK5i0;BK0~K?svlrnE=T!?~fSb{zd1Y7rQvb zxR{#>-aWcSIa%~^mkHAO7)WusFS&m z8T$;RV-KFR#d9e5>fKF)r$r}JC;4Orj<1UX`HEx^~6CHRFi0gtIieuSmJ_rhN$yw4D9SF$~hXUgNd)d8H(;}8(Q zf-S6)=XIo9L!{S(1?F?0o~To7x82>i-p|APe$D>Un^Y3cxC`>MZtri;<&tCvHSX`0 z-T@1RK?z6Em)curviD`8&L$4a9V3P;gZcT$XyYG3`_M1OV!|)6V6zu3K+^{##y#a! z5vYb;-!Qp!NA&iwWuPtkt=O6!rU~-piH&rH?9DvjY_0DR(MW8`NK)x+wUVEC9F*9x zs<_XB-iJ9PBeUZe>n75g0quDF<}Iq1JL)7do(%%MqkNDU)A}3 zKd>;h=K+I*L&@Iw4o@R>#MEhJn}{KQLg;1nZRx@xK4Y_VhxIW+Xl@hnM$k}6A?7DMo5WiBYkA-tHm+^1Cl=-q{-tq_zpr=RQhd1; zy^U=EQ|UF$9o=MZz*e&;M!?jMYBd=qKmqTdwf~gr@vEZuf3Avu!Z3I=|MZ>=-(ScF zrrGN3q4kl=2V2tQiD^U%1EZ^arz~M#YM9ogDKOu^YncemypA2UJMr>!*zoZS+7|2m zIT8H5wLojs!kLdwXxd650rk00k~n0Qpzxa9MUu-qOgGV{vi^Glt6g1dx`c7{w!cgY z9iRB&X_i$hBg7LFBzRXQ3@!e7V{Qzx`nKHnhoFktI!k;2qu|TtSjN2CX?sqE!%oA* zw#sY|xM~#RaL;)tqE38o>hYvgJbNMm(>xs?@@NVVA^He+jO?-bGK5m8K&HhRJIBA| zHul`iyYD6S0BP@V0>Eud@VQ3i2uz>$do15=@tD0=AkvueC>uSKwt|3fv`(8kO5p zZEt`4;p6(x$FhtvCP33*5vdnypziX!$7TU$^tv2G-J!r~7a%{D=Juk+8L|pTWY)lF zjjC#J>9CNf@>5ltt95e>60C&$Aoq!{gFNUfvW^chFKO86Z0HW^?IV&Tm&}oikCI|t;v~It3-j^AE1=5jj45H{`816O%i-kg&?g@FuKeO%y)l|PD zm^aUnWO-!X##@f7&&!UacvauHBzZC{>0Z7NV3RWosuYDB>yoXT2IWHP?RgJ79|y%$ zRnmOZc%9JC)EoM4kx>8Qqt~F%%&X98-X)GH=WmPNus&Hkg1`OH%FBm}3QQ~M2wcF! zlQe{SL5ejSkndY-jBQb%8|5boK2Ma=>~|$6CU-3=e$Q_MS42H<^!&Ypj%|NeR>qsG z?X+2|hN`C>aD^8Pm)nGpyG^H+>b+PH}7#fNm?956~I1s`V##rxlh>_d}?v6owJA>+`az^qKitpgJ&Rnlp(Js z);EB4*4rrgS${MDINy{d!@XQi?cL!_F1C~-8mZ8t6Ik|nSFPO(ZY`FW3QUmEHLdB^ zr8)NQMdqH)5V2w47LX80e)@&NApqYX6|uM?3De;ZoL0>vuN>qN(8ruMkrx%Hm8^RO zU-m_qCS=JZpFuVSKO|xN9mBAeml=${@tsJQtH9XYhBx@zrdIGdeg_I>VfmYdYxMO3 zNED?FQz0%&5cv6mlN;gx+<2DB*2`uAvUt>hTZ{@Ij52-68`UICohWiU|IY942#;ZC zO(-%hIsN;_NTo0^Tt>WUXw?AnEU~%+7lb#N3H_*del(xk_&R!-_>Qo<+hc&3(w8s_ zmi0gvd3<1Pl>GXp@9Pre{0Sf{i&P+?6r!?8CC)g=5>J>ouLrqI}7tvxN7!KPO2fD!I#;S_$BU2tY>C>eXr{Qt~yAE;ng0q zRtD=gBD*2)$s{70k*M!V2#V1p#U`3Tj-P!Fr^#dGN_^!kb^Jx(#VGJgotGD$3nvDD z)rr?hSNd}_depq&x#t5)o~Ea=AA$(?Ixk4~&caI!pIvyYH>W`6g-@}S`*pAuLw4yA z)W>2mx8zn}!JpJ`?Oxhi(S55P1Ee$pZ$I$?mMeY@<-VKuthag0B;%dD#N5j}I~93C zR_CqoORTrQQ}zQ>ilv0AZ%5)hQ$0uE$vpfMzD!5^%v(5;BF$of(3jj-9#8*O@K{Ti zt!LPp8#)0fj$C6vVLrIS%=SaKS z=BAPRAF%aL-)m(=;V%g-?)=0u_1$C$Y2|Q?VmN*q^!)#}LBRSl#5l*hf4Y^%wPgiJ|+z^h0~O)YxZ>54r8ER!DEyiA0%_uaWSY%~=g z0!Nx@&9aK8zFBO?gNy6QP?6ih+JQLA4&e}Pvaok$c>iHzcnnw^;C1o0Q{!=358^l9^5|PDrex8x8o=R(K7B}oeBNl) znWkt<1Xz#1tjV6kDf_sE%#Qv~p{>s_aBf*GaYt&^Jg;|DG2 zAXUKMhFHM)-s2#M=(&w}Gk8<@#HnKUqt{n-Ly45O!ChV-J80;5sc+>yaGXvYh}czmxAdz2VlmoM$i_T)U^cPOEv#${2Rs4n&;Zdw zBz$LG#v$ij#xzaj+YNSTfqbE!dj^s4ziW1v+TOP@u~feb3SJ-tan)O++P)>Bh^!dM z*<0FfH;Qh1Cb#ZKEXyGP%ksE~P!m}p^$uU6KU6~Sw{PZ-KAdHNJi6ZzAKs_jUdGnC zb$@XlT05GVr<-Oy$1!9b4-LGdAQq8dZsf)khNs$5>G3S}8)rGdb9O{~>KPib@hA6R zCnQk>UixY?G~DODCz^UV^Mt}PWa0S^B=Ippqn~)k`?%oQNdgylvirp%`X*(h5-*#qb$Zc%X+C**Qvq{_NAqLmpkm4W!rnQScrRt=KPldw6I@ zH5`};z{tI`!t12)KF0kU`TwWg0pIOV(M}LFmx(n^b{YSn5CvhHsA&Sb}WsZrvYpOBc23R;aN{qB8mdFb>5X`D(Gay&zb)DCB z#XS+O==oz32H`IeQOleB_F+Qg)`*3K%=~m!7*m)g_G)L*@Cc6j3g~m&;UJVWzug$# z=yFc*T78}RV!L=QwJ5O6UGTWhKJu}~DAeqL@UTYgHmx>c{!XoNnR^#Akx*|=_?6y6 zrVo}r^WM+48_9I<2wCA>N{6tI0R%@n>@wKgMUloN|L4ZM?ktL=^?GV}aFr5Lg;{TtAlx&LtfHSUVGt_CI=u@)i2 z*t$j@2dR{D7IHELNR<~Yi2!hgfr))9qas$h(O#TGA3VVLAU-HVe`MOzc`Db*(fz z`wg&xRJ}q*(7qi(s7`$Cv-^pW_bQny0j1%$`!!n{EnG2*-{(&n?@XHzFUD(>!a81C zJf`5g-=zMACC6k|YnnCbWUyGWC&Bj>6(;5O$89dkJAoG;E{3BJW>X{z8Vj=@h(0`f z=K&!)h#_4+sHOgRBSKoJ^~vM$w^l1>-#h*7PR{(UqhTnYwx>*Q|Ga0YPwOKSO`-se z#e$ZAdT!|b_vWlBI`YG5RpR&@m`pu|!V>6NX8|_e1}3RiWv}FeB4B+}AJ%jl8ho<_ zdW-ofXu?Q~dV?>6U6?&+hdQ;-Ejmv$AR(pPXtjoj%!B-15a&+YN=ST$(j>%5il2guri8FvkPv zfd>(m&PQfxRi}vY9=aKC2`K0hVGL!wrXI3Rv7X6!GCuxhnAgqh+ed1Z)JNL$$i<%E zYS~d(>p6a?0Z#~rF+O{*VLE#K zvT44wq~I^`&&v$jUczIyl~6>|MK13CI3tE&KxrTat;E7R8+0NtCz@&Wcrhq!CGlPySfa8zRIUq2feeD&;(32(_&$`0Y2y|F62 zaGDL@epS~@6(9&tZ4BA5tPt2<<-AwrKB*|Xqr~`=0rQ^rHiGBU7x!e~60CbE z_pw^XmUCO0+kifk1d8g&!f&&;nce9n$sh_oAD2nfaJW+-$;koqHn>rOz8h5^n^~^&`Q+J6<|T5Ewq?G8}5-PhqjH_k#yw3lIoV@6r1Vwq+D z!MDM-%5VrkpRIG&5Ip(t33B-ULJ@_SVk19&nh*0I;&lL!ps6De-?V$|9k@@pqXjnL z$$0HEOtNkxyhGG4QYMP^g*}Tqna++T@YBET9ErIDEBz)roFo^8Fm^yQn8yRv39_862ktiwo90B;>Ue5_qx@yl3WUEklH>a4Fa>YCjb_8YNAZGO zfi{yIZ)hItC(BV4AD#&l9THsR=%u{kNDYaLVt+brwUR=LdCh_sWK%OoV=V9eY}g^v zz(JPF0qfr*)Xzbr~hCeq6(~ux-F)dH--e1;y z{lLjs#hI8(&dZLGV$e~5A@RcLCyZYR7!R2ybKAadFORFFV03!UlZ|`nhMMz2=vaud zB9xg*r#c)3NG-SQ=;m88D%Xrc1onYzjiLOLkZJI+yt6!aZ@lI7_aX({oB6qDE5lLS z4#`g3J8kWnR(R&)v|*GGxj$1LO_i0ib#*m?Eb{DqS7ZQ%GVcNpsi+QH0?MPa6vFL- zjeq74!}^5Px#ejOBuk^uHDaKDN@{B@cs2w5bU+6nglC`=0Awo*6xvgW(Dz1IJY|J8 zYFHAnRtCcQEvMgajs&))k^+WAu=2`w8{ZcBCZid3dyt5H;OPRg84v4uF7HJZtB5-c z#W;52MlSFR`uP7Y=#c+5>{2qp@EI3WJ`af z!_r^g#h1_~DKZy&#opM#hnQ7SMqxQN-jtj2R56AM5Ob?Ahn$LH>VCu_%XxZyM$ zVyJ9Wo3q9hL<{?@tj)x6!R9fxpCldCulCS6DJ>j7OCe5pVla6`e4ltCAIr<-`0Ec;O$&w2ryyG*~3SU%0V;i_Cm-KRNx~v z#Rl_`(TAh{5zPJAQ`QDiy;563iJ>Za%DW%LJ=rtWB=97(H3{k6tgcpGuz{4ehgj!a z1qCMhmb2qH=(AGE5X!rFSK|%se@qVvx#ER^zhVJ?4Q$lnljd<}N_aI)LW#C{x}@E- z9QMr#*^fKp0I4bK3yV}xSeasGts$fKd7Pym{_n(2x_>gc;J-eJ-qsha$b zP$p0jrLedGA);O;8hP_xNdHn;oe=OOn4lXW_I6;LOqNN3Y(lK_a;IaPkV7F5*F|-^ z6eE?H4v+%ODdG`}4a21Aw%7aD_;dhWZllacMS>>5ld2DBvh1z^p{d_5ST0Cl+yr*P z{LLGzGzeyFye&Q#JQBRx>08vGP|mcIQHZR%7Y2m{^LJOuY}E#*;!KN^Z7<`ojsAjW z5y80U$^?WhZBoH*>j1FtwI5s99Mr4>9DerVx9D}gFPMm%05q4oN1er2H5@zs03hWi zf_JXirfwvKvX9Sp0Fg7TfpQA0EGmLz`Iy)78WKCjQPrlp7lzifDJWQvF}1R~e+k9@ z6l87k@Uu}}{kh1VMVp@f2te+ANR?ix|LZvlQFL3S_eeSkF9*+;R`S{VAuq)HHbmuH z{DKz_cF)_sukWvraSIs6Gg7Qy#VhXD*o)cQp%|Nm6wRiOqiQ2mnTz}W{*7*Rxx(~C zav!${3ymu<*8f)c{F8K*Jgjfc=zP|$rxdb*+&Z>8zLLQYs!ZXL5KKQ%9kV2umQc;u zh730ew# z^)O8j$YxWfG0g0nz}r zr_qZ0WaLht9b>-##CyELlo~3jEAH7)g$eT+KfOOVkSM;b*!nCy>6J(()20^@lg*zm zs{=vu{G7I|>kwk&3Jvi|M~ERKGhT6S8&a)IdA9`VT}noHGj}OiF?jX5 z6&*`(=xbl;kvdFSSxY%-RSTsupnUQ9AGr&wN+fh9BMaN})_B6oR0 zqf0LHQOJ1+)WQhq%Sr9Gj{=WkbRQly0@l5Bc$i=jlhhq;KU9@=+5rP^uZTTX`;(3DjBT~ zKV<(?xn(@>U7M$8y2=~9?7Gz8df)RaaP{58mF4x##4yCikSl<-1sfAtR$Z$f3(E7_ zjX9tE$bG9a?ZOcF>1GtVP(V*D9_xy9Yo-RA*}waNM;Qa7osZUI6JVZ9SsdEB;Jp zn*JdD!QBU69)NfbOqpA*VjPUtqPf}ogO;+=il0&(=BcDY_%uP_*5kVyl_g;1$-sHg zMn=*ZDND9V(yXt>&{Br$+#5Pb3M1;uXt&xA(fh{=ykmqMY2uxiO8;r{jWL0)!t=Bu zP~D4F)kZ2vK|ZvjC3GTiau2*_Bu>~V%-&%mr{)v-9c9?63& z8syz)LI9%weuSVRBSQw`S{0=q)~*fV4!Xk^6XteULe4V73&^N4EOx72t0;Av49i~H zd}sZq$~2`c5qiwSf>D><=eQ_!V4yqm{b&8;B!-Rr;c+W;b z0Xpnf2GJZjyyxEMufofA0~ zW7DX9bh0Y)#*EA8j~pSIJE|6#p$r?i7%&SM-7qt5AC69&n1?ih^L&1AY;Df3v~RmH z?3(hXJHGx>8e1qb0@ss=PxfA|6JFW=6&bn8*~iy^a^o(R7W{2J)Gv--^wr_EQRIi1 znU}FeeXMuZuL)&SRN!dXq5&kqA zeswop=bcSb<6?9|Ia2iNomKtHFxDNp;gXIc+=yVCU?6wV%Y8rNsTX^qF$#JeEHVj5n5t&?tjrCF_<=WNrj)Dm#X+#sk$<*v;PL8(lduYC9sW5oDG%a z&^uYq^JVgmVbgczyz9Im^R81}mY!MyRZYFf+)f&3#$Lv$&%ZrqjwYy8OP?f$wD$lW&A@lRrKmBVJ@^U1YvsU4*zkNiH9 zZ>xj*t5YxbZnSIa2P&I@? z0{%<4&3=*8gg)=-ouysmA zl+Vk$w!68{@DnqIe~mn_6U9a@AtRbFtqF(oYJ(xS3pq}Q+R5KitmU`p*9}B7L)%}y z2emi{4UHK{w%6y#{%UMKw@_Mfl$sONgDll~)eoC)zYQx#R&9sX;u*8<@#-*a>5de# z3e;UYXbsC5wt9>Aob&&#zG*-0&d|GyeQpx#9EMr4m}L7YmTHlPY<-6hu{7dyC%K*e z2e=KXAO4O{hp1Yc!x9wap|Z7d7|&H%6S&dbo_!{3(#7MEi{y2G_Eykpnoa}cGzYa+ zT>8A8rPk!lgPC~amF!>TYzMnMA??Zs^gYlEqM+blmwATywU`x)!iqfF}?+ih8}3tq9~Lv6~$vKr-DAfQ+CLz$EoI1Pk}WBGlI zk-vv*BBVqzVfuv983#hFLWG5*OfyUh^c$Nl!(TzMw6Zrhs-sq=`ZT8SD?V_hr&*=i zkp#hb%L_#Rl9|iWXiap{pFscdSfi?X3rNff#uel~r#<1Z%~Nj)ud@>L=y)1EkTJx< z#Q<*NT34gFC(WHU+0qoxyohM&T6=_TN`Wq0-L#Zk2cvNw#uqJ+X87Qlx^J_D8kFWL z@RlOA4S@8&o*k$Hjpwq!`@bjM{&ype#_*JzfH1$WND=T4xpZd68SYq_;?N3@6kSc2 z_35ocx%}5wie<4ZoZ>VlKChFMr3)@nG}VhFPVTKgr#6*dOMa)WTrQ#wL#=>()1Ua5 z7mpp`ejOb{Mg_vHoSsni!!=|)RTmMpVIsxI2D(v6zm}BIi&{MB_;Q}YE$*& zLM}fUD#5vWd%4@runj*tuuzgB$lpR|-Apzs2$nl~wjisTL`GJR>dJuACtJ_FvXTDx z)_ZHU%U-KS@6@M6QXKm9fY`yf7(bK@*kmVw?EvEc#UvBks04$Uv}CUJ#?RdPmVhOr z2^W~SF7!^W-Z_#i)Cxwx+`7BN~zb4OYHe!u*xt# zkj%pcw6V#Vg_`b?f#x^RK3w3Ok~!mgeZdfMVPa$6{Kdd{0(CUB#y&;tj}PlAHU*g^ z(mak$N!baOc7ExNU-KFvA>dbTyMs(^5j2)w!Dv z$VnipXwu*Q#YdOFwoh?jjMweMIH)*L|B<|SnPWluLy9v04$ge+|MU{HN{N7v2YLG= zi6n8B!=`X-&HmxK@?z_e$Gw2?YtaEmGj{kSU@!0#sLFZt#O-V69w~7YSkkNRTvn{ma%iD*s!H-8cFG+%4 zN3(D_ZLyN0W>(IP4AYB0p`Bl$ic1FRKiAR37>QbO^%R+2>c?CH1NxmALpGvD)@T-V zlzzh}>h&P1KCif%2UATOR&Xdv5y-0hRWKyp2h*Y zXY-KGER6?7P?Ob0qiZve$MmpmKz))}oW>`adXraDh;7U0-uA1CBvm75(W^_dgVv0{ zhZgY6{==Yf6Z+*)CbPpc7lwCS0fS7OmuoIZ7lRNVxt*CLPrv8>!F#A54k^hywSD(^ zuwtPWrJJsH$Mph`qK^zKV;OG{oh8)?n=A!ohZ|dsf12RA-CZ{$Dg6C5 zGmOG>o<(bIg&(2npvhUG1uFvG(|`zUcdkPG!`;fe%Ky|B5?U%{dzZzvevYf)%4BsY z^#ggp4QS}r%b%Lu8bDYM&Hset?8?4+mcLz!<;ARC90602U+n>3kwK+aT4GfB+7q|^ zZc749ojWIBhI~&23CC^R=>A8 z51j2Jt@d!^A(8BgMidMn;uXo6lfPyro^myywNw5he%eh-iNb?FFpkgf_zAaZe)9m@-b2tK5zLU;acZ`(X zZ3qCJjv_-->tY$Pa5Co>e7^Ow84Vgq{qifBjs@oXJkV zE}-!Y^sL@9HQ;w$u=`{idf4<9(CL*W{d$aO86>-!M+MM$U`Mx}Nx+so8wB(+S07fT zj`|JqU#sDQgZYpJ+5YIG(;BGW%1%j3uwrzs&)Y^ZQt#)v$AAhxXnn%^s!03uHd!2)OYA}rNKFot3C1SOAvh=BlcMl(7`Vp@Ey$M(w>w$%sdA-y-T%Qnv7Ys=g{_D9;P@H z;B4BBLSM7q+9iu>jhQOSLs(28^Q#aXhi+dHvr7S+oF~OyPg_<4-(GAa()?B)?(o+b z5b~}*r9q<1%MCQYdG@Hy0Cbc-_y{!h#LDW}6l5jWve6X1Y zYC4X8z+LIb(-CgoK|(x(Nwt9tj`RlxyiNABJRN$4B89RNuH}PpAHbIUp*bL(xW?l& zSi{pZmi+z$fntmU+r-Y1InHQVUZJ5t9oSz@Q zPNDhvusm5^7(yKMW00rzt3a%@B`mf^F3pOk(qd@Hq}H@EIY}D^pZV5q-QU#(nj`Qo zlzPmO`Sj3(nASi;km-q12`ZNG3IWSWKGAAYp6Y4z^0NDK9+VldaqIE>zhdqAW@b`= zDH3<@K~4m_mTGx2Nt1U$$GP954$C9xyXzv$ z4Fg;W&OTdVig^PWK`keKwOnOu1FQ9=j_n#Sit+LOjRU7#_q>s&2z3mJ$;rJ9-q-aS zJ65+2A@^xzQ_@_lCD)2eeN37@8oy~&)jsN8s#?*q(d@stT)TIgH8`!1w$1RSkiXG>f&LAgeaRqE%WHdV z%kv@On}9L!=V&d`je}XP(O7H5YU6H@>|usGqy%t$H^ZM&Y_Tp>XWIu)@ASRP>*3xa zzS^Q|@zL#6y+JhkqOaXkDWOXbHoX z4eL3na$$dgBH6grpp{>T#?CFwK(W8XvjumU*qvU7b0i<=SgU=gU3q!HjpWJ{_gsWUS66DV-MyT&+1sbr+Y!Bg||fdW)VEx_|Z z9DTnKqh8h%MAS`Y#ACAmO>PF}lDD!(?{Md}5XWdpMJvGXa&E7HP6!cdKPF?J)@z?o zq&}WAVdqH37-&otl2_xzc1niG0>YstdgAnd=uIKB##z#;`MXXruD>NW&!#tX%+c_x zYf)Pke#Wa$I59%>i(Ms^a;sNm6pGe75-Shu9{6vQGj8Z8TggkdEyFX&K_(u~K>iWw z5~calUBHq^?{k(QHL`Y0Fpi4rL=z^Sv_90M9hhuyn{0GFVYFyu-1ZgT*0as=X75b1 z?GD7{4!hydcCh=&TSm#sodO!~r4;U+%1d)thndNj9q5zArIxJbgO*}>!P$iFMC0}S z4~^!xHVh6};a$%V%caPx@@ZS#*sv`+tBEXl9scTt$& zG1-=ekOo`W$?t|hK|+jNgLg6AH*t!IM3Dc!)`W#0!IgH6^E_3)uAB4czitTep@6;m z-E1(Iq5ik-t6{O7CnLm(@M>uxZaH+O&NCs9Yu8DQu+`Ul=i7!Ys!Ywp1G75OFo{%MLbolaO+MywvnU#;DwJx?Z8zM1jWH}7ANo-x0B8w z>w}i&(l~roCky%M@bH=orM5XY zb|?4Dj4qPa$4@CuD1qm9_b-pqMBknP=lo2iVbZ|tN7sj^;6qneFh6R5xCwFkHzC$D zbmb{k&gI|L?d>yGmf4S`iq~k7&HP4Y`Nw@C7Q^$!_LPT`Y}-BEP^w&}s7Y&>ypIg3 z47lqG8O;2=U(Pj>gxk2v!R8)>EVq&SWvZOeZFC#P_Y|{xSWTF4FAsi&MXyxjWmDi= z=vtYINF9gs)t}cyM{GSBK{o>7sFb+e0#K3{#dZMhH@Bzb8G}^&793?lok@}VUM8W& z>ZH}F&muvai8sR|>pp5`S-fwiR2CiSUL6IQ555@sE-Z^vf(DlRldffmL`kq-dy@a_ zDl;n^E=L<(E@wl-_{wSH-Z8<2J_f2rzmf4RhJSqZ$SNCbuzfJV?%j5gAGL|&a-Y58 z2znhpSEldrWGpFhnR3oLw7wUYltVSKl2Uv?{ozdCw-^$>BM4EYE-ZumQaz|Mug)m5 zbSDYQXf_MCvZ?ED`g{2Jq&JRvGM7g@_|W92`e?a_z(pBU$Sw=K}H6xig=xuAao zj<&M?;(*Zm@dBPhL>&%U+OHn!eKO1(&H2F(#L{L3@5^{pC3oflt4IJ}_}7#KA8j-V zF@QXlYS>4EEwD`?Yez>d5shnOS;e@;o4Uup3I6BC*CFwsp@jCTgDv|Hp{$3|%$qK) z{8Wb^rd;S6W@5VUy+VuRP3uc2Z~_Cx7eJ8ivsAR)DiD}>$*=O%o}gx*-TksJU#mAj zZHtS~Y3}MXMtQ~{=g+vp7C(R_3zNoII}gVmMDdnqu^9*kV8|Y1UNY|#E1WpPL7{c^ znA)@b+b$%my>6eG&u<+P^|T8NbME3(SicdZ^5JvsD&noyrx_PNUc#i2^BuyTyd%m| zAJ&oJT?k2&R$d>2h;n~=&|=ZRVZ&03?m^?sY?F1hr^_bWE1BIgEk$PQjFppJPJ6OM z6XlQf`8(=asPY~o66v-5y<|mj`82o;_)XneR^>nk7 z$H&&H-yELO54;Qz<@ zJ-8Aa{F21)v3eg6(zN%^Dbn4C*II;7w&|P8Nv>D@QqK*uW9BLF$(h9jsE}qpDv5Ha zKoi|s79}z5(t>|wS-ABF^1*<-aT)j~uT5sH{4QG{KQB*Tg%h$24KCO4#w;~VvLIz# zkYchdMO;q6cDT{!3rM>7?A5I2+P{ub_`Ldv27Ob#N+1|uec6FNvJIYpUf|+r_KxGk zGLeVm?2cAGndYQV?9RukhMhU(5=8S`?kzsAEdId=-kRpGiTr%y(sx#w0);Ye%?re9 zzkLGeV^F*d_%<#8rKu!tGR1%Ijk=RtTj!N`jUkacvPGpp@8S1IZ@O^w3vFbq!mi;T z%B$Rpi-@H=UocNrl)mrOvXp9^&RSczJg4?H;MQxRS6hxN#69iJsuEu3v?hfLx;yBL zpNL*QM8ANe%S*c9vJk8{bq{TteMW@k;9~Q2i@Dl={KPwIFq-i{vXHYx^i}S;kdNzR zy27qfp?FGZ9PzJbK^Mm!#6h20Tu{RyDn8;tl!A86@e6U}oEu1FAvD1Lm4)Xf@_V@0 zIpx!_E9l)LP#wkLo#;8^{N3JU3=@}eNMHNn44pgpm`Ry9D3+#WIpn@9_3d$$xRi}f7&#x zm;hVjoBH5p!NI#L|C6BpCru3lF$Mb4H<{<{2JnqO>Hk)UF~OIv><13zRIo1{!spf~ z4|^?C3G$)2s`k>(N@ z(0rS3`1Y=|}HtelT8l%R{s)-my8LHYZu;x#}dM8F1YElOsO$?%*@@R)R;qpfU z4>ox;`q)M&h{J}uqAM<}aF z-{j19vO;uJ$UZc_T(P~XYto>m7fG~PRJ zxHBN?I0l1OORbZ{tU$rhZZmBC{SpRtS&tX2H*x9*A#ULc?NO}c^F1^%yLNImcBjIR zchsmqLjO;7=l;+1|NsA2qD18sDkA4XR*0OJ2s!0^+*TpUDd#q3t&$|6g@jooIj$Tt zMkgfMh8N2@$IWrEv4-(|dc9un_xttv6TV$8{nDlJ+~fIpKJNGX?RLH2pMgGJsY^rL zN}u7{+|R?me86eD|C2}pmjl}W_&iRx!Zh_nw}sMCpGU0AVTY|1sV%VH{??a$wD_De zfVmPHV-HDUqKFU9ti>fnH`7<%Kg8aLzqMTa^*l+24-q3u?D%m&y-SxcZ+v0-2MT`m*={{F|tpSy^1fpz| z7>C9yBc|Zmq5;ie_OBXb_#?N@ji-uXcI`UqjyCA}pwG|x>hG4An`B%%$GR?xbVo0_ z=}EZ=oK2&3AXEKvhb(ekH)EWzy5EwEzBLgeQydx=v|{$$Tpq~+hn^w}9XK=fJX#x{ zHr;t{UX=&Q+tMUDg-PEfb>rt!2lFE@!68ldex953J56E}8_YgZe7I*=V!6GRE1j%M zy%zFe@V(DfFM~H=TB_LAIW%S$&r#I->iyaxblYjT6ncrdD@IH1x&i*j$m{xxmAaw} zU0=1)+Hzi=n4*!!LVaGnBO1Y)TYft&D>r~~ zLR+1x=MQ488XZOfacHU8kU?B;n$&vSSgOC+$kSbKTV;QcYerzywwjc=jc(6uCO=j1 zgOBEcLv>#x_c6J%tt*jhj;%{mHA;yuCz@2?qnC1Hs9Oe4z^Nc|q9}d#xv?GszmGNl z2RuZA2IS`Zvt_EXztugFUNrK+euutEqe^??$gh(Zq!H07DrW;-dsFn{=k0PSRFXIwB3gb(^Gt<0Zu_NlXt8R7Kn!8f{^AeMo zl-dvKRzpesZ#g{|aqVfecu^xWTw+R~IgVOs5${vPyWR&=T{A3Bo?csfX>_!M^Y*-5?K{YoD4*EL$RYh<>XLUpf}xI_)!! zsV-0A+DLj5Ly4V3f)E_+P5;Qu4YgivsgO5=E8_IA%tlQ0VAN|nXU1YKC_jJmm8s4V zzOHv&Iy^=*T9w>0%68~VBQX*BX4ofLjEJc;D&`Sdq?4IKnPX~9;k_QzhcaS+n7)dU zBm3ulW1a~TKG8nJ1Z5u|hLvYS~Rccn2ed8AQtl*UY2SQ$B{E;S2Np&-=Z_&Bd4?2w0XUaoq5{Y=Vq_GXin zPG=&=DyNp*OA5uVLB>kMWy;pQdE3i+?Nucb@qLLS9WBW3#tU|n z10AWIBY9fYxr4@@#ztFjfie)u8DH60`8*9%4Iyn`<~c4hF7qjF!yVb;tW}JcTVC5t zoo2qErsG?TH*7~bmlrrIc{QTVo0XN2W@Rc|JGL&a@vB-9fYY5)@nYUU6?kMh+ou|< z1OEVmVdI?r5jmAKcL-V5gH%b5cUkd}yg+tkudXk+t#YG8xshw8MEsY&O^AvpvMvG} zrXUCZ(<6P}7BAP5S+5k3WnDH&^q(t_Z~_#$tDk0z7mPC0$Mn6S(oN82ueej*lU*Fi z47H*8hu<-4{WpC+pr&;#WEgVIBc*74te2e7#V1ey){vf-IS2_3 zp-cvCyXb@@Hm3HqR?}qv(^`ER#*)?)B(W?ygXznK#dUls zyZ>DfujFornBzN79EY<=RuqDBnwrFQ#DB^Q$tB8FVbtlNnn75+I*>_ znLex<9=~kcwXfGZBfq*rDwwM2VU4e4mzEzZJ8Pcc>2K`Te+K@bcuaxeRUhCR!2}yI zd!s%AWEFF5Q3u_8F0Qam<(L{LMiq#h2|(ohYg^4O3FV_U5OKy*IPZ)U3lWo+y(q`aUJuS#!9vMx449d@2j+_3deLCbb8&Va>YF& zTKmRrK|I`!?v0=Yv)9SLBO%o8g#dG`8*MXYTMzFdnfw8t&m7D63Fx8&HW+07)h?@< zAMFe4j_68q@@4dB`FhhVVyUrm!#N+t$f13Fv=~`^KF=`gtmkyErmOtqbQG#--FgQu z%Ux`7-#Rflb$BC25>;9HqfC5OzO?t9L{hoV<#YY0hP4&Lt|g1)PfdNk1ev+-w>#Gr zq{M?FItZBofmg#uIM2G68!&&|= zFHdovQ3lexcFndFhlTxWQ>ECmVZMFO9M6{I7mr@VWo2hl#WjiugfxXk&+EO+yKkPt zk2iUfLJv7163T3Go!m-hWxyENrO{+~?q{yQss9yQs- z;qOO!pR6y;l77Wd+g?N*{MwcnX`6O)%RMk<3!4oF1uJ7nVwy_l;j~FkbvFX1Q?ILvzAI&=qmS2<=n%cGnp|W~@WADikpaeIS^o#|Tbn7+GB5Rrhs!8+LK_;eghjR|r^ouJ z(;Luyq-*)?T?p0$p>`Y22p2@bqs6B7vfy zi09O|+5q!03%VGvbcTSX-HdPvH(K#7YusX;6^aH42I9(nUi;@hPHL9<+LqbOO9)a` z3%TKboWg0%H6i^s?WJ;PVT1MUnd;_fZ0qlmOhz{KOpr-sot~sF0UpUYVu_#yl-uA@cQj39;8|&I4i6! zzUBONqGQH4tAl=VbN4e+DM!n|S+-j8?-v`wx;Tu1+_9AxOy-~-ESSlyI_|fshZl^@ z_D95qLU82h6(+tY&{Y56OawMjgf6sKlrC$?N62-;r)E8F)FAh!+3}Vb*6)ifd0^L~ zvH#N5`U6fG1bzh9$ge5t!80&*kZqlqyi#qkbb+AyYAixC7OG>Ca%{MN(#i`BNw%EcwJN})5z)daJdyHXn=8SCWMyc>fn)i z-s7Mf%)az)vZxV<<7+szfchD(*BNl< zTYmtkSn}kJA?a!0=gLz|-1!jj16f-1=x7J$&eQ>0*c&Lsdd?Ws>FS2nREo^@=#H@L0;Cl zczfoUB>dqArU<=R-gaX1XGZj%mC*kF7SSGg{fuK-%z8c%dD)d>nAp{8PiX;7J*4(i zEl)Hk$vjFXFmJQV2=2;QKqC|A9ckl1_MW6*HtBE-6B;Fgg%(>*HrZ|rT{@VLl9@JP zuT1%v^G;_hj=324Dc!7@?^ddO*gN#$xuf$yUF7ZE>vpKDO?=5#&g$M^!-yhZpi94%#LatrcIkn}j zHS01Oik+XfLr^^1(40Hcf0YFSD z`CCjvf!RC|QdNio#r$qkxrp4rMVxicj6QI&N8~B8vPsV^jrV7jLb7a}L}Qf<02Ibq zSWceRgHO70!ZlCwLj0OV=lmgFf^zRutsF75JPle*1#Rcm7;IOG_teM8l(uK7pms8lOIA+-u_* z`L*iK!@bS$cni)dy~)i*7@7UCI1(QN7@q)`u5$&0`a@S5IMpa5?%F){n^(Om+TL?6$1;rOy* z!Z1~r^c#}kY?WL-Fly|$z1?;cR{HMig)eS-V3c3k#ajMIksGsnHyn>GJ`E@pIE{hi z3(A|lGS#q(kNkFo^T9-1ZJPRSA4>&Do{lI|bPa%{R*;8V3dIGoHQzda{L6p# z><6U5ZY16nXJW$9vwW?TG19SjW2IUHUD-^PzhZ{-gBgT%X8X6xuH5{yXFgJnqI*%9 zVvc(?rttK1R|A3F7i%>iiUA9a92Pz*k9}2zv;lVMOx_qQKo0io%0?rw-JHHuSQT_0 zrglWgqGhm*NIPmidQp_#QTzQUiG#||<3a;~3PUh_DgVuK*3!8E713T}0Td3? z4bPa^lI51#7yY7v%(U9MZ+<`oS!|7)T{OQ%fy?CPWcHaqW#}avQ zEN6Zgk%wLqr8Dw)kP_!{sAN?NAW;Ye5BnP=Pj9g_rS^&(n)MjbKsR@%gY7+@&U*`@ zL!QyEXKQ$OMk$5sWYIN4T;a_Kp?YvAN}0Ucb*9o*Gm2+K+1-}etWOgvXF|I;)<}}< zo-VjkEy0)uWLjk+{H)ueBWJf>pS{Gp9&3+|Z$?$5P(%CABK*U>Z?wdpCY}Jcea|{I zMn&c0gds#M1Kd(0$U0RBZmGy1`JqDsyOy5L0KEnKSTNN4y1@W_by>Z{p(sQI&IF}D4X|iABp=6w}7HM>>)CpA{ zE*)9-vG@d~OcXNXLx}Gr|7e@LoEHX_pigV{F!9#0VW=={XM_@k{hfz+yA7^N>0Jf- zEAa$4e^yWb@QJU-J~c(3s@4kXc?ThD1talKoQQ^5X-|%4H@VJFaOPStQQ+_)-^FDy zx)zY$e2eRiazc}2)i*!#V8-D`QMddL(=XuwpQejv#3cdSSu;R_zZ7*dLt8b|uen=I z63gz|QUE;{E==U7u6`i!&y^=Ts~}_UnY|_@B6~CiS=SC=vUX*iD8sRFPE2qH>)Re z3e14EnP<$tCP}-YF3#ua$nf0yUzPl@(fH62)c zk>@!n`?FAc!5myqTHV&H`v8#AZEqF&v(Z2mPyJX023SazvF~@FIc{|TKQJt@HOC7t ztNq@+d`*kE#TnAV&DTeLv{WTAt0`ky*bQ5%Y`(35I66Mvby9(1z@3iV=hmXm3)sD81~a zK^AyS;1FHZD#O7V17fiVS0tB6Gm(4Ema0{{8P9si9n|=_`L@Kx2xDu@i8t)2E?8V?ozpN zO3S%Ef2rlG&NaUgBXyVgN{z!nTdx3*0c7{9K=9+$V!MJYK7#(m^TC(D;$dxA0oKEo z+^x@RfEv=s*>-!l9Z7>dnOXf8P>}vj1QnCF3x6WWd0{_Oe<|L&Z+o0|v*FxrV(I92r9U&x zYqE-BJ(o|)H6Re1=exx9+`LVF3$xy92B(F6UtP(K*%z7L&|(7dU7RnA*jl|Qqby~n zX!ev7>6kVzGH|l5YXvc(K6UvDMGfCss<)`rIuwMuOAbU-E59*A_0cTwf?x^LLUv)Wt-=|c6qnfunnm#!`Joq&ot@huj2<# zR~&OeW379qe)Hgd?)m>j2im&(b%`YARV6h5uHVa=)lqB z6S`dQ=1gRARc7$fCwES__FY}&7j^PJHhKt1Q?bEYdn?Y%h(dS?By2NU!RQh=Wjcqg zy5+Dg`*}OO9=jYcpYQC;M(J_$Etm*iJxrM3QPjpKFjJMa1USr=S*0W!ob2`?^&ZC~ za*Gs~);H%1Is&c5ORaDj7rxw&Yl$BXIVvCXEkhMqeCvsW!vOw_r0Lx0Sz_5+mo#5u4oizZirDBGEA z5D8eLjJS@54vlx4&7x|vH79#YPE1a(zCzw!uMUr{_81;g!Z_zr_i2mx#Tw9{6W&7E zdwZUEn)z^y&FwCV(9S-!*14tVv+7ctSVp+qr^?*!x)9?Csf&ps8$I z$%Oe6_|yuWU(Q%DC3JHJKTEM$129D9^kX>(&@%H`Jt!iZq4sGbZt}V|-XaZiRTgl; zLHqCXv%;)cgl`j`NtGQ}OCD!_vnvS_Kho@^FFW9x_iOgkQzn%{5sd)UT)P3MT)k^uO`g;n1(v+@wE6cbywU z765Wmbf&H3-2Q9UPjZ4k@Qb^mBaA+wHe=2h4d^?XI)p1mpd7QA@$agE|4XQ;$;@Tg z*galo{A9fMJNwVqQH?H7=L0tijm-hNs2y7Ns_|aUgRIOA&FJR9#()cywYBbW*x^A` ziB1&1#mai~!jHjsiS3j?hoSkgm{H6q-y{&jR|6e7Ns`73N$y$x#J4pDdKV7i+J7go zT4xyI16&Y$Yvcwr^)I*K2TWV<5mh?@PSQ3dp!UbBu z*pGh2S993iO)~$nxPLaEp+Ir$!I}Dxdqf%xz>xhNaY%6Z7g9WN#C7_h-V}BTfZ-+{75I>9mXIm)`XosP;BW z%*KygY^yn0P@yghMWiZ83@-Ea^uWRwT4ZWrUU3E^$0=SSTk?pSS970KW}f;Sy% z(EKqlj{al%Q=`dwYdeRwDzB}*Z(il22gER zkcx<*>h4Af|B*+m!{U$6+)th#W@J_{j3OeCYbCd4;HqR=EXw|92lIsgh<9^F0-);`c%D0jh!W~Xob6J0&o8@daL@QA^w0Uq!wnsBEqyxUbj0o(y=-a>5w*K z6$fcHKiqGE-@Z#-OUMj@z^S7mpbz(?`tQcv~* z(<^TKWLMUXxTF}gA^G);-`+quA8zMPkFva5Bf~`%yXIRP#xCY5w?rJLNNx3S{*_R~ z)*Nn~SK{HK9($une=YD|>pM!M{hP1<=ejXqm_caLsOjt5Zc6&TVQOJe1#)5XbCMzf zo1dnI8+vG-;~bz8-=^X=`A}a)0b8JIR|NOYmbvgoNNXv|=X8H5bB27ME>FCWZM-nd zOc{|WyfDz*y=mMy7dM`>n-}pJrf(9=#e7R#+wNnbj{H+fNTEI`cKhtZ$X4;3&bhsCKG|5cpA#UTQ zV$LVN^^r81tr~PPNww@W2rrg;?c1udsS$Xs_PPJd0*Y7eULby61z7n|(3sT>`#+v@ z&pls6`MYH0wjt}4M1_ZP_cuR(B*6Y0GvOhA1a*USTlVi{BrR;zUy1(C^<4J7;_vK2 zU*tuIC5?i4L-{W51rS5@=sA;nF8R8co1Z%wVPBcb1*NXf+!XQi8-p_Y&ZHdQ<>TC4 zS5mroA(v{jyHRw`R>#wPL5C8sU#@Kf*a%ZV!I>yP7QH;a!YGwaArKQxK>N~Ox)dQ(ECB4^eqzsey)tfcxo8)i;`f-faJzrdV9i=n4Oyf zHQ%^VKG78Ta@6&8yG%=%<+ALa$Holi)f+N*U%`i0c>x_kWi#wA9l`S!?Z^!UqY0P% zF*i8i&L&=BjglUQy7)1?%#Uf1hr)U?v!-Z?;`H(0p>`FL5#Nu(vtoA-tQa(%V@uAx z^IiYZ_O_Ncy0tL<%O6$TL{b9W;N=srY+SnBUqGbi2go71@k-N>&Tu!$n7Umb(J*XN zkm`5ud?*i6+l7rxUIp5!EU+8+145o(gs%|;&q!JCX)_s%`%acbz0|95PgVHbc<9ry z?}Zj!ezDbYDJpvb-cnNHZ}rHFrE=~l@_AD?#!%K%AQXNP&qF-X6NM(6d2nD)3PWqk zka4?Wu)QZ8m**ZcU4aIN4qqB}TX|wx-Y$U_F`r#eifDPaVZHh(=?j+A`L4c0EWGu&7+3@S3UH`B7E|KgkWQqvIGkw;S!qQrZytCLS>!hZ8h z(xdul7ru=&-$$sQJj^g+JkVy#QkTCCuci^c*o8ts!t{Hf<9}uBM?7{NrdwQ#R3l3{ z=UM2rG*)Pnl{lFS!`DW(iHy2)q2c+vlOyn5VlTVxJ5jPpGKK4L=RO)pypz#gs?!#d z>U`OL?eUp~F#gHpy(t9fpLn?_4zN?2ZwUVda=PMT1G+C_V<1tZ)>2F0t4m^CuV&L$ znr4;nU3Xz$rAjj5*q+AP2M0AN2Nat1f;1>Qy~ZjCs`ZIF3@S#i-)D0og}vVTJN(>i z@UirHfbVOz{QF0vaIscrj>1R%rIwy*vqiAI9n0Y{G-vboCr%0U5c8OTK>6t~rUD*# zq`5Zgjr`5go2BpS>r*p(^Tfo5yb=osPyZxPJJ#a#ASv@}YfV7E7^oll3h`!VTo2WL zZ4JcF9Sx*6m0Lfyo~z5EU#3K#f9 z@@Hz{8v?sT2Nk_3Cq-r6n}6}nFi2P=0deIlp-f=pB<0xRP`g|e0U|lsMqx)$3(LZM z;s69Xwd2HYwDH~Z$N$>n&=1HoT6Zhl+C#<*msQqn9@5B%aL`PAIMZ%Xe-S-;F?VA) zho7ZwlKfe>wx`}YX-R?9JKxW@E-yxpmTNwC_zYj8TJe_X2qXa@fIV_LfTQfg)adjd z|N9lF54V7gO)C7xH0^_1%MB)-#y=v}C*wPJP|fGIprdY*ovVws$)|>4*v$!H`c+Mc zB%QW__jH*{9M00909wkQ0x;MvX=T!}a`&Z^l62H5$-Q`~z)*^i-zT!PluzQ>hYdFQm za4L$yx`I(gcGt<7AiH?iGh@hMuhwQOT+UK$@=ccVgrOwe_E4qxULw?GmRe9OnH;2!C9^ z-sA<}+WcKSy*f*qULEjPGAcUTuoqf z$fpbgZn>Fn9@vc0YK(gh5m(+LkSW~ zXD7Uu!oqG5;~IZIopvX$cJ`uDJA1iV{kHqUg$UsMAaa%)v%V8ZPsO3M)(%i-$C?Aq zQfTP-<;)p*b||UEcT$D+SrAsWe10FRti@R!*kJt3>S>_0E|VdFQU|~&H9^j{ed?H+ zoCX_mup}tU&{CR=1&wKJeH~@4azb}7#3=O-DO2&Q42DLthKA53sk7e<_mO)_VNmcZ-2%AFGwr-TZBk|lJiz`oFU3B zod8#y=ymnj0>DL^Tp1fsg}-O7@V*Zj1*px828{Dt!| zD$P3{7=7bPgdS9yG*Ts*-$>@In|4bbAbpafU>FU#V~2DMo?Qsn%Z@zWZN1`M+5|v% zx}%r(@3Q zyuQh*=4Q3p9xOW`09TZUs16Fc(D_*+I|96-GHtNIA;Fvy|9UKZg z(!oPxkWCK}d};XaDHWHfvipU~7wr8(bA0PdMdQ+QW|sCeU?ht#cTaSKb1*YGI z5oAhc)-vew6sLWN_+Orw(Xv~XK6)k)mpS}I^s>MF<)Rmmj^FFd&LU3;nK@rg?rdic zP4{i)5$dH*C89PA1PdvBd+RwNn9c6%fj24_~VOEwotb(#D@&+sg0+2m5I&i4svNhi+XPn4|jGs z@l)-wF03=@5-^yoGdwqj|F>V15$AzGHb|b<6+A3l0;X}kF~#>>`MHbd7{5FJfBjmu z-Qz$jM*m_zv%LP`v(4UwGH31NVH-b|@5XHxNUp2cwu3l*C@~>=APKBnPdg|M*uyB; zlFgHv>ax^Kz$)&nwBWs7ySnz$35WX1i-@nQJ*zIy=?EJtsnxzIOfX zNCasb88L{`_SQU*6^*enR7>h+lLqoL*$f!yvZI9qyJ2~CKChE%&0`d7l;56C*efzq za$NGN#wlvld5wnj1IK3B4|xH6(?-`n%4$OwVxlqMt}jIbtxRNe_+@4GFt-ENWHF`8To?B3$_3+c`!>_D#Isptj#ESr4wmjLf zyjO{gp_WS0d!9mCW~$DyQvf$QVxp6g@z~Ay^;25E=!hc{n@hxcCp-Ue^J0@-$~cjR zsB(UmS$L<%Y{{m*gmRG*L8HC*+%VGnjaj~(T?!ARO_v)}^2uJUx5&{1$>7e)H&#j( zx7?7H(Xg+YW7ZUH!N;3>y^W35ui#CAlR0R5Kw$Ujw`REAy99T-XHXxAepim8pxj^S=o9>mZ~c5AT0`%q z8l^~o%rzKKdJ2P#1tGfaaQLMLb;>k?HODL*QF#!-pmolY0+aze*GO^_vC8RKNN~Uc z1++*HBcn^&NEsM)_H(wyI`Ix`3}|70wvRQ^Vb$MnwAhqZwf0Jx9H8On?S$5lV<+BNUS;*S0% zM`R__;Sjw+*llsBwlusUT5rQ!h`R}Ic)*GU35^})BWUy&T{z#r#yt_dVZ8H*XrVQp zdedwU+cT_Hf0XeU1x#wZB-al_qQNxyC&nS5dnQ$w4*p=8{_z;<><$DHpoWgukA2BY zr~!p4P+-_CM`1JpEZKcijPAb|r1cJ2{kz8#x9xs|93nw?G!GJ^7gy7=e>Oql79lv3 z1At%V#*E<`&xLHa=67bP$Yud~%hOx{0kOXK!&YJTaN3jk-n-x|tsa0VN&m7Uli29S zL3+pO)`fc7PXZV@fVFig5x_$G(r1ACS25a)GRZa~Z*iBqgOw>tJ%OU}$%d$IQWOfH zt2Fa_Im#2G5HvRbUM1o3nS08$GHa_UwHB zYhL$n?ngu2(DCotKL$TR%=|n?2~+SNN9fCHlwRT4;HPxN*Le=$t#$Jv^0sr+hcE*EX@}8IQ zeclH10d(cTC?23jWV?;ep>fI8WadEzpuATI;@gWPw4hPiSW{w5oUWt`B@vD+C<#y1|m4aS_1ax>dkqH$P#meh}XkZis1J z5=L(sTjC^@2r`Wrtv*k0VE(J$>V|ELt{F~+y?#oBp0f-vjS1K)g{yz9N z8g+X45WN}DJopgu-6QhOCc7b}5yAdD7|cnrA$ir#2`|r{VwlMQC?A5o@BBKEpH zP@6D3(D1)hh7v#3UW00Uf^`xUH~tzPk~xB4TOkKpw>KOl!^6c(o&Z!b^5 zECYvj2ayz`z#qBWptKMit+Nj69Iz#QI?-)@t!%?HWxn7z#VI`xI77CUfvaIep(|s| zzVac$yYyioJ7d=sfR&q7KE5~)oMiq#fc7;*rycPGs5oZsX#Jn;<8VHh9W;sE7wIKV zu)6<(k5DsiXI`OkKI0GKZ)`_fgkE7|T-#~~Xn+b#_jt_1EwSwI1V;L8%S6?2n&kad zydspIGqpa$4E&-HAq_PzB?nF}11C4|-Q*3#|9qMn9u+N?t-<4oH`R6NGjYh614gmb z747)(BZ1nk=#Vse_(eqBuF0=WyDo%o=L@hFpMp^lTVBcy(=C(ytf|_f5tIimREiHV zIt$h@r~Aki4FA~-X#?ZIStv|{y9Ax%eG$iOnuZJRKK<*D|J*E~MtWMT0NMJbh<`F% zErK}Gz<({hI0TLzW^b|NG2cXcLK-u`Fd~T)4~#|D=ny$gDO>ldb`$G*~H)H=MJsj&DXbUHWY9vHL|mTeWXZ&u=R zrK7!1On4OlQ&-2qSFFc@u!Q;=zjcdUdYlrRV-SwsDxxf2@!zl0x_h@GP|1z>=Z%N zPBWc4kW)Gr)oIk22fgqtO}25?4DT!5Seuv+5BrUP*KM2DUITS9ZE#soTHaUJMXn+g zX!4=T%PrQd24*~(&=0F@2g;YbW|H0NQoKFSL-L*DFOIsIy_3O{<2#`jD($%&=Z{i5 z3kOiupH_hezK%+d!LsB{W2}+Wg!>kIdKQ$h@pPEO&#F2{X7nmix)!$rb)koUr=RIP z@|4|}JEoqxhCS(kjkA@qSE9hA3=X}in>{M)D6WB5I`t#ff2VNCnFfYkW-Cgzv7#3M zzr)u=O-iSSY6J|0`@BgcwK(hCw2WU>Ka*|?ymuYc=l)1ND>fFtFLE~XXUJId{6KS_ z#F8NfuKf7}_xit!aIhg@Z3D~;b=vDoHN{QTIZv-y^gd7{0h2Wxe3O4?eDlVDvrcrb zg+Gi`uWyY8V1hYB4+C^*%h@d1a8-mBuwd2qYl3{&jCw)!O{WRwVNd9Jx!|u(9Zeph z^eEujz|>v>ykFM3?Fvzz%Gq=K2; zDYzczoU)M{5qM<53yhwvPjFqFJzK;B3C#hSdY2AYckL=Wz=PnI3O^Pe5?c#&v~Xzr zk=PDH%+;UqT6e&*ACLc|?OZV`qYPO`rlS~ccQ2QI&9@eRd;b2VsAvZbnWJRoELW;u zm_~tvYHQThY#t}G~DwbBud9rjUjy|!^43)Z|j3vXse0uRF_ zt6v0$u9daX6t}3a!JGVX?(yR3fKjN7Zb(!Je}EW>hL>d+IW3OU8dI}kF;hXTw3I)P zqs0uPo%XRpV)0yabPA2wsC)I0j&G2xL5X3(DtPO=_g6iSM(DGefg-vu+<&{Fq)T2Lb@KJmp$?n!3GR1 zSBNA)(Tkw0jr5LSKg0%o+nBgS%_&^(5?`FY4a^}S6#ik;aH~F~7zq>`XZ|WSj{aMG zy7=KOkC9n{eW4}G`Hlqd`c%0*ST|8(CQu%;cvr8^IW!icCsmooNUjUJaz97&#V!BF zZDBZxLo*Egjob~y^}FD`JQ6%REzVBpjm{5tiGWp7Fy!|pR}YI_gb33w?t5XqDlF$| z*xrb!jA7PIf3{u;vq(8X^*DK^*ZIxHH`W>csO`!J_5WPTu`qK@aE+;JpX_ z%SpGtgAO@D_bS+MD@+Ch^|n(&E@TuXxDw_9-Eg_isK`1!cH;=7BXV+2%G|}2nX9)) z{+RJ*UnebA`p-uX4bXB?r0VDXXa0}6UOpR)p9`?6x;^V<_d&vTARl*nIH0$Ykd0 zmE5@qtkLGyuHC0kpZ`x<<*xnN96IZisWDDAlHdg&y-chrY)ot5*39~e#XMo zUWSZUDl>$zZ}_g=ZVYxPYjm6K!Kelq*@A<7gS=>9LTb% zC!fyWUwN+-%PweAdpY{nw57L8a{N1I#z2zxH3A!DI1lKOGTL81uKvTr+~Kv{NxTei zQCpIm%@WFNOHo;yFLs&>u7h?<+*LbD;Sj5W)kZT?=Ee>Y=cNo+LEQJ4N z`R8_gm3+%kPpjeJa=e)gZ1b{YHvLq%&!gv#$Qx~+;5h)HQWGqTu>EV_xnrb{|1U{0 z1P+W&d2V92t0P!6gcHT@D;QI)yD|zNA_KLrgy0JKOcI zUwQrxdPAXSQIiN7KVl&-bSlNiiv1Fg*8*Q*{qYB1o0ZvlXWB+c!#mtrCnIN;0x1zH z`S-FJ{(IR#JIe-(jS(6jXn5XaZwmi$n{t}a_`p&DN13SfM=+mq*-Myhl_>4?6Y!a;ufu zs(&Xie=Wg@TlGiWVFLr+Au&0Pe+~b5KFs;|jgkj$6l_0BO`p+h zsC@0TF7}GyPRR$2s z)8?=Zv#G?!Y|C+&@m$^Ahu`x&*B`iE+pc}TpYP}Vet!u5lzY7>TP1M;#Iu)KQH-epoIH7Q{kdDW*4-=EXSM#G#RCh>QN=sx>)9dKPIfEbv5wKrS!W-6 zV@`Rky-DiOvDMMrn%^8F)F47dmtg`g?$msI#r&9I%$504r^}knPlMX=4Dl}I4;+|2 z^LVPDC)A9367tUlK(_C&SozN#1Y%M0zjy!lYujz)jsJTmWVOuy-u!sH}jLghVt({dNTf>#25)yKs135WG%O-bf2aQ#>8mAf%Dg}gB0?UZV>yMGp zgR)JE+HE;M8Lt zS=JIs39)OAsBjBHf$`bIR)Lo0w!MYp9ogat(Fc6S0}KM%rf0))Sp!6zSIU&sydw08 z*r%ci(NMAf!kgfxn3a%&7XR*HO@@Koba(f8+35}9(6dL^(qQEBAF#`zG3Hw$SMa1Y zT1dbx$iM5V3xm6P?y=BHhoFKkH_}o37^N7P)%hD7^%Xn_`sII52J~4iMzykD zK_?)fKb!B7$&HQihmIP0H8zZXh#VJbOe15u zG7YPz3Xz%+H1@v-;-AVa91JkAP_YD;1X+0S-tG!$sk zrQN3Y^|=`nUxpYad3Ta8#H?M1S{}VRcPJ^u)?C!c>ANOlVZII{_v;8Z2kwv|GO8Fv zI^~cd-f^xbu*}rRASZSH*@x448K~u+_rXH(?-$&?FPv%;F&;E#Hb;vD6TZ>ml%tW@ z4H#Wo7qeR089g<-N31IuGStYHx*Ke?5_fgXhQ#TO_Y1eia7cOH zu3mNCD|oF+Wewa0=79maCOZaE?Gyt{-yh3m8sWJy;qiL~ryY|hP|;#JEZ}Q}{#cEa zIkkshraqd`#~uGFEuEf2o)VuXgc7xChBTU{e)OA`qNqq2OEDG@`iYIY58?J#xFj5U zF&aH?#MrQcXwXOHUzm*+cU!QR8d6UUDnZD9{1oTU%cmr11y4~HVc~aMws0rbnJ@_7 zv^PlbV;rfveSTkkM55J2QZTDis&T1HCK1p(eV3r6kwieFLi^K)w< zWUK#c?IA@t?7EMNat8$|>N>yk&i|;uFkM7uiqF>uuH*%N2$qWz zJn~-gsnD?-XIAms{xWyEi(=y&a%I-Bvb7A3IVPgKScaIu$n{5( zks26@@sVC>=~hhR*M^Q#p`Wq`yWEK}Vfm)Q)O>b+0Vn(0OrP{GR|3=g>w4fTFuIG9-zu*S{v}N;dr4108E@()`QiaY8m_0F*DxDDw#d6sV<6h1PUO!=0v#Rax5ggmwywJ@ z7pY#w&2eRY*cECfhc@^hOOmdk0jhznt04%xD@lGf7kxUTd(N?4WE(Brz0nnz7MIs$ zljocJ-(yztX1BVQ#?!^FUM5)(c9ZW7Vc+rDqP2rHoi6BH@wuC9EqL)A&`P3mmFvbl z*m5l+7#;f1S;FPId_B1>czKs`T5r$*$0fm%!3Tc<|o3!OepIZ5u50L&mE?t`ce#HFPpYfCC_g6lEF7}?8d~56a z&;uZ6Bv1TW=Ajtm0wM1!sT1F3<-A51!DD!SPpTna{AA`uIOQ|) zMsv*B+VVff1-%e<$h(d*VgFA_j0P(75kqrwhC)AiRR?OQG%wd-qzN@6IP^n~;%i6o@u6>)vOGT&n&%kere7=DeAH_d4Uru!??*wk z2C3uH*$Q_;f`*9$5U;P#Q(F~$L)3?z+~vqESu2>1_dJ<#*Ue^QBL|)!{>9{dRw=^}#Co;EoJ>z6??MxM~-XZ=nWb zd@R35kRhs{D4Nu#y9CO@PqzY?{T`SRDq2mxbxu`g zVMg+TSG4f}%kikzhu*a^#%N1RG^aovAasg-NW!PFsrb%c+IrA)aFsbr!T!*Rqy7QFx5;X{=@-92| zRr5P{i@z<#>s!SR&2w)wHeecYy@Gun%nS6fn8B@$=W=q?d`7RUYU%{{Q0c2_!=J7` z^D^WqucirF*B)XZ)7C5SjmuIjIZI>50WA$pdcxg;t4YgMD|o_&9~z0MuNJ#R#&hdH zg)w?fa2A_R#n}tQbHrWm{}sv2(B+S(2d#h54oS}H{3DX%!TjI{V3ZkMU_=aRQGD$4 zORP*y2F6Z{`C9`odPbR7I~BFC6mGgj)f0-%nM-9>&iPotD~PRDohi=?QJ*4(ooCHz zG%3@1upDVzXB_izwB$_&m0_lHr@MjaXeT0!#`O--94*SH3L{JNzS_x z$`twIW<5XfldkFGxwcUUs-Q;o@WH-*K_pu5(a;fQ$E%7;;Z;-Y@$FGWOK7L1)r8ln7fx6XfRx5I-YR(nvO(|5U;*tox3 ziCz3shFH?V4rx`sJ9E?W(Ntj+qpSjO6RGF(dtHmh#H3UiVaab8K2l(}R`>(PZhYgL zkuM?hWr*IJ!r)JTJV1t~6;ti$Nc>op#@1vb3s+Y~)?xEdo|$gk#Lj=0_lLyJ|LTtc z(GL^G++t(#qp6`0zU4m}Q}Dx9wN257ZPBHMX$)HWxC)?ss5n&NR9E1hU+AUQZ@4kM z^j_WmTX{LpsnTspA_9Vhz254rP;%J0)P0YyO8`%)3O3WPt9SuM&r|aWTAsGLSqV#U zOKQ|9XCp1&x#kbf>jDqS(?F6Y z%=u023cuuyG2D2mtn2&%it0d_&hzwAjllLie5mWX3(o6J@4ej1x!TOBZKnv~S?eOLpvwR4{H)(u<%0kvRO3^*&2x`rIa_mVL`OpaOr!(v8U zb>;uOO9gxv7^T;3Je_9B$%c4w=33jo(;dnUHM7O`550{!W)6Gv`W%>dnB1lxof+Ss zsp##7&%Jt9J0ra;%CcwX0ki&U4?}^amhefo`2`0}Gkj)+#H`$241M@4bX{8~FN6_WA!iaLDea^Pr zh0=zfYH)@!ljrhKBpp3*Skj)D@6Gw*Grxa}6#=~^&x0=O!PtzCIYUQD&L_8od*k(_O!Ml;E}S~@N78X|irr^dvnpCZ zIVH-ID4kbRd00GXo`)nUcGpy${~FSbGBT|XSoYm>>hQz|_D9L)DL)yudJ0tQuRQnY zZj}bEnYi#g@VO4SyvMEK+T-b+ywJF%qa6+{)K)T8uu)GMCe|W&nA`s&bhP&_3zb^^ zgbmxKb2Vo7){g7oIx7@i<=c&pwPqo#>XHmygIDW~O!NNU9OOGI2cV zg<9yL-`^vMY1qZJQPs#lNlV)5pZycU3Niqs{l!IAUf=1|y8hBLgITK@J8U9Sb8TR( zOhn>8wL54TyMF1CAUu(WULOk@8irxPLMcS7d z^MT`4Z6E6r5PmOoIt_)nyB$a|1nnI!O(Myf@ujh=z!caE3hU`hR;D#gd6Yd6+dz-h zB7`0nTQ!g3#u%+z59QMLT8kI@+b3P5Qbk%?9aV{imni(#EZTemw4UShk1q0~{H=il zZQ8lLXZ1QB<{L~skmkZ5Y7a6KBJ}$kQnz3R7qSKYtST8|N9d>6&V_zN9B2kxMcKLd ze$Xc(0b-|Ea`y6gUX1Y;3?SWQhp+pq>GA8plGn?`t$;2D&Jdbm$?jo6s1d>H$h{VD6u-XHAQ)XSwAx5pyxeqbM`$iFqa&I$k(rG*v$K5F zakfcx#Ke1f=z+aFuWpI3q@=#}-vSikvw1zn_8_QcjJF2(9tM%EG-z~&8LH{*F1O&Z zTHSwl5?&k25ksWDT++nCuz^J;sIb7`bzlJTp|HebvF=%g;VU}om6wfaZR(533T9oOhIH|qEE=4@bb&rR?9>VS7;mRU4fbMkHS z|8r7*4lMAOYWD);Ufoyq`JiF!R*!LTS2DA{Rq>f!HUYX0kq5d$ZKu^NTbWk(@trIq z!!Y%s0cfA?@D10$7V2i0+j^-lpv+-g;tUp4F@RX*UB~)=wE=+qREIwU*jHh~S(Ef` zvl3Jt!_DO&l0+Z;!;4?vYzFP;_ygAuBgmRamI|eonF*1?fgQm)A#m1A`q;PhG2ZdG zhF#x}o5o}N@v(EmWF9qR)`NTAPUk%(vrg;z^l=OGzQ$F=P5&I2&r^+Yt&!A}ls8_1 z5x`@fv4&ROi=pxT80kK=edpA3Qx=!2QLZC^bT}Q8oxZcty?L$h(i)tcbRjQzx2|Do zoO^yk!o%HZP5ygZ4w2?wWgHFQWXCL7b9bI%en^=4?V3)is*a9lx$f@!&*v^p5k_oe z7VNa(%I-Sb3F$KgeJp9shN+|nU{?L-ehLtN9Z|Z1&72xETZMgDH&mwO&WlBx&1d|` zT=RER?{$^2v9UeafXa;~eCs{mn_er_q}I#>>8j!tC16MQZy;$>etlR(|1-m7p@-XGdR- z5pFv6JXN^TH9Tlvy2uZ$Ih4EW@9y!tdl1yIH6Cb6Q%t(T#jQl!8%{Q8GcbxSq1*gc z@D9|5yMBlXI7Tl6@?T#YM%v;?XV!IDc@CL0k+Ys_c=arK!dSO+GlMdOdoJB7c_kOg zPT9e7KDMp zT^6CAh2vKz=VMe{?o>?7r0s`=z}YS{^FQ?-m%BDUPo-}qo@j1P;UU4oD&J?WeHs7k z41h3qOAzdHYl?V2ekSK7Y_gALr)~xo@~S;%=1SxGV1289BpYE0G5x;j+BnN{#FTr ztC=Y6LmD=dn)}LJW^8ZUGKD$8JU6#2_^GDi7AcWb6)v6QXgV-)l@i&HV3(c<2k{Wk zK5ap9g+IqEDt2v`KAp-&1&y~%@Tm84OC2efOIi#K>m$tuf{Kt9)XxiJbVt&DuzyOI ze(Ugyn)Lw0siG}(T3(QM`3CRir>$1pRIChfX>I*shfQr#H5qE48uzf59V%Jq5Tj|A zS2?jocG|n)m>sCTjT#%T%wF#CQu4G)byAfaI(z(;dgLOE<=niAhJSD&gZ8B2BGh~5 zr$WZy4({-EK8Y@XE_w_szx!=c_VXiAHXrO#`~VimtB|2Sf7KS~&P&#(aUMC%OE()! zuns$HW>E{n_0pI@$IVB0G7DhxzqPj4=dE(Z#vVWTRMJ57zFwWOZ_YlK5}`>KC>Hwa ziY=ZCYt)m6m0|lg3DPKFt6%Xtwx2Wb`*Fj*SI(>NsJ5u)JR$|{@iQM%SFB*Yd|llF zP8@{M0P&Ess?|a=#Zrn`Jik(Q#3F5w(W($;cVMuO$T;ZqfU;t`a?#(T4Z$| zY>t0!oyYLAPbSUL9Z$2p_s4(SxI2^lqX3C-A)ntk-kjBXsZ9eW8_DKI=XOW6m^j?N z@s}VqLi@}dZ*VSSD{{fIFBYr-fPxi1^Nz+Sj-Dpeu$?i*9mak-| zhnZz47bsK9fq7GWLJZ^ zj)c#Z@aL?9K8|vcNg7zp`8}61d_E}fX5)F1jU|oT(}0V<1=Y$x0*HQ#(e4J#weiLy z?0VH(f%UBJrOYHB<0rwr5^Ny40^Me6-e2KuANKA{l84Qk=g^~cd{->Jih1^;Yu`lg z!L!Vkgahh|%Pe1O`uH~9g^l0F+{cbpJ67zoh^Z^gSMY@JY#vUFF_f>hIA!lAJ~1 zTE$!9b6N*F*4u~+ByIeGF-qs50FUUrd$Q9{BiwicPMa`g3xmmAEJ06^8meGc=JmXF zJc>TkIp1~2G>fm5V*6+G znoFiT3UwCJQ638+y~{i17HqTXI)Nc*3$tnNBh_$lDPsB6;_@~+-cW0{>*Io8D0c-9 zCPMsm0B8b!i+3M9y^+0HG((T&37)*D*5e&P|zT#j9CDlYMR z1*l{E-!Wfac=O0Avmnnk;G4@8qQtX^op1uG*0Y)xW$w8;WABwpjV5ddV)IK$vn`jC z{KvwUK~!NxI(~)Lt*FE(k^yZB)-~1(EAs9!7*wtth?pu_h?_lq{kd)4WznCcpn`~( zd=?)<{=Cgsw-!&O9PE_l8Zb^UJs+Zy0c=azx$W7~=Q^bbkLyVL$TdRK7n^vq$Z;w+ zi5|!3N!lgyriV2NO+Jz`OQ)ZY*+2#KhI+v40##2)FGHlrKMqJ{0Bb@Z6+PE2umcLw(xP#>*{x;5TC)il1`Y7h;eDU;-A zEcM9GxUQ|S$`kw2MHSgi^=@=q(t5vx#hhY%G*t^X2NQzg8PF@A;p2Y+p^=uXuNgWr z{Bk$;2Z)!X67$9-vaz}3-sgg-05(KBRO;)&cJDdCRKvJl4H7EVZ`?fm>IVL1CP6k* zy=63xn{=9J+bXmH3`x35hGpe5>?GQ$8(stz#n0J}n+JBEoKDI#u8it8<4aIXyj?`P z#8VZQjYw=Isjal?Jw0iR`zA=)H+#G<);VK3*4Zf!l~-}r4V|kN%;IMm#`K~P#&yll ze~O6$uDZ12u+pJCdPTb*Y#k=12Mp!qq-&1{!14nt>NcZ1)WiZHd`SJ9 zK@f%4@c#XzGQASwMZXpTtJhH6*^^qQs1CxuW#>9QoX_$UP=1KJDHTX3HHi94x?2j+ z5P}X#tgVi!L(s7zn-#2t#?!YGb%^VX`(|0#FKgH*+;~B9o`bQqRAL1@$6&}SCo$OW z8vJ_jXdl!)==futOSEo|AgVT^z{*9kvt>9>OA}@W#C?8~P$}t3RHmiK29duz8F2f! zl#(E;%8}{|5x3PbjTusy_lbM+qIDggj^a94KCAF!W8>5@YDwCJw*Yb9^KiLotR)3n z-%^$=#=T*kKvzgUCnU>;C)q~#&cxD}%tEDC4HsL3G`w8ZtiT}PC;k%@pc1dZ|J22N zWUior5Rq63!y_(xyPWlS0qTl}msey$@X4(~6*WhjeOZjDEb(IhSR`^hZeDvxq28vs z=wcHyCqiJP69*93G)lP*2y!={D}HpMtzrMjz(tXAi1}#CVdeciUXx@UD?hdY!S(tnqI81F^`b7oXO-4L7MF_*- z6=7*-r{bSIr3Nxb4x8!xS2o1dfw1AJ|RC z19}R6Z}OxZEVs!XEv%R+J6JzPOZh$a#&Zw6dVx~@BiVXK3m$GlnvFNqM%?zgXGOKi848ED zRK4+cW6kJ9+{O+GwsI)>Qn7shA#U~42wNv~MZSOWK1`alJo>6$_q9gYeBabdbJrw_ zg3Lk{@nqLnB>qh*utkWPy*9wcK<&ayT^eSO04IboSQ_4}n=g5QcvYHV-T_{s?g_g%q3) zyCN6K3UAoInQFqEm5e2u!mJmFgRG+#bwQ>~)D~d7OY8h$>DnAmMSa)|f|(d2N37mN zk76yUCEcJgcRRe&R; zJSyX|e7q!OscADqv<7a1c#tz1EjRtDxssrDwRJS%!fgny(7QRMiL*Eib`87{U8EU9 z7OKhol1fw>$n+V^DdpHJ{bfKH1mWIMQRZ}r{y@nJ-jj|8%5T{D=@(=ii}Y#j{qHNV zJM2%Q^=6OUvub^bc)$nK1^~Xbq`*GF@MQk^^`=S$V2c;!?8~d2BdG#`5V)Oo0*Z}_T6;}>IC#K9 z{Jt<=lI@$0DCI)3;o+|C+BSO-eP&&;TG2JztM?IPS5`j$hy zgFu4^xbwo9PHQ>JO8DgR%5PuYd$Rzhmj-gm-}*Pq;G*+X=Dyr{pQUWi3f#n_DoDe( z!5o#Q0n*uc*OjWj*e*Uj&Q}kF$URB`DM&tLKDxeH=u%m>wR-=V$g-1{ z6?ksLfo`j=h>JVox}xN!U*-)JMua9drdJ&-^e;TkA5N%Y(wR3sfh!^)J3SDn^RVg; zWazb9hGYw#;%c(1x!Op}on?yz)iHyU(TP{co);5a(`ToJ_)mPrOLAY*o0P^ABxbsx zI}8%;1M7Wbj^=p|<<}u+csX>w+JG^P1bbx7EUsmt7TDa4d@!7j775LRxw(%LZl@vE z;LWt2fJwqMu!H+;vA*9I#ShJ{a{;~<&IdiI?@`Vl6x)tZK+V0E)QtYw`qSPg)!5_+ zkMBCWqu+h?Udb~!=~%M9>_E(ZV}Hd{!Ab&QFh2!k9ST2>rTo@-I|H#iL43!-d^l$T}Sl!G1hfHAY9F_G6*=Mt{ioXUU<)PHO<K-H9MI9xkxC|F{;FFh( zf{Mzk%I-JXH+D3j|!TjPaZ0y$F#9HwaLVk{C%vtMDGO>aCh-jS<- zdXic^D1K@(ea>PF$Snac0CF%m_(M@>ui_8TriIa{LVhq>n0!G<%W4*G9Rq14*u(7} zt!(A=d)P40b1C)=RFws61pi1wS>}a0ucon2UmbM(!Pr~ZRUhy;N~wzP=eos)N_uP1 zXdl!0DX4D%Z*=lvE^g_7Is&9=!Ova44q&U0)}3*EaU(NR04%L+zeVNY%;9zo0}H4+ zagz=6IB3_Gg(Vv>{5e5HAutO|tF!_A_uy2=!a3++!`jf;9e8NuepYTr8Uj$uwH4o~xGT!-cxxH8?o(qX@z_dOY`!(u zff~+2R2A>NqR2wAB%Mq>wpz=A@M|#dnbW#TwmFw(HD!bBjq;N*!M2?DAA6%|K`s1xOnyv#(1wFMRqgn$5`H!JKFVP+1|49 zL&CzT{DT)Hh4~TRm8P?T0lPOabH<@GOQub8LHa52x#;6CtB=RyII1Onh!*P#z##2K z>uXYc2L6lqYpu>oz(x z;>o}_3?2Jc(;rG?9<7uJ!LSd3CIs&}X0go43z<%60!)LO#A?8#R$E`F<_ZLIC01M#6RfN5kk3~O_sNCpzN z9<%bP#O1<;^8DFumP&S01}syDV=Zpj7k1p!T&rG9@x&pErukBw3zne5%z^_s?_RvI zJN<|X91T5Hng3)83IUJ`DO9w{$uUQI?>eBI2z~6&QnV>YEN#?;#;=%I!JpwhXjDow)kPnR6UdDD=z3f zW=`4M4?%$_i>{^ky;?R@<;UAfgP>v{fvClJ=$*6sQdiBubZ(iuvcumQ4w{&(Z($B1 zKP=G|>x-@Ay5`FAi);~NTAE(B={noV|Lo=IJjRN&YwVddGKI%8K?y8$gyIxd^0xj2 z0JmcBqj>W`&X}>f;WE`_l=~CX;2qg}+Wi)Dq8;wGR&lD#-^(C8|j7PLrYz5MO3BvHS+0)bSCk+ z(Vr6HhX8J-4#3#1z=KM!Ol6vL7G7{|?xj>sVTR?ivQTEhMo9&P=_r!awcTSVXkbYa zhBF_W)UUtz0$y6g#uXoVoB*@uLyxsrxa@h0S+!GDL+ba|6rhiZFPRJWM ztV7$is+2gRoW#~0RD!$yVLAK2oOKdlx~08>Q{%((C;e{|?x`Tka0Wki;J)jF+=_!e zG7CL5?aZ~ylhigii}NvO1iC+dLi>t@^8D>}AFdp3S#zO0XWSWi7yYmH`~(v%Z+$$N zRG_$~W~hs{mG&+&9)jlVvPkpsUpRW&La&{fbzIkRXqzi6MgzN3MLGaTa_Es>qu8DWX#;#sSpu_urjG7@RV5s)hus}Y96Unx z!qI>QX?q=h;R$edh6v-Ok%Nrq`k7=+Ns9kx?x4m}LoB~V@-NL90;x@em~KdlQGCTx ztdbaQY!h|lGR(9UH(icjG6z61EKHE|4EYr^5lxxbZFNdJCH|vLV%a=j*;Ms7I{y0TP}!(dt?=7%bz)3v9UmXF z{lCeXfR!Lk&Ti0j8+M)7n~Noz573eN&Q1gGF$%nb%neyW31`ubs7!97MFQ$7xdSP2 z$3ZT$lY4_W@XX@iLi0Z^06az-XjxCNan@F!E7r&lMlrOgg$BKcbfZ~Y16erz9@_&J zDp#!&M6Yrp)ps!KO7sD99X`_NC-v$;!`{Rtl27u_wRTKjB|eF5 z0@8Ft_d(1!ftugfhNCsBL+Xt;jzf>a`AaG0hgkjB+RX+@xOnKtsV36E+?IdfU2tJP zx0vGWR)($`wDe+Ms^`hOl0JsXuwG+4h$VBCw}C^!9xobpjpF|6pE#F^%X zF7}-$YbH#OIcaA)(~QTsKyiCo%Kx)Y6Vv*Wl!>XtR|<2#iE_@$H5-(+c;LDkH~JtWLP^;hQwASX360V`%&bN4gmp%JyQxuhSRBSGZm z&DdLc^B)r)59R+e$n;C1ySBa1eH|}#FnHK4)!f5P6|B@om7Z{=&o2n-GpHchJ(OcI zIv+JRQn_Eec#P|PY!&utsPU2@#^upF`s?Zlkl^zx{{dI}(_VLNdQql0n*{f=AE898 zt5t|N1JVLcS_=qi($c&vyG?MT;Ni^*rWR^ZJ9)$8V-tXQA^bVFS#8t^STsFi8hgse z-*_qU#6Qy>zqH4U_00IJ(3J4#_;PD24%fXnALCGL$$1AA?eV`Y1xgJnw<=N=8-qrd zJ{r&d0V zGNHV~PgO-U^({xJ{1o}mi)I5lpEZB}go`F|aF0`E)p^%N!IKSIIj41-r`{e!okf(w zoXT|gnSdE3Yd|t!Up@KJsaS5R8XYz{O8)+n25@|y(r}t6JR8xfGL{UfmR8UQfbXRTpn^*t(cJ$Z3M$5p-vO=c4z5S!1p+}LB zSH9xMLg;;v_MtR#81qFGD}~A|K%9Wq84!j(%45Ng+1bzPJ@h>|HGzr1grOu~uWn7x z8-K2?wz0dw2BhwNUiv1^Yb>hy9toCIe^0)>0D=9JKbaXCw^s`Gt3K0~cs?jByj-#B z=I_U-!cvn^4)=a)dQx9eYfuxRy0e>lMbFyx=Sx%%L*a&>$=k2F+V-v?L1bSH zri0ACQ$kP>j@{n|!nd|vosMe{z#4R(X#DnbbZmcn<(kFSdo3zMB2fHrJ%GjTvx~b6 zKEEi6TEo4R5clUM;a^e&e?)G{|IP6}7|{=AUUVF20?akc)1`mUfk7bBkfN))ep5R9{~rjp)zQ>y-Y+Upz|IYTSxBlIiuqGx>gQMAX|m z`43|M{Au>T_D?fh_#g)HH_HUgW;~3j4mH-vi6{6({bC&by_dPm*`d3 zLju3Hn3m>s6GW7#88c?IA;Tb|?&oBpH?$GjH{1?1GAWyfEF*edmu<8Cv6mpz*L(R#@*ju99L0| zxVpBJi6sTRjMWF38v|$mRSuqdlG<$MXsmUUFI4ed@Z8HVT!4q-c+1Ty5aD~;RYW`W zwM?p+RJ4QZAp_j3JWR`VP1@uaT+%=!X99bn}ZP@~~{_Z1)qlQSs!5sg3L>jUgo8?KSa{jOE5&Qvlo z_XI*uw{aGhkg%D1Se`R0Y4N_-X}3Oe>fQTtvHNH&;EJ)cP2*2nS!eiLPg!#2&o{3% z(e4fUmVM-2?x8``FH6V&<%(fKgI5*!oO<&#;ZVhwlNr8*w=iqf+h!p%*yN=%8NO~G zUP-gC+onnsJ8<3t5ITmmC6G(9&T+dFDrL2Dv|t~IGPjlQ9tOF&t`ytlT&~aZj}~zL zwop(nyFvVP%BqiqF)s6rrl!4}GnQGH%en2E;Ju@ka8+a{-d5T&$o7}NIn|xQv~ohZ ztY@nc-ga@iVIVU}%(qW z9=e#_y4xhLugZL|2tv+`c!tbP0gNqpA~~dBymd!N@r@X>nz-9&ryn%W-`aDCd$SCW zPE&=)xybxC3cP4uA2Z@*gfQW(`T5ea*S*sjMj8JGPMWLYTTT25v1>Utj<$5kqiJ{L z2IH&Qvzb2@{2MirqM~}StO~G~5M2xJZsI!W=c?N`2*TN~8Gst3?+XlB#V%ua^)oaV2Iz#en%F9 zzZqM7(U}fmE58}wT1Z0Q9fUm^ar)uCNtXYNK0hhX^YWBKIgT7oYV%T}Ne*R>#=%DI zF`E`*srM>?(vEz|%`*m(2k=0lcZQk1*(1uAbn&?d{iUqTjq&0ls71{3+OFtvrJYev z5VXP7|E7588(&>0E~|6mRf81|jf;ES>IgwHL>8rf|Elw!o}fen)0eq z`^T6|@a!d&kG!wq7;49pSsTmKPByjb7=R0F>Q|GyCwmEw_O9JKg&*pR)2{7yJm^d? z_k;1kSy$!tDMS@sLygQ6Xnju}J$7vh_GIHmj7kew;X=r8@D$fhG1>`GWJ$I~C!VgP z1)gK8+K$=~)Mq=x!r=DE?KHbxe&7%Tuk^6%y!+h$WX1}qQXr|mGK-pQkBw|SufJwA z9GwdrcEKHYnRC~7beuypF>&MN8&X*iYi7{_qRK|fyt}vmjf~k3rpY=oaoz!Z!b_;T zNuL4_=1%dL-O81>p;!+@tjvLRN}!%aol?=V=MZu@=?~^QePQ0q@@&lE{GHKYmsusS5``=gPpYP0iO@M{vU~u;6T&5YR<%fKS ztJOt+)7b)-=2*e~r~zSBkPS;Tqfw>|e2M4MlFuP3Oaw|Ds?Pl9J&0$)d1=+BLkw4x zhif@EfP?OM+0mT_`&&F*K_Yw|tjV~d0QrefGmA4vc)U@=nVOc!#4)+mpn7{fZ5^MD zr{1Wzn4#u&&&n<5vSiWMnu-Wskm!FGaf8iLW(Jk?$UiWMAp7%ThyRr3_!>M zsdd0gSkC2tYDOCshvvQmSchP|V(OiU`s%|iQ zjGB1)`O{L!&rNKRYFrdGRy*ynfD&EDQ&zy>-*)~}exTRCfH>Un?*3CgVqs78r|4a2;2mT6xr&yDfs|EjZVUxjQd=G&SQZ$_Q_|gA# z?JU1MjT0c48nd{|>W|2KXXheGFx^+Ei!UkziKj|HSPidTF?71bwK}94$79T=X<%4B zr>yCI_sJ}i#OO36xBv$D^vM*Ybhb)q6HpaZSw zDVkE76h|=D?_dDXJ2yil;v};F47V@hB7Ey4+4Bpi-J*#Czjgcn)k6#IdcL79=~K{M zgxsBK+SSmQrgOBH+rwxK3uY{h+nDjxhEKxERt6p$*d0d5Z!&d=Dy!h1J6OmcO@7$t z=tppucfHCDNh>Y~U^TcoJ;gc%+S_m+;Xq!i4jj=sjfGW5$wtN;7$rir>o0|r>B=l@ zZ};t4XqD-HT1`0&xOG=M%3{MV)B>5eA)%-)$WijiB}NAf%=UjzJvWUgql+U z=n!~q7`(srAJhsQZ&=|m@9K;zhZC5k{reR|00DH0zv3YHXawd;;$``@v6k1mF}ORG z4||jE^ygJaw;lR@MYr@Z)69c=n!5v9ORQE{U=1bZ`PF$uLT|0$5e>^Klo4`$1+VpA zBsH2oslGRN*x*NHe>jEzzl+YRj3{VmHFL%8h8BpJ`0Ak0MyOJW@o_*@bnl|UMV(gF zTy9@qf2k@)_eV#eKAq1Q`p?ta)8crrgwHrkkv;FlLESX>(@fkM{yA`#c5~7hgYTb} zgu%L(9$a`sz)hY%=9}1H%r)4nkuA$9eZE~cXq4|_#)5dgbKNoOF+|??el<9T*RWsl zR(6FE2+IIy(Ec(*N?UePKt4?7r z;54RNBB4x%*!Jvn4=JjM>1tTH4cTe@PEp6%i9#fK2jPSX3QzqaNrZbukV`53?h?Tt zVEN$olaxR>07FyTG-_5mSVM9oA0~yME2OWePjxD$tftFz@I=U5?jaPrVppYP4>Z{M zUH1w0ki`IL=GlgEPH~E2RVVM<^ za1`zfceRuL_ira4IbU23>YhB`=#jJ+Thiv?Kse2NjK9s8JvsoVaq;;s7YZCdYZ#O` zZy!qVYVN9}oyxzVygN!S9>`DgLK-Vs!bZ#2S=}h^xQjq}7agcouBpyPL}$3$@Q<+_ z8OpSpsS?_GnVcnNG8bmmnhx<|JYiOjQ$h0TXhc5#rYe?*DC9dQF_mc_2>6Z!dr7`I z)AR+u0{Jnwn1t#QH{N)bt7}?jJQ(*f7#5={Y~8jHOLoTaalrZ@@{HxP)@n(APFe8l zlFZWsmA5OdnU_TTpW41Yp6T`fe;1Z}>-BoRp0DR_kdyEOsn;E!JoK}WMP@pz*^6r5a|cxv?1^aQCR8Dcfr!CQ7k;^^ z6JwBu5Uq`6^wWgwFdd0-(6Ln|VE$VWWnKOrIop@NvTV!6>in)p>=>D!2ajfDXe7+o zUNb0!LP&2ZcEx3yeaf5T;D6ZmmbAL)VUE=e&v_oe3ll2f4T74+otLVdT*R7&)^2*G z%9khixhHvS7lp3MllGTqUMj%bJc#V(y9^~y2<{$xYI!*7#x-^;J-=h!k55cqRi*j}uy=(FqOLVz#BZeN&J0EP9(P$<_mKvZ_2P*%1Ve><0^zst@DWnX~ zvaJ7M(^xL)(1T;%D@&To4~kJ;FbnC4n62S4+;t;th(XD4&dJgC$>0ZI(2!`1`L~&t zqUZioMkvj^+UWut24@>Hs7VlYE)I7&DB&MSw^YUpVbH7ofgUAnKQWUaNm5W|`vIP+JvzeNmtCyepPjcgOuJzl{eRpYIb5h zPSaSzNw#WKpKp8MNuT9TwXjD;sDfbgY{bbe=K1Il@7H^@Ckn1C!CA)Rad5N#V8dyQ zvOLn8U|FW>65Lwaxsv_Bvqx7jpSTek~)~ z+-2fL%GiIY1gXKfo|s~`digugJ1Z1^YfpEtP*L=VZ@1TDM;%@a{+iz;ITTuxE~{A@ zV4@jr1^p?y=~pDOC^JE*xo7FV+`Ry!S^XPM$?E2&gW`SxJdqV?bm2TJrMckye*)k~6TQ>E|_TG#GVtb#Fv9y;Hl*I}%xy>>~fCX{DEr zR{Myy1hi&cx_Y%ML8z{1oGBLQ!E-M>sGOfQEz>{*{U5jmn3$l)U$Z^`;tzyH<(`E} zOHlpFIQTMye_dR!7tU}059=d+!9NZ-8FJ1NU@&dUf-=FmFBF%5o3~!nZULN^-s$Ps z$hf#jQwK*!)a>U4)gB`wBX{P0UzgGPXSo$wqFccodvyGa@V`JN;}zIFfrRbTH+QAl|T&@YUBmM*4c)p@_Oq2zEc zOMj|4{w2knyOrgyz|D{9`~3HF0GRzdpx}@1`{lSZJ$L+md_eGnhzcjcTSig>1m8`< z1!EFv3`UOzS8v4L?c|5dGkXR1#~csWhOubNaW`4pX?Z&@rKdZdCZgh-5(Hyop?B?! zwTlHWH=?X!W^Z&zqjnT?9?;xuWiVca$0vQect;2WQka+I=v2%x{n{AEYi{ZbZ_H8N?M=y7&XForlIAL&Scg z!Pv>UnqV8N(sxgleqj68A~d*`XPU_QydI^~r zOF~5UnjT;r?{VApYP!cVTt((cR> zXTNRsmE6OJ?dZq#<=@rxZoLPq3TYBu*{?y2l@kVXTtHPz2NP%1P{uWV;%%Z|=-VUT zK<7NLg!Y0By|e&*iaT7;i0(BFfq;0G^2=1pFf|3U=SXgMfyN43PUV`Y)*F+d*J$hg z<_8!%PZetYrIp8P(raYgQv8=S0u0Bxsn6$DX-UuA&mK+hUB$Vizp=6y*bn(Ud6%4g#N9RXLXB0)Os!8FPv&pAD^n1jdAf+Kc(}IV zRRWoOLn%meS@XE!db>_xS{8y3}&|80xONh zwa0beqvtj7VQdR4#j5CpF;k4B)UcybYt@IxKUw%=2lbGpPP#9TTB?-uY zv14eWEg{s^F|$KZocAZvp$Lgk(_(u=LP-{uXf^oxmSk1Bz=!=#pd?F2k?X5Nbh=_E zEK2txWPK~2jp!%r6FC(EM zx8~W4UW0A36HL-X7dA8_zaJ%i)SIR(TLoRGl-#be3YJ}t{zW=jmJxmUH%Ti+z1>|; z+Pj_}HqHXfNwpsy1DP59_pZni182q)v=doalb>C#>3lBNc7M%!o$8&Pe7dK6J{a|$ zi|r5hO1#IOZ5IL(UbBD>^K0?LhgW}m^?ZE=r|A$OjHu+5nK+wh=)7Sk@r>!cvv0*m~0rfw)M zC;C!~sdWdHIPo6xBChmW(K%VdrOrJEjLG=|$ zh41qo7syj*o|^zAClvLPVFuen=+gCp=np9rq4|y(XISz(Pd{d(yq0&Wt?v%Bb43QJ z;0fNF4hUXt54ZE$b+L+9!hF-6T#GEZoN^)u#wG=|Q@yb(+LSu^VpgBQifS}Rxz^66 z8~5YP>F^nG4hqsg-{ zf_I8ntE!Wh8h;LU8P}X1_Q1o z9Rxitr5vxt2>20(&HaPsPe|9$hRafS#;Ee``zCvag9qff0+Uevki>4V3bU+D8XH*S3k*}lvk;@_bVmksO-AIr zsT!N!N}33dIw2e5BO2Uc`P~JD9(^4U@F4m8pMUe;&%On1MwdpT*X%Sm_4!0I+0kQF zTanY)+)rks?6!L#Pv{-LXE-%iCn1oavS>zy7^%j5_F-dT*~!{eI4P-Q~7v2x=Z+#Yzx~CFF5YP#4=V&L1q{hrKpz$@?6R^M}`QrZ&e#-O7 z1e0(n{mmiXdrC!)!MPe`OX|uz23r>i&GcMy zjSf1Ib-P2kK4ro{p1Lm2@IJ~Gp%fB7Qef@KutgYEtDS#*(;O)8g(W3q=EiE}x#cK@ zu4I7<3w*(-_?2br0OU{xqd9bzh`C@%lX?atZo6n>=~VSRjnC|zAK)|O(a-|a)!IF_ z?9B7FvZ6hULr$Pq!EtOK0TXs-i%Ug*g_Kr7*{%q0Ko^B3Zevz99PEx>4s0VHWz5Lu zRW;tn1#S&8XVugypp`}9?ic6cN2qJ^RIWf^mc?JlFY_nlD6lvkUiKEt;J9dt8ofQ+ zytRjKx%wCD4v$q!u9I;myn%!>D)p}Ax3InAe#1ehkX0!2LLl_C0eQl=+i>9s z`4ZXlQ>|Ja(zfxtHmUH?0db{Y_`qavrky^|a-r;oN6OR=M#zdX4n(E2UylFr=K$i# zW!zYzJwfLs^i;4nk{4cEe(`Cs#H`9m31DUF;CBU7g-nZGRUq$ay+I-x3<7ZDjq1u|K?etDsNvp5dn_4w?;yn_URG&EWQ zNG#lu;@T^Yb15GU-(70z9eY0YrH3$I5P{Wdx(^Kl>g`Nw1gW&dx9l12ys#JWRT~0+L*FC89ej6)_Hy@23QwpBU zwbB|0J{O1OSMjXOO`+i`oLU~U**qJnZt9CF$>-E-M4cF`Z)x}HTgNb@!z`#98yF%< zrr@BA=s>&URp`Ag6foibCcZji-~#`v7D==-1OYVJbn4I+34zE5hgjNRa$E%DWYuY--bL83UP4%|^)#(eWCjT(LlxkA&9a#gNJ@yvv+EZAwUw*=bwtVRpdm1Awm zUO>r@soBu}r1UdgGWBU#5<=oPxI}oX=WW(^-W%psQvJH4AtNuH9GecwU{-r3W43Z7 z+rz;QFs!T9KAdzf+B;#*W0*y`m@e=eoKmeTb??Wn(8teVlvxh4*Z{BfacVs#)^2j~ zI!MZ{A1@+n%43_`H}!S4Inc7&PIhRAWzTLztl1ezbYol1#0>5Mmi`n_X=v(a!!Hpr zLHMs%Wqe2k*+(QPDQ-|p+u@~T-h)lfFi=NxjSuCW(fWXU^rUcB^vAGW^ z>Pxc<)}Qyx&!b2QX8lshhJy1_oAQEwC+Ixbm5?y%{>Ebdm0vLBn-;OPmKxsV2Pcoy zFj$ETu`~(S*T9>#CFq32jELNOXFuVCzWd&jPP{WUb=@=Ff|-_l{#7`vWf-Ui%$oay zP!`Qdh<+jJzFqo5rz{QdTo^J!AR1v_L93lVZ5Rtp#77N8Ogf7_e!g$ft}9XMP?&^I zjvCL+ezC=p>~+NYRfGULRI3ck7YK*o=G(K@WjiqIW7Oa{!quF~1=Yz2y|Fo}E-veq zt`FuU9!La=-PaJ6<1g_e=IVUTT4t$aIZjKz5GMd+|1L&nRjw?|K#u#t6WV5OI@s64 z2W9OrAl(qXHH-c+SPK{)+zhe&Ow)LrvF^CcPl&_+_J)uJV(Tswfo6560!-B@BNhac zFNd7A>HpBFWoC8c3iSE%Q`9-h&`j(m%BI0-Qk@z%G<~vl_Tl7%xaK&si_n+d7To>6 zH&};rhn(>YDB;vRkn~!z3GgBT2fhyn#1_F&y})l;ng!ssJRKR4JmA4MsAH`rA0)Mh zE^>YV;Y1gXlLaax6E)3vfrr9>e$H|+CR;GRRo~9O^yt_U)&pJy;uDJ`6L40rM7jcf z`wyrF_8+3tPddK|LoYP(-8%{P#l^)>KYl!DW~;5Oqo*KE8Hm7Q)vNibsZc6LE1!9f z_xbbZiO*c3LQ;FzdciJOo@5E_zftE?g!n&ZO%kxML|PFuo25L5g@ObQqo@-)N7OkMF_prIL$EkSH+?t?`1 z!f8v|H(U$Wc@Su6{EcUt} zJzQ5@)6fu^2KUWcYI`OK8) z#S(Uzy`Rawci}|1{s(!mnNvq4r>%@6!J%gC4zQH~@#aywbYG6Tz$_?!s+X^jDLvUy zLn)pbhX8{@uAxoI@F@T#+$e=T6Gvp;-l^8@ul;DQf{?re5Kuq@?l)Tbhh^M82@?d) zgW|PxYfqCN6NE63UA{VI&~($xcDcb;!0Kqv5Y$#WC;;J#(dkC8@Mc7P9-nimtgP$b z%~F7U`tHI;1RnmG`EOo*hMv_NWuz6n=}OK~kWQbw&CW%6O2!7v1M!yqsm1p}{&}&=woCMAbsT-YAQ^Br+vDv!H zF%YmHAOye*uKEz9jQ1`HvqwJ14%TSXe$CT>W5|5dvQ>FZ*m2~nnN^szk3=tVBWv9A zTxEMEKj7$alB}1#qz==LdG5U0t+r@Vm^=W;`2s(mHzZ+=4Xr$t(4bo5^Ya+tMTH(M@2ov?IwMT7AN`sHm{`CM5y)wUl%ktFEXp$#$CPBuG-5gX zIL7`hF>=BaZP6ZHWJqJ}UxHKYOu^0w+4+JVDU9%8$*yyL?2IRssjcf5enj#pYI#!n ze#o`RfNl?O0OcMXN9sWd>};3>SJU?oo9>x^I-Ce##aA$Pt|NeSYRPp1QlqpbS4%4h zMfx<>fwakeiQ~pf=~XpR!%-7vGQBnIdyuUHVfS|}fZIuZwtKZ*$D+cz%{>9+aYDdZOM*o+Q(Pk!*qaE|cSf1FVnq&-r0Wu^Z~qiA|Mt?q`pL zei{@(kAHwQn=cN4@3OjoX%Aek>o$fqC8?&@cw71T-gtPk+w?-%B(9n4J?i3zVaQ@m zcpv}hy&9QncpsVP-4n)6ezf-ine+7O%C^_^fd5&-{c}{x8igB7&f(> ze+CkGB+ayl1C&M*HZW6wX!(CCn>j2a^#37T2@GDgxX8jx#1_w;5er> z_bB~+lFeJlTDTNf#n$ETx{=nXC}z|<)YZWnm=`MFO_?6OCzfN+YtCeA&{lLtr~3>T z+M_ZU)(G5>aaRq)91=L^ak4i12#-Ax7~I|OEJA5;rH3)$ z_pH;gKhD%E821m)bI5LZI>NEt7122vCXT=JRvOcvKB3s`r~2jog%(&h-FTirJ+59{ z=a5g|!Or=aK7V*1bwDSWC=*Y2TU6SS%A_HB1I?dsd#G|^1l(kV^Zx6hetx=(sTNLx#bG1om(3^QXE3AoF zFfAUI_JpJmL87#%=i(_^Ff{caU+q+!))&$70UA>qlT~#-DOjGw*Uag*ZJTA>ql50u4 zeH^Vcn4m?2sE9+7z8PD})`%FieOp^N`1ZpS!iIhP_#DH|C`3ocz*v0F9QIspitLPn z$v%$+{Cx8=SWNMk%%SgUIo$QNR#w^?SvOV;=q33}jYOyH{VcXB56(&Neuf-2_vboV zW$Zz3b{877{l$^qI3ti`2M)~telB;SDdXwXx~LtO+1|z1+U$id!!l!u*~ilNjE$Eo zibiMod|Ec;y`dZ)uREFWQIBY3sp9r?NV)SA!)nafEiV{~vTx~Nm%Fyd7|Bi#ysG>R zTj%g4xz@Z=@TR8Ak)uFD5+4>;V_3rxcX|(eS6|Y$sVOIZ(q-O6z z9`C;TR|6?s7MgAHO9j{{Maa7LoivJNduZN6`NN zwUcXAxh4#2`GEiC^9|REF0Uw3W-ZG`t(V>Q-+S&a^+{CIT_scsKy}ndvU$5&-?TF{_)%G=8xUn_S*CLcs>sI$K!r~?#;Y1_u-OP zQV$?eR5ai|EX7pLs{Ro9j+yBAZi&8n{rdGko*&(Q_jKI#)sAPVb1NQizjJh{!?7*L zm2W+JaNx|U6~}Jx{kl5ODE+VVD>sB6ypTBhoE6}nNIyuK^_7-VA`|9(Dn(8GC=sfF zEqnCj6{%8GbE^)PBY7!=uYvr}>+n*5^FMDOkmaUSrT@OYz5M^Yt5Vkc@Bh%ZA^-c` zhX0=ro>#H{=X2<{v6nAz;PR%5EoJ$H%*;&A@8A0Y`7r9mY(MgJZ~VUK6upK!0pdIv zS@yY^g8)vsy4u8J;`NWUOmvVQ4jQmbA&`#3>o)&++|@l6mh0f;6lh)AaE`#bzV-DF zPr6^@MqyF$jjWomr%hIR^S}S3^9%HVs6_Kkm#q9q`2#xHIxuoN)W*VK%5&?<3<^+? zPnr1T%`9CPfse}1r@UvC*So4hsLubh`IV22mI#W9{!|tyHxO9mde|^wbt|lni_0$? zUW0823W&%AZE;#%>ECT$H5+N(-*6Z`WUM5SOdKg<_b04K#9hCnk{q=fy96@vhrQ}Q zJJ?MP-vI;-YjYS(ueW%{=l8yS8M&*d>=KhAEgT zG5-0zzJ|;cjla*ddL%8f)r8o4E&cZlmwA7JAXo3x!%DX-d`HBe|NW+APh=Xtymh!nXKkf zV(zTvF^X&t0A&B_>e@}1Y$S!p($65YWcR@y#hwLD;u8fQ$?8Zqc7cVG1JWHifCMOU zF85+j6dT{{gQk<)7}xA6Msp~5r>Otb<}#)o-C{gU37h?Y4d8&)C<{M({;{)Z7MHGb zqn%-MaL!xhXo%KeqeCPk33R0XRaEWff%B)7`rpbOCaSa&#_~^`p_Xmg#h3OnSUv$~ z%sIBf-|cf^O#t08$x;rH_maXFhKicRjuG)mxV~tuS_a|~?UKA6`Kb{b zh(PVXj`?#PmSh?caobA^dry|ELQjfp25cYcqQ`v)Paku@TuW3TCkT16Hz5r%XuR8J zODz3B$RGCWZ{+eOMor#UdsH&AS|cxG>T~yd=?!j!XWN`{5IbTBQk;SuAOE6MiP%gZ z3X(P<^suGwmtL{YTG}VRi)Eb^23nAaUbwRFsn_5LU~0B<6P8(G4^yhQx%wP{tvfIX zVZr0^sPfoaS+}0u)P|?`;Y8yQdhzJR8ECpL)>zwc5P=WtL0HDfcmtS_YfJynrq;)3 z#;JUESB}d<*T025$>*|S+gyAi!eEz7eB%!J;GTLkj{gYb4Q{T~!iIe))7T}X#EZ7n zzYV0h(M?VO`Pw{jPtW|XMtj!_ew$Q2B>RzXLshwZh8l>u-gNUdRx_LH$2SCiV#9E&Hd{i@MZtsJM~ zo}kj?ftv81kOWz70gq81*!f%_x*m8$2c7dcp4sc0ZJ5pyR=Uxh^!@D z`HWl!rZM!^mDCS-HCX$vebydBLmg_L?|K|E7Hn*%mt@%_E{WqtMIK7T?Qx&>hqb73 zM4JV^DZCr^AW)=5te+Q(=?`f8iB0V37ateyCI$Q4EKwFTMQHyrGAX<3v_A3P^2GS0 z+oNA0tWzheOuHM2s=6(&M{Yp4qO`co!&| zgA!1+36=AJmgMVe1Sh^A(q`}!K+tSBMNYT*HT_{=bh)eSRDiU%r3%;p71ZD1@a1h~ zFlmpXNe4L#bR97V5)3yi5q;p2ZU_-pd!`h0oBh}0v z(Hhtlf32g8X|IH)NJOA!jxSZd%Nq`KFRIGanxSS9yef8cZJ0# zvFyzvhU0ayr53~0eNauRSVVR_VdY!rFwz>1I_;^gl>E`=VNFA$kJ~VOBkM4_K*!Cz z%RH!=P|xoR@#5dppt}D-R@1fD;krb1_D;fABEcXq&!b7}?Ou!hDp{E=v-yKdAc1SJ zs)y0QDRjDAgX)@I$e9s$Eme2YZvN?VcdI?W;6|sk`>cT%=U8w-fAjCNiQgYhODpFM z76HnJS_r&$IEqo>&dra2g_WoTN21GU9JQ>D+FEGQJ>|H3*MSGs>Po~8yD)bYtK_te z>2_bQy3*^{ghjXeVGi8AQs8DhW{MC8=v%?ZWce}R|QO+tgzvQxQKg<$^ zI#AK-@s#TMwCyG>GD)(!tx8Ga)((LyPN7#}>7Wz-VQ*~jtXRfjv{|29U9pGdb9PQ) zoCzV@)LJFwy-o5)jAwXwK3uinKTFnQnR~cu9X(%mRfg9(Cg^S<7H3t)ECJ$f#1hSP zA2bF&z7?V=*n{Y&JE2h+<=JXPGHi}fzeG^-6@m=h0SFU^M4Dlyla2mi4Z!6V%lFY> zQ4**t3((7{E2!NVB}o^+T)qhvY8bmlSs=>SckULNN4z&}VXpez%V<%2_S`4+t@Gcr zuoy_kQqBhl?{!MVuK+qZF^JI9+^WXv#LHi<#WHCHj9ZLlS@E6OVgwq0y6==(u!(&l z?6+idSkhlp$LFl4JeE2cHh&P+{XiZqLcZnFLmiK8T*}cgzOIGyqcF2=|DUA+N|Hf! zgu@}q!&{v)15af^0(Y05bM_FFmVDF?D1nOGi?7^UJXU*Tg0O+o5jqTEEh|t9VIi@z z9{o3cwIi0qS-tQ`Nqyel0Azaw6?a9-) zQJm_iJELL9?!>Xg1tU#kA#FE@0%5OiuAZ!eA+)XN9iHK`{Ab!c!|2lf0mzxzc8aXxhfe zl(35Dh{jici}yJP6r#fVkF2sG4JQqVz9LF@uot61>3C2jtMR~keiT%e`DFd)9a?~U zSx_E{*ds$&WbHOX=pQq!>0mCq);@7r&lSMW($&t~s7N3PYafue+b4ED4Z4wMoxl(> zAuh!etQHEB?|lS63>6PpJF7$!#i1tStbI^TgkGrH?RftS9j{49OT8E;P|dHsT4Hv)wON ziTSK!rg^$PQMQ@SvUv5Iu1gZlgI}813>bA)C$VXLs>1tP!C@<-rOEw2HV^;5_7Ug+ zzyVKeIqfsEmu6KKX#>`WBn$Fl~D_iEKI6THG#U&PNQmT?N*c_Poqqc2x1i|1vCyv#{KNbgxz zTk}pFic+!RTfGSg#`;giT=rp9AgrG5?Cl#S>0`sH(KZ+Uz_F48_Ms9xM6~KI4o!{WK7I*w#`>^4QmFKMrZuvoKJhtXyU8PRnWrWM-O4CM3PTt_q9_g!iu zr5`Wtv2z|F?b0Q@%Yk z^SdTaOqhqzG#&NZh5~$Ojui3KkWHq}lqZ|lxeDU4)n4;9Bp(9kvA~WCRjae_{pO?> zsN2WTLpHPKH;_Dda7D`HS8q{^y%k!zwcaDS;(fz41WwKPV(87|v&Lgg?WI3)xAl*V zCJyav^K1CrEFl=7IMVUXE>WQHLYi*wt26P+M>(Sqd-}i0hqsL5lj$7I*Ezw0qgHV?tslj;OJ)%E!_Y&&0E-pj=$Sr1RQ1MKnPDas~)vCgFKKDpeN;H%x z;Jw0NYISVmqXU;NI19wyCwzyfc8>r8`ENPsg1|?9zT+=%PzW3B_Uot?+b*Xf0>&f` zuh{b6(o2OliaI?#KFnMjs7sjKoeb8gYp1@S?tQ;{@rmgA9lIL>o9gy#grHGQXniVn z8RrDS;IL+Md{aWNnhO>OfjfxT89Y;8?`d+|jxhtkGQ_-FlKA0Dn7nY_*3C=&fo*%z zj%C0X&bE-IWS~t>DIQG86ljkJEXI0<=6%;mp8K;0zsxNCRwbHlH}8$K5tuN>p*<5WcD_2 zYpXpb=@NZ%lR_5su~6nt@s=IHr=Tp3wg?7>7OerR=K&nqR+ z4s~m?K1|WHM%dGSL95HQK?knrITAOQ0V@hZ#r~E2JvFwwIMMaPIzY2eK-rYhELL_Ce0cHA-0=FnxiX@dx;N^9(7y?tePme$3vP2UrRND)1a;{%U0a5We>Ce zb5MPU{)xWH#8qT`7>HA5v)|aZ(!NjpV^Rdd->s zlF!F$4A)T`=i^wja$&yC!9=SSt?{4PUvJ16kWNb8ropkeQ++mD5{eQ1= zwNi2&1b)t_k!L>PKQD;ss~shuCF~v2iZiFm?JP^^Uv?2KzZNrM4n-=P#5;+zxx#oTQqM*Vt>Zrh~1| zC0Mll?Qh7LEWn^V=CG6-{Mi3kYhp2}LuJ{GNzXsUwvhW6=98VLbHXuiM$eP|zEvg9 z?)HoRS~=ZOuZl|ENt6zw-U5L6k~mz)?25w5m)06-5V98|k=bC!+1N#<4yujgyPuA& zvzFGwMb7~K4tsIZgLOAIVYuP|d@5s3<5l&3`NGZ!y zl7wg>#qUNuIwrhs5&NsbNl4v+Xs!ADE58oK&h)Eog&j-0+Db7yIe3hb8l{)l{y*2_7%B6}y>-9^ z1z7hGrz)`&A^v|Y3ix+bH%>&&zazrl^qgzwKD^Zsih<3mPpPS&LwXpB3X!8a zLFCTHTPvx2XuY05vK~L>9I?fopMj)y;#Z--nMpLZIeGR~sgKg{r%9I@gfqpQ=1~5$ z{4FCI1pl`h$s@UsQ^2f`EMAVb4KffVMK<&O(#;6 zpL~Gjt*}F0%Ql$zo+F1M#aH?6U_<$Z-^K7i-84#edX0$FK)N^B+N`x9~`vA&K?lg?mnA!Hl>~x+$zxnS}^Y> zP6SffjPZXRxZQ^;WoK`Dr4j zND4TQn5#xj{FyNtrXCNi_l*Px#)wf`i?_N&zc0eXn1I`Ku9?VnpUCx`UO+o7M}#41 z(5=H~>w@fns=niB)aX?WzI07B@eZTi;l?i`!?~ANJX*|BZRviPg>rtH!u#TP$p)&S zP;{*>DCm9r_QGI#2BHP|C;Acxd1{E8naSJbJE6>bXt!9`CRI6V&J*(L-`$wJbjMlv zSjsEEgTY}=WAFG+1(Ihk7NEkFtjQb5CRd>7ig|*}&GtQ;qcaO4E*#jTH8<-2n~m@?S2-}&a*l9du)1aj2^{;`jjN>#gQ zo%FrTp~&BgsGHOM9{rSpfxfRR8#ZiMU2}Feh%fr}J(OA6du%kZ1mp%LVtzZ7y+bcX zlthUbwQ=?x?$t3)B|!e^9{GADcjIJiS11V94(1}B4+5_UEbIHQn-bK|F$%7BE%sJ8 zQPbImOVNr1{QB${zXnZ-q7JKs(om}4L9B($4Uuo&4 zjfee-8Y5;-+YO0{@55?alD}(Tgz5S0!cc@OvhKB>#8mB%8(Zn}^H<}?90T5Z)G{Pq zW~&MwA9jjJzX;9Y|CX?$NkYB(!#1xN%rE-2|JPVFN+sZ0VXrLwowf5FSh# zPzV9bgXWS%kch=#@G;WsI#0ZTO6zLbg=$1X*}Vyu@)xg z*Ni#dH4l!Izj4h-$~K^jrO#3w$$KhSk=A1YJJBpFH{>{aD4i}Zk+0W?un=myG|lYd z%KMyU9vFh}ta70vB9J3xL^{XXG42_=Mhrsli&3yF^}754 z0ahy5|I0hgF1ekcd_zb}qz%W~n2q|QZRu%?-tfji$(nHly=u@I?ROsnunZRQFG`8X z*Duc-9kVTMrwuS*#BuGXe`Z`?aq8SPCi`H;6E|?QpkF9R0&xP@t2y~woo2j`i(Qb2 zK2gl^uA7=2U{GwYvpxh6rac4Ve3Xcya@Y>ZC{%G5QnAfVe6U$zs(xlUqSM$<91v!r zV4ETFK4A2L`1?)XD$R_-qN3226PpDPxcG-)-;_eU-_|xg!-p;HTPBp3@!h!qd=)j` zf7Pm0sR#G6*dTX_Wor}t8!VRH;YH{3C#|8gCIL=pQKK z?$Ig?K;uiY5O90+kePex6(KId5`)v(VPQMMTkc zX!CdZ8B)zSwlPs2M(U1{fF2`U@Mvu3&k~(NaKQ0;NA0IQ(TOJVO)5*0l-fa(3)r{7 z-+gG5uB%F{Po$`*9m~_lq>03u%Z%*lNH2fZZ30}}>Fw=#fMgDGkTdZohw{=Rn-Zi` z4BpVDdoB2tDjMxGk0@@CzxE!`fg{F#HcDR-x?|g`8=JN53=n3LAZQfabz^(ui@M^u zv9h~AtdB$u+9c{p5`}b=w+{hcW;1(;JEJak68nse^`^7^Qe{IvFlRSH4Xk)VEEbsy z@?VU@b1+`_TS`s>@Tc&mE6Aw(AEo`Ud~{V~l{SXX4NLrzpZVo1W$)xO;=yI(ITO;z zX4@EEvLGml|Cmr4w%s%${PH;JHRU+cp5NZlGk3aaJ6H28$FSCc*uy|K+)il09w*G#WSd6#BV{DY*0HBC`-uX>}Zlx!88acQ{vJ2_JAAw4os z(U!8w{ZpAV3CJKPT63VVoDj?;*oQWhDoK8?8&Oa`@vaR+JsWxcr-n;in|@0_v|Pbb z;wu-C3;fcd<6NKo5Atg^QSKPWyFVrcozYT$h|7hOAjhIARgI?pow~y6wcF|JdARqd zh`ejKLicBCC&QMr{TXtP=LC#%8Z8r#5K{?sq9NVDyu=}tSvdE*XUXI)YgLbhi7n3nE1k0Q3v~bcNuHw0OFC!t6+THUb~$I@9i;Q zGrIsjh!ySA9m|H>&F6&oVs)1Z!qE6vhfF!OYpYJa`Ws|x>;&;@eJn(=ohe9FZq1X4 zZLHe9KheRGu4a1tX|misHluXbT%F+` z#?>4GU|+=WdfGOkD`CvA~Z#OX_vkroIO9~7d23^#yyMdmn}Kuv3lYr zjd+U8BwAYaokW;#+f`N{0Oa@__=8|hrWb)bEhQM-3!P;;*)KO{=^b#fJ}y1#H&OAM5A4{4sG)%P>Pv}8nwEU*jSmr<@|Ax z8or!fA-hVgu8M6`{evLLeO@Ma;vfQReA0p8OwY7Ub zs1;}tfgi{g$90K;C5SJ99bDsmjo9O8y!^7+IpW^<5lP*|;LQsYb|G>6h@&s~$0n_a zs>hB^qrO^s(!-XJYqM&J2{<>3X&(P3j7M&4kt9#}UU}w(36LJ`-`frCo#n=({eo6j zo7TpE3As=xwQ!kA!&hyaTZuCAOq5cb;So$3||7gadCy+>{6B||i`w7`j?WFxQ4*nR!$7J^0p z((qv0u+>?qjszy>SHOaQuKQx}$bL)l}XhImhj_{HBL72eJD zEAlw`lY3}#*?upnF_VT%0s`ZDl?8hOyLy$)RE~IspT~VGE{Cvg#v~7UJ%1z#KI0$v zj#PNL=Cj{0e0jdsS3@@TLDQ-mZ{x4Mubp-i*Bu`SLLwQl+@1|Tnqf)OOU!S3!oH(5 zNspS2mtV12^I^!?COy;;xgJ;M+I)&qvfZAdR?m&?+GI1;*ssStlnw+w|CkQ4iQ@7O zOZBb8G<;fM;NrAa-^jj2BA)cy8Vgs4iDVwPvOoRREUG!{ci|YFaVRQ}lY}<2lGDsr z#^t5cmnjS6yM(r)u6&L&@Omu)5+#N9y~m`c%_|_lT#XQBomjV{FUs29eD7Z!xA^aI z<)s~570gfF7GOV<^Yj>I^3s2D&G__jhYCWyavUx_7IuSKvc+-iO6MyjqB41pY4WVc zI--OB8L#h6>;jm|o0}o-qQCK=m|_bqcsPO-Y7z0rwOuXnl2(U}psm8^B5>aX&?9F! zVM_*UrNP7T&=!-xk0k451CPFkj8qjT+C>qf0sn-$YEsXDt+S1h$;G;LXTHc&qFMy9 z-fgPVI^GLaU68fz&;4;`-_HTDEg{sA@Ui-Iz%5-+MgvpAfOrG<6XW?^cp9 zZu^&y$Y{wm#|SB?I#H1P8@)s0BHWie(W^CIy9V1Ci}6)U-W4dC1xGE&-On~30lL3e zO1tX-MsP%yL})&yn(L}KB8wS^M+9Cw0*Z}Bea!2x{N~-7#EhzAD#QjtlQlyEO1#83 zQMwt7{F`|iT9dUubXb;K%6kW~b>q_}*Db+GD$JDxsC5W-R^lrfl#>0f{0;MD-#8nI z_Otxts*bPrcvs4O%RCH5I&jhMkWypopUf$FlL>&SQ`G9p1GCcc73$A3HQFDcUVxTX zw&ug#JUK%_2m<%(1CI*ElfTTpUGhp#1Uw)St@-k=5BBwW28G-|mW~r!Ov>sid%1OC zzg$~2F3;jmbfIuIKN*O};z-hS+D&Gm-Bv`pL^UImhodS0}0>y4)>~cfO|#a zMUYkTdTM65U)cvVl_cL71qaf3wnROCksPdv`{eo)dw(6AxhknlBQY*vm|^?u6%(Wh z*uL_=e~jROf^$k$C77Xz`Mu+@Ebc-!-4K}&7?A(uad^U^I~6W1Sf2Y8_rMP^c1t<7 z7g9B-I+wDK1Cbv4K@!ScbiV$rZ}-s8$Q<6Du`iX&IFVi=lP-8pIkob$ta8DI1vfO* zBhIcaN1WOBu#ks5<&zcx`}9*d{s(+hYIx3k1%TtJ%~y+!l3ckR;%!(L^@fPKq(p+m zq-)oTNV`9y0&8s&UZkC2FrnPNG>Z1p$)?$BQ~ww*-*?0>@8jGb|3qyvY_Q!VGC*Yk zV~;2cg6z%!@iTi3D*gJupO?Ki{WRcjTEHzQhcZ_^%MA{M&EHx@brQ?o0uAY^r1g50 zkt(Uy%c-|aPt$ED7&gT6#AvRE2oMA#?$x*5rMd(P@>Ws}0RQNahSdo=>11B2#AAAo zM##Tn0(Wz{Jcdg_%=C_VkbBw0#hK3T!CFkY$3t4gu?>a(P14QIEwu}_Z1^0fqb#U} zex%h(ya3av*=xClpaRMwzhRS&X>c@!pw@8L;mmYUOZ}9Luog=~$V^t6G57stX!;-) z#+7g!IopT|5%Or5+PPjM>(@$!>1Rc@rS2}&^-J2T2HB^bvOjJixz-N}O^{Wg?sxZ0 z6%4jhAf#WzWCM{Oqv~Cx({piQM|@-Ep73ce;9LUv^VjFJnFLLKZON>wdhMa~z^!fR zO%S*X|CFD-0qB`MARbGAjT!UvuMbAsr>j<(_RC+^u>sziu(O=0Mb{F43CM z+y~T)XnfBE`PU~KV_mYQ8Kz?SUdm(BHfPLa!au381cYI5?obp0J850_OChSW$#E!F zzP;ydjs`WNeXQ{O!AUmVxvBr2MDuM-J@r1tdd+f^|%{nnw<@>5x)J-sXs{Cbf zPKFhgPh=GJ6|<5!z%I?9{>2p*wDC}N=7~q`uE&0tu`%biQRxHh$wD*ne2HK-PsUS}=&<-xy&f|2$;84s!;Tc+uS@b@lu ze4sZL>9x=p3}^D)AXJT9{)RB`)Tom(_R(STDsXmPL~J1o0xv13@K4{5E4LFUgC^=d zoc2&LxYAN-8)5k7vx_}hB|U7HWgNTTbwl@;{Q9Nv8u#K5_p%$l%YI;e$G1y-8y3?c z)gnPVq~Cm;MZ5fNE!Kx@~8^ds^f zchhAJN;r}+nQXkt^^gi@6TSL%<-1C1d_~1o(jI6;wOd)$&L3YNS~R}L=|2$f{qqNj zQC#)8nV+<7f!6~WG#%_cioBl=$fLX)6yb4Qy+&8H5} zuo8soG|6ebrJU7i4>engY6o^x8yk=Fn6l9@*7d zNQo!?(<%kPUR{zsLL~+@&>*ZQh%WWb$s?#&HBY>b_*~8K2qXfALlNjm@y;(KzpFeRvOeIifhi|Rnr`|7n`zoT({C=|e#D(E4 z_Z=91Tj%`pVLD&8STxrE$aj^)kwf??+J>w|MT$sV4>HpzW`eM(BbGjLb2?;QnPWPL zIaWyv8n@QTl7JG*38t>CV8J{)%X6{yEe)8t_(griEe)#n`F@#UOM#P@iISf3)S+ta=cxys*?5E@F1^}TB8i7El=?X zH*e0B86KK%e$ST9{Rg1AL9* zM65h|Xc?!ygI2iubAfA?>;;(OHN-D z?0k{e);17XIQxNFU`CW%{l#$6V_tHS?&7l+;ROhSJS@e9pQq}8Ai;1qM{m0k`ka+3hWQeoY$IYX1WN>Yx+8x&A*C3gu zu~sYex=E+rU0#mkS|3N!c6(Ux_nUhYK3qC;;BRo+i^PYRu8COg7cT@rXVVCnRc^i* zC3*R)n^)nk67D7Kn{3S5VHZho<{vBDT{%I@or=!~XvX3Dv+8qw#18xi+)^)kA*GvL zPPoLfV-kb|UdY4GpQmTmf3FP-c%X5{J>-wD>U-PI!xWKwf*VX_nI-H;9&2r)4pJ-P zq!uPpj1%R=NGRjFp_M&sH`jKr9uw6C&Uiu~)B7JJd_uvYI94aH{-LV}j@M?@h3{LzLXf8g0|(4nv^CDX7rw=f{NE@tgx6G0Ez zZTQUY@)GIAPS?r#>T3y!g*ThcSFp8rR;;W~kIKjvehE{lMOtN|Uc~lReSgp`D6Z8m zh3lpc2H{Gi&uwvGJ&FooAY!v4?+m;)u>m1VaEd;oqH5D_+-R3PL^Oi}dMAdX2qX^F z{p@gGbkE1?msN)x$^m3HO;tE0&>v43q_iVk7pm{ah0c4*^NGcU&_BPC7+F;(1G1Q^ z7R_y=U%yXy8LBKE(mPcS1(i}RGM6sQkw+_A#q?9jmRs#*C$`39GX$CmHaghOjx@>E zGoky11>;9N48emM1vQpepnPy3!cc5fZx1q$18_UP6rTSAGQQCM7j^G&y;m2R9I_-L zM>;d%kE7Yj?_uyB9$?n^XP6(C29M6^pHiC!p{MwD2yQYQ&AaIt*;~-Y)1Lt4(B6T$ zb=f4{En~P=FoBw$j(vx30;eIS2T!R>dc1Q$ptB>tU#rQY9Uk(GezyWp`6{dKtZhp4 zyO75!wIKv%h4UC2UhJme<-0<`iubqi4p6;S;6vUT)e=sgmB_2i%Zy9Hgs7r!0#(_$M>B>hdsChG>%Bcy{KzTj(E=XOOdmw*| zVMiOTaNHGIT0w3AIft=_iJ)KqB|ki~VeTtpQcTDRs_2rupa`A+!7GQaIxy76FV^LK zwWFA>34?temRpDl=rky`$oKI$ zn!nz9y`HPM?TI?2(6%t>QGaRX<h%8b|HuV04INd*>9LY{RLVe7Y_q}>%2j_&*G$=rRt1jpMv*Y#-^ zoYt*(Y^j8A=oB1cQ{!}}f`;NUX=4Z^_`F*tMB2T=f;M`G;h z+eg50mTz5EM$+o8SOaj7Fzl1Z zJH$!y_Wf(4TwhhWhwL1m*bv9d?YA104ijW;Jr-qr4r-Dm9(Qi{ERmiQ4sByUw|yEO z_8j4xcyi@|C-D(e`2+PX;L>Lf|0b`17HI+F) z$T<+%gigQi4idQ`#ZCHJ&35wfz9FA-MNX)lyq8u6_}6J;4fy#-Kt1l5aA2#E>q=j= zJT-Le8G;*TH>vo1&qHvt#GUUq`VND$tZ$J7*!kl=d%*-gXjF{WhfJ%9g4v&*A_|jj z+jvg;+DgC7>gu(BbH(}BE6<|w?$_&I)ObX!(skK%dOT#fgjUQkFd5hT-dH&Z-3@#W z*tm$2yOwgo8dmA9Ao5cpDp9X~RSL!LL<6ZFrYp5mZmtZg?WjLh>HVbvRGuaLOdnnQ z+ZGd*mtGvODi+O?&biF8fCC|k3LD=KR}bbs9zoy(hCc(&ppJ_+`J36R)cG3IZ-7Wo za8=ZX6edV&m)V@zZYb6u47E;Juvl29a`O?I_Jq-p$+;B zfm#($>v1vRJ1C+01~dCz3o2&4QfN_{!3{_u{A|`Wpb4+SNlvh(6Ue!w@Wi(*_YplG zG1HR>C#_SGQ`6LnvBG1!bQ33CkC>RO@uzXkqeK^7MW~fjXL*gBJSS3R?~!`40&Kxc z3=Fy>W-?d9Z;Ml4(yz`4oums6p}B`K3K zn)oo!B4T9qgjc1aVDn8oXWS<7jsbYxU=u%K*<_L2$^j7|6(Es>iXd<)&3l06Tb+kgAC?u*msu3m_NhccEVKfr53sEDN;JvAybJPDDU zxYgcn5U>?mtKzQIYF(p2`%Qu72_zJx*~RBv=%?Gps}voB6=O`0Mw)EGfcJ5z;-K0~ zUNZ1fKq)|$TP_Q=R{U=K&2CTYzL@!dV(m3nd1sx8D?GtaFf$agIlNhrlf^NO!Yog= zbw3_n`B~bcxbNS1FR!0s^!EYXZ&W$vK)rlhYA28)4|*&qbIn>ZeAoDY z;sHFLA#k(|3R7a0z?&~gWm(m>rgqtWuES@^Y3KQK!(J8VJiHxlyJ8>Lf{MR0bICUA zu!43|H+sM!_26)U|lsqns%I6#>k`-%gNph!mt* zuh!AZ!qirRDNwx$J~tmKD6;cRZz!NrREFPJ$64IA$*ym*n_ZA}h@Mn*{b&|tpaR|u z)UVK9pk&(D$5VU^dv~R|L`2a*?gq73gu%5>8!f`6JPj(+Bdui`5ycy+`1LM6qiBm+ zUnnylXp~3?86}q^e>}tuA_s(tY9|gYy`xspi6?SH(PrKsHtq4GRaLya&viuG3T3SG zbPT;&Vy1}DObJHZ_wxW8nS2Fk}Irh7q!wPS=8)f|-krCv8P z9{K=*7s%ebjnW4uOM*&!rsK;V9r+Y{=4++?cx6hy zY!kLK5!EPf97cpor}l&5Mv2`LOXJr0`SlWC@#1&^G$&fR*uFg7abu!vtJI+)W82-j{Jp~<1Fq%tr(&Z>LfV84 zGmNt%hXH|%U%DvfvFl$mnzA!hCeAg3dS6yhc(TNbhmpz|#~$1XdJbj^*1sYtP&TmO zi?iTJe2{qI4eplZXu$86ue(e$c{K)!<{nB?OPmo+e z(~9hT1)RKnEgxQ*i{FQv=+__E3T%HBCQZJZD+yv{QDvL4((u{m?U(j|f=_Nh({%X+ zbID@|%w*)h-gNmNwL+IFXVkTCxbru7zbYf)(xmEIlT@bGjYEviCN9>ba>qA2PjTKs z>KLfx`go|Nr9r>R?n2R|5^+$PEPH?>C<`#JC5m~;-+3;;+p_waW5 zq4Ba}9?2i}QN-F9 zfUX?3kz7lYklgD+Qm7R_sOL73E62D&1vBc=;I$e$>Q_8Pk!FrZL?FJOjefBli6F0}AQJ}iojg;1$1Tvb*32FZ-D!|?U_si={s^}M5#egB{J@RaAOk*02 z|dm$jma^995|1xd!5en+y6ykw99xn5uSvl%nhseMnMldvm2N@v1 zIPz~O+|&bcn=~O_KkPbHjBP z;#N-N`O@>GJI7X;sa&*W6Mu(B!GO`w53fCxZcRCN!YG6(x{QE$SzeleNVYq7Wd3JM z#Nh>j^TFq%Aa!}1Up077_V|#^kWE_WoWFeVxPXJmEnoWlfgf_JgAE&PSA!SN@WE`R zo@}sh#e_5PYVbD#_g4+?z(OS~QwM>g?l|DZy)o<)DSh;h-f;nccsrq23AVa~Za}2i zB=a&)M3)t7rzV+R5++J=7gV{cCeIlqAxd;BeXpV}*T!ygu_Nn}=LfPl3(>DHoy*~j zB%WOFv4%Qq2g0x`tye`Nunyop(zGN(SuYY4gjyANVyFGrCEw*;ZH& z9QiKL8XlbvUO`w;8c+?5jg7k(uHz?jr>D1D@85r4clGg}(!Q462f^P$*w~4FhMAT4 zE61Hj2h(a>gPGtzmfq1z1yul?d}l#wRhEq z!bjKq^Ci2Fg6gp}ykE0^1DW-+9&kLFnSy7?<#Q&t?Df>XWZK4$TGWg6j22{v6jV}l zT`8u^^CsTya)E#nMi|`o?@tOqu%AHPjrAI=!D{TEXa{v_JhZYiESfpQ2$9BF<4l#T z12z(Wa5(?{p$N3ea_V14m+i9m_h;=V6|^IIBQU3c(lyv6I^f@eUxTmH-Gc(h&7aGK z$&_G^h{8KDA9~ltMuQharlh(j(SAdW4#|LLeO;rTa4q=!~S>S0%mF8{LVuyum25`e;y9r-N<&+XzU=-O+QL^#Pw8=P(X_vnUg_u-8z27qzILq z>^&!qvxOJR*Gc-A50Xq*l@*7PqT&KQ&eH6WT;R$?Ra2jfc4C;@%PJVG} zz^6Y`_r#N9_7V|cJBMl-wx7$}ly~Rj5u@)V1ePcB{zqqHie6ZF+#dPMKh!LYd8Y#F z&7NF_p`QZ2Eevo(_j3lsyrU1gd6Rf)EMIh`wz@WV4w-zUea4F3a6Ieospe+Wz}9&G zKG-0##l6G&038%l?f-g5`a`!UNF4k_!g_7~MB`p*TM$Kf42elc&I)H`zHM;0U+2Lu z@*&iG(^TU&N?3Gc`MKqUUCYB2uW+H?UQ&FX0`U7SF!sY|nO|JR^%K&nrq`QV@_5MZ znK)#~+ozID@-p$z3wUV(>s4A{HTWwGV7FU!9bx`)j|?7@A{_ezK|(|o^m2X-U@C{} z&ZxBvZ`RZMV5ix~-X~(9umDV9wy862W#Z?aD-~js15ZA#mceuGT-pSy8B$MUffvmd z@jnPF`nNZ|YB@;>l^7!Y9{l~z>PA>no7;Jq*7Y%DOY38^8>iI#x%BNjg^j~%t9v|_ zF{BmL!rK}a+3P(D>$hL&W|Ov77GE}=v$=5~F|v7M$7`3Xi8&9x-YlCSS#pkrg*a3} z2`@`V*XO$q1(7mW{3_!!U+uw-4@Cm_sP+F}ZC@S_W&8F$h^*BeQY3ZXMTL|?StsoY z*=8&aNp?fB&luN4bSFz2lEzXYyX@OgsbtBFeHlg)gE2G8n8{3d&+fbB_x$sG-sk;1 z@BHE8nz@#9KaTTze2?$7%>;LZW$138YVkM1ZR8582h{Zhsj-8*9x#z!P$E!WPdA&! z9#&9dp362=C9=c=^QQgpC?C2XrgLc9xl)%%oU*6ONvd9Q|Jo6j%CAXM@iu0= zc(P2~jNF8HLdD&V;wv^ivCW5=vo>1GsffF~RG|dMm}ip9rlQCIWC_R~N5b4z4gIvT znYqjM!iIH5_Of)5t=c=!&aspp@#UAX$)f~ffD`59kiYOwrj~njncdN#va3TxBk>6! zSMWrKN#I9BlXY_#DNKXzy%6`2tZ3)C4wUGLMZ2(GjX7}>N^ZNz^ek77Nt(XA^=b}8 zbg9>yVR_j*omE!ex!i>sWp;bnU=27cv-;hLq+~l!eGiw^o!o<-96s2GaIdNbumT)~ zMVIwUNu6fv{8gnImQp8G*GNWTr&2xUdJlHhWaU7?EwvS5QrhrpQ&uDO;AjAs#Wu*! z7mqvlNWn_%m#R#W<%VFWX=f-e)~0A%MbluvMr-?ZGu@o!T@5Qk+fF&7WP|(ANI-=M zu7GkO(&N5R-@}v1?Lk763fMc5lD$%dH^u9$Lx)uDiO=}AWqoFjcxlxsAaVgiW|4xc zVy*RD60jRSvU~qkR()hdG@N}`k%@tSykVY>E^lG}dcJHj7r%A_$`!^biGW0H(s8A(}!dDq^+48WlIJFIUln>qwjRj=vI zunAkgfVdO92hK`)Ug#}=ad1h+t}o}juY&FwSetHV4SBt~;o$kf=g;e$B8>-u2Ro8Y zg#}IJ_lT3VvT{3YlMt_rq<+_dHR$wrZqZbE);&a>>mG%ez$0*T#AGrmtZ=^4Co^-N z5>ZrNR(|fVuaKlcO3pK--QgyxVT3gZT{*5iNIkd9Dsp&e=*POO0OF5(mVAfYAIZvO z3zxYf>k!{#xpNJ;ywN10r}Q7zk%sKXPsZv|siI0;;d_`?@lJ>zstx#ZaOvN{n2;|4{ywq@QGh$)28B zXQ>0Kg=+TB45r3~?Cc_HyMr($OYLe@ff|TUzr09|o!7VFofv9#18B?j664A7b`ki) zL+6#REQ*oooQ#U@A=CR@7NC-^cd}5$=@y5^0+kX+0&{~6^E>Ngk>_?PW53>yeAe* z!QJTLJeO)Ul*Z!fsAa0pdtSY(u;~dh)>42n3-tYEX~x7z+&xCD@8~KS#QBnYD;`$) ztXk9tn@DesrA|gjeP0LYUU+K1ty>hD5)vE|Ou``utw(fYm?6VY^EIWB`Q6J4OG}Te zkBcsjNj%P^tTdG35Kwx9TyXdK;ZK4(a1Bb!LeH_*PHs0HsYCurLW8<7UI{ z#=m@@={5CVH2f;dB8D7Y``jF_ctfd9yYMq-@xDS@3R_jLyc3V5i=e~05gRERENE8N z2}gEDUz|0!n=B712~;XGE(pM8HXV4#x8p!HVYrtnaO9X;y-tPH z?)f!n=3PR;_^5O|ibL!^`$eCJF;K1NR_WQn3!2N7Z6wR|++1s4 zEvTi@(~^LE4%MzA7t?15q&j~1e$#6hHyo!@M#Sy&ubYXNmpywL*q7u9p~T5(*pu6> zG5U5d3F;>bH|-s)D)=U|Zs=lh8&T@aKn^}c|F3sv&+_V2Gtl_r;c}t#Rjwoj*{to>E!2yNDTUYrvLt; zC6%?MpZOy&z)_2X?HCXp+s?- zD%n@E7=+&^ev2`d8=b}sk}LAqXEJVqIIZaA=dC1uL8k3#(b*6od9WVLD%3T(M!5qL z9ltF_jLlfG7Lx(-`>KvACaQm_chAh$OE$%jR1PCqkl1bRVkalWu4>T?c~h%+@C)+} z4#B}{ecmx-9zKXV2xx8Yxo@TUL9T-A!kZ@^Znbd_-|@@I35=9lt!UFw{}#Y)QJ2=u z+jIE6L;_oaw--wW9!-j8`9>{or zlUZO?TTmOsLYE?Bs7q_6ea}DGv?)(%xWw*(dXr{@7ruDmIqL!9f_wDznIqmd0;jPp zY;Lep*kVdysm040PF(R-13MP{-uG4q6P07@#xhB5Rb`f2LIXST&W>ouL!QS;1KEw< zs)z4`P%$ElZY!^G7ivPDSq6EIzgql5@kPY#oU7I1rWKDhy^V%6v#cz)*|%5)Ku0rd zV(gu78txRs;t|;!2avg;k)Fz4>$dYFubf}|p1u#&qP?+=N$Px%~ZflYoR<+H_chIhD%b(8& zES<9cR_=^Gn>Kq4!<;?HF?1!|YotnbupMVmearoNuE?O2akD8h*W2=1dbRzt<$kb6 zneZz05JPy2duK6(NZIH++JSWe3-9r} zi}rN1i5vA8-`7<4EBTmnEcdcjnS$k0;SJ)>?T%Kd-ndqyE)6Nus6AmO!}Rlsd7K z?ZakLZtHrAU%26^R-llnbEGSi?vkor?zBjRYnwhzO+CkIvA5e?LMM6#wo}Q z!x0asGX>Cz+1>5vF;?B?c3yxQB&~Pgc%R^g%lRXmy(JuQ%9V{fvgaiYB4rHE2puC8S+hiGfII0TD0J;JHM9QpO z4tDMT6Hd~R6sX(l_LQ*jt>Y3UH%8`nfvkb1qZ!^1GvPRPgVww`vj77(N*Up1pV6i` z=uwMq;>zmxCo3u{3fCIVLcnRIH+pO7*3IB$|IrZn{&?#rnI|XT^Z_uZ7AKW#V8#r9 zj8c3;X6*0Mz5bLz-=ao;s=FH;8rsY8+_Bchj-@O%Z^lj`WW+fWA3X5qa~L(mT}$6n zDt~@r{`l2L-@p2=M+XqUU={Ei0F8fwEZ>h@{*~qYvH$*mxbpk+z~}$+k$8dNt5-KF zZrz$*`Q*uyvtv+r_||HmLkYd@{9`i$Q*t*%Z6}e|s+Y*$+|D%Ny%~4rkOY61AAe0E z)-8AHi9*R}0>G{!isR$Y-8bx<;`I-4O-Be`>_^v}pK=d{wIs_SqZGOQ02OH&9it@1 zD7p-#NvqzC@Mk2YS#0_}2=<>>wMU0eNkf@_h3IwG_{MUgLeajGH)v=zlhGp}N)C9N z{t0CxE9KPaGSrEW1N_5(>kHI*rq$F7A4zlou;s~QbW&(g!SK}EezM5)LFoYZfhXG~ zJrq>QJ)?EHS2@Mo)`%A3TFO!dw&?w~F)zq~y(Ed)hyk*s&_f_osw2Fx?0epLzhpvX z(qegtk9UCHpg*5@9XkzDh}F`z+-AaI@TNLjvh|LVE>X0JSb#Xaz6}g{54)r9qs013|Dk>?h4;0Oz zTPqcna{~s(mI{qd;V?=gjUS~m>06@Z*U}CBKnnEQ>Dgngq&P~oUkgLM)#ztykiEg+ z?;3n}d}I?X`J)kKh#LQvb&fzG=??W^j9gBnN6Ikr^e_|DT2@yR>0D>hI%{0Qlcs^7 z=@9^JbwZIFmdhLk+{%;~-_!UK3Q9^8VmFBe8MgZO20qfvPQ6!>kzwcaX*f89(`opZ zCns!_s1-cUTIh7e1^q=;I$Y!lasqnI?~%^E`T#Jo=_$Hjv6_uSZG?_a2jXjPSv!z@ z#f_uL+*wSSpfPut@kww|h|B6@k6?~YE|MT#k2xoMU^!&$2#O2D#HRRN9K90S!wWY$ zt<-BtnEqC6L0$k~a?!5Y!#Q}*f?|Q?E0pw^oVRYB!tXH%-H1LI(0fexA1=>Gb9_g5 z=iIX0z{SqaKEB0)-#7})dWwbllZAGLmcVVEEU*60#&O^IT|lI=yh#CM%zcGYA68sU z9C+Qv2T3RsWf6$M12=vzqPojvMoW{G5ik8#4T`d(&BZlGu5_b$Z)#rg^c!3Jk8Q5b zHWfd;(jb=AdFp79RL-qISV7)$vuwi@%E8`h|GhVkPbj?_1*I?3d9hL+-S+nIg89l0 ztvwgbtrn@zw6s9{%TKGJ*h;{S8s>TTl-yXDFzW=@8mGb_rC~Jc)K~_ml@(+x_InGB zR&ZZ7LV!2Ov!OokJyg4!zByP@UV#vIjNRapy|@EqP6*NRdv%RvyXn&0;b3a~W3i1K zT4VYd>Av$I`)XDF4zI)R+D~8HBW?LjEuwGO=`uu58L=n2BTv@@dgq25;cW3qfR$=| zBOo+##Qe7^*wRNTBQ0ndo4(X!ITP;ooiHXOB9|d2sVA=*ig)T*|g-&V|lM zGEhLno>UosOp&Yq(pfKn*;raHz1`=q5C5d}JvZJ>;;O4)Am4Gic8)oh&c_lZz@dQ- z|AOqR`iba95~BOGL=Zp3^ z`4@tk{()!ywLb(2vh23|QpqDue|y_Fr;@H9@5bmRNEI^1hLUo%cUx{A7a93pGc3dY z&V#p7w1PQr{dWQw1k`qSF+3c=PbB*jV?ZHMWSXgWM^<#!zP!_O&>E=OwTj!}21k?s zRFA97DCldkn|w9uCGQKH11QnJ#i*~EHqbVaUGR_+13H%Rdacbyk63=EJ8si2qna7y z3%A*II#?*K@WZH7@M5az))tX|&Sn}SxTLA}A!2o_?r!vKc-!>}?PnHyn2z*I_+nVz zUWGC*R}P4;*+hQYM_kbE4Qf7IUs5OKhz@md)B3W#0AzI(q?V_rC^kbt>J@0W3@iwt z9m1N6&Y!PZKeWg=&b8LH^YsoQ1P*OYB_vraT$ZSR54`S9CXCz8BbR(oFNj<)K zIGvi39uR+ex=&=<)uhm&j?1Z&y~yxkxfS0>?_C2@4L=b~y7>K*|1|3XpWPl&eM_lt zI^f#ZD;Pp;+Ilm)di{-sr>iuk7)q>SYasT@Wi5^zN*;3Ka;l_vjLcI+ro*IN4{cI* z%B2~d6X4yEwyz(UDE{Ji)j^1RfjxEzsJTEtHeMG+2Q#0UnM6tn|6$nD%6mhUA{P|* zS339h4xVEU43lkFM_&7v33}Iult{gw%Wu^P!YjsQM9Cf^f=K3bKkj$@PdUn`8d;S;e!br%o4C)f#u~_ZdP9H6I_L7Diwq9OL z(=;7Z;2q8q4AjeeI*sd)AdkMNIvc=k|4!cDDe3T7AFZ)?bRBtAY&Vl6%lCCPE%V+B zCk@{t{93AG@DX9fPVWZLXId`_KQ?T!u(g!QM5-f;KBz8H3ny}gk;5w0Hz)3k6{6>d zqG80@C67h-;l?h=o&b3EGXXf>`Tp3K`+#uC+${OB0E6^MZhFSf?NdH68Vxg`xet`Cop#76 zyP0m<&k?mz5ncj==(ihER4#z&gPfQYSk7+i=ULDBm01Y~f%B`L?GvTxP*&HOPv20t zITK{cU5HL6y_)N;B|bn0aH-5{12QtXpT{AvF5#)>04ojNc+BDrOmAHcE2BQ8ZoMG; zi_f0$QJs*uB72-N5QweZCGvYll?Vma&(?Y@)~PEonIuSY_HgrtOz%W{1!J(K@tXNG zChNE~*T-<#AQWiAwHd#MD|U?<1j>~4>`v)S?mjp&L)~hh0Xn>Fq)54}Ei~mQ-qF!s zEW%W`b=avnUvc-$uXu+WPMu)ykvuf?D(|(g0 zb6%n%^R=*n9G}%V5k!FD0t|?jzqGbtnV_1C^;o9dN25Jbbj#Cs8UA2WYilMnq(k(BkV$N_i=w5J4DGy1`L zM+)brS>qJ5==s}(gK7eGaTl7fJM^oukE@-b^B`MNY=GvT^a}aFqa-MRc=&QTc0je^ z%O2FI8=jgSk`5z=2-N9$9(ReS+!$?LXUEDIz53WgFxK!G4T!)~%3-Jso2eS2IC(%Y zUQs@`TNxGoVCQ)ixH;d)hPxfUetLGSsZmp}iWphw$dTB`&}!;)iEG@I*)u8$Klf%)TR}ITaCCTp z@?sxumdT{d@ApI(dvV0beXKV9kZRA$OZNmYsmZd?y4+UrEopO6fCYvM0|S6gOmj`6 zMoEyoop*7uG8-tQHq0uUSvN^tum@6Q)g9qUTtl|)Cd?d2QDf03gE#sDS=TF%=*NCI z=9%f(M6{D=ea7-vOZQ&|Lb)L+dB^=w;*sbpo|N|kr_Qd8H*7g02zd)1w#&LERfujK zf1Fv8kV9&T(KjR&GeEXfOmTM^B4P~V!k+W47znnp_ji=Z+_SqUb|k-|+j5LvyoR{i zftATj`_P^1b(F$t0R7o&r2VK}+^XPQ#OAj!g93IV@5=CJnHulPilVLO*()SC)7RN6AJ%*egJH@JAI4_!ap)i- z21LF%}cx8{tQdG7C3{Jhiy&i*7ovQ~LYk1bTFW*4PriO_Bc`9OVX!~*3 zJOSA5a^8hqy|*|hKOjk+Th#hQ!kG)CsOCC8|BQdbL{DTl+k8auk@fub{A6%(zv%oFEG*l`RqToAH_9tP!ZYQ2;eX?066{NkZxc=nDoaOOpS&^a1?!^*#TTWEED^L9k8#E;nnca>|0uGCCN%W1i}8WajnfPqky~>IDfC{;Sg(4N7rq<8q_E zF8uV5@oT^p+-;P`c@sWRlzjecM0|ga^)b9-50IBci8c)QDh5x5{@WDa|Khzw$=csq(mh`UB5*tP?KcQ*TwVGPSZU-!t7BlSi+T0|DVH<2a< zn>P3jp4|S{{cV^qJv&MFr%*;0>NLJ@iG$Uu)isJSkRmqN%W&{{{)05f(1idw#N!Ac z$;RD%)lXq=P8^Q^CEeg&=kysr&t$ANUhb4G!jpF6<)tH%g^DGe7^#3eabl z0uR8Abai7(=xlbc9^B~2k(-GFRTT^Kd&V6nS85Ux6DJnV=$%?d*`R7$VEfd%jnSth2hx zVOG|9s@)QtUR@0_FnFV?uj?!-@Tx%QdZ76q=IEe3Tb*{=1#jEbY;TjFBYi5lgj9_q z)hZbUmAntQU_g3l!F?36C9^3aY}kNLu6!@b4WDY@z9i-%NtOe;-1g9~wE`Hy(E&(f ztT5g1n+wBmi+Ym@yrDqDtSxlVFWQIX9oy60)strEFWjNv!#M5R^)XZp(G)Tcn&6(zj$kM7`1 zy=+=nZ2S`0;uj$rMAwoHhx$twzO{#Lw`1wZ%=g1^JqtSzHx|H%F~yWZlcW#TF{M^W zV~VK&<`rd4vkUh$9w#^29hRs)SBz}z@sWH`?l`o$%P;vs-58nITe+p~OPt-ktau5R ztZ?mvKD=T}1^$D6*!=3*e|Ih~{scl9Z6wF7jdOK9tW}N&0}4?ZTMcs;!0`2JhiSMF z-inzZ`x|;}^ie@^`v^AsR;?kHdCydENc47PvG-vWc!|K3I$pUU8s5ZEjzU$8hY3y= zA_$r6USWf=?8`w3!mJzE{wHonWj)xT^i}MvxoI| zdZ&B{DST2Zb#npNduJ&uL%vsqO*Gw-kvm)0D z-wY04mcnX_zggzA;h+TgKx9sbJcN{pFCiu2f)*Kz* zcM6p@^KI`43?_>F{+0uIH(KG>723>p)}%iBxw{cRw2EWsA}ugG#H=F=_{Kp0?qKjI z)=XD=S10{39#9|Tw-;`%viJa#!@2e3UdBH7>X)VPTyD`0|5x+_i-oML+sn!osKsLn zs(%Wsl&2H_6~+p3Zz=S@rHe^INmJdaw$$h0He3A0TaR-$9`Q3HTd>z%gGvd znJ~k#s%zd-!9C0RISI9XQH93wmZ;ij`imbkGo%s}E*}RIh6_lXD@%eqou`WlF&RAeShH#NwjMPiUr4^({IS%CXUo^-iqRLlj=GzJf0#~5U zlL^mgWh4X042Jjr)i8o%YqIMdar;w>$LJ%u38lV5>^zGxt;&2G zXq_0h`xu5}@qjIN891pr*}j)ff6&iPw;L-)GA@%U3xG0))x;uu33fd@_}+^z=bfe#k|oJZzy}xtuT1C?mF~_5eqgq1kG!(*lNsGf4ek}90V1v0K zqytc}`sl+4=8e`={q ztP$tHhYU3u9VHu?U%p&bzr4-91e1~1wHyL2W$C!TP4sMD^@)Lf!2AIJ)&7JlDKrx@ zD?~D9R(yB8-X^ii=01_M=r|-KqmxISVyA?4ezcEc&$`=DHT*!)k+eM^iC1uWANC&0=&(AC3oKWZM7 z;&f=VN6d7L5HZcPh_@E{9$f&dLV@@04`wusAO+KRkl6E{ zy3(lV;&g1J!CmlUgYrb%!dNh&rPIU79-p2-_j&Ma2gYkBE0r`q&sCy(`!pZb%lZIp z+^Bm85#sT)wfSR8BUlWerF|mO7oG-}Ie(a&u&Z`{8bh-}&BA<6{Wc!g|2}qZJ7(zlQp5Su?5}! z3>v+kRDQ@V<9gecGZ}Y2*cBA^kXCAW@5RJ`K4!6kntRlV$oU9h@_jO#gBq*Ds~Xo; zLxc#}3*f8&<9=?E1^Dyatsf?V_}vvV`N7D#=m+M&i@QJO+@J!O;-x!(T*d)vo-nnW zKQCLttOPAiE`)`k936SeFg$VNPgMKG4>tfjOi}UQcp4c0rN4fMpIZClC3>*JU-33_ z*RQYvAeH}9P8cjO@V~j||Kyg?|A%}2Pi_hQ@b~}sst){ZR6ktmnSfTNNH1RgypY#_ zcYyroylu;`ytl)hUZi{|6ZvmX|F3twb_(khwJIP**tB8QV}|~{Z_b=BJ6>#jG4lTa DgR{!l diff --git a/README.md b/README.md index 6f0cf205c..7554e2737 100644 --- a/README.md +++ b/README.md @@ -295,7 +295,7 @@ LiveKit server is licensed under Apache License v2.0.
- + From 6198aac7a8d310c42f2326c2cf5906ad1e257fdd Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 3 Jul 2023 23:49:03 +0530 Subject: [PATCH 268/324] Make a default config variable (#1848) * Make a default config variable * decoder.Decode needs pointer to config --- pkg/config/config.go | 323 ++++++++++++++++++++++--------------------- 1 file changed, 162 insertions(+), 161 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 77bfe4adf..8b7b340e7 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -264,171 +264,172 @@ func DefaultAPIConfig() APIConfig { } } +var DefaultConfig = Config{ + Port: 7880, + RTC: RTCConfig{ + RTCConfig: rtcconfig.RTCConfig{ + UseExternalIP: false, + TCPPort: 7881, + UDPPort: 0, + ICEPortRangeStart: 0, + ICEPortRangeEnd: 0, + STUNServers: []string{}, + }, + PacketBufferSize: 500, + StrictACKs: true, + PLIThrottle: PLIThrottleConfig{ + LowQuality: 500 * time.Millisecond, + MidQuality: time.Second, + HighQuality: time.Second, + }, + CongestionControl: CongestionControlConfig{ + Enabled: true, + AllowPause: false, + ProbeMode: CongestionControlProbeModePadding, + ProbeConfig: CongestionControlProbeConfig{ + BaseInterval: 3 * time.Second, + BackoffFactor: 1.5, + MaxInterval: 2 * time.Minute, + + SettleWait: 250 * time.Millisecond, + SettleWaitMax: 10 * time.Second, + + TrendWait: 2 * time.Second, + + OveragePct: 120, + MinBps: 200_000, + MinDuration: 200 * time.Millisecond, + MaxDuration: 20 * time.Second, + DurationOverflowFactor: 1.25, + DurationIncreaseFactor: 1.5, + }, + }, + }, + Audio: AudioConfig{ + ActiveLevel: 35, // -35dBov + MinPercentile: 40, + UpdateInterval: 400, + SmoothIntervals: 2, + }, + Video: VideoConfig{ + DynacastPauseDelay: 5 * time.Second, + StreamTracker: StreamTrackersConfig{ + Video: StreamTrackerConfig{ + StreamTrackerType: StreamTrackerTypePacket, + BitrateReportInterval: map[int32]time.Duration{ + 0: 1 * time.Second, + 1: 1 * time.Second, + 2: 1 * time.Second, + }, + PacketTracker: map[int32]StreamTrackerPacketConfig{ + 0: { + SamplesRequired: 1, + CyclesRequired: 4, + CycleDuration: 500 * time.Millisecond, + }, + 1: { + SamplesRequired: 5, + CyclesRequired: 20, + CycleDuration: 500 * time.Millisecond, + }, + 2: { + SamplesRequired: 5, + CyclesRequired: 20, + CycleDuration: 500 * time.Millisecond, + }, + }, + FrameTracker: map[int32]StreamTrackerFrameConfig{ + 0: { + MinFPS: 5.0, + }, + 1: { + MinFPS: 5.0, + }, + 2: { + MinFPS: 5.0, + }, + }, + }, + Screenshare: StreamTrackerConfig{ + StreamTrackerType: StreamTrackerTypePacket, + BitrateReportInterval: map[int32]time.Duration{ + 0: 4 * time.Second, + 1: 4 * time.Second, + 2: 4 * time.Second, + }, + PacketTracker: map[int32]StreamTrackerPacketConfig{ + 0: { + SamplesRequired: 1, + CyclesRequired: 1, + CycleDuration: 2 * time.Second, + }, + 1: { + SamplesRequired: 1, + CyclesRequired: 1, + CycleDuration: 2 * time.Second, + }, + 2: { + SamplesRequired: 1, + CyclesRequired: 1, + CycleDuration: 2 * time.Second, + }, + }, + FrameTracker: map[int32]StreamTrackerFrameConfig{ + 0: { + MinFPS: 0.5, + }, + 1: { + MinFPS: 0.5, + }, + 2: { + MinFPS: 0.5, + }, + }, + }, + }, + }, + Redis: redisLiveKit.RedisConfig{}, + Room: RoomConfig{ + AutoCreate: true, + EnabledCodecs: []CodecSpec{ + {Mime: webrtc.MimeTypeOpus}, + {Mime: "audio/red"}, + {Mime: webrtc.MimeTypeVP8}, + {Mime: webrtc.MimeTypeH264}, + // {Mime: webrtc.MimeTypeAV1}, + // {Mime: webrtc.MimeTypeVP9}, + }, + EmptyTimeout: 5 * 60, + }, + Logging: LoggingConfig{ + PionLevel: "error", + }, + TURN: TURNConfig{ + Enabled: false, + }, + NodeSelector: NodeSelectorConfig{ + Kind: "any", + SortBy: "random", + SysloadLimit: 0.9, + CPULoadLimit: 0.9, + }, + SignalRelay: SignalRelayConfig{ + Enabled: false, + RetryTimeout: 7500 * time.Millisecond, + MinRetryInterval: 500 * time.Millisecond, + MaxRetryInterval: 4 * time.Second, + StreamBufferSize: 1000, + }, + Keys: map[string]string{}, +} + func NewConfig(confString string, strictMode bool, c *cli.Context, baseFlags []cli.Flag) (*Config, error) { // start with defaults - conf := &Config{ - Port: 7880, - RTC: RTCConfig{ - RTCConfig: rtcconfig.RTCConfig{ - UseExternalIP: false, - TCPPort: 7881, - UDPPort: 0, - ICEPortRangeStart: 0, - ICEPortRangeEnd: 0, - STUNServers: []string{}, - }, - PacketBufferSize: 500, - StrictACKs: true, - PLIThrottle: PLIThrottleConfig{ - LowQuality: 500 * time.Millisecond, - MidQuality: time.Second, - HighQuality: time.Second, - }, - CongestionControl: CongestionControlConfig{ - Enabled: true, - AllowPause: false, - ProbeMode: CongestionControlProbeModePadding, - ProbeConfig: CongestionControlProbeConfig{ - BaseInterval: 3 * time.Second, - BackoffFactor: 1.5, - MaxInterval: 2 * time.Minute, - - SettleWait: 250 * time.Millisecond, - SettleWaitMax: 10 * time.Second, - - TrendWait: 2 * time.Second, - - OveragePct: 120, - MinBps: 200_000, - MinDuration: 200 * time.Millisecond, - MaxDuration: 20 * time.Second, - DurationOverflowFactor: 1.25, - DurationIncreaseFactor: 1.5, - }, - }, - }, - Audio: AudioConfig{ - ActiveLevel: 35, // -35dBov - MinPercentile: 40, - UpdateInterval: 400, - SmoothIntervals: 2, - }, - Video: VideoConfig{ - DynacastPauseDelay: 5 * time.Second, - StreamTracker: StreamTrackersConfig{ - Video: StreamTrackerConfig{ - StreamTrackerType: StreamTrackerTypePacket, - BitrateReportInterval: map[int32]time.Duration{ - 0: 1 * time.Second, - 1: 1 * time.Second, - 2: 1 * time.Second, - }, - PacketTracker: map[int32]StreamTrackerPacketConfig{ - 0: { - SamplesRequired: 1, - CyclesRequired: 4, - CycleDuration: 500 * time.Millisecond, - }, - 1: { - SamplesRequired: 5, - CyclesRequired: 20, - CycleDuration: 500 * time.Millisecond, - }, - 2: { - SamplesRequired: 5, - CyclesRequired: 20, - CycleDuration: 500 * time.Millisecond, - }, - }, - FrameTracker: map[int32]StreamTrackerFrameConfig{ - 0: { - MinFPS: 5.0, - }, - 1: { - MinFPS: 5.0, - }, - 2: { - MinFPS: 5.0, - }, - }, - }, - Screenshare: StreamTrackerConfig{ - StreamTrackerType: StreamTrackerTypePacket, - BitrateReportInterval: map[int32]time.Duration{ - 0: 4 * time.Second, - 1: 4 * time.Second, - 2: 4 * time.Second, - }, - PacketTracker: map[int32]StreamTrackerPacketConfig{ - 0: { - SamplesRequired: 1, - CyclesRequired: 1, - CycleDuration: 2 * time.Second, - }, - 1: { - SamplesRequired: 1, - CyclesRequired: 1, - CycleDuration: 2 * time.Second, - }, - 2: { - SamplesRequired: 1, - CyclesRequired: 1, - CycleDuration: 2 * time.Second, - }, - }, - FrameTracker: map[int32]StreamTrackerFrameConfig{ - 0: { - MinFPS: 0.5, - }, - 1: { - MinFPS: 0.5, - }, - 2: { - MinFPS: 0.5, - }, - }, - }, - }, - }, - Redis: redisLiveKit.RedisConfig{}, - Room: RoomConfig{ - AutoCreate: true, - EnabledCodecs: []CodecSpec{ - {Mime: webrtc.MimeTypeOpus}, - {Mime: "audio/red"}, - {Mime: webrtc.MimeTypeVP8}, - {Mime: webrtc.MimeTypeH264}, - // {Mime: webrtc.MimeTypeAV1}, - // {Mime: webrtc.MimeTypeVP9}, - }, - EmptyTimeout: 5 * 60, - }, - Logging: LoggingConfig{ - PionLevel: "error", - }, - TURN: TURNConfig{ - Enabled: false, - }, - NodeSelector: NodeSelectorConfig{ - Kind: "any", - SortBy: "random", - SysloadLimit: 0.9, - CPULoadLimit: 0.9, - }, - SignalRelay: SignalRelayConfig{ - Enabled: false, - RetryTimeout: 7500 * time.Millisecond, - MinRetryInterval: 500 * time.Millisecond, - MaxRetryInterval: 4 * time.Second, - StreamBufferSize: 1000, - }, - Keys: map[string]string{}, - } - + conf := DefaultConfig if confString != "" { decoder := yaml.NewDecoder(strings.NewReader(confString)) decoder.KnownFields(strictMode) - if err := decoder.Decode(conf); err != nil { + if err := decoder.Decode(&conf); err != nil { return nil, fmt.Errorf("could not parse config: %v", err) } } @@ -473,7 +474,7 @@ func NewConfig(confString string, strictMode bool, c *cli.Context, baseFlags []c conf.Environment = "dev" } - return conf, nil + return &conf, nil } func (conf *Config) IsTURNSEnabled() bool { From 4afb0e0b9b2a0a83e3d9637557180e76fcd77f75 Mon Sep 17 00:00:00 2001 From: David Colburn Date: Wed, 5 Jul 2023 15:03:18 -0700 Subject: [PATCH 269/324] retry egress on timeout/resource exhausted (#1852) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 2a5b7b9e9..c0b20b832 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230612070454-d5299b956135 - github.com/livekit/protocol v1.5.9-0.20230701042848-e5323cdb4ab3 + github.com/livekit/protocol v1.5.9-0.20230705215502-424b21fd0e92 github.com/livekit/psrpc v0.3.1 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 diff --git a/go.sum b/go.sum index 1c1db1c6c..6c8d2f96e 100644 --- a/go.sum +++ b/go.sum @@ -124,8 +124,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20230612070454-d5299b956135 h1:lWYbsondvqG69czxoACDwaJ/BoyD57BahCo70ZH+m4U= github.com/livekit/mediatransportutil v0.0.0-20230612070454-d5299b956135/go.mod h1:MRc0zSOSzXuFt0X218SgabzlaKevkvCckPgBEoHYc34= -github.com/livekit/protocol v1.5.9-0.20230701042848-e5323cdb4ab3 h1:OUWOLcsgEJ3o5p5NAlebqHzN0xJHBWl1HBUwMe/PZv4= -github.com/livekit/protocol v1.5.9-0.20230701042848-e5323cdb4ab3/go.mod h1:B6hJiuXT84dHsUgaKHBo+ZLPX4XhklptYA2UbANSiNg= +github.com/livekit/protocol v1.5.9-0.20230705215502-424b21fd0e92 h1:R0Y13wZIsH+82MqqOl5FMW3T8iRNcdW16vSLyfpUxck= +github.com/livekit/protocol v1.5.9-0.20230705215502-424b21fd0e92/go.mod h1:B6hJiuXT84dHsUgaKHBo+ZLPX4XhklptYA2UbANSiNg= github.com/livekit/psrpc v0.3.1 h1:KfylgJHvoLQcc22t/oflwMOeSnx0c14G7cWsS+9MYS4= github.com/livekit/psrpc v0.3.1/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= From 919355c873a35afe3aa5e0fc3add2d17431d82ad Mon Sep 17 00:00:00 2001 From: David Zhao Date: Thu, 6 Jul 2023 23:38:01 -0700 Subject: [PATCH 270/324] Log additional details when updating participant permissions (#1855) To help track down sporadic updateParticipant failures --- pkg/rtc/participant.go | 2 ++ pkg/service/roomservice.go | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index a89315c10..2d2f43fc4 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -370,6 +370,8 @@ func (p *ParticipantImpl) SetPermission(permission *livekit.ParticipantPermissio return false } + p.GetLogger().Infow("updating participant permission", "permission", permission) + video.UpdateFromPermission(permission) p.dirty.Store(true) diff --git a/pkg/service/roomservice.go b/pkg/service/roomservice.go index 749e367de..31b368cf0 100644 --- a/pkg/service/roomservice.go +++ b/pkg/service/roomservice.go @@ -2,6 +2,7 @@ package service import ( "context" + "fmt" "strconv" "time" @@ -14,6 +15,7 @@ import ( "github.com/livekit/livekit-server/pkg/routing" "github.com/livekit/livekit-server/pkg/rtc" "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" "github.com/livekit/protocol/rpc" ) @@ -273,20 +275,24 @@ func (s *RoomService) UpdateParticipant(ctx context.Context, req *livekit.Update } var participant *livekit.ParticipantInfo + var detailedError error err = s.confirmExecution(func() error { participant, err = s.roomStore.LoadParticipant(ctx, livekit.RoomName(req.Room), livekit.ParticipantIdentity(req.Identity)) if err != nil { return err } if req.Metadata != "" && participant.Metadata != req.Metadata { + detailedError = fmt.Errorf("metadata does not match") return ErrOperationFailed } if req.Permission != nil && !proto.Equal(req.Permission, participant.Permission) { + detailedError = fmt.Errorf("permissions do not match, expected: %v, actual: %v", req.Permission, participant.Permission) return ErrOperationFailed } return nil }) if err != nil { + logger.Warnw("could not confirm participant update", detailedError) return nil, err } From 873c87f24b73a8733e501a838e46d44472e735f4 Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Fri, 7 Jul 2023 15:46:18 +0800 Subject: [PATCH 271/324] Fix nack issue for svc codecs (#1856) * Fix nack issue for svc codecs * Fix test --- pkg/sfu/forwarder.go | 6 ++++-- pkg/sfu/videolayerselector/dependencydescriptor.go | 6 +++--- pkg/sfu/videolayerselector/dependencydescriptor_test.go | 7 ++++--- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 75e00baa9..92fae3ea7 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -1599,8 +1599,10 @@ func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer in if !result.IsSelected { tp.shouldDrop = true if f.started && result.IsRelevant { - f.rtpMunger.UpdateAndGetSnTs(extPkt) // call to update highest incoming sequence number and other internal structures - f.rtpMunger.PacketDropped(extPkt) + // call to update highest incoming sequence number and other internal structures + if _, err := f.rtpMunger.UpdateAndGetSnTs(extPkt); err == nil { + f.rtpMunger.PacketDropped(extPkt) + } } return tp, nil } diff --git a/pkg/sfu/videolayerselector/dependencydescriptor.go b/pkg/sfu/videolayerselector/dependencydescriptor.go index 101c73e1a..3cc5fcf70 100644 --- a/pkg/sfu/videolayerselector/dependencydescriptor.go +++ b/pkg/sfu/videolayerselector/dependencydescriptor.go @@ -46,6 +46,9 @@ func (d *DependencyDescriptor) IsOvershootOkay() bool { } func (d *DependencyDescriptor) Select(extPkt *buffer.ExtPacket, _layer int32) (result VideoLayerSelectorResult) { + // a packet is always relevant for the svc codec + result.IsRelevant = true + ddwdt := extPkt.DependencyDescriptor if ddwdt == nil { // packet doesn't have dependency descriptor @@ -54,9 +57,6 @@ func (d *DependencyDescriptor) Select(extPkt *buffer.ExtPacket, _layer int32) (r dd := ddwdt.Descriptor - // a packet is relevant as long as it has DD extension - result.IsRelevant = true - frameNum := d.frameNum.Update(dd.FrameNumber) extFrameNum := frameNum.ExtendedVal diff --git a/pkg/sfu/videolayerselector/dependencydescriptor_test.go b/pkg/sfu/videolayerselector/dependencydescriptor_test.go index 21e416691..0a4ead314 100644 --- a/pkg/sfu/videolayerselector/dependencydescriptor_test.go +++ b/pkg/sfu/videolayerselector/dependencydescriptor_test.go @@ -4,11 +4,12 @@ import ( "sort" "testing" + "github.com/pion/rtp" + "github.com/stretchr/testify/require" + "github.com/livekit/livekit-server/pkg/sfu/buffer" dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" "github.com/livekit/protocol/logger" - "github.com/pion/rtp" - "github.com/stretchr/testify/require" ) func TestDecodeTarget(t *testing.T) { @@ -124,7 +125,7 @@ func TestDependencyDescriptor(t *testing.T) { // no dd ext, dropped ret := ddSelector.Select(&buffer.ExtPacket{}, 0) require.False(t, ret.IsSelected) - require.False(t, ret.IsRelevant) + require.True(t, ret.IsRelevant) // non key frame, dropped ret = ddSelector.Select(&buffer.ExtPacket{ From 3e71ea3d77cc4d4f2d645bba938a4eb4c120fc88 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Fri, 7 Jul 2023 13:36:15 -0700 Subject: [PATCH 272/324] Fixed hidden participant update (#1857) --- pkg/rtc/participant.go | 7 +++---- pkg/rtc/participant_signal.go | 4 ++++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 2d2f43fc4..5de6904c1 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -625,14 +625,13 @@ func (p *ParticipantImpl) removeMutedTrackNotFired(mt *MediaTrack) { // AddTrack is called when client intends to publish track. // records track details and lets client know it's ok to proceed func (p *ParticipantImpl) AddTrack(req *livekit.AddTrackRequest) { - p.lock.Lock() - defer p.lock.Unlock() - - if !p.grants.Video.GetCanPublishSource(req.Source) { + if !p.CanPublishSource(req.Source) { p.params.Logger.Warnw("no permission to publish track", nil) return } + p.lock.Lock() + defer p.lock.Unlock() ti := p.addPendingTrackLocked(req) if ti == nil { return diff --git a/pkg/rtc/participant_signal.go b/pkg/rtc/participant_signal.go index 192ef0b48..305e920e5 100644 --- a/pkg/rtc/participant_signal.go +++ b/pkg/rtc/participant_signal.go @@ -86,6 +86,10 @@ func (p *ParticipantImpl) SendParticipantUpdate(participantsToUpdate []*livekit. isValid = false } } + if pi.Permission != nil && pi.Permission.Hidden && pi.Sid != string(p.params.SID) { + p.params.Logger.Debugw("skipping hidden participant update", "otherParticipant", pi.Identity) + isValid = false + } if isValid { p.updateCache.Add(pID, participantUpdateInfo{version: pi.Version, state: pi.State, updatedAt: time.Now()}) validUpdates = append(validUpdates, pi) From bf3732b898fd8ec29a30028bdc9fd05b27fb5df2 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sat, 8 Jul 2023 11:58:56 +0530 Subject: [PATCH 273/324] Remove noisy debug logs. (#1858) --- pkg/sfu/buffer/rtpstats.go | 45 -------------------------------------- 1 file changed, 45 deletions(-) diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index ce702409b..c21938d36 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -821,22 +821,6 @@ func (r *RTPStats) SetRtcpSenderReportData(srData *RTCPSenderReportData) { "highestTS", r.highestTS, "highestTime", r.highestTime.String(), ) - } else { - packetDriftResult, reportDriftResult := r.getDrift() - r.logger.Debugw( - "received sender report", - "ntp", srData.NTPTimestamp.Time().String(), - "rtp", srData.RTPTimestamp, - "arrival", srData.At.String(), - "ntpDiffSinceLast", ntpDiffSinceLast.Seconds(), - "rtpDiffSinceLast", int32(rtpDiffSinceLast), - "arrivalDiffSinceLast", arrivalDiffSinceLast.Seconds(), - "expectedTimeDiffSinceLast", expectedTimeDiffSinceLast, - "packetDrift", packetDriftResult.String(), - "reportDrift", reportDriftResult.String(), - "highestTS", r.highestTS, - "highestTime", r.highestTime.String(), - ) } } @@ -872,19 +856,6 @@ func (r *RTPStats) GetExpectedRTPTimestamp(at time.Time) (uint32, uint64, error) if r.srNewest != nil { minTS = r.srNewest.RTPTimestampExt } - r.logger.Debugw( - "expected RTP timestamp", - "firstTime", r.firstTime.String(), - "checkAt", at.String(), - "timeDiff", timeDiff, - "firstRTP", r.extStartTS, - "expectedRTPDiff", expectedRTPDiff, - "expectedExtRTP", expectedExtRTP, - "expectedRTP", uint32(expectedExtRTP), - "minTS", minTS, - "highestTS", r.highestTS, - "highestTime", r.highestTime.String(), - ) return uint32(expectedExtRTP), minTS, nil } @@ -968,22 +939,6 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32, srFirst *RTCPSenderReportDat "highestTS", r.highestTS, "highestTime", r.highestTime.String(), ) - } else { - packetDriftResult, reportDriftResult := r.getDrift() - r.logger.Debugw( - "sending sender report", - "ntp", nowNTP.Time().String(), - "rtp", nowRTP, - "departure", now.String(), - "ntpDiffSinceLast", ntpDiffSinceLast.Seconds(), - "rtpDiffSinceLast", int32(rtpDiffSinceLast), - "departureDiffSinceLast", departureDiffSinceLast.Seconds(), - "expectedTimeDiffSinceLast", expectedTimeDiffSinceLast, - "packetDrift", packetDriftResult.String(), - "reportDrift", reportDriftResult.String(), - "highestTS", r.highestTS, - "highestTime", r.highestTime.String(), - ) } return &rtcp.SenderReport{ From cbec68ae44371323d9cbada9364d1836aa6adc72 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Sat, 8 Jul 2023 12:06:31 -0700 Subject: [PATCH 274/324] Do not use cancellable context for Redis operations (#1859) --- pkg/service/redisstore.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/service/redisstore.go b/pkg/service/redisstore.go index a11c318ad..6656757da 100644 --- a/pkg/service/redisstore.go +++ b/pkg/service/redisstore.go @@ -246,9 +246,9 @@ func (s *RedisStore) LockRoom(_ context.Context, roomName livekit.RoomName, dura return "", ErrRoomLockFailed } -func (s *RedisStore) UnlockRoom(ctx context.Context, roomName livekit.RoomName, uid string) error { +func (s *RedisStore) UnlockRoom(_ context.Context, roomName livekit.RoomName, uid string) error { key := RoomLockPrefix + string(roomName) - res, err := s.unlockScript.Run(ctx, s.rc, []string{key}, uid).Result() + res, err := s.unlockScript.Run(s.ctx, s.rc, []string{key}, uid).Result() if err != nil { return err } @@ -708,7 +708,7 @@ func (s *RedisStore) LoadIngressFromStreamKey(_ context.Context, streamKey strin } } -func (s *RedisStore) ListIngress(ctx context.Context, roomName livekit.RoomName) ([]*livekit.IngressInfo, error) { +func (s *RedisStore) ListIngress(_ context.Context, roomName livekit.RoomName) ([]*livekit.IngressInfo, error) { var infos []*livekit.IngressInfo if roomName == "" { From 42a7d522724691d1796d3aa9f04da45efb589b02 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Sat, 8 Jul 2023 22:30:49 -0700 Subject: [PATCH 275/324] Return 404 with DeleteRoom/RemoveParticipant when deleting non-existent resources (#1860) Fixes #1587 --- pkg/service/roomservice.go | 10 ++++++++++ pkg/service/roomservice_test.go | 9 +++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/pkg/service/roomservice.go b/pkg/service/roomservice.go index 31b368cf0..e3859d4b3 100644 --- a/pkg/service/roomservice.go +++ b/pkg/service/roomservice.go @@ -127,6 +127,11 @@ func (s *RoomService) DeleteRoom(ctx context.Context, req *livekit.DeleteRoomReq if err := EnsureCreatePermission(ctx); err != nil { return nil, twirpAuthError(err) } + + if _, _, err := s.roomStore.LoadRoom(ctx, livekit.RoomName(req.Room), false); err == ErrRoomNotFound { + return nil, twirp.NotFoundError("room not found") + } + err := s.router.WriteRoomRTC(ctx, livekit.RoomName(req.Room), &livekit.RTCNodeMessage{ Message: &livekit.RTCNodeMessage_DeleteRoom{ DeleteRoom: req, @@ -187,6 +192,11 @@ func (s *RoomService) GetParticipant(ctx context.Context, req *livekit.RoomParti func (s *RoomService) RemoveParticipant(ctx context.Context, req *livekit.RoomParticipantIdentity) (*livekit.RemoveParticipantResponse, error) { AppendLogFields(ctx, "room", req.Room, "participant", req.Identity) + + if _, err := s.roomStore.LoadParticipant(ctx, livekit.RoomName(req.Room), livekit.ParticipantIdentity(req.Identity)); err == ErrParticipantNotFound { + return nil, twirp.NotFoundError("participant not found") + } + err := s.writeParticipantMessage(ctx, livekit.RoomName(req.Room), livekit.ParticipantIdentity(req.Identity), &livekit.RTCNodeMessage{ Message: &livekit.RTCNodeMessage_RemoveParticipant{ RemoveParticipant: req, diff --git a/pkg/service/roomservice_test.go b/pkg/service/roomservice_test.go index 744e18442..a8706bd64 100644 --- a/pkg/service/roomservice_test.go +++ b/pkg/service/roomservice_test.go @@ -17,7 +17,7 @@ import ( ) func TestDeleteRoom(t *testing.T) { - t.Run("normal deletion", func(t *testing.T) { + t.Run("delete non-existent", func(t *testing.T) { svc := newTestRoomService(config.RoomConfig{}) grant := &auth.ClaimGrants{ Video: &auth.VideoGrant{ @@ -29,7 +29,12 @@ func TestDeleteRoom(t *testing.T) { _, err := svc.DeleteRoom(ctx, &livekit.DeleteRoomRequest{ Room: "testroom", }) - require.NoError(t, err) + require.Error(t, err) + if terr, ok := err.(twirp.Error); ok { + require.Equal(t, twirp.NotFound, terr.Code()) + } else { + require.Fail(t, "should be twirp error") + } }) t.Run("missing permissions", func(t *testing.T) { From 0e15619355fb6fdad6e9689bb462523a72f1567e Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 8 Jul 2023 23:31:45 -0700 Subject: [PATCH 276/324] Update module github.com/livekit/protocol to v1.5.9 (#1846) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index c0b20b832..ee8f41f21 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230612070454-d5299b956135 - github.com/livekit/protocol v1.5.9-0.20230705215502-424b21fd0e92 + github.com/livekit/protocol v1.5.9 github.com/livekit/psrpc v0.3.1 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 @@ -101,6 +101,6 @@ require ( golang.org/x/text v0.10.0 // indirect golang.org/x/tools v0.9.3 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect - google.golang.org/grpc v1.55.0 // indirect + google.golang.org/grpc v1.56.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 6c8d2f96e..37d59cdba 100644 --- a/go.sum +++ b/go.sum @@ -124,8 +124,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20230612070454-d5299b956135 h1:lWYbsondvqG69czxoACDwaJ/BoyD57BahCo70ZH+m4U= github.com/livekit/mediatransportutil v0.0.0-20230612070454-d5299b956135/go.mod h1:MRc0zSOSzXuFt0X218SgabzlaKevkvCckPgBEoHYc34= -github.com/livekit/protocol v1.5.9-0.20230705215502-424b21fd0e92 h1:R0Y13wZIsH+82MqqOl5FMW3T8iRNcdW16vSLyfpUxck= -github.com/livekit/protocol v1.5.9-0.20230705215502-424b21fd0e92/go.mod h1:B6hJiuXT84dHsUgaKHBo+ZLPX4XhklptYA2UbANSiNg= +github.com/livekit/protocol v1.5.9 h1:fqPOLgKkWmkmUMnpfj2KDZlidHgAazAPU2T3FewyLew= +github.com/livekit/protocol v1.5.9/go.mod h1:GMTlFbc0JypUGo+PMilDrL45AX+yUBiQ1Cl/FGZibDY= github.com/livekit/psrpc v0.3.1 h1:KfylgJHvoLQcc22t/oflwMOeSnx0c14G7cWsS+9MYS4= github.com/livekit/psrpc v0.3.1/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= @@ -411,8 +411,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc= google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= -google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= +google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ= +google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= From 52212a6902859b250f108ac0b6438b65472b3c15 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Sat, 8 Jul 2023 23:47:33 -0700 Subject: [PATCH 277/324] v1.4.4 (#1861) --- CHANGELOG | 31 +++++++++++++++++++++++++++++++ version/version.go | 2 +- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index 018bc8baf..f4893fd8c 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,6 +2,37 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.4.4] - 2023-07-08 + +### Added +- Add dependency descriptor stream tracker for svc codecs (#1788) +- Full reconnect on publication mismatch on resume. (#1823) +- Pacer interface in down stream path. (#1835) +- retry egress on timeout/resource exhausted (#1852) + +### Fixed +- Send Room metadata updates immediately after update (#1787) +- Do not send ParticipantJoined webhook if connection was resumed (#1795) +- Reduce memory leaks by avoiding references in closure. (#1809) +- Honor bind address passed as `--bind` also for RTC ports (#1815) +- Avoid dangling downtracks by always deleting them in receiver close. (#1842) +- Better cleanup of subscriptions with needsCleanup. (#1845) +- Fix nack issue for svc codecs (#1856) +- Fixed hidden participant update were still sent when track is published (#1857) +- Fixed Redis lockup when unlocking room with canceled request context (#1859) + +### Changed +- Improvements to A/V sync (#1773 #1781 #1784 ) +- Improved probing to be less disruptive in low bandwidth scenarios (#1782 #1834 #1839) +- Do not mute forwarder when paused due to bandwidth congestion. (#1796) +- Improvements to congestion controller (#1800 #1802 ) +- Close participant on full reconnect. (#1818) +- Do not process events after participant close. (#1824) +- Improvements to dependency descriptor based selection forwarder (#1808) +- Discount out-of-order packets in downstream score. (#1831) +- Adaptive stream to select highest layer of equal dimensions (#1841) +- Return 404 with DeleteRoom/RemoveParticipant when deleting non-existent resources (#1860) + ## [1.4.3] - 2023-06-03 ### Added diff --git a/version/version.go b/version/version.go index 7f79727bd..c44e5ca30 100644 --- a/version/version.go +++ b/version/version.go @@ -1,3 +1,3 @@ package version -const Version = "1.4.3" +const Version = "1.4.4" From 8a1fc223da570ca8c4d378405c1e515e65dd8156 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Sun, 9 Jul 2023 16:55:37 -0700 Subject: [PATCH 278/324] Fix RTC IP when binding to 0.0.0.0 (#1862) --- cmd/server/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/server/main.go b/cmd/server/main.go index 1092bd539..42c3cf93f 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -217,7 +217,7 @@ func getConfig(c *cli.Context) (*config.Config, error) { // our IP discovery ignores loopback addresses for _, addr := range conf.BindAddresses { ip := net.ParseIP(addr) - if ip != nil && !ip.IsLoopback() { + if ip != nil && !ip.IsLoopback() && !ip.IsUnspecified() { shouldMatchRTCIP = true } } From e6f5f2f344b89c7edd5e5fabc46c265ac38b4789 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 10 Jul 2023 08:39:52 +0530 Subject: [PATCH 279/324] Prevent anachronous sample reading. (#1863) * Prevenet anachronous sample reading. Not so pretty way of solving this. Please let me know if you have thoughts. Passing in time allows testing easier. But, that also leads to time reversal problems. Example scenario 1. Connection stats worker gets a time and initiates quality calculation. 2. A layer transition is recorded after that. 3. By the time, scorer is called to calculate score with time from Step 1, there is time reversal and results in anachronous sample. One option is to use a scorer lock in connection stats module and wrap all calls to scorer in that lock, but that does not prevent the passed in time stamps themselves getting out of order. Also, stand alond use of scorer in some other context will be problematic. Doing the hybrid thing of taking current time in scorer if passed in time is zero so that scorer lock domain controls it. * use zero time everywhere in normal flow * make APIs with and without time passed in as Paul suggested --- pkg/sfu/connectionquality/connectionstats.go | 81 +++++++++++---- .../connectionquality/connectionstats_test.go | 98 +++++++++---------- pkg/sfu/connectionquality/scorer.go | 84 ++++++++++++++-- pkg/sfu/downtrack.go | 6 +- pkg/sfu/receiver.go | 19 ++-- 5 files changed, 200 insertions(+), 88 deletions(-) diff --git a/pkg/sfu/connectionquality/connectionstats.go b/pkg/sfu/connectionquality/connectionstats.go index 2b2601b74..ce821b9ee 100644 --- a/pkg/sfu/connectionquality/connectionstats.go +++ b/pkg/sfu/connectionquality/connectionstats.go @@ -60,16 +60,27 @@ func NewConnectionStats(params ConnectionStatsParams) *ConnectionStats { } } -func (cs *ConnectionStats) Start(trackInfo *livekit.TrackInfo, at time.Time) { +func (cs *ConnectionStats) start(trackInfo *livekit.TrackInfo) { + cs.isVideo.Store(trackInfo.Type == livekit.TrackType_VIDEO) + go cs.updateStatsWorker() +} + +func (cs *ConnectionStats) StartAt(trackInfo *livekit.TrackInfo, at time.Time) { if cs.isStarted.Swap(true) { return } - cs.isVideo.Store(trackInfo.Type == livekit.TrackType_VIDEO) + cs.scorer.StartAt(at) + cs.start(trackInfo) +} - cs.scorer.Start(at) +func (cs *ConnectionStats) Start(trackInfo *livekit.TrackInfo) { + if cs.isStarted.Swap(true) { + return + } - go cs.updateStatsWorker() + cs.scorer.Start() + cs.start(trackInfo) } func (cs *ConnectionStats) Close() { @@ -80,36 +91,68 @@ func (cs *ConnectionStats) OnStatsUpdate(fn func(cs *ConnectionStats, stat *live cs.onStatsUpdate = fn } -func (cs *ConnectionStats) UpdateMute(isMuted bool, at time.Time) { +func (cs *ConnectionStats) UpdateMuteAt(isMuted bool, at time.Time) { if cs.done.IsBroken() { return } - cs.scorer.UpdateMute(isMuted, at) + cs.scorer.UpdateMuteAt(isMuted, at) } -func (cs *ConnectionStats) AddBitrateTransition(bitrate int64, at time.Time) { +func (cs *ConnectionStats) UpdateMute(isMuted bool) { if cs.done.IsBroken() { return } - cs.scorer.AddBitrateTransition(bitrate, at) + cs.scorer.UpdateMute(isMuted) } -func (cs *ConnectionStats) UpdateLayerMute(isMuted bool, at time.Time) { +func (cs *ConnectionStats) AddBitrateTransitionAt(bitrate int64, at time.Time) { if cs.done.IsBroken() { return } - cs.scorer.UpdateLayerMute(isMuted, at) + cs.scorer.AddBitrateTransitionAt(bitrate, at) } -func (cs *ConnectionStats) AddLayerTransition(distance float64, at time.Time) { +func (cs *ConnectionStats) AddBitrateTransition(bitrate int64) { if cs.done.IsBroken() { return } - cs.scorer.AddLayerTransition(distance, at) + cs.scorer.AddBitrateTransition(bitrate) +} + +func (cs *ConnectionStats) UpdateLayerMuteAt(isMuted bool, at time.Time) { + if cs.done.IsBroken() { + return + } + + cs.scorer.UpdateLayerMuteAt(isMuted, at) +} + +func (cs *ConnectionStats) UpdateLayerMute(isMuted bool) { + if cs.done.IsBroken() { + return + } + + cs.scorer.UpdateLayerMute(isMuted) +} + +func (cs *ConnectionStats) AddLayerTransitionAt(distance float64, at time.Time) { + if cs.done.IsBroken() { + return + } + + cs.scorer.AddLayerTransitionAt(distance, at) +} + +func (cs *ConnectionStats) AddLayerTransition(distance float64) { + if cs.done.IsBroken() { + return + } + + cs.scorer.AddLayerTransition(distance) } func (cs *ConnectionStats) GetScoreAndQuality() (float32, livekit.ConnectionQuality) { @@ -129,7 +172,11 @@ func (cs *ConnectionStats) updateScoreWithAggregate(agg *buffer.RTPDeltaInfo, at stat.rttMax = agg.RttMax stat.jitterMax = agg.JitterMax } - cs.scorer.Update(&stat, at) + if at.IsZero() { + cs.scorer.Update(&stat) + } else { + cs.scorer.UpdateAt(&stat, at) + } mos, _ := cs.scorer.GetMOSAndQuality() return mos @@ -182,7 +229,7 @@ func (cs *ConnectionStats) updateScoreFromReceiverReport(at time.Time) (float32, return cs.updateScoreWithAggregate(agg, at), streams } -func (cs *ConnectionStats) updateScore(at time.Time) (float32, map[uint32]*buffer.StreamStatsWithLayers) { +func (cs *ConnectionStats) updateScoreAt(at time.Time) (float32, map[uint32]*buffer.StreamStatsWithLayers) { if cs.params.GetDeltaStats == nil { return MinMOS, nil } @@ -227,8 +274,8 @@ func (cs *ConnectionStats) clearStreamingStart() { cs.lock.Unlock() } -func (cs *ConnectionStats) getStat(at time.Time) { - score, streams := cs.updateScore(at) +func (cs *ConnectionStats) getStat() { + score, streams := cs.updateScoreAt(time.Time{}) if cs.onStatsUpdate != nil && len(streams) != 0 { analyticsStreams := make([]*livekit.AnalyticsStream, 0, len(streams)) @@ -279,7 +326,7 @@ func (cs *ConnectionStats) updateStatsWorker() { return } - cs.getStat(time.Now()) + cs.getStat() } } } diff --git a/pkg/sfu/connectionquality/connectionstats_test.go b/pkg/sfu/connectionquality/connectionstats_test.go index dce11601d..86d1302de 100644 --- a/pkg/sfu/connectionquality/connectionstats_test.go +++ b/pkg/sfu/connectionquality/connectionstats_test.go @@ -39,11 +39,11 @@ func TestConnectionQuality(t *testing.T) { duration := 5 * time.Second now := time.Now() - cs.Start(&livekit.TrackInfo{Type: livekit.TrackType_AUDIO}, now.Add(-duration)) - cs.UpdateMute(false, now.Add(-1*time.Second)) + cs.StartAt(&livekit.TrackInfo{Type: livekit.TrackType_AUDIO}, now.Add(-duration)) + cs.UpdateMuteAt(false, now.Add(-1*time.Second)) // no data and not enough unmute time should return default state which is EXCELLENT quality - cs.updateScore(now) + cs.updateScoreAt(now) mos, quality := cs.GetScoreAndQuality() require.Greater(t, float32(4.6), mos) require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality) @@ -58,7 +58,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(now.Add(duration)) + cs.updateScoreAt(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(4.6), mos) require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality) @@ -83,7 +83,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(now.Add(duration)) + cs.updateScoreAt(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(2.1), mos) require.Equal(t, livekit.ConnectionQuality_POOR, quality) @@ -101,7 +101,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(now.Add(duration)) + cs.updateScoreAt(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(4.1), mos) require.Equal(t, livekit.ConnectionQuality_GOOD, quality) @@ -117,7 +117,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(now.Add(duration)) + cs.updateScoreAt(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(4.1), mos) require.Equal(t, livekit.ConnectionQuality_GOOD, quality) @@ -133,7 +133,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(now.Add(duration)) + cs.updateScoreAt(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(4.6), mos) require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality) @@ -150,7 +150,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(now.Add(duration)) + cs.updateScoreAt(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(4.1), mos) require.Equal(t, livekit.ConnectionQuality_GOOD, quality) @@ -166,7 +166,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(now.Add(duration)) + cs.updateScoreAt(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(4.1), mos) require.Equal(t, livekit.ConnectionQuality_GOOD, quality) @@ -182,7 +182,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(now.Add(duration)) + cs.updateScoreAt(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(4.6), mos) require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality) @@ -199,20 +199,20 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(now.Add(duration)) + cs.updateScoreAt(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(2.1), mos) require.Equal(t, livekit.ConnectionQuality_POOR, quality) now = now.Add(duration) - cs.UpdateMute(true, now.Add(1*time.Second)) + cs.UpdateMuteAt(true, now.Add(1*time.Second)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(4.6), mos) require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality) // unmute at time so that next window does not satisfy the unmute time threshold. // that means even if the next update has 0 packets, it should hold state and stay at EXCELLENT quality - cs.UpdateMute(false, now.Add(3*time.Second)) + cs.UpdateMuteAt(false, now.Add(3*time.Second)) streams = map[uint32]*buffer.StreamStatsWithLayers{ 1: { @@ -223,7 +223,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(now.Add(duration)) + cs.updateScoreAt(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(4.6), mos) require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality) @@ -239,15 +239,15 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(now.Add(duration)) + cs.updateScoreAt(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(2.1), mos) require.Equal(t, livekit.ConnectionQuality_POOR, quality) // mute/unmute to bring quality back up now = now.Add(duration) - cs.UpdateMute(true, now.Add(1*time.Second)) - cs.UpdateMute(false, now.Add(2*time.Second)) + cs.UpdateMuteAt(true, now.Add(1*time.Second)) + cs.UpdateMuteAt(false, now.Add(2*time.Second)) // with lesser number of packet (simulating DTX). // even higher loss (like 10%) should not knock down quality due to quadratic weighting of packet loss ratio @@ -261,15 +261,15 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(now.Add(duration)) + cs.updateScoreAt(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(4.6), mos) require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality) // mute/unmute to bring quality back up now = now.Add(duration) - cs.UpdateMute(true, now.Add(1*time.Second)) - cs.UpdateMute(false, now.Add(2*time.Second)) + cs.UpdateMuteAt(true, now.Add(1*time.Second)) + cs.UpdateMuteAt(false, now.Add(2*time.Second)) // RTT and jitter can knock quality down. // at 2% loss, quality should stay at EXCELLENT purely based on loss, but with added RTT/jitter, should drop to GOOD @@ -285,19 +285,19 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(now.Add(duration)) + cs.updateScoreAt(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(4.1), mos) require.Equal(t, livekit.ConnectionQuality_GOOD, quality) // mute/unmute to bring quality back up now = now.Add(duration) - cs.UpdateMute(true, now.Add(1*time.Second)) - cs.UpdateMute(false, now.Add(2*time.Second)) + cs.UpdateMuteAt(true, now.Add(1*time.Second)) + cs.UpdateMuteAt(false, now.Add(2*time.Second)) // bitrate based calculation can drop quality even if there is no loss - cs.AddBitrateTransition(1_000_000, now) - cs.AddBitrateTransition(2_000_000, now.Add(2*time.Second)) + cs.AddBitrateTransitionAt(1_000_000, now) + cs.AddBitrateTransitionAt(2_000_000, now.Add(2*time.Second)) streams = map[uint32]*buffer.StreamStatsWithLayers{ 1: { @@ -309,21 +309,21 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(now.Add(duration)) + cs.updateScoreAt(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(4.1), mos) require.Equal(t, livekit.ConnectionQuality_GOOD, quality) // a transition to 0 (all layers stopped) should flip quality to EXCELLENT now = now.Add(duration) - cs.AddBitrateTransition(0, now) + cs.AddBitrateTransitionAt(0, now) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(4.6), mos) require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality) // test layer mute via UpdateLayerMute API - cs.AddBitrateTransition(1_000_000, now) - cs.AddBitrateTransition(2_000_000, now.Add(2*time.Second)) + cs.AddBitrateTransitionAt(1_000_000, now) + cs.AddBitrateTransitionAt(2_000_000, now.Add(2*time.Second)) streams = map[uint32]*buffer.StreamStatsWithLayers{ 1: { @@ -335,20 +335,20 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(now.Add(duration)) + cs.updateScoreAt(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(4.1), mos) require.Equal(t, livekit.ConnectionQuality_GOOD, quality) now = now.Add(duration) - cs.UpdateLayerMute(true, now) + cs.UpdateLayerMuteAt(true, now) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(4.6), mos) require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality) // setting bit rate after layer mute should layer unmute automatically - cs.AddBitrateTransition(1_000_000, now) - cs.AddBitrateTransition(2_000_000, now.Add(2*time.Second)) + cs.AddBitrateTransitionAt(1_000_000, now) + cs.AddBitrateTransitionAt(2_000_000, now.Add(2*time.Second)) streams = map[uint32]*buffer.StreamStatsWithLayers{ 1: { @@ -360,7 +360,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(now.Add(duration)) + cs.updateScoreAt(now.Add(duration)) mos, quality = cs.GetScoreAndQuality() require.Greater(t, float32(4.1), mos) require.Equal(t, livekit.ConnectionQuality_GOOD, quality) @@ -375,8 +375,8 @@ func TestConnectionQuality(t *testing.T) { duration := 5 * time.Second now := time.Now() - cs.Start(&livekit.TrackInfo{Type: livekit.TrackType_AUDIO}, now.Add(-duration)) - cs.UpdateMute(false, now.Add(-1*time.Second)) + cs.StartAt(&livekit.TrackInfo{Type: livekit.TrackType_AUDIO}, now.Add(-duration)) + cs.UpdateMuteAt(false, now.Add(-1*time.Second)) // RTT does not knock quality down because it is dependent and hence not taken into account // at 2% loss, quality should stay at EXCELLENT purely based on loss. With high RTT (700 ms) @@ -392,7 +392,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(now.Add(duration)) + cs.updateScoreAt(now.Add(duration)) mos, quality := cs.GetScoreAndQuality() require.Greater(t, float32(4.6), mos) require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality) @@ -407,8 +407,8 @@ func TestConnectionQuality(t *testing.T) { duration := 5 * time.Second now := time.Now() - cs.Start(&livekit.TrackInfo{Type: livekit.TrackType_AUDIO}, now.Add(-duration)) - cs.UpdateMute(false, now.Add(-1*time.Second)) + cs.StartAt(&livekit.TrackInfo{Type: livekit.TrackType_AUDIO}, now.Add(-duration)) + cs.UpdateMuteAt(false, now.Add(-1*time.Second)) // Jitter does not knock quality down because it is dependent and hence not taken into account // at 2% loss, quality should stay at EXCELLENT purely based on loss. With high jitter (200 ms) @@ -424,7 +424,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(now.Add(duration)) + cs.updateScoreAt(now.Add(duration)) mos, quality := cs.GetScoreAndQuality() require.Greater(t, float32(4.6), mos) require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality) @@ -576,7 +576,7 @@ func TestConnectionQuality(t *testing.T) { duration := 5 * time.Second now := time.Now() - cs.Start(&livekit.TrackInfo{Type: livekit.TrackType_AUDIO}, now.Add(-duration)) + cs.StartAt(&livekit.TrackInfo{Type: livekit.TrackType_AUDIO}, now.Add(-duration)) for _, eq := range tc.expectedQualities { streams = map[uint32]*buffer.StreamStatsWithLayers{ @@ -589,7 +589,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(now.Add(duration)) + cs.updateScoreAt(now.Add(duration)) mos, quality := cs.GetScoreAndQuality() require.Greater(t, eq.expectedMOS, mos) require.Equal(t, eq.expectedQuality, quality) @@ -673,10 +673,10 @@ func TestConnectionQuality(t *testing.T) { duration := 5 * time.Second now := time.Now() - cs.Start(&livekit.TrackInfo{Type: livekit.TrackType_VIDEO}, now) + cs.StartAt(&livekit.TrackInfo{Type: livekit.TrackType_VIDEO}, now) for _, tr := range tc.transitions { - cs.AddBitrateTransition(tr.bitrate, now.Add(tr.offset)) + cs.AddBitrateTransitionAt(tr.bitrate, now.Add(tr.offset)) } streams = map[uint32]*buffer.StreamStatsWithLayers{ @@ -689,7 +689,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(now.Add(duration)) + cs.updateScoreAt(now.Add(duration)) mos, quality := cs.GetScoreAndQuality() require.Greater(t, tc.expectedMOS, mos) require.Equal(t, tc.expectedQuality, quality) @@ -764,10 +764,10 @@ func TestConnectionQuality(t *testing.T) { duration := 5 * time.Second now := time.Now() - cs.Start(&livekit.TrackInfo{Type: livekit.TrackType_VIDEO}, now) + cs.StartAt(&livekit.TrackInfo{Type: livekit.TrackType_VIDEO}, now) for _, tr := range tc.transitions { - cs.AddLayerTransition(tr.distance, now.Add(tr.offset)) + cs.AddLayerTransitionAt(tr.distance, now.Add(tr.offset)) } streams = map[uint32]*buffer.StreamStatsWithLayers{ @@ -779,7 +779,7 @@ func TestConnectionQuality(t *testing.T) { }, }, } - cs.updateScore(now.Add(duration)) + cs.updateScoreAt(now.Add(duration)) mos, quality := cs.GetScoreAndQuality() require.Greater(t, tc.expectedMOS, mos) require.Equal(t, tc.expectedQuality, quality) diff --git a/pkg/sfu/connectionquality/scorer.go b/pkg/sfu/connectionquality/scorer.go index 5084d6ce8..a855246f2 100644 --- a/pkg/sfu/connectionquality/scorer.go +++ b/pkg/sfu/connectionquality/scorer.go @@ -181,17 +181,25 @@ func newQualityScorer(params qualityScorerParams) *qualityScorer { } } -func (q *qualityScorer) Start(at time.Time) { - q.lock.Lock() - defer q.lock.Unlock() - +func (q *qualityScorer) startAtLocked(at time.Time) { q.lastUpdateAt = at } -func (q *qualityScorer) UpdateMute(isMuted bool, at time.Time) { +func (q *qualityScorer) StartAt(at time.Time) { q.lock.Lock() defer q.lock.Unlock() + q.startAtLocked(at) +} + +func (q *qualityScorer) Start() { + q.lock.Lock() + defer q.lock.Unlock() + + q.startAtLocked(time.Now()) +} + +func (q *qualityScorer) updateMuteAtLocked(isMuted bool, at time.Time) { if isMuted { q.mutedAt = at q.score = maxScore @@ -200,10 +208,21 @@ func (q *qualityScorer) UpdateMute(isMuted bool, at time.Time) { } } -func (q *qualityScorer) AddBitrateTransition(bitrate int64, at time.Time) { +func (q *qualityScorer) UpdateMuteAt(isMuted bool, at time.Time) { q.lock.Lock() defer q.lock.Unlock() + q.updateMuteAtLocked(isMuted, at) +} + +func (q *qualityScorer) UpdateMute(isMuted bool) { + q.lock.Lock() + defer q.lock.Unlock() + + q.updateMuteAtLocked(isMuted, time.Now()) +} + +func (q *qualityScorer) addBitrateTransitionAtLocked(bitrate int64, at time.Time) { q.aggregateBitrate.AddSampleAt(bitrate, at) if bitrate == 0 { @@ -218,10 +237,21 @@ func (q *qualityScorer) AddBitrateTransition(bitrate int64, at time.Time) { } } -func (q *qualityScorer) UpdateLayerMute(isMuted bool, at time.Time) { +func (q *qualityScorer) AddBitrateTransitionAt(bitrate int64, at time.Time) { q.lock.Lock() defer q.lock.Unlock() + q.addBitrateTransitionAtLocked(bitrate, at) +} + +func (q *qualityScorer) AddBitrateTransition(bitrate int64) { + q.lock.Lock() + defer q.lock.Unlock() + + q.addBitrateTransitionAtLocked(bitrate, time.Now()) +} + +func (q *qualityScorer) updateLayerMuteAtLocked(isMuted bool, at time.Time) { if isMuted { if !q.isLayerMuted() { q.aggregateBitrate.AddSampleAt(0, at) @@ -236,17 +266,39 @@ func (q *qualityScorer) UpdateLayerMute(isMuted bool, at time.Time) { } } -func (q *qualityScorer) AddLayerTransition(distance float64, at time.Time) { +func (q *qualityScorer) UpdateLayerMuteAt(isMuted bool, at time.Time) { q.lock.Lock() defer q.lock.Unlock() + q.updateLayerMuteAtLocked(isMuted, at) +} + +func (q *qualityScorer) UpdateLayerMute(isMuted bool) { + q.lock.Lock() + defer q.lock.Unlock() + + q.updateLayerMuteAtLocked(isMuted, time.Now()) +} + +func (q *qualityScorer) addLayerTransitionAtLocked(distance float64, at time.Time) { q.layerDistance.AddSampleAt(distance, at) } -func (q *qualityScorer) Update(stat *windowStat, at time.Time) { +func (q *qualityScorer) AddLayerTransitionAt(distance float64, at time.Time) { q.lock.Lock() defer q.lock.Unlock() + q.addLayerTransitionAtLocked(distance, at) +} + +func (q *qualityScorer) AddLayerTransition(distance float64) { + q.lock.Lock() + defer q.lock.Unlock() + + q.addLayerTransitionAtLocked(distance, time.Now()) +} + +func (q *qualityScorer) updateAtLocked(stat *windowStat, at time.Time) { // always update transitions expectedBitrate, _, err := q.aggregateBitrate.GetAggregateAndRestartAt(at) if err != nil { @@ -331,6 +383,20 @@ func (q *qualityScorer) Update(stat *windowStat, at time.Time) { q.lastUpdateAt = at } +func (q *qualityScorer) UpdateAt(stat *windowStat, at time.Time) { + q.lock.Lock() + defer q.lock.Unlock() + + q.updateAtLocked(stat, at) +} + +func (q *qualityScorer) Update(stat *windowStat) { + q.lock.Lock() + defer q.lock.Unlock() + + q.updateAtLocked(stat, time.Now()) +} + func (q *qualityScorer) isMuted() bool { return !q.mutedAt.IsZero() && (q.unmutedAt.IsZero() || q.mutedAt.After(q.unmutedAt)) } diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 5e7644449..013dca537 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -396,7 +396,7 @@ func (d *DownTrack) TrackInfoAvailable() { if ti == nil { return } - d.connectionStats.Start(ti, time.Now()) + d.connectionStats.Start(ti) } func (d *DownTrack) SetStreamAllocatorListener(listener DownTrackStreamAllocatorListener) { @@ -740,7 +740,7 @@ func (d *DownTrack) handleMute(muted bool, isPub bool, changed bool, maxLayer bu return } - d.connectionStats.UpdateMute(d.forwarder.IsAnyMuted(), time.Now()) + d.connectionStats.UpdateMute(d.forwarder.IsAnyMuted()) // // Subscriber mute changes trigger a max layer notification. @@ -955,7 +955,7 @@ func (d *DownTrack) maybeAddTransition(_bitrate int64, distance float64) { return } - d.connectionStats.AddLayerTransition(distance, time.Now()) + d.connectionStats.AddLayerTransition(distance) } func (d *DownTrack) UpTrackBitrateReport(availableLayers []int32, bitrates Bitrates) { diff --git a/pkg/sfu/receiver.go b/pkg/sfu/receiver.go index 26ca98ad7..837b534ab 100644 --- a/pkg/sfu/receiver.go +++ b/pkg/sfu/receiver.go @@ -215,7 +215,7 @@ func NewWebRTCReceiver( w.onStatsUpdate(w, stat) } }) - w.connectionStats.Start(w.trackInfo, time.Now()) + w.connectionStats.Start(w.trackInfo) for _, ext := range receiver.GetParameters().HeaderExtensions { if ext.URI == dd.ExtensionUrl { @@ -375,7 +375,7 @@ func (w *WebRTCReceiver) SetUpTrackPaused(paused bool) { } w.bufferMu.RUnlock() - w.connectionStats.UpdateMute(paused, time.Now()) + w.connectionStats.UpdateMute(paused) } func (w *WebRTCReceiver) AddDownTrack(track TrackSender) error { @@ -398,12 +398,11 @@ func (w *WebRTCReceiver) AddDownTrack(track TrackSender) error { func (w *WebRTCReceiver) SetMaxExpectedSpatialLayer(layer int32) { w.streamTrackerManager.SetMaxExpectedSpatialLayer(layer) - now := time.Now() if layer == buffer.InvalidLayerSpatial { - w.connectionStats.UpdateLayerMute(true, now) + w.connectionStats.UpdateLayerMute(true) } else { - w.connectionStats.UpdateLayerMute(false, now) - w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired(), now) + w.connectionStats.UpdateLayerMute(false) + w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired()) } } @@ -413,7 +412,7 @@ func (w *WebRTCReceiver) OnAvailableLayersChanged() { dt.UpTrackLayersChange() } - w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired(), time.Now()) + w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired()) } // StreamTrackerManagerListener.OnBitrateAvailabilityChanged @@ -429,7 +428,7 @@ func (w *WebRTCReceiver) OnMaxPublishedLayerChanged(maxPublishedLayer int32) { dt.UpTrackMaxPublishedLayerChange(maxPublishedLayer) } - w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired(), time.Now()) + w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired()) } // StreamTrackerManagerListener.OnMaxTemporalLayerSeenChanged @@ -438,7 +437,7 @@ func (w *WebRTCReceiver) OnMaxTemporalLayerSeenChanged(maxTemporalLayerSeen int3 dt.UpTrackMaxTemporalLayerSeenChange(maxTemporalLayerSeen) } - w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired(), time.Now()) + w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired()) } // StreamTrackerManagerListener.OnMaxAvailableLayerChanged @@ -458,7 +457,7 @@ func (w *WebRTCReceiver) OnBitrateReport(availableLayers []int32, bitrates Bitra dt.UpTrackBitrateReport(availableLayers, bitrates) } - w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired(), time.Now()) + w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired()) } func (w *WebRTCReceiver) GetLayeredBitrate() ([]int32, Bitrates) { From 1cb74b9e1b4f9c0f05b11d103940b464bf114584 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Mon, 10 Jul 2023 13:20:57 +0530 Subject: [PATCH 280/324] Check for desired before clean up. (#1865) Fix a potential race between needsCleanup checking and a re-subscribe setting desired back to true. --- pkg/rtc/subscriptionmanager.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/rtc/subscriptionmanager.go b/pkg/rtc/subscriptionmanager.go index fbf83dd73..1ed7d3f8c 100644 --- a/pkg/rtc/subscriptionmanager.go +++ b/pkg/rtc/subscriptionmanager.go @@ -369,6 +369,7 @@ func (m *SubscriptionManager) reconcileSubscription(s *trackSubscription) { // successfully unsubscribed, remove from map m.lock.Lock() if !s.isDesired() { + s.logger.Debugw("unsubscribe removing subscription") delete(m.subscriptions, s.trackID) } m.lock.Unlock() @@ -389,7 +390,10 @@ func (m *SubscriptionManager) reconcileSubscription(s *trackSubscription) { if s.needsCleanup() { m.lock.Lock() - delete(m.subscriptions, s.trackID) + if !s.isDesired() { + s.logger.Debugw("cleanup removing subscription") + delete(m.subscriptions, s.trackID) + } m.lock.Unlock() } } From 9abc580e8bd30731bc7da9a49cd2f1d8a261a0f9 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 10 Jul 2023 21:56:47 -0700 Subject: [PATCH 281/324] Update golang.org/x/exp digest to fffb143 (#1866) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ee8f41f21..7fe6d00e8 100644 --- a/go.mod +++ b/go.mod @@ -45,7 +45,7 @@ require ( github.com/urfave/cli/v2 v2.25.7 github.com/urfave/negroni/v3 v3.0.0 go.uber.org/atomic v1.11.0 - golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df + golang.org/x/exp v0.0.0-20230711023510-fffb14384f22 golang.org/x/sync v0.3.0 google.golang.org/protobuf v1.31.0 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index 37d59cdba..80203a14b 100644 --- a/go.sum +++ b/go.sum @@ -292,8 +292,8 @@ golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= -golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME= -golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/exp v0.0.0-20230711023510-fffb14384f22 h1:FqrVOBQxQ8r/UwwXibI0KMolVhvFiGobSfdE33deHJM= +golang.org/x/exp v0.0.0-20230711023510-fffb14384f22/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= From 5459bd2931112781a9633ce7c45808b4474f874b Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 11 Jul 2023 15:29:35 +0530 Subject: [PATCH 282/324] Push track quality to poor on a bandwidth constrained pause. (#1867) * Push track quality to poor on a bandwidth constrained pause. * add tests * scale distance by divisor * fix test distance to desired * wait longer for subscription manager to reconcile --- pkg/sfu/connectionquality/connectionstats.go | 16 +++++ .../connectionquality/connectionstats_test.go | 41 ++++++++---- pkg/sfu/connectionquality/scorer.go | 66 ++++++++++++++----- pkg/sfu/downtrack.go | 18 +++-- pkg/sfu/forwarder.go | 9 ++- pkg/sfu/forwarder_test.go | 6 +- test/singlenode_test.go | 2 +- 7 files changed, 118 insertions(+), 40 deletions(-) diff --git a/pkg/sfu/connectionquality/connectionstats.go b/pkg/sfu/connectionquality/connectionstats.go index ce821b9ee..4c1c9e545 100644 --- a/pkg/sfu/connectionquality/connectionstats.go +++ b/pkg/sfu/connectionquality/connectionstats.go @@ -139,6 +139,22 @@ func (cs *ConnectionStats) UpdateLayerMute(isMuted bool) { cs.scorer.UpdateLayerMute(isMuted) } +func (cs *ConnectionStats) UpdatePauseAt(isPaused bool, at time.Time) { + if cs.done.IsBroken() { + return + } + + cs.scorer.UpdatePauseAt(isPaused, at) +} + +func (cs *ConnectionStats) UpdatePause(isPaused bool) { + if cs.done.IsBroken() { + return + } + + cs.scorer.UpdatePause(isPaused) +} + func (cs *ConnectionStats) AddLayerTransitionAt(distance float64, at time.Time) { if cs.done.IsBroken() { return diff --git a/pkg/sfu/connectionquality/connectionstats_test.go b/pkg/sfu/connectionquality/connectionstats_test.go index 86d1302de..ff3474cda 100644 --- a/pkg/sfu/connectionquality/connectionstats_test.go +++ b/pkg/sfu/connectionquality/connectionstats_test.go @@ -30,7 +30,7 @@ func newConnectionStats( } func TestConnectionQuality(t *testing.T) { - t.Run("quality scorer state machine", func(t *testing.T) { + t.Run("quality scorer operation", func(t *testing.T) { var streams map[uint32]*buffer.StreamStatsWithLayers getDeltaStats := func() map[uint32]*buffer.StreamStatsWithLayers { return streams @@ -314,13 +314,6 @@ func TestConnectionQuality(t *testing.T) { require.Greater(t, float32(4.1), mos) require.Equal(t, livekit.ConnectionQuality_GOOD, quality) - // a transition to 0 (all layers stopped) should flip quality to EXCELLENT - now = now.Add(duration) - cs.AddBitrateTransitionAt(0, now) - mos, quality = cs.GetScoreAndQuality() - require.Greater(t, float32(4.6), mos) - require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality) - // test layer mute via UpdateLayerMute API cs.AddBitrateTransitionAt(1_000_000, now) cs.AddBitrateTransitionAt(2_000_000, now.Add(2*time.Second)) @@ -346,10 +339,36 @@ func TestConnectionQuality(t *testing.T) { require.Greater(t, float32(4.6), mos) require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality) - // setting bit rate after layer mute should layer unmute automatically - cs.AddBitrateTransitionAt(1_000_000, now) - cs.AddBitrateTransitionAt(2_000_000, now.Add(2*time.Second)) + // unmute layer + cs.UpdateLayerMuteAt(false, now.Add(2*time.Second)) + streams = map[uint32]*buffer.StreamStatsWithLayers{ + 1: { + RTPStats: &buffer.RTPDeltaInfo{ + StartTime: now, + Duration: duration, + Packets: 250, + Bytes: 8_000_000 / 8 / 5, + }, + }, + } + cs.updateScoreAt(now.Add(duration)) + mos, quality = cs.GetScoreAndQuality() + require.Greater(t, float32(4.6), mos) + require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality) + + // pause + now = now.Add(duration) + cs.UpdatePauseAt(true, now) + mos, quality = cs.GetScoreAndQuality() + require.Greater(t, float32(2.1), mos) + require.Equal(t, livekit.ConnectionQuality_POOR, quality) + + // resume + cs.UpdatePauseAt(false, now.Add(2*time.Second)) + + // although conditions are perfect, climbing back from POOR (because of pause above) + // will only climb to GOOD. streams = map[uint32]*buffer.StreamStatsWithLayers{ 1: { RTPStats: &buffer.RTPDeltaInfo{ diff --git a/pkg/sfu/connectionquality/scorer.go b/pkg/sfu/connectionquality/scorer.go index a855246f2..5391406a9 100644 --- a/pkg/sfu/connectionquality/scorer.go +++ b/pkg/sfu/connectionquality/scorer.go @@ -162,6 +162,9 @@ type qualityScorer struct { layerMutedAt time.Time layerUnmutedAt time.Time + pausedAt time.Time + resumedAt time.Time + maxPPS float64 aggregateBitrate *utils.TimedAggregator[int64] @@ -224,17 +227,6 @@ func (q *qualityScorer) UpdateMute(isMuted bool) { func (q *qualityScorer) addBitrateTransitionAtLocked(bitrate int64, at time.Time) { q.aggregateBitrate.AddSampleAt(bitrate, at) - - if bitrate == 0 { - if !q.isLayerMuted() { - q.layerMutedAt = at - q.score = maxScore - } - } else { - if q.isLayerMuted() { - q.layerUnmutedAt = at - } - } } func (q *qualityScorer) AddBitrateTransitionAt(bitrate int64, at time.Time) { @@ -254,8 +246,9 @@ func (q *qualityScorer) AddBitrateTransition(bitrate int64) { func (q *qualityScorer) updateLayerMuteAtLocked(isMuted bool, at time.Time) { if isMuted { if !q.isLayerMuted() { - q.aggregateBitrate.AddSampleAt(0, at) - q.layerDistance.AddSampleAt(0, at) + q.aggregateBitrate.Reset() + q.layerDistance.Reset() + q.layerMutedAt = at q.score = maxScore } @@ -280,6 +273,36 @@ func (q *qualityScorer) UpdateLayerMute(isMuted bool) { q.updateLayerMuteAtLocked(isMuted, time.Now()) } +func (q *qualityScorer) updatePauseAtLocked(isPaused bool, at time.Time) { + if isPaused { + if !q.isPaused() { + q.aggregateBitrate.Reset() + q.layerDistance.Reset() + + q.pausedAt = at + q.score = poorScore + } + } else { + if q.isPaused() { + q.resumedAt = at + } + } +} + +func (q *qualityScorer) UpdatePauseAt(isPaused bool, at time.Time) { + q.lock.Lock() + defer q.lock.Unlock() + + q.updatePauseAtLocked(isPaused, at) +} + +func (q *qualityScorer) UpdatePause(isPaused bool) { + q.lock.Lock() + defer q.lock.Unlock() + + q.updatePauseAtLocked(isPaused, time.Now()) +} + func (q *qualityScorer) addLayerTransitionAtLocked(distance float64, at time.Time) { q.layerDistance.AddSampleAt(distance, at) } @@ -311,11 +334,14 @@ func (q *qualityScorer) updateAtLocked(stat *windowStat, at time.Time) { // nothing to do when muted or not unmuted for long enough // NOTE: it is possible that unmute -> mute -> unmute transition happens in the - // same analysis window. On a transition to mute, state immediately moves - // to stable and quality EXCELLENT for responsiveness. On an unmute, the - // entire window data is considered (as long as enough time has passed since - // unmute) including the data before mute. - if q.isMuted() || !q.isUnmutedEnough(at) || q.isLayerMuted() { + // same analysis window. On a transition to mute, quality is immediately moved + // EXCELLENT for responsiveness. On an unmute, the entire window data is + // considered (as long as enough time has passed since unmute). + // + // Similarly, when paused (possibly due to congestion), score is immediately + // set to poorScore for responsiveness. The layer transision is reest. + // On a resume, quality climbs back up using normal operation. + if q.isMuted() || !q.isUnmutedEnough(at) || q.isLayerMuted() || q.isPaused() { q.lastUpdateAt = at return } @@ -430,6 +456,10 @@ func (q *qualityScorer) isLayerMuted() bool { return !q.layerMutedAt.IsZero() && (q.layerUnmutedAt.IsZero() || q.layerMutedAt.After(q.layerUnmutedAt)) } +func (q *qualityScorer) isPaused() bool { + return !q.pausedAt.IsZero() && (q.resumedAt.IsZero() || q.pausedAt.After(q.resumedAt)) +} + func (q *qualityScorer) getPacketLossWeight(stat *windowStat) float64 { if stat == nil || stat.duration == 0 { return q.params.PacketLossWeight diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 013dca537..f0463a1e0 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -950,18 +950,24 @@ func (d *DownTrack) UpTrackMaxTemporalLayerSeenChange(maxTemporalLayerSeen int32 } } -func (d *DownTrack) maybeAddTransition(_bitrate int64, distance float64) { +func (d *DownTrack) maybeAddTransition(_bitrate int64, distance float64, pauseReason VideoPauseReason) { if d.kind == webrtc.RTPCodecTypeAudio { return } - d.connectionStats.AddLayerTransition(distance) + if pauseReason == VideoPauseReasonBandwidth { + d.connectionStats.UpdatePause(true) + } else { + d.connectionStats.UpdatePause(false) + d.connectionStats.AddLayerTransition(distance) + } } func (d *DownTrack) UpTrackBitrateReport(availableLayers []int32, bitrates Bitrates) { d.maybeAddTransition( d.forwarder.GetOptimalBandwidthNeeded(bitrates), d.forwarder.DistanceToDesired(availableLayers, bitrates), + d.forwarder.PauseReason(), ) } @@ -1011,7 +1017,7 @@ func (d *DownTrack) AllocateOptimal(allowOvershoot bool) VideoAllocation { al, brs := d.receiver.GetLayeredBitrate() allocation := d.forwarder.AllocateOptimal(al, brs, allowOvershoot) d.maybeStartKeyFrameRequester() - d.maybeAddTransition(allocation.BandwidthNeeded, allocation.DistanceToDesired) + d.maybeAddTransition(allocation.BandwidthNeeded, allocation.DistanceToDesired, allocation.PauseReason) return allocation } @@ -1039,7 +1045,7 @@ func (d *DownTrack) ProvisionalAllocateGetBestWeightedTransition() VideoTransiti func (d *DownTrack) ProvisionalAllocateCommit() VideoAllocation { allocation := d.forwarder.ProvisionalAllocateCommit() d.maybeStartKeyFrameRequester() - d.maybeAddTransition(allocation.BandwidthNeeded, allocation.DistanceToDesired) + d.maybeAddTransition(allocation.BandwidthNeeded, allocation.DistanceToDesired, allocation.PauseReason) return allocation } @@ -1047,7 +1053,7 @@ func (d *DownTrack) AllocateNextHigher(availableChannelCapacity int64, allowOver al, brs := d.receiver.GetLayeredBitrate() allocation, available := d.forwarder.AllocateNextHigher(availableChannelCapacity, al, brs, allowOvershoot) d.maybeStartKeyFrameRequester() - d.maybeAddTransition(allocation.BandwidthNeeded, allocation.DistanceToDesired) + d.maybeAddTransition(allocation.BandwidthNeeded, allocation.DistanceToDesired, allocation.PauseReason) return allocation, available } @@ -1062,7 +1068,7 @@ func (d *DownTrack) Pause() VideoAllocation { al, brs := d.receiver.GetLayeredBitrate() allocation := d.forwarder.Pause(al, brs) d.maybeStartKeyFrameRequester() - d.maybeAddTransition(allocation.BandwidthNeeded, allocation.DistanceToDesired) + d.maybeAddTransition(allocation.BandwidthNeeded, allocation.DistanceToDesired, allocation.PauseReason) return allocation } diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 92fae3ea7..33c196b86 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -526,6 +526,13 @@ func (f *Forwarder) IsDeficient() bool { return f.isDeficientLocked() } +func (f *Forwarder) PauseReason() VideoPauseReason { + f.lock.RLock() + defer f.lock.RUnlock() + + return f.lastAllocation.PauseReason +} + func (f *Forwarder) BandwidthRequested(brs Bitrates) int64 { f.lock.RLock() defer f.lock.RUnlock() @@ -1865,7 +1872,7 @@ done: ((adjustedMaxLayer.Spatial - adjustedTargetLayer.Spatial) * (maxSeenLayer.Temporal + 1)) + (adjustedMaxLayer.Temporal - adjustedTargetLayer.Temporal) if !targetLayer.IsValid() { - distance++ + distance += (maxSeenLayer.Temporal + 1) } return float64(distance) / float64(maxSeenLayer.Temporal+1) diff --git a/pkg/sfu/forwarder_test.go b/pkg/sfu/forwarder_test.go index df82fa86d..a0e340800 100644 --- a/pkg/sfu/forwarder_test.go +++ b/pkg/sfu/forwarder_test.go @@ -562,7 +562,7 @@ func TestForwarderProvisionalAllocate(t *testing.T) { TargetLayer: expectedTargetLayer, RequestLayerSpatial: expectedTargetLayer.Spatial, MaxLayer: expectedMaxLayer, - DistanceToDesired: 0.25, + DistanceToDesired: 1.0, } result = f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) @@ -595,7 +595,7 @@ func TestForwarderProvisionalAllocate(t *testing.T) { TargetLayer: buffer.InvalidLayer, RequestLayerSpatial: buffer.InvalidLayerSpatial, MaxLayer: expectedMaxLayer, - DistanceToDesired: 0.25, + DistanceToDesired: 1.0, } result = f.ProvisionalAllocateCommit() require.Equal(t, expectedResult, result) @@ -1114,7 +1114,7 @@ func TestForwarderPause(t *testing.T) { TargetLayer: buffer.InvalidLayer, RequestLayerSpatial: buffer.InvalidLayerSpatial, MaxLayer: buffer.DefaultMaxLayer, - DistanceToDesired: 3, + DistanceToDesired: 3.75, } result := f.Pause(nil, bitrates) require.Equal(t, expectedResult, result) diff --git a/test/singlenode_test.go b/test/singlenode_test.go index 8d9acbdc7..fe84de51d 100644 --- a/test/singlenode_test.go +++ b/test/singlenode_test.go @@ -620,7 +620,7 @@ func TestSubscribeToCodecUnsupported(t *testing.T) { require.Equal(t, h264TrackID, sr.TrackSid) require.Equal(t, livekit.SubscriptionError_SE_CODEC_UNSUPPORTED, sr.Err) return true - }, time.Second, 10*time.Millisecond, "did not receive subscription response") + }, 5*time.Second, 10*time.Millisecond, "did not receive subscription response") // publish another vp8 track again, ensure the transport recovered by sfu and c2 can receive it t4, err := c1.AddStaticTrack("video/vp8", "video2", "webcam2") From 49e72fb2523aab25d8811ce87a8c9e141e2d5704 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 11 Jul 2023 09:02:52 -0700 Subject: [PATCH 283/324] Update golang.org/x/exp digest to 06a737e (#1868) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7fe6d00e8..57516561d 100644 --- a/go.mod +++ b/go.mod @@ -45,7 +45,7 @@ require ( github.com/urfave/cli/v2 v2.25.7 github.com/urfave/negroni/v3 v3.0.0 go.uber.org/atomic v1.11.0 - golang.org/x/exp v0.0.0-20230711023510-fffb14384f22 + golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb golang.org/x/sync v0.3.0 google.golang.org/protobuf v1.31.0 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index 80203a14b..e22cfb93e 100644 --- a/go.sum +++ b/go.sum @@ -292,8 +292,8 @@ golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= -golang.org/x/exp v0.0.0-20230711023510-fffb14384f22 h1:FqrVOBQxQ8r/UwwXibI0KMolVhvFiGobSfdE33deHJM= -golang.org/x/exp v0.0.0-20230711023510-fffb14384f22/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb h1:xIApU0ow1zwMa2uL1VDNeQlNVFTWMQxZUZCMDy0Q4Us= +golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= From ed867fafe5c29fe0df0295bd2725ab4351136da7 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 12 Jul 2023 10:28:36 +0530 Subject: [PATCH 284/324] Log unexpected ICE connection states (#1870) --- pkg/rtc/transport.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index d924a141e..9b47f33d5 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -597,6 +597,11 @@ func (t *PCTransport) onICEConnectionStateChange(state webrtc.ICEConnectionState case webrtc.ICEConnectionStateChecking: t.setICEStartedAt(time.Now()) + + case webrtc.ICEConnectionStateDisconnected: + fallthrough + case webrtc.ICEConnectionStateFailed: + t.params.Logger.Infow("ice connection state change unexpected", "state", state.String()) } } From 8dc2c005c33cf45270324793191dc581a13b91e0 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 12 Jul 2023 14:12:00 +0530 Subject: [PATCH 285/324] Add ability to roll back video layer selection. (#1871) * Add ability to roll back video layer selection. Not currently useful, but it is possible to do things like not applying a layer switch if the switch point time stamp is too far back. Add ability to roll back a layer switch and invoke rollback if a packet was selected for forwarding, but a subsequent error or decision to drop the packet can rollback layer switch if that was the switching packet. In current code, the paths where a packet can be dropped after selection does not happen at switch points. So, it was okay to apply the selection unconditionally. But, adding the call to rollback in the current code also in all paths where packet is dropped after selection for consistent code flow. * separate switch for temporal layer --- pkg/sfu/forwarder.go | 13 +- pkg/sfu/videolayerselector/base.go | 78 ++++++-- .../dependencydescriptor.go | 21 +- pkg/sfu/videolayerselector/simulcast.go | 188 ++++++++---------- .../videolayerselector/videolayerselector.go | 4 +- pkg/sfu/videolayerselector/vp9.go | 3 + 6 files changed, 179 insertions(+), 128 deletions(-) diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 33c196b86..8bb21c387 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -1594,6 +1594,12 @@ func (f *Forwarder) getTranslationParamsAudio(extPkt *buffer.ExtPacket, layer in // should be called with lock held func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer int32) (*TranslationParams, error) { + maybeRollback := func(isSwitching bool) { + if isSwitching { + f.vls.Rollback() + } + } + tp := &TranslationParams{} if !f.vls.GetTarget().IsValid() { @@ -1640,20 +1646,23 @@ func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer in // To differentiate between the two cases, drop only when in DEFICIENT state. // tp.shouldDrop = true + maybeRollback(result.IsSwitching) return tp, nil } _, err := f.getTranslationParamsCommon(extPkt, layer, tp) if tp.shouldDrop || len(extPkt.Packet.Payload) == 0 { + maybeRollback(result.IsSwitching) return tp, err } // codec specific forwarding check and any needed packet munging + tl, isSwitching := f.vls.SelectTemporal(extPkt) codecBytes, err := f.codecMunger.UpdateAndGet( extPkt, tp.rtp.snOrdering == SequenceNumberOrderingOutOfOrder, tp.rtp.snOrdering == SequenceNumberOrderingGap, - f.vls.SelectTemporal(extPkt), + tl, ) if err != nil { tp.rtp = nil @@ -1663,9 +1672,11 @@ func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer in // filtered temporal layer, update sequence number offset to prevent holes f.rtpMunger.PacketDropped(extPkt) } + maybeRollback(result.IsSwitching || isSwitching) return tp, nil } + maybeRollback(result.IsSwitching || isSwitching) return tp, err } diff --git a/pkg/sfu/videolayerselector/base.go b/pkg/sfu/videolayerselector/base.go index 534cb751b..7323fc7ae 100644 --- a/pkg/sfu/videolayerselector/base.go +++ b/pkg/sfu/videolayerselector/base.go @@ -11,25 +11,33 @@ type Base struct { tls temporallayerselector.TemporalLayerSelector - maxLayer buffer.VideoLayer - targetLayer buffer.VideoLayer + maxLayer buffer.VideoLayer + maxSeenLayer buffer.VideoLayer + + targetLayer buffer.VideoLayer + previousTargetLayer buffer.VideoLayer + requestSpatial int32 - maxSeenLayer buffer.VideoLayer - parkedLayer buffer.VideoLayer + parkedLayer buffer.VideoLayer + previousParkedLayer buffer.VideoLayer - currentLayer buffer.VideoLayer + currentLayer buffer.VideoLayer + previousLayer buffer.VideoLayer } func NewBase(logger logger.Logger) *Base { return &Base{ - logger: logger, - maxLayer: buffer.InvalidLayer, - targetLayer: buffer.InvalidLayer, // start off with nothing, let streamallocator/opportunistic forwarder set the target - requestSpatial: buffer.InvalidLayerSpatial, - maxSeenLayer: buffer.InvalidLayer, - parkedLayer: buffer.InvalidLayer, - currentLayer: buffer.InvalidLayer, + logger: logger, + maxLayer: buffer.InvalidLayer, + maxSeenLayer: buffer.InvalidLayer, + targetLayer: buffer.InvalidLayer, // start off with nothing, let streamallocator/opportunistic forwarder set the target + previousTargetLayer: buffer.InvalidLayer, + requestSpatial: buffer.InvalidLayerSpatial, + parkedLayer: buffer.InvalidLayer, + previousParkedLayer: buffer.InvalidLayer, + currentLayer: buffer.InvalidLayer, + previousLayer: buffer.InvalidLayer, } } @@ -58,6 +66,7 @@ func (b *Base) GetMax() buffer.VideoLayer { } func (b *Base) SetTarget(targetLayer buffer.VideoLayer) { + b.previousTargetLayer = targetLayer b.targetLayer = targetLayer } @@ -115,12 +124,49 @@ func (b *Base) Select(_extPkt *buffer.ExtPacket, _layer int32) (result VideoLaye return } -func (b *Base) SelectTemporal(extPkt *buffer.ExtPacket) int32 { +func (b *Base) Rollback() { + b.logger.Infow( + "rolling back", + "previous", b.previousLayer, + "current", b.currentLayer, + "previousParked", b.previousParkedLayer, + "parked", b.parkedLayer, + "previousTarget", b.previousTargetLayer, + "target", b.targetLayer, + "max", b.maxLayer, + "req", b.requestSpatial, + "maxSeen", b.maxSeenLayer, + ) + b.parkedLayer = b.previousParkedLayer + b.currentLayer = b.previousLayer + b.targetLayer = b.previousTargetLayer +} + +func (b *Base) SelectTemporal(extPkt *buffer.ExtPacket) (int32, bool) { if b.tls != nil { + isSwitching := false this, next := b.tls.Select(extPkt, b.currentLayer.Temporal, b.targetLayer.Temporal) - b.currentLayer.Temporal = next - return this + if next != b.currentLayer.Temporal { + isSwitching = true + + b.previousLayer = b.currentLayer + b.currentLayer.Temporal = next + + b.logger.Infow( + "updating temporal layer", + "previous", b.previousLayer, + "current", b.currentLayer, + "previousParked", b.previousParkedLayer, + "parked", b.parkedLayer, + "previousTarget", b.previousTargetLayer, + "target", b.targetLayer, + "max", b.maxLayer, + "req", b.requestSpatial, + "maxSeen", b.maxSeenLayer, + ) + } + return this, isSwitching } - return b.currentLayer.Temporal + return b.currentLayer.Temporal, false } diff --git a/pkg/sfu/videolayerselector/dependencydescriptor.go b/pkg/sfu/videolayerselector/dependencydescriptor.go index 3cc5fcf70..7b8c95310 100644 --- a/pkg/sfu/videolayerselector/dependencydescriptor.go +++ b/pkg/sfu/videolayerselector/dependencydescriptor.go @@ -16,8 +16,9 @@ type DependencyDescriptor struct { frameNum *utils.WrapAround[uint16, uint64] decisions *SelectorDecisionCache - activeDecodeTargetsBitmask *uint32 - structure *dede.FrameDependencyStructure + previousActiveDecodeTargetsBitmask *uint32 + activeDecodeTargetsBitmask *uint32 + structure *dede.FrameDependencyStructure chains []*FrameChain @@ -183,6 +184,7 @@ func (d *DependencyDescriptor) Select(extPkt *buffer.ExtPacket, _layer int32) (r } if d.currentLayer != highestDecodeTarget.Layer { + result.IsSwitching = true if !d.currentLayer.IsValid() { result.IsResuming = true d.logger.Infow( @@ -196,8 +198,13 @@ func (d *DependencyDescriptor) Select(extPkt *buffer.ExtPacket, _layer int32) (r "feed", extPkt.Packet.SSRC, ) } + + d.previousLayer = d.currentLayer d.currentLayer = highestDecodeTarget.Layer + + d.previousActiveDecodeTargetsBitmask = d.activeDecodeTargetsBitmask d.activeDecodeTargetsBitmask = buffer.GetActiveDecodeTargetBitmask(d.currentLayer, ddwdt.DecodeTargets) + if d.currentLayer.Spatial == d.requestSpatial { result.IsSwitchingToRequestSpatial = true } @@ -206,7 +213,9 @@ func (d *DependencyDescriptor) Select(extPkt *buffer.ExtPacket, _layer int32) (r result.MaxSpatialLayer = d.currentLayer.Spatial d.logger.Infow( "reached max layer", + "previous", d.previousLayer, "current", d.currentLayer, + "previousTarget", d.previousTargetLayer, "target", d.targetLayer, "max", d.maxLayer, "layer", fd.SpatialId, @@ -255,12 +264,10 @@ func (d *DependencyDescriptor) Select(extPkt *buffer.ExtPacket, _layer int32) (r return } -func (d *DependencyDescriptor) SetTarget(targetLayer buffer.VideoLayer) { - if targetLayer == d.targetLayer { - return - } +func (d *DependencyDescriptor) Rollback() { + d.activeDecodeTargetsBitmask = d.previousActiveDecodeTargetsBitmask - d.Base.SetTarget(targetLayer) + d.Base.Rollback() } func (d *DependencyDescriptor) updateDependencyStructure(structure *dede.FrameDependencyStructure, decodeTargets []buffer.DependencyDescriptorDecodeTarget) { diff --git a/pkg/sfu/videolayerselector/simulcast.go b/pkg/sfu/videolayerselector/simulcast.go index fe6f33659..ebb80d113 100644 --- a/pkg/sfu/videolayerselector/simulcast.go +++ b/pkg/sfu/videolayerselector/simulcast.go @@ -26,111 +26,12 @@ func (s *Simulcast) IsOvershootOkay() bool { } func (s *Simulcast) Select(extPkt *buffer.ExtPacket, layer int32) (result VideoLayerSelectorResult) { - if s.currentLayer.Spatial != s.targetLayer.Spatial { - // Three things to check when not locked to target - // 1. Resumable layer - don't need a key frame - // 2. Opportunistic layer upgrade - needs a key frame - // 3. Need to downgrade - needs a key frame - isActive := s.currentLayer.IsValid() - found := false - if s.parkedLayer.IsValid() { - if s.parkedLayer.Spatial == layer { - s.logger.Infow( - "resuming at parked layer", - "current", s.currentLayer, - "target", s.targetLayer, - "max", s.maxLayer, - "parked", s.parkedLayer, - "req", s.requestSpatial, - "maxSeen", s.maxSeenLayer, - "feed", extPkt.Packet.SSRC, - ) - s.currentLayer = s.parkedLayer - found = true - } - } else { - if extPkt.KeyFrame { - if layer > s.currentLayer.Spatial && layer <= s.targetLayer.Spatial { - s.logger.Infow( - "upgrading layer", - "current", s.currentLayer, - "target", s.targetLayer, - "max", s.maxLayer, - "layer", layer, - "req", s.requestSpatial, - "maxSeen", s.maxSeenLayer, - "feed", extPkt.Packet.SSRC, - ) - found = true - } - - if layer < s.currentLayer.Spatial && layer >= s.targetLayer.Spatial { - s.logger.Infow( - "downgrading layer", - "current", s.currentLayer, - "target", s.targetLayer, - "max", s.maxLayer, - "layer", layer, - "req", s.requestSpatial, - "maxSeen", s.maxSeenLayer, - "feed", extPkt.Packet.SSRC, - ) - found = true - } - - if found { - s.currentLayer.Spatial = layer - s.currentLayer.Temporal = extPkt.VideoLayer.Temporal - } - } + populateSwitches := func(isActive bool, reason string) { + result.IsSwitching = true + if !isActive { + result.IsResuming = true } - if found { - if !isActive { - result.IsResuming = true - } - - s.SetParked(buffer.InvalidLayer) - - if s.currentLayer.Spatial == s.requestSpatial { - result.IsSwitchingToRequestSpatial = true - } - - if s.currentLayer.Spatial >= s.maxLayer.Spatial { - result.IsSwitchingToMaxSpatial = true - result.MaxSpatialLayer = s.currentLayer.Spatial - s.logger.Infow( - "reached max layer", - "current", s.currentLayer, - "target", s.targetLayer, - "max", s.maxLayer, - "layer", layer, - "req", s.requestSpatial, - "maxSeen", s.maxSeenLayer, - "feed", extPkt.Packet.SSRC, - ) - } - - if s.currentLayer.Spatial >= s.maxLayer.Spatial || s.currentLayer.Spatial == s.maxSeenLayer.Spatial { - s.targetLayer.Spatial = s.currentLayer.Spatial - } - } - } - - // if locked to higher than max layer due to overshoot, check if it can be dialed back - if s.currentLayer.Spatial > s.maxLayer.Spatial && layer <= s.maxLayer.Spatial && extPkt.KeyFrame { - s.logger.Infow( - "adjusting overshoot", - "current", s.currentLayer, - "target", s.targetLayer, - "max", s.maxLayer, - "layer", layer, - "req", s.requestSpatial, - "maxSeen", s.maxSeenLayer, - "feed", extPkt.Packet.SSRC, - ) - s.currentLayer.Spatial = layer - if s.currentLayer.Spatial == s.requestSpatial { result.IsSwitchingToRequestSpatial = true } @@ -138,11 +39,92 @@ func (s *Simulcast) Select(extPkt *buffer.ExtPacket, layer int32) (result VideoL if s.currentLayer.Spatial >= s.maxLayer.Spatial { result.IsSwitchingToMaxSpatial = true result.MaxSpatialLayer = s.currentLayer.Spatial + if reason != "" { + reason += ", " + } + reason += "reached max layer" } + if reason != "" { + s.logger.Infow( + reason, + "previous", s.previousLayer, + "current", s.currentLayer, + "previousParked", s.previousParkedLayer, + "parked", s.parkedLayer, + "previousTarget", s.previousTargetLayer, + "target", s.targetLayer, + "max", s.maxLayer, + "layer", layer, + "req", s.requestSpatial, + "maxSeen", s.maxSeenLayer, + "feed", extPkt.Packet.SSRC, + ) + } + } + + if s.currentLayer.Spatial != s.targetLayer.Spatial { + currentLayer := s.currentLayer + + // Three things to check when not locked to target + // 1. Resumable layer - don't need a key frame + // 2. Opportunistic layer upgrade - needs a key frame + // 3. Need to downgrade - needs a key frame + isActive := s.currentLayer.IsValid() + found := false + reason := "" + if s.parkedLayer.IsValid() { + if s.parkedLayer.Spatial == layer { + reason = "resuming at parked layer" + currentLayer = s.parkedLayer + found = true + } + } else { + if extPkt.KeyFrame { + if layer > s.currentLayer.Spatial && layer <= s.targetLayer.Spatial { + reason = "upgrading layer" + found = true + } + + if layer < s.currentLayer.Spatial && layer >= s.targetLayer.Spatial { + reason = "downgrading layer" + found = true + } + + if found { + currentLayer.Spatial = layer + currentLayer.Temporal = extPkt.VideoLayer.Temporal + } + } + } + + if found { + s.previousParkedLayer = s.parkedLayer + s.parkedLayer = buffer.InvalidLayer + + s.previousLayer = s.currentLayer + s.currentLayer = currentLayer + + s.previousTargetLayer = s.targetLayer + if s.currentLayer.Spatial >= s.maxLayer.Spatial || s.currentLayer.Spatial == s.maxSeenLayer.Spatial { + s.targetLayer.Spatial = s.currentLayer.Spatial + } + + populateSwitches(isActive, reason) + } + } + + // if locked to higher than max layer due to overshoot, check if it can be dialed back + if s.currentLayer.Spatial > s.maxLayer.Spatial && layer <= s.maxLayer.Spatial && extPkt.KeyFrame { + s.previousLayer = s.currentLayer + s.currentLayer.Spatial = layer + + s.previousTargetLayer = s.targetLayer if s.currentLayer.Spatial >= s.maxLayer.Spatial || s.currentLayer.Spatial == s.maxSeenLayer.Spatial { s.targetLayer.Spatial = layer } + + populateSwitches(true, "adjusting overshoot") } result.RTPMarker = extPkt.Packet.Marker diff --git a/pkg/sfu/videolayerselector/videolayerselector.go b/pkg/sfu/videolayerselector/videolayerselector.go index b108976bf..ffbb9f42c 100644 --- a/pkg/sfu/videolayerselector/videolayerselector.go +++ b/pkg/sfu/videolayerselector/videolayerselector.go @@ -8,6 +8,7 @@ import ( type VideoLayerSelectorResult struct { IsSelected bool IsRelevant bool + IsSwitching bool IsResuming bool IsSwitchingToRequestSpatial bool IsSwitchingToMaxSpatial bool @@ -46,5 +47,6 @@ type VideoLayerSelector interface { GetCurrent() buffer.VideoLayer Select(extPkt *buffer.ExtPacket, layer int32) VideoLayerSelectorResult - SelectTemporal(extPkt *buffer.ExtPacket) int32 + SelectTemporal(extPkt *buffer.ExtPacket) (int32, bool) + Rollback() } diff --git a/pkg/sfu/videolayerselector/vp9.go b/pkg/sfu/videolayerselector/vp9.go index 4a77f3675..ed1a165b6 100644 --- a/pkg/sfu/videolayerselector/vp9.go +++ b/pkg/sfu/videolayerselector/vp9.go @@ -75,6 +75,7 @@ func (v *VP9) Select(extPkt *buffer.ExtPacket, _layer int32) (result VideoLayerS } if updatedLayer != v.currentLayer { + result.IsSwitching = true if !v.currentLayer.IsValid() && updatedLayer.IsValid() { result.IsResuming = true } @@ -89,6 +90,7 @@ func (v *VP9) Select(extPkt *buffer.ExtPacket, _layer int32) (result VideoLayerS v.logger.Infow( "reached max layer", "current", v.currentLayer, + "updated", updatedLayer, "target", v.targetLayer, "max", v.maxLayer, "layer", extPkt.VideoLayer.Spatial, @@ -98,6 +100,7 @@ func (v *VP9) Select(extPkt *buffer.ExtPacket, _layer int32) (result VideoLayerS ) } + v.previousLayer = v.currentLayer v.currentLayer = updatedLayer } } From 9cf190bdf7c60bce1392d1be24a362016124e68c Mon Sep 17 00:00:00 2001 From: David Colburn Date: Wed, 12 Jul 2023 18:46:48 -0700 Subject: [PATCH 286/324] update protocol, psrpc (#1872) --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 57516561d..09dbd2d90 100644 --- a/go.mod +++ b/go.mod @@ -18,8 +18,8 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230612070454-d5299b956135 - github.com/livekit/protocol v1.5.9 - github.com/livekit/psrpc v0.3.1 + github.com/livekit/protocol v1.5.10-0.20230713013703-71d539707e4c + github.com/livekit/psrpc v0.3.2 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 github.com/maxbrunsfeld/counterfeiter/v6 v6.6.2 @@ -101,6 +101,6 @@ require ( golang.org/x/text v0.10.0 // indirect golang.org/x/tools v0.9.3 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect - google.golang.org/grpc v1.56.1 // indirect + google.golang.org/grpc v1.56.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index e22cfb93e..496e76719 100644 --- a/go.sum +++ b/go.sum @@ -124,10 +124,10 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20230612070454-d5299b956135 h1:lWYbsondvqG69czxoACDwaJ/BoyD57BahCo70ZH+m4U= github.com/livekit/mediatransportutil v0.0.0-20230612070454-d5299b956135/go.mod h1:MRc0zSOSzXuFt0X218SgabzlaKevkvCckPgBEoHYc34= -github.com/livekit/protocol v1.5.9 h1:fqPOLgKkWmkmUMnpfj2KDZlidHgAazAPU2T3FewyLew= -github.com/livekit/protocol v1.5.9/go.mod h1:GMTlFbc0JypUGo+PMilDrL45AX+yUBiQ1Cl/FGZibDY= -github.com/livekit/psrpc v0.3.1 h1:KfylgJHvoLQcc22t/oflwMOeSnx0c14G7cWsS+9MYS4= -github.com/livekit/psrpc v0.3.1/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= +github.com/livekit/protocol v1.5.10-0.20230713013703-71d539707e4c h1:o2xIq3un7M7un8wU7jYw+6mV1uSp7oWA2lCHRm5m3+Y= +github.com/livekit/protocol v1.5.10-0.20230713013703-71d539707e4c/go.mod h1:eRzojAYSPJuNgDHMlvLji/CPauj9hrgvb6rVPUj6MoU= +github.com/livekit/psrpc v0.3.2 h1:eAaJhASme33gtoBhCRLH9jsnWcdm1tHWf0WzaDk56ew= +github.com/livekit/psrpc v0.3.2/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= github.com/mackerelio/go-osstat v0.2.4/go.mod h1:Zy+qzGdZs3A9cuIqmgbJvwbmLQH9dJvtio5ZjJTbdlQ= github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= @@ -411,8 +411,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc= google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ= -google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +google.golang.org/grpc v1.56.2 h1:fVRFRnXvU+x6C4IlHZewvJOVHoOv1TUuQyoRsYnB4bI= +google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= From e746fe14e1267052b43386dd9949c9fa722ef675 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 13 Jul 2023 10:42:23 +0530 Subject: [PATCH 287/324] Mark active when switching to parked layer. (#1873) * Mark active when switching to parked layer. Parked layer lock is not a switch. It is just a restart at the same layer. * make explicit bool for switching --- pkg/sfu/videolayerselector/simulcast.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/pkg/sfu/videolayerselector/simulcast.go b/pkg/sfu/videolayerselector/simulcast.go index ebb80d113..4d0c9294d 100644 --- a/pkg/sfu/videolayerselector/simulcast.go +++ b/pkg/sfu/videolayerselector/simulcast.go @@ -26,8 +26,11 @@ func (s *Simulcast) IsOvershootOkay() bool { } func (s *Simulcast) Select(extPkt *buffer.ExtPacket, layer int32) (result VideoLayerSelectorResult) { - populateSwitches := func(isActive bool, reason string) { - result.IsSwitching = true + populateSwitches := func(isSwitching bool, isActive bool, reason string) { + if isSwitching { + result.IsSwitching = true + } + if !isActive { result.IsResuming = true } @@ -70,6 +73,7 @@ func (s *Simulcast) Select(extPkt *buffer.ExtPacket, layer int32) (result VideoL // 1. Resumable layer - don't need a key frame // 2. Opportunistic layer upgrade - needs a key frame // 3. Need to downgrade - needs a key frame + isSwitching := true isActive := s.currentLayer.IsValid() found := false reason := "" @@ -77,6 +81,7 @@ func (s *Simulcast) Select(extPkt *buffer.ExtPacket, layer int32) (result VideoL if s.parkedLayer.Spatial == layer { reason = "resuming at parked layer" currentLayer = s.parkedLayer + isSwitching = false found = true } } else { @@ -110,7 +115,7 @@ func (s *Simulcast) Select(extPkt *buffer.ExtPacket, layer int32) (result VideoL s.targetLayer.Spatial = s.currentLayer.Spatial } - populateSwitches(isActive, reason) + populateSwitches(isSwitching, isActive, reason) } } @@ -124,7 +129,7 @@ func (s *Simulcast) Select(extPkt *buffer.ExtPacket, layer int32) (result VideoL s.targetLayer.Spatial = layer } - populateSwitches(true, "adjusting overshoot") + populateSwitches(true, true, "adjusting overshoot") } result.RTPMarker = extPkt.Packet.Marker From 557fe7c9d3e722fe58eaacfe7d1f9e7ea4fb5d11 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Thu, 13 Jul 2023 16:33:04 -0700 Subject: [PATCH 288/324] Mark room as dirty after track published changes (#1878) Ensure that we are recomputing NumPublished when needed --- pkg/rtc/room.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index dcd5a6b1c..a90b90676 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -828,6 +828,7 @@ func (r *Room) createJoinResponseLocked(participant types.LocalParticipant, iceS func (r *Room) onTrackPublished(participant types.LocalParticipant, track types.MediaTrack) { // publish participant update, since track state is changed r.broadcastParticipantState(participant, broadcastOptions{skipSource: true}) + r.protoProxy.MarkDirty(false) r.lock.RLock() // subscribe all existing participants to this MediaTrack @@ -887,6 +888,7 @@ func (r *Room) onTrackUpdated(p types.LocalParticipant, _ types.MediaTrack) { func (r *Room) onTrackUnpublished(p types.LocalParticipant, track types.MediaTrack) { r.trackManager.RemoveTrack(track) + r.protoProxy.MarkDirty(false) if !p.IsClosed() { r.broadcastParticipantState(p, broadcastOptions{skipSource: true}) } From 68e5fa8e1c85975d98b4c91b00c077a10eae3557 Mon Sep 17 00:00:00 2001 From: Benjamin Pracht Date: Fri, 14 Jul 2023 09:11:55 +0800 Subject: [PATCH 289/324] Allow listing ingress by id (#1874) --- go.mod | 2 +- go.sum | 4 ++-- pkg/service/ingress.go | 17 +++++++++++++---- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 09dbd2d90..894e44d78 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230612070454-d5299b956135 - github.com/livekit/protocol v1.5.10-0.20230713013703-71d539707e4c + github.com/livekit/protocol v1.5.10-0.20230714010226-3c53edc91962 github.com/livekit/psrpc v0.3.2 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 diff --git a/go.sum b/go.sum index 496e76719..dba27c2ea 100644 --- a/go.sum +++ b/go.sum @@ -124,8 +124,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20230612070454-d5299b956135 h1:lWYbsondvqG69czxoACDwaJ/BoyD57BahCo70ZH+m4U= github.com/livekit/mediatransportutil v0.0.0-20230612070454-d5299b956135/go.mod h1:MRc0zSOSzXuFt0X218SgabzlaKevkvCckPgBEoHYc34= -github.com/livekit/protocol v1.5.10-0.20230713013703-71d539707e4c h1:o2xIq3un7M7un8wU7jYw+6mV1uSp7oWA2lCHRm5m3+Y= -github.com/livekit/protocol v1.5.10-0.20230713013703-71d539707e4c/go.mod h1:eRzojAYSPJuNgDHMlvLji/CPauj9hrgvb6rVPUj6MoU= +github.com/livekit/protocol v1.5.10-0.20230714010226-3c53edc91962 h1:y+rtYNMGmvpEgQlNG/wOUO16S497ygh83wQSUBKHpG4= +github.com/livekit/protocol v1.5.10-0.20230714010226-3c53edc91962/go.mod h1:eRzojAYSPJuNgDHMlvLji/CPauj9hrgvb6rVPUj6MoU= github.com/livekit/psrpc v0.3.2 h1:eAaJhASme33gtoBhCRLH9jsnWcdm1tHWf0WzaDk56ew= github.com/livekit/psrpc v0.3.2/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= diff --git a/pkg/service/ingress.go b/pkg/service/ingress.go index 04d4a6bd8..5fa7990ee 100644 --- a/pkg/service/ingress.go +++ b/pkg/service/ingress.go @@ -216,10 +216,19 @@ func (s *IngressService) ListIngress(ctx context.Context, req *livekit.ListIngre return nil, ErrIngressNotConnected } - infos, err := s.store.ListIngress(ctx, livekit.RoomName(req.RoomName)) - if err != nil { - logger.Errorw("could not list ingress info", err) - return nil, err + var infos []*livekit.IngressInfo + if req.IngressId != "" { + info, err := s.store.LoadIngress(ctx, req.IngressId) + if err != nil { + return nil, err + } + infos = []*livekit.IngressInfo{info} + } else { + infos, err = s.store.ListIngress(ctx, livekit.RoomName(req.RoomName)) + if err != nil { + logger.Errorw("could not list ingress info", err) + return nil, err + } } return &livekit.ListIngressResponse{Items: infos}, nil From 4c02a6d71720062a0c61bdb88a6106cc62dbe392 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 14 Jul 2023 11:47:07 +0530 Subject: [PATCH 290/324] Time stamp adjustments v2 (I think) (#1875) * WIP commit * WIP commit * WIP commit * Some clean up - Removed a chatty debug log - some spelling, punctuation correction in comments - missed an `Abs` in check, add it. --- pkg/rtc/wrappedreceiver.go | 7 +- pkg/sfu/buffer/rtpstats.go | 93 ++++----- pkg/sfu/downtrack.go | 5 +- pkg/sfu/forwarder.go | 321 +++++++++++++++++--------------- pkg/sfu/receiver.go | 6 +- pkg/sfu/streamtrackermanager.go | 141 +++++++++----- 6 files changed, 320 insertions(+), 253 deletions(-) diff --git a/pkg/rtc/wrappedreceiver.go b/pkg/rtc/wrappedreceiver.go index 545ac1b78..7028084fb 100644 --- a/pkg/rtc/wrappedreceiver.go +++ b/pkg/rtc/wrappedreceiver.go @@ -12,7 +12,6 @@ import ( "github.com/livekit/protocol/logger" "github.com/livekit/livekit-server/pkg/sfu" - "github.com/livekit/livekit-server/pkg/sfu/buffer" ) // wrapper around WebRTC receiver, overriding its ID @@ -297,11 +296,11 @@ func (d *DummyReceiver) GetRedReceiver() sfu.TrackReceiver { return d } -func (d *DummyReceiver) GetRTCPSenderReportData(layer int32) (*buffer.RTCPSenderReportData, *buffer.RTCPSenderReportData) { +func (d *DummyReceiver) GetCalculatedClockRate(layer int32) uint32 { if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { - return r.GetRTCPSenderReportData(layer) + return r.GetCalculatedClockRate(layer) } - return nil, nil + return 0 } func (d *DummyReceiver) GetReferenceLayerRTPTimestamp(ts uint32, layer int32, referenceLayer int32) (uint32, error) { diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index c21938d36..8bf34b958 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -750,13 +750,13 @@ func (r *RTPStats) GetRtt() uint32 { } func (r *RTPStats) SetRtcpSenderReportData(srData *RTCPSenderReportData) { - if srData == nil { - return - } - r.lock.Lock() defer r.lock.Unlock() + if srData == nil || !r.initialized { + return + } + // prevent against extreme case of anachronous sender reports if r.srNewest != nil && r.srNewest.NTPTimestamp > srData.NTPTimestamp { r.logger.Infow( @@ -767,29 +767,6 @@ func (r *RTPStats) SetRtcpSenderReportData(srData *RTCPSenderReportData) { return } - // monitor and log RTP timestamp anomalies - var ntpDiffSinceLast time.Duration - var rtpDiffSinceLast uint32 - var arrivalDiffSinceLast time.Duration - var expectedTimeDiffSinceLast float64 - var reason string - if r.srNewest != nil { - ntpDiffSinceLast = srData.NTPTimestamp.Time().Sub(r.srNewest.NTPTimestamp.Time()) - rtpDiffSinceLast = srData.RTPTimestamp - r.srNewest.RTPTimestamp - arrivalDiffSinceLast = srData.At.Sub(r.srNewest.At) - - expectedTimeDiffSinceLast = float64(rtpDiffSinceLast) / float64(r.params.ClockRate) - - if (srData.RTPTimestamp - r.srNewest.RTPTimestamp) > (1 << 31) { - reason = "received sender report, out-of-order" // should not happen, just a sanity check - } else { - if math.Abs(expectedTimeDiffSinceLast-ntpDiffSinceLast.Seconds()) > 0.2 { - // more than 200 ms away from expected delta - reason = "received sender report, time warp" - } - } - } - cycles := uint64(0) if r.srNewest != nil { cycles = r.srNewest.RTPTimestampExt & 0xFF_FF_FF_FF_00_00_00_00 @@ -800,15 +777,47 @@ func (r *RTPStats) SetRtcpSenderReportData(srData *RTCPSenderReportData) { srDataCopy := *srData srDataCopy.RTPTimestampExt = uint64(srDataCopy.RTPTimestamp) + cycles + + // monitor and log RTP timestamp anomalies + var ntpDiffSinceLast time.Duration + var rtpDiffSinceLast uint32 + var arrivalDiffSinceLast time.Duration + var expectedTimeDiffSinceLast float64 + var isWarped bool + if r.srNewest != nil { + if srDataCopy.RTPTimestampExt < r.srNewest.RTPTimestampExt { + // This can happen when a track is replaced with a null and then restored - + // i. e. muting replacing with null and unmute restoring the original track. + // Under such a condition reset the sender reports to start from this point. + // Resetting will ensure sample rate calculations do not go haywire due to negative time. + r.logger.Infow( + "received sender report, out-of-order, resetting", + "prevTSExt", r.srNewest.RTPTimestampExt, + "currTSExt", srDataCopy.RTPTimestampExt, + ) + r.srFirst = &srDataCopy + r.srNewest = &srDataCopy + } + + ntpDiffSinceLast = srDataCopy.NTPTimestamp.Time().Sub(r.srNewest.NTPTimestamp.Time()) + rtpDiffSinceLast = srDataCopy.RTPTimestamp - r.srNewest.RTPTimestamp + arrivalDiffSinceLast = srDataCopy.At.Sub(r.srNewest.At) + expectedTimeDiffSinceLast = float64(rtpDiffSinceLast) / float64(r.params.ClockRate) + if math.Abs(expectedTimeDiffSinceLast-ntpDiffSinceLast.Seconds()) > 0.2 { + // more than 200 ms away from expected delta + isWarped = true + } + } + r.srNewest = &srDataCopy if r.srFirst == nil { r.srFirst = &srDataCopy } - if reason != "" { + if isWarped { packetDriftResult, reportDriftResult := r.getDrift() r.logger.Infow( - reason, + "received sender report, time warp", "ntp", srData.NTPTimestamp.Time().String(), "rtp", srData.RTPTimestamp, "arrival", srData.At.String(), @@ -840,26 +849,22 @@ func (r *RTPStats) GetRtcpSenderReportData() (srFirst *RTCPSenderReportData, srN return } -func (r *RTPStats) GetExpectedRTPTimestamp(at time.Time) (uint32, uint64, error) { +func (r *RTPStats) GetExpectedRTPTimestamp(at time.Time) (expectedTSExt uint64, err error) { r.lock.RLock() defer r.lock.RUnlock() if !r.initialized { - return 0, 0, errors.New("uninitilaized") + err = errors.New("uninitilaized") + return } timeDiff := at.Sub(r.firstTime) expectedRTPDiff := timeDiff.Nanoseconds() * int64(r.params.ClockRate) / 1e9 - expectedExtRTP := r.extStartTS + uint64(expectedRTPDiff) - - minTS := ^uint64(0) - if r.srNewest != nil { - minTS = r.srNewest.RTPTimestampExt - } - return uint32(expectedExtRTP), minTS, nil + expectedTSExt = r.extStartTS + uint64(expectedRTPDiff) + return } -func (r *RTPStats) GetRtcpSenderReport(ssrc uint32, srFirst *RTCPSenderReportData, srNewest *RTCPSenderReportData) *rtcp.SenderReport { +func (r *RTPStats) GetRtcpSenderReport(ssrc uint32, calculatedClockRate uint32) *rtcp.SenderReport { r.lock.Lock() defer r.lock.Unlock() @@ -877,18 +882,14 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32, srFirst *RTCPSenderReportDat // It is possible that publisher is pacing at a slower rate. // That would make `highestTS` to be lagging the RTP time stamp in the RTCP Sender Report from publisher. - // Check for that and use the later time stamp if applicable. + // Check for that using calculated clock rate and use the later time stamp if applicable. tsCycles := r.tsCycles if nowRTP < r.highestTS { tsCycles++ } nowRTPExt := getExtTS(nowRTP, tsCycles) - if srFirst != nil && srNewest != nil && srFirst.RTPTimestamp != srNewest.RTPTimestamp { - // use incoming rate as a guide - tsf := srNewest.NTPTimestamp.Time().Sub(srFirst.NTPTimestamp.Time()) - rdsf := srNewest.RTPTimestampExt - srFirst.RTPTimestampExt - sr := float64(rdsf) / tsf.Seconds() - nowRTPExtUsingRate := r.extStartTS + uint64(sr*timeSinceFirst.Seconds()) + if calculatedClockRate != 0 { + nowRTPExtUsingRate := r.extStartTS + uint64(float64(calculatedClockRate)*timeSinceFirst.Seconds()) if nowRTPExtUsingRate > nowRTPExt { nowRTPExt = nowRTPExtUsingRate nowRTP = uint32(nowRTPExtUsingRate) diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index f0463a1e0..c6dbb34fd 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -1102,8 +1102,7 @@ func (d *DownTrack) CreateSenderReport() *rtcp.SenderReport { return nil } - srFirst, srNewest := d.receiver.GetRTCPSenderReportData(d.forwarder.GetReferenceLayerSpatial()) - return d.rtpStats.GetRtcpSenderReport(d.ssrc, srFirst, srNewest) + return d.rtpStats.GetRtcpSenderReport(d.ssrc, d.receiver.GetCalculatedClockRate(d.forwarder.CurrentLayer().Spatial)) } func (d *DownTrack) writeBlankFrameRTP(duration float32, generation uint32) chan struct{} { @@ -1542,7 +1541,7 @@ func (d *DownTrack) DebugInfo() map[string]interface{} { } } -func (d *DownTrack) getExpectedRTPTimestamp(at time.Time) (uint32, uint64, error) { +func (d *DownTrack) getExpectedRTPTimestamp(at time.Time) (uint64, error) { return d.rtpStats.GetExpectedRTPTimestamp(at) } diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 8bb21c387..1da413ee8 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -26,6 +26,10 @@ const ( FlagFilterRTX = true TransitionCostSpatial = 10 ParkedLayerWaitDuration = 2 * time.Second + + ResumeBehindThresholdSeconds = float64(0.1) // 100ms + LayerSwitchBehindThresholdSeconds = float64(0.05) // 50ms + SwitchAheadThresholdSeconds = float64(0.025) // 25ms ) // ------------------------------------------------------------------- @@ -171,7 +175,7 @@ type Forwarder struct { kind webrtc.RTPCodecType logger logger.Logger getReferenceLayerRTPTimestamp func(ts uint32, layer int32, referenceLayer int32) (uint32, error) - getExpectedRTPTimestamp func(at time.Time) (uint32, uint64, error) + getExpectedRTPTimestamp func(at time.Time) (uint64, error) muted bool pubMuted bool @@ -202,7 +206,7 @@ func NewForwarder( kind webrtc.RTPCodecType, logger logger.Logger, getReferenceLayerRTPTimestamp func(ts uint32, layer int32, referenceLayer int32) (uint32, error), - getExpectedRTPTimestamp func(at time.Time) (uint32, uint64, error), + getExpectedRTPTimestamp func(at time.Time) (uint64, error), ) *Forwarder { f := &Forwarder{ kind: kind, @@ -508,13 +512,6 @@ func (f *Forwarder) TargetLayer() buffer.VideoLayer { return f.vls.GetTarget() } -func (f *Forwarder) GetReferenceLayerSpatial() int32 { - f.lock.RLock() - defer f.lock.RUnlock() - - return f.referenceLayerSpatial -} - func (f *Forwarder) isDeficientLocked() bool { return f.lastAllocation.IsDeficient } @@ -1477,103 +1474,176 @@ func (f *Forwarder) GetTranslationParams(extPkt *buffer.ExtPacket, layer int32) return nil, ErrUnknownKind } +func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) error { + if !f.started { + f.started = true + f.referenceLayerSpatial = layer + f.rtpMunger.SetLastSnTs(extPkt) + f.codecMunger.SetLast(extPkt) + f.logger.Infow( + "starting forwarding", + "sequenceNumber", extPkt.Packet.SequenceNumber, + "timestamp", extPkt.Packet.Timestamp, + "layer", layer, + "referenceLayerSpatial", f.referenceLayerSpatial, + ) + return nil + } + + if f.referenceLayerSpatial == buffer.InvalidLayerSpatial { + // on a resume, reference layer may not be set, so only set when it is invalid + f.referenceLayerSpatial = layer + } + + // Compute how much time passed between the previous forwarded packet + // and the current incoming (to be forwarded) packet and calculate + // timestamp offset on source change. + // + // There are three timestamps to consider here + // 1. lastTS -> timestamp of last sent packet + // 2. refTS -> timestamp of this packet (after munging) calculated using feed's RTCP sender report + // 3. expectedTS -> expected timestamp of this packet calculated based on elapsed time since first packet + // Ideally, refTS and expectedTS should be very close and lastTS should be before both of those. + // But, cases like muting/unmuting, clock vagaries, pacing, etc. make them not satisfy those conditions always. + lastTS := f.rtpMunger.GetLast().LastTS + refTS := lastTS + expectedTS := lastTS + switchingAt := time.Now() + if f.getReferenceLayerRTPTimestamp != nil { + ts, err := f.getReferenceLayerRTPTimestamp(extPkt.Packet.Timestamp, layer, f.referenceLayerSpatial) + if err == nil { + refTS = ts + } + // AVSYNC-TODO: can error out here if refTS is not available. It can happen when there is no sender report + // for the layer being switched to. Can especially happen at the start of the track when layer switches are + // potentially happening very quickly. Erroring out and waiting for a layer for which a sender report has been + // received will calculate a better offset, but may result in initial adaptation to take a bit longer depending + // on how often publisher/remote side sends RTCP sender report. + } + + if f.getExpectedRTPTimestamp != nil { + tsExt, err := f.getExpectedRTPTimestamp(switchingAt) + if err == nil { + expectedTS = uint32(tsExt) + } else { + rtpDiff := uint32(0) + if !f.preStartTime.IsZero() && f.refTSOffset == 0 { + timeSinceFirst := time.Since(f.preStartTime) + rtpDiff = uint32(timeSinceFirst.Nanoseconds() * int64(f.codec.ClockRate) / 1e9) + f.refTSOffset = f.firstTS + rtpDiff - refTS + f.logger.Infow( + "calculating refTSOffset", + "preStartTime", f.preStartTime.String(), + "firstTS", f.firstTS, + "timeSinceFirst", timeSinceFirst, + "rtpDiff", rtpDiff, + "refTS", refTS, + "refTSOffset", f.refTSOffset, + ) + } + expectedTS += rtpDiff + } + } + refTS += f.refTSOffset + + var nextTS uint32 + if f.lastSSRC == 0 { + // If resuming (e. g. on unmute), keep next timestamp close to expected timestamp. + // + // Rationale: + // Case 1: If mute is implemented via something like stopping a track and resuming it on unmute, + // the RTP timestamp may not have jumped across mute valley. In this case, old timestamp + // should not be used. + // + // Case 2: OTOH, something like pacing may be adding latency in the publisher path (even if + // the timestamps incremented correctly across the mute valley). In this case, reference + // timestamp should be used as things will catch up to real time when channel capacity + // increases and pacer starts sending at faster rate. + // + // But, the challenege is distinguishing between the two cases. As a compromise, the difference + // between expectedTS and refTS is thresholded. Difference below the threshold is treated as Case 2 + // and above as Case 1. + // + // In the event of refTS > expectedTS, another threshold is used to pick the next timestamp. + // Ideally, refTS should not be ahead of expectedTS, but expectedTS uses the first packet's + // wall clock time. So, if the first packet experienced abmormal latency, it is possible + // for refTS > expectedTS + diffSeconds := float64(expectedTS-refTS) / float64(f.codec.ClockRate) + if diffSeconds >= 0.0 { + if diffSeconds > ResumeBehindThresholdSeconds { + f.logger.Infow("resume, reference too far behind", "expectedTS", expectedTS, "refTS", refTS, "diffSeconds", diffSeconds) + nextTS = expectedTS + } else { + nextTS = refTS + } + } else { + if math.Abs(diffSeconds) > SwitchAheadThresholdSeconds { + f.logger.Infow("resume, reference too far ahead", "expectedTS", expectedTS, "refTS", refTS, "diffSeconds", math.Abs(diffSeconds)) + nextTS = expectedTS + } else { + nextTS = refTS + } + } + } else { + // switching between layers, check if refTS is too far behind the last sent + diffSeconds := float64(refTS-lastTS) / float64(f.codec.ClockRate) + if diffSeconds < 0.0 { + if math.Abs(diffSeconds) > LayerSwitchBehindThresholdSeconds { + // AVSYNC-TODO: This could be due to pacer trickling out this layer. Should potentially return error here and wait for a more opportune time + // or some forcing function (like "have waited for too long for layer switch, nothing available, switch to whatever is available" kind of condition) + // to do the switch. Just logging it for now. + f.logger.Infow("layer switch, reference too far behind", "expectedTS", expectedTS, "refTS", refTS, "lastTS", lastTS, "diffSeconds", math.Abs(diffSeconds)) + } + // use a nominal increase to ensure that timestamp is always moving forward + nextTS = lastTS + 1 + } else { + diffSeconds = float64(expectedTS-refTS) / float64(f.codec.ClockRate) + if diffSeconds < 0.0 && math.Abs(diffSeconds) > SwitchAheadThresholdSeconds { + f.logger.Infow("layer switch, reference too far ahead", "expectedTS", expectedTS, "refTS", refTS, "diffSeconds", math.Abs(diffSeconds)) + nextTS = expectedTS + } else { + nextTS = refTS + } + } + } + + if nextTS-lastTS == 0 || nextTS-lastTS > (1<<31) { + f.logger.Infow("next timestamp is before last, adjusting", "nextTS", nextTS, "lastTS", lastTS) + // nominal increase + nextTS = lastTS + 1 + } + f.logger.Infow( + "next timestamp on switch", + "switchingAt", switchingAt.String(), + "layer", layer, + "lastTS", lastTS, + "refTS", refTS, + "refTSOffset", f.refTSOffset, + "referenceLayerSpatial", f.referenceLayerSpatial, + "expectedTS", expectedTS, + "nextTS", nextTS, + "jump", nextTS-lastTS, + ) + + f.rtpMunger.UpdateSnTsOffsets(extPkt, 1, nextTS-lastTS) + f.codecMunger.UpdateOffsets(extPkt) + return nil +} + // should be called with lock held func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer int32, tp *TranslationParams) (*TranslationParams, error) { + if tp == nil { + tp = &TranslationParams{} + } if f.lastSSRC != extPkt.Packet.SSRC { - if !f.started { - f.started = true - f.referenceLayerSpatial = layer - f.rtpMunger.SetLastSnTs(extPkt) - f.codecMunger.SetLast(extPkt) - f.logger.Infow( - "starting forwarding", - "sequenceNumber", extPkt.Packet.SequenceNumber, - "timestamp", extPkt.Packet.Timestamp, - "layer", layer, - "referenceLayerSpatial", f.referenceLayerSpatial, - ) - } else { - if f.referenceLayerSpatial == buffer.InvalidLayerSpatial { - // on a resume, reference layer may not be set, so only set when it is invalid - f.referenceLayerSpatial = layer - } - - // Compute how much time passed between the old RTP extPkt - // and the current packet, and fix timestamp on source change - // - // There are three time stamps to consider here - // 1. lastTS -> time stamp of last sent packet - // 2. refTS -> time stamp of this packet (after munging) calculated using feed's RTCP sender report - // 3. expectedTS -> time stamp of this packet (after munging) calculated using this stream's RTCP sender report - // Ideally, refTS and expectedTS should be very close and lastTS should be before both of those. - // But, cases like muting/unmuting, clock vagaries make them not satisfy those conditions always. - // - // There are 6 orderings to consider (considering only inequalities). Resolve them using following rules - // 1. Timestamp has to move forward - // 2. Keep next time stamp close to expected - lastTS := f.rtpMunger.GetLast().LastTS - refTS := lastTS - expectedTS := lastTS - minTS := ^uint64(0) - switchingAt := time.Now() - if f.getReferenceLayerRTPTimestamp != nil { - ts, err := f.getReferenceLayerRTPTimestamp(extPkt.Packet.Timestamp, layer, f.referenceLayerSpatial) - if err == nil { - refTS = ts - } - } - if f.getExpectedRTPTimestamp != nil { - ts, min, err := f.getExpectedRTPTimestamp(switchingAt) - if err == nil { - expectedTS = ts - minTS = min - } else { - rtpDiff := uint32(0) - if !f.preStartTime.IsZero() && f.refTSOffset == 0 { - timeSinceFirst := time.Since(f.preStartTime) - rtpDiff = uint32(timeSinceFirst.Nanoseconds() * int64(f.codec.ClockRate) / 1e9) - f.refTSOffset = f.firstTS + rtpDiff - refTS - f.logger.Infow( - "calculating refTSOffset", - "preStartTime", f.preStartTime.String(), - "firstTS", f.firstTS, - "timeSinceFirst", timeSinceFirst, - "rtpDiff", rtpDiff, - "refTS", refTS, - "refTSOffset", f.refTSOffset, - ) - } - expectedTS += rtpDiff - } - } - refTS += f.refTSOffset - nextTS, explain := getNextTimestamp(lastTS, refTS, expectedTS, minTS) - f.logger.Infow( - "next timestamp on switch", - "switchingAt", switchingAt.String(), - "layer", layer, - "lastTS", lastTS, - "refTS", refTS, - "refTSOffset", f.refTSOffset, - "referenceLayerSpatial", f.referenceLayerSpatial, - "expectedTS", expectedTS, - "minTS", minTS, - "nextTS", nextTS, - "jump", nextTS-lastTS, - "explanation", explain, - ) - - f.rtpMunger.UpdateSnTsOffsets(extPkt, 1, nextTS-lastTS) - f.codecMunger.UpdateOffsets(extPkt) + if err := f.processSourceSwitch(extPkt, layer); err != nil { + tp.shouldDrop = true + return tp, err } - f.logger.Debugw("switching feed", "from", f.lastSSRC, "to", extPkt.Packet.SSRC) f.lastSSRC = extPkt.Packet.SSRC } - if tp == nil { - tp = &TranslationParams{} - } tpRTP, err := f.rtpMunger.UpdateAndGetSnTs(extPkt) if err != nil { tp.shouldDrop = true @@ -1696,7 +1766,7 @@ func (f *Forwarder) maybeStart() { Packet: &rtp.Packet{ Header: rtp.Header{ SequenceNumber: uint16(rand.Intn(1<<14)) + uint16(1<<15), // a random number in third quartile of sequence number space - Timestamp: uint32(rand.Intn(1<<30)) + uint32(1<<31), // a random number in third quartile of time stamp space + Timestamp: uint32(rand.Intn(1<<30)) + uint32(1<<31), // a random number in third quartile of timestamp space }, }, } @@ -1741,16 +1811,16 @@ func (f *Forwarder) GetSnTsForBlankFrames(frameRate uint32, numPackets int) ([]S lastTS := f.rtpMunger.GetLast().LastTS expectedTS := lastTS - minTS := ^uint64(0) if f.getExpectedRTPTimestamp != nil { - ts, min, err := f.getExpectedRTPTimestamp(time.Now()) + tsExt, err := f.getExpectedRTPTimestamp(time.Now()) if err == nil { - expectedTS = ts - minTS = min + expectedTS = uint32(tsExt) } } - nextTS, _ := getNextTimestamp(lastTS, expectedTS, expectedTS, minTS) - snts, err := f.rtpMunger.UpdateAndGetPaddingSnTs(numPackets, f.codec.ClockRate, frameRate, frameEndNeeded, nextTS) + if expectedTS-lastTS == 0 || expectedTS-lastTS > (1<<31) { + expectedTS = lastTS + 1 + } + snts, err := f.rtpMunger.UpdateAndGetPaddingSnTs(numPackets, f.codec.ClockRate, frameRate, frameEndNeeded, expectedTS) return snts, frameEndNeeded, err } @@ -1888,44 +1958,3 @@ done: return float64(distance) / float64(maxSeenLayer.Temporal+1) } - -func getNextTimestamp(lastTS uint32, refTS uint32, expectedTS uint32, minTS uint64) (uint32, string) { - isInOrder := func(val1, val2 uint32) bool { - diff := val1 - val2 - return diff != 0 && diff < (1<<31) - } - - rl := isInOrder(refTS, lastTS) - el := isInOrder(expectedTS, lastTS) - er := isInOrder(expectedTS, refTS) - - nextTS := lastTS + 1 - explain := "l = r = e" - - switch { - case rl && el && er: // lastTS < refTS < expectedTS - nextTS = uint32(float64(refTS) + 0.05*float64(expectedTS-refTS)) - explain = fmt.Sprintf("l < r < e, %d, %d", refTS-lastTS, expectedTS-refTS) - case rl && el && !er: // lastTS < expectedTS < refTS - nextTS = uint32(float64(expectedTS) + 0.5*float64(refTS-expectedTS)) - explain = fmt.Sprintf("l < e < r, %d, %d", expectedTS-lastTS, refTS-expectedTS) - case !rl && el && er: // refTS < lastTS < expectedTS - nextTS = uint32(float64(lastTS) + 0.5*float64(expectedTS-lastTS)) - explain = fmt.Sprintf("r < l < e, %d, %d", lastTS-refTS, expectedTS-lastTS) - case !rl && !el && er: // refTS < expectedTS < lastTS - nextTS = lastTS + 1 - explain = fmt.Sprintf("r < e < l, %d, %d", expectedTS-refTS, lastTS-expectedTS) - case rl && !el && !er: // expectedTS < lastTS < refTS - nextTS = uint32(float64(lastTS) + 0.75*float64(refTS-lastTS)) - explain = fmt.Sprintf("e < l < r, %d, %d", lastTS-expectedTS, refTS-lastTS) - case !rl && !el && !er: // expectedTS < refTS < lastTS - nextTS = lastTS + 1 - explain = fmt.Sprintf("e < r < l, %d, %d", refTS-expectedTS, lastTS-refTS) - } - - if minTS != ^uint64(0) && !isInOrder(nextTS, uint32(minTS)) { - nextTS = uint32(minTS) + 1 - } - - return nextTS, explain -} diff --git a/pkg/sfu/receiver.go b/pkg/sfu/receiver.go index 837b534ab..410143844 100644 --- a/pkg/sfu/receiver.go +++ b/pkg/sfu/receiver.go @@ -66,7 +66,7 @@ type TrackReceiver interface { GetTemporalLayerFpsForSpatial(layer int32) []float32 - GetRTCPSenderReportData(layer int32) (*buffer.RTCPSenderReportData, *buffer.RTCPSenderReportData) + GetCalculatedClockRate(layer int32) uint32 GetReferenceLayerRTPTimestamp(ts uint32, layer int32, referenceLayer int32) (uint32, error) } @@ -752,8 +752,8 @@ func (w *WebRTCReceiver) GetTemporalLayerFpsForSpatial(layer int32) []float32 { return b.GetTemporalLayerFpsForSpatial(layer) } -func (w *WebRTCReceiver) GetRTCPSenderReportData(layer int32) (*buffer.RTCPSenderReportData, *buffer.RTCPSenderReportData) { - return w.streamTrackerManager.GetRTCPSenderReportData(layer) +func (w *WebRTCReceiver) GetCalculatedClockRate(layer int32) uint32 { + return w.streamTrackerManager.GetCalculatedClockRate(layer) } func (w *WebRTCReceiver) GetReferenceLayerRTPTimestamp(ts uint32, layer int32, referenceLayer int32) (uint32, error) { diff --git a/pkg/sfu/streamtrackermanager.go b/pkg/sfu/streamtrackermanager.go index ef92d6618..1a5d24ffe 100644 --- a/pkg/sfu/streamtrackermanager.go +++ b/pkg/sfu/streamtrackermanager.go @@ -2,6 +2,7 @@ package sfu import ( "fmt" + "math" "sort" "sync" "time" @@ -15,6 +16,14 @@ import ( "github.com/livekit/protocol/logger" ) +const ( + senderReportThresholdSeconds = float64(60.0) + + minDurationForClockRateCalculation = 15 * time.Second +) + +// --------------------------------------------------- + type StreamTrackerManagerListener interface { OnAvailableLayersChanged() OnBitrateAvailabilityChanged() @@ -24,9 +33,12 @@ type StreamTrackerManagerListener interface { OnBitrateReport(availableLayers []int32, bitrates Bitrates) } +// --------------------------------------------------- + type endsSenderReport struct { - first *buffer.RTCPSenderReportData - newest *buffer.RTCPSenderReportData + first *buffer.RTCPSenderReportData + newest *buffer.RTCPSenderReportData + lastUpdated time.Time } type StreamTrackerManager struct { @@ -50,6 +62,7 @@ type StreamTrackerManager struct { senderReportMu sync.RWMutex senderReports [buffer.DefaultMaxLayerSpatial + 1]endsSenderReport + layerOffsets [buffer.DefaultMaxLayerSpatial + 1][buffer.DefaultMaxLayerSpatial + 1]uint32 closed core.Fuse @@ -517,6 +530,40 @@ func (s *StreamTrackerManager) maxExpectedLayerFromTrackInfo() { } } +func (s *StreamTrackerManager) updateLayerOffsetLocked(ref, other int32) { + srRef := s.senderReports[ref].newest + srOther := s.senderReports[other].newest + if srRef == nil || srRef.NTPTimestamp == 0 || srOther == nil || srOther.NTPTimestamp == 0 { + return + } + + ntpDiff := srRef.NTPTimestamp.Time().Sub(srOther.NTPTimestamp.Time()) + if math.Abs(ntpDiff.Seconds()) > senderReportThresholdSeconds { + // offset is updated only if the layers' sender reports are close enough. + // + // Rationale: higher layers could be paused for extended periods of time + // due to adaptive stream/dynacast or publisher constraints like CPU/bandwidth. + // The check is to avoid using very old reports. + return + } + rtpDiff := ntpDiff.Nanoseconds() * int64(s.clockRate) / 1e9 + + // calculate other layer's time stamp at the same time as ref layer's NTP time + normalizedOtherTS := srOther.RTPTimestamp + uint32(rtpDiff) + + // now both layers' time stamp refer to the same NTP time and the diff is the offset between the layers + offset := srRef.RTPTimestamp - normalizedOtherTS + + // use minimal offset to indicate value availability in the extremely unlikely case of + // both layers using the same timestamp + if offset == 0 { + s.logger.Infow("using default offset", "ref", ref, "other", other) + offset = 1 + } + + s.layerOffsets[ref][other] = offset +} + func (s *StreamTrackerManager) SetRTCPSenderReportData(layer int32, srFirst *buffer.RTCPSenderReportData, srNewest *buffer.RTCPSenderReportData) { s.senderReportMu.Lock() defer s.senderReportMu.Unlock() @@ -527,74 +574,66 @@ func (s *StreamTrackerManager) SetRTCPSenderReportData(layer int32, srFirst *buf s.senderReports[layer].first = srFirst s.senderReports[layer].newest = srNewest + s.senderReports[layer].lastUpdated = time.Now() + + // (re)fill offsets as necessary for received layer. + for i := int32(0); i < buffer.DefaultMaxLayerSpatial+1; i++ { + if i == layer { + continue + } + + // treating layer for which report was received as reference layer + s.updateLayerOffsetLocked(layer, i) + + // and the other way + s.updateLayerOffsetLocked(i, layer) + } } -func (s *StreamTrackerManager) GetRTCPSenderReportData(layer int32) (*buffer.RTCPSenderReportData, *buffer.RTCPSenderReportData) { +func (s *StreamTrackerManager) GetCalculatedClockRate(layer int32) uint32 { s.senderReportMu.RLock() defer s.senderReportMu.RUnlock() if layer < 0 || int(layer) >= len(s.senderReports) { - return nil, nil + // invalid layer + return 0 } - return s.senderReports[layer].first, s.senderReports[layer].newest + srFirst := s.senderReports[layer].first + srNewest := s.senderReports[layer].newest + if srFirst == nil || srFirst.NTPTimestamp == 0 || srNewest == nil || srNewest.NTPTimestamp == 0 || srFirst.RTPTimestamp == srNewest.RTPTimestamp { + // sender reports invalid or same + return 0 + } + + if s.senderReports[layer].lastUpdated.IsZero() || time.Since(s.senderReports[layer].lastUpdated).Seconds() > senderReportThresholdSeconds { + // sender report updated too far back + return 0 + } + + tsf := srNewest.NTPTimestamp.Time().Sub(srFirst.NTPTimestamp.Time()) + if tsf < minDurationForClockRateCalculation { + // not enough time has elapsed to get a stable clock rate calculation + return 0 + } + + rdsf := srNewest.RTPTimestampExt - srFirst.RTPTimestampExt + return uint32(float64(rdsf) / tsf.Seconds()) } func (s *StreamTrackerManager) GetReferenceLayerRTPTimestamp(ts uint32, layer int32, referenceLayer int32) (uint32, error) { s.senderReportMu.RLock() defer s.senderReportMu.RUnlock() - if layer < 0 || referenceLayer < 0 { + if layer < 0 || int(layer) >= len(s.layerOffsets[0]) || referenceLayer < 0 || int(referenceLayer) >= len(s.layerOffsets) { return 0, fmt.Errorf("invalid layer, target: %d, reference: %d", layer, referenceLayer) } - if layer == referenceLayer { - return ts, nil + if layer != referenceLayer && s.layerOffsets[referenceLayer][layer] == 0 { + return 0, fmt.Errorf("offset unavailable, target: %d, reference: %d", layer, referenceLayer) } - var srLayer *buffer.RTCPSenderReportData - if int(layer) < len(s.senderReports) { - srLayer = s.senderReports[layer].newest - } - if srLayer == nil || srLayer.NTPTimestamp == 0 { - return 0, fmt.Errorf("layer rtcp sender report not available: %d", layer) - } - - var srRef *buffer.RTCPSenderReportData - if int(referenceLayer) < len(s.senderReports) { - srRef = s.senderReports[referenceLayer].newest - } - if srRef == nil || srRef.NTPTimestamp == 0 { - return 0, fmt.Errorf("reference layer rtcp sender report not available: %d", referenceLayer) - } - - // line up the RTP time stamps using NTP time of most recent sender report of layer and referenceLayer - // NOTE: It is possible that reference layer has stopped (due to dynacast/adaptive streaming OR publisher - // constraints). It should be okay even if the layer has stopped for a long time when using modulo arithmetic for - // RTP time stamp (uint32 arithmetic). - ntpDiff := srRef.NTPTimestamp.Time().Sub(srLayer.NTPTimestamp.Time()) - rtpDiff := ntpDiff.Nanoseconds() * int64(s.clockRate) / 1e9 - normalizedTS := srLayer.RTPTimestamp + uint32(rtpDiff) - s.logger.Infow( - "getting reference timestamp", - "layer", layer, - "referenceLayer", referenceLayer, - "incomingTS", ts, - "layerNTP", srLayer.NTPTimestamp.Time().String(), - "refNTP", srRef.NTPTimestamp.Time().String(), - "ntpDiff", ntpDiff.String(), - "layerRTP", srLayer.RTPTimestamp, - "refRTP", srRef.RTPTimestamp, - "rtpDiff", rtpDiff, - "normalizedTS", normalizedTS, - "mappedTS", ts+(srRef.RTPTimestamp-normalizedTS), - ) - - // now that both RTP timestamps correspond to roughly the same NTP time, - // the diff between them is the offset in RTP timestamp units between layer and referenceLayer. - // Add the offset to layer's ts to map it to corresponding RTP timestamp in - // the reference layer. - return ts + (srRef.RTPTimestamp - normalizedTS), nil + return ts + s.layerOffsets[referenceLayer][layer], nil } func (s *StreamTrackerManager) GetMaxTemporalLayerSeen() int32 { From 469f1cd073b2806c463a94f3831bba5ae4aa6748 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sat, 15 Jul 2023 12:43:05 +0530 Subject: [PATCH 291/324] Minor changes to publisher bool. (#1880) * Minor changes to publisher bool. * address feedback --- pkg/rtc/participant.go | 61 ++++++++++++++++++++++-------------------- pkg/rtc/room.go | 3 +-- 2 files changed, 33 insertions(+), 31 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 5de6904c1..e0d5214d4 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -370,13 +370,14 @@ func (p *ParticipantImpl) SetPermission(permission *livekit.ParticipantPermissio return false } - p.GetLogger().Infow("updating participant permission", "permission", permission) + p.params.Logger.Infow("updating participant permission", "permission", permission) video.UpdateFromPermission(permission) p.dirty.Store(true) canPublish := video.GetCanPublish() canSubscribe := video.GetCanSubscribe() + onParticipantUpdate := p.onParticipantUpdate onClaimsChanged := p.onClaimsChanged @@ -387,13 +388,7 @@ func (p *ParticipantImpl) SetPermission(permission *livekit.ParticipantPermissio // publish permission has been revoked then remove offending tracks for _, track := range p.GetPublishedTracks() { if !video.GetCanPublishSource(track.Source()) { - p.RemovePublishedTrack(track, false, false) - if p.ProtocolVersion().SupportsUnpublish() { - p.sendTrackUnpublished(track.ID()) - } else { - // for older clients that don't support unpublish, mute to avoid them sending data - p.sendTrackMuted(track.ID(), true) - } + p.removePublishedTrack(track) } } @@ -1190,22 +1185,24 @@ func (p *ParticipantImpl) updateState(state livekit.ParticipantInfo_State) { } func (p *ParticipantImpl) setIsPublisher(isPublisher bool) { - if p.isPublisher.Swap(isPublisher) != isPublisher { - p.lock.Lock() - p.requireBroadcast = true - p.lock.Unlock() + if p.isPublisher.Swap(isPublisher) == isPublisher { + return + } - p.dirty.Store(true) + p.lock.Lock() + p.requireBroadcast = true + p.lock.Unlock() - // trigger update as well if participant is already fully connected - if p.State() == livekit.ParticipantInfo_ACTIVE { - p.lock.RLock() - onParticipantUpdate := p.onParticipantUpdate - p.lock.RUnlock() + p.dirty.Store(true) - if onParticipantUpdate != nil { - onParticipantUpdate(p) - } + // trigger update as well if participant is already fully connected + if p.State() == livekit.ParticipantInfo_ACTIVE { + p.lock.RLock() + onParticipantUpdate := p.onParticipantUpdate + p.lock.RUnlock() + + if onParticipantUpdate != nil { + onParticipantUpdate(p) } } } @@ -1220,6 +1217,16 @@ func (p *ParticipantImpl) onSubscriberOffer(offer webrtc.SessionDescription) err }) } +func (p *ParticipantImpl) removePublishedTrack(track types.MediaTrack) { + p.RemovePublishedTrack(track, false, false) + if p.ProtocolVersion().SupportsUnpublish() { + p.sendTrackUnpublished(track.ID()) + } else { + // for older clients that don't support unpublish, mute to avoid them sending data + p.sendTrackMuted(track.ID(), true) + } +} + // when a new remoteTrack is created, creates a Track and adds it to room func (p *ParticipantImpl) onMediaTrack(track *webrtc.TrackRemote, rtpReceiver *webrtc.RTPReceiver) { if p.IsDisconnected() { @@ -1242,12 +1249,12 @@ func (p *ParticipantImpl) onMediaTrack(track *webrtc.TrackRemote, rtpReceiver *w p.params.Logger.Warnw("no permission to publish mediaTrack", nil, "source", publishedTrack.Source(), ) + p.removePublishedTrack(publishedTrack) return } - if !p.IsPublisher() { - p.setIsPublisher(true) - } + p.setIsPublisher(true) + p.dirty.Store(true) p.params.Logger.Infow("mediaTrack published", "kind", track.Kind().String(), @@ -1258,8 +1265,6 @@ func (p *ParticipantImpl) onMediaTrack(track *webrtc.TrackRemote, rtpReceiver *w "mime", track.Codec().MimeType, ) - p.dirty.Store(true) - if !isNewTrack && !publishedTrack.HasPendingCodec() && p.IsReady() { p.lock.RLock() onTrackUpdated := p.onTrackUpdated @@ -1300,9 +1305,7 @@ func (p *ParticipantImpl) onDataMessage(kind livekit.DataPacket_Kind, data []byt p.params.Logger.Warnw("received unsupported data packet", nil, "payload", payload) } - if !p.IsPublisher() { - p.setIsPublisher(true) - } + p.setIsPublisher(true) } func (p *ParticipantImpl) onICECandidate(c *webrtc.ICECandidate, target livekit.SignalTarget) error { diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index a90b90676..7602659ac 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -828,7 +828,6 @@ func (r *Room) createJoinResponseLocked(participant types.LocalParticipant, iceS func (r *Room) onTrackPublished(participant types.LocalParticipant, track types.MediaTrack) { // publish participant update, since track state is changed r.broadcastParticipantState(participant, broadcastOptions{skipSource: true}) - r.protoProxy.MarkDirty(false) r.lock.RLock() // subscribe all existing participants to this MediaTrack @@ -888,7 +887,6 @@ func (r *Room) onTrackUpdated(p types.LocalParticipant, _ types.MediaTrack) { func (r *Room) onTrackUnpublished(p types.LocalParticipant, track types.MediaTrack) { r.trackManager.RemoveTrack(track) - r.protoProxy.MarkDirty(false) if !p.IsClosed() { r.broadcastParticipantState(p, broadcastOptions{skipSource: true}) } @@ -898,6 +896,7 @@ func (r *Room) onTrackUnpublished(p types.LocalParticipant, track types.MediaTra } func (r *Room) onParticipantUpdate(p types.LocalParticipant) { + r.protoProxy.MarkDirty(false) // immediately notify when permissions or metadata changed r.broadcastParticipantState(p, broadcastOptions{immediate: true}) if r.onParticipantChanged != nil { From 06d8459234b04a20abe8475ab335a35ac21c0451 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sat, 15 Jul 2023 14:09:22 +0530 Subject: [PATCH 292/324] Pick up proto proxy no update on no change (#1881) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 894e44d78..fad272d8d 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230612070454-d5299b956135 - github.com/livekit/protocol v1.5.10-0.20230714010226-3c53edc91962 + github.com/livekit/protocol v1.5.10-0.20230715082801-9d0d6c9e9427 github.com/livekit/psrpc v0.3.2 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 diff --git a/go.sum b/go.sum index dba27c2ea..1c95ae879 100644 --- a/go.sum +++ b/go.sum @@ -124,8 +124,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20230612070454-d5299b956135 h1:lWYbsondvqG69czxoACDwaJ/BoyD57BahCo70ZH+m4U= github.com/livekit/mediatransportutil v0.0.0-20230612070454-d5299b956135/go.mod h1:MRc0zSOSzXuFt0X218SgabzlaKevkvCckPgBEoHYc34= -github.com/livekit/protocol v1.5.10-0.20230714010226-3c53edc91962 h1:y+rtYNMGmvpEgQlNG/wOUO16S497ygh83wQSUBKHpG4= -github.com/livekit/protocol v1.5.10-0.20230714010226-3c53edc91962/go.mod h1:eRzojAYSPJuNgDHMlvLji/CPauj9hrgvb6rVPUj6MoU= +github.com/livekit/protocol v1.5.10-0.20230715082801-9d0d6c9e9427 h1:NyNdCT8+glCnGGdkgbEO2rpX1iRscwp4HCf1u21Clzo= +github.com/livekit/protocol v1.5.10-0.20230715082801-9d0d6c9e9427/go.mod h1:eRzojAYSPJuNgDHMlvLji/CPauj9hrgvb6rVPUj6MoU= github.com/livekit/psrpc v0.3.2 h1:eAaJhASme33gtoBhCRLH9jsnWcdm1tHWf0WzaDk56ew= github.com/livekit/psrpc v0.3.2/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= From 11e1eb00fa5d7fedc89ff9b2700738b259707e51 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sun, 16 Jul 2023 23:28:20 +0530 Subject: [PATCH 293/324] Attempt to avoid out-of-order max subscribed layer notifications. (#1882) * Check for request layer lock only in the goroutine * check before sending PLI * max layer notifier worker * test cleanup * clean up * do notification in the callback --- pkg/sfu/downtrack.go | 192 +++++++++++------- pkg/sfu/forwarder.go | 69 ++++--- pkg/sfu/forwarder_test.go | 34 ++-- pkg/sfu/streamallocator/streamallocator.go | 4 + .../dependencydescriptor.go | 20 -- pkg/sfu/videolayerselector/simulcast.go | 13 -- .../videolayerselector/videolayerselector.go | 3 - pkg/sfu/videolayerselector/vp9.go | 20 -- 8 files changed, 183 insertions(+), 172 deletions(-) diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index c6dbb34fd..e3dd65a04 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -42,6 +42,8 @@ type TrackSender interface { HandleRTCPSenderReportData(payloadType webrtc.PayloadType, layer int32, srData *buffer.RTCPSenderReportData) error } +// ------------------------------------------------------------------- + const ( RTPPaddingMaxPayloadSize = 255 RTPPaddingEstimatedHeaderSize = 20 @@ -60,6 +62,8 @@ const ( maxPaddingOnMuteDuration = 5 * time.Second ) +// ------------------------------------------------------------------- + var ( ErrUnknownKind = errors.New("unknown kind of codec") ErrOutOfOrderSequenceNumberCacheMiss = errors.New("out-of-order sequence number not found in cache") @@ -197,14 +201,13 @@ type DownTrack struct { transceiver *webrtc.RTPTransceiver writeStream webrtc.TrackLocalWriter rtcpReader *buffer.RTCPReader - onCloseHandler func(willBeResumed bool) - onBinding func(error) listenerLock sync.RWMutex receiverReportListeners []ReceiverReportListener - bindLock sync.Mutex - bound atomic.Bool + bindLock sync.Mutex + bound atomic.Bool + onBinding func(error) isClosed atomic.Bool connected atomic.Bool @@ -235,14 +238,13 @@ type DownTrack struct { pacer pacer.Pacer - // update stats - onStatsUpdate func(dt *DownTrack, stat *livekit.AnalyticsStat) + maxLayerNotifierCh chan struct{} - // when max subscribed layer changes + cbMu sync.RWMutex + onStatsUpdate func(dt *DownTrack, stat *livekit.AnalyticsStat) onMaxSubscribedLayerChanged func(dt *DownTrack, layer int32) - - // update rtt - onRttUpdate func(dt *DownTrack, rtt uint32) + onRttUpdate func(dt *DownTrack, rtt uint32) + onCloseHandler func(willBeResumed bool) } // NewDownTrack returns a DownTrack. @@ -266,17 +268,18 @@ func NewDownTrack( } d := &DownTrack{ - logger: logger, - id: r.TrackID(), - subscriberID: subID, - maxTrack: mt, - streamID: r.StreamID(), - bufferFactory: bf, - receiver: r, - upstreamCodecs: codecs, - kind: kind, - codec: codecs[0].RTPCodecCapability, - pacer: pacer, + logger: logger, + id: r.TrackID(), + subscriberID: subID, + maxTrack: mt, + streamID: r.StreamID(), + bufferFactory: bf, + receiver: r, + upstreamCodecs: codecs, + kind: kind, + codec: codecs[0].RTPCodecCapability, + pacer: pacer, + maxLayerNotifierCh: make(chan struct{}, 20), } d.forwarder = NewForwarder( d.kind, @@ -307,11 +310,15 @@ func NewDownTrack( Logger: d.logger.WithValues("direction", "down"), }) d.connectionStats.OnStatsUpdate(func(_cs *connectionquality.ConnectionStats, stat *livekit.AnalyticsStat) { - if d.onStatsUpdate != nil { - d.onStatsUpdate(d, stat) + if onStatsUpdate := d.getOnStatsUpdate(); onStatsUpdate != nil { + onStatsUpdate(d, stat) } }) + if d.kind == webrtc.RTPCodecTypeVideo { + go d.maxLayerNotifierWorker() + } + return d, nil } @@ -541,6 +548,7 @@ func (d *DownTrack) keyFrameRequester(generation uint32, layer int32) { if d.IsClosed() || layer == buffer.InvalidLayerSpatial { return } + interval := 2 * d.rtpStats.GetRtt() if interval < keyFrameIntervalMin { interval = keyFrameIntervalMin @@ -550,7 +558,13 @@ func (d *DownTrack) keyFrameRequester(generation uint32, layer int32) { } ticker := time.NewTicker(time.Duration(interval) * time.Millisecond) defer ticker.Stop() + for { + locked, _ := d.forwarder.CheckSync() + if locked { + return + } + if d.connected.Load() { d.logger.Debugw("sending PLI for layer lock", "generation", generation, "layer", layer) d.receiver.SendPLI(layer, false) @@ -565,6 +579,34 @@ func (d *DownTrack) keyFrameRequester(generation uint32, layer int32) { } } +func (d *DownTrack) postMaxLayerNotifierEvent() { + if d.IsClosed() { + return + } + + select { + case d.maxLayerNotifierCh <- struct{}{}: + default: + d.logger.Warnw("max layer notifier event queue full", nil) + } +} + +func (d *DownTrack) maxLayerNotifierWorker() { + more := true + for more { + _, more = <-d.maxLayerNotifierCh + + maxLayerSpatial := buffer.InvalidLayerSpatial + if more { + maxLayerSpatial = d.forwarder.GetMaxSubscribedSpatial() + } + if onMaxSubscribedLayerChanged := d.getOnMaxLayerChanged(); onMaxSubscribedLayerChanged != nil { + d.logger.Infow("max subscribed layer changed", maxLayerSpatial) + onMaxSubscribedLayerChanged(d, maxLayerSpatial) + } + } +} + // WriteRTP writes an RTP Packet to the DownTrack func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { if !d.bound.Load() || !d.connected.Load() { @@ -725,17 +767,17 @@ func (d *DownTrack) WritePaddingRTP(bytesToSend int, paddingOnMute bool, forceMa // Mute enables or disables media forwarding - subscriber triggered func (d *DownTrack) Mute(muted bool) { - changed, maxLayer := d.forwarder.Mute(muted) - d.handleMute(muted, false, changed, maxLayer) + changed := d.forwarder.Mute(muted) + d.handleMute(muted, changed) } // PubMute enables or disables media forwarding - publisher side func (d *DownTrack) PubMute(pubMuted bool) { - changed, maxLayer := d.forwarder.PubMute(pubMuted) - d.handleMute(pubMuted, true, changed, maxLayer) + changed := d.forwarder.PubMute(pubMuted) + d.handleMute(pubMuted, changed) } -func (d *DownTrack) handleMute(muted bool, isPub bool, changed bool, maxLayer buffer.VideoLayer) { +func (d *DownTrack) handleMute(muted bool, changed bool) { if !changed { return } @@ -762,18 +804,7 @@ func (d *DownTrack) handleMute(muted bool, isPub bool, changed bool, maxLayer bu // Note that while publisher mute is active, subscriber changes can also happen // and that could turn on/off layers on publisher side. // - if !isPub && d.onMaxSubscribedLayerChanged != nil && d.kind == webrtc.RTPCodecTypeVideo { - notifyLayer := buffer.InvalidLayerSpatial - if !muted { - // - // When unmuting, don't wait for layer lock as - // client might need to be notified to start layers - // before locking can happen in the forwarder. - // - notifyLayer = maxLayer.Spatial - } - d.onMaxSubscribedLayerChanged(d, notifyLayer) - } + d.postMaxLayerNotifierEvent() if sal := d.getStreamAllocatorListener(); sal != nil { sal.OnSubscriptionChanged(d) @@ -856,12 +887,10 @@ func (d *DownTrack) CloseWithFlush(flush bool) { d.rtpStats.Stop() d.logger.Infow("rtp stats", "direction", "downstream", "mime", d.mime, "ssrc", d.ssrc, "stats", d.rtpStats.ToString()) - if d.onMaxSubscribedLayerChanged != nil && d.kind == webrtc.RTPCodecTypeVideo { - d.onMaxSubscribedLayerChanged(d, buffer.InvalidLayerSpatial) - } + close(d.maxLayerNotifierCh) - if d.onCloseHandler != nil { - d.onCloseHandler(!flush) + if onCloseHandler := d.getOnCloseHandler(); onCloseHandler != nil { + onCloseHandler(!flush) } d.stopKeyFrameRequester() @@ -869,21 +898,12 @@ func (d *DownTrack) CloseWithFlush(flush bool) { } func (d *DownTrack) SetMaxSpatialLayer(spatialLayer int32) { - changed, maxLayer, currentLayer := d.forwarder.SetMaxSpatialLayer(spatialLayer) + changed, maxLayer := d.forwarder.SetMaxSpatialLayer(spatialLayer) if !changed { return } - if d.onMaxSubscribedLayerChanged != nil && d.kind == webrtc.RTPCodecTypeVideo && maxLayer.SpatialGreaterThanOrEqual(currentLayer) { - // - // Notify when new max is - // 1. Equal to current -> already locked to the new max - // 2. Greater than current -> two scenarios - // a. is higher than previous max -> client may need to start higher layer before forwarder can lock - // b. is lower than previous max -> client can stop higher layer(s) - // - d.onMaxSubscribedLayerChanged(d, maxLayer.Spatial) - } + d.postMaxLayerNotifierEvent() if sal := d.getStreamAllocatorListener(); sal != nil { sal.OnSubscribedLayerChanged(d, maxLayer) @@ -891,7 +911,7 @@ func (d *DownTrack) SetMaxSpatialLayer(spatialLayer int32) { } func (d *DownTrack) SetMaxTemporalLayer(temporalLayer int32) { - changed, maxLayer, _ := d.forwarder.SetMaxTemporalLayer(temporalLayer) + changed, maxLayer := d.forwarder.SetMaxTemporalLayer(temporalLayer) if !changed { return } @@ -973,10 +993,23 @@ func (d *DownTrack) UpTrackBitrateReport(availableLayers []int32, bitrates Bitra // OnCloseHandler method to be called on remote tracked removed func (d *DownTrack) OnCloseHandler(fn func(willBeResumed bool)) { + d.cbMu.Lock() + defer d.cbMu.Unlock() + d.onCloseHandler = fn } +func (d *DownTrack) getOnCloseHandler() func(willBeResumed bool) { + d.cbMu.RLock() + defer d.cbMu.RUnlock() + + return d.onCloseHandler +} + func (d *DownTrack) OnBinding(fn func(error)) { + d.bindLock.Lock() + defer d.bindLock.Unlock() + d.onBinding = fn } @@ -988,17 +1021,47 @@ func (d *DownTrack) AddReceiverReportListener(listener ReceiverReportListener) { } func (d *DownTrack) OnStatsUpdate(fn func(dt *DownTrack, stat *livekit.AnalyticsStat)) { + d.cbMu.Lock() + defer d.cbMu.Unlock() + d.onStatsUpdate = fn } +func (d *DownTrack) getOnStatsUpdate() func(dt *DownTrack, stat *livekit.AnalyticsStat) { + d.cbMu.RLock() + defer d.cbMu.RUnlock() + + return d.onStatsUpdate +} + func (d *DownTrack) OnRttUpdate(fn func(dt *DownTrack, rtt uint32)) { + d.cbMu.Lock() + defer d.cbMu.Unlock() + d.onRttUpdate = fn } +func (d *DownTrack) getOnRttUpdate() func(dt *DownTrack, rtt uint32) { + d.cbMu.RLock() + defer d.cbMu.RUnlock() + + return d.onRttUpdate +} + func (d *DownTrack) OnMaxLayerChanged(fn func(dt *DownTrack, layer int32)) { + d.cbMu.Lock() + defer d.cbMu.Unlock() + d.onMaxSubscribedLayerChanged = fn } +func (d *DownTrack) getOnMaxLayerChanged() func(dt *DownTrack, layer int32) { + d.cbMu.RLock() + defer d.cbMu.RUnlock() + + return d.onMaxSubscribedLayerChanged +} + func (d *DownTrack) IsDeficient() bool { return d.forwarder.IsDeficient() } @@ -1355,8 +1418,8 @@ func (d *DownTrack) handleRTCP(bytes []byte) { d.sequencer.setRTT(rttToReport) } - if d.onRttUpdate != nil { - d.onRttUpdate(d, rttToReport) + if onRttUpdate := d.getOnRttUpdate(); onRttUpdate != nil { + onRttUpdate(d, rttToReport) } } } @@ -1744,15 +1807,8 @@ func (d *DownTrack) packetSent(md interface{}, hdr *rtp.Header, payloadSize int, } if spmd.tp != nil { - if spmd.tp.isSwitchingToMaxSpatial && d.onMaxSubscribedLayerChanged != nil && d.kind == webrtc.RTPCodecTypeVideo { - d.onMaxSubscribedLayerChanged(d, spmd.tp.maxSpatialLayer) - } - - if spmd.tp.isSwitchingToRequestSpatial { - locked, _ := d.forwarder.CheckSync() - if locked { - d.stopKeyFrameRequester() - } + if spmd.tp.isSwitching { + d.postMaxLayerNotifierEvent() } if spmd.tp.isResuming { diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 1da413ee8..23a8d694d 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -129,15 +129,13 @@ func (v VideoTransition) String() string { // ------------------------------------------------------------------- type TranslationParams struct { - shouldDrop bool - isResuming bool - isSwitchingToRequestSpatial bool - isSwitchingToMaxSpatial bool - maxSpatialLayer int32 - rtp *TranslationParamsRTP - codecBytes []byte - ddBytes []byte - marker bool + shouldDrop bool + isResuming bool + isSwitching bool + rtp *TranslationParamsRTP + codecBytes []byte + ddBytes []byte + marker bool } // ------------------------------------------------------------------- @@ -360,12 +358,12 @@ func (f *Forwarder) SeedState(state ForwarderState) { f.refTSOffset = state.RefTSOffset } -func (f *Forwarder) Mute(muted bool) (bool, buffer.VideoLayer) { +func (f *Forwarder) Mute(muted bool) bool { f.lock.Lock() defer f.lock.Unlock() if f.muted == muted { - return false, f.vls.GetMax() + return false } // Do not mute when paused due to bandwidth limitation. @@ -384,7 +382,7 @@ func (f *Forwarder) Mute(muted bool) (bool, buffer.VideoLayer) { // the case of intentional mute. if muted && f.isDeficientLocked() && f.lastAllocation.PauseReason == VideoPauseReasonBandwidth { f.logger.Infow("ignoring forwarder mute, paused due to congestion") - return false, f.vls.GetMax() + return false } f.logger.Debugw("setting forwarder mute", "muted", muted) @@ -395,7 +393,7 @@ func (f *Forwarder) Mute(muted bool) (bool, buffer.VideoLayer) { f.resyncLocked() } - return true, f.vls.GetMax() + return true } func (f *Forwarder) IsMuted() bool { @@ -405,12 +403,12 @@ func (f *Forwarder) IsMuted() bool { return f.muted } -func (f *Forwarder) PubMute(pubMuted bool) (bool, buffer.VideoLayer) { +func (f *Forwarder) PubMute(pubMuted bool) bool { f.lock.Lock() defer f.lock.Unlock() if f.pubMuted == pubMuted { - return false, f.vls.GetMax() + return false } f.logger.Debugw("setting forwarder pub mute", "pubMuted", pubMuted) @@ -432,7 +430,7 @@ func (f *Forwarder) PubMute(pubMuted bool) (bool, buffer.VideoLayer) { } } - return true, f.vls.GetMax() + return true } func (f *Forwarder) IsPubMuted() bool { @@ -449,17 +447,17 @@ func (f *Forwarder) IsAnyMuted() bool { return f.muted || f.pubMuted } -func (f *Forwarder) SetMaxSpatialLayer(spatialLayer int32) (bool, buffer.VideoLayer, buffer.VideoLayer) { +func (f *Forwarder) SetMaxSpatialLayer(spatialLayer int32) (bool, buffer.VideoLayer) { f.lock.Lock() defer f.lock.Unlock() if f.kind == webrtc.RTPCodecTypeAudio { - return false, buffer.InvalidLayer, buffer.InvalidLayer + return false, buffer.InvalidLayer } existingMax := f.vls.GetMax() if spatialLayer == existingMax.Spatial { - return false, existingMax, f.vls.GetCurrent() + return false, existingMax } f.logger.Debugw("setting max spatial layer", "layer", spatialLayer) @@ -467,20 +465,20 @@ func (f *Forwarder) SetMaxSpatialLayer(spatialLayer int32) (bool, buffer.VideoLa f.clearParkedLayer() - return true, f.vls.GetMax(), f.vls.GetCurrent() + return true, f.vls.GetMax() } -func (f *Forwarder) SetMaxTemporalLayer(temporalLayer int32) (bool, buffer.VideoLayer, buffer.VideoLayer) { +func (f *Forwarder) SetMaxTemporalLayer(temporalLayer int32) (bool, buffer.VideoLayer) { f.lock.Lock() defer f.lock.Unlock() if f.kind == webrtc.RTPCodecTypeAudio { - return false, buffer.InvalidLayer, buffer.InvalidLayer + return false, buffer.InvalidLayer } existingMax := f.vls.GetMax() if temporalLayer == existingMax.Temporal { - return false, existingMax, f.vls.GetCurrent() + return false, existingMax } f.logger.Debugw("setting max temporal layer", "layer", temporalLayer) @@ -488,7 +486,7 @@ func (f *Forwarder) SetMaxTemporalLayer(temporalLayer int32) (bool, buffer.Video f.clearParkedLayer() - return true, f.vls.GetMax(), f.vls.GetCurrent() + return true, f.vls.GetMax() } func (f *Forwarder) MaxLayer() buffer.VideoLayer { @@ -512,6 +510,25 @@ func (f *Forwarder) TargetLayer() buffer.VideoLayer { return f.vls.GetTarget() } +func (f *Forwarder) GetMaxSubscribedSpatial() int32 { + f.lock.RLock() + defer f.lock.RUnlock() + + layer := buffer.InvalidLayerSpatial // covers muted case + if !f.muted { + layer = f.vls.GetMax().Spatial + + // If current is higher, mark the current layer as max subscribed layer + // to prevent the current layer from stopping before forwarder switches + // to the new and lower max layer, + if layer < f.vls.GetCurrent().Spatial { + layer = f.vls.GetCurrent().Spatial + } + } + + return layer +} + func (f *Forwarder) isDeficientLocked() bool { return f.lastAllocation.IsDeficient } @@ -1690,9 +1707,7 @@ func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer in return tp, nil } tp.isResuming = result.IsResuming - tp.isSwitchingToRequestSpatial = result.IsSwitchingToRequestSpatial - tp.isSwitchingToMaxSpatial = result.IsSwitchingToMaxSpatial - tp.maxSpatialLayer = result.MaxSpatialLayer + tp.isSwitching = result.IsSwitching tp.ddBytes = result.DependencyDescriptorExtension tp.marker = result.RTPMarker diff --git a/pkg/sfu/forwarder_test.go b/pkg/sfu/forwarder_test.go index a0e340800..2da1c524b 100644 --- a/pkg/sfu/forwarder_test.go +++ b/pkg/sfu/forwarder_test.go @@ -26,13 +26,13 @@ func newForwarder(codec webrtc.RTPCodecCapability, kind webrtc.RTPCodecType) *Fo func TestForwarderMute(t *testing.T) { f := newForwarder(testutils.TestOpusCodec, webrtc.RTPCodecTypeAudio) require.False(t, f.IsMuted()) - muted, _ := f.Mute(false) + muted := f.Mute(false) require.False(t, muted) // no change in mute state require.False(t, f.IsMuted()) - muted, _ = f.Mute(true) + muted = f.Mute(true) require.True(t, muted) require.True(t, f.IsMuted()) - muted, _ = f.Mute(false) + muted = f.Mute(false) require.True(t, muted) require.False(t, f.IsMuted()) } @@ -45,15 +45,13 @@ func TestForwarderLayersAudio(t *testing.T) { require.Equal(t, buffer.InvalidLayer, f.CurrentLayer()) require.Equal(t, buffer.InvalidLayer, f.TargetLayer()) - changed, maxLayer, currentLayer := f.SetMaxSpatialLayer(1) + changed, maxLayer := f.SetMaxSpatialLayer(1) require.False(t, changed) require.Equal(t, buffer.InvalidLayer, maxLayer) - require.Equal(t, buffer.InvalidLayer, currentLayer) - changed, maxLayer, currentLayer = f.SetMaxTemporalLayer(1) + changed, maxLayer = f.SetMaxTemporalLayer(1) require.False(t, changed) require.Equal(t, buffer.InvalidLayer, maxLayer) - require.Equal(t, buffer.InvalidLayer, currentLayer) require.Equal(t, buffer.InvalidLayer, f.MaxLayer()) } @@ -72,12 +70,11 @@ func TestForwarderLayersVideo(t *testing.T) { Spatial: buffer.DefaultMaxLayerSpatial, Temporal: buffer.DefaultMaxLayerTemporal, } - changed, maxLayer, currentLayer := f.SetMaxSpatialLayer(buffer.DefaultMaxLayerSpatial) + changed, maxLayer := f.SetMaxSpatialLayer(buffer.DefaultMaxLayerSpatial) require.True(t, changed) require.Equal(t, expectedLayers, maxLayer) - require.Equal(t, buffer.InvalidLayer, currentLayer) - changed, maxLayer, currentLayer = f.SetMaxSpatialLayer(buffer.DefaultMaxLayerSpatial - 1) + changed, maxLayer = f.SetMaxSpatialLayer(buffer.DefaultMaxLayerSpatial - 1) require.True(t, changed) expectedLayers = buffer.VideoLayer{ Spatial: buffer.DefaultMaxLayerSpatial - 1, @@ -85,21 +82,18 @@ func TestForwarderLayersVideo(t *testing.T) { } require.Equal(t, expectedLayers, maxLayer) require.Equal(t, expectedLayers, f.MaxLayer()) - require.Equal(t, buffer.InvalidLayer, currentLayer) f.vls.SetCurrent(buffer.VideoLayer{Spatial: 0, Temporal: 1}) - changed, maxLayer, currentLayer = f.SetMaxSpatialLayer(buffer.DefaultMaxLayerSpatial - 1) + changed, maxLayer = f.SetMaxSpatialLayer(buffer.DefaultMaxLayerSpatial - 1) require.False(t, changed) require.Equal(t, expectedLayers, maxLayer) require.Equal(t, expectedLayers, f.MaxLayer()) - require.Equal(t, buffer.VideoLayer{Spatial: 0, Temporal: 1}, currentLayer) - changed, maxLayer, currentLayer = f.SetMaxTemporalLayer(buffer.DefaultMaxLayerTemporal) + changed, maxLayer = f.SetMaxTemporalLayer(buffer.DefaultMaxLayerTemporal) require.False(t, changed) require.Equal(t, expectedLayers, maxLayer) - require.Equal(t, buffer.VideoLayer{Spatial: 0, Temporal: 1}, currentLayer) - changed, maxLayer, currentLayer = f.SetMaxTemporalLayer(buffer.DefaultMaxLayerTemporal - 1) + changed, maxLayer = f.SetMaxTemporalLayer(buffer.DefaultMaxLayerTemporal - 1) require.True(t, changed) expectedLayers = buffer.VideoLayer{ Spatial: buffer.DefaultMaxLayerSpatial - 1, @@ -107,7 +101,6 @@ func TestForwarderLayersVideo(t *testing.T) { } require.Equal(t, expectedLayers, maxLayer) require.Equal(t, expectedLayers, f.MaxLayer()) - require.Equal(t, buffer.VideoLayer{Spatial: 0, Temporal: 1}, currentLayer) } func TestForwarderAllocateOptimal(t *testing.T) { @@ -1404,8 +1397,8 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { marshalledVP8, err := expectedVP8.Marshal() require.NoError(t, err) expectedTP = TranslationParams{ - isSwitchingToMaxSpatial: true, - isResuming: true, + isSwitching: true, + isResuming: true, rtp: &TranslationParamsRTP{ snOrdering: SequenceNumberOrderingContiguous, sequenceNumber: 23333, @@ -1716,8 +1709,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { marshalledVP8, err = expectedVP8.Marshal() require.NoError(t, err) expectedTP = TranslationParams{ - isSwitchingToMaxSpatial: true, - maxSpatialLayer: 1, + isSwitching: true, rtp: &TranslationParamsRTP{ snOrdering: SequenceNumberOrderingContiguous, sequenceNumber: 23339, diff --git a/pkg/sfu/streamallocator/streamallocator.go b/pkg/sfu/streamallocator/streamallocator.go index 475b38e94..f0c17db9e 100644 --- a/pkg/sfu/streamallocator/streamallocator.go +++ b/pkg/sfu/streamallocator/streamallocator.go @@ -548,6 +548,10 @@ func (s *StreamAllocator) postEvent(event Event) { func (s *StreamAllocator) processEvents() { for event := range s.eventCh { + if s.isStopped.Load() { + break + } + s.handleEvent(&event) } diff --git a/pkg/sfu/videolayerselector/dependencydescriptor.go b/pkg/sfu/videolayerselector/dependencydescriptor.go index 7b8c95310..34f8b0b55 100644 --- a/pkg/sfu/videolayerselector/dependencydescriptor.go +++ b/pkg/sfu/videolayerselector/dependencydescriptor.go @@ -204,26 +204,6 @@ func (d *DependencyDescriptor) Select(extPkt *buffer.ExtPacket, _layer int32) (r d.previousActiveDecodeTargetsBitmask = d.activeDecodeTargetsBitmask d.activeDecodeTargetsBitmask = buffer.GetActiveDecodeTargetBitmask(d.currentLayer, ddwdt.DecodeTargets) - - if d.currentLayer.Spatial == d.requestSpatial { - result.IsSwitchingToRequestSpatial = true - } - if d.currentLayer.Spatial == d.maxLayer.Spatial { - result.IsSwitchingToMaxSpatial = true - result.MaxSpatialLayer = d.currentLayer.Spatial - d.logger.Infow( - "reached max layer", - "previous", d.previousLayer, - "current", d.currentLayer, - "previousTarget", d.previousTargetLayer, - "target", d.targetLayer, - "max", d.maxLayer, - "layer", fd.SpatialId, - "req", d.requestSpatial, - "maxSeen", d.maxSeenLayer, - "feed", extPkt.Packet.SSRC, - ) - } } ddExtension := &dede.DependencyDescriptorExtension{ diff --git a/pkg/sfu/videolayerselector/simulcast.go b/pkg/sfu/videolayerselector/simulcast.go index 4d0c9294d..06e0bad72 100644 --- a/pkg/sfu/videolayerselector/simulcast.go +++ b/pkg/sfu/videolayerselector/simulcast.go @@ -35,19 +35,6 @@ func (s *Simulcast) Select(extPkt *buffer.ExtPacket, layer int32) (result VideoL result.IsResuming = true } - if s.currentLayer.Spatial == s.requestSpatial { - result.IsSwitchingToRequestSpatial = true - } - - if s.currentLayer.Spatial >= s.maxLayer.Spatial { - result.IsSwitchingToMaxSpatial = true - result.MaxSpatialLayer = s.currentLayer.Spatial - if reason != "" { - reason += ", " - } - reason += "reached max layer" - } - if reason != "" { s.logger.Infow( reason, diff --git a/pkg/sfu/videolayerselector/videolayerselector.go b/pkg/sfu/videolayerselector/videolayerselector.go index ffbb9f42c..f17d745d1 100644 --- a/pkg/sfu/videolayerselector/videolayerselector.go +++ b/pkg/sfu/videolayerselector/videolayerselector.go @@ -10,9 +10,6 @@ type VideoLayerSelectorResult struct { IsRelevant bool IsSwitching bool IsResuming bool - IsSwitchingToRequestSpatial bool - IsSwitchingToMaxSpatial bool - MaxSpatialLayer int32 RTPMarker bool DependencyDescriptorExtension []byte } diff --git a/pkg/sfu/videolayerselector/vp9.go b/pkg/sfu/videolayerselector/vp9.go index ed1a165b6..508cdf289 100644 --- a/pkg/sfu/videolayerselector/vp9.go +++ b/pkg/sfu/videolayerselector/vp9.go @@ -80,26 +80,6 @@ func (v *VP9) Select(extPkt *buffer.ExtPacket, _layer int32) (result VideoLayerS result.IsResuming = true } - if v.currentLayer.Spatial != v.requestSpatial && updatedLayer.Spatial == v.requestSpatial { - result.IsSwitchingToRequestSpatial = true - } - - if v.currentLayer.Spatial != v.maxLayer.Spatial && updatedLayer.Spatial == v.maxLayer.Spatial { - result.IsSwitchingToMaxSpatial = true - result.MaxSpatialLayer = updatedLayer.Spatial - v.logger.Infow( - "reached max layer", - "current", v.currentLayer, - "updated", updatedLayer, - "target", v.targetLayer, - "max", v.maxLayer, - "layer", extPkt.VideoLayer.Spatial, - "req", v.requestSpatial, - "maxSeen", v.maxSeenLayer, - "feed", extPkt.Packet.SSRC, - ) - } - v.previousLayer = v.currentLayer v.currentLayer = updatedLayer } From e6a47a24a726caca1aab58b45fa4df20c39f821d Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 16 Jul 2023 13:24:06 -0700 Subject: [PATCH 294/324] Update livekit deps (#1869) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index fad272d8d..34d36beea 100644 --- a/go.mod +++ b/go.mod @@ -17,8 +17,8 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.4 github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 - github.com/livekit/mediatransportutil v0.0.0-20230612070454-d5299b956135 - github.com/livekit/protocol v1.5.10-0.20230715082801-9d0d6c9e9427 + github.com/livekit/mediatransportutil v0.0.0-20230716190407-fc4944cbc33a + github.com/livekit/protocol v1.5.10 github.com/livekit/psrpc v0.3.2 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 diff --git a/go.sum b/go.sum index 1c95ae879..417006f39 100644 --- a/go.sum +++ b/go.sum @@ -122,10 +122,10 @@ github.com/lithammer/shortuuid/v4 v4.0.0 h1:QRbbVkfgNippHOS8PXDkti4NaWeyYfcBTHtw github.com/lithammer/shortuuid/v4 v4.0.0/go.mod h1:Zs8puNcrvf2rV9rTH51ZLLcj7ZXqQI3lv67aw4KiB1Y= github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkDaKb5iXdynYrzB84ErPPO4LbRASk58= github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= -github.com/livekit/mediatransportutil v0.0.0-20230612070454-d5299b956135 h1:lWYbsondvqG69czxoACDwaJ/BoyD57BahCo70ZH+m4U= -github.com/livekit/mediatransportutil v0.0.0-20230612070454-d5299b956135/go.mod h1:MRc0zSOSzXuFt0X218SgabzlaKevkvCckPgBEoHYc34= -github.com/livekit/protocol v1.5.10-0.20230715082801-9d0d6c9e9427 h1:NyNdCT8+glCnGGdkgbEO2rpX1iRscwp4HCf1u21Clzo= -github.com/livekit/protocol v1.5.10-0.20230715082801-9d0d6c9e9427/go.mod h1:eRzojAYSPJuNgDHMlvLji/CPauj9hrgvb6rVPUj6MoU= +github.com/livekit/mediatransportutil v0.0.0-20230716190407-fc4944cbc33a h1:JWpPHcMFuw0fP4swE89CfMgeUXiSN5IKvCJL/5HLI3A= +github.com/livekit/mediatransportutil v0.0.0-20230716190407-fc4944cbc33a/go.mod h1:xirUXW8xnLGmfCwUeAv/nj1VGo1OO1BmgxrYP7jK/14= +github.com/livekit/protocol v1.5.10 h1:lnaHMa27cbRkHybi/jvOVuRSaLsho2wCLRjKiC6ce2Y= +github.com/livekit/protocol v1.5.10/go.mod h1:eRzojAYSPJuNgDHMlvLji/CPauj9hrgvb6rVPUj6MoU= github.com/livekit/psrpc v0.3.2 h1:eAaJhASme33gtoBhCRLH9jsnWcdm1tHWf0WzaDk56ew= github.com/livekit/psrpc v0.3.2/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= From 7dc60bb1bf59394dcd05d5f8ff9046aabd9c4834 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Sun, 16 Jul 2023 13:40:53 -0700 Subject: [PATCH 295/324] start reading signal messages before session handler finishes (#1883) * start reading signal messages before session handler finishes * fix err scope --- pkg/service/signal.go | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/pkg/service/signal.go b/pkg/service/signal.go index c6907dadb..aa6ab5578 100644 --- a/pkg/service/signal.go +++ b/pkg/service/signal.go @@ -139,9 +139,6 @@ func (r *signalService) RelaySignal(stream psrpc.ServerStream[*rpc.RelaySignalRe "connID", ss.ConnectionId, ) - reqChan := routing.NewDefaultMessageChannel(livekit.ConnectionID(ss.ConnectionId)) - defer reqChan.Close() - sink := routing.NewSignalMessageSink(routing.SignalSinkParams[*rpc.RelaySignalResponse, *rpc.RelaySignalRequest]{ Logger: l, Stream: stream, @@ -149,6 +146,19 @@ func (r *signalService) RelaySignal(stream psrpc.ServerStream[*rpc.RelaySignalRe Writer: signalResponseMessageWriter{}, ConnectionID: livekit.ConnectionID(ss.ConnectionId), }) + reqChan := routing.NewDefaultMessageChannel(livekit.ConnectionID(ss.ConnectionId)) + + go func() { + err := routing.CopySignalStreamToMessageChannel[*rpc.RelaySignalResponse, *rpc.RelaySignalRequest]( + stream, + reqChan, + signalRequestMessageReader{}, + r.config, + ) + l.Infow("signal stream closed", "error", err) + + reqChan.Close() + }() err = r.sessionHandler(ctx, livekit.RoomName(ss.RoomName), *pi, livekit.ConnectionID(ss.ConnectionId), reqChan, sink) if err != nil { @@ -156,9 +166,7 @@ func (r *signalService) RelaySignal(stream psrpc.ServerStream[*rpc.RelaySignalRe return } - err = routing.CopySignalStreamToMessageChannel[*rpc.RelaySignalResponse, *rpc.RelaySignalRequest](stream, reqChan, signalRequestMessageReader{}, r.config) - l.Infow("signal stream closed", "error", err) - + stream.Hijack() return } From 8784449fc6c476f45889189e85478e15139a998e Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Sun, 16 Jul 2023 15:03:47 -0700 Subject: [PATCH 296/324] manually cancel signal relay context (#1884) --- pkg/service/signal.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/pkg/service/signal.go b/pkg/service/signal.go index aa6ab5578..2c28465d8 100644 --- a/pkg/service/signal.go +++ b/pkg/service/signal.go @@ -112,12 +112,6 @@ type signalService struct { } func (r *signalService) RelaySignal(stream psrpc.ServerStream[*rpc.RelaySignalResponse, *rpc.RelaySignalRequest]) (err error) { - // copy the context to prevent a race between the session handler closing - // and the delivery of any parting messages from the client. take care to - // copy the incoming rpc headers to avoid dropping any session vars. - ctx, cancel := context.WithCancel(metadata.NewContextWithIncomingHeader(context.Background(), metadata.IncomingHeader(stream.Context()))) - defer cancel() - req, ok := <-stream.Channel() if !ok { return nil @@ -139,6 +133,11 @@ func (r *signalService) RelaySignal(stream psrpc.ServerStream[*rpc.RelaySignalRe "connID", ss.ConnectionId, ) + // copy the context to prevent a race between the session handler closing + // and the delivery of any parting messages from the client. take care to + // copy the incoming rpc headers to avoid dropping any session vars. + ctx, cancel := context.WithCancel(metadata.NewContextWithIncomingHeader(context.Background(), metadata.IncomingHeader(stream.Context()))) + sink := routing.NewSignalMessageSink(routing.SignalSinkParams[*rpc.RelaySignalResponse, *rpc.RelaySignalRequest]{ Logger: l, Stream: stream, @@ -158,11 +157,13 @@ func (r *signalService) RelaySignal(stream psrpc.ServerStream[*rpc.RelaySignalRe l.Infow("signal stream closed", "error", err) reqChan.Close() + cancel() }() err = r.sessionHandler(ctx, livekit.RoomName(ss.RoomName), *pi, livekit.ConnectionID(ss.ConnectionId), reqChan, sink) if err != nil { l.Errorw("could not handle new participant", err) + cancel() return } From 5535916ff27d985430cf95acae1f4007c3676d1e Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Sun, 16 Jul 2023 19:01:53 -0700 Subject: [PATCH 297/324] prevent signal context from closing before room setup finishes (#1885) --- pkg/service/signal.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/service/signal.go b/pkg/service/signal.go index 2c28465d8..299df055a 100644 --- a/pkg/service/signal.go +++ b/pkg/service/signal.go @@ -138,6 +138,12 @@ func (r *signalService) RelaySignal(stream psrpc.ServerStream[*rpc.RelaySignalRe // copy the incoming rpc headers to avoid dropping any session vars. ctx, cancel := context.WithCancel(metadata.NewContextWithIncomingHeader(context.Background(), metadata.IncomingHeader(stream.Context()))) + // wait until room setup finishes to cancel the session context even if the + // stream has closed. this is required to complete setup i/o after room api + // has discarded the temp client. + sessionHandlerDone := make(chan struct{}) + defer close(sessionHandlerDone) + sink := routing.NewSignalMessageSink(routing.SignalSinkParams[*rpc.RelaySignalResponse, *rpc.RelaySignalRequest]{ Logger: l, Stream: stream, @@ -157,13 +163,14 @@ func (r *signalService) RelaySignal(stream psrpc.ServerStream[*rpc.RelaySignalRe l.Infow("signal stream closed", "error", err) reqChan.Close() + + <-sessionHandlerDone cancel() }() err = r.sessionHandler(ctx, livekit.RoomName(ss.RoomName), *pi, livekit.ConnectionID(ss.ConnectionId), reqChan, sink) if err != nil { l.Errorw("could not handle new participant", err) - cancel() return } From 5d1d454a9861226dace41f2ee8b7ea86e361b9f2 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Sun, 16 Jul 2023 20:05:41 -0700 Subject: [PATCH 298/324] Fix missed label arg in logger (#1886) --- pkg/sfu/downtrack.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index e3dd65a04..e21a001f9 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -601,7 +601,7 @@ func (d *DownTrack) maxLayerNotifierWorker() { maxLayerSpatial = d.forwarder.GetMaxSubscribedSpatial() } if onMaxSubscribedLayerChanged := d.getOnMaxLayerChanged(); onMaxSubscribedLayerChanged != nil { - d.logger.Infow("max subscribed layer changed", maxLayerSpatial) + d.logger.Infow("max subscribed layer changed", "maxLayerSpatial", maxLayerSpatial) onMaxSubscribedLayerChanged(d, maxLayerSpatial) } } From 9f3c975b1c415b4bd16310a77c12ed4c0c814c7b Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Sun, 16 Jul 2023 20:24:11 -0700 Subject: [PATCH 299/324] leave signal context open after stream closes (#1887) --- pkg/service/signal.go | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/pkg/service/signal.go b/pkg/service/signal.go index 299df055a..3de03c1ae 100644 --- a/pkg/service/signal.go +++ b/pkg/service/signal.go @@ -133,17 +133,6 @@ func (r *signalService) RelaySignal(stream psrpc.ServerStream[*rpc.RelaySignalRe "connID", ss.ConnectionId, ) - // copy the context to prevent a race between the session handler closing - // and the delivery of any parting messages from the client. take care to - // copy the incoming rpc headers to avoid dropping any session vars. - ctx, cancel := context.WithCancel(metadata.NewContextWithIncomingHeader(context.Background(), metadata.IncomingHeader(stream.Context()))) - - // wait until room setup finishes to cancel the session context even if the - // stream has closed. this is required to complete setup i/o after room api - // has discarded the temp client. - sessionHandlerDone := make(chan struct{}) - defer close(sessionHandlerDone) - sink := routing.NewSignalMessageSink(routing.SignalSinkParams[*rpc.RelaySignalResponse, *rpc.RelaySignalRequest]{ Logger: l, Stream: stream, @@ -163,11 +152,13 @@ func (r *signalService) RelaySignal(stream psrpc.ServerStream[*rpc.RelaySignalRe l.Infow("signal stream closed", "error", err) reqChan.Close() - - <-sessionHandlerDone - cancel() }() + // copy the context to prevent a race between the session handler closing + // and the delivery of any parting messages from the client. take care to + // copy the incoming rpc headers to avoid dropping any session vars. + ctx := metadata.NewContextWithIncomingHeader(context.Background(), metadata.IncomingHeader(stream.Context())) + err = r.sessionHandler(ctx, livekit.RoomName(ss.RoomName), *pi, livekit.ConnectionID(ss.ConnectionId), reqChan, sink) if err != nil { l.Errorw("could not handle new participant", err) From f41b93657e6e9c3663445ea9563adb5a806d4e29 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 18 Jul 2023 09:14:41 +0530 Subject: [PATCH 300/324] Log a bit more in sender report warp report. (#1888) --- pkg/sfu/buffer/rtpstats.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index 8bf34b958..1774eb28c 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -888,8 +888,9 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32, calculatedClockRate uint32) tsCycles++ } nowRTPExt := getExtTS(nowRTP, tsCycles) + var nowRTPExtUsingRate uint64 if calculatedClockRate != 0 { - nowRTPExtUsingRate := r.extStartTS + uint64(float64(calculatedClockRate)*timeSinceFirst.Seconds()) + nowRTPExtUsingRate = r.extStartTS + uint64(float64(calculatedClockRate)*timeSinceFirst.Seconds()) if nowRTPExtUsingRate > nowRTPExt { nowRTPExt = nowRTPExtUsingRate nowRTP = uint32(nowRTPExtUsingRate) @@ -939,6 +940,9 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32, calculatedClockRate uint32) "reportDrift", reportDriftResult.String(), "highestTS", r.highestTS, "highestTime", r.highestTime.String(), + "calculatedClockRate", calculatedClockRate, + "nowRTPExt", nowRTPExt, + "nowRTPExtUsingRate", nowRTPExtUsingRate, ) } From 66de9ff4a0c2c9a3832d61bf3b9383b40bcc3713 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 18 Jul 2023 23:21:06 +0530 Subject: [PATCH 301/324] Add debug log for RTCP sender report. (#1890) * Add debug log for RTCP sender report. Temporary to collect more data. Hitting scenarios under congestion where the sender report gets off sync. Need some data to pore through and understand and implement changes. * Debugw --- pkg/sfu/buffer/rtpstats.go | 19 +++++++++++++++++++ pkg/sfu/downtrack.go | 6 +++++- pkg/sfu/forwarder.go | 19 +++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index 1774eb28c..f5bb0a5d9 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -944,6 +944,25 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32, calculatedClockRate uint32) "nowRTPExt", nowRTPExt, "nowRTPExtUsingRate", nowRTPExtUsingRate, ) + } else { + packetDriftResult, reportDriftResult := r.getDrift() + r.logger.Debugw( + "sending sender report", + "ntp", nowNTP.Time().String(), + "rtp", nowRTP, + "departure", now.String(), + "ntpDiffSinceLast", ntpDiffSinceLast.Seconds(), + "rtpDiffSinceLast", int32(rtpDiffSinceLast), + "departureDiffSinceLast", departureDiffSinceLast.Seconds(), + "expectedTimeDiffSinceLast", expectedTimeDiffSinceLast, + "packetDrift", packetDriftResult.String(), + "reportDrift", reportDriftResult.String(), + "highestTS", r.highestTS, + "highestTime", r.highestTime.String(), + "calculatedClockRate", calculatedClockRate, + "nowRTPExt", nowRTPExt, + "nowRTPExtUsingRate", nowRTPExtUsingRate, + ) } return &rtcp.SenderReport{ diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index e21a001f9..d943b7c75 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -1165,7 +1165,11 @@ func (d *DownTrack) CreateSenderReport() *rtcp.SenderReport { return nil } - return d.rtpStats.GetRtcpSenderReport(d.ssrc, d.receiver.GetCalculatedClockRate(d.forwarder.CurrentLayer().Spatial)) + clockLayer := d.forwarder.CurrentLayer().Spatial + if clockLayer == buffer.InvalidLayerSpatial { + clockLayer = d.forwarder.GetReferenceLayerSpatial() + } + return d.rtpStats.GetRtcpSenderReport(d.ssrc, d.receiver.GetCalculatedClockRate(clockLayer)) } func (d *DownTrack) writeBlankFrameRTP(duration float32, generation uint32) chan struct{} { diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 23a8d694d..79b2b48c1 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -529,6 +529,13 @@ func (f *Forwarder) GetMaxSubscribedSpatial() int32 { return layer } +func (f *Forwarder) GetReferenceLayerSpatial() int32 { + f.lock.RLock() + defer f.lock.RUnlock() + + return f.referenceLayerSpatial +} + func (f *Forwarder) isDeficientLocked() bool { return f.lastAllocation.IsDeficient } @@ -1970,6 +1977,18 @@ done: if !targetLayer.IsValid() { distance += (maxSeenLayer.Temporal + 1) } + // TODO-REMOVE-AFTER-DEBUG + logger.Debugw( + "distance to desired", + "maxSeenLauer", maxSeenLayer, + "availableLayers", availableLayers, + "brs", brs, + "targetLayer", targetLayer, + "maxLayer", maxLayer, + "adjustedMaxLayer", adjustedMaxLayer, + "maxAvailableSpatial", maxAvailableSpatial, + "maxAvailableTemporal", maxAvailableTemporal, + ) return float64(distance) / float64(maxSeenLayer.Temporal+1) } From cf8cf1a87f7ed8a1df34e00d41f2875308e271a8 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 19 Jul 2023 10:22:51 +0530 Subject: [PATCH 302/324] Forgot to log important bits :-( (#1891) --- pkg/sfu/forwarder.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 79b2b48c1..02f711d06 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -1988,6 +1988,8 @@ done: "adjustedMaxLayer", adjustedMaxLayer, "maxAvailableSpatial", maxAvailableSpatial, "maxAvailableTemporal", maxAvailableTemporal, + "distance", distance, + "distanceToDesired", float64(distance)/float64(maxSeenLayer.Temporal+1), ) return float64(distance) / float64(maxSeenLayer.Temporal+1) From dd995899bf25120aed3e418f173bd745eff71422 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 19 Jul 2023 12:50:03 +0530 Subject: [PATCH 303/324] Handle extreme case of sender report lagging. (#1892) --- pkg/sfu/buffer/rtpstats.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index f5bb0a5d9..075fe72fe 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -793,7 +793,9 @@ func (r *RTPStats) SetRtcpSenderReportData(srData *RTCPSenderReportData) { r.logger.Infow( "received sender report, out-of-order, resetting", "prevTSExt", r.srNewest.RTPTimestampExt, + "prevNTP", r.srNewest.NTPTimestamp.Time().String(), "currTSExt", srDataCopy.RTPTimestampExt, + "currNTP", srDataCopy.NTPTimestamp.Time().String(), ) r.srFirst = &srDataCopy r.srNewest = &srDataCopy @@ -897,6 +899,27 @@ func (r *RTPStats) GetRtcpSenderReport(ssrc uint32, calculatedClockRate uint32) } } + if r.srNewest != nil && nowRTPExt < r.srNewest.RTPTimestampExt { + // If report being generated is behind, use the time different and clock rate of codec to produce next report. + // Current report could be behind due to the following + // - Publisher pacing + // - Due to above, report from publisher side is ahead of packet timestamps. + // Note that report will map wall clock to timestamp at capture time and happens before the pacer. + // - Pause/Mute followed by resume, some combination of events that could + // result in this module not having calculated clock rate of publisher side. + // - When the above happens, current will be generated using highestTS which could be behind. + // That could end up behind the last report's timestamp in extreme cases + r.logger.Infow( + "sending sender report, out-of-order, repairing", + "prevTSExt", r.srNewest.RTPTimestampExt, + "prevNTP", r.srNewest.NTPTimestamp.Time().String(), + "currTSExt", nowRTPExt, + "currNTP", nowNTP.Time().String(), + ) + ntpDiffSinceLast := nowNTP.Time().Sub(r.srNewest.NTPTimestamp.Time()) + nowRTPExt = r.srNewest.RTPTimestampExt + uint64(ntpDiffSinceLast.Seconds()*float64(r.params.ClockRate)) + } + // monitor and log RTP timestamp anomalies var ntpDiffSinceLast time.Duration var rtpDiffSinceLast uint32 From cf4801064d2a37d61be9be425dcd0a5d0de41bb2 Mon Sep 17 00:00:00 2001 From: kannonski Date: Wed, 19 Jul 2023 23:23:30 +0200 Subject: [PATCH 304/324] changing key file permissions control (#1893) --- pkg/config/config.go | 5 +++-- pkg/service/wire.go | 5 +++-- pkg/service/wire_gen.go | 5 +++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 8b7b340e7..f13cd49e4 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -36,7 +36,7 @@ const ( ) var ( - ErrKeyFileIncorrectPermission = errors.New("key file must have 0600 permission") + ErrKeyFileIncorrectPermission = errors.New("key file others permissions must be set to 0") ErrKeysNotSet = errors.New("one of key-file or keys must be provided") ) @@ -547,9 +547,10 @@ func (conf *Config) ToCLIFlagNames(existingFlags []cli.Flag) map[string]reflect. func (conf *Config) ValidateKeys() error { // prefer keyfile if set if conf.KeyFile != "" { + var otherFilter os.FileMode = 0007 if st, err := os.Stat(conf.KeyFile); err != nil { return err - } else if st.Mode().Perm() != 0600 { + } else if st.Mode().Perm()&otherFilter != 0000 { return ErrKeyFileIncorrectPermission } f, err := os.Open(conf.KeyFile) diff --git a/pkg/service/wire.go b/pkg/service/wire.go index 126542780..bb8451e05 100644 --- a/pkg/service/wire.go +++ b/pkg/service/wire.go @@ -87,10 +87,11 @@ func getNodeID(currentNode routing.LocalNode) livekit.NodeID { func createKeyProvider(conf *config.Config) (auth.KeyProvider, error) { // prefer keyfile if set if conf.KeyFile != "" { + var otherFilter os.FileMode = 0007 if st, err := os.Stat(conf.KeyFile); err != nil { return nil, err - } else if st.Mode().Perm() != 0600 { - return nil, fmt.Errorf("key file must have permission set to 600") + } else if st.Mode().Perm()&otherFilter != 0000 { + return nil, fmt.Errorf("key file others permissions must be set to 0") } f, err := os.Open(conf.KeyFile) if err != nil { diff --git a/pkg/service/wire_gen.go b/pkg/service/wire_gen.go index 1ce19893a..ec817449d 100644 --- a/pkg/service/wire_gen.go +++ b/pkg/service/wire_gen.go @@ -132,10 +132,11 @@ func getNodeID(currentNode routing.LocalNode) livekit.NodeID { func createKeyProvider(conf *config.Config) (auth.KeyProvider, error) { if conf.KeyFile != "" { + var otherFilter os.FileMode = 0007 if st, err := os.Stat(conf.KeyFile); err != nil { return nil, err - } else if st.Mode().Perm() != 0600 { - return nil, fmt.Errorf("key file must have permission set to 600") + } else if st.Mode().Perm()&otherFilter != 0000 { + return nil, fmt.Errorf("key file others permission must be set to 0") } f, err := os.Open(conf.KeyFile) if err != nil { From 6ad1e1598d3ddf864d7b5ab4f2abcdbab5ebbf65 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Thu, 20 Jul 2023 19:13:27 -0700 Subject: [PATCH 305/324] move signal server start to server start (#1894) * move signal server start to server start * fix test --- pkg/service/server.go | 4 ++++ pkg/service/signal.go | 13 +++++++------ pkg/service/signal_test.go | 5 ++++- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/pkg/service/server.go b/pkg/service/server.go index c21b98fe0..390432116 100644 --- a/pkg/service/server.go +++ b/pkg/service/server.go @@ -227,6 +227,10 @@ func (s *LivekitServer) Start() error { go s.promServer.Serve(promLn) } + if err := s.signalServer.Start(); err != nil { + return err + } + httpGroup := &errgroup.Group{} for _, ln := range listeners { l := ln diff --git a/pkg/service/signal.go b/pkg/service/signal.go index 3de03c1ae..817344914 100644 --- a/pkg/service/signal.go +++ b/pkg/service/signal.go @@ -28,6 +28,7 @@ type SessionHandler func( type SignalServer struct { server rpc.TypedSignalServer + nodeID livekit.NodeID } func NewSignalServer( @@ -47,12 +48,7 @@ func NewSignalServer( if err != nil { return nil, err } - logger.Debugw("starting relay signal server", "topic", nodeID) - if err := s.RegisterRelaySignalTopic(nodeID); err != nil { - return nil, err - } - - return &SignalServer{s}, nil + return &SignalServer{s, nodeID}, nil } func NewDefaultSignalServer( @@ -101,6 +97,11 @@ func NewDefaultSignalServer( return NewSignalServer(livekit.NodeID(currentNode.Id), currentNode.Region, bus, config, sessionHandler) } +func (s *SignalServer) Start() error { + logger.Debugw("starting relay signal server", "topic", s.nodeID) + return s.server.RegisterRelaySignalTopic(s.nodeID) +} + func (r *SignalServer) Stop() { r.server.Kill() } diff --git a/pkg/service/signal_test.go b/pkg/service/signal_test.go index 946675864..ec83c5c3e 100644 --- a/pkg/service/signal_test.go +++ b/pkg/service/signal_test.go @@ -44,7 +44,7 @@ func TestSignal(t *testing.T) { client, err := routing.NewSignalClient(livekit.NodeID("node0"), bus, cfg) require.NoError(t, err) - _, err = NewSignalServer(livekit.NodeID("node1"), "region", bus, cfg, func( + server, err := NewSignalServer(livekit.NodeID("node1"), "region", bus, cfg, func( ctx context.Context, roomName livekit.RoomName, pi routing.ParticipantInit, @@ -62,6 +62,9 @@ func TestSignal(t *testing.T) { }) require.NoError(t, err) + err = server.Start() + require.NoError(t, err) + _, reqSink, resSource, err := client.StartParticipantSignal( context.Background(), livekit.RoomName("room1"), From 3980d049c9c3dff519de69387f26ced62aa690b5 Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Thu, 20 Jul 2023 19:23:35 -0700 Subject: [PATCH 306/324] close disconnected participants when signal channel fails (#1895) * close disconnected participants when signal channel fails * fix typefake * update reason --- pkg/rtc/participant.go | 9 ++++++ pkg/rtc/types/interfaces.go | 1 + .../typesfakes/fake_local_participant.go | 30 +++++++++++++++++++ pkg/service/roommanager.go | 2 +- 4 files changed, 41 insertions(+), 1 deletion(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index e0d5214d4..587879833 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -516,6 +516,15 @@ func (p *ParticipantImpl) OnClaimsChanged(callback func(types.LocalParticipant)) p.lock.Unlock() } +func (p *ParticipantImpl) HandleSignalSourceClose() { + p.TransportManager.SetSignalSourceValid(false) + + if !p.TransportManager.HasPublisherEverConnected() && !p.TransportManager.HasSubscriberEverConnected() { + p.params.Logger.Infow("closing disconnected participant") + _ = p.Close(false, types.ParticipantCloseReasonJoinFailed, false) + } +} + // HandleOffer an offer from remote participant, used when clients make the initial connection func (p *ParticipantImpl) HandleOffer(offer webrtc.SessionDescription) { p.params.Logger.Debugw("received offer", "transport", livekit.SignalTarget_PUBLISHER) diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index 443a0a3d5..eb9336121 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -295,6 +295,7 @@ type LocalParticipant interface { CloseSignalConnection(reason SignallingCloseReason) UpdateLastSeenSignal() SetSignalSourceValid(valid bool) + HandleSignalSourceClose() // permissions ClaimGrants() *auth.ClaimGrants diff --git a/pkg/rtc/types/typesfakes/fake_local_participant.go b/pkg/rtc/types/typesfakes/fake_local_participant.go index deb07909f..b071877d9 100644 --- a/pkg/rtc/types/typesfakes/fake_local_participant.go +++ b/pkg/rtc/types/typesfakes/fake_local_participant.go @@ -326,6 +326,10 @@ type FakeLocalParticipant struct { handleReconnectAndSendResponseReturnsOnCall map[int]struct { result1 error } + HandleSignalSourceCloseStub func() + handleSignalSourceCloseMutex sync.RWMutex + handleSignalSourceCloseArgsForCall []struct { + } HasPermissionStub func(livekit.TrackID, livekit.ParticipantIdentity) bool hasPermissionMutex sync.RWMutex hasPermissionArgsForCall []struct { @@ -2481,6 +2485,30 @@ func (fake *FakeLocalParticipant) HandleReconnectAndSendResponseReturnsOnCall(i }{result1} } +func (fake *FakeLocalParticipant) HandleSignalSourceClose() { + fake.handleSignalSourceCloseMutex.Lock() + fake.handleSignalSourceCloseArgsForCall = append(fake.handleSignalSourceCloseArgsForCall, struct { + }{}) + stub := fake.HandleSignalSourceCloseStub + fake.recordInvocation("HandleSignalSourceClose", []interface{}{}) + fake.handleSignalSourceCloseMutex.Unlock() + if stub != nil { + fake.HandleSignalSourceCloseStub() + } +} + +func (fake *FakeLocalParticipant) HandleSignalSourceCloseCallCount() int { + fake.handleSignalSourceCloseMutex.RLock() + defer fake.handleSignalSourceCloseMutex.RUnlock() + return len(fake.handleSignalSourceCloseArgsForCall) +} + +func (fake *FakeLocalParticipant) HandleSignalSourceCloseCalls(stub func()) { + fake.handleSignalSourceCloseMutex.Lock() + defer fake.handleSignalSourceCloseMutex.Unlock() + fake.HandleSignalSourceCloseStub = stub +} + func (fake *FakeLocalParticipant) HasPermission(arg1 livekit.TrackID, arg2 livekit.ParticipantIdentity) bool { fake.hasPermissionMutex.Lock() ret, specificReturn := fake.hasPermissionReturnsOnCall[len(fake.hasPermissionArgsForCall)] @@ -5626,6 +5654,8 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.handleOfferMutex.RUnlock() fake.handleReconnectAndSendResponseMutex.RLock() defer fake.handleReconnectAndSendResponseMutex.RUnlock() + fake.handleSignalSourceCloseMutex.RLock() + defer fake.handleSignalSourceCloseMutex.RUnlock() fake.hasPermissionMutex.RLock() defer fake.hasPermissionMutex.RUnlock() fake.hiddenMutex.RLock() diff --git a/pkg/service/roommanager.go b/pkg/service/roommanager.go index bd09448dd..b0cefd31c 100644 --- a/pkg/service/roommanager.go +++ b/pkg/service/roommanager.go @@ -549,7 +549,7 @@ func (r *RoomManager) rtcSessionWorker(room *rtc.Room, participant types.LocalPa // this means ICE restart isn't possible in single node mode if obj == nil { if room.GetParticipantRequestSource(participant.Identity()) == requestSource { - participant.SetSignalSourceValid(false) + participant.HandleSignalSourceClose() } return } From 6c20c7eb152018ab6b2cc0cb99bdf63fbb82177f Mon Sep 17 00:00:00 2001 From: Paul Wells Date: Thu, 20 Jul 2023 21:21:40 -0700 Subject: [PATCH 307/324] add test for removing disconnected participants on signal close (#1896) * add test for removing disconnected participants on signal close * cleanup --- pkg/service/wire_gen.go | 2 +- test/client/client.go | 201 +++++++++++++++++++++++----------------- test/multinode_test.go | 44 +++++++++ 3 files changed, 163 insertions(+), 84 deletions(-) diff --git a/pkg/service/wire_gen.go b/pkg/service/wire_gen.go index ec817449d..b051fb954 100644 --- a/pkg/service/wire_gen.go +++ b/pkg/service/wire_gen.go @@ -136,7 +136,7 @@ func createKeyProvider(conf *config.Config) (auth.KeyProvider, error) { if st, err := os.Stat(conf.KeyFile); err != nil { return nil, err } else if st.Mode().Perm()&otherFilter != 0000 { - return nil, fmt.Errorf("key file others permission must be set to 0") + return nil, fmt.Errorf("key file others permissions must be set to 0") } f, err := os.Open(conf.KeyFile) if err != nil { diff --git a/test/client/client.go b/test/client/client.go index fa9b485a1..776d356a7 100644 --- a/test/client/client.go +++ b/test/client/client.go @@ -28,6 +28,11 @@ import ( "github.com/livekit/livekit-server/pkg/rtc/types" ) +type SignalRequestHandler func(msg *livekit.SignalRequest) error +type SignalRequestInterceptor func(msg *livekit.SignalRequest, next SignalRequestHandler) error +type SignalResponseHandler func(msg *livekit.SignalResponse) error +type SignalResponseInterceptor func(msg *livekit.SignalResponse, next SignalResponseHandler) error + type RTCClient struct { id livekit.ParticipantID conn *websocket.Conn @@ -45,6 +50,9 @@ type RTCClient struct { localParticipant *livekit.ParticipantInfo remoteParticipants map[livekit.ParticipantID]*livekit.ParticipantInfo + signalRequestInterceptor SignalRequestInterceptor + signalResponseInterceptor SignalResponseInterceptor + subscriberAsPrimary atomic.Bool publisherFullyEstablished atomic.Bool subscriberFullyEstablished atomic.Bool @@ -83,10 +91,12 @@ var ( ) type Options struct { - AutoSubscribe bool - Publish string - ClientInfo *livekit.ClientInfo - DisabledCodecs []webrtc.RTPCodecCapability + AutoSubscribe bool + Publish string + ClientInfo *livekit.ClientInfo + DisabledCodecs []webrtc.RTPCodecCapability + SignalRequestInterceptor SignalRequestInterceptor + SignalResponseInterceptor SignalResponseInterceptor } func NewWebSocketConn(host, token string, opts *Options) (*websocket.Conn, error) { @@ -265,6 +275,11 @@ func NewRTCClient(conn *websocket.Conn, opts *Options) (*RTCClient, error) { }) }) + if opts != nil { + c.signalRequestInterceptor = opts.SignalRequestInterceptor + c.signalResponseInterceptor = opts.SignalResponseInterceptor + } + return c, nil } @@ -290,89 +305,101 @@ func (c *RTCClient) Run() error { logger.Errorw("error while reading", err) return err } - switch msg := res.Message.(type) { - case *livekit.SignalResponse_Join: - c.localParticipant = msg.Join.Participant - c.id = livekit.ParticipantID(msg.Join.Participant.Sid) - c.lock.Lock() - for _, p := range msg.Join.OtherParticipants { - c.remoteParticipants[livekit.ParticipantID(p.Sid)] = p - } - c.lock.Unlock() - // if publish only, negotiate - if !msg.Join.SubscriberPrimary { - c.subscriberAsPrimary.Store(false) - c.publisher.Negotiate(false) - } else { - c.subscriberAsPrimary.Store(true) - } - - logger.Infow("join accepted, awaiting offer", "participant", msg.Join.Participant.Identity) - case *livekit.SignalResponse_Answer: - // logger.Debugw("received server answer", - // "participant", c.localParticipant.Identity, - // "answer", msg.Answer.Sdp) - c.handleAnswer(rtc.FromProtoSessionDescription(msg.Answer)) - case *livekit.SignalResponse_Offer: - logger.Infow("received server offer", - "participant", c.localParticipant.Identity, - ) - desc := rtc.FromProtoSessionDescription(msg.Offer) - c.handleOffer(desc) - case *livekit.SignalResponse_Trickle: - candidateInit, err := rtc.FromProtoTrickle(msg.Trickle) - if err != nil { - return err - } - if msg.Trickle.Target == livekit.SignalTarget_PUBLISHER { - c.publisher.AddICECandidate(candidateInit) - } else { - c.subscriber.AddICECandidate(candidateInit) - } - case *livekit.SignalResponse_Update: - c.lock.Lock() - for _, p := range msg.Update.Participants { - if livekit.ParticipantID(p.Sid) != c.id { - if p.State != livekit.ParticipantInfo_DISCONNECTED { - c.remoteParticipants[livekit.ParticipantID(p.Sid)] = p - } else { - delete(c.remoteParticipants, livekit.ParticipantID(p.Sid)) - } - } - } - c.lock.Unlock() - - case *livekit.SignalResponse_TrackPublished: - logger.Debugw("track published", "trackID", msg.TrackPublished.Track.Name, "participant", c.localParticipant.Sid, - "cid", msg.TrackPublished.Cid, "trackSid", msg.TrackPublished.Track.Sid) - c.lock.Lock() - c.pendingPublishedTracks[msg.TrackPublished.Cid] = msg.TrackPublished.Track - c.lock.Unlock() - case *livekit.SignalResponse_RefreshToken: - c.lock.Lock() - c.refreshToken = msg.RefreshToken - c.lock.Unlock() - case *livekit.SignalResponse_TrackUnpublished: - sid := msg.TrackUnpublished.TrackSid - c.lock.Lock() - sender := c.trackSenders[sid] - if sender != nil { - if err := c.publisher.RemoveTrack(sender); err != nil { - logger.Errorw("Could not unpublish track", err) - } - c.publisher.Negotiate(false) - } - delete(c.trackSenders, sid) - delete(c.localTracks, sid) - c.lock.Unlock() - case *livekit.SignalResponse_Pong: - c.pongReceivedAt.Store(msg.Pong) - case *livekit.SignalResponse_SubscriptionResponse: - c.subscriptionResponse.Store(msg.SubscriptionResponse) + if c.signalResponseInterceptor != nil { + err = c.signalResponseInterceptor(res, c.handleSignalResponse) + } else { + err = c.handleSignalResponse(res) + } + if err != nil { + return err } } } +func (c *RTCClient) handleSignalResponse(res *livekit.SignalResponse) error { + switch msg := res.Message.(type) { + case *livekit.SignalResponse_Join: + c.localParticipant = msg.Join.Participant + c.id = livekit.ParticipantID(msg.Join.Participant.Sid) + c.lock.Lock() + for _, p := range msg.Join.OtherParticipants { + c.remoteParticipants[livekit.ParticipantID(p.Sid)] = p + } + c.lock.Unlock() + // if publish only, negotiate + if !msg.Join.SubscriberPrimary { + c.subscriberAsPrimary.Store(false) + c.publisher.Negotiate(false) + } else { + c.subscriberAsPrimary.Store(true) + } + + logger.Infow("join accepted, awaiting offer", "participant", msg.Join.Participant.Identity) + case *livekit.SignalResponse_Answer: + // logger.Debugw("received server answer", + // "participant", c.localParticipant.Identity, + // "answer", msg.Answer.Sdp) + c.handleAnswer(rtc.FromProtoSessionDescription(msg.Answer)) + case *livekit.SignalResponse_Offer: + logger.Infow("received server offer", + "participant", c.localParticipant.Identity, + ) + desc := rtc.FromProtoSessionDescription(msg.Offer) + c.handleOffer(desc) + case *livekit.SignalResponse_Trickle: + candidateInit, err := rtc.FromProtoTrickle(msg.Trickle) + if err != nil { + return err + } + if msg.Trickle.Target == livekit.SignalTarget_PUBLISHER { + c.publisher.AddICECandidate(candidateInit) + } else { + c.subscriber.AddICECandidate(candidateInit) + } + case *livekit.SignalResponse_Update: + c.lock.Lock() + for _, p := range msg.Update.Participants { + if livekit.ParticipantID(p.Sid) != c.id { + if p.State != livekit.ParticipantInfo_DISCONNECTED { + c.remoteParticipants[livekit.ParticipantID(p.Sid)] = p + } else { + delete(c.remoteParticipants, livekit.ParticipantID(p.Sid)) + } + } + } + c.lock.Unlock() + + case *livekit.SignalResponse_TrackPublished: + logger.Debugw("track published", "trackID", msg.TrackPublished.Track.Name, "participant", c.localParticipant.Sid, + "cid", msg.TrackPublished.Cid, "trackSid", msg.TrackPublished.Track.Sid) + c.lock.Lock() + c.pendingPublishedTracks[msg.TrackPublished.Cid] = msg.TrackPublished.Track + c.lock.Unlock() + case *livekit.SignalResponse_RefreshToken: + c.lock.Lock() + c.refreshToken = msg.RefreshToken + c.lock.Unlock() + case *livekit.SignalResponse_TrackUnpublished: + sid := msg.TrackUnpublished.TrackSid + c.lock.Lock() + sender := c.trackSenders[sid] + if sender != nil { + if err := c.publisher.RemoveTrack(sender); err != nil { + logger.Errorw("Could not unpublish track", err) + } + c.publisher.Negotiate(false) + } + delete(c.trackSenders, sid) + delete(c.localTracks, sid) + c.lock.Unlock() + case *livekit.SignalResponse_Pong: + c.pongReceivedAt.Store(msg.Pong) + case *livekit.SignalResponse_SubscriptionResponse: + c.subscriptionResponse.Store(msg.SubscriptionResponse) + } + return nil +} + func (c *RTCClient) WaitUntilConnected() error { ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() @@ -486,6 +513,14 @@ func (c *RTCClient) SendPing() error { } func (c *RTCClient) SendRequest(msg *livekit.SignalRequest) error { + if c.signalRequestInterceptor != nil { + return c.signalRequestInterceptor(msg, c.sendRequest) + } else { + return c.sendRequest(msg) + } +} + +func (c *RTCClient) sendRequest(msg *livekit.SignalRequest) error { payload, err := proto.Marshal(msg) if err != nil { return err diff --git a/test/multinode_test.go b/test/multinode_test.go index 57a2ed57e..fe973aefd 100644 --- a/test/multinode_test.go +++ b/test/multinode_test.go @@ -11,6 +11,7 @@ import ( "github.com/livekit/livekit-server/pkg/rtc" "github.com/livekit/livekit-server/pkg/testutils" + "github.com/livekit/livekit-server/test/client" ) func TestMultiNodeRouting(t *testing.T) { @@ -261,3 +262,46 @@ func TestMultiNodeRevokePublishPermission(t *testing.T) { return "" }) } + +func TestCloseDisconnectedParticipantOnSignalClose(t *testing.T) { + _, _, finish := setupMultiNodeTest("TestCloseDisconnectedParticipantOnSignalClose") + defer finish() + + c1 := createRTCClient("c1", secondServerPort, nil) + waitUntilConnected(t, c1) + + c2 := createRTCClient("c2", defaultServerPort, &client.Options{ + SignalRequestInterceptor: func(msg *livekit.SignalRequest, next client.SignalRequestHandler) error { + switch msg.Message.(type) { + case *livekit.SignalRequest_Offer, *livekit.SignalRequest_Answer, *livekit.SignalRequest_Leave: + return nil + default: + return next(msg) + } + }, + SignalResponseInterceptor: func(msg *livekit.SignalResponse, next client.SignalResponseHandler) error { + switch msg.Message.(type) { + case *livekit.SignalResponse_Offer, *livekit.SignalResponse_Answer: + return nil + default: + return next(msg) + } + }, + }) + + testutils.WithTimeout(t, func() string { + if len(c1.RemoteParticipants()) != 1 { + return "c1 did not see c2 join" + } + return "" + }) + + c2.Stop() + + testutils.WithTimeout(t, func() string { + if len(c1.RemoteParticipants()) != 0 { + return "c1 did not see c2 removed" + } + return "" + }) +} From 7e6aa0042618ca2ba47d63e482ca8b50f7be43fd Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Fri, 21 Jul 2023 16:23:00 +0530 Subject: [PATCH 308/324] Remove unused fields left over from refactor (#1897) --- pkg/sfu/streamallocator/channelobserver.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pkg/sfu/streamallocator/channelobserver.go b/pkg/sfu/streamallocator/channelobserver.go index 0e094bdad..94b888c57 100644 --- a/pkg/sfu/streamallocator/channelobserver.go +++ b/pkg/sfu/streamallocator/channelobserver.go @@ -72,10 +72,6 @@ type ChannelObserver struct { estimateTrend *TrendDetector nackTracker *NackTracker - - nackWindowStartTime time.Time - packets uint32 - repeatedNacks uint32 } func NewChannelObserver(params ChannelObserverParams, logger logger.Logger) *ChannelObserver { From 43fa6f57d1fb2b99fce6c01da743b3eb4a073b8b Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sun, 23 Jul 2023 10:11:35 +0530 Subject: [PATCH 309/324] A very simple leaky bucket pacer. (#1899) --- pkg/sfu/pacer/base.go | 17 +++-- pkg/sfu/pacer/leaky_bucket.go | 137 ++++++++++++++++++++++++++++++++++ pkg/sfu/pacer/pacer.go | 3 + 3 files changed, 152 insertions(+), 5 deletions(-) create mode 100644 pkg/sfu/pacer/leaky_bucket.go diff --git a/pkg/sfu/pacer/base.go b/pkg/sfu/pacer/base.go index fef7b413e..4b5da20c9 100644 --- a/pkg/sfu/pacer/base.go +++ b/pkg/sfu/pacer/base.go @@ -22,7 +22,13 @@ func NewBase(logger logger.Logger) *Base { } } -func (b *Base) SendPacket(p *Packet) error { +func (b *Base) SetInterval(_interval time.Duration) { +} + +func (b *Base) SetBitrate(_bitrate int) { +} + +func (b *Base) SendPacket(p *Packet) (int, error) { var sendingAt time.Time var err error defer func() { @@ -34,18 +40,19 @@ func (b *Base) SendPacket(p *Packet) error { sendingAt, err = b.writeRTPHeaderExtensions(p) if err != nil { b.logger.Errorw("writing rtp header extensions err", err) - return err + return 0, err } - _, err = p.WriteStream.WriteRTP(p.Header, p.Payload) + var written int + written, err = p.WriteStream.WriteRTP(p.Header, p.Payload) if err != nil { if !errors.Is(err, io.ErrClosedPipe) { b.logger.Errorw("write rtp packet failed", err) } - return err + return 0, err } - return nil + return written, nil } // writes RTP header extensions of track diff --git a/pkg/sfu/pacer/leaky_bucket.go b/pkg/sfu/pacer/leaky_bucket.go new file mode 100644 index 000000000..6e16f7f4e --- /dev/null +++ b/pkg/sfu/pacer/leaky_bucket.go @@ -0,0 +1,137 @@ +package pacer + +import ( + "sync" + "time" + + "github.com/gammazero/deque" + "github.com/livekit/protocol/logger" +) + +const ( + maxOvershootFactor = 2.0 +) + +type LeakyBucket struct { + *Base + + logger logger.Logger + + lock sync.RWMutex + packets deque.Deque[Packet] + interval time.Duration + bitrate int + isStopped bool +} + +func NewLeakyBucket(logger logger.Logger, interval time.Duration, bitrate int) *LeakyBucket { + l := &LeakyBucket{ + Base: NewBase(logger), + logger: logger, + interval: interval, + bitrate: bitrate, + } + l.packets.SetMinCapacity(9) + + go l.sendWorker() + return l +} + +func (l *LeakyBucket) SetInterval(interval time.Duration) { + l.lock.Lock() + defer l.lock.Unlock() + + l.interval = interval +} + +func (l *LeakyBucket) SetBitrate(bitrate int) { + l.lock.Lock() + defer l.lock.Unlock() + + l.bitrate = bitrate +} + +func (l *LeakyBucket) Stop() { + l.lock.Lock() + if l.isStopped { + l.lock.Unlock() + return + } + + l.isStopped = true + l.lock.Unlock() +} + +func (l *LeakyBucket) Enqueue(p Packet) { + l.lock.Lock() + defer l.lock.Unlock() + + if !l.isStopped { + l.packets.PushBack(p) + } +} + +func (l *LeakyBucket) sendWorker() { + l.lock.RLock() + interval := l.interval + bitrate := l.bitrate + l.lock.RUnlock() + + timer := time.NewTimer(interval) + overage := 0 + + for { + <-timer.C + + l.lock.RLock() + interval = l.interval + bitrate = l.bitrate + l.lock.RUnlock() + + // calculate number of bytes that can be sent in this interval + // adjusting for overage. + intervalBytes := int(interval.Seconds() * float64(bitrate) / 8.0) + maxOvershootBytes := int(float64(intervalBytes) * maxOvershootFactor) + toSendBytes := intervalBytes - overage + if toSendBytes < 0 { + // too much overage, wait for next interval + overage = -toSendBytes + timer.Reset(interval) + continue + } + + // do not allow too much overshoot in an interval + if toSendBytes > maxOvershootBytes { + toSendBytes = maxOvershootBytes + } + + for { + l.lock.Lock() + if l.isStopped { + l.lock.Unlock() + return + } + + if l.packets.Len() == 0 { + l.lock.Unlock() + // allow overshoot in next interval with shortage in this interval + overage = -toSendBytes + timer.Reset(interval) + break + } + p := l.packets.PopFront() + l.lock.Unlock() + + written, _ := l.Base.SendPacket(&p) + toSendBytes -= written + if toSendBytes < 0 { + // overage, wait for next interval + overage = -toSendBytes + timer.Reset(interval) + break + } + } + } +} + +// ------------------------------------------------ diff --git a/pkg/sfu/pacer/pacer.go b/pkg/sfu/pacer/pacer.go index 3be8a8a86..2288d8639 100644 --- a/pkg/sfu/pacer/pacer.go +++ b/pkg/sfu/pacer/pacer.go @@ -26,6 +26,9 @@ type Packet struct { type Pacer interface { Enqueue(p Packet) Stop() + + SetInterval(interval time.Duration) + SetBitrate(bitrate int) } // ------------------------------------------------ From ffd6dc221048cab7e7f271d1efb644c7ccfa9dac Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 25 Jul 2023 13:53:21 +0530 Subject: [PATCH 310/324] Packet level ddebug logs. (#1900) Only for debugging for a bit. Not for deploy. --- pkg/sfu/downtrack.go | 95 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 90 insertions(+), 5 deletions(-) diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index d943b7c75..77226e7b5 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -621,11 +621,27 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { return err } + var iPID, iTL0PICIDX, iKEYIDX, iTID, oPID, oTL0PICIDX, oKEYIDX, oTID int // TOOD-REMOVE-AFTER-DEBUG var payload []byte pool := PacketFactory.Get().(*[]byte) if len(tp.codecBytes) != 0 { - incomingVP8, _ := extPkt.Payload.(buffer.VP8) - payload = d.translateVP8PacketTo(extPkt.Packet, &incomingVP8, tp.codecBytes, pool) + incomingVP8, ok := extPkt.Payload.(buffer.VP8) + if ok { + iPID = int(incomingVP8.PictureID) + iTL0PICIDX = int(incomingVP8.TL0PICIDX) + iKEYIDX = int(incomingVP8.KEYIDX) + iTID = int(incomingVP8.TID) + + var outgoingVP8 buffer.VP8 + if uerr := outgoingVP8.Unmarshal(append(tp.codecBytes, 0x0)); uerr == nil { + oPID = int(outgoingVP8.PictureID) + oTL0PICIDX = int(outgoingVP8.TL0PICIDX) + oKEYIDX = int(outgoingVP8.KEYIDX) + oTID = int(outgoingVP8.TID) + } + + payload = d.translateVP8PacketTo(extPkt.Packet, &incomingVP8, tp.codecBytes, pool) + } } if payload == nil { payload = (*pool)[:len(extPkt.Packet.Payload)] @@ -652,6 +668,28 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { return err } + if d.kind == webrtc.RTPCodecTypeVideo { + d.logger.Debugw( + "packet debug (forwarding)", + "layer", layer, + "isn", extPkt.Packet.SequenceNumber, + "its", extPkt.Packet.Timestamp, + "im", extPkt.Packet.Marker, + "osn", hdr.SequenceNumber, + "ots", hdr.Timestamp, + "om", hdr.Marker, + "len", len(payload), + "iPID", iPID, + "iTL0PICIDX", iTL0PICIDX, + "iKEYIDX", iKEYIDX, + "iTID", iTID, + "oPID", oPID, + "oTL0PICIDX", oTL0PICIDX, + "oKEYIDX", oKEYIDX, + "oTID", oTID, + ) + } + d.pacer.Enqueue(pacer.Packet{ Header: hdr, Extensions: []pacer.ExtensionData{{ID: uint8(d.dependencyDescriptorExtID), Payload: tp.ddBytes}}, @@ -735,6 +773,16 @@ func (d *DownTrack) WritePaddingRTP(bytesToSend int, paddingOnMute bool, forceMa // last byte of padding has padding size including that byte payload[RTPPaddingMaxPayloadSize-1] = byte(RTPPaddingMaxPayloadSize) + if d.kind == webrtc.RTPCodecTypeVideo { + d.logger.Debugw( + "packet debug (padding)", + "osn", hdr.SequenceNumber, + "ots", hdr.Timestamp, + "om", hdr.Marker, + "len", len(payload), + ) + } + d.pacer.Enqueue(pacer.Packet{ Header: &hdr, Payload: payload, @@ -1488,18 +1536,22 @@ func (d *DownTrack) retransmitPackets(nacks []uint16) { numRepeatedNACKs++ } + var incomingHdr rtp.Header // TODO-REMOVE-AFTER-DEBUG var pkt rtp.Packet if err = pkt.Unmarshal(pktBuff[:n]); err != nil { + d.logger.Errorw("unmarshalling rtp packet failed in retransmit", err) continue } + incomingHdr = pkt.Header pkt.Header.SequenceNumber = meta.targetSeqNo pkt.Header.Timestamp = meta.timestamp pkt.Header.SSRC = d.ssrc pkt.Header.PayloadType = d.payloadType + var iPID, iTL0PICIDX, iKEYIDX, iTID, oPID, oTL0PICIDX, oKEYIDX, oTID int // TOOD-REMOVE-AFTER-DEBUG var payload []byte pool := PacketFactory.Get().(*[]byte) - if d.mime == "video/vp8" && len(pkt.Payload) > 0 { + if d.mime == "video/vp8" && len(pkt.Payload) > 0 && len(meta.codecBytes) != 0 { var incomingVP8 buffer.VP8 if err = incomingVP8.Unmarshal(pkt.Payload); err != nil { d.logger.Errorw("unmarshalling VP8 packet err", err) @@ -1507,15 +1559,48 @@ func (d *DownTrack) retransmitPackets(nacks []uint16) { continue } - if len(meta.codecBytes) != 0 { - payload = d.translateVP8PacketTo(&pkt, &incomingVP8, meta.codecBytes, pool) + iPID = int(incomingVP8.PictureID) + iTL0PICIDX = int(incomingVP8.TL0PICIDX) + iKEYIDX = int(incomingVP8.KEYIDX) + iTID = int(incomingVP8.TID) + + var outgoingVP8 buffer.VP8 + if uerr := outgoingVP8.Unmarshal(append(meta.codecBytes, 0x0)); uerr == nil { + oPID = int(outgoingVP8.PictureID) + oTL0PICIDX = int(outgoingVP8.TL0PICIDX) + oKEYIDX = int(outgoingVP8.KEYIDX) + oTID = int(outgoingVP8.TID) } + + payload = d.translateVP8PacketTo(&pkt, &incomingVP8, meta.codecBytes, pool) } if payload == nil { payload = (*pool)[:len(pkt.Payload)] copy(payload, pkt.Payload) } + if d.kind == webrtc.RTPCodecTypeVideo { + d.logger.Debugw( + "packet debug (retransmit)", + "layer", meta.layer, + "isn", incomingHdr.SequenceNumber, + "its", incomingHdr.Timestamp, + "im", incomingHdr.Marker, + "osn", pkt.Header.SequenceNumber, + "ots", pkt.Header.Timestamp, + "om", pkt.Header.Marker, + "len", len(payload), + "iPID", iPID, + "iTL0PICIDX", iTL0PICIDX, + "iKEYIDX", iKEYIDX, + "iTID", iTID, + "oPID", oPID, + "oTL0PICIDX", oTL0PICIDX, + "oKEYIDX", oKEYIDX, + "oTID", oTID, + ) + } + d.pacer.Enqueue(pacer.Packet{ Header: &pkt.Header, Extensions: []pacer.ExtensionData{{ID: uint8(d.dependencyDescriptorExtID), Payload: meta.ddBytes}}, From 5ae1387c6812d82221a5f4d0aea2e9ea61cc8607 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Tue, 25 Jul 2023 19:00:43 +0530 Subject: [PATCH 311/324] Return a copy of down tracks from spreader. (#1902) As shadow copy can change, do not return as is. Also use the broacast function to broadcast up track changes to down tracks. --- pkg/sfu/downtrackspreader.go | 6 +++++- pkg/sfu/receiver.go | 16 ++++++++-------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/pkg/sfu/downtrackspreader.go b/pkg/sfu/downtrackspreader.go index 04463eb8d..8d986120e 100644 --- a/pkg/sfu/downtrackspreader.go +++ b/pkg/sfu/downtrackspreader.go @@ -34,7 +34,11 @@ func (d *DownTrackSpreader) GetDownTracks() []TrackSender { d.downTrackMu.RLock() defer d.downTrackMu.RUnlock() - return d.downTracksShadow + downTracks := make([]TrackSender, 0, len(d.downTracksShadow)) + for _, dt := range d.downTracksShadow { + downTracks = append(downTracks, dt) + } + return downTracks } func (d *DownTrackSpreader) ResetAndGetDownTracks() []TrackSender { diff --git a/pkg/sfu/receiver.go b/pkg/sfu/receiver.go index 410143844..75ba72aa5 100644 --- a/pkg/sfu/receiver.go +++ b/pkg/sfu/receiver.go @@ -408,34 +408,34 @@ func (w *WebRTCReceiver) SetMaxExpectedSpatialLayer(layer int32) { // StreamTrackerManagerListener.OnAvailableLayersChanged func (w *WebRTCReceiver) OnAvailableLayersChanged() { - for _, dt := range w.downTrackSpreader.GetDownTracks() { + w.downTrackSpreader.Broadcast(func(dt TrackSender) { dt.UpTrackLayersChange() - } + }) w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired()) } // StreamTrackerManagerListener.OnBitrateAvailabilityChanged func (w *WebRTCReceiver) OnBitrateAvailabilityChanged() { - for _, dt := range w.downTrackSpreader.GetDownTracks() { + w.downTrackSpreader.Broadcast(func(dt TrackSender) { dt.UpTrackBitrateAvailabilityChange() - } + }) } // StreamTrackerManagerListener.OnMaxPublishedLayerChanged func (w *WebRTCReceiver) OnMaxPublishedLayerChanged(maxPublishedLayer int32) { - for _, dt := range w.downTrackSpreader.GetDownTracks() { + w.downTrackSpreader.Broadcast(func(dt TrackSender) { dt.UpTrackMaxPublishedLayerChange(maxPublishedLayer) - } + }) w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired()) } // StreamTrackerManagerListener.OnMaxTemporalLayerSeenChanged func (w *WebRTCReceiver) OnMaxTemporalLayerSeenChanged(maxTemporalLayerSeen int32) { - for _, dt := range w.downTrackSpreader.GetDownTracks() { + w.downTrackSpreader.Broadcast(func(dt TrackSender) { dt.UpTrackMaxTemporalLayerSeenChange(maxTemporalLayerSeen) - } + }) w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired()) } From 34a7b6099134883f3b194f8dff8a0d8aa6ad3904 Mon Sep 17 00:00:00 2001 From: "taegu.kang" <115057375+tgkang-jocoos@users.noreply.github.com> Date: Wed, 26 Jul 2023 14:12:26 +0900 Subject: [PATCH 312/324] Update config-sample.yaml (#1904) fix typo --- config-sample.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config-sample.yaml b/config-sample.yaml index e1cd63731..bdf63e6a0 100644 --- a/config-sample.yaml +++ b/config-sample.yaml @@ -212,7 +212,7 @@ keys: # # set UDP port range for TURN relay to connect to LiveKit SFU, by default it uses a any available port # relay_range_start: 1024 # relay_range_end: 30000 -# # set external_tl to true if using a L4 load balancer to terminate TLS. when enabled, +# # set external_tls to true if using a L4 load balancer to terminate TLS. when enabled, # # LiveKit expects unencrypted traffic on tls_port, and still advertise tls_port as a TURN/TLS candidate. # external_tls: true # # needs to match tls cert domain From 0484a68342138ac6f6c07f90d8adb42875077c59 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 26 Jul 2023 13:36:58 +0530 Subject: [PATCH 313/324] Plug a couple of holes in stream transitions. (#1905) * Plug a couple of holes in stream transitions. 1. Missed negative sign meant stealing bits from other tracks was not working. 2. When a track change (mute, unmute, subscription change) cannot be allocated, explicitly pause so that stream state update happens. Refactor stream state update a bit to make it a bit cleaner. * correct comment --- pkg/sfu/forwarder.go | 4 +- pkg/sfu/forwarder_test.go | 2 +- pkg/sfu/streamallocator/streamallocator.go | 69 +++++++++++--------- pkg/sfu/streamallocator/streamstateupdate.go | 32 +++++---- pkg/sfu/streamallocator/track.go | 10 +-- 5 files changed, 67 insertions(+), 50 deletions(-) diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 02f711d06..b8343e756 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -1013,7 +1013,7 @@ func (f *Forwarder) ProvisionalAllocateGetBestWeightedTransition() VideoTransiti bandwidthDelta := int64(math.Max(float64(0), float64(existingBandwidthNeeded-f.provisional.Bitrates[s][t]))) transitionCost := int32(0) - // LK-TODO: SVC will need a different cost transition + // SVC-TODO: SVC will need a different cost transition if targetLayer.Spatial != s { transitionCost = TransitionCostSpatial } @@ -1036,7 +1036,7 @@ func (f *Forwarder) ProvisionalAllocateGetBestWeightedTransition() VideoTransiti return VideoTransition{ From: targetLayer, To: bestLayer, - BandwidthDelta: bestBandwidthDelta, + BandwidthDelta: -bestBandwidthDelta, } } diff --git a/pkg/sfu/forwarder_test.go b/pkg/sfu/forwarder_test.go index 2da1c524b..a99095839 100644 --- a/pkg/sfu/forwarder_test.go +++ b/pkg/sfu/forwarder_test.go @@ -847,7 +847,7 @@ func TestForwarderProvisionalAllocateGetBestWeightedTransition(t *testing.T) { expectedTransition := VideoTransition{ From: f.TargetLayer(), To: buffer.VideoLayer{Spatial: 2, Temporal: 0}, - BandwidthDelta: 2, + BandwidthDelta: -2, } transition := f.ProvisionalAllocateGetBestWeightedTransition() require.Equal(t, expectedTransition, transition) diff --git a/pkg/sfu/streamallocator/streamallocator.go b/pkg/sfu/streamallocator/streamallocator.go index f0c17db9e..4e12bceec 100644 --- a/pkg/sfu/streamallocator/streamallocator.go +++ b/pkg/sfu/streamallocator/streamallocator.go @@ -79,7 +79,7 @@ func (s streamAllocatorState) String() string { case streamAllocatorStateDeficient: return "DEFICIENT" default: - return fmt.Sprintf("%d", int(s)) + return fmt.Sprintf("UNKNOWN: %d", int(s)) } } @@ -696,8 +696,8 @@ func (s *StreamAllocator) handleSignalResume(event *Event) { if track != nil { update := NewStreamStateUpdate() - if track.SetPaused(false) { - update.HandleStreamingChange(false, track) + if track.SetStreamState(StreamStateActive) { + update.HandleStreamingChange(track, StreamStateActive) } s.maybeSendUpdate(update) } @@ -840,9 +840,7 @@ func (s *StreamAllocator) allocateTrack(track *Track) { if !s.params.Config.Enabled || s.state == streamAllocatorStateStable || !track.IsManaged() { update := NewStreamStateUpdate() allocation := track.AllocateOptimal(FlagAllowOvershootWhileOptimal) - if allocation.PauseReason == sfu.VideoPauseReasonBandwidth && track.SetPaused(true) { - update.HandleStreamingChange(true, track) - } + updateStreamStateChange(track, allocation, update) s.maybeSendUpdate(update) return } @@ -867,9 +865,7 @@ func (s *StreamAllocator) allocateTrack(track *Track) { allocation := track.ProvisionalAllocateCommit() update := NewStreamStateUpdate() - if allocation.PauseReason == sfu.VideoPauseReasonBandwidth && track.SetPaused(true) { - update.HandleStreamingChange(true, track) - } + updateStreamStateChange(track, allocation, update) s.maybeSendUpdate(update) s.adjustState() @@ -910,9 +906,7 @@ func (s *StreamAllocator) allocateTrack(track *Track) { // commit the tracks that contributed for _, t := range contributingTracks { allocation := t.ProvisionalAllocateCommit() - if allocation.PauseReason == sfu.VideoPauseReasonBandwidth && track.SetPaused(true) { - update.HandleStreamingChange(true, t) - } + updateStreamStateChange(t, allocation, update) } // STREAM-ALLOCATOR-TODO if got too much extra, can potentially give it to some deficient track @@ -921,9 +915,11 @@ func (s *StreamAllocator) allocateTrack(track *Track) { // commit the track that needs change if enough could be acquired or pause not allowed if !s.allowPause || bandwidthAcquired >= transition.BandwidthDelta { allocation := track.ProvisionalAllocateCommit() - if allocation.PauseReason == sfu.VideoPauseReasonBandwidth && track.SetPaused(true) { - update.HandleStreamingChange(true, track) - } + updateStreamStateChange(track, allocation, update) + } else { + // explicitly pause to ensure stream state update happens if a track coming out of mute cannot be allocated + allocation := track.Pause() + updateStreamStateChange(track, allocation, update) } s.maybeSendUpdate(update) @@ -989,9 +985,7 @@ func (s *StreamAllocator) maybeBoostDeficientTracks() { continue } - if allocation.PauseReason == sfu.VideoPauseReasonBandwidth && track.SetPaused(true) { - update.HandleStreamingChange(true, track) - } + updateStreamStateChange(track, allocation, update) availableChannelCapacity -= allocation.BandwidthDelta if availableChannelCapacity <= 0 { @@ -1053,9 +1047,7 @@ func (s *StreamAllocator) allocateAllTracks() { } allocation := track.AllocateOptimal(FlagAllowOvershootExemptTrackWhileDeficient) - if allocation.PauseReason == sfu.VideoPauseReasonBandwidth && track.SetPaused(true) { - update.HandleStreamingChange(true, track) - } + updateStreamStateChange(track, allocation, update) // STREAM-ALLOCATOR-TODO: optimistic allocation before bitrate is available will return 0. How to account for that? availableChannelCapacity -= allocation.BandwidthRequested @@ -1072,9 +1064,7 @@ func (s *StreamAllocator) allocateAllTracks() { } allocation := track.Pause() - if allocation.PauseReason == sfu.VideoPauseReasonBandwidth && track.SetPaused(true) { - update.HandleStreamingChange(true, track) - } + updateStreamStateChange(track, allocation, update) } } else { sorted := s.getSorted() @@ -1101,9 +1091,7 @@ func (s *StreamAllocator) allocateAllTracks() { for _, track := range sorted { allocation := track.ProvisionalAllocateCommit() - if allocation.PauseReason == sfu.VideoPauseReasonBandwidth && track.SetPaused(true) { - update.HandleStreamingChange(true, track) - } + updateStreamStateChange(track, allocation, update) } } @@ -1225,9 +1213,7 @@ func (s *StreamAllocator) maybeProbeWithMedia() { } update := NewStreamStateUpdate() - if allocation.PauseReason == sfu.VideoPauseReasonBandwidth && track.SetPaused(true) { - update.HandleStreamingChange(true, track) - } + updateStreamStateChange(track, allocation, update) s.maybeSendUpdate(update) s.probeController.Reset() @@ -1354,3 +1340,26 @@ func (s *StreamAllocator) getTracksHistory() map[livekit.TrackID]string { } // ------------------------------------------------ + +func updateStreamStateChange(track *Track, allocation sfu.VideoAllocation, update *StreamStateUpdate) { + updated := false + streamState := StreamStateInactive + switch allocation.PauseReason { + case sfu.VideoPauseReasonMuted: + fallthrough + + case sfu.VideoPauseReasonPubMuted: + streamState = StreamStateInactive + updated = track.SetStreamState(streamState) + + case sfu.VideoPauseReasonBandwidth: + streamState = StreamStatePaused + updated = track.SetStreamState(streamState) + } + + if updated { + update.HandleStreamingChange(track, streamState) + } +} + +// ------------------------------------------------ diff --git a/pkg/sfu/streamallocator/streamstateupdate.go b/pkg/sfu/streamallocator/streamstateupdate.go index e5b3156f7..b7bca6a2d 100644 --- a/pkg/sfu/streamallocator/streamstateupdate.go +++ b/pkg/sfu/streamallocator/streamstateupdate.go @@ -1,6 +1,8 @@ package streamallocator import ( + "fmt" + "github.com/livekit/protocol/livekit" ) @@ -9,18 +11,21 @@ import ( type StreamState int const ( - StreamStateActive StreamState = iota + StreamStateInactive StreamState = iota + StreamStateActive StreamStatePaused ) func (s StreamState) String() string { switch s { + case StreamStateInactive: + return "INACTIVE" case StreamStateActive: - return "active" + return "ACTIVE" case StreamStatePaused: - return "paused" + return "PAUSED" default: - return "unknown" + return fmt.Sprintf("UNKNOWN: %d", int(s)) } } @@ -40,19 +45,22 @@ func NewStreamStateUpdate() *StreamStateUpdate { return &StreamStateUpdate{} } -func (s *StreamStateUpdate) HandleStreamingChange(isPaused bool, track *Track) { - if isPaused { - s.StreamStates = append(s.StreamStates, &StreamStateInfo{ - ParticipantID: track.PublisherID(), - TrackID: track.ID(), - State: StreamStatePaused, - }) - } else { +func (s *StreamStateUpdate) HandleStreamingChange(track *Track, streamState StreamState) { + switch streamState { + case StreamStateInactive: + // inactive is not a notification, could get into this state because of mute + case StreamStateActive: s.StreamStates = append(s.StreamStates, &StreamStateInfo{ ParticipantID: track.PublisherID(), TrackID: track.ID(), State: StreamStateActive, }) + case StreamStatePaused: + s.StreamStates = append(s.StreamStates, &StreamStateInfo{ + ParticipantID: track.PublisherID(), + TrackID: track.ID(), + State: StreamStatePaused, + }) } } diff --git a/pkg/sfu/streamallocator/track.go b/pkg/sfu/streamallocator/track.go index 4c1a125e4..7a74d605a 100644 --- a/pkg/sfu/streamallocator/track.go +++ b/pkg/sfu/streamallocator/track.go @@ -42,7 +42,7 @@ type Track struct { isDirty bool - isPaused bool + streamState StreamState } func NewTrack( @@ -61,7 +61,7 @@ func NewTrack( nackInfos: make(map[uint16]sfu.NackInfo), nackHistory: make([]string, 0, 10), receiverReportHistory: make([]string, 0, 10), - isPaused: true, + streamState: StreamStateInactive, } t.SetPriority(0) t.SetMaxLayer(downTrack.MaxLayer()) @@ -78,12 +78,12 @@ func (t *Track) SetDirty(isDirty bool) bool { return true } -func (t *Track) SetPaused(isPaused bool) bool { - if t.isPaused == isPaused { +func (t *Track) SetStreamState(streamState StreamState) bool { + if t.streamState == streamState { return false } - t.isPaused = isPaused + t.streamState = streamState return true } From 9702d3b541d8058fceb80ae681933581250b6b65 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Wed, 26 Jul 2023 15:35:07 +0530 Subject: [PATCH 314/324] A couple of more opportunities in stream allocator. (#1906) 1. When re-allocating for a track in DEFICIENT state, try to use available headroom to accommodate change before trying to steal bits from other tracks. 2. If the changing track gives back bits (because of muting or moving to a lower layer subscription), use the returned bits to try and boost deficient track(s). --- pkg/rtc/participant.go | 1 - pkg/sfu/downtrack.go | 4 + pkg/sfu/forwarder.go | 7 ++ pkg/sfu/streamallocator/streamallocator.go | 131 ++++++++++++++------- pkg/sfu/streamallocator/track.go | 4 + 5 files changed, 106 insertions(+), 41 deletions(-) diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 587879833..fbc17ac6c 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -1894,7 +1894,6 @@ func (p *ParticipantImpl) getPendingTrack(clientId string, kind livekit.TrackTyp if pendingInfo == nil { track_loop: for cid, pti := range p.pendingTracks { - ti := pti.trackInfos[0] for _, c := range ti.Codecs { if c.Cid == clientId { diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 77226e7b5..f2c89ea0d 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -1137,6 +1137,10 @@ func (d *DownTrack) ProvisionalAllocatePrepare() { d.forwarder.ProvisionalAllocatePrepare(al, brs) } +func (d *DownTrack) ProvisionalAllocateReset() { + d.forwarder.ProvisionalAllocateReset() +} + func (d *DownTrack) ProvisionalAllocate(availableChannelCapacity int64, layers buffer.VideoLayer, allowPause bool, allowOvershoot bool) int64 { return d.forwarder.ProvisionalAllocate(availableChannelCapacity, layers, allowPause, allowOvershoot) } diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index b8343e756..c0471e7b3 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -747,6 +747,13 @@ func (f *Forwarder) ProvisionalAllocatePrepare(availableLayers []int32, Bitrates copy(f.provisional.availableLayers, availableLayers) } +func (f *Forwarder) ProvisionalAllocateReset() { + f.lock.Lock() + defer f.lock.Unlock() + + f.provisional.allocatedLayer = buffer.InvalidLayer +} + func (f *Forwarder) ProvisionalAllocate(availableChannelCapacity int64, layer buffer.VideoLayer, allowPause bool, allowOvershoot bool) int64 { f.lock.Lock() defer f.lock.Unlock() diff --git a/pkg/sfu/streamallocator/streamallocator.go b/pkg/sfu/streamallocator/streamallocator.go index 4e12bceec..7a9302dac 100644 --- a/pkg/sfu/streamallocator/streamallocator.go +++ b/pkg/sfu/streamallocator/streamallocator.go @@ -847,10 +847,22 @@ func (s *StreamAllocator) allocateTrack(track *Track) { // // In DEFICIENT state, - // 1. Find cooperative transition from track that needs allocation. - // 2. If track is currently streaming at minimum, do not do anything. - // 3. If that track is giving back bits, apply the transition. - // 4. If this track needs more, ask for best offer from others and try to use it. + // Two possibilities + // 1. Available headroom is enough to accommodate track that needs change. + // Note that the track could be muted, hence stopping. + // 2. Have to steal bits from other tracks currently streaming. + // + // For both cases, do + // a. Find cooperative transition from track that needs allocation. + // b. If track is currently streaming at minimum, do not do anything. + // c. If track is giving back bits, apply the transition and use bits given + // back to boost any deficient track(s). + // + // If track needs more bits, i.e. upward transition (may need resume or higher layer subscription), + // a. Try to allocate using existing headroom. This can be tried to get the best + // possible fit for the available headroom. + // b. If there is not enough headroom to allocate anything, ask for best offer from + // other tracks that are currently streaming and try to use it. // track.ProvisionalAllocatePrepare() transition := track.ProvisionalAllocateGetCooperativeTransition(FlagAllowOvershootWhileDeficient) @@ -869,18 +881,56 @@ func (s *StreamAllocator) allocateTrack(track *Track) { s.maybeSendUpdate(update) s.adjustState() - return - // STREAM-ALLOCATOR-TODO-START - // Should use the bits given back to start any paused track. + + // Use the bits given back to boost deficient track(s). // Note layer downgrade may actually have positive delta (i.e. consume more bits) - // because of when the measurement is done. Watch for that. - // STREAM_ALLOCATOR-TODO-END + // because of when the measurement is done. But, only available headroom after + // applying the transition will be used to boost deficient track(s). + s.maybeBoostDeficientTracks() + return } - // - // This track is currently not streaming and needs bits to start. - // Try to redistribute starting with tracks that are closest to their desired. - // + // this track is currently not streaming and needs bits to start. + // first try an allocation using available headroom + availableChannelCapacity := s.getAvailableHeadroom(false) + if availableChannelCapacity > 0 { + track.ProvisionalAllocateReset() // to reset allocation from co-operative transition above and try fresh + + bestLayer := buffer.InvalidLayer + + alloc_loop: + for spatial := int32(0); spatial <= buffer.DefaultMaxLayerSpatial; spatial++ { + for temporal := int32(0); temporal <= buffer.DefaultMaxLayerTemporal; temporal++ { + layer := buffer.VideoLayer{ + Spatial: spatial, + Temporal: temporal, + } + + usedChannelCapacity := track.ProvisionalAllocate(availableChannelCapacity, layer, s.allowPause, FlagAllowOvershootWhileDeficient) + if availableChannelCapacity < usedChannelCapacity { + break alloc_loop + } + + bestLayer = layer + } + } + + if bestLayer.IsValid() { + // found layer that can fit in available headroom + update := NewStreamStateUpdate() + allocation := track.ProvisionalAllocateCommit() + updateStreamStateChange(track, allocation, update) + s.maybeSendUpdate(update) + + s.adjustState() + return + } + + track.ProvisionalAllocateReset() + transition = track.ProvisionalAllocateGetCooperativeTransition(FlagAllowOvershootWhileDeficient) // get transition again to reset above allocation attempt using available headroom + } + + // if there is not enough headroom, try to redistribute starting with tracks that are closest to their desired. bandwidthAcquired := int64(0) var contributingTracks []*Track @@ -963,16 +1013,7 @@ func (s *StreamAllocator) onProbeDone(isNotFailing bool, isGoalReached bool) { } func (s *StreamAllocator) maybeBoostDeficientTracks() { - committedChannelCapacity := s.committedChannelCapacity - if s.params.Config.MinChannelCapacity > committedChannelCapacity { - committedChannelCapacity = s.params.Config.MinChannelCapacity - s.params.Logger.Debugw( - "stream allocator: overriding channel capacity", - "actual", s.committedChannelCapacity, - "override", committedChannelCapacity, - ) - } - availableChannelCapacity := committedChannelCapacity - s.getExpectedBandwidthUsage() + availableChannelCapacity := s.getAvailableHeadroom(false) if availableChannelCapacity <= 0 { return } @@ -1018,23 +1059,7 @@ func (s *StreamAllocator) allocateAllTracks() { // update := NewStreamStateUpdate() - availableChannelCapacity := s.committedChannelCapacity - if s.params.Config.MinChannelCapacity > availableChannelCapacity { - availableChannelCapacity = s.params.Config.MinChannelCapacity - s.params.Logger.Debugw( - "stream allocator: overriding channel capacity with min channel capacity", - "actual", s.committedChannelCapacity, - "override", availableChannelCapacity, - ) - } - if s.overriddenChannelCapacity > 0 { - availableChannelCapacity = s.overriddenChannelCapacity - s.params.Logger.Debugw( - "stream allocator: overriding channel capacity", - "actual", s.committedChannelCapacity, - "override", availableChannelCapacity, - ) - } + availableChannelCapacity := s.getAvailableChannelCapacity(true) // // This pass is to find out if there is any leftover channel capacity after allocating exempt tracks. @@ -1120,6 +1145,28 @@ func (s *StreamAllocator) maybeSendUpdate(update *StreamStateUpdate) { } } +func (s *StreamAllocator) getAvailableChannelCapacity(allowOverride bool) int64 { + availableChannelCapacity := s.committedChannelCapacity + if s.params.Config.MinChannelCapacity > availableChannelCapacity { + availableChannelCapacity = s.params.Config.MinChannelCapacity + s.params.Logger.Debugw( + "stream allocator: overriding channel capacity with min channel capacity", + "actual", s.committedChannelCapacity, + "override", availableChannelCapacity, + ) + } + if allowOverride && s.overriddenChannelCapacity > 0 { + availableChannelCapacity = s.overriddenChannelCapacity + s.params.Logger.Debugw( + "stream allocator: overriding channel capacity", + "actual", s.committedChannelCapacity, + "override", availableChannelCapacity, + ) + } + + return availableChannelCapacity +} + func (s *StreamAllocator) getExpectedBandwidthUsage() int64 { expected := int64(0) for _, track := range s.getTracks() { @@ -1129,6 +1176,10 @@ func (s *StreamAllocator) getExpectedBandwidthUsage() int64 { return expected } +func (s *StreamAllocator) getAvailableHeadroom(allowOverride bool) int64 { + return s.getAvailableChannelCapacity(allowOverride) - s.getExpectedBandwidthUsage() +} + func (s *StreamAllocator) getNackDelta() (uint32, uint32) { aggPacketDelta := uint32(0) aggRepeatedNackDelta := uint32(0) diff --git a/pkg/sfu/streamallocator/track.go b/pkg/sfu/streamallocator/track.go index 7a74d605a..63d201bc8 100644 --- a/pkg/sfu/streamallocator/track.go +++ b/pkg/sfu/streamallocator/track.go @@ -146,6 +146,10 @@ func (t *Track) ProvisionalAllocatePrepare() { t.downTrack.ProvisionalAllocatePrepare() } +func (t *Track) ProvisionalAllocateReset() { + t.downTrack.ProvisionalAllocateReset() +} + func (t *Track) ProvisionalAllocate(availableChannelCapacity int64, layer buffer.VideoLayer, allowPause bool, allowOvershoot bool) int64 { return t.downTrack.ProvisionalAllocate(availableChannelCapacity, layer, allowPause, allowOvershoot) } From 886e7b024c5096a790ddfeb11a775d63f3018564 Mon Sep 17 00:00:00 2001 From: Jonas Schell Date: Wed, 26 Jul 2023 12:13:52 +0200 Subject: [PATCH 315/324] update readme (#1907) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7554e2737..3692b94e5 100644 --- a/README.md +++ b/README.md @@ -295,7 +295,7 @@ LiveKit server is licensed under Apache License v2.0.
LiveKit Ecosystem
Client SDKsComponents · JavaScript · Rust · iOS/macOS · Android · Flutter · Unity (web) · React Native (beta)
Client SDKsComponents · JavaScript · Rust · iOS/macOS · Android · Flutter · Unity (web) · Python · React Native (beta)
Server SDKsNode.js · Golang · Ruby · Java/Kotlin · PHP (community) · Python (community)
ServicesLivekit server · Egress · Ingress
ResourcesDocs · Example apps · Cloud · Self-hosting · CLI
- + From 7a10f60be7daa0fc268751d7fbbdc15207e6af10 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 27 Jul 2023 10:04:04 +0530 Subject: [PATCH 316/324] Remove packet debug. (#1909) Not showing anything too useful. --- pkg/sfu/downtrack.go | 86 +------------------------------------------- pkg/sfu/forwarder.go | 14 -------- 2 files changed, 1 insertion(+), 99 deletions(-) diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index f2c89ea0d..74f47f84b 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -621,25 +621,11 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { return err } - var iPID, iTL0PICIDX, iKEYIDX, iTID, oPID, oTL0PICIDX, oKEYIDX, oTID int // TOOD-REMOVE-AFTER-DEBUG var payload []byte pool := PacketFactory.Get().(*[]byte) if len(tp.codecBytes) != 0 { incomingVP8, ok := extPkt.Payload.(buffer.VP8) if ok { - iPID = int(incomingVP8.PictureID) - iTL0PICIDX = int(incomingVP8.TL0PICIDX) - iKEYIDX = int(incomingVP8.KEYIDX) - iTID = int(incomingVP8.TID) - - var outgoingVP8 buffer.VP8 - if uerr := outgoingVP8.Unmarshal(append(tp.codecBytes, 0x0)); uerr == nil { - oPID = int(outgoingVP8.PictureID) - oTL0PICIDX = int(outgoingVP8.TL0PICIDX) - oKEYIDX = int(outgoingVP8.KEYIDX) - oTID = int(outgoingVP8.TID) - } - payload = d.translateVP8PacketTo(extPkt.Packet, &incomingVP8, tp.codecBytes, pool) } } @@ -668,28 +654,6 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { return err } - if d.kind == webrtc.RTPCodecTypeVideo { - d.logger.Debugw( - "packet debug (forwarding)", - "layer", layer, - "isn", extPkt.Packet.SequenceNumber, - "its", extPkt.Packet.Timestamp, - "im", extPkt.Packet.Marker, - "osn", hdr.SequenceNumber, - "ots", hdr.Timestamp, - "om", hdr.Marker, - "len", len(payload), - "iPID", iPID, - "iTL0PICIDX", iTL0PICIDX, - "iKEYIDX", iKEYIDX, - "iTID", iTID, - "oPID", oPID, - "oTL0PICIDX", oTL0PICIDX, - "oKEYIDX", oKEYIDX, - "oTID", oTID, - ) - } - d.pacer.Enqueue(pacer.Packet{ Header: hdr, Extensions: []pacer.ExtensionData{{ID: uint8(d.dependencyDescriptorExtID), Payload: tp.ddBytes}}, @@ -773,16 +737,6 @@ func (d *DownTrack) WritePaddingRTP(bytesToSend int, paddingOnMute bool, forceMa // last byte of padding has padding size including that byte payload[RTPPaddingMaxPayloadSize-1] = byte(RTPPaddingMaxPayloadSize) - if d.kind == webrtc.RTPCodecTypeVideo { - d.logger.Debugw( - "packet debug (padding)", - "osn", hdr.SequenceNumber, - "ots", hdr.Timestamp, - "om", hdr.Marker, - "len", len(payload), - ) - } - d.pacer.Enqueue(pacer.Packet{ Header: &hdr, Payload: payload, @@ -1540,19 +1494,16 @@ func (d *DownTrack) retransmitPackets(nacks []uint16) { numRepeatedNACKs++ } - var incomingHdr rtp.Header // TODO-REMOVE-AFTER-DEBUG var pkt rtp.Packet if err = pkt.Unmarshal(pktBuff[:n]); err != nil { d.logger.Errorw("unmarshalling rtp packet failed in retransmit", err) continue } - incomingHdr = pkt.Header pkt.Header.SequenceNumber = meta.targetSeqNo pkt.Header.Timestamp = meta.timestamp pkt.Header.SSRC = d.ssrc pkt.Header.PayloadType = d.payloadType - var iPID, iTL0PICIDX, iKEYIDX, iTID, oPID, oTL0PICIDX, oKEYIDX, oTID int // TOOD-REMOVE-AFTER-DEBUG var payload []byte pool := PacketFactory.Get().(*[]byte) if d.mime == "video/vp8" && len(pkt.Payload) > 0 && len(meta.codecBytes) != 0 { @@ -1563,19 +1514,6 @@ func (d *DownTrack) retransmitPackets(nacks []uint16) { continue } - iPID = int(incomingVP8.PictureID) - iTL0PICIDX = int(incomingVP8.TL0PICIDX) - iKEYIDX = int(incomingVP8.KEYIDX) - iTID = int(incomingVP8.TID) - - var outgoingVP8 buffer.VP8 - if uerr := outgoingVP8.Unmarshal(append(meta.codecBytes, 0x0)); uerr == nil { - oPID = int(outgoingVP8.PictureID) - oTL0PICIDX = int(outgoingVP8.TL0PICIDX) - oKEYIDX = int(outgoingVP8.KEYIDX) - oTID = int(outgoingVP8.TID) - } - payload = d.translateVP8PacketTo(&pkt, &incomingVP8, meta.codecBytes, pool) } if payload == nil { @@ -1583,28 +1521,6 @@ func (d *DownTrack) retransmitPackets(nacks []uint16) { copy(payload, pkt.Payload) } - if d.kind == webrtc.RTPCodecTypeVideo { - d.logger.Debugw( - "packet debug (retransmit)", - "layer", meta.layer, - "isn", incomingHdr.SequenceNumber, - "its", incomingHdr.Timestamp, - "im", incomingHdr.Marker, - "osn", pkt.Header.SequenceNumber, - "ots", pkt.Header.Timestamp, - "om", pkt.Header.Marker, - "len", len(payload), - "iPID", iPID, - "iTL0PICIDX", iTL0PICIDX, - "iKEYIDX", iKEYIDX, - "iTID", iTID, - "oPID", oPID, - "oTL0PICIDX", oTL0PICIDX, - "oKEYIDX", oKEYIDX, - "oTID", oTID, - ) - } - d.pacer.Enqueue(pacer.Packet{ Header: &pkt.Header, Extensions: []pacer.ExtensionData{{ID: uint8(d.dependencyDescriptorExtID), Payload: meta.ddBytes}}, @@ -1892,7 +1808,7 @@ func (d *DownTrack) packetSent(md interface{}, hdr *rtp.Header, payloadSize int, d.isNACKThrottled.Store(false) d.rtpStats.UpdateKeyFrame(1) d.logger.Debugw( - "forwarding key frame", + "forwarded key frame", "layer", spmd.layer, "rtpsn", hdr.SequenceNumber, "rtpts", hdr.Timestamp, diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index c0471e7b3..36edeb558 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -1984,20 +1984,6 @@ done: if !targetLayer.IsValid() { distance += (maxSeenLayer.Temporal + 1) } - // TODO-REMOVE-AFTER-DEBUG - logger.Debugw( - "distance to desired", - "maxSeenLauer", maxSeenLayer, - "availableLayers", availableLayers, - "brs", brs, - "targetLayer", targetLayer, - "maxLayer", maxLayer, - "adjustedMaxLayer", adjustedMaxLayer, - "maxAvailableSpatial", maxAvailableSpatial, - "maxAvailableTemporal", maxAvailableTemporal, - "distance", distance, - "distanceToDesired", float64(distance)/float64(maxSeenLayer.Temporal+1), - ) return float64(distance) / float64(maxSeenLayer.Temporal+1) } From ee1c23eb0260ae52e459f2626e6548991610e1d7 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 27 Jul 2023 11:48:22 +0530 Subject: [PATCH 317/324] Move congestion controller channel observer params to config (#1910) --- pkg/config/config.go | 42 ++++++++++++++++++---- pkg/sfu/streamallocator/channelobserver.go | 26 ++++++-------- pkg/sfu/streamallocator/streamallocator.go | 42 ++++++++-------------- 3 files changed, 60 insertions(+), 50 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index f13cd49e4..9de991d08 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -123,13 +123,25 @@ type CongestionControlProbeConfig struct { DurationIncreaseFactor float64 `yaml:"duration_increase_factor,omitempty"` } +type CongestionControlChannelObserverConfig struct { + EstimateRequiredSamples int `yaml:"estimate_required_samples,omitmpety"` + EstimateDownwardTrendThreshold float64 `yaml:"estimate_downward_trend_threshold,omitempty"` + EstimateCollapseThreshold time.Duration `yaml:"estimate_collapse_threshold,omitempty"` + EstimateValidityWindow time.Duration `yaml:"estimate_validity_window,omitempty"` + NackWindowMinDuration time.Duration `yaml:"nack_window_min_duration,omitempty"` + NackWindowMaxDuration time.Duration `yaml:"nack_window_max_duration,omitempty"` + NackRatioThreshold float64 `yaml:"nack_ratio_threshold,omitempty"` +} + type CongestionControlConfig struct { - Enabled bool `yaml:"enabled"` - AllowPause bool `yaml:"allow_pause"` - UseSendSideBWE bool `yaml:"send_side_bandwidth_estimation,omitempty"` - ProbeMode CongestionControlProbeMode `yaml:"padding_mode,omitempty"` - MinChannelCapacity int64 `yaml:"min_channel_capacity,omitempty"` - ProbeConfig CongestionControlProbeConfig `yaml:"probe_config,omitempty"` + Enabled bool `yaml:"enabled"` + AllowPause bool `yaml:"allow_pause"` + UseSendSideBWE bool `yaml:"send_side_bandwidth_estimation,omitempty"` + ProbeMode CongestionControlProbeMode `yaml:"padding_mode,omitempty"` + MinChannelCapacity int64 `yaml:"min_channel_capacity,omitempty"` + ProbeConfig CongestionControlProbeConfig `yaml:"probe_config,omitempty"` + ChannelObserverProbeConfig CongestionControlChannelObserverConfig `yaml:"channel_observer_probe_config,omitempty"` + ChannelObserverNonProbeConfig CongestionControlChannelObserverConfig `yaml:"channel_observer_non_probe_config,omitempty"` } type AudioConfig struct { @@ -303,6 +315,24 @@ var DefaultConfig = Config{ DurationOverflowFactor: 1.25, DurationIncreaseFactor: 1.5, }, + ChannelObserverProbeConfig: CongestionControlChannelObserverConfig{ + EstimateRequiredSamples: 3, + EstimateDownwardTrendThreshold: 0.0, + EstimateCollapseThreshold: 0, + EstimateValidityWindow: 10 * time.Second, + NackWindowMinDuration: 500 * time.Millisecond, + NackWindowMaxDuration: 1 * time.Second, + NackRatioThreshold: 0.04, + }, + ChannelObserverNonProbeConfig: CongestionControlChannelObserverConfig{ + EstimateRequiredSamples: 8, + EstimateDownwardTrendThreshold: -0.5, + EstimateCollapseThreshold: 500 * time.Millisecond, + EstimateValidityWindow: 10 * time.Second, + NackWindowMinDuration: 1 * time.Second, + NackWindowMaxDuration: 2 * time.Second, + NackRatioThreshold: 0.08, + }, }, }, Audio: AudioConfig{ diff --git a/pkg/sfu/streamallocator/channelobserver.go b/pkg/sfu/streamallocator/channelobserver.go index 94b888c57..9776f8c8a 100644 --- a/pkg/sfu/streamallocator/channelobserver.go +++ b/pkg/sfu/streamallocator/channelobserver.go @@ -2,8 +2,8 @@ package streamallocator import ( "fmt" - "time" + "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/protocol/logger" ) @@ -56,14 +56,8 @@ func (c ChannelCongestionReason) String() string { // ------------------------------------------------ type ChannelObserverParams struct { - Name string - EstimateRequiredSamples int - EstimateDownwardTrendThreshold float64 - EstimateCollapseThreshold time.Duration - EstimateValidityWindow time.Duration - NackWindowMinDuration time.Duration - NackWindowMaxDuration time.Duration - NackRatioThreshold float64 + Name string + Config config.CongestionControlChannelObserverConfig } type ChannelObserver struct { @@ -81,17 +75,17 @@ func NewChannelObserver(params ChannelObserverParams, logger logger.Logger) *Cha estimateTrend: NewTrendDetector(TrendDetectorParams{ Name: params.Name + "-estimate", Logger: logger, - RequiredSamples: params.EstimateRequiredSamples, - DownwardTrendThreshold: params.EstimateDownwardTrendThreshold, - CollapseThreshold: params.EstimateCollapseThreshold, - ValidityWindow: params.EstimateValidityWindow, + RequiredSamples: params.Config.EstimateRequiredSamples, + DownwardTrendThreshold: params.Config.EstimateDownwardTrendThreshold, + CollapseThreshold: params.Config.EstimateCollapseThreshold, + ValidityWindow: params.Config.EstimateValidityWindow, }), nackTracker: NewNackTracker(NackTrackerParams{ Name: params.Name + "-nack", Logger: logger, - WindowMinDuration: params.NackWindowMinDuration, - WindowMaxDuration: params.NackWindowMaxDuration, - RatioThreshold: params.NackRatioThreshold, + WindowMinDuration: params.Config.NackWindowMinDuration, + WindowMaxDuration: params.Config.NackWindowMaxDuration, + RatioThreshold: params.Config.NackRatioThreshold, }), } } diff --git a/pkg/sfu/streamallocator/streamallocator.go b/pkg/sfu/streamallocator/streamallocator.go index 7a9302dac..30d7d8616 100644 --- a/pkg/sfu/streamallocator/streamallocator.go +++ b/pkg/sfu/streamallocator/streamallocator.go @@ -39,32 +39,6 @@ const ( // --------------------------------------------------------------------------- -var ( - ChannelObserverParamsProbe = ChannelObserverParams{ - Name: "probe", - EstimateRequiredSamples: 3, - EstimateDownwardTrendThreshold: 0.0, - EstimateCollapseThreshold: 0, - EstimateValidityWindow: 10 * time.Second, - NackWindowMinDuration: 500 * time.Millisecond, - NackWindowMaxDuration: 1 * time.Second, - NackRatioThreshold: 0.04, - } - - ChannelObserverParamsNonProbe = ChannelObserverParams{ - Name: "non-probe", - EstimateRequiredSamples: 8, - EstimateDownwardTrendThreshold: -0.5, - EstimateCollapseThreshold: 500 * time.Millisecond, - EstimateValidityWindow: 10 * time.Second, - NackWindowMinDuration: 1 * time.Second, - NackWindowMaxDuration: 2 * time.Second, - NackRatioThreshold: 0.08, - } -) - -// --------------------------------------------------------------------------- - type streamAllocatorState int const ( @@ -1193,11 +1167,23 @@ func (s *StreamAllocator) getNackDelta() (uint32, uint32) { } func (s *StreamAllocator) newChannelObserverProbe() *ChannelObserver { - return NewChannelObserver(ChannelObserverParamsProbe, s.params.Logger) + return NewChannelObserver( + ChannelObserverParams{ + Name: "probe", + Config: s.params.Config.ChannelObserverProbeConfig, + }, + s.params.Logger, + ) } func (s *StreamAllocator) newChannelObserverNonProbe() *ChannelObserver { - return NewChannelObserver(ChannelObserverParamsNonProbe, s.params.Logger) + return NewChannelObserver( + ChannelObserverParams{ + Name: "non-probe", + Config: s.params.Config.ChannelObserverNonProbeConfig, + }, + s.params.Logger, + ) } func (s *StreamAllocator) initProbe(probeGoalDeltaBps int64) { From 38c4eba5a36131a148d61ddbf797df2699f4c2df Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 27 Jul 2023 12:02:12 +0530 Subject: [PATCH 318/324] Fix spelling (#1911) --- pkg/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 9de991d08..5a7626759 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -124,7 +124,7 @@ type CongestionControlProbeConfig struct { } type CongestionControlChannelObserverConfig struct { - EstimateRequiredSamples int `yaml:"estimate_required_samples,omitmpety"` + EstimateRequiredSamples int `yaml:"estimate_required_samples,omitempty"` EstimateDownwardTrendThreshold float64 `yaml:"estimate_downward_trend_threshold,omitempty"` EstimateCollapseThreshold time.Duration `yaml:"estimate_collapse_threshold,omitempty"` EstimateValidityWindow time.Duration `yaml:"estimate_validity_window,omitempty"` From fc7d4bd01eca3b043ae67dd2f03ac32fed6de96e Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 27 Jul 2023 16:50:18 +0530 Subject: [PATCH 319/324] E2EE trailer for server injected packets. (#1908) * Ability to use trailer with server injected frames A 32-byte trailer generated per room. Trailer appended when track encryption is enabled. * E2EE trailer for server injected packets. - Generate a 32-byte per room trailer. Too reasons for longer length o Laziness: utils generates a 32 byte string. o Longer length random string reduces chances of colliding with real data. - Trailer sent in JoinResponse - Trailer added to server injected frames (not to padding only packets) * generate * add a length check * pass trailer in as an argument --- pkg/rtc/mediatrackreceiver.go | 7 ++ pkg/rtc/mediatracksubscriptions.go | 5 ++ pkg/rtc/participant.go | 7 ++ pkg/rtc/room.go | 13 ++++ pkg/rtc/types/interfaces.go | 3 + .../typesfakes/fake_local_media_track.go | 65 +++++++++++++++++++ .../typesfakes/fake_local_participant.go | 65 +++++++++++++++++++ pkg/rtc/types/typesfakes/fake_media_track.go | 65 +++++++++++++++++++ pkg/service/roommanager.go | 1 + pkg/sfu/downtrack.go | 35 +++++++--- 10 files changed, 257 insertions(+), 9 deletions(-) diff --git a/pkg/rtc/mediatrackreceiver.go b/pkg/rtc/mediatrackreceiver.go index 8fc33f20f..a35b328a4 100644 --- a/pkg/rtc/mediatrackreceiver.go +++ b/pkg/rtc/mediatrackreceiver.go @@ -800,4 +800,11 @@ func (t *MediaTrackReceiver) GetTemporalLayerForSpatialFps(spatial int32, fps ui return buffer.DefaultMaxLayerTemporal } +func (t *MediaTrackReceiver) IsEncrypted() bool { + t.lock.RLock() + defer t.lock.RUnlock() + + return t.trackInfo.Encryption != livekit.Encryption_NONE +} + // --------------------------- diff --git a/pkg/rtc/mediatracksubscriptions.go b/pkg/rtc/mediatracksubscriptions.go index 8887176cd..2b107d9bd 100644 --- a/pkg/rtc/mediatracksubscriptions.go +++ b/pkg/rtc/mediatracksubscriptions.go @@ -98,6 +98,10 @@ func (t *MediaTrackSubscriptions) AddSubscriber(sub types.LocalParticipant, wr * for _, c := range codecs { c.RTCPFeedback = rtcpFeedback } + var trailer []byte + if t.params.MediaTrack.IsEncrypted() { + trailer = sub.GetTrailer() + } downTrack, err := sfu.NewDownTrack( codecs, wr, @@ -105,6 +109,7 @@ func (t *MediaTrackSubscriptions) AddSubscriber(sub types.LocalParticipant, wr * subscriberID, t.params.ReceiverConfig.PacketBufferSize, sub.GetPacer(), + trailer, LoggerWithTrack(sub.GetLogger(), trackID, t.params.IsRelayed), ) if err != nil { diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index fbc17ac6c..759b39523 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -68,6 +68,7 @@ type ParticipantParams struct { VideoConfig config.VideoConfig ProtocolVersion types.ProtocolVersion Telemetry telemetry.TelemetryService + Trailer []byte PLIThrottleConfig config.PLIThrottleConfig CongestionControlConfig config.CongestionControlConfig EnabledCodecs []*livekit.Codec @@ -224,6 +225,12 @@ func NewParticipant(params ParticipantParams) (*ParticipantImpl, error) { return p, nil } +func (p *ParticipantImpl) GetTrailer() []byte { + trailer := make([]byte, len(p.params.Trailer)) + copy(trailer, p.params.Trailer) + return trailer +} + func (p *ParticipantImpl) GetLogger() logger.Logger { return p.params.Logger } diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index 7602659ac..a609a25d2 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -77,6 +77,8 @@ type Room struct { leftAt atomic.Int64 closed chan struct{} + trailer []byte + onParticipantChanged func(p types.LocalParticipant) onRoomUpdated func() onClose func() @@ -111,6 +113,7 @@ func NewRoom( bufferFactory: buffer.NewFactoryOfBufferFactory(config.Receiver.PacketBufferSize), batchedUpdates: make(map[livekit.ParticipantIdentity]*livekit.ParticipantInfo), closed: make(chan struct{}), + trailer: []byte(utils.RandomSecret()), } r.protoProxy = utils.NewProtoProxy[*livekit.Room](roomUpdateInterval, r.updateProto) if r.protoRoom.EmptyTimeout == 0 { @@ -139,6 +142,15 @@ func (r *Room) ID() livekit.RoomID { return livekit.RoomID(r.protoRoom.Sid) } +func (r *Room) Trailer() []byte { + r.lock.RLock() + defer r.lock.RUnlock() + + trailer := make([]byte, len(r.trailer)) + copy(trailer, r.trailer) + return trailer +} + func (r *Room) GetParticipant(identity livekit.ParticipantIdentity) types.LocalParticipant { r.lock.RLock() defer r.lock.RUnlock() @@ -821,6 +833,7 @@ func (r *Room) createJoinResponseLocked(participant types.LocalParticipant, iceS ServerInfo: r.serverInfo, ServerVersion: r.serverInfo.Version, ServerRegion: r.serverInfo.Region, + SifTrailer: r.trailer, } } diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index eb9336121..6a426b521 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -277,6 +277,7 @@ type LocalParticipant interface { ToProtoWithVersion() (*livekit.ParticipantInfo, utils.TimedVersion) // getters + GetTrailer() []byte GetLogger() logger.Logger GetAdaptiveStream() bool ProtocolVersion() ProtocolVersion @@ -449,6 +450,8 @@ type MediaTrack interface { Receivers() []sfu.TrackReceiver ClearAllReceivers(willBeResumed bool) + + IsEncrypted() bool } //counterfeiter:generate . LocalMediaTrack diff --git a/pkg/rtc/types/typesfakes/fake_local_media_track.go b/pkg/rtc/types/typesfakes/fake_local_media_track.go index f6ee5b324..0b9a4517c 100644 --- a/pkg/rtc/types/typesfakes/fake_local_media_track.go +++ b/pkg/rtc/types/typesfakes/fake_local_media_track.go @@ -128,6 +128,16 @@ type FakeLocalMediaTrack struct { iDReturnsOnCall map[int]struct { result1 livekit.TrackID } + IsEncryptedStub func() bool + isEncryptedMutex sync.RWMutex + isEncryptedArgsForCall []struct { + } + isEncryptedReturns struct { + result1 bool + } + isEncryptedReturnsOnCall map[int]struct { + result1 bool + } IsMutedStub func() bool isMutedMutex sync.RWMutex isMutedArgsForCall []struct { @@ -928,6 +938,59 @@ func (fake *FakeLocalMediaTrack) IDReturnsOnCall(i int, result1 livekit.TrackID) }{result1} } +func (fake *FakeLocalMediaTrack) IsEncrypted() bool { + fake.isEncryptedMutex.Lock() + ret, specificReturn := fake.isEncryptedReturnsOnCall[len(fake.isEncryptedArgsForCall)] + fake.isEncryptedArgsForCall = append(fake.isEncryptedArgsForCall, struct { + }{}) + stub := fake.IsEncryptedStub + fakeReturns := fake.isEncryptedReturns + fake.recordInvocation("IsEncrypted", []interface{}{}) + fake.isEncryptedMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalMediaTrack) IsEncryptedCallCount() int { + fake.isEncryptedMutex.RLock() + defer fake.isEncryptedMutex.RUnlock() + return len(fake.isEncryptedArgsForCall) +} + +func (fake *FakeLocalMediaTrack) IsEncryptedCalls(stub func() bool) { + fake.isEncryptedMutex.Lock() + defer fake.isEncryptedMutex.Unlock() + fake.IsEncryptedStub = stub +} + +func (fake *FakeLocalMediaTrack) IsEncryptedReturns(result1 bool) { + fake.isEncryptedMutex.Lock() + defer fake.isEncryptedMutex.Unlock() + fake.IsEncryptedStub = nil + fake.isEncryptedReturns = struct { + result1 bool + }{result1} +} + +func (fake *FakeLocalMediaTrack) IsEncryptedReturnsOnCall(i int, result1 bool) { + fake.isEncryptedMutex.Lock() + defer fake.isEncryptedMutex.Unlock() + fake.IsEncryptedStub = nil + if fake.isEncryptedReturnsOnCall == nil { + fake.isEncryptedReturnsOnCall = make(map[int]struct { + result1 bool + }) + } + fake.isEncryptedReturnsOnCall[i] = struct { + result1 bool + }{result1} +} + func (fake *FakeLocalMediaTrack) IsMuted() bool { fake.isMutedMutex.Lock() ret, specificReturn := fake.isMutedReturnsOnCall[len(fake.isMutedArgsForCall)] @@ -1947,6 +2010,8 @@ func (fake *FakeLocalMediaTrack) Invocations() map[string][][]interface{} { defer fake.hasSdpCidMutex.RUnlock() fake.iDMutex.RLock() defer fake.iDMutex.RUnlock() + fake.isEncryptedMutex.RLock() + defer fake.isEncryptedMutex.RUnlock() fake.isMutedMutex.RLock() defer fake.isMutedMutex.RUnlock() fake.isOpenMutex.RLock() diff --git a/pkg/rtc/types/typesfakes/fake_local_participant.go b/pkg/rtc/types/typesfakes/fake_local_participant.go index b071877d9..6e3863a45 100644 --- a/pkg/rtc/types/typesfakes/fake_local_participant.go +++ b/pkg/rtc/types/typesfakes/fake_local_participant.go @@ -304,6 +304,16 @@ type FakeLocalParticipant struct { getSubscribedTracksReturnsOnCall map[int]struct { result1 []types.SubscribedTrack } + GetTrailerStub func() []byte + getTrailerMutex sync.RWMutex + getTrailerArgsForCall []struct { + } + getTrailerReturns struct { + result1 []byte + } + getTrailerReturnsOnCall map[int]struct { + result1 []byte + } HandleAnswerStub func(webrtc.SessionDescription) handleAnswerMutex sync.RWMutex handleAnswerArgsForCall []struct { @@ -2359,6 +2369,59 @@ func (fake *FakeLocalParticipant) GetSubscribedTracksReturnsOnCall(i int, result }{result1} } +func (fake *FakeLocalParticipant) GetTrailer() []byte { + fake.getTrailerMutex.Lock() + ret, specificReturn := fake.getTrailerReturnsOnCall[len(fake.getTrailerArgsForCall)] + fake.getTrailerArgsForCall = append(fake.getTrailerArgsForCall, struct { + }{}) + stub := fake.GetTrailerStub + fakeReturns := fake.getTrailerReturns + fake.recordInvocation("GetTrailer", []interface{}{}) + fake.getTrailerMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) GetTrailerCallCount() int { + fake.getTrailerMutex.RLock() + defer fake.getTrailerMutex.RUnlock() + return len(fake.getTrailerArgsForCall) +} + +func (fake *FakeLocalParticipant) GetTrailerCalls(stub func() []byte) { + fake.getTrailerMutex.Lock() + defer fake.getTrailerMutex.Unlock() + fake.GetTrailerStub = stub +} + +func (fake *FakeLocalParticipant) GetTrailerReturns(result1 []byte) { + fake.getTrailerMutex.Lock() + defer fake.getTrailerMutex.Unlock() + fake.GetTrailerStub = nil + fake.getTrailerReturns = struct { + result1 []byte + }{result1} +} + +func (fake *FakeLocalParticipant) GetTrailerReturnsOnCall(i int, result1 []byte) { + fake.getTrailerMutex.Lock() + defer fake.getTrailerMutex.Unlock() + fake.GetTrailerStub = nil + if fake.getTrailerReturnsOnCall == nil { + fake.getTrailerReturnsOnCall = make(map[int]struct { + result1 []byte + }) + } + fake.getTrailerReturnsOnCall[i] = struct { + result1 []byte + }{result1} +} + func (fake *FakeLocalParticipant) HandleAnswer(arg1 webrtc.SessionDescription) { fake.handleAnswerMutex.Lock() fake.handleAnswerArgsForCall = append(fake.handleAnswerArgsForCall, struct { @@ -5648,6 +5711,8 @@ func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { defer fake.getSubscribedParticipantsMutex.RUnlock() fake.getSubscribedTracksMutex.RLock() defer fake.getSubscribedTracksMutex.RUnlock() + fake.getTrailerMutex.RLock() + defer fake.getTrailerMutex.RUnlock() fake.handleAnswerMutex.RLock() defer fake.handleAnswerMutex.RUnlock() fake.handleOfferMutex.RLock() diff --git a/pkg/rtc/types/typesfakes/fake_media_track.go b/pkg/rtc/types/typesfakes/fake_media_track.go index de0e870f4..bb0c07e31 100644 --- a/pkg/rtc/types/typesfakes/fake_media_track.go +++ b/pkg/rtc/types/typesfakes/fake_media_track.go @@ -93,6 +93,16 @@ type FakeMediaTrack struct { iDReturnsOnCall map[int]struct { result1 livekit.TrackID } + IsEncryptedStub func() bool + isEncryptedMutex sync.RWMutex + isEncryptedArgsForCall []struct { + } + isEncryptedReturns struct { + result1 bool + } + isEncryptedReturnsOnCall map[int]struct { + result1 bool + } IsMutedStub func() bool isMutedMutex sync.RWMutex isMutedArgsForCall []struct { @@ -689,6 +699,59 @@ func (fake *FakeMediaTrack) IDReturnsOnCall(i int, result1 livekit.TrackID) { }{result1} } +func (fake *FakeMediaTrack) IsEncrypted() bool { + fake.isEncryptedMutex.Lock() + ret, specificReturn := fake.isEncryptedReturnsOnCall[len(fake.isEncryptedArgsForCall)] + fake.isEncryptedArgsForCall = append(fake.isEncryptedArgsForCall, struct { + }{}) + stub := fake.IsEncryptedStub + fakeReturns := fake.isEncryptedReturns + fake.recordInvocation("IsEncrypted", []interface{}{}) + fake.isEncryptedMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeMediaTrack) IsEncryptedCallCount() int { + fake.isEncryptedMutex.RLock() + defer fake.isEncryptedMutex.RUnlock() + return len(fake.isEncryptedArgsForCall) +} + +func (fake *FakeMediaTrack) IsEncryptedCalls(stub func() bool) { + fake.isEncryptedMutex.Lock() + defer fake.isEncryptedMutex.Unlock() + fake.IsEncryptedStub = stub +} + +func (fake *FakeMediaTrack) IsEncryptedReturns(result1 bool) { + fake.isEncryptedMutex.Lock() + defer fake.isEncryptedMutex.Unlock() + fake.IsEncryptedStub = nil + fake.isEncryptedReturns = struct { + result1 bool + }{result1} +} + +func (fake *FakeMediaTrack) IsEncryptedReturnsOnCall(i int, result1 bool) { + fake.isEncryptedMutex.Lock() + defer fake.isEncryptedMutex.Unlock() + fake.IsEncryptedStub = nil + if fake.isEncryptedReturnsOnCall == nil { + fake.isEncryptedReturnsOnCall = make(map[int]struct { + result1 bool + }) + } + fake.isEncryptedReturnsOnCall[i] = struct { + result1 bool + }{result1} +} + func (fake *FakeMediaTrack) IsMuted() bool { fake.isMutedMutex.Lock() ret, specificReturn := fake.isMutedReturnsOnCall[len(fake.isMutedArgsForCall)] @@ -1522,6 +1585,8 @@ func (fake *FakeMediaTrack) Invocations() map[string][][]interface{} { defer fake.getTemporalLayerForSpatialFpsMutex.RUnlock() fake.iDMutex.RLock() defer fake.iDMutex.RUnlock() + fake.isEncryptedMutex.RLock() + defer fake.isEncryptedMutex.RUnlock() fake.isMutedMutex.RLock() defer fake.isMutedMutex.RUnlock() fake.isOpenMutex.RLock() diff --git a/pkg/service/roommanager.go b/pkg/service/roommanager.go index b0cefd31c..1920ecda3 100644 --- a/pkg/service/roommanager.go +++ b/pkg/service/roommanager.go @@ -351,6 +351,7 @@ func (r *RoomManager) StartSession( VideoConfig: r.config.Video, ProtocolVersion: pv, Telemetry: r.telemetry, + Trailer: room.Trailer(), PLIThrottleConfig: r.config.RTC.PLIThrottle, CongestionControlConfig: r.config.RTC.CongestionControl, EnabledCodecs: protoRoom.EnabledCodecs, diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 74f47f84b..4147b1650 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -240,6 +240,8 @@ type DownTrack struct { maxLayerNotifierCh chan struct{} + trailer []byte + cbMu sync.RWMutex onStatsUpdate func(dt *DownTrack, stat *livekit.AnalyticsStat) onMaxSubscribedLayerChanged func(dt *DownTrack, layer int32) @@ -255,6 +257,7 @@ func NewDownTrack( subID livekit.ParticipantID, mt int, pacer pacer.Pacer, + trailer []byte, logger logger.Logger, ) (*DownTrack, error) { var kind webrtc.RTPCodecType @@ -279,6 +282,7 @@ func NewDownTrack( kind: kind, codec: codecs[0].RTPCodecCapability, pacer: pacer, + trailer: trailer, maxLayerNotifierCh: make(chan struct{}, 20), } d.forwarder = NewForwarder( @@ -1273,19 +1277,30 @@ func (d *DownTrack) writeBlankFrameRTP(duration float32, generation uint32) chan return done } +func (d *DownTrack) maybeAddTrailer(buf []byte) int { + if len(buf) < len(d.trailer) { + d.logger.Warnw("trailer too big", nil, "bufLen", len(buf), "trailerLen", len(d.trailer)) + return 0 + } + + copy(buf, d.trailer) + return len(d.trailer) +} + func (d *DownTrack) getOpusBlankFrame(_frameEndNeeded bool) ([]byte, error) { // silence frame // Used shortly after muting to ensure residual noise does not keep // generating noise at the decoder after the stream is stopped // i. e. comfort noise generation actually not producing something comfortable. - payload := make([]byte, len(OpusSilenceFrame)) + payload := make([]byte, 1000) copy(payload[0:], OpusSilenceFrame) - return payload, nil + trailerLen := d.maybeAddTrailer(payload[len(OpusSilenceFrame):]) + return payload[:len(OpusSilenceFrame)+trailerLen], nil } func (d *DownTrack) getOpusRedBlankFrame(_frameEndNeeded bool) ([]byte, error) { // primary only silence frame for opus/red, there is no need to contain redundant silent frames - payload := make([]byte, len(OpusSilenceFrame)+1) + payload := make([]byte, 1000) // primary header // 0 1 2 3 4 5 6 7 @@ -1294,7 +1309,8 @@ func (d *DownTrack) getOpusRedBlankFrame(_frameEndNeeded bool) ([]byte, error) { // +-+-+-+-+-+-+-+-+ payload[0] = opusPT copy(payload[1:], OpusSilenceFrame) - return payload, nil + trailerLen := d.maybeAddTrailer(payload[1+len(OpusSilenceFrame):]) + return payload[:1+len(OpusSilenceFrame)+trailerLen], nil } func (d *DownTrack) getVP8BlankFrame(frameEndNeeded bool) ([]byte, error) { @@ -1307,17 +1323,18 @@ func (d *DownTrack) getVP8BlankFrame(frameEndNeeded bool) ([]byte, error) { // Used even when closing out a previous frame. Looks like receivers // do not care about content (it will probably end up being an undecodable // frame, but that should be okay as there are key frames following) - payload := make([]byte, len(blankVP8)+len(VP8KeyFrame8x8)) + payload := make([]byte, 1000) copy(payload[:len(blankVP8)], blankVP8) copy(payload[len(blankVP8):], VP8KeyFrame8x8) - return payload, nil + trailerLen := d.maybeAddTrailer(payload[len(blankVP8)+len(VP8KeyFrame8x8):]) + return payload[:len(blankVP8)+len(VP8KeyFrame8x8)+trailerLen], nil } func (d *DownTrack) getH264BlankFrame(_frameEndNeeded bool) ([]byte, error) { // TODO - Jie Zeng // now use STAP-A to compose sps, pps, idr together, most decoder support packetization-mode 1. // if client only support packetization-mode 0, use single nalu unit packet - buf := make([]byte, 1462) + buf := make([]byte, 1000) offset := 0 buf[0] = 0x18 // STAP-A offset++ @@ -1327,8 +1344,8 @@ func (d *DownTrack) getH264BlankFrame(_frameEndNeeded bool) ([]byte, error) { copy(buf[offset:offset+len(payload)], payload) offset += len(payload) } - payload := buf[:offset] - return payload, nil + offset += d.maybeAddTrailer(buf[offset:]) + return buf[:offset], nil } func (d *DownTrack) handleRTCP(bytes []byte) { From 887f6580ec5319b4682ee31861320c38b3489cc1 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Thu, 27 Jul 2023 17:08:14 +0530 Subject: [PATCH 320/324] Cache marker in sequencer and use it while retransmit. (#1912) With SVC codecs, input marker and fowarded marker could be different. So, cache it in sequence and use it on retransmit. @cndderrauber - this could have affected SVC under packet loss. --- pkg/sfu/downtrack.go | 24 +++++++++++--------- pkg/sfu/sequencer.go | 12 +++++++++- pkg/sfu/sequencer_test.go | 48 ++++++++++++++++++++++----------------- 3 files changed, 51 insertions(+), 33 deletions(-) diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 4147b1650..0ad86315c 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -638,17 +638,6 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { copy(payload, extPkt.Packet.Payload) } - if d.sequencer != nil { - d.sequencer.push( - extPkt.Packet.SequenceNumber, - tp.rtp.sequenceNumber, - tp.rtp.timestamp, - int8(layer), - tp.codecBytes, - tp.ddBytes, - ) - } - hdr, err := d.getTranslatedRTPHeader(extPkt, tp) if err != nil { d.logger.Errorw("write rtp packet failed", err) @@ -658,6 +647,18 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { return err } + if d.sequencer != nil { + d.sequencer.push( + extPkt.Packet.SequenceNumber, + tp.rtp.sequenceNumber, + tp.rtp.timestamp, + hdr.Marker, + int8(layer), + tp.codecBytes, + tp.ddBytes, + ) + } + d.pacer.Enqueue(pacer.Packet{ Header: hdr, Extensions: []pacer.ExtensionData{{ID: uint8(d.dependencyDescriptorExtID), Payload: tp.ddBytes}}, @@ -1516,6 +1517,7 @@ func (d *DownTrack) retransmitPackets(nacks []uint16) { d.logger.Errorw("unmarshalling rtp packet failed in retransmit", err) continue } + pkt.Header.Marker = meta.marker pkt.Header.SequenceNumber = meta.targetSeqNo pkt.Header.Timestamp = meta.timestamp pkt.Header.SSRC = d.ssrc diff --git a/pkg/sfu/sequencer.go b/pkg/sfu/sequencer.go index 4a8b31f9c..333fd93ff 100644 --- a/pkg/sfu/sequencer.go +++ b/pkg/sfu/sequencer.go @@ -39,6 +39,8 @@ type packetMeta struct { // Modified timestamp for current associated // down track. timestamp uint32 + // Modified marker + marker bool // The last time this packet was nack requested. // Sometimes clients request the same packet more than once, so keep // track of the requested packets helps to avoid writing multiple times @@ -93,7 +95,14 @@ func (s *sequencer) setRTT(rtt uint32) { } } -func (s *sequencer) push(sn, offSn uint16, timeStamp uint32, layer int8, codecBytes []byte, ddBytes []byte) { +func (s *sequencer) push( + sn, offSn uint16, + timeStamp uint32, + marker bool, + layer int8, + codecBytes []byte, + ddBytes []byte, +) { s.Lock() defer s.Unlock() @@ -106,6 +115,7 @@ func (s *sequencer) push(sn, offSn uint16, timeStamp uint32, layer int8, codecBy sourceSeqNo: sn, targetSeqNo: offSn, timestamp: timeStamp, + marker: marker, layer: layer, codecBytes: append([]byte{}, codecBytes...), ddBytes: append([]byte{}, ddBytes...), diff --git a/pkg/sfu/sequencer_test.go b/pkg/sfu/sequencer_test.go index 94a74f791..87434bf03 100644 --- a/pkg/sfu/sequencer_test.go +++ b/pkg/sfu/sequencer_test.go @@ -15,11 +15,11 @@ func Test_sequencer(t *testing.T) { off := uint16(15) for i := uint16(1); i < 518; i++ { - seq.push(i, i+off, 123, 2, nil, nil) + seq.push(i, i+off, 123, true, 2, nil, nil) } // send the last two out-of-order - seq.push(519, 519+off, 123, 2, nil, nil) - seq.push(518, 518+off, 123, 2, nil, nil) + seq.push(519, 519+off, 123, false, 2, nil, nil) + seq.push(518, 518+off, 123, true, 2, nil, nil) time.Sleep(60 * time.Millisecond) req := []uint16{57, 58, 62, 63, 513, 514, 515, 516, 517} @@ -41,11 +41,11 @@ func Test_sequencer(t *testing.T) { require.Equal(t, val.layer, int8(2)) } - seq.push(521, 521+off, 123, 1, nil, nil) + seq.push(521, 521+off, 123, true, 1, nil, nil) m := seq.getPacketsMeta([]uint16{521 + off}) require.Equal(t, 1, len(m)) - seq.push(505, 505+off, 123, 1, nil, nil) + seq.push(505, 505+off, 123, false, 1, nil, nil) m = seq.getPacketsMeta([]uint16{505 + off}) require.Equal(t, 1, len(m)) } @@ -55,13 +55,15 @@ func Test_sequencer_getNACKSeqNo(t *testing.T) { seqNo []uint16 } type fields struct { - input []uint16 - padding []uint16 - offset uint16 - codecBytesOdd []byte + input []uint16 + padding []uint16 + offset uint16 + markerOdd bool + markerEven bool + codecBytesOdd []byte codecBytesEven []byte - ddBytesOdd []byte - ddBytesEven []byte + ddBytesOdd []byte + ddBytesEven []byte } tests := []struct { @@ -73,13 +75,15 @@ func Test_sequencer_getNACKSeqNo(t *testing.T) { { name: "Should get correct seq numbers", fields: fields{ - input: []uint16{2, 3, 4, 7, 8, 11}, - padding: []uint16{9, 10}, - offset: 5, - codecBytesOdd: []byte{1, 2, 3, 4}, + input: []uint16{2, 3, 4, 7, 8, 11}, + padding: []uint16{9, 10}, + offset: 5, + markerOdd: true, + markerEven: false, + codecBytesOdd: []byte{1, 2, 3, 4}, codecBytesEven: []byte{5, 6, 7}, - ddBytesOdd: []byte{8, 9, 10}, - ddBytesEven: []byte{11, 12}, + ddBytesOdd: []byte{8, 9, 10}, + ddBytesEven: []byte{11, 12}, }, args: args{ seqNo: []uint16{4 + 5, 5 + 5, 8 + 5, 9 + 5, 10 + 5, 11 + 5}, @@ -93,10 +97,10 @@ func Test_sequencer_getNACKSeqNo(t *testing.T) { n := newSequencer(5, 10, logger.GetLogger()) for _, i := range tt.fields.input { - if i % 2 == 0 { - n.push(i, i+tt.fields.offset, 123, 3, tt.fields.codecBytesEven, tt.fields.ddBytesEven) + if i%2 == 0 { + n.push(i, i+tt.fields.offset, 123, tt.fields.markerEven, 3, tt.fields.codecBytesEven, tt.fields.ddBytesEven) } else { - n.push(i, i+tt.fields.offset, 123, 3, tt.fields.codecBytesOdd, tt.fields.ddBytesOdd) + n.push(i, i+tt.fields.offset, 123, tt.fields.markerOdd, 3, tt.fields.codecBytesOdd, tt.fields.ddBytesOdd) } } for _, i := range tt.fields.padding { @@ -107,10 +111,12 @@ func Test_sequencer_getNACKSeqNo(t *testing.T) { var got []uint16 for _, sn := range g { got = append(got, sn.sourceSeqNo) - if sn.sourceSeqNo % 2 == 0 { + if sn.sourceSeqNo%2 == 0 { + require.Equal(t, tt.fields.markerEven, sn.marker) require.Equal(t, tt.fields.codecBytesEven, sn.codecBytes) require.Equal(t, tt.fields.ddBytesEven, sn.ddBytes) } else { + require.Equal(t, tt.fields.markerOdd, sn.marker) require.Equal(t, tt.fields.codecBytesOdd, sn.codecBytes) require.Equal(t, tt.fields.ddBytesOdd, sn.ddBytes) } From 981fb7cac7f65f0f5d4b141c2dd5ed78a5bfe2ea Mon Sep 17 00:00:00 2001 From: David Zhao Date: Thu, 27 Jul 2023 16:43:19 -0700 Subject: [PATCH 321/324] Adding license notices (#1913) * Adding license notices * remove from config --- .github/workflows/buildtest.yaml | 14 ++++++++++++++ .github/workflows/docker.yaml | 14 ++++++++++++++ .github/workflows/release.yaml | 14 ++++++++++++++ .goreleaser.yaml | 14 ++++++++++++++ Dockerfile | 14 ++++++++++++++ NOTICE | 13 +++++++++++++ bootstrap.sh | 14 ++++++++++++++ cmd/server/commands.go | 14 ++++++++++++++ cmd/server/main.go | 14 ++++++++++++++ cmd/server/main_test.go | 14 ++++++++++++++ install-livekit.sh | 14 ++++++++++++++ magefile.go | 14 ++++++++++++++ magefile_unix.go | 14 ++++++++++++++ magefile_windows.go | 14 ++++++++++++++ pkg/clientconfiguration/conf.go | 14 ++++++++++++++ pkg/clientconfiguration/conf_test.go | 14 ++++++++++++++ pkg/clientconfiguration/match.go | 14 ++++++++++++++ pkg/clientconfiguration/staticconfiguration.go | 14 ++++++++++++++ pkg/clientconfiguration/types.go | 14 ++++++++++++++ pkg/config/config.go | 14 ++++++++++++++ pkg/config/config_test.go | 14 ++++++++++++++ pkg/routing/errors.go | 14 ++++++++++++++ pkg/routing/interfaces.go | 14 ++++++++++++++ pkg/routing/localrouter.go | 14 ++++++++++++++ pkg/routing/messagechannel.go | 14 ++++++++++++++ pkg/routing/messagechannel_test.go | 14 ++++++++++++++ pkg/routing/node.go | 14 ++++++++++++++ pkg/routing/redis.go | 14 ++++++++++++++ pkg/routing/redisrouter.go | 14 ++++++++++++++ pkg/routing/selector/any.go | 14 ++++++++++++++ pkg/routing/selector/cpuload.go | 14 ++++++++++++++ pkg/routing/selector/cpuload_test.go | 14 ++++++++++++++ pkg/routing/selector/errors.go | 14 ++++++++++++++ pkg/routing/selector/interfaces.go | 14 ++++++++++++++ pkg/routing/selector/regionaware.go | 14 ++++++++++++++ pkg/routing/selector/regionaware_test.go | 14 ++++++++++++++ pkg/routing/selector/sortby_test.go | 14 ++++++++++++++ pkg/routing/selector/sysload.go | 14 ++++++++++++++ pkg/routing/selector/sysload_test.go | 14 ++++++++++++++ pkg/routing/selector/utils.go | 14 ++++++++++++++ pkg/routing/selector/utils_test.go | 14 ++++++++++++++ pkg/routing/signal.go | 14 ++++++++++++++ pkg/routing/utils.go | 14 ++++++++++++++ pkg/routing/utils_test.go | 14 ++++++++++++++ pkg/rtc/clientinfo.go | 14 ++++++++++++++ pkg/rtc/config.go | 14 ++++++++++++++ pkg/rtc/dynacastmanager.go | 14 ++++++++++++++ pkg/rtc/dynacastmanager_test.go | 14 ++++++++++++++ pkg/rtc/dynacastquality.go | 14 ++++++++++++++ pkg/rtc/errors.go | 14 ++++++++++++++ pkg/rtc/helper_test.go | 14 ++++++++++++++ pkg/rtc/mediaengine.go | 14 ++++++++++++++ pkg/rtc/mediaengine_test.go | 14 ++++++++++++++ pkg/rtc/medialossproxy.go | 14 ++++++++++++++ pkg/rtc/mediatrack.go | 14 ++++++++++++++ pkg/rtc/mediatrack_test.go | 14 ++++++++++++++ pkg/rtc/mediatrackreceiver.go | 14 ++++++++++++++ pkg/rtc/mediatracksubscriptions.go | 14 ++++++++++++++ pkg/rtc/participant.go | 14 ++++++++++++++ pkg/rtc/participant_internal_test.go | 14 ++++++++++++++ pkg/rtc/participant_sdp.go | 14 ++++++++++++++ pkg/rtc/participant_signal.go | 14 ++++++++++++++ pkg/rtc/room.go | 14 ++++++++++++++ pkg/rtc/room_egress.go | 14 ++++++++++++++ pkg/rtc/room_test.go | 14 ++++++++++++++ pkg/rtc/signalhandler.go | 14 ++++++++++++++ pkg/rtc/subscribedtrack.go | 14 ++++++++++++++ pkg/rtc/supervisor/participant_supervisor.go | 14 ++++++++++++++ pkg/rtc/supervisor/publication_monitor.go | 14 ++++++++++++++ pkg/rtc/transport.go | 14 ++++++++++++++ pkg/rtc/transport_test.go | 14 ++++++++++++++ pkg/rtc/transportmanager.go | 14 ++++++++++++++ pkg/rtc/types/interfaces.go | 14 ++++++++++++++ pkg/rtc/types/protocol_version.go | 14 ++++++++++++++ pkg/rtc/unhandlesimulcast.go | 14 ++++++++++++++ pkg/rtc/uptrackmanager.go | 14 ++++++++++++++ pkg/rtc/uptrackmanager_test.go | 14 ++++++++++++++ pkg/rtc/utils.go | 14 ++++++++++++++ pkg/rtc/utils_test.go | 14 ++++++++++++++ pkg/rtc/wrappedreceiver.go | 14 ++++++++++++++ pkg/service/auth.go | 14 ++++++++++++++ pkg/service/auth_test.go | 14 ++++++++++++++ pkg/service/egress.go | 14 ++++++++++++++ pkg/service/errors.go | 14 ++++++++++++++ pkg/service/ingress.go | 14 ++++++++++++++ pkg/service/interfaces.go | 14 ++++++++++++++ pkg/service/ioinfo.go | 14 ++++++++++++++ pkg/service/localstore.go | 14 ++++++++++++++ pkg/service/redisstore.go | 14 ++++++++++++++ pkg/service/redisstore_test.go | 14 ++++++++++++++ pkg/service/roomallocator.go | 14 ++++++++++++++ pkg/service/roomallocator_test.go | 14 ++++++++++++++ pkg/service/roommanager.go | 14 ++++++++++++++ pkg/service/roomservice.go | 14 ++++++++++++++ pkg/service/roomservice_test.go | 14 ++++++++++++++ pkg/service/rtcservice.go | 14 ++++++++++++++ pkg/service/server.go | 14 ++++++++++++++ pkg/service/signal.go | 14 ++++++++++++++ pkg/service/signal_test.go | 14 ++++++++++++++ pkg/service/turn.go | 14 ++++++++++++++ pkg/service/utils.go | 14 ++++++++++++++ pkg/service/utils_test.go | 14 ++++++++++++++ pkg/service/wire.go | 14 ++++++++++++++ pkg/service/wsprotocol.go | 14 ++++++++++++++ pkg/sfu/audio/audiolevel.go | 14 ++++++++++++++ pkg/sfu/audio/audiolevel_test.go | 14 ++++++++++++++ pkg/sfu/buffer/buffer.go | 14 ++++++++++++++ pkg/sfu/buffer/buffer_test.go | 14 ++++++++++++++ pkg/sfu/buffer/datastats.go | 14 ++++++++++++++ pkg/sfu/buffer/datastats_test.go | 14 ++++++++++++++ pkg/sfu/buffer/dependencydescriptorparser.go | 14 ++++++++++++++ pkg/sfu/buffer/factory.go | 14 ++++++++++++++ pkg/sfu/buffer/fps.go | 14 ++++++++++++++ pkg/sfu/buffer/fps_test.go | 14 ++++++++++++++ pkg/sfu/buffer/helpers.go | 14 ++++++++++++++ pkg/sfu/buffer/helpers_test.go | 14 ++++++++++++++ pkg/sfu/buffer/rtcpreader.go | 14 ++++++++++++++ pkg/sfu/buffer/rtpstats.go | 14 ++++++++++++++ pkg/sfu/buffer/rtpstats_test.go | 14 ++++++++++++++ pkg/sfu/buffer/streamstats.go | 14 ++++++++++++++ pkg/sfu/buffer/videolayer.go | 14 ++++++++++++++ pkg/sfu/buffer/videolayerutils.go | 14 ++++++++++++++ pkg/sfu/buffer/videolayerutils_test.go | 14 ++++++++++++++ pkg/sfu/codecmunger/codecmunger.go | 14 ++++++++++++++ pkg/sfu/codecmunger/null.go | 14 ++++++++++++++ pkg/sfu/codecmunger/vp8.go | 14 ++++++++++++++ pkg/sfu/codecmunger/vp8_test.go | 14 ++++++++++++++ pkg/sfu/connectionquality/connectionstats.go | 14 ++++++++++++++ pkg/sfu/connectionquality/connectionstats_test.go | 14 ++++++++++++++ pkg/sfu/connectionquality/scorer.go | 14 ++++++++++++++ pkg/sfu/dependencydescriptor/bitstreamreader.go | 14 ++++++++++++++ pkg/sfu/dependencydescriptor/bitstreamwriter.go | 14 ++++++++++++++ .../dependencydescriptorextension.go | 14 ++++++++++++++ .../dependencydescriptorextension_test.go | 14 ++++++++++++++ .../dependencydescriptorreader.go | 14 ++++++++++++++ .../dependencydescriptorwriter.go | 14 ++++++++++++++ pkg/sfu/downtrack.go | 14 ++++++++++++++ pkg/sfu/downtrackspreader.go | 14 ++++++++++++++ pkg/sfu/errors.go | 14 ++++++++++++++ pkg/sfu/forwarder.go | 14 ++++++++++++++ pkg/sfu/forwarder_test.go | 14 ++++++++++++++ pkg/sfu/helpers.go | 14 ++++++++++++++ pkg/sfu/pacer/base.go | 14 ++++++++++++++ pkg/sfu/pacer/leaky_bucket.go | 14 ++++++++++++++ pkg/sfu/pacer/no_queue.go | 14 ++++++++++++++ pkg/sfu/pacer/pacer.go | 14 ++++++++++++++ pkg/sfu/pacer/packet_time.go | 14 ++++++++++++++ pkg/sfu/pacer/pass_through.go | 14 ++++++++++++++ pkg/sfu/receiver.go | 14 ++++++++++++++ pkg/sfu/receiver_test.go | 14 ++++++++++++++ pkg/sfu/redprimaryreceiver.go | 14 ++++++++++++++ pkg/sfu/redreceiver.go | 14 ++++++++++++++ pkg/sfu/redreceiver_test.go | 14 ++++++++++++++ pkg/sfu/rtpmunger.go | 14 ++++++++++++++ pkg/sfu/rtpmunger_test.go | 14 ++++++++++++++ pkg/sfu/sequencer.go | 14 ++++++++++++++ pkg/sfu/sequencer_test.go | 14 ++++++++++++++ pkg/sfu/sfu.go | 14 ++++++++++++++ pkg/sfu/streamallocator/channelobserver.go | 14 ++++++++++++++ pkg/sfu/streamallocator/nacktracker.go | 14 ++++++++++++++ pkg/sfu/streamallocator/probe_controller.go | 14 ++++++++++++++ pkg/sfu/streamallocator/prober.go | 14 ++++++++++++++ pkg/sfu/streamallocator/ratemonitor.go | 14 ++++++++++++++ pkg/sfu/streamallocator/streamallocator.go | 14 ++++++++++++++ pkg/sfu/streamallocator/streamstateupdate.go | 14 ++++++++++++++ pkg/sfu/streamallocator/track.go | 14 ++++++++++++++ pkg/sfu/streamallocator/trenddetector.go | 14 ++++++++++++++ pkg/sfu/streamtracker/interfaces.go | 14 ++++++++++++++ pkg/sfu/streamtracker/streamtracker.go | 14 ++++++++++++++ pkg/sfu/streamtracker/streamtracker_dd.go | 14 ++++++++++++++ pkg/sfu/streamtracker/streamtracker_dd_test.go | 14 ++++++++++++++ pkg/sfu/streamtracker/streamtracker_frame.go | 14 ++++++++++++++ pkg/sfu/streamtracker/streamtracker_packet.go | 14 ++++++++++++++ pkg/sfu/streamtracker/streamtracker_packet_test.go | 14 ++++++++++++++ pkg/sfu/streamtrackermanager.go | 14 ++++++++++++++ pkg/sfu/testutils/data.go | 14 ++++++++++++++ pkg/sfu/utils/wraparound.go | 14 ++++++++++++++ pkg/sfu/utils/wraparound_test.go | 14 ++++++++++++++ pkg/sfu/videolayerselector/base.go | 14 ++++++++++++++ pkg/sfu/videolayerselector/decodetarget.go | 14 ++++++++++++++ pkg/sfu/videolayerselector/dependencydescriptor.go | 14 ++++++++++++++ .../dependencydescriptor_test.go | 14 ++++++++++++++ pkg/sfu/videolayerselector/framechain.go | 14 ++++++++++++++ pkg/sfu/videolayerselector/null.go | 14 ++++++++++++++ .../videolayerselector/selectordecisioncache.go | 14 ++++++++++++++ pkg/sfu/videolayerselector/simulcast.go | 14 ++++++++++++++ .../temporallayerselector/null.go | 14 ++++++++++++++ .../temporallayerselector/temporallayerselector.go | 14 ++++++++++++++ .../temporallayerselector/vp8.go | 14 ++++++++++++++ pkg/sfu/videolayerselector/videolayerselector.go | 14 ++++++++++++++ pkg/sfu/videolayerselector/vp9.go | 14 ++++++++++++++ pkg/telemetry/analyticsservice.go | 14 ++++++++++++++ pkg/telemetry/events.go | 14 ++++++++++++++ pkg/telemetry/events_test.go | 14 ++++++++++++++ pkg/telemetry/prometheus/node.go | 14 ++++++++++++++ pkg/telemetry/prometheus/node_linux.go | 14 ++++++++++++++ pkg/telemetry/prometheus/node_nonlinux.go | 14 ++++++++++++++ pkg/telemetry/prometheus/packets.go | 14 ++++++++++++++ pkg/telemetry/prometheus/psrpc.go | 14 ++++++++++++++ pkg/telemetry/prometheus/quality.go | 14 ++++++++++++++ pkg/telemetry/prometheus/rooms.go | 14 ++++++++++++++ pkg/telemetry/signalanddatastats.go | 14 ++++++++++++++ pkg/telemetry/stats.go | 14 ++++++++++++++ pkg/telemetry/stats_test.go | 14 ++++++++++++++ pkg/telemetry/statsconn.go | 14 ++++++++++++++ pkg/telemetry/statsworker.go | 14 ++++++++++++++ pkg/telemetry/telemetryservice.go | 14 ++++++++++++++ pkg/testutils/timeout.go | 14 ++++++++++++++ pkg/utils/math.go | 14 ++++++++++++++ pkg/utils/opsqueue.go | 14 ++++++++++++++ test/client/client.go | 14 ++++++++++++++ test/client/trackwriter.go | 14 ++++++++++++++ test/integration_helpers.go | 14 ++++++++++++++ test/multinode_roomservice_test.go | 14 ++++++++++++++ test/multinode_test.go | 14 ++++++++++++++ test/scenarios.go | 14 ++++++++++++++ test/singlenode_test.go | 14 ++++++++++++++ test/webhook_test.go | 14 ++++++++++++++ tools/tools.go | 14 ++++++++++++++ version/version.go | 14 ++++++++++++++ 220 files changed, 3079 insertions(+) create mode 100644 NOTICE diff --git a/.github/workflows/buildtest.yaml b/.github/workflows/buildtest.yaml index fcf886631..13719e47c 100644 --- a/.github/workflows/buildtest.yaml +++ b/.github/workflows/buildtest.yaml @@ -1,3 +1,17 @@ +# Copyright 2023 LiveKit, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + name: Test on: diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 03c4ea14d..e08c7f1ad 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -1,3 +1,17 @@ +# Copyright 2023 LiveKit, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + name: Release to Docker # Controls when the action will run. diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index a93876991..7e0110d0a 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,3 +1,17 @@ +# Copyright 2023 LiveKit, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + name: Release on: diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 4cd59d959..f8e54c4e1 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -1,3 +1,17 @@ +# Copyright 2023 LiveKit, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + before: hooks: - go mod tidy diff --git a/Dockerfile b/Dockerfile index a02bbb617..f35b65ada 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,17 @@ +# Copyright 2023 LiveKit, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + FROM golang:1.20-alpine as builder ARG TARGETPLATFORM diff --git a/NOTICE b/NOTICE new file mode 100644 index 000000000..692adc992 --- /dev/null +++ b/NOTICE @@ -0,0 +1,13 @@ +Copyright 2023 LiveKit, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/bootstrap.sh b/bootstrap.sh index 4d7085f29..3109ddc38 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -1,4 +1,18 @@ #!/bin/bash +# Copyright 2023 LiveKit, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + if ! command -v mage &> /dev/null then diff --git a/cmd/server/commands.go b/cmd/server/commands.go index 927668a0a..99499a50f 100644 --- a/cmd/server/commands.go +++ b/cmd/server/commands.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package main import ( diff --git a/cmd/server/main.go b/cmd/server/main.go index 42c3cf93f..58664107c 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package main import ( diff --git a/cmd/server/main_test.go b/cmd/server/main_test.go index 931692a64..dc82bf086 100644 --- a/cmd/server/main_test.go +++ b/cmd/server/main_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package main import ( diff --git a/install-livekit.sh b/install-livekit.sh index 3dd0f27ee..e0b243a81 100755 --- a/install-livekit.sh +++ b/install-livekit.sh @@ -1,4 +1,18 @@ #!/usr/bin/env bash +# Copyright 2023 LiveKit, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # LiveKit install script for Linux set -u diff --git a/magefile.go b/magefile.go index 1742de3df..1b3fdfa81 100644 --- a/magefile.go +++ b/magefile.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //go:build mage // +build mage diff --git a/magefile_unix.go b/magefile_unix.go index a84892371..186f6f4d4 100644 --- a/magefile_unix.go +++ b/magefile_unix.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //go:build mage && !windows // +build mage,!windows diff --git a/magefile_windows.go b/magefile_windows.go index 3276726bb..9e25fe722 100644 --- a/magefile_windows.go +++ b/magefile_windows.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //go:build mage // +build mage diff --git a/pkg/clientconfiguration/conf.go b/pkg/clientconfiguration/conf.go index 916cd153f..8c37b7ba4 100644 --- a/pkg/clientconfiguration/conf.go +++ b/pkg/clientconfiguration/conf.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package clientconfiguration import ( diff --git a/pkg/clientconfiguration/conf_test.go b/pkg/clientconfiguration/conf_test.go index 46f271735..093a98f19 100644 --- a/pkg/clientconfiguration/conf_test.go +++ b/pkg/clientconfiguration/conf_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package clientconfiguration import ( diff --git a/pkg/clientconfiguration/match.go b/pkg/clientconfiguration/match.go index a060d83cc..3c3514220 100644 --- a/pkg/clientconfiguration/match.go +++ b/pkg/clientconfiguration/match.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package clientconfiguration import ( diff --git a/pkg/clientconfiguration/staticconfiguration.go b/pkg/clientconfiguration/staticconfiguration.go index 83f9c2dfb..2071d9112 100644 --- a/pkg/clientconfiguration/staticconfiguration.go +++ b/pkg/clientconfiguration/staticconfiguration.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package clientconfiguration import ( diff --git a/pkg/clientconfiguration/types.go b/pkg/clientconfiguration/types.go index 5e7a8ca2f..b014518cf 100644 --- a/pkg/clientconfiguration/types.go +++ b/pkg/clientconfiguration/types.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package clientconfiguration import ( diff --git a/pkg/config/config.go b/pkg/config/config.go index 5a7626759..6b285da12 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package config import ( diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 912aacc33..0e4719fff 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package config import ( diff --git a/pkg/routing/errors.go b/pkg/routing/errors.go index 050b6b90f..4b6af0686 100644 --- a/pkg/routing/errors.go +++ b/pkg/routing/errors.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package routing import "errors" diff --git a/pkg/routing/interfaces.go b/pkg/routing/interfaces.go index 3bc54d9c9..7a450bd16 100644 --- a/pkg/routing/interfaces.go +++ b/pkg/routing/interfaces.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package routing import ( diff --git a/pkg/routing/localrouter.go b/pkg/routing/localrouter.go index 0b604a73d..b0fcbbccb 100644 --- a/pkg/routing/localrouter.go +++ b/pkg/routing/localrouter.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package routing import ( diff --git a/pkg/routing/messagechannel.go b/pkg/routing/messagechannel.go index bf914e1aa..e761f3add 100644 --- a/pkg/routing/messagechannel.go +++ b/pkg/routing/messagechannel.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package routing import ( diff --git a/pkg/routing/messagechannel_test.go b/pkg/routing/messagechannel_test.go index 25bf7fdf1..5d78c2104 100644 --- a/pkg/routing/messagechannel_test.go +++ b/pkg/routing/messagechannel_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package routing_test import ( diff --git a/pkg/routing/node.go b/pkg/routing/node.go index e39f5ac8c..16dc769a2 100644 --- a/pkg/routing/node.go +++ b/pkg/routing/node.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package routing import ( diff --git a/pkg/routing/redis.go b/pkg/routing/redis.go index 054613e61..5d81ce088 100644 --- a/pkg/routing/redis.go +++ b/pkg/routing/redis.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package routing import ( diff --git a/pkg/routing/redisrouter.go b/pkg/routing/redisrouter.go index 5ad1a5c1a..ef679261f 100644 --- a/pkg/routing/redisrouter.go +++ b/pkg/routing/redisrouter.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package routing import ( diff --git a/pkg/routing/selector/any.go b/pkg/routing/selector/any.go index 399ad4947..71f09ba87 100644 --- a/pkg/routing/selector/any.go +++ b/pkg/routing/selector/any.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package selector import ( diff --git a/pkg/routing/selector/cpuload.go b/pkg/routing/selector/cpuload.go index 61197907b..1cd04c4c0 100644 --- a/pkg/routing/selector/cpuload.go +++ b/pkg/routing/selector/cpuload.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package selector import ( diff --git a/pkg/routing/selector/cpuload_test.go b/pkg/routing/selector/cpuload_test.go index c8afd5bcc..33bca4717 100644 --- a/pkg/routing/selector/cpuload_test.go +++ b/pkg/routing/selector/cpuload_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package selector_test import ( diff --git a/pkg/routing/selector/errors.go b/pkg/routing/selector/errors.go index 9c05a269c..c011f67af 100644 --- a/pkg/routing/selector/errors.go +++ b/pkg/routing/selector/errors.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package selector import "errors" diff --git a/pkg/routing/selector/interfaces.go b/pkg/routing/selector/interfaces.go index aee027564..60d001e18 100644 --- a/pkg/routing/selector/interfaces.go +++ b/pkg/routing/selector/interfaces.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package selector import ( diff --git a/pkg/routing/selector/regionaware.go b/pkg/routing/selector/regionaware.go index 257862247..61f494f70 100644 --- a/pkg/routing/selector/regionaware.go +++ b/pkg/routing/selector/regionaware.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package selector import ( diff --git a/pkg/routing/selector/regionaware_test.go b/pkg/routing/selector/regionaware_test.go index 1645c4526..75a5bef31 100644 --- a/pkg/routing/selector/regionaware_test.go +++ b/pkg/routing/selector/regionaware_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package selector_test import ( diff --git a/pkg/routing/selector/sortby_test.go b/pkg/routing/selector/sortby_test.go index 1e391c650..31de0027b 100644 --- a/pkg/routing/selector/sortby_test.go +++ b/pkg/routing/selector/sortby_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package selector_test import ( diff --git a/pkg/routing/selector/sysload.go b/pkg/routing/selector/sysload.go index 821f092ab..909311a0e 100644 --- a/pkg/routing/selector/sysload.go +++ b/pkg/routing/selector/sysload.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package selector import ( diff --git a/pkg/routing/selector/sysload_test.go b/pkg/routing/selector/sysload_test.go index 1941d8e7c..ac7d59a25 100644 --- a/pkg/routing/selector/sysload_test.go +++ b/pkg/routing/selector/sysload_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package selector_test import ( diff --git a/pkg/routing/selector/utils.go b/pkg/routing/selector/utils.go index 7ab021374..2ba0b3876 100644 --- a/pkg/routing/selector/utils.go +++ b/pkg/routing/selector/utils.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package selector import ( diff --git a/pkg/routing/selector/utils_test.go b/pkg/routing/selector/utils_test.go index 46038be7f..4f62f6db4 100644 --- a/pkg/routing/selector/utils_test.go +++ b/pkg/routing/selector/utils_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package selector_test import ( diff --git a/pkg/routing/signal.go b/pkg/routing/signal.go index aa94b12dc..ecb1c6c7c 100644 --- a/pkg/routing/signal.go +++ b/pkg/routing/signal.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package routing import ( diff --git a/pkg/routing/utils.go b/pkg/routing/utils.go index 38a458b63..2e11fdbe2 100644 --- a/pkg/routing/utils.go +++ b/pkg/routing/utils.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package routing import ( diff --git a/pkg/routing/utils_test.go b/pkg/routing/utils_test.go index 10a21a60f..8ae1e9b4c 100644 --- a/pkg/routing/utils_test.go +++ b/pkg/routing/utils_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package routing import ( diff --git a/pkg/rtc/clientinfo.go b/pkg/rtc/clientinfo.go index efef60bef..7912968b0 100644 --- a/pkg/rtc/clientinfo.go +++ b/pkg/rtc/clientinfo.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( diff --git a/pkg/rtc/config.go b/pkg/rtc/config.go index c47bcecbc..efe2a6a1f 100644 --- a/pkg/rtc/config.go +++ b/pkg/rtc/config.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( diff --git a/pkg/rtc/dynacastmanager.go b/pkg/rtc/dynacastmanager.go index 76427798a..edaadb4f2 100644 --- a/pkg/rtc/dynacastmanager.go +++ b/pkg/rtc/dynacastmanager.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( diff --git a/pkg/rtc/dynacastmanager_test.go b/pkg/rtc/dynacastmanager_test.go index ad7b065bb..ee1c97c70 100644 --- a/pkg/rtc/dynacastmanager_test.go +++ b/pkg/rtc/dynacastmanager_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( diff --git a/pkg/rtc/dynacastquality.go b/pkg/rtc/dynacastquality.go index 5be1e9976..7f0f90495 100644 --- a/pkg/rtc/dynacastquality.go +++ b/pkg/rtc/dynacastquality.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( diff --git a/pkg/rtc/errors.go b/pkg/rtc/errors.go index 20c41acb9..383afde0d 100644 --- a/pkg/rtc/errors.go +++ b/pkg/rtc/errors.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import "errors" diff --git a/pkg/rtc/helper_test.go b/pkg/rtc/helper_test.go index c26e7aebc..c47b5a658 100644 --- a/pkg/rtc/helper_test.go +++ b/pkg/rtc/helper_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( diff --git a/pkg/rtc/mediaengine.go b/pkg/rtc/mediaengine.go index 4178dfb15..c2866697b 100644 --- a/pkg/rtc/mediaengine.go +++ b/pkg/rtc/mediaengine.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( diff --git a/pkg/rtc/mediaengine_test.go b/pkg/rtc/mediaengine_test.go index 19f81529c..353f6f966 100644 --- a/pkg/rtc/mediaengine_test.go +++ b/pkg/rtc/mediaengine_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( diff --git a/pkg/rtc/medialossproxy.go b/pkg/rtc/medialossproxy.go index 4b25d479a..0ac54b7e6 100644 --- a/pkg/rtc/medialossproxy.go +++ b/pkg/rtc/medialossproxy.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( diff --git a/pkg/rtc/mediatrack.go b/pkg/rtc/mediatrack.go index 4095013fa..72f0a6f3a 100644 --- a/pkg/rtc/mediatrack.go +++ b/pkg/rtc/mediatrack.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( diff --git a/pkg/rtc/mediatrack_test.go b/pkg/rtc/mediatrack_test.go index 2eb172fdc..9a0587447 100644 --- a/pkg/rtc/mediatrack_test.go +++ b/pkg/rtc/mediatrack_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( diff --git a/pkg/rtc/mediatrackreceiver.go b/pkg/rtc/mediatrackreceiver.go index a35b328a4..cff226ca0 100644 --- a/pkg/rtc/mediatrackreceiver.go +++ b/pkg/rtc/mediatrackreceiver.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( diff --git a/pkg/rtc/mediatracksubscriptions.go b/pkg/rtc/mediatracksubscriptions.go index 2b107d9bd..51499a5ba 100644 --- a/pkg/rtc/mediatracksubscriptions.go +++ b/pkg/rtc/mediatracksubscriptions.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 759b39523..aea4c9e94 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( diff --git a/pkg/rtc/participant_internal_test.go b/pkg/rtc/participant_internal_test.go index 8dce027b5..55f773879 100644 --- a/pkg/rtc/participant_internal_test.go +++ b/pkg/rtc/participant_internal_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( diff --git a/pkg/rtc/participant_sdp.go b/pkg/rtc/participant_sdp.go index 427c846a5..01a50b6b0 100644 --- a/pkg/rtc/participant_sdp.go +++ b/pkg/rtc/participant_sdp.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( diff --git a/pkg/rtc/participant_signal.go b/pkg/rtc/participant_signal.go index 305e920e5..a9ae4dfab 100644 --- a/pkg/rtc/participant_signal.go +++ b/pkg/rtc/participant_signal.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index a609a25d2..c0fce9d25 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( diff --git a/pkg/rtc/room_egress.go b/pkg/rtc/room_egress.go index 989270713..d632a007b 100644 --- a/pkg/rtc/room_egress.go +++ b/pkg/rtc/room_egress.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( diff --git a/pkg/rtc/room_test.go b/pkg/rtc/room_test.go index 9afba5250..0fb46b0af 100644 --- a/pkg/rtc/room_test.go +++ b/pkg/rtc/room_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( diff --git a/pkg/rtc/signalhandler.go b/pkg/rtc/signalhandler.go index c29baf10f..3c90209b6 100644 --- a/pkg/rtc/signalhandler.go +++ b/pkg/rtc/signalhandler.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( diff --git a/pkg/rtc/subscribedtrack.go b/pkg/rtc/subscribedtrack.go index 7bb8ab294..5a83e0a1f 100644 --- a/pkg/rtc/subscribedtrack.go +++ b/pkg/rtc/subscribedtrack.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( diff --git a/pkg/rtc/supervisor/participant_supervisor.go b/pkg/rtc/supervisor/participant_supervisor.go index 99126739c..1dc6b9d32 100644 --- a/pkg/rtc/supervisor/participant_supervisor.go +++ b/pkg/rtc/supervisor/participant_supervisor.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package supervisor import ( diff --git a/pkg/rtc/supervisor/publication_monitor.go b/pkg/rtc/supervisor/publication_monitor.go index f7af4ed23..c5c61c557 100644 --- a/pkg/rtc/supervisor/publication_monitor.go +++ b/pkg/rtc/supervisor/publication_monitor.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package supervisor import ( diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index 9b47f33d5..c3f314eec 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( diff --git a/pkg/rtc/transport_test.go b/pkg/rtc/transport_test.go index e7365531e..eb59df779 100644 --- a/pkg/rtc/transport_test.go +++ b/pkg/rtc/transport_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( diff --git a/pkg/rtc/transportmanager.go b/pkg/rtc/transportmanager.go index 78dd8dee2..3f698ebe1 100644 --- a/pkg/rtc/transportmanager.go +++ b/pkg/rtc/transportmanager.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index 6a426b521..6431dc661 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package types import ( diff --git a/pkg/rtc/types/protocol_version.go b/pkg/rtc/types/protocol_version.go index 4c0257734..93449ae79 100644 --- a/pkg/rtc/types/protocol_version.go +++ b/pkg/rtc/types/protocol_version.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package types type ProtocolVersion int diff --git a/pkg/rtc/unhandlesimulcast.go b/pkg/rtc/unhandlesimulcast.go index 0fd611443..568c7dc1b 100644 --- a/pkg/rtc/unhandlesimulcast.go +++ b/pkg/rtc/unhandlesimulcast.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( diff --git a/pkg/rtc/uptrackmanager.go b/pkg/rtc/uptrackmanager.go index 10e75227c..961134c83 100644 --- a/pkg/rtc/uptrackmanager.go +++ b/pkg/rtc/uptrackmanager.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( diff --git a/pkg/rtc/uptrackmanager_test.go b/pkg/rtc/uptrackmanager_test.go index e75d92881..46a8c96cf 100644 --- a/pkg/rtc/uptrackmanager_test.go +++ b/pkg/rtc/uptrackmanager_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( diff --git a/pkg/rtc/utils.go b/pkg/rtc/utils.go index 0b49d02af..b9b532d8b 100644 --- a/pkg/rtc/utils.go +++ b/pkg/rtc/utils.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( diff --git a/pkg/rtc/utils_test.go b/pkg/rtc/utils_test.go index 9e13e8fe2..813f910d8 100644 --- a/pkg/rtc/utils_test.go +++ b/pkg/rtc/utils_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( diff --git a/pkg/rtc/wrappedreceiver.go b/pkg/rtc/wrappedreceiver.go index 7028084fb..593a62487 100644 --- a/pkg/rtc/wrappedreceiver.go +++ b/pkg/rtc/wrappedreceiver.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rtc import ( diff --git a/pkg/service/auth.go b/pkg/service/auth.go index c83cfbb1a..af940e83a 100644 --- a/pkg/service/auth.go +++ b/pkg/service/auth.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service import ( diff --git a/pkg/service/auth_test.go b/pkg/service/auth_test.go index f61d9fbc0..8a70d6a18 100644 --- a/pkg/service/auth_test.go +++ b/pkg/service/auth_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service_test import ( diff --git a/pkg/service/egress.go b/pkg/service/egress.go index 29ce6f07a..1ae4b5a69 100644 --- a/pkg/service/egress.go +++ b/pkg/service/egress.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service import ( diff --git a/pkg/service/errors.go b/pkg/service/errors.go index b856579da..a1473c3a2 100644 --- a/pkg/service/errors.go +++ b/pkg/service/errors.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service import ( diff --git a/pkg/service/ingress.go b/pkg/service/ingress.go index 5fa7990ee..45c0bb789 100644 --- a/pkg/service/ingress.go +++ b/pkg/service/ingress.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service import ( diff --git a/pkg/service/interfaces.go b/pkg/service/interfaces.go index e3b1bcef0..36da68fc5 100644 --- a/pkg/service/interfaces.go +++ b/pkg/service/interfaces.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service import ( diff --git a/pkg/service/ioinfo.go b/pkg/service/ioinfo.go index 612020916..b8fedd8ac 100644 --- a/pkg/service/ioinfo.go +++ b/pkg/service/ioinfo.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service import ( diff --git a/pkg/service/localstore.go b/pkg/service/localstore.go index 53518022b..a651c24a0 100644 --- a/pkg/service/localstore.go +++ b/pkg/service/localstore.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service import ( diff --git a/pkg/service/redisstore.go b/pkg/service/redisstore.go index 6656757da..e8fe4ecc9 100644 --- a/pkg/service/redisstore.go +++ b/pkg/service/redisstore.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service import ( diff --git a/pkg/service/redisstore_test.go b/pkg/service/redisstore_test.go index 949480d28..32e550378 100644 --- a/pkg/service/redisstore_test.go +++ b/pkg/service/redisstore_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service_test import ( diff --git a/pkg/service/roomallocator.go b/pkg/service/roomallocator.go index 8c82f6754..9b38b9768 100644 --- a/pkg/service/roomallocator.go +++ b/pkg/service/roomallocator.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service import ( diff --git a/pkg/service/roomallocator_test.go b/pkg/service/roomallocator_test.go index ddf305d66..4397e22c4 100644 --- a/pkg/service/roomallocator_test.go +++ b/pkg/service/roomallocator_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service_test import ( diff --git a/pkg/service/roommanager.go b/pkg/service/roommanager.go index 1920ecda3..fe4008cb3 100644 --- a/pkg/service/roommanager.go +++ b/pkg/service/roommanager.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service import ( diff --git a/pkg/service/roomservice.go b/pkg/service/roomservice.go index e3859d4b3..3e8997587 100644 --- a/pkg/service/roomservice.go +++ b/pkg/service/roomservice.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service import ( diff --git a/pkg/service/roomservice_test.go b/pkg/service/roomservice_test.go index a8706bd64..a7433a090 100644 --- a/pkg/service/roomservice_test.go +++ b/pkg/service/roomservice_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service_test import ( diff --git a/pkg/service/rtcservice.go b/pkg/service/rtcservice.go index 32dfca410..5259622bc 100644 --- a/pkg/service/rtcservice.go +++ b/pkg/service/rtcservice.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service import ( diff --git a/pkg/service/server.go b/pkg/service/server.go index 390432116..69191ff81 100644 --- a/pkg/service/server.go +++ b/pkg/service/server.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service import ( diff --git a/pkg/service/signal.go b/pkg/service/signal.go index 817344914..862d79e44 100644 --- a/pkg/service/signal.go +++ b/pkg/service/signal.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service import ( diff --git a/pkg/service/signal_test.go b/pkg/service/signal_test.go index ec83c5c3e..3e202636e 100644 --- a/pkg/service/signal_test.go +++ b/pkg/service/signal_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service import ( diff --git a/pkg/service/turn.go b/pkg/service/turn.go index 325eca32b..08d971fb4 100644 --- a/pkg/service/turn.go +++ b/pkg/service/turn.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service import ( diff --git a/pkg/service/utils.go b/pkg/service/utils.go index 42f0c9c41..32076455f 100644 --- a/pkg/service/utils.go +++ b/pkg/service/utils.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service import ( diff --git a/pkg/service/utils_test.go b/pkg/service/utils_test.go index d46cb0707..99c19ac35 100644 --- a/pkg/service/utils_test.go +++ b/pkg/service/utils_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service_test import ( diff --git a/pkg/service/wire.go b/pkg/service/wire.go index bb8451e05..955f2ecef 100644 --- a/pkg/service/wire.go +++ b/pkg/service/wire.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //go:build wireinject // +build wireinject diff --git a/pkg/service/wsprotocol.go b/pkg/service/wsprotocol.go index 06e971719..50a4dc2fb 100644 --- a/pkg/service/wsprotocol.go +++ b/pkg/service/wsprotocol.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package service import ( diff --git a/pkg/sfu/audio/audiolevel.go b/pkg/sfu/audio/audiolevel.go index 15b549c9c..7e834fda7 100644 --- a/pkg/sfu/audio/audiolevel.go +++ b/pkg/sfu/audio/audiolevel.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package audio import ( diff --git a/pkg/sfu/audio/audiolevel_test.go b/pkg/sfu/audio/audiolevel_test.go index aadd4ca29..8b8f03eba 100644 --- a/pkg/sfu/audio/audiolevel_test.go +++ b/pkg/sfu/audio/audiolevel_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package audio import ( diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go index b242662dd..4ed4c0a8b 100644 --- a/pkg/sfu/buffer/buffer.go +++ b/pkg/sfu/buffer/buffer.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package buffer import ( diff --git a/pkg/sfu/buffer/buffer_test.go b/pkg/sfu/buffer/buffer_test.go index 1e22bd991..7f1e97357 100644 --- a/pkg/sfu/buffer/buffer_test.go +++ b/pkg/sfu/buffer/buffer_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package buffer import ( diff --git a/pkg/sfu/buffer/datastats.go b/pkg/sfu/buffer/datastats.go index 880e172af..515341cc0 100644 --- a/pkg/sfu/buffer/datastats.go +++ b/pkg/sfu/buffer/datastats.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package buffer import ( diff --git a/pkg/sfu/buffer/datastats_test.go b/pkg/sfu/buffer/datastats_test.go index f2369b7c5..5803fe9d9 100644 --- a/pkg/sfu/buffer/datastats_test.go +++ b/pkg/sfu/buffer/datastats_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package buffer import ( diff --git a/pkg/sfu/buffer/dependencydescriptorparser.go b/pkg/sfu/buffer/dependencydescriptorparser.go index 71b4aac2e..17ed0732d 100644 --- a/pkg/sfu/buffer/dependencydescriptorparser.go +++ b/pkg/sfu/buffer/dependencydescriptorparser.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package buffer import ( diff --git a/pkg/sfu/buffer/factory.go b/pkg/sfu/buffer/factory.go index d3436a6ca..d0a9979f8 100644 --- a/pkg/sfu/buffer/factory.go +++ b/pkg/sfu/buffer/factory.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package buffer import ( diff --git a/pkg/sfu/buffer/fps.go b/pkg/sfu/buffer/fps.go index 518b8e38c..0ba7c5764 100644 --- a/pkg/sfu/buffer/fps.go +++ b/pkg/sfu/buffer/fps.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package buffer import ( diff --git a/pkg/sfu/buffer/fps_test.go b/pkg/sfu/buffer/fps_test.go index b090d2770..4a85d2808 100644 --- a/pkg/sfu/buffer/fps_test.go +++ b/pkg/sfu/buffer/fps_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package buffer import ( diff --git a/pkg/sfu/buffer/helpers.go b/pkg/sfu/buffer/helpers.go index 01bb33403..f52464b4c 100644 --- a/pkg/sfu/buffer/helpers.go +++ b/pkg/sfu/buffer/helpers.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package buffer import ( diff --git a/pkg/sfu/buffer/helpers_test.go b/pkg/sfu/buffer/helpers_test.go index 6ce0ad860..378bfbebf 100644 --- a/pkg/sfu/buffer/helpers_test.go +++ b/pkg/sfu/buffer/helpers_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package buffer import ( diff --git a/pkg/sfu/buffer/rtcpreader.go b/pkg/sfu/buffer/rtcpreader.go index 5d322abcd..1f8f365df 100644 --- a/pkg/sfu/buffer/rtcpreader.go +++ b/pkg/sfu/buffer/rtcpreader.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package buffer import ( diff --git a/pkg/sfu/buffer/rtpstats.go b/pkg/sfu/buffer/rtpstats.go index 075fe72fe..cd9c96066 100644 --- a/pkg/sfu/buffer/rtpstats.go +++ b/pkg/sfu/buffer/rtpstats.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package buffer import ( diff --git a/pkg/sfu/buffer/rtpstats_test.go b/pkg/sfu/buffer/rtpstats_test.go index 70c852e7b..c5c4138ba 100644 --- a/pkg/sfu/buffer/rtpstats_test.go +++ b/pkg/sfu/buffer/rtpstats_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package buffer import ( diff --git a/pkg/sfu/buffer/streamstats.go b/pkg/sfu/buffer/streamstats.go index 04c02a65b..cdd8e1333 100644 --- a/pkg/sfu/buffer/streamstats.go +++ b/pkg/sfu/buffer/streamstats.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package buffer type StreamStatsWithLayers struct { diff --git a/pkg/sfu/buffer/videolayer.go b/pkg/sfu/buffer/videolayer.go index eedf0b2c8..761cc1c13 100644 --- a/pkg/sfu/buffer/videolayer.go +++ b/pkg/sfu/buffer/videolayer.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package buffer import "fmt" diff --git a/pkg/sfu/buffer/videolayerutils.go b/pkg/sfu/buffer/videolayerutils.go index 32abdb45d..b18c83d84 100644 --- a/pkg/sfu/buffer/videolayerutils.go +++ b/pkg/sfu/buffer/videolayerutils.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package buffer import ( diff --git a/pkg/sfu/buffer/videolayerutils_test.go b/pkg/sfu/buffer/videolayerutils_test.go index b57bb2de0..bb103f72c 100644 --- a/pkg/sfu/buffer/videolayerutils_test.go +++ b/pkg/sfu/buffer/videolayerutils_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package buffer import ( diff --git a/pkg/sfu/codecmunger/codecmunger.go b/pkg/sfu/codecmunger/codecmunger.go index eec4f2437..850cecb8d 100644 --- a/pkg/sfu/codecmunger/codecmunger.go +++ b/pkg/sfu/codecmunger/codecmunger.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package codecmunger import ( diff --git a/pkg/sfu/codecmunger/null.go b/pkg/sfu/codecmunger/null.go index f9c327a2d..32e3d9ee9 100644 --- a/pkg/sfu/codecmunger/null.go +++ b/pkg/sfu/codecmunger/null.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package codecmunger import ( diff --git a/pkg/sfu/codecmunger/vp8.go b/pkg/sfu/codecmunger/vp8.go index dbe8f0665..bb270387d 100644 --- a/pkg/sfu/codecmunger/vp8.go +++ b/pkg/sfu/codecmunger/vp8.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package codecmunger import ( diff --git a/pkg/sfu/codecmunger/vp8_test.go b/pkg/sfu/codecmunger/vp8_test.go index 93c27086c..c72965189 100644 --- a/pkg/sfu/codecmunger/vp8_test.go +++ b/pkg/sfu/codecmunger/vp8_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package codecmunger import ( diff --git a/pkg/sfu/connectionquality/connectionstats.go b/pkg/sfu/connectionquality/connectionstats.go index 4c1c9e545..565eccabb 100644 --- a/pkg/sfu/connectionquality/connectionstats.go +++ b/pkg/sfu/connectionquality/connectionstats.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package connectionquality import ( diff --git a/pkg/sfu/connectionquality/connectionstats_test.go b/pkg/sfu/connectionquality/connectionstats_test.go index ff3474cda..a8aec4b81 100644 --- a/pkg/sfu/connectionquality/connectionstats_test.go +++ b/pkg/sfu/connectionquality/connectionstats_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package connectionquality import ( diff --git a/pkg/sfu/connectionquality/scorer.go b/pkg/sfu/connectionquality/scorer.go index 5391406a9..340f32368 100644 --- a/pkg/sfu/connectionquality/scorer.go +++ b/pkg/sfu/connectionquality/scorer.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package connectionquality import ( diff --git a/pkg/sfu/dependencydescriptor/bitstreamreader.go b/pkg/sfu/dependencydescriptor/bitstreamreader.go index 1ce60e390..62f4d100c 100644 --- a/pkg/sfu/dependencydescriptor/bitstreamreader.go +++ b/pkg/sfu/dependencydescriptor/bitstreamreader.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package dependencydescriptor import ( diff --git a/pkg/sfu/dependencydescriptor/bitstreamwriter.go b/pkg/sfu/dependencydescriptor/bitstreamwriter.go index 0461792aa..1e5ffacbe 100644 --- a/pkg/sfu/dependencydescriptor/bitstreamwriter.go +++ b/pkg/sfu/dependencydescriptor/bitstreamwriter.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package dependencydescriptor import ( diff --git a/pkg/sfu/dependencydescriptor/dependencydescriptorextension.go b/pkg/sfu/dependencydescriptor/dependencydescriptorextension.go index 7493f1bfc..b573d7d60 100644 --- a/pkg/sfu/dependencydescriptor/dependencydescriptorextension.go +++ b/pkg/sfu/dependencydescriptor/dependencydescriptorextension.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package dependencydescriptor import ( diff --git a/pkg/sfu/dependencydescriptor/dependencydescriptorextension_test.go b/pkg/sfu/dependencydescriptor/dependencydescriptorextension_test.go index 6580c8842..95c3aba35 100644 --- a/pkg/sfu/dependencydescriptor/dependencydescriptorextension_test.go +++ b/pkg/sfu/dependencydescriptor/dependencydescriptorextension_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package dependencydescriptor import ( diff --git a/pkg/sfu/dependencydescriptor/dependencydescriptorreader.go b/pkg/sfu/dependencydescriptor/dependencydescriptorreader.go index 68f00b863..04ae1ce7c 100644 --- a/pkg/sfu/dependencydescriptor/dependencydescriptorreader.go +++ b/pkg/sfu/dependencydescriptor/dependencydescriptorreader.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package dependencydescriptor import ( diff --git a/pkg/sfu/dependencydescriptor/dependencydescriptorwriter.go b/pkg/sfu/dependencydescriptor/dependencydescriptorwriter.go index 0c4d5164f..37ce7bcf8 100644 --- a/pkg/sfu/dependencydescriptor/dependencydescriptorwriter.go +++ b/pkg/sfu/dependencydescriptor/dependencydescriptorwriter.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package dependencydescriptor import ( diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 0ad86315c..16c8f4456 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package sfu import ( diff --git a/pkg/sfu/downtrackspreader.go b/pkg/sfu/downtrackspreader.go index 8d986120e..dd7ac59c6 100644 --- a/pkg/sfu/downtrackspreader.go +++ b/pkg/sfu/downtrackspreader.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package sfu import ( diff --git a/pkg/sfu/errors.go b/pkg/sfu/errors.go index 22831db4c..10743808c 100644 --- a/pkg/sfu/errors.go +++ b/pkg/sfu/errors.go @@ -1 +1,15 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package sfu diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 36edeb558..9839fd76c 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package sfu import ( diff --git a/pkg/sfu/forwarder_test.go b/pkg/sfu/forwarder_test.go index a99095839..935009792 100644 --- a/pkg/sfu/forwarder_test.go +++ b/pkg/sfu/forwarder_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package sfu import ( diff --git a/pkg/sfu/helpers.go b/pkg/sfu/helpers.go index b6dcafd9f..1f4101910 100644 --- a/pkg/sfu/helpers.go +++ b/pkg/sfu/helpers.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package sfu import ( diff --git a/pkg/sfu/pacer/base.go b/pkg/sfu/pacer/base.go index 4b5da20c9..83e0efc43 100644 --- a/pkg/sfu/pacer/base.go +++ b/pkg/sfu/pacer/base.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pacer import ( diff --git a/pkg/sfu/pacer/leaky_bucket.go b/pkg/sfu/pacer/leaky_bucket.go index 6e16f7f4e..9ac2a1350 100644 --- a/pkg/sfu/pacer/leaky_bucket.go +++ b/pkg/sfu/pacer/leaky_bucket.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pacer import ( diff --git a/pkg/sfu/pacer/no_queue.go b/pkg/sfu/pacer/no_queue.go index b34b994ae..927236394 100644 --- a/pkg/sfu/pacer/no_queue.go +++ b/pkg/sfu/pacer/no_queue.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pacer import ( diff --git a/pkg/sfu/pacer/pacer.go b/pkg/sfu/pacer/pacer.go index 2288d8639..48b20efea 100644 --- a/pkg/sfu/pacer/pacer.go +++ b/pkg/sfu/pacer/pacer.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pacer import ( diff --git a/pkg/sfu/pacer/packet_time.go b/pkg/sfu/pacer/packet_time.go index 3dce57e3a..eac3866b4 100644 --- a/pkg/sfu/pacer/packet_time.go +++ b/pkg/sfu/pacer/packet_time.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pacer import ( diff --git a/pkg/sfu/pacer/pass_through.go b/pkg/sfu/pacer/pass_through.go index ccbefbd61..8c33d808f 100644 --- a/pkg/sfu/pacer/pass_through.go +++ b/pkg/sfu/pacer/pass_through.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pacer import ( diff --git a/pkg/sfu/receiver.go b/pkg/sfu/receiver.go index 75ba72aa5..46508acb5 100644 --- a/pkg/sfu/receiver.go +++ b/pkg/sfu/receiver.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package sfu import ( diff --git a/pkg/sfu/receiver_test.go b/pkg/sfu/receiver_test.go index e451c3fe3..a1f475e2a 100644 --- a/pkg/sfu/receiver_test.go +++ b/pkg/sfu/receiver_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package sfu import ( diff --git a/pkg/sfu/redprimaryreceiver.go b/pkg/sfu/redprimaryreceiver.go index 5f6fd8183..eb0965627 100644 --- a/pkg/sfu/redprimaryreceiver.go +++ b/pkg/sfu/redprimaryreceiver.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package sfu import ( diff --git a/pkg/sfu/redreceiver.go b/pkg/sfu/redreceiver.go index 7a9adc700..57cf49e8b 100644 --- a/pkg/sfu/redreceiver.go +++ b/pkg/sfu/redreceiver.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package sfu import ( diff --git a/pkg/sfu/redreceiver_test.go b/pkg/sfu/redreceiver_test.go index 72d8d9a0e..2aae30182 100644 --- a/pkg/sfu/redreceiver_test.go +++ b/pkg/sfu/redreceiver_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package sfu import ( diff --git a/pkg/sfu/rtpmunger.go b/pkg/sfu/rtpmunger.go index 452019507..52faa8124 100644 --- a/pkg/sfu/rtpmunger.go +++ b/pkg/sfu/rtpmunger.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package sfu import ( diff --git a/pkg/sfu/rtpmunger_test.go b/pkg/sfu/rtpmunger_test.go index b4d764ca3..63a611a1f 100644 --- a/pkg/sfu/rtpmunger_test.go +++ b/pkg/sfu/rtpmunger_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package sfu import ( diff --git a/pkg/sfu/sequencer.go b/pkg/sfu/sequencer.go index 333fd93ff..e0f89d2c7 100644 --- a/pkg/sfu/sequencer.go +++ b/pkg/sfu/sequencer.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package sfu import ( diff --git a/pkg/sfu/sequencer_test.go b/pkg/sfu/sequencer_test.go index 87434bf03..a2303742d 100644 --- a/pkg/sfu/sequencer_test.go +++ b/pkg/sfu/sequencer_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package sfu import ( diff --git a/pkg/sfu/sfu.go b/pkg/sfu/sfu.go index cd15dc01f..648c1e563 100644 --- a/pkg/sfu/sfu.go +++ b/pkg/sfu/sfu.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package sfu import ( diff --git a/pkg/sfu/streamallocator/channelobserver.go b/pkg/sfu/streamallocator/channelobserver.go index 9776f8c8a..e8c432dc8 100644 --- a/pkg/sfu/streamallocator/channelobserver.go +++ b/pkg/sfu/streamallocator/channelobserver.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package streamallocator import ( diff --git a/pkg/sfu/streamallocator/nacktracker.go b/pkg/sfu/streamallocator/nacktracker.go index 74104d625..b353781e5 100644 --- a/pkg/sfu/streamallocator/nacktracker.go +++ b/pkg/sfu/streamallocator/nacktracker.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package streamallocator import ( diff --git a/pkg/sfu/streamallocator/probe_controller.go b/pkg/sfu/streamallocator/probe_controller.go index 1d6bfd352..0d7bb52b7 100644 --- a/pkg/sfu/streamallocator/probe_controller.go +++ b/pkg/sfu/streamallocator/probe_controller.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package streamallocator import ( diff --git a/pkg/sfu/streamallocator/prober.go b/pkg/sfu/streamallocator/prober.go index be774f9b2..c30202454 100644 --- a/pkg/sfu/streamallocator/prober.go +++ b/pkg/sfu/streamallocator/prober.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // Design of Prober // // Probing is used to check for existence of excess channel capacity. diff --git a/pkg/sfu/streamallocator/ratemonitor.go b/pkg/sfu/streamallocator/ratemonitor.go index 06445ea44..9c2ba243e 100644 --- a/pkg/sfu/streamallocator/ratemonitor.go +++ b/pkg/sfu/streamallocator/ratemonitor.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package streamallocator import ( diff --git a/pkg/sfu/streamallocator/streamallocator.go b/pkg/sfu/streamallocator/streamallocator.go index 30d7d8616..aa274be3e 100644 --- a/pkg/sfu/streamallocator/streamallocator.go +++ b/pkg/sfu/streamallocator/streamallocator.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package streamallocator import ( diff --git a/pkg/sfu/streamallocator/streamstateupdate.go b/pkg/sfu/streamallocator/streamstateupdate.go index b7bca6a2d..53156de3c 100644 --- a/pkg/sfu/streamallocator/streamstateupdate.go +++ b/pkg/sfu/streamallocator/streamstateupdate.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package streamallocator import ( diff --git a/pkg/sfu/streamallocator/track.go b/pkg/sfu/streamallocator/track.go index 63d201bc8..6ccae215b 100644 --- a/pkg/sfu/streamallocator/track.go +++ b/pkg/sfu/streamallocator/track.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package streamallocator import ( diff --git a/pkg/sfu/streamallocator/trenddetector.go b/pkg/sfu/streamallocator/trenddetector.go index ed0ae1d71..c54d29efa 100644 --- a/pkg/sfu/streamallocator/trenddetector.go +++ b/pkg/sfu/streamallocator/trenddetector.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package streamallocator import ( diff --git a/pkg/sfu/streamtracker/interfaces.go b/pkg/sfu/streamtracker/interfaces.go index a9135e631..f3ad5e699 100644 --- a/pkg/sfu/streamtracker/interfaces.go +++ b/pkg/sfu/streamtracker/interfaces.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package streamtracker import ( diff --git a/pkg/sfu/streamtracker/streamtracker.go b/pkg/sfu/streamtracker/streamtracker.go index 1be7dc0fc..b45723a79 100644 --- a/pkg/sfu/streamtracker/streamtracker.go +++ b/pkg/sfu/streamtracker/streamtracker.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package streamtracker import ( diff --git a/pkg/sfu/streamtracker/streamtracker_dd.go b/pkg/sfu/streamtracker/streamtracker_dd.go index 2d7301f7c..c19b3fe85 100644 --- a/pkg/sfu/streamtracker/streamtracker_dd.go +++ b/pkg/sfu/streamtracker/streamtracker_dd.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package streamtracker import ( diff --git a/pkg/sfu/streamtracker/streamtracker_dd_test.go b/pkg/sfu/streamtracker/streamtracker_dd_test.go index 1c8ae8200..f638e4e2d 100644 --- a/pkg/sfu/streamtracker/streamtracker_dd_test.go +++ b/pkg/sfu/streamtracker/streamtracker_dd_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package streamtracker import ( diff --git a/pkg/sfu/streamtracker/streamtracker_frame.go b/pkg/sfu/streamtracker/streamtracker_frame.go index 87273bcc1..db08d9343 100644 --- a/pkg/sfu/streamtracker/streamtracker_frame.go +++ b/pkg/sfu/streamtracker/streamtracker_frame.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package streamtracker import ( diff --git a/pkg/sfu/streamtracker/streamtracker_packet.go b/pkg/sfu/streamtracker/streamtracker_packet.go index 78866e40e..e95629580 100644 --- a/pkg/sfu/streamtracker/streamtracker_packet.go +++ b/pkg/sfu/streamtracker/streamtracker_packet.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package streamtracker import ( diff --git a/pkg/sfu/streamtracker/streamtracker_packet_test.go b/pkg/sfu/streamtracker/streamtracker_packet_test.go index a7beb2d3c..276483e5e 100644 --- a/pkg/sfu/streamtracker/streamtracker_packet_test.go +++ b/pkg/sfu/streamtracker/streamtracker_packet_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package streamtracker import ( diff --git a/pkg/sfu/streamtrackermanager.go b/pkg/sfu/streamtrackermanager.go index 1a5d24ffe..30ba98322 100644 --- a/pkg/sfu/streamtrackermanager.go +++ b/pkg/sfu/streamtrackermanager.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package sfu import ( diff --git a/pkg/sfu/testutils/data.go b/pkg/sfu/testutils/data.go index 2ab28767a..38640d96b 100644 --- a/pkg/sfu/testutils/data.go +++ b/pkg/sfu/testutils/data.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package testutils import ( diff --git a/pkg/sfu/utils/wraparound.go b/pkg/sfu/utils/wraparound.go index ea9a09887..299002fbf 100644 --- a/pkg/sfu/utils/wraparound.go +++ b/pkg/sfu/utils/wraparound.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package utils import ( diff --git a/pkg/sfu/utils/wraparound_test.go b/pkg/sfu/utils/wraparound_test.go index e9b6bd7a2..9e3b8e555 100644 --- a/pkg/sfu/utils/wraparound_test.go +++ b/pkg/sfu/utils/wraparound_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package utils import ( diff --git a/pkg/sfu/videolayerselector/base.go b/pkg/sfu/videolayerselector/base.go index 7323fc7ae..37b223948 100644 --- a/pkg/sfu/videolayerselector/base.go +++ b/pkg/sfu/videolayerselector/base.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package videolayerselector import ( diff --git a/pkg/sfu/videolayerselector/decodetarget.go b/pkg/sfu/videolayerselector/decodetarget.go index 45055ac06..7204b329f 100644 --- a/pkg/sfu/videolayerselector/decodetarget.go +++ b/pkg/sfu/videolayerselector/decodetarget.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package videolayerselector import ( diff --git a/pkg/sfu/videolayerselector/dependencydescriptor.go b/pkg/sfu/videolayerselector/dependencydescriptor.go index 34f8b0b55..ef63f6526 100644 --- a/pkg/sfu/videolayerselector/dependencydescriptor.go +++ b/pkg/sfu/videolayerselector/dependencydescriptor.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package videolayerselector import ( diff --git a/pkg/sfu/videolayerselector/dependencydescriptor_test.go b/pkg/sfu/videolayerselector/dependencydescriptor_test.go index 0a4ead314..8b4aad771 100644 --- a/pkg/sfu/videolayerselector/dependencydescriptor_test.go +++ b/pkg/sfu/videolayerselector/dependencydescriptor_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package videolayerselector import ( diff --git a/pkg/sfu/videolayerselector/framechain.go b/pkg/sfu/videolayerselector/framechain.go index 60cb4ea73..6dc460c28 100644 --- a/pkg/sfu/videolayerselector/framechain.go +++ b/pkg/sfu/videolayerselector/framechain.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package videolayerselector import ( diff --git a/pkg/sfu/videolayerselector/null.go b/pkg/sfu/videolayerselector/null.go index d1b87fb1b..644a8f957 100644 --- a/pkg/sfu/videolayerselector/null.go +++ b/pkg/sfu/videolayerselector/null.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package videolayerselector import ( diff --git a/pkg/sfu/videolayerselector/selectordecisioncache.go b/pkg/sfu/videolayerselector/selectordecisioncache.go index c98d13984..ea60086b0 100644 --- a/pkg/sfu/videolayerselector/selectordecisioncache.go +++ b/pkg/sfu/videolayerselector/selectordecisioncache.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package videolayerselector import ( diff --git a/pkg/sfu/videolayerselector/simulcast.go b/pkg/sfu/videolayerselector/simulcast.go index 06e0bad72..7d7e173a8 100644 --- a/pkg/sfu/videolayerselector/simulcast.go +++ b/pkg/sfu/videolayerselector/simulcast.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package videolayerselector import ( diff --git a/pkg/sfu/videolayerselector/temporallayerselector/null.go b/pkg/sfu/videolayerselector/temporallayerselector/null.go index d39cab106..51f2fb591 100644 --- a/pkg/sfu/videolayerselector/temporallayerselector/null.go +++ b/pkg/sfu/videolayerselector/temporallayerselector/null.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package temporallayerselector import ( diff --git a/pkg/sfu/videolayerselector/temporallayerselector/temporallayerselector.go b/pkg/sfu/videolayerselector/temporallayerselector/temporallayerselector.go index 8219aea2d..1d691b401 100644 --- a/pkg/sfu/videolayerselector/temporallayerselector/temporallayerselector.go +++ b/pkg/sfu/videolayerselector/temporallayerselector/temporallayerselector.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package temporallayerselector import "github.com/livekit/livekit-server/pkg/sfu/buffer" diff --git a/pkg/sfu/videolayerselector/temporallayerselector/vp8.go b/pkg/sfu/videolayerselector/temporallayerselector/vp8.go index a83765526..2d2660897 100644 --- a/pkg/sfu/videolayerselector/temporallayerselector/vp8.go +++ b/pkg/sfu/videolayerselector/temporallayerselector/vp8.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package temporallayerselector import ( diff --git a/pkg/sfu/videolayerselector/videolayerselector.go b/pkg/sfu/videolayerselector/videolayerselector.go index f17d745d1..545196eae 100644 --- a/pkg/sfu/videolayerselector/videolayerselector.go +++ b/pkg/sfu/videolayerselector/videolayerselector.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package videolayerselector import ( diff --git a/pkg/sfu/videolayerselector/vp9.go b/pkg/sfu/videolayerselector/vp9.go index 508cdf289..62d5d35d9 100644 --- a/pkg/sfu/videolayerselector/vp9.go +++ b/pkg/sfu/videolayerselector/vp9.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package videolayerselector import ( diff --git a/pkg/telemetry/analyticsservice.go b/pkg/telemetry/analyticsservice.go index 0373c33b6..8611337f1 100644 --- a/pkg/telemetry/analyticsservice.go +++ b/pkg/telemetry/analyticsservice.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package telemetry import ( diff --git a/pkg/telemetry/events.go b/pkg/telemetry/events.go index b1e731bdc..35eaf668e 100644 --- a/pkg/telemetry/events.go +++ b/pkg/telemetry/events.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package telemetry import ( diff --git a/pkg/telemetry/events_test.go b/pkg/telemetry/events_test.go index 77529b211..e83b61ba9 100644 --- a/pkg/telemetry/events_test.go +++ b/pkg/telemetry/events_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package telemetry_test import ( diff --git a/pkg/telemetry/prometheus/node.go b/pkg/telemetry/prometheus/node.go index 5dc04a3c8..dad09977e 100644 --- a/pkg/telemetry/prometheus/node.go +++ b/pkg/telemetry/prometheus/node.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package prometheus import ( diff --git a/pkg/telemetry/prometheus/node_linux.go b/pkg/telemetry/prometheus/node_linux.go index 9854f056e..baa6919f8 100644 --- a/pkg/telemetry/prometheus/node_linux.go +++ b/pkg/telemetry/prometheus/node_linux.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //go:build linux // +build linux diff --git a/pkg/telemetry/prometheus/node_nonlinux.go b/pkg/telemetry/prometheus/node_nonlinux.go index 5cd36c9c1..22fc11c7b 100644 --- a/pkg/telemetry/prometheus/node_nonlinux.go +++ b/pkg/telemetry/prometheus/node_nonlinux.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //go:build !linux package prometheus diff --git a/pkg/telemetry/prometheus/packets.go b/pkg/telemetry/prometheus/packets.go index 09525053c..3048367a8 100644 --- a/pkg/telemetry/prometheus/packets.go +++ b/pkg/telemetry/prometheus/packets.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package prometheus import ( diff --git a/pkg/telemetry/prometheus/psrpc.go b/pkg/telemetry/prometheus/psrpc.go index 958704605..f07c90034 100644 --- a/pkg/telemetry/prometheus/psrpc.go +++ b/pkg/telemetry/prometheus/psrpc.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package prometheus import ( diff --git a/pkg/telemetry/prometheus/quality.go b/pkg/telemetry/prometheus/quality.go index 708c654f0..481b7558f 100644 --- a/pkg/telemetry/prometheus/quality.go +++ b/pkg/telemetry/prometheus/quality.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package prometheus import ( diff --git a/pkg/telemetry/prometheus/rooms.go b/pkg/telemetry/prometheus/rooms.go index d50ce2d15..ef320ad73 100644 --- a/pkg/telemetry/prometheus/rooms.go +++ b/pkg/telemetry/prometheus/rooms.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package prometheus import ( diff --git a/pkg/telemetry/signalanddatastats.go b/pkg/telemetry/signalanddatastats.go index 0ff189ca7..840f126c3 100644 --- a/pkg/telemetry/signalanddatastats.go +++ b/pkg/telemetry/signalanddatastats.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package telemetry import ( diff --git a/pkg/telemetry/stats.go b/pkg/telemetry/stats.go index 81e9c2556..79d6717cb 100644 --- a/pkg/telemetry/stats.go +++ b/pkg/telemetry/stats.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package telemetry import ( diff --git a/pkg/telemetry/stats_test.go b/pkg/telemetry/stats_test.go index 5dd8b82c3..1758c8b45 100644 --- a/pkg/telemetry/stats_test.go +++ b/pkg/telemetry/stats_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package telemetry_test import ( diff --git a/pkg/telemetry/statsconn.go b/pkg/telemetry/statsconn.go index 91f60af94..10ac45fb5 100644 --- a/pkg/telemetry/statsconn.go +++ b/pkg/telemetry/statsconn.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package telemetry import ( diff --git a/pkg/telemetry/statsworker.go b/pkg/telemetry/statsworker.go index f963097ee..b86175368 100644 --- a/pkg/telemetry/statsworker.go +++ b/pkg/telemetry/statsworker.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package telemetry import ( diff --git a/pkg/telemetry/telemetryservice.go b/pkg/telemetry/telemetryservice.go index 5b4c4b9c1..c74619eda 100644 --- a/pkg/telemetry/telemetryservice.go +++ b/pkg/telemetry/telemetryservice.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package telemetry import ( diff --git a/pkg/testutils/timeout.go b/pkg/testutils/timeout.go index af70a48f9..11debef92 100644 --- a/pkg/testutils/timeout.go +++ b/pkg/testutils/timeout.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package testutils import ( diff --git a/pkg/utils/math.go b/pkg/utils/math.go index 0ac6319af..9e75c1ad8 100644 --- a/pkg/utils/math.go +++ b/pkg/utils/math.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package utils import "sort" diff --git a/pkg/utils/opsqueue.go b/pkg/utils/opsqueue.go index 473430a3a..3e992461c 100644 --- a/pkg/utils/opsqueue.go +++ b/pkg/utils/opsqueue.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package utils import ( diff --git a/test/client/client.go b/test/client/client.go index 776d356a7..70b135a6d 100644 --- a/test/client/client.go +++ b/test/client/client.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package client import ( diff --git a/test/client/trackwriter.go b/test/client/trackwriter.go index e475a2ed9..9104f9cd2 100644 --- a/test/client/trackwriter.go +++ b/test/client/trackwriter.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package client import ( diff --git a/test/integration_helpers.go b/test/integration_helpers.go index ec0ec7fb2..82f50f825 100644 --- a/test/integration_helpers.go +++ b/test/integration_helpers.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package test import ( diff --git a/test/multinode_roomservice_test.go b/test/multinode_roomservice_test.go index f822ebdb1..a1ec1cb8b 100644 --- a/test/multinode_roomservice_test.go +++ b/test/multinode_roomservice_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package test import ( diff --git a/test/multinode_test.go b/test/multinode_test.go index fe973aefd..8b1eb1d32 100644 --- a/test/multinode_test.go +++ b/test/multinode_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package test import ( diff --git a/test/scenarios.go b/test/scenarios.go index d5c7878e0..d72578614 100644 --- a/test/scenarios.go +++ b/test/scenarios.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package test import ( diff --git a/test/singlenode_test.go b/test/singlenode_test.go index fe84de51d..fe2e55b07 100644 --- a/test/singlenode_test.go +++ b/test/singlenode_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package test import ( diff --git a/test/webhook_test.go b/test/webhook_test.go index 22e901e88..678c48c80 100644 --- a/test/webhook_test.go +++ b/test/webhook_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package test import ( diff --git a/tools/tools.go b/tools/tools.go index 5a341066b..f77dcd861 100644 --- a/tools/tools.go +++ b/tools/tools.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //go:build tools // +build tools diff --git a/version/version.go b/version/version.go index c44e5ca30..1df2fdae5 100644 --- a/version/version.go +++ b/version/version.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package version const Version = "1.4.4" From a87232f42a83868721912b714bcff93a7439d0d1 Mon Sep 17 00:00:00 2001 From: cnderrauber Date: Fri, 28 Jul 2023 14:40:33 +0800 Subject: [PATCH 322/324] Frame integrity check for svc codec (#1914) * Frame integrity check for svc codec * Fix test * Spell --- pkg/sfu/buffer/dependencydescriptorparser.go | 19 +- pkg/sfu/buffer/frameintegrity.go | 211 ++++++++++++++++++ pkg/sfu/buffer/frameintegrity_test.go | 72 ++++++ .../dependencydescriptor.go | 45 +--- .../dependencydescriptor_test.go | 4 + pkg/sfu/videolayerselector/framechain.go | 5 + 6 files changed, 317 insertions(+), 39 deletions(-) create mode 100644 pkg/sfu/buffer/frameintegrity.go create mode 100644 pkg/sfu/buffer/frameintegrity_test.go diff --git a/pkg/sfu/buffer/dependencydescriptorparser.go b/pkg/sfu/buffer/dependencydescriptorparser.go index 17ed0732d..a3a2be794 100644 --- a/pkg/sfu/buffer/dependencydescriptorparser.go +++ b/pkg/sfu/buffer/dependencydescriptorparser.go @@ -33,10 +33,12 @@ type DependencyDescriptorParser struct { onMaxLayerChanged func(int32, int32) decodeTargets []DependencyDescriptorDecodeTarget - wrapAround *utils.WrapAround[uint16, uint64] + seqWrapAround *utils.WrapAround[uint16, uint64] + frameWrapAround *utils.WrapAround[uint16, uint64] structureExtSeq uint64 activeDecodeTargetsExtSeq uint64 activeDecodeTargetsMask uint32 + frameChecker *FrameIntegrityChecker } func NewDependencyDescriptorParser(ddExtID uint8, logger logger.Logger, onMaxLayerChanged func(int32, int32)) *DependencyDescriptorParser { @@ -45,7 +47,9 @@ func NewDependencyDescriptorParser(ddExtID uint8, logger logger.Logger, onMaxLay ddExtID: ddExtID, logger: logger, onMaxLayerChanged: onMaxLayerChanged, - wrapAround: utils.NewWrapAround[uint16, uint64](), + seqWrapAround: utils.NewWrapAround[uint16, uint64](), + frameWrapAround: utils.NewWrapAround[uint16, uint64](), + frameChecker: NewFrameIntegrityChecker(180, 1024), // 2seconds for L3T3 30fps video } } @@ -55,6 +59,8 @@ type ExtDependencyDescriptor struct { DecodeTargets []DependencyDescriptorDecodeTarget StructureUpdated bool ActiveDecodeTargetsUpdated bool + Integrity bool + ExtFrameNum uint64 } func (r *DependencyDescriptorParser) Parse(pkt *rtp.Packet) (*ExtDependencyDescriptor, VideoLayer, error) { @@ -75,14 +81,19 @@ func (r *DependencyDescriptorParser) Parse(pkt *rtp.Packet) (*ExtDependencyDescr return nil, videoLayer, err } - extSeq := r.wrapAround.Update(pkt.SequenceNumber).ExtendedVal + extSeq := r.seqWrapAround.Update(pkt.SequenceNumber).ExtendedVal if ddVal.FrameDependencies != nil { videoLayer.Spatial, videoLayer.Temporal = int32(ddVal.FrameDependencies.SpatialId), int32(ddVal.FrameDependencies.TemporalId) } + extFN := r.frameWrapAround.Update(ddVal.FrameNumber).ExtendedVal + r.frameChecker.AddPacket(extSeq, extFN, &ddVal) + extDD := &ExtDependencyDescriptor{ - Descriptor: &ddVal, + Descriptor: &ddVal, + ExtFrameNum: extFN, + Integrity: r.frameChecker.FrameIntegrity(extFN), } if ddVal.AttachedStructure != nil { diff --git a/pkg/sfu/buffer/frameintegrity.go b/pkg/sfu/buffer/frameintegrity.go new file mode 100644 index 000000000..1b712652e --- /dev/null +++ b/pkg/sfu/buffer/frameintegrity.go @@ -0,0 +1,211 @@ +package buffer + +import ( + dd "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" +) + +type FrameEntity struct { + startSeq *uint64 + endSeq *uint64 + integrity bool + + packetsConsective func(uint64, uint64) bool +} + +func (fe *FrameEntity) AddPacket(extSeq uint64, ddVal *dd.DependencyDescriptor) { + // duplicate packet + if fe.integrity { + return + } + + if fe.startSeq == nil && ddVal.FirstPacketInFrame { + fe.startSeq = &extSeq + } + if fe.endSeq == nil && ddVal.LastPacketInFrame { + fe.endSeq = &extSeq + } + + if fe.startSeq != nil && fe.endSeq != nil { + if fe.packetsConsective(*fe.startSeq, *fe.endSeq) { + fe.integrity = true + } + } +} + +func (fe *FrameEntity) Reset() { + fe.integrity = false + fe.startSeq, fe.endSeq = nil, nil +} + +func (fe *FrameEntity) Integrity() bool { + return fe.integrity +} + +// ------------------------------ + +type PacketHistory struct { + base uint64 + last uint64 + bits []uint64 + packetCount int + inited bool +} + +func NewPacketHistory(packetCount int) *PacketHistory { + packetCount = (packetCount + 63) / 64 * 64 + return &PacketHistory{ + bits: make([]uint64, packetCount/64), + packetCount: packetCount, + } +} + +func (ph *PacketHistory) AddPacket(extSeq uint64) { + if !ph.inited { + ph.inited = true + ph.base = uint64(extSeq) + // set base to extSeq-100 to avoid out-of-order packets belongs to first frame to be dropped + if ph.base > 100 { + ph.base -= 100 + } else { + ph.base = 0 + } + ph.last = uint64(extSeq) + ph.set(extSeq, true) + return + } + + if extSeq <= ph.base { + // too old + return + } + + if extSeq <= ph.last { + if ph.last-extSeq < uint64(ph.packetCount) { + ph.set(extSeq, true) + } + return + } + + for i := ph.last + 1; i < extSeq; i++ { + ph.set(i, false) + } + + ph.set(extSeq, true) + ph.last = extSeq +} + +func (ph *PacketHistory) getPos(seq uint64) (index, offset int) { + idx := (seq - ph.base) % uint64(ph.packetCount) + return int(idx >> 6), int(idx % 64) +} + +func (ph *PacketHistory) set(seq uint64, received bool) { + idx, offset := ph.getPos(seq) + if !received { + ph.bits[idx] &= ^(1 << offset) + } else { + ph.bits[idx] |= 1 << (offset) + } +} + +func (ph *PacketHistory) PacketsConsecutive(start, end uint64) bool { + if start > end { + return false + } + + if end-start >= uint64(ph.packetCount) { + return false + } + + startIndex, startOffset := ph.getPos(start) + endIndex, endOffset := ph.getPos(end) + + if startIndex == endIndex && end-start <= 64 { + testBits := uint64((1<<(endOffset-startOffset+1))-1) << startOffset + return ph.bits[startIndex]&testBits == testBits + } + + if (ph.bits[startIndex]>>(startOffset))+1 != 1<<(64-startOffset) { + return false + } + + for i := startIndex + 1; i != endIndex; i++ { + if i == len(ph.bits) { + i = 0 + if i == endIndex { + break + } + } + if ph.bits[i]+1 != 0 { + return false + } + } + + testBits := uint64((1 << (endOffset + 1)) - 1) + return ph.bits[endIndex]&testBits == testBits +} + +// ------------------------------ + +type FrameIntegrityChecker struct { + frameCount int + frames []FrameEntity + base uint64 + last uint64 + + pktHistory *PacketHistory + inited bool +} + +func NewFrameIntegrityChecker(frameCount, packetCount int) *FrameIntegrityChecker { + fc := &FrameIntegrityChecker{ + frames: make([]FrameEntity, frameCount), + pktHistory: NewPacketHistory(packetCount), + frameCount: frameCount, + } + + for i := range fc.frames { + fc.frames[i].packetsConsective = fc.pktHistory.PacketsConsecutive + fc.frames[i].Reset() + } + return fc +} + +func (fc *FrameIntegrityChecker) AddPacket(extSeq uint64, extFrameNum uint64, ddVal *dd.DependencyDescriptor) { + fc.pktHistory.AddPacket(extSeq) + + if !fc.inited { + fc.inited = true + fc.base = extFrameNum + fc.last = extFrameNum + } + + if extFrameNum < fc.base { + // frame too old + return + } + + if extFrameNum <= fc.last { + if fc.last-extFrameNum >= uint64(fc.frameCount) { + // frame too old + return + } + fc.frames[int(extFrameNum-fc.base)%fc.frameCount].AddPacket(extSeq, ddVal) + return + } + + // reset missing frames + for i := fc.last + 1; i <= extFrameNum; i++ { + fc.frames[int(i-fc.base)%fc.frameCount].Reset() + } + fc.frames[int(extFrameNum-fc.base)%fc.frameCount].AddPacket(extSeq, ddVal) + fc.last = extFrameNum +} + +func (fc *FrameIntegrityChecker) FrameIntegrity(extFrameNum uint64) bool { + if extFrameNum < fc.base || extFrameNum > fc.last || fc.last-extFrameNum >= uint64(fc.frameCount) { + return false + } + + return fc.frames[int(extFrameNum-fc.base)%fc.frameCount].Integrity() +} diff --git a/pkg/sfu/buffer/frameintegrity_test.go b/pkg/sfu/buffer/frameintegrity_test.go new file mode 100644 index 000000000..3e505efdd --- /dev/null +++ b/pkg/sfu/buffer/frameintegrity_test.go @@ -0,0 +1,72 @@ +package buffer + +import ( + "math/rand" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" +) + +func TestFrameIntegrityChecker(t *testing.T) { + fc := NewFrameIntegrityChecker(100, 1000) + + // first frame out of order + fc.AddPacket(10, 10, &dependencydescriptor.DependencyDescriptor{}) + require.False(t, fc.FrameIntegrity(10)) + fc.AddPacket(9, 10, &dependencydescriptor.DependencyDescriptor{FirstPacketInFrame: true}) + require.False(t, fc.FrameIntegrity(10)) + fc.AddPacket(11, 10, &dependencydescriptor.DependencyDescriptor{LastPacketInFrame: true}) + require.True(t, fc.FrameIntegrity(10)) + + // single packet frame + fc.AddPacket(100, 100, &dependencydescriptor.DependencyDescriptor{FirstPacketInFrame: true, LastPacketInFrame: true}) + require.True(t, fc.FrameIntegrity(100)) + require.False(t, fc.FrameIntegrity(101)) + require.False(t, fc.FrameIntegrity(99)) + + // frame too old than first frame + fc.AddPacket(99, 99, &dependencydescriptor.DependencyDescriptor{FirstPacketInFrame: true, LastPacketInFrame: true}) + + // multiple packet frame, out of order + fc.AddPacket(2001, 2001, &dependencydescriptor.DependencyDescriptor{}) + require.False(t, fc.FrameIntegrity(2001)) + require.False(t, fc.FrameIntegrity(1999)) + // out of frame count(100) + require.False(t, fc.FrameIntegrity(100)) + require.False(t, fc.FrameIntegrity(1900)) + + fc.AddPacket(2000, 2001, &dependencydescriptor.DependencyDescriptor{FirstPacketInFrame: true}) + require.False(t, fc.FrameIntegrity(2001)) + fc.AddPacket(2002, 2001, &dependencydescriptor.DependencyDescriptor{LastPacketInFrame: true}) + require.True(t, fc.FrameIntegrity(2001)) + // duplicate packet + fc.AddPacket(2001, 2001, &dependencydescriptor.DependencyDescriptor{}) + require.True(t, fc.FrameIntegrity(2001)) + + // frame too old + fc.AddPacket(900, 1900, &dependencydescriptor.DependencyDescriptor{FirstPacketInFrame: true, LastPacketInFrame: true}) + require.False(t, fc.FrameIntegrity(1900)) + + for frame := uint64(2002); frame < 2102; frame++ { + // large frame (1000 packets) out of order / retransmitted + firstFrame := uint64(3000 + (frame-2002)*1000) + lastFrame := uint64(3999 + (frame-2002)*1000) + frames := make([]uint64, 0, lastFrame-firstFrame+1) + for i := firstFrame; i <= lastFrame; i++ { + frames = append(frames, i) + } + require.False(t, fc.FrameIntegrity(frame)) + rand.Seed(int64(frame)) + rand.Shuffle(len(frames), func(i, j int) { frames[i], frames[j] = frames[j], frames[i] }) + for i, f := range frames { + fc.AddPacket(f, frame, &dependencydescriptor.DependencyDescriptor{ + FirstPacketInFrame: f == firstFrame, + LastPacketInFrame: f == lastFrame, + }) + require.Equal(t, i == len(frames)-1, fc.FrameIntegrity(frame), i) + } + require.True(t, fc.FrameIntegrity(frame)) + } +} diff --git a/pkg/sfu/videolayerselector/dependencydescriptor.go b/pkg/sfu/videolayerselector/dependencydescriptor.go index ef63f6526..555437c6d 100644 --- a/pkg/sfu/videolayerselector/dependencydescriptor.go +++ b/pkg/sfu/videolayerselector/dependencydescriptor.go @@ -20,14 +20,12 @@ import ( "github.com/livekit/livekit-server/pkg/sfu/buffer" dede "github.com/livekit/livekit-server/pkg/sfu/dependencydescriptor" - "github.com/livekit/livekit-server/pkg/sfu/utils" "github.com/livekit/protocol/logger" ) type DependencyDescriptor struct { *Base - frameNum *utils.WrapAround[uint16, uint64] decisions *SelectorDecisionCache previousActiveDecodeTargetsBitmask *uint32 @@ -43,7 +41,6 @@ type DependencyDescriptor struct { func NewDependencyDescriptor(logger logger.Logger) *DependencyDescriptor { return &DependencyDescriptor{ Base: NewBase(logger), - frameNum: utils.NewWrapAround[uint16, uint64](), decisions: NewSelectorDecisionCache(256, 80), } } @@ -51,7 +48,6 @@ func NewDependencyDescriptor(logger logger.Logger) *DependencyDescriptor { func NewDependencyDescriptorFromNull(vls VideoLayerSelector) *DependencyDescriptor { return &DependencyDescriptor{ Base: vls.(*Null).Base, - frameNum: utils.NewWrapAround[uint16, uint64](), decisions: NewSelectorDecisionCache(256, 80), } } @@ -72,8 +68,7 @@ func (d *DependencyDescriptor) Select(extPkt *buffer.ExtPacket, _layer int32) (r dd := ddwdt.Descriptor - frameNum := d.frameNum.Update(dd.FrameNumber) - extFrameNum := frameNum.ExtendedVal + extFrameNum := ddwdt.ExtFrameNum fd := dd.FrameDependencies incomingLayer := buffer.VideoLayer{ @@ -117,6 +112,8 @@ func (d *DependencyDescriptor) Select(extPkt *buffer.ExtPacket, _layer int32) (r } var dti dede.DecodeTargetIndication d.decodeTargetsLock.RLock() + + // decodeTargets be sorted from high to low, find the highest decode target that is active and integrity for _, dt := range d.decodeTargets { if !dt.Active() || dt.Layer.Spatial > d.targetLayer.Spatial || dt.Layer.Temporal > d.targetLayer.Temporal { continue @@ -133,28 +130,14 @@ func (d *DependencyDescriptor) Select(extPkt *buffer.ExtPacket, _layer int32) (r return } - // Keep forwarding the lower spatial with temporal layer 0 to keep the lower frame chain intact, - // it will cost a few extra bits as those frames might not be present in the current target - // but will make the subscriber switch to lower layer seamlessly without pli. if frameResult.TargetValid { - if highestDecodeTarget.Target == -1 { - highestDecodeTarget = dt.DependencyDescriptorDecodeTarget - dti = frameResult.DTI - } else if dt.Layer.Spatial < highestDecodeTarget.Layer.Spatial && dt.Layer.Temporal == 0 && - frameResult.DTI != dede.DecodeTargetNotPresent && frameResult.DTI != dede.DecodeTargetDiscardable { - dti = frameResult.DTI - } + highestDecodeTarget = dt.DependencyDescriptorDecodeTarget + dti = frameResult.DTI + break } } d.decodeTargetsLock.RUnlock() - // DD-TODO : we don't have a rtp queue to ensure the order of packets now, - // so we don't know packet is lost/out of order, that cause us can't detect - // frame integrity, entire frame is forwareded, whether frame chain is broken. - // So use a simple check here, assume all the reference frame is forwarded and - // only check DTI of the active decode target. - // it is not effeciency, at last we need check frame chain integrity. - if highestDecodeTarget.Target < 0 { // no active decode target, do not select // d.logger.Debugw(fmt.Sprintf("drop packet for no target found, decodeTargets %v, tagetLayer %v, incoming %v", @@ -218,6 +201,7 @@ func (d *DependencyDescriptor) Select(extPkt *buffer.ExtPacket, _layer int32) (r d.previousActiveDecodeTargetsBitmask = d.activeDecodeTargetsBitmask d.activeDecodeTargetsBitmask = buffer.GetActiveDecodeTargetBitmask(d.currentLayer, ddwdt.DecodeTargets) + d.logger.Debugw("switch to target", "highest", highestDecodeTarget.Layer, "current", d.currentLayer, "bitmask", *d.activeDecodeTargetsBitmask) } ddExtension := &dede.DependencyDescriptorExtension{ @@ -241,18 +225,9 @@ func (d *DependencyDescriptor) Select(extPkt *buffer.ExtPacket, _layer int32) (r result.DependencyDescriptorExtension = bytes } - // DD-TODO START - // Ideally should add this frame only on the last packet of the frame and if all packets of the frame have been selected. - // But, adding on any packet so that any out-of-order packets within a frame can be fowarded. - // But, that could result in decodability/chain integrity to erroneously pass (i. e. in the case of lost packet in this - // frame, this frame is not decodable and hence the chain is broken). - // - // Note that packets can get lost in the forwarded path also. That will be handled by receiver sending PLI. - // - // Within SFU, there is more work to do to ensure integrity of forwarded packets/frames to adhere to the complete design - // goal of dependency descriptor - // DD-TODO END - d.decisions.AddForwarded(extFrameNum) + if ddwdt.Integrity { + d.decisions.AddForwarded(extFrameNum) + } result.RTPMarker = extPkt.Packet.Header.Marker || (dd.LastPacketInFrame && d.currentLayer.Spatial == int32(fd.SpatialId)) result.IsSelected = true return diff --git a/pkg/sfu/videolayerselector/dependencydescriptor_test.go b/pkg/sfu/videolayerselector/dependencydescriptor_test.go index 8b4aad771..e9e158fda 100644 --- a/pkg/sfu/videolayerselector/dependencydescriptor_test.go +++ b/pkg/sfu/videolayerselector/dependencydescriptor_test.go @@ -321,6 +321,8 @@ func createDDFrames(maxLayer buffer.VideoLayer, startFrameNumber uint16) []*buff DecodeTargets: decodeTargets, StructureUpdated: true, ActiveDecodeTargetsUpdated: true, + Integrity: true, + ExtFrameNum: uint64(startFrameNumber), }, Packet: &rtp.Packet{ Header: rtp.Header{ @@ -370,6 +372,8 @@ func createDDFrames(maxLayer buffer.VideoLayer, startFrameNumber uint16) []*buff }, }, DecodeTargets: decodeTargets, + Integrity: true, + ExtFrameNum: uint64(startFrameNumber), }, Packet: &rtp.Packet{ Header: rtp.Header{ diff --git a/pkg/sfu/videolayerselector/framechain.go b/pkg/sfu/videolayerselector/framechain.go index 6dc460c28..5edddfad9 100644 --- a/pkg/sfu/videolayerselector/framechain.go +++ b/pkg/sfu/videolayerselector/framechain.go @@ -87,10 +87,15 @@ func (fc *FrameChain) OnFrame(extFrameNum uint64, fd *dd.FrameDependencyTemplate } func (fc *FrameChain) OnExpectFrameChanged(frameNum uint64, decision selectorDecision) { + if fc.broken { + return + } + for i, f := range fc.expectFrames { if f == frameNum { if decision != selectorDecisionForwarded { fc.broken = true + fc.logger.Debugw("frame chain broken", "chanIdx", fc.chainIdx, "sd", decision, "frame", frameNum) } fc.expectFrames[i] = fc.expectFrames[len(fc.expectFrames)-1] fc.expectFrames = fc.expectFrames[:len(fc.expectFrames)-1] From cfee506f51187be965f1fcc1f259bf5133554132 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 28 Jul 2023 16:39:58 -0700 Subject: [PATCH 323/324] Update golang.org/x/exp digest to b0cb94b (#1877) Generated by renovateBot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 34d36beea..793e9df8d 100644 --- a/go.mod +++ b/go.mod @@ -45,7 +45,7 @@ require ( github.com/urfave/cli/v2 v2.25.7 github.com/urfave/negroni/v3 v3.0.0 go.uber.org/atomic v1.11.0 - golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb + golang.org/x/exp v0.0.0-20230728194245-b0cb94b80691 golang.org/x/sync v0.3.0 google.golang.org/protobuf v1.31.0 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index 417006f39..40eb15323 100644 --- a/go.sum +++ b/go.sum @@ -292,8 +292,8 @@ golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= -golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb h1:xIApU0ow1zwMa2uL1VDNeQlNVFTWMQxZUZCMDy0Q4Us= -golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/exp v0.0.0-20230728194245-b0cb94b80691 h1:/yRP+0AN7mf5DkD3BAI6TOFnd51gEoDEb8o35jIFtgw= +golang.org/x/exp v0.0.0-20230728194245-b0cb94b80691/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= From b6394d5aa64c1192af149f8568f59bd0833f6889 Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sat, 29 Jul 2023 18:26:57 +0530 Subject: [PATCH 324/324] De-dupe ICE candidates, makes logging cleaner. (#1916) --- go.mod | 10 +++++----- go.sum | 24 +++++++++++------------- pkg/rtc/transport.go | 41 ++++++++++++++++++++++++----------------- 3 files changed, 40 insertions(+), 35 deletions(-) diff --git a/go.mod b/go.mod index 793e9df8d..1d7e21bf7 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jxskiss/base62 v1.1.0 github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/mediatransportutil v0.0.0-20230716190407-fc4944cbc33a - github.com/livekit/protocol v1.5.10 + github.com/livekit/protocol v1.5.11-0.20230729124740-d45d830f69e2 github.com/livekit/psrpc v0.3.2 github.com/mackerelio/go-osstat v0.2.4 github.com/magefile/mage v1.15.0 @@ -26,14 +26,14 @@ require ( github.com/mitchellh/go-homedir v1.1.0 github.com/olekukonko/tablewriter v0.0.5 github.com/pion/dtls/v2 v2.2.7 - github.com/pion/ice/v2 v2.3.8 + github.com/pion/ice/v2 v2.3.9 github.com/pion/interceptor v0.1.17 github.com/pion/rtcp v1.2.10 - github.com/pion/rtp v1.7.13 + github.com/pion/rtp v1.8.0 github.com/pion/sdp/v3 v3.0.6 github.com/pion/transport/v2 v2.2.1 github.com/pion/turn/v2 v2.1.2 - github.com/pion/webrtc/v3 v3.2.11 + github.com/pion/webrtc/v3 v3.2.13 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.16.0 github.com/redis/go-redis/v9 v9.0.5 @@ -101,6 +101,6 @@ require ( golang.org/x/text v0.10.0 // indirect golang.org/x/tools v0.9.3 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect - google.golang.org/grpc v1.56.2 // indirect + google.golang.org/grpc v1.57.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 40eb15323..9a2e52f3c 100644 --- a/go.sum +++ b/go.sum @@ -124,8 +124,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20230716190407-fc4944cbc33a h1:JWpPHcMFuw0fP4swE89CfMgeUXiSN5IKvCJL/5HLI3A= github.com/livekit/mediatransportutil v0.0.0-20230716190407-fc4944cbc33a/go.mod h1:xirUXW8xnLGmfCwUeAv/nj1VGo1OO1BmgxrYP7jK/14= -github.com/livekit/protocol v1.5.10 h1:lnaHMa27cbRkHybi/jvOVuRSaLsho2wCLRjKiC6ce2Y= -github.com/livekit/protocol v1.5.10/go.mod h1:eRzojAYSPJuNgDHMlvLji/CPauj9hrgvb6rVPUj6MoU= +github.com/livekit/protocol v1.5.11-0.20230729124740-d45d830f69e2 h1:KxQIooCpXmn+qzxQxNbxBtRXstEFd2/7ihH4Pp1dOc4= +github.com/livekit/protocol v1.5.11-0.20230729124740-d45d830f69e2/go.mod h1:3Dt53NrYnuA7pAJjAjXLJ2q5rU3JKoebvMttZPZWDH8= github.com/livekit/psrpc v0.3.2 h1:eAaJhASme33gtoBhCRLH9jsnWcdm1tHWf0WzaDk56ew= github.com/livekit/psrpc v0.3.2/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= @@ -184,8 +184,8 @@ github.com/pion/datachannel v1.5.5 h1:10ef4kwdjije+M9d7Xm9im2Y3O6A6ccQb0zcqZcJew github.com/pion/datachannel v1.5.5/go.mod h1:iMz+lECmfdCMqFRhXhcA/219B0SQlbpoR2V118yimL0= github.com/pion/dtls/v2 v2.2.7 h1:cSUBsETxepsCSFSxC3mc/aDo14qQLMSL+O6IjG28yV8= github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= -github.com/pion/ice/v2 v2.3.8 h1:/4vM7uFPJez3PhNhlqUcJhboYaDNWo+R8oAuMj2cKsA= -github.com/pion/ice/v2 v2.3.8/go.mod h1:DoMA9FvsfNTBVnjyRf2t4EhUkSp9tNrH77fMtPFYygQ= +github.com/pion/ice/v2 v2.3.9 h1:7yZpHf3PhPxJGT4JkMj1Y8Rl5cQ6fB709iz99aeMd/U= +github.com/pion/ice/v2 v2.3.9/go.mod h1:lT3kv5uUIlHfXHU/ZRD7uKD/ufM202+eTa3C/umgGf4= github.com/pion/interceptor v0.1.17 h1:prJtgwFh/gB8zMqGZoOgJPHivOwVAp61i2aG61Du/1w= github.com/pion/interceptor v0.1.17/go.mod h1:SY8kpmfVBvrbUzvj2bsXz7OJt5JvmVNZ+4Kjq7FcwrI= github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= @@ -196,8 +196,9 @@ github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= github.com/pion/rtcp v1.2.10 h1:nkr3uj+8Sp97zyItdN60tE/S6vk4al5CPRR6Gejsdjc= github.com/pion/rtcp v1.2.10/go.mod h1:ztfEwXZNLGyF1oQDttz/ZKIBaeeg/oWbRYqzBM9TL1I= -github.com/pion/rtp v1.7.13 h1:qcHwlmtiI50t1XivvoawdCGTP4Uiypzfrsap+bijcoA= github.com/pion/rtp v1.7.13/go.mod h1:bDb5n+BFZxXx0Ea7E5qe+klMuqiBrP+w8XSjiWtCUko= +github.com/pion/rtp v1.8.0 h1:SYD7040IR+NqrGBOc2GDU5iDjAR+0m5rnX/EWCUMNhw= +github.com/pion/rtp v1.8.0/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= github.com/pion/sctp v1.8.5/go.mod h1:SUFFfDpViyKejTAdwD1d/HQsCu+V/40cCs2nZIvC3s0= github.com/pion/sctp v1.8.7 h1:JnABvFakZueGAn4KU/4PSKg+GWbF6QWbKTWZOSGJjXw= github.com/pion/sctp v1.8.7/go.mod h1:g1Ul+ARqZq5JEmoFy87Q/4CePtKnTJ1QCL9dBBdN6AU= @@ -205,8 +206,6 @@ github.com/pion/sdp/v3 v3.0.6 h1:WuDLhtuFUUVpTfus9ILC4HRyHsW6TdugjEX/QY9OiUw= github.com/pion/sdp/v3 v3.0.6/go.mod h1:iiFWFpQO8Fy3S5ldclBkpXqmWy02ns78NOKoLLL0YQw= github.com/pion/srtp/v2 v2.0.15 h1:+tqRtXGsGwHC0G0IUIAzRmdkHvriF79IHVfZGfHrQoA= github.com/pion/srtp/v2 v2.0.15/go.mod h1:b/pQOlDrbB0HEH5EUAQXzSYxikFbNcNuKmF8tM0hCtw= -github.com/pion/stun v0.4.0/go.mod h1:QPsh1/SbXASntw3zkkrIk3ZJVKz4saBY2G7S10P3wCw= -github.com/pion/stun v0.6.0/go.mod h1:HPqcfoeqQn9cuaet7AOmB5e5xkObu9DwBdurwLKO9oA= github.com/pion/stun v0.6.1 h1:8lp6YejULeHBF8NmV8e2787BogQhduZugh5PdhDyyN4= github.com/pion/stun v0.6.1/go.mod h1:/hO7APkX4hZKu/D0f2lHzNyvdkTGtIy3NDmLR7kSz/8= github.com/pion/transport v0.14.1 h1:XSM6olwW+o8J4SCmOBb/BpwZypkHeyM0PGFCxNQBr40= @@ -216,11 +215,10 @@ github.com/pion/transport/v2 v2.1.0/go.mod h1:AdSw4YBZVDkZm8fpoz+fclXyQwANWmZAlD github.com/pion/transport/v2 v2.2.0/go.mod h1:AdSw4YBZVDkZm8fpoz+fclXyQwANWmZAlDuQdctTThQ= github.com/pion/transport/v2 v2.2.1 h1:7qYnCBlpgSJNYMbLCKuSY9KbQdBFoETvPNETv0y4N7c= github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g= -github.com/pion/turn/v2 v2.1.0/go.mod h1:yrT5XbXSGX1VFSF31A3c1kCNB5bBZgk/uu5LET162qs= github.com/pion/turn/v2 v2.1.2 h1:wj0cAoGKltaZ790XEGW9HwoUewqjliwmhtxCuB2ApyM= github.com/pion/turn/v2 v2.1.2/go.mod h1:1kjnPkBcex3dhCU2Am+AAmxDcGhLX3WnMfmkNpvSTQU= -github.com/pion/webrtc/v3 v3.2.11 h1:lfGKYZcG7ghCTQWn+zsD+icIIWL3qIfclEjBGk537+s= -github.com/pion/webrtc/v3 v3.2.11/go.mod h1:fejQio1v8tKG4ntq4u8H4uDHsCNX6eX7bT093t4H+0E= +github.com/pion/webrtc/v3 v3.2.13 h1://ltbnahZewBWHvQYunlyLVWrHrsoyxYDfi3Ux6V4Gk= +github.com/pion/webrtc/v3 v3.2.13/go.mod h1:KS57v8u+fNMYAVM6gNsceIHtciyHlnfPNXU/7klJMFU= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -289,7 +287,6 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= -golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= golang.org/x/exp v0.0.0-20230728194245-b0cb94b80691 h1:/yRP+0AN7mf5DkD3BAI6TOFnd51gEoDEb8o35jIFtgw= @@ -385,6 +382,7 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -411,8 +409,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc= google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/grpc v1.56.2 h1:fVRFRnXvU+x6C4IlHZewvJOVHoOv1TUuQyoRsYnB4bI= -google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= +google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index c3f314eec..0eaabd1ed 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -38,6 +38,7 @@ import ( "github.com/livekit/protocol/logger" "github.com/livekit/protocol/logger/pionlogger" lksdp "github.com/livekit/protocol/sdp" + "github.com/livekit/protocol/utils" "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/rtc/types" @@ -65,6 +66,8 @@ const ( minConnectTimeoutAfterICE = 10 * time.Second maxConnectTimeoutAfterICE = 20 * time.Second // max duration for waiting pc to connect after ICE is connected + maxICECandidates = 20 + shortConnectionThreshold = 90 * time.Second ) @@ -227,10 +230,10 @@ type PCTransport struct { pendingRestartIceOffer *webrtc.SessionDescription // for cleaner logging - allowedLocalCandidates []string - allowedRemoteCandidates []string - filteredLocalCandidates []string - filteredRemoteCandidates []string + allowedLocalCandidates *utils.DedupedSlice[string] + allowedRemoteCandidates *utils.DedupedSlice[string] + filteredLocalCandidates *utils.DedupedSlice[string] + filteredRemoteCandidates *utils.DedupedSlice[string] } type TransportParams struct { @@ -371,6 +374,10 @@ func NewPCTransport(params TransportParams) (*PCTransport, error) { eventCh: make(chan event, 50), previousTrackDescription: make(map[string]*trackDescription), canReuseTransceiver: true, + allowedLocalCandidates: utils.NewDedupedSlice[string](maxICECandidates), + allowedRemoteCandidates: utils.NewDedupedSlice[string](maxICECandidates), + filteredLocalCandidates: utils.NewDedupedSlice[string](maxICECandidates), + filteredRemoteCandidates: utils.NewDedupedSlice[string](maxICECandidates), } if params.IsSendSide { t.streamAllocator = streamallocator.NewStreamAllocator(streamallocator.StreamAllocatorParams{ @@ -1167,7 +1174,7 @@ func (t *PCTransport) GetICEConnectionType() types.ICEConnectionType { // Pion would have created a prflx candidate with the same address as the relay candidate. // to report an accurate connection type, we'll compare to see if existing relay candidates match t.lock.RLock() - allowedRemoteCandidates := t.allowedRemoteCandidates + allowedRemoteCandidates := t.allowedRemoteCandidates.Get() t.lock.RUnlock() for _, ci := range allowedRemoteCandidates { @@ -1487,12 +1494,12 @@ func (t *PCTransport) clearLocalDescriptionSent() { t.cacheLocalCandidates = true t.cachedLocalCandidates = nil - t.allowedLocalCandidates = nil + t.allowedLocalCandidates.Clear() t.lock.Lock() - t.allowedRemoteCandidates = nil + t.allowedRemoteCandidates.Clear() t.lock.Unlock() - t.filteredLocalCandidates = nil - t.filteredRemoteCandidates = nil + t.filteredLocalCandidates.Clear() + t.filteredRemoteCandidates.Clear() } func (t *PCTransport) handleLocalICECandidate(e *event) error { @@ -1502,7 +1509,7 @@ func (t *PCTransport) handleLocalICECandidate(e *event) error { if t.preferTCP.Load() && c != nil && c.Protocol != webrtc.ICEProtocolTCP { cstr := c.String() t.params.Logger.Debugw("filtering out local candidate", "candidate", cstr) - t.filteredLocalCandidates = append(t.filteredLocalCandidates, cstr) + t.filteredLocalCandidates.Add(cstr) filtered = true } @@ -1511,7 +1518,7 @@ func (t *PCTransport) handleLocalICECandidate(e *event) error { } if c != nil { - t.allowedLocalCandidates = append(t.allowedLocalCandidates, c.String()) + t.allowedLocalCandidates.Add(c.String()) } if t.cacheLocalCandidates { t.cachedLocalCandidates = append(t.cachedLocalCandidates, c) @@ -1531,7 +1538,7 @@ func (t *PCTransport) handleRemoteICECandidate(e *event) error { filtered := false if t.preferTCP.Load() && !strings.Contains(c.Candidate, "tcp") { t.params.Logger.Debugw("filtering out remote candidate", "candidate", c.Candidate) - t.filteredRemoteCandidates = append(t.filteredRemoteCandidates, c.Candidate) + t.filteredRemoteCandidates.Add(c.Candidate) filtered = true } @@ -1540,7 +1547,7 @@ func (t *PCTransport) handleRemoteICECandidate(e *event) error { } t.lock.Lock() - t.allowedRemoteCandidates = append(t.allowedRemoteCandidates, c.Candidate) + t.allowedRemoteCandidates.Add(c.Candidate) t.lock.Unlock() if t.pc.RemoteDescription() == nil { @@ -1558,10 +1565,10 @@ func (t *PCTransport) handleRemoteICECandidate(e *event) error { func (t *PCTransport) handleLogICECandidates(e *event) error { t.params.Logger.Infow( "ice candidates", - "lc", t.allowedLocalCandidates, - "rc", t.allowedRemoteCandidates, - "lc (filtered)", t.filteredLocalCandidates, - "rc (filtered)", t.filteredRemoteCandidates, + "lc", t.allowedLocalCandidates.Get(), + "rc", t.allowedRemoteCandidates.Get(), + "lc (filtered)", t.filteredLocalCandidates.Get(), + "rc (filtered)", t.filteredRemoteCandidates.Get(), ) return nil
LiveKit Ecosystem
Client SDKsComponents · JavaScript · Rust · iOS/macOS · Android · Flutter · Unity (web) · Python · React Native (beta)
Client SDKsComponents · JavaScript · iOS/macOS · Android · Flutter · React Native · Rust · Python · Unity (web) · Unity (beta)
Server SDKsNode.js · Golang · Ruby · Java/Kotlin · PHP (community) · Python (community)
ServicesLivekit server · Egress · Ingress
ResourcesDocs · Example apps · Cloud · Self-hosting · CLI