From aeefbb080eaee4c6827739e099866c548d241ddc Mon Sep 17 00:00:00 2001 From: Raja Subramanian Date: Sun, 19 Mar 2023 18:34:56 +0530 Subject: [PATCH] Account for time before measurement available in connection quality. (#1528) --- pkg/sfu/connectionquality/scorer.go | 3 ++ pkg/sfu/downtrack.go | 23 +++++---- pkg/sfu/forwarder.go | 72 +++++++++++++++++++++++++---- pkg/sfu/forwarder_test.go | 60 ++++++++++++------------ pkg/sfu/receiver.go | 2 + pkg/sfu/streamtrackermanager.go | 10 +++- 6 files changed, 119 insertions(+), 51 deletions(-) diff --git a/pkg/sfu/connectionquality/scorer.go b/pkg/sfu/connectionquality/scorer.go index 9e7a1b047..a1329dfca 100644 --- a/pkg/sfu/connectionquality/scorer.go +++ b/pkg/sfu/connectionquality/scorer.go @@ -124,6 +124,7 @@ type qualityScorer struct { lastUpdateAt time.Time score float64 + stat windowStat mutedAt time.Time unmutedAt time.Time @@ -276,6 +277,7 @@ func (q *qualityScorer) Update(stat *windowStat, at time.Time) { "reason", reason, "prevScore", q.score, "prevQuality", scoreToConnectionQuality(q.score), + "prevStat", q.stat, "score", score, "quality", scoreToConnectionQuality(score), "stat", stat, @@ -285,6 +287,7 @@ func (q *qualityScorer) Update(stat *windowStat, at time.Time) { } q.score = score + q.stat = *stat q.lastUpdateAt = at } diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index 6bd4c0ea0..79abe27cb 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -954,8 +954,11 @@ func (d *DownTrack) maybeAddTransition(_bitrate int64, distance float64) { d.connectionStats.AddLayerTransition(distance, time.Now()) } -func (d *DownTrack) UpTrackBitrateReport(_availableLayers []int32, bitrates Bitrates) { - d.maybeAddTransition(d.forwarder.GetOptimalBandwidthNeeded(bitrates), d.forwarder.DistanceToDesired(bitrates)) +func (d *DownTrack) UpTrackBitrateReport(availableLayers []int32, bitrates Bitrates) { + d.maybeAddTransition( + d.forwarder.GetOptimalBandwidthNeeded(bitrates), + d.forwarder.DistanceToDesired(availableLayers, bitrates), + ) } // OnCloseHandler method to be called on remote tracked removed @@ -996,8 +999,8 @@ func (d *DownTrack) BandwidthRequested() int64 { } func (d *DownTrack) DistanceToDesired() float64 { - _, brs := d.receiver.GetLayeredBitrate() - return d.forwarder.DistanceToDesired(brs) + al, brs := d.receiver.GetLayeredBitrate() + return d.forwarder.DistanceToDesired(al, brs) } func (d *DownTrack) AllocateOptimal(allowOvershoot bool) VideoAllocation { @@ -1009,8 +1012,8 @@ func (d *DownTrack) AllocateOptimal(allowOvershoot bool) VideoAllocation { } func (d *DownTrack) ProvisionalAllocatePrepare() { - _, brs := d.receiver.GetLayeredBitrate() - d.forwarder.ProvisionalAllocatePrepare(brs) + al, brs := d.receiver.GetLayeredBitrate() + d.forwarder.ProvisionalAllocatePrepare(al, brs) } func (d *DownTrack) ProvisionalAllocate(availableChannelCapacity int64, layers VideoLayers, allowPause bool, allowOvershoot bool) int64 { @@ -1037,8 +1040,8 @@ func (d *DownTrack) ProvisionalAllocateCommit() VideoAllocation { } func (d *DownTrack) AllocateNextHigher(availableChannelCapacity int64, allowOvershoot bool) (VideoAllocation, bool) { - _, brs := d.receiver.GetLayeredBitrate() - allocation, available := d.forwarder.AllocateNextHigher(availableChannelCapacity, brs, allowOvershoot) + al, brs := d.receiver.GetLayeredBitrate() + allocation, available := d.forwarder.AllocateNextHigher(availableChannelCapacity, al, brs, allowOvershoot) d.maybeStartKeyFrameRequester() d.maybeAddTransition(allocation.bandwidthNeeded, allocation.distanceToDesired) return allocation, available @@ -1052,8 +1055,8 @@ func (d *DownTrack) GetNextHigherTransition(allowOvershoot bool) (VideoTransitio } func (d *DownTrack) Pause() VideoAllocation { - _, brs := d.receiver.GetLayeredBitrate() - allocation := d.forwarder.Pause(brs) + al, brs := d.receiver.GetLayeredBitrate() + allocation := d.forwarder.Pause(al, brs) d.maybeStartKeyFrameRequester() d.maybeAddTransition(allocation.bandwidthNeeded, allocation.distanceToDesired) return allocation diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index d75c890a7..c5dd4df47 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -98,6 +98,7 @@ type VideoAllocationProvisional struct { pubMuted bool maxPublishedLayer int32 maxTemporalLayerSeen int32 + availableLayers []int32 bitrates Bitrates maxLayers VideoLayers currentLayers VideoLayers @@ -497,11 +498,20 @@ func (f *Forwarder) BandwidthRequested(brs Bitrates) int64 { return brs[f.targetLayers.Spatial][f.targetLayers.Temporal] } -func (f *Forwarder) DistanceToDesired(brs Bitrates) float64 { +func (f *Forwarder) DistanceToDesired(availableLayers []int32, brs Bitrates) float64 { f.lock.RLock() defer f.lock.RUnlock() - return getDistanceToDesired(f.muted, f.pubMuted, f.maxPublishedLayer, f.maxTemporalLayerSeen, brs, f.targetLayers, f.maxLayers) + return getDistanceToDesired( + f.muted, + f.pubMuted, + f.maxPublishedLayer, + f.maxTemporalLayerSeen, + availableLayers, + brs, + f.targetLayers, + f.maxLayers, + ) } func (f *Forwarder) GetOptimalBandwidthNeeded(brs Bitrates) int64 { @@ -620,12 +630,21 @@ func (f *Forwarder) AllocateOptimal(availableLayers []int32, brs Bitrates, allow alloc.bandwidthRequested = optimalBandwidthNeeded } alloc.bandwidthDelta = alloc.bandwidthRequested - f.lastAllocation.bandwidthRequested - alloc.distanceToDesired = getDistanceToDesired(f.muted, f.pubMuted, f.maxPublishedLayer, f.maxTemporalLayerSeen, brs, alloc.targetLayers, f.maxLayers) + alloc.distanceToDesired = getDistanceToDesired( + f.muted, + f.pubMuted, + f.maxPublishedLayer, + f.maxTemporalLayerSeen, + availableLayers, + brs, + alloc.targetLayers, + f.maxLayers, + ) return f.updateAllocation(alloc, "optimal") } -func (f *Forwarder) ProvisionalAllocatePrepare(bitrates Bitrates) { +func (f *Forwarder) ProvisionalAllocatePrepare(availableLayers []int32, bitrates Bitrates) { f.lock.Lock() defer f.lock.Unlock() @@ -640,6 +659,9 @@ func (f *Forwarder) ProvisionalAllocatePrepare(bitrates Bitrates) { currentLayers: f.currentLayers, parkedLayers: f.parkedLayers, } + + f.provisional.availableLayers = make([]int32, len(availableLayers)) + copy(f.provisional.availableLayers, availableLayers) } func (f *Forwarder) ProvisionalAllocate(availableChannelCapacity int64, layers VideoLayers, allowPause bool, allowOvershoot bool) int64 { @@ -945,6 +967,7 @@ func (f *Forwarder) ProvisionalAllocateCommit() VideoAllocation { f.provisional.pubMuted, f.provisional.maxPublishedLayer, f.provisional.maxTemporalLayerSeen, + f.provisional.availableLayers, f.provisional.bitrates, f.provisional.allocatedLayers, f.provisional.maxLayers, @@ -1002,7 +1025,7 @@ func (f *Forwarder) ProvisionalAllocateCommit() VideoAllocation { return f.updateAllocation(alloc, "cooperative") } -func (f *Forwarder) AllocateNextHigher(availableChannelCapacity int64, brs Bitrates, allowOvershoot bool) (VideoAllocation, bool) { +func (f *Forwarder) AllocateNextHigher(availableChannelCapacity int64, availableLayers []int32, brs Bitrates, allowOvershoot bool) (VideoAllocation, bool) { f.lock.Lock() defer f.lock.Unlock() @@ -1053,7 +1076,16 @@ func (f *Forwarder) AllocateNextHigher(availableChannelCapacity int64, brs Bitra targetLayers: targetLayers, requestLayerSpatial: targetLayers.Spatial, maxLayers: f.maxLayers, - distanceToDesired: getDistanceToDesired(f.muted, f.pubMuted, f.maxPublishedLayer, f.maxTemporalLayerSeen, brs, targetLayers, f.maxLayers), + distanceToDesired: getDistanceToDesired( + f.muted, + f.pubMuted, + f.maxPublishedLayer, + f.maxTemporalLayerSeen, + availableLayers, + brs, + targetLayers, + f.maxLayers, + ), } if targetLayers.GreaterThan(f.maxLayers) || bandwidthRequested >= optimalBandwidthNeeded { alloc.isDeficient = false @@ -1187,7 +1219,7 @@ func (f *Forwarder) GetNextHigherTransition(brs Bitrates, allowOvershoot bool) ( return VideoTransition{}, false } -func (f *Forwarder) Pause(brs Bitrates) VideoAllocation { +func (f *Forwarder) Pause(availableLayers []int32, brs Bitrates) VideoAllocation { f.lock.Lock() defer f.lock.Unlock() @@ -1200,7 +1232,16 @@ func (f *Forwarder) Pause(brs Bitrates) VideoAllocation { targetLayers: InvalidLayers, requestLayerSpatial: InvalidLayerSpatial, maxLayers: f.maxLayers, - distanceToDesired: getDistanceToDesired(f.muted, f.pubMuted, f.maxPublishedLayer, f.maxTemporalLayerSeen, brs, InvalidLayers, f.maxLayers), + distanceToDesired: getDistanceToDesired( + f.muted, + f.pubMuted, + f.maxPublishedLayer, + f.maxTemporalLayerSeen, + availableLayers, + brs, + InvalidLayers, + f.maxLayers, + ), } switch { @@ -1703,6 +1744,7 @@ func getDistanceToDesired( pubMuted bool, maxPublishedLayer int32, maxTemporalLayerSeen int32, + availableLayers []int32, brs Bitrates, targetLayers VideoLayers, maxLayers VideoLayers, @@ -1713,11 +1755,13 @@ func getDistanceToDesired( adjustedMaxLayers := maxLayers + maxAvailableSpatial := InvalidLayerSpatial + maxAvailableTemporal := InvalidLayerTemporal + // max available spatial is min(subscribedMax, publishedMax, availableMax) // subscribedMax = subscriber requested max spatial layer // publishedMax = max spatial layer ever published // availableMax = based on bit rate measurement, available max spatial layer - maxAvailableSpatial := InvalidLayerSpatial done: for s := int32(len(brs)) - 1; s >= 0; s-- { for t := int32(len(brs[0])) - 1; t >= 0; t-- { @@ -1727,6 +1771,15 @@ done: } } } + + // before bit rate measurement is available, stream tracker could declare layer seen, account for that + for _, layer := range availableLayers { + if layer > maxAvailableSpatial { + maxAvailableSpatial = layer + maxAvailableTemporal = maxTemporalLayerSeen // till bit rate measurement is available, assume max seen as temporal + } + } + if maxAvailableSpatial < adjustedMaxLayers.Spatial { adjustedMaxLayers.Spatial = maxAvailableSpatial } @@ -1739,7 +1792,6 @@ done: // subscribedMax = subscriber requested max temporal layer // temporalLayerSeenMax = max temporal layer ever published/seen // availableMax = based on bit rate measurement, available max temporal in the adjusted max spatial layer - maxAvailableTemporal := InvalidLayerTemporal if adjustedMaxLayers.Spatial != InvalidLayerSpatial { for t := int32(len(brs[0])) - 1; t >= 0; t-- { if brs[adjustedMaxLayers.Spatial][t] != 0 { diff --git a/pkg/sfu/forwarder_test.go b/pkg/sfu/forwarder_test.go index 5398a5a61..2c3784ea3 100644 --- a/pkg/sfu/forwarder_test.go +++ b/pkg/sfu/forwarder_test.go @@ -312,7 +312,7 @@ func TestForwarderAllocateOptimal(t *testing.T) { targetLayers: DefaultMaxLayers, requestLayerSpatial: 2, maxLayers: DefaultMaxLayers, - distanceToDesired: -2.75, + distanceToDesired: -1.0, } result = f.AllocateOptimal([]int32{0, 1}, emptyBitrates, false) require.Equal(t, expectedResult, result) @@ -375,7 +375,7 @@ func TestForwarderAllocateOptimal(t *testing.T) { targetLayers: expectedTargetLayers, requestLayerSpatial: 0, maxLayers: f.maxLayers, - distanceToDesired: -0.25, + distanceToDesired: 0.0, } result = f.AllocateOptimal([]int32{0, 1}, emptyBitrates, true) require.Equal(t, expectedResult, result) @@ -397,7 +397,7 @@ func TestForwarderAllocateOptimal(t *testing.T) { targetLayers: expectedTargetLayers, requestLayerSpatial: 2, maxLayers: f.maxLayers, - distanceToDesired: -2.75, + distanceToDesired: -1.5, } result = f.AllocateOptimal([]int32{0, 1}, emptyBitrates, true) require.Equal(t, expectedResult, result) @@ -417,7 +417,7 @@ func TestForwarderProvisionalAllocate(t *testing.T) { {9, 10, 11, 12}, } - f.ProvisionalAllocatePrepare(bitrates) + f.ProvisionalAllocatePrepare(nil, bitrates) usedBitrate := f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 0, Temporal: 0}, true, false) require.Equal(t, bitrates[0][0], usedBitrate) @@ -458,7 +458,7 @@ func TestForwarderProvisionalAllocate(t *testing.T) { // when nothing fits and pausing disallowed, should allocate (0, 0) f.targetLayers = InvalidLayers - f.ProvisionalAllocatePrepare(bitrates) + f.ProvisionalAllocatePrepare(nil, bitrates) usedBitrate = f.ProvisionalAllocate(0, VideoLayers{Spatial: 0, Temporal: 0}, false, false) require.Equal(t, int64(1), usedBitrate) @@ -494,7 +494,7 @@ func TestForwarderProvisionalAllocate(t *testing.T) { {9, 10, 11, 12}, } - f.ProvisionalAllocatePrepare(bitrates) + f.ProvisionalAllocatePrepare(nil, bitrates) usedBitrate = f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 0, Temporal: 0}, false, true) require.Equal(t, int64(0), usedBitrate) @@ -540,7 +540,7 @@ func TestForwarderProvisionalAllocate(t *testing.T) { } f.currentLayers = VideoLayers{Spatial: 0, Temporal: 2} - f.ProvisionalAllocatePrepare(bitrates) + f.ProvisionalAllocatePrepare(nil, bitrates) // all the provisional allocations should not succeed because the feed is dry usedBitrate = f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 0, Temporal: 0}, false, true) @@ -578,7 +578,7 @@ func TestForwarderProvisionalAllocate(t *testing.T) { // Same case as above, but current is above max, so target should go to invalid // f.currentLayers = VideoLayers{Spatial: 1, Temporal: 2} - f.ProvisionalAllocatePrepare(bitrates) + f.ProvisionalAllocatePrepare(nil, bitrates) // all the provisional allocations below should not succeed because the feed is dry usedBitrate = f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 0, Temporal: 0}, false, true) @@ -621,7 +621,7 @@ func TestForwarderProvisionalAllocateMute(t *testing.T) { } f.Mute(true) - f.ProvisionalAllocatePrepare(bitrates) + f.ProvisionalAllocatePrepare(nil, bitrates) usedBitrate := f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 0, Temporal: 0}, true, false) require.Equal(t, int64(0), usedBitrate) @@ -659,7 +659,7 @@ func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { {9, 10, 0, 0}, } - f.ProvisionalAllocatePrepare(bitrates) + f.ProvisionalAllocatePrepare(nil, bitrates) // from scratch (InvalidLayers) should give back layer (0, 0) expectedTransition := VideoTransition{ @@ -732,7 +732,7 @@ func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { // mute f.Mute(true) - f.ProvisionalAllocatePrepare(bitrates) + f.ProvisionalAllocatePrepare(nil, bitrates) // mute should send target to InvalidLayers expectedTransition = VideoTransition{ @@ -758,7 +758,7 @@ func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { } f.targetLayers = InvalidLayers - f.ProvisionalAllocatePrepare(bitrates) + f.ProvisionalAllocatePrepare(nil, bitrates) // from scratch (InvalidLayers) should go to a layer past maximum as overshoot is allowed expectedTransition = VideoTransition{ @@ -797,7 +797,7 @@ func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { f.currentLayers = VideoLayers{Spatial: 0, Temporal: 2} f.targetLayers = InvalidLayers - f.ProvisionalAllocatePrepare(bitrates) + f.ProvisionalAllocatePrepare(nil, bitrates) // from scratch (InvalidLayers) should go to current layer // NOTE: targetLayer is set to InvalidLayers for testing, but in practice current layers valid and target layers invalid should not happen @@ -852,7 +852,7 @@ func TestForwarderProvisionalAllocateGetBestWeightedTransition(t *testing.T) { {9, 10, 11, 12}, } - f.ProvisionalAllocatePrepare(bitrates) + f.ProvisionalAllocatePrepare(nil, bitrates) f.targetLayers = VideoLayers{Spatial: 2, Temporal: 2} f.lastAllocation.bandwidthRequested = bitrates[2][2] @@ -878,7 +878,7 @@ func TestForwarderAllocateNextHigher(t *testing.T) { {0, 7, 0, 0}, } - result, boosted := f.AllocateNextHigher(ChannelCapacityInfinity, bitrates, false) + result, boosted := f.AllocateNextHigher(ChannelCapacityInfinity, nil, bitrates, false) require.Equal(t, VideoAllocationDefault, result) // no layer for audio require.False(t, boosted) @@ -889,7 +889,7 @@ func TestForwarderAllocateNextHigher(t *testing.T) { f.SetMaxTemporalLayerSeen(DefaultMaxLayerTemporal) // when not in deficient state, does not boost - result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, bitrates, false) + result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, nil, bitrates, false) require.Equal(t, VideoAllocationDefault, result) require.False(t, boosted) @@ -898,7 +898,7 @@ func TestForwarderAllocateNextHigher(t *testing.T) { Spatial: 0, Temporal: 0, } - result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, bitrates, false) + result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, nil, bitrates, false) require.Equal(t, VideoAllocationDefault, result) require.False(t, boosted) @@ -924,14 +924,14 @@ func TestForwarderAllocateNextHigher(t *testing.T) { maxLayers: DefaultMaxLayers, distanceToDesired: 2.0, } - result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, bitrates, false) + result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, nil, bitrates, false) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) require.Equal(t, expectedTargetLayers, f.TargetLayers()) require.True(t, boosted) // empty bitrates cannot increase layer, i. e. last allocation is left unchanged - result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, emptyBitrates, false) + result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, nil, emptyBitrates, false) require.Equal(t, expectedResult, result) require.False(t, boosted) @@ -952,7 +952,7 @@ func TestForwarderAllocateNextHigher(t *testing.T) { maxLayers: DefaultMaxLayers, distanceToDesired: 1.25, } - result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, bitrates, false) + result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, nil, bitrates, false) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) require.Equal(t, expectedTargetLayers, f.TargetLayers()) @@ -976,7 +976,7 @@ func TestForwarderAllocateNextHigher(t *testing.T) { maxLayers: DefaultMaxLayers, distanceToDesired: 0.5, } - result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, bitrates, false) + result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, nil, bitrates, false) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) require.Equal(t, expectedTargetLayers, f.TargetLayers()) @@ -998,7 +998,7 @@ func TestForwarderAllocateNextHigher(t *testing.T) { maxLayers: DefaultMaxLayers, distanceToDesired: 0.0, } - result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, bitrates, false) + result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, nil, bitrates, false) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) require.Equal(t, expectedTargetLayers, f.TargetLayers()) @@ -1007,7 +1007,7 @@ func TestForwarderAllocateNextHigher(t *testing.T) { // ask again, should return not boosted as there is no room to go higher f.currentLayers.Spatial = 2 f.currentLayers.Temporal = 1 - result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, bitrates, false) + result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, nil, bitrates, false) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) require.Equal(t, expectedTargetLayers, f.TargetLayers()) @@ -1033,7 +1033,7 @@ func TestForwarderAllocateNextHigher(t *testing.T) { maxLayers: DefaultMaxLayers, distanceToDesired: 2.25, } - result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, bitrates, false) + result, boosted = f.AllocateNextHigher(ChannelCapacityInfinity, nil, bitrates, false) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) require.Equal(t, expectedTargetLayers, f.TargetLayers()) @@ -1051,7 +1051,7 @@ func TestForwarderAllocateNextHigher(t *testing.T) { maxLayers: DefaultMaxLayers, distanceToDesired: 2.25, } - result, boosted = f.AllocateNextHigher(0, bitrates, false) + result, boosted = f.AllocateNextHigher(0, nil, bitrates, false) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) require.Equal(t, expectedTargetLayers, f.TargetLayers()) @@ -1086,7 +1086,7 @@ func TestForwarderAllocateNextHigher(t *testing.T) { distanceToDesired: -1.0, } // overshoot should return (1, 0) even if there is not enough capacity - result, boosted = f.AllocateNextHigher(bitrates[1][0]-1, bitrates, true) + result, boosted = f.AllocateNextHigher(bitrates[1][0]-1, nil, bitrates, true) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) require.Equal(t, expectedTargetLayers, f.TargetLayers()) @@ -1106,7 +1106,7 @@ func TestForwarderPause(t *testing.T) { {9, 10, 11, 12}, } - f.ProvisionalAllocatePrepare(bitrates) + f.ProvisionalAllocatePrepare(nil, bitrates) f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 0, Temporal: 0}, true, false) // should have set target at (0, 0) f.ProvisionalAllocateCommit() @@ -1123,7 +1123,7 @@ func TestForwarderPause(t *testing.T) { maxLayers: DefaultMaxLayers, distanceToDesired: 3, } - result := f.Pause(bitrates) + result := f.Pause(nil, bitrates) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) require.Equal(t, InvalidLayers, f.TargetLayers()) @@ -1141,7 +1141,7 @@ func TestForwarderPauseMute(t *testing.T) { {9, 10, 11, 12}, } - f.ProvisionalAllocatePrepare(bitrates) + f.ProvisionalAllocatePrepare(nil, bitrates) f.ProvisionalAllocate(bitrates[2][3], VideoLayers{Spatial: 0, Temporal: 0}, true, true) // should have set target at (0, 0) f.ProvisionalAllocateCommit() @@ -1157,7 +1157,7 @@ func TestForwarderPauseMute(t *testing.T) { maxLayers: DefaultMaxLayers, distanceToDesired: 0, } - result := f.Pause(bitrates) + result := f.Pause(nil, bitrates) require.Equal(t, expectedResult, result) require.Equal(t, expectedResult, f.lastAllocation) require.Equal(t, InvalidLayers, f.TargetLayers()) diff --git a/pkg/sfu/receiver.go b/pkg/sfu/receiver.go index 52ba65fde..bc5ea1705 100644 --- a/pkg/sfu/receiver.go +++ b/pkg/sfu/receiver.go @@ -403,6 +403,8 @@ func (w *WebRTCReceiver) OnAvailableLayersChanged() { for _, dt := range w.downTrackSpreader.GetDownTracks() { dt.UpTrackLayersChange() } + + w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired(), time.Now()) } // StreamTrackerManagerListener.OnBitrateAvailabilityChanged diff --git a/pkg/sfu/streamtrackermanager.go b/pkg/sfu/streamtrackermanager.go index 332c4ed9f..d04b03a54 100644 --- a/pkg/sfu/streamtrackermanager.go +++ b/pkg/sfu/streamtrackermanager.go @@ -292,7 +292,7 @@ func (s *StreamTrackerManager) DistanceToDesired() float64 { return 0 } - _, brs := s.getLayeredBitrateLocked() + al, brs := s.getLayeredBitrateLocked() maxLayers := InvalidLayers done: @@ -308,6 +308,14 @@ done: } } + // before bit rate measurement is available, stream tracker could declare layer seen, account for that + for _, layer := range al { + if layer > maxLayers.Spatial { + maxLayers.Spatial = layer + maxLayers.Temporal = s.maxTemporalLayerSeen // till bit rate measurement is available, assume max seen as temporal + } + } + adjustedMaxLayers := maxLayers if !maxLayers.IsValid() { adjustedMaxLayers = VideoLayers{Spatial: 0, Temporal: 0}