From 10b0c9b9ffd40584f7ede67f5e2fb630cc873db3 Mon Sep 17 00:00:00 2001
From: Raja Subramanian <raja.gobi@tutanota.com>
Date: Mon, 30 May 2022 13:51:59 +0530
Subject: [PATCH] Allow overshooting maximum when there are no bandwidth
 constraints. (#739)

* Allow overshooting maximum when there are no bandwidth constraints.

Some clients prioritize sending higher layer of video (for e.g.
Firefox). They may get into a state where congestion does not allow
sending lower layers.

That combined with adaptive streaming capping the max layer at a
certain level, it is possible to get into a state where video is
not streamed.

To make this experience better, allow overshoot beyond max layers
in case of optimal (i. e. no congestion) allocation.

NOTE: This means that the video could freeze when there is congestion
even if `AllowPause` is not disabled as in congested state, we do
not overshoot the max layer.

* log about allowing overshoot
---
 pkg/sfu/forwarder.go      | 34 ++++++++++++++++++++++++++++++++++
 pkg/sfu/forwarder_test.go | 39 ++++++++++++++++++++++++++++++++++-----
 2 files changed, 68 insertions(+), 5 deletions(-)

diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go
index b3c2da4d8..e0e53ea50 100644
--- a/pkg/sfu/forwarder.go
+++ b/pkg/sfu/forwarder.go
@@ -482,6 +482,40 @@ func (f *Forwarder) AllocateOptimal(brs Bitrates) VideoAllocation {
 				break
 			}
 		}
+
+		if bandwidthRequested == 0 {
+			// if we cannot allocate anything below max layer,
+			// look for a layer above. It is okay to overshoot
+			// in optimal allocation (i. e. no bandwidth restricstions).
+			// It is possible that clients send only a higher layer.
+			// To accommodate cases like that, try finding a layer
+			// above the requested maximum to ensure streaming
+			for s := DefaultMaxLayerSpatial; s >= 0; s-- {
+				for t := DefaultMaxLayerTemporal; t >= 0; t-- {
+					if brs[s][t] == 0 {
+						continue
+					}
+
+					targetLayers = VideoLayers{
+						Spatial:  s,
+						Temporal: t,
+					}
+
+					bandwidthRequested = brs[s][t]
+					state = VideoAllocationStateOptimal
+
+					if f.targetLayers == InvalidLayers {
+						change = VideoStreamingChangeResuming
+					}
+					f.logger.Infow("allowing overshoot", "maxLayer", f.maxLayers, "targetLayers", targetLayers)
+					break
+				}
+
+				if bandwidthRequested != 0 {
+					break
+				}
+			}
+		}
 	}
 
 	if !targetLayers.IsValid() {
diff --git a/pkg/sfu/forwarder_test.go b/pkg/sfu/forwarder_test.go
index eea98e7e2..67976001c 100644
--- a/pkg/sfu/forwarder_test.go
+++ b/pkg/sfu/forwarder_test.go
@@ -233,6 +233,35 @@ func TestForwarderAllocate(t *testing.T) {
 	require.Equal(t, InvalidLayers, f.CurrentLayers())
 
 	// allocate using bitrates, allocation should choose optimal
+	f.UpTrackLayersChange([]int32{0, 1, 2})
+	f.maxLayers = VideoLayers{Spatial: 1, Temporal: 3}
+	expectedTargetLayers = VideoLayers{
+		Spatial:  1,
+		Temporal: 3,
+	}
+	expectedResult = VideoAllocation{
+		state:              VideoAllocationStateOptimal,
+		change:             VideoStreamingChangeNone,
+		bandwidthRequested: bitrates[1][3],
+		bandwidthDelta:     bitrates[1][3],
+		availableLayers:    []int32{0, 1, 2},
+		bitrates:           bitrates,
+		targetLayers:       expectedTargetLayers,
+		distanceToDesired:  0,
+	}
+	result = f.AllocateOptimal(bitrates)
+	require.Equal(t, expectedResult, result)
+	require.Equal(t, expectedResult, f.lastAllocation)
+	require.Equal(t, InvalidLayers, f.CurrentLayers())
+	require.Equal(t, expectedTargetLayers, f.TargetLayers())
+
+	// allocate using bitrates above maximum layer
+	f.UpTrackLayersChange([]int32{2})
+	sparseBitrates := Bitrates{
+		{0, 0, 0, 0},
+		{0, 0, 0, 0},
+		{0, 7, 0, 0},
+	}
 	expectedTargetLayers = VideoLayers{
 		Spatial:  2,
 		Temporal: 1,
@@ -240,14 +269,14 @@ func TestForwarderAllocate(t *testing.T) {
 	expectedResult = VideoAllocation{
 		state:              VideoAllocationStateOptimal,
 		change:             VideoStreamingChangeNone,
-		bandwidthRequested: bitrates[2][1],
-		bandwidthDelta:     bitrates[2][1],
-		availableLayers:    []int32{0},
-		bitrates:           bitrates,
+		bandwidthRequested: sparseBitrates[2][1],
+		bandwidthDelta:     sparseBitrates[2][1] - bitrates[1][3],
+		availableLayers:    []int32{2},
+		bitrates:           sparseBitrates,
 		targetLayers:       expectedTargetLayers,
 		distanceToDesired:  0,
 	}
-	result = f.AllocateOptimal(bitrates)
+	result = f.AllocateOptimal(sparseBitrates)
 	require.Equal(t, expectedResult, result)
 	require.Equal(t, expectedResult, f.lastAllocation)
 	require.Equal(t, InvalidLayers, f.CurrentLayers())